From 898bfc914e425dec6b4f740bd5462421a0552b22 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 1 Oct 2026 11:06:46 -0700 Subject: [PATCH 1/4] chore(search): remove legacy indexed enterprise search --- .github/workflows/desktop-e2e.yml | 1 - .../self-hosting/integrations-oauth.mdx | 2 +- apps/docs/content/docs/search/index.mdx | 2 +- apps/docs/content/docs/search/mcp.mdx | 2 +- .../connectors/directory-sync/route.test.ts | 19 - apps/sim/app/api/knowledge/search/route.ts | 64 +- .../api/knowledge/sim-search/connect/route.ts | 20 - .../sim-search/integrations/overview/route.ts | 23 - .../sim-search/personal-source-setup/route.ts | 70 -- .../sim-search/sources/overview/route.ts | 23 - .../sim-search/sources/progress/route.ts | 23 - .../sim-search/sources/route.test.ts | 44 +- .../api/knowledge/sim-search/stats/route.ts | 24 - .../[id]/connected-accounts/indexing/route.ts | 22 - apps/sim/app/api/v1/knowledge/search/route.ts | 3 - .../components/composer/composer.test.tsx | 9 +- .../components/get-started/get-started.tsx | 14 +- .../home/organization-home.test.tsx | 16 +- .../home/organization-home.tsx | 10 +- .../indexed/github-member-integration.tsx | 91 -- .../integrations/indexed/index.ts | 1 - .../indexed/member-integration-row.tsx | 223 ---- .../indexed/member-integrations-list.tsx | 268 ---- .../integrations/indexed/source-status.ts | 44 - .../indexed/use-member-enrollment.test.tsx | 358 ------ .../indexed/use-member-enrollment.ts | 416 ------ .../integrations/integrations.test.tsx | 1041 --------------- .../integrations/integrations.tsx | 12 +- .../[knowledgeBaseId]/[documentId]/page.tsx | 91 -- .../[documentId]/search-params.ts | 25 - .../components/integrations/indexed/index.ts | 1 - ...xed-organization-integrations-settings.tsx | 139 --- .../organization-integrations-setup.tsx | 188 --- .../organization-search-stats-period.tsx | 89 -- .../indexed/organization-source-people.tsx | 79 -- .../indexed/organization-source-stats.tsx | 226 ---- ...rganization-integrations-settings.test.tsx | 456 ------- .../organization-integrations-settings.tsx | 9 +- .../organization-search-status.ts | 27 - .../integrations/search-source-setup.test.tsx | 150 +-- .../integrations/search-source-setup.tsx | 21 +- .../providers/[connectorType]/page.test.tsx | 23 +- .../providers/[connectorType]/page.tsx | 3 +- .../[connectorType]/provider-detail.tsx | 125 +- .../sources/[connectorId]/search-params.ts | 14 - .../[connectorId]/source-detail.test.tsx | 68 +- .../sources/[connectorId]/source-detail.tsx | 168 +-- .../knowledge-search-results/indexed/index.ts | 1 - .../indexed/indexed-search-results.tsx | 214 ---- .../knowledge-search-results.tsx | 5 +- .../search-transitions.test.tsx | 49 +- .../search-integration-connection.tsx | 30 +- .../atlassian-source-setup-modal.tsx | 254 ---- .../search-sources/source-setup-modal.tsx | 131 -- .../home/hooks/use-chat.dom.test.tsx | 4 +- .../[workspaceId]/home/hooks/use-chat.ts | 7 +- .../add-connector-modal.tsx | 3 +- .../connector-documents.tsx | 12 - .../connector-selector-field.test.tsx | 51 - .../connector-selector-field.tsx | 22 +- .../connector-settings-fields.tsx | 3 +- .../use-connector-settings-form.test.tsx | 13 +- .../use-connector-settings-form.ts | 3 +- apps/sim/bootstrap.ts | 2 - apps/sim/connectors/coda/README.md | 2 +- .../search-params.ts | 22 - apps/sim/executor/utils/credential-token.ts | 5 +- apps/sim/hooks/queries/kb/connectors.test.ts | 2 +- apps/sim/hooks/queries/kb/connectors.ts | 222 +--- apps/sim/hooks/queries/kb/knowledge.test.ts | 47 +- apps/sim/hooks/queries/kb/knowledge.ts | 32 +- .../hooks/queries/organization-accounts.ts | 4 - .../queries/organization-search-stats.ts | 28 - .../hooks/queries/personal-source-setup.ts | 67 - apps/sim/hooks/queries/selectors.test.tsx | 78 -- apps/sim/hooks/queries/selectors.ts | 127 +- .../reset-organization-search-access.test.ts | 55 +- .../utils/reset-organization-search-access.ts | 8 +- .../hooks/queries/utils/search-source-keys.ts | 12 - .../use-personal-source-account.test.tsx | 130 -- apps/sim/hooks/use-personal-source-account.ts | 135 -- ...use-search-integration-connection.test.tsx | 46 +- .../use-search-integration-connection.ts | 144 +-- .../api/contracts/desktop-source-connect.ts | 6 - .../lib/api/contracts/knowledge/connectors.ts | 176 +-- apps/sim/lib/api/contracts/knowledge/mcp.ts | 48 - .../knowledge/personal-integrations.ts | 13 - .../knowledge/personal-source-setup.test.ts | 36 - .../knowledge/personal-source-setup.ts | 86 -- .../api/contracts/knowledge/search-stats.ts | 76 -- .../contracts/mothership-search-sources.ts | 1 - .../api/contracts/organization-accounts.ts | 26 - apps/sim/lib/api/contracts/workspaces.ts | 1 - apps/sim/lib/core/config/deployment-shape.ts | 2 - apps/sim/lib/core/config/env-flags.ts | 4 - apps/sim/lib/core/config/env.ts | 3 - apps/sim/lib/credential-groups/README.md | 4 +- .../organization-account-indexing.test.ts | 134 -- .../organization-account-indexing.ts | 62 - .../organization-settings-delegation.test.ts | 2 - .../self-enrollment-oauth.ts | 2 +- .../credential-groups/slack-provider.test.ts | 67 +- .../lib/credential-groups/slack-provider.ts | 8 +- ...esolve-organization-personal-token.test.ts | 54 +- .../resolve-organization-personal-token.ts | 59 +- apps/sim/lib/desktop/source-browser.ts | 16 +- .../__integration__/coda-live.integration.ts | 110 +- ...dormant-processing-recovery.integration.ts | 5 - .../dormant-search-processing.integration.ts | 7 +- .../embedding-insert-batches.integration.ts | 16 +- .../excluded-member-documents.integration.ts | 11 - .../github-member.integration.ts | 313 +---- .../gitlab-live.integration.ts | 614 +++++---- .../gmail-member.integration.ts | 11 - .../google-calendar-member.integration.ts | 11 - .../jira-member.integration.ts | 11 - .../kb-block-search.integration.ts | 41 +- .../knowledge-projection.integration.ts | 1111 ++++------------- .../organization-mcp-search.integration.ts | 988 --------------- ...rganization-search-overview.integration.ts | 432 ------- .../processing-lock-scope.integration.ts | 16 +- ...rovider-processing-recovery.integration.ts | 93 +- .../read-indexed-document.integration.ts | 343 ----- .../search-latency.integration.ts | 765 +----------- .../search-source-pagination.integration.ts | 79 +- .../search-source-progress.integration.ts | 91 +- .../search-source-setup.integration.ts | 96 +- .../stored-document-recovery.integration.ts | 68 +- .../unfilled-projection-source.integration.ts | 402 ------ ...orkspace-kb-document-access.integration.ts | 15 +- .../knowledge/access/predicate.integration.ts | 208 +-- .../lib/knowledge/access/predicate.test.ts | 116 -- apps/sim/lib/knowledge/api/route-policies.ts | 21 +- ...onnect-personal-search-integration.test.ts | 36 - .../connect-personal-search-integration.ts | 30 +- .../knowledge/application/connectors.test.ts | 91 -- .../lib/knowledge/application/connectors.ts | 106 +- .../lib/knowledge/application/operations.ts | 79 -- .../organization-search-overview.test.ts | 181 --- .../organization-search-overview.ts | 307 ----- .../organization-search-stats.test.ts | 94 -- .../application/organization-search-stats.ts | 24 - .../personal-search-account.test.ts | 125 -- .../application/personal-search-account.ts | 68 - .../personal-search-integration-pages.ts | 49 - .../personal-search-integrations.test.ts | 117 +- .../personal-search-integrations.ts | 13 - .../application/personal-source-setup.test.ts | 271 ---- .../application/personal-source-setup.ts | 305 ----- .../application/search-diagnostics.ts | 27 - .../application/search-integrations.test.ts | 52 +- .../application/search-integrations.ts | 50 +- .../application/search-source-overview.ts | 220 ---- .../application/search-source-progress.ts | 110 -- .../application/search-sources.test.ts | 145 +-- .../knowledge/application/search-sources.ts | 144 +-- .../lib/knowledge/application/search.test.ts | 51 +- apps/sim/lib/knowledge/application/search.ts | 16 +- .../knowledge/application/sim-search.test.ts | 256 +--- .../lib/knowledge/application/sim-search.ts | 285 +---- .../connectors/indexing-policy.test.ts | 27 - .../knowledge/connectors/indexing-policy.ts | 5 +- .../organization-account-indexing.test.ts | 110 -- .../organization-account-indexing.ts | 136 -- .../connectors/viewer-member-sync-error.ts | 33 - .../connectors/viewer-source-accounts.test.ts | 76 -- .../connectors/viewer-source-accounts.ts | 76 -- apps/sim/lib/knowledge/constants.ts | 1 - .../lib/knowledge/documents/ocr-recovery.md | 2 +- .../lib/knowledge/mcp/route-handler.test.ts | 5 +- apps/sim/lib/knowledge/mcp/route-handler.ts | 3 +- .../lib/knowledge/mcp/server.protocol.test.ts | 165 +-- apps/sim/lib/knowledge/mcp/server.test.ts | 89 +- apps/sim/lib/knowledge/mcp/server.ts | 172 ++- .../lib/knowledge/orchestration/connectors.ts | 11 +- apps/sim/lib/knowledge/projection/enqueue.ts | 12 +- apps/sim/lib/knowledge/projection/run.ts | 23 +- .../lib/knowledge/search/activity-stats.ts | 100 -- .../sim/lib/knowledge/search/activity.test.ts | 67 - apps/sim/lib/knowledge/search/activity.ts | 38 - apps/sim/lib/knowledge/search/author.ts | 34 - .../knowledge/search/connection-attempt.ts | 1 - .../search/connection-target.test.ts | 5 +- .../lib/knowledge/search/connection-target.ts | 29 +- apps/sim/lib/knowledge/search/diagnostics.ts | 44 +- .../lib/knowledge/search/keyword-ranking.ts | 4 +- apps/sim/lib/knowledge/search/prewarm.test.ts | 9 +- apps/sim/lib/knowledge/search/prewarm.ts | 8 +- apps/sim/lib/knowledge/search/queries.ts | 26 +- .../search/source-vector-indexes.test.ts | 21 - .../knowledge/search/source-vector-indexes.ts | 24 - apps/sim/lib/knowledge/search/stats.test.ts | 63 - apps/sim/lib/knowledge/search/stats.ts | 74 -- .../load-search-integrations.test.ts | 75 +- .../application/load-search-integrations.ts | 10 - .../assistant/connected-account-tool.test.ts | 9 +- .../assistant/connected-account-tool.ts | 8 +- .../lib/mothership/assistant/tool-policy.ts | 5 +- apps/sim/lib/mothership/chat/payload.ts | 8 +- .../server/knowledge/workspace-search.test.ts | 229 +--- .../server/knowledge/workspace-search.ts | 237 +--- .../tools/server/search-sources.test.ts | 10 +- .../server/settings-connected-accounts.ts | 13 - .../application/execute-selector.test.ts | 38 - .../selectors/application/execute-selector.ts | 30 +- .../lib/selectors/server/credentials.test.ts | 88 -- apps/sim/lib/selectors/server/credentials.ts | 54 - .../providers/credential-bundle.test.ts | 49 - .../server/providers/credential-bundle.ts | 15 +- apps/sim/lib/selectors/server/types.ts | 5 - apps/sim/lib/selectors/types.ts | 7 - apps/sim/lib/sim-search/connectors.ts | 38 +- apps/sim/lib/sim-search/indexed/README.md | 39 - .../documents/read-indexed-document.ts | 230 ---- .../documents/read-search-document.test.ts | 267 ---- .../indexed/documents/read-search-document.ts | 173 --- apps/sim/lib/sim-search/indexed/gate.ts | 42 - apps/sim/lib/sim-search/indexed/index.ts | 18 - .../personal-account-ownership.ts | 29 - .../integrations/personal-inventory.ts | 43 - .../personal-search-integrations.ts | 164 --- .../sim-search/indexed/mcp/register-tools.ts | 148 --- .../indexed/retrieval/access-plan.ts | 188 --- .../lib/sim-search/indexed/retrieval/index.ts | 10 - .../sim-search/indexed/retrieval/keyword.ts | 325 ----- .../sim-search/indexed/retrieval/legs.test.ts | 995 --------------- .../lib/sim-search/indexed/retrieval/legs.ts | 128 -- .../sim-search/indexed/retrieval/permitted.ts | 424 ------- .../indexed/retrieval/projection-access.ts | 234 ---- .../indexed/retrieval/projection-fill.test.ts | 78 -- .../indexed/retrieval/projection-fill.ts | 102 -- .../retrieval/source-vector-indexes.ts | 33 - .../retrieval/tin-keyword-readiness.test.ts | 22 - .../indexed/retrieval/tin-keyword.test.ts | 41 - .../indexed/retrieval/tin-keyword.ts | 65 - .../indexed/retrieval/tin-query.test.ts | 23 - .../sim-search/indexed/retrieval/tin-query.ts | 195 --- .../sim-search/indexed/retrieval/vector.ts | 491 -------- .../search/scoped-search.activity.test.ts | 135 -- .../indexed/search/scoped-search.test.ts | 100 -- .../indexed/search/scoped-search.ts | 184 --- apps/sim/lib/sim-search/live/README.md | 2 +- apps/sim/lib/sim-search/live/application.ts | 8 - .../lib/sim-search/personal-source-setup.ts | 2 - apps/sim/lib/slack-search/connections.test.ts | 3 +- .../fixtures/desktop-source-connect.tsx | 25 +- apps/sim/tools/index.test.ts | 3 - apps/sim/tools/index.ts | 4 +- docker/app.Dockerfile | 2 - packages/db/knowledge-projection.test.ts | 245 ---- packages/db/knowledge-projection.ts | 272 +--- packages/db/schema.ts | 8 +- .../indexed-search-retirement.md | 152 +++ .../search-embedding-retirement.md | 4 +- .../src/mocks/deployment-shape.mock.ts | 2 - packages/testing/src/mocks/env-flags.mock.ts | 2 - .../src/mocks/indexed-org-search.mock.ts | 22 - .../src/mocks/kb-connectors-queries.mock.ts | 15 +- scripts/test-patterns-baseline.json | 1 - 259 files changed, 1538 insertions(+), 23430 deletions(-) delete mode 100644 apps/sim/app/api/knowledge/sim-search/connect/route.ts delete mode 100644 apps/sim/app/api/knowledge/sim-search/integrations/overview/route.ts delete mode 100644 apps/sim/app/api/knowledge/sim-search/personal-source-setup/route.ts delete mode 100644 apps/sim/app/api/knowledge/sim-search/sources/overview/route.ts delete mode 100644 apps/sim/app/api/knowledge/sim-search/sources/progress/route.ts delete mode 100644 apps/sim/app/api/knowledge/sim-search/stats/route.ts delete mode 100644 apps/sim/app/api/organizations/[id]/connected-accounts/indexing/route.ts delete mode 100644 apps/sim/app/o/[organizationId]/integrations/indexed/github-member-integration.tsx delete mode 100644 apps/sim/app/o/[organizationId]/integrations/indexed/index.ts delete mode 100644 apps/sim/app/o/[organizationId]/integrations/indexed/member-integration-row.tsx delete mode 100644 apps/sim/app/o/[organizationId]/integrations/indexed/member-integrations-list.tsx delete mode 100644 apps/sim/app/o/[organizationId]/integrations/indexed/source-status.ts delete mode 100644 apps/sim/app/o/[organizationId]/integrations/indexed/use-member-enrollment.test.tsx delete mode 100644 apps/sim/app/o/[organizationId]/integrations/indexed/use-member-enrollment.ts delete mode 100644 apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx delete mode 100644 apps/sim/app/o/[organizationId]/knowledge/[knowledgeBaseId]/[documentId]/page.tsx delete mode 100644 apps/sim/app/o/[organizationId]/knowledge/[knowledgeBaseId]/[documentId]/search-params.ts delete mode 100644 apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/index.ts delete mode 100644 apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/indexed-organization-integrations-settings.tsx delete mode 100644 apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-integrations-setup.tsx delete mode 100644 apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-search-stats-period.tsx delete mode 100644 apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-source-people.tsx delete mode 100644 apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-source-stats.tsx delete mode 100644 apps/sim/app/o/[organizationId]/settings/components/integrations/organization-integrations-settings.test.tsx delete mode 100644 apps/sim/app/o/[organizationId]/settings/components/integrations/organization-search-status.ts delete mode 100644 apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/search-params.ts delete mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/indexed/index.ts delete mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/indexed/indexed-search-results.tsx delete mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/search-sources/atlassian-source-setup-modal.tsx delete mode 100644 apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx delete mode 100644 apps/sim/ee/organization-search-stats/search-params.ts delete mode 100644 apps/sim/hooks/queries/organization-search-stats.ts delete mode 100644 apps/sim/hooks/queries/personal-source-setup.ts delete mode 100644 apps/sim/hooks/use-personal-source-account.test.tsx delete mode 100644 apps/sim/hooks/use-personal-source-account.ts delete mode 100644 apps/sim/lib/api/contracts/knowledge/personal-source-setup.test.ts delete mode 100644 apps/sim/lib/api/contracts/knowledge/personal-source-setup.ts delete mode 100644 apps/sim/lib/api/contracts/knowledge/search-stats.ts delete mode 100644 apps/sim/lib/credential-groups/application/organization-account-indexing.test.ts delete mode 100644 apps/sim/lib/credential-groups/application/organization-account-indexing.ts delete mode 100644 apps/sim/lib/knowledge/__integration__/organization-mcp-search.integration.ts delete mode 100644 apps/sim/lib/knowledge/__integration__/organization-search-overview.integration.ts delete mode 100644 apps/sim/lib/knowledge/__integration__/read-indexed-document.integration.ts delete mode 100644 apps/sim/lib/knowledge/__integration__/unfilled-projection-source.integration.ts delete mode 100644 apps/sim/lib/knowledge/application/organization-search-overview.test.ts delete mode 100644 apps/sim/lib/knowledge/application/organization-search-overview.ts delete mode 100644 apps/sim/lib/knowledge/application/organization-search-stats.test.ts delete mode 100644 apps/sim/lib/knowledge/application/organization-search-stats.ts delete mode 100644 apps/sim/lib/knowledge/application/personal-search-account.test.ts delete mode 100644 apps/sim/lib/knowledge/application/personal-search-account.ts delete mode 100644 apps/sim/lib/knowledge/application/personal-search-integration-pages.ts delete mode 100644 apps/sim/lib/knowledge/application/personal-source-setup.test.ts delete mode 100644 apps/sim/lib/knowledge/application/personal-source-setup.ts delete mode 100644 apps/sim/lib/knowledge/application/search-source-overview.ts delete mode 100644 apps/sim/lib/knowledge/application/search-source-progress.ts delete mode 100644 apps/sim/lib/knowledge/connectors/indexing-policy.test.ts delete mode 100644 apps/sim/lib/knowledge/connectors/organization-account-indexing.test.ts delete mode 100644 apps/sim/lib/knowledge/connectors/organization-account-indexing.ts delete mode 100644 apps/sim/lib/knowledge/connectors/viewer-member-sync-error.ts delete mode 100644 apps/sim/lib/knowledge/connectors/viewer-source-accounts.test.ts delete mode 100644 apps/sim/lib/knowledge/connectors/viewer-source-accounts.ts delete mode 100644 apps/sim/lib/knowledge/search/activity-stats.ts delete mode 100644 apps/sim/lib/knowledge/search/activity.test.ts delete mode 100644 apps/sim/lib/knowledge/search/activity.ts delete mode 100644 apps/sim/lib/knowledge/search/author.ts delete mode 100644 apps/sim/lib/knowledge/search/source-vector-indexes.test.ts delete mode 100644 apps/sim/lib/knowledge/search/source-vector-indexes.ts delete mode 100644 apps/sim/lib/knowledge/search/stats.test.ts delete mode 100644 apps/sim/lib/knowledge/search/stats.ts delete mode 100644 apps/sim/lib/sim-search/indexed/README.md delete mode 100644 apps/sim/lib/sim-search/indexed/documents/read-indexed-document.ts delete mode 100644 apps/sim/lib/sim-search/indexed/documents/read-search-document.test.ts delete mode 100644 apps/sim/lib/sim-search/indexed/documents/read-search-document.ts delete mode 100644 apps/sim/lib/sim-search/indexed/gate.ts delete mode 100644 apps/sim/lib/sim-search/indexed/index.ts delete mode 100644 apps/sim/lib/sim-search/indexed/integrations/personal-account-ownership.ts delete mode 100644 apps/sim/lib/sim-search/indexed/integrations/personal-inventory.ts delete mode 100644 apps/sim/lib/sim-search/indexed/integrations/personal-search-integrations.ts delete mode 100644 apps/sim/lib/sim-search/indexed/mcp/register-tools.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/access-plan.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/index.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/keyword.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/legs.test.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/legs.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/permitted.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/projection-access.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/projection-fill.test.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/projection-fill.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/source-vector-indexes.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/tin-keyword-readiness.test.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/tin-keyword.test.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/tin-keyword.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/tin-query.test.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/tin-query.ts delete mode 100644 apps/sim/lib/sim-search/indexed/retrieval/vector.ts delete mode 100644 apps/sim/lib/sim-search/indexed/search/scoped-search.activity.test.ts delete mode 100644 apps/sim/lib/sim-search/indexed/search/scoped-search.test.ts delete mode 100644 apps/sim/lib/sim-search/indexed/search/scoped-search.ts delete mode 100644 apps/sim/lib/sim-search/personal-source-setup.ts delete mode 100644 packages/db/knowledge-projection.test.ts create mode 100644 packages/db/script-migrations/indexed-search-retirement.md delete mode 100644 packages/testing/src/mocks/indexed-org-search.mock.ts diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml index c9d4f92a539..83964f3aee1 100644 --- a/.github/workflows/desktop-e2e.yml +++ b/.github/workflows/desktop-e2e.yml @@ -20,7 +20,6 @@ on: - 'apps/sim/hooks/queries/personal-search-integrations.ts' - 'apps/sim/hooks/use-search-integration-connection.ts' - 'apps/sim/hooks/use-github-installation-setup.ts' - - 'apps/sim/app/o/**/integrations/indexed/use-member-enrollment.ts' - 'apps/sim/lib/api/contracts/desktop-source-connect.ts' - 'apps/sim/scripts/fixtures/desktop-source-connect.tsx' - 'apps/sim/app/workspace/**/browser-session/**' diff --git a/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx b/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx index 392b0785f8d..99c522a3f49 100644 --- a/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx +++ b/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx @@ -207,7 +207,7 @@ The private key must include its PEM header, footer, and contents. Sim accepts a Keep the private key and client secret in the deployment's server configuration; organization admins select installations in Sim without entering these secrets. -The live Search application needs these five variables. If a worker also indexes ordinary GitHub knowledge bases, or Search has explicitly reverted to `SIM_SEARCH_LIVE=false`, configure the variables in that worker’s matching environment too. Updating the app’s secret store does not update a separately configured worker. Live Search does not schedule a GitHub indexing worker. +The live Search application needs these five variables. If a worker also indexes ordinary GitHub knowledge bases, configure the variables in that worker’s matching environment too. Updating the app’s secret store does not update a separately configured worker. Live Search does not schedule a GitHub indexing worker. The **Client ID** is different from the numeric **App ID**. Use credentials from **Developer settings → GitHub Apps**. `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` belong to the separate GitHub sign-in integration and remain unchanged. Search does not read `GITHUB_REPO_CLIENT_ID` or `GITHUB_REPO_CLIENT_SECRET`. diff --git a/apps/docs/content/docs/search/index.mdx b/apps/docs/content/docs/search/index.mdx index cb498ac484c..59c23f27f13 100644 --- a/apps/docs/content/docs/search/index.mdx +++ b/apps/docs/content/docs/search/index.mdx @@ -80,6 +80,6 @@ Conversations remain private to their author. Connecting an external account doe ## Legacy indexing and workspace knowledge bases -Live Search is the default. An operator can explicitly set `SIM_SEARCH_LIVE=false` to restore the legacy indexed Search backend. Its content sync, document processing, and stored permission maintenance are specific to that mode. These guides describe live Search. +Enterprise Search uses live provider queries. Search sources do not run content indexing or stored permission maintenance; ordinary knowledge-base connectors retain their indexing behavior. Ordinary workspace [knowledge bases](/knowledgebase) still ingest, chunk, and index documents for their own features. Their connector setup, processing status, and access settings remain separate. diff --git a/apps/docs/content/docs/search/mcp.mdx b/apps/docs/content/docs/search/mcp.mdx index 25fab917a3d..0e838e09fca 100644 --- a/apps/docs/content/docs/search/mcp.mdx +++ b/apps/docs/content/docs/search/mcp.mdx @@ -44,7 +44,7 @@ The connection applies to the organization whose URL you copied. Sim checks curr `chat` accepts questions up to 8,192 characters. Tool responses are limited to 1 MiB. API rate limits apply; follow any retry delay. Provider failures or incomplete retrieval are reported explicitly, and the tools do not fall back to an old index when a live provider is unavailable. -These are the live-backend schemas. If an operator explicitly selects legacy Search with `SIM_SEARCH_LIVE=false`, the server advertises its indexed search/read schemas instead. Refresh your MCP client's tool discovery after changing backends. +These are the Search MCP schemas. Refresh tool discovery in clients that previously connected to the retired indexed backend. ## Reconnect or revoke access diff --git a/apps/sim/app/api/knowledge/connectors/directory-sync/route.test.ts b/apps/sim/app/api/knowledge/connectors/directory-sync/route.test.ts index f9c6c7c53ae..d3fded53e48 100644 --- a/apps/sim/app/api/knowledge/connectors/directory-sync/route.test.ts +++ b/apps/sim/app/api/knowledge/connectors/directory-sync/route.test.ts @@ -4,7 +4,6 @@ import { hasMockCondition, resetEnvFlagsMock, schemaMock, - setEnvFlags, } from '@sim/testing' import { authInternalMock, authInternalMockFns } from '@sim/testing/mocks/auth-internal.mock' import { dbChainMockFns } from '@sim/testing/mocks/database.mock' @@ -108,24 +107,6 @@ describe('connector directory sync scheduler', () => { await expect(run()).resolves.toMatchObject({ dispatched: 1, failed: 1 }) }) - it.each([true, false])( - 'excludes Search directories from scheduled pages only when live Search is %s', - async (liveSearch) => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: liveSearch }) - mockConnectorRows.mockResolvedValue([]) - await run() - expect( - hasMockCondition( - mockWhere.mock.calls[0][0], - (node) => - node.type === 'eq' && - node.left === schemaMock.knowledgeBase.isSearchIndex && - node.right === false - ) - ).toBe(liveSearch) - } - ) - it('does not enqueue a connector another scheduler claimed or paused', async () => { mockConnectorRows.mockResolvedValue([connector()]) mockClaim.mockResolvedValueOnce([]) diff --git a/apps/sim/app/api/knowledge/search/route.ts b/apps/sim/app/api/knowledge/search/route.ts index 1186b6da0f0..610507ea53e 100644 --- a/apps/sim/app/api/knowledge/search/route.ts +++ b/apps/sim/app/api/knowledge/search/route.ts @@ -6,68 +6,9 @@ import { } from '@/lib/api/server/routes' import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { DEFAULT_RERANKER_MODEL } from '@/lib/knowledge/reranker-models' -import { sourceAuthor } from '@/lib/knowledge/search/author' -import { searchScopedKnowledge } from '@/lib/sim-search/indexed' -import { isIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' import { searchLiveKnowledge } from '@/lib/sim-search/live/application' -const DIRECT_SEARCH_VECTOR_BUDGET_MS = 3000 - -const indexedSearchRoute = defineInternalJsonRoute({ - contract: searchWorkspaceKnowledgeContract, - auth: internalSessionAuth, - operation: knowledgeOperations.search, - rateLimit: internalRateLimits.none({ - reason: - 'A person typing queries; the embedding call is metered against the canonical search owner', - }), - errorPolicy: internalKnowledgeErrorPolicies.search, - mapInput: ({ body }, { request }) => ({ - workspaceId: body.workspaceId, - organizationId: body.organizationId, - filters: body.filters, - query: body.query, - topK: body.topK, - allowPartialResults: true, - vectorBudgetMs: DIRECT_SEARCH_VECTOR_BUDGET_MS, - /** - * A person's search is reranked by a cross-encoder whenever the workspace or the platform - * holds a key for one; the use case checks that before spending a call, and reranking stays - * best-effort, so a provider outage leaves the fused order in place. - */ - rerankerEnabled: true, - rerankerModel: DEFAULT_RERANKER_MODEL, - surface: 'dashboard' as const, - signal: request.signal, - }), - useCase: searchScopedKnowledge, - present: ({ results, knowledgeBases, retrieval }, { input }) => { - const knowledgeBaseNames = new Map(knowledgeBases.map((kb) => [kb.id, kb.name])) - return { - success: true as const, - data: { - query: input.query ?? '', - retrieval, - results: results.map((result) => ({ - documentId: result.documentId, - knowledgeBaseId: result.knowledgeBaseId, - knowledgeBaseName: knowledgeBaseNames.get(result.knowledgeBaseId) ?? '', - documentName: result.documentName, - sourceUrl: result.sourceUrl, - connectorType: result.connectorType, - sourceModifiedAt: result.sourceModifiedAt?.toISOString() ?? null, - author: sourceAuthor(result.metadata), - content: result.content, - chunkIndex: result.chunkIndex, - similarity: result.similarity, - })), - }, - } - }, -}) - -const liveSearchRoute = defineInternalJsonRoute({ +export const POST = defineInternalJsonRoute({ contract: searchWorkspaceKnowledgeContract, auth: internalSessionAuth, operation: knowledgeOperations.search, @@ -80,6 +21,3 @@ const liveSearchRoute = defineInternalJsonRoute({ useCase: searchLiveKnowledge, present: (data) => ({ success: true as const, data }), }) - -/** Indexed organization search is dormant unless its gate is on; Live Search serves otherwise. */ -export const POST = isIndexedOrgSearchEnabled() ? indexedSearchRoute : liveSearchRoute diff --git a/apps/sim/app/api/knowledge/sim-search/connect/route.ts b/apps/sim/app/api/knowledge/sim-search/connect/route.ts deleted file mode 100644 index 521b6b570ef..00000000000 --- a/apps/sim/app/api/knowledge/sim-search/connect/route.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { connectSimSearchConnectorContract } from '@/lib/api/contracts/knowledge' -import { - defineInternalJsonRoute, - internalRateLimits, - internalSessionAuth, -} from '@/lib/api/server/routes' -import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { connectSimSearchConnector } from '@/lib/knowledge/application/sim-search' - -export const POST = defineInternalJsonRoute({ - contract: connectSimSearchConnectorContract, - auth: internalSessionAuth, - operation: knowledgeOperations.simSearchConnect, - rateLimit: internalRateLimits.none({ reason: 'One click per source; mints a single-use link' }), - errorPolicy: internalKnowledgeErrorPolicies.connectAccount, - mapInput: ({ body }) => body, - useCase: connectSimSearchConnector, - present: (result) => ({ success: true as const, data: result }), -}) diff --git a/apps/sim/app/api/knowledge/sim-search/integrations/overview/route.ts b/apps/sim/app/api/knowledge/sim-search/integrations/overview/route.ts deleted file mode 100644 index 325f40ea7c8..00000000000 --- a/apps/sim/app/api/knowledge/sim-search/integrations/overview/route.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { readOrganizationSearchOverviewContract } from '@/lib/api/contracts/knowledge/connectors' -import { - defineInternalJsonRoute, - internalRateLimits, - internalSessionAuth, -} from '@/lib/api/server/routes' -import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { readOrganizationSearchOverview } from '@/lib/knowledge/application/organization-search-overview' - -export const GET = defineInternalJsonRoute({ - contract: readOrganizationSearchOverviewContract, - auth: internalSessionAuth, - operation: knowledgeOperations.readOrganizationSearchOverview, - rateLimit: internalRateLimits.none({ - reason: 'Bounded provider operational aggregates for organization integration settings', - }), - errorPolicy: internalKnowledgeErrorPolicies.connectors, - mapInput: ({ query }) => query, - useCase: readOrganizationSearchOverview, - present: (overview) => ({ success: true as const, data: overview }), - staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, -}) diff --git a/apps/sim/app/api/knowledge/sim-search/personal-source-setup/route.ts b/apps/sim/app/api/knowledge/sim-search/personal-source-setup/route.ts deleted file mode 100644 index 8fff7a8ff6e..00000000000 --- a/apps/sim/app/api/knowledge/sim-search/personal-source-setup/route.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { - listPersonalSourceSetupAccountsContract, - personalSourceSetupContract, -} from '@/lib/api/contracts/knowledge/personal-source-setup' -import { - defineInternalJsonRoute, - extendInternalErrorPolicy, - internalErrorResponse, - internalRateLimits, - internalSessionAuth, -} from '@/lib/api/server/routes' -import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { - listPersonalSourceSetupAccounts, - personalSourceSetup, -} from '@/lib/knowledge/application/personal-source-setup' -import { - SelectorConnectionUnavailableError, - SelectorContextUnavailableError, - SelectorOptionsUnavailableError, -} from '@/lib/selectors/server/errors' -import { IntegrationNotAllowedError } from '@/ee/access-control/utils/permission-check' - -const errorPolicy = extendInternalErrorPolicy( - internalKnowledgeErrorPolicies.connectAccount, - (error) => { - if (error instanceof SelectorConnectionUnavailableError) - return internalErrorResponse(error.status, { - error: 'Reconnect your account to choose projects or spaces', - }) - if (error instanceof SelectorContextUnavailableError) - return internalErrorResponse(400, { - error: 'Enter your Atlassian site to choose projects or spaces', - }) - if (error instanceof SelectorOptionsUnavailableError) - return internalErrorResponse(error.status, { - error: - 'Could not load projects or spaces. Check the site and account access, then try again.', - }) - if (error instanceof IntegrationNotAllowedError) - return internalErrorResponse(403, { error: error.message }) - return null - } -) - -export const GET = defineInternalJsonRoute({ - contract: listPersonalSourceSetupAccountsContract, - auth: internalSessionAuth, - operation: knowledgeOperations.listPersonalSourceSetupAccounts, - rateLimit: internalRateLimits.user({ bucketName: 'knowledge.search.personal-setup.accounts' }), - errorPolicy, - mapInput: ({ query }) => query, - useCase: listPersonalSourceSetupAccounts, - present: (data) => ({ success: true as const, data }), - staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, -}) - -export const POST = defineInternalJsonRoute({ - contract: personalSourceSetupContract, - auth: internalSessionAuth, - operation: knowledgeOperations.personalSourceSetup, - rateLimit: internalRateLimits.user({ bucketName: 'knowledge.search.personal-setup' }), - errorPolicy, - parseOptions: { maxBodyBytes: 384 * 1024 }, - mapInput: ({ body }) => body, - useCase: personalSourceSetup, - present: (data) => ({ success: true as const, data }), - staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, -}) diff --git a/apps/sim/app/api/knowledge/sim-search/sources/overview/route.ts b/apps/sim/app/api/knowledge/sim-search/sources/overview/route.ts deleted file mode 100644 index f6bdf9abb5e..00000000000 --- a/apps/sim/app/api/knowledge/sim-search/sources/overview/route.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { readSearchSourceOverviewContract } from '@/lib/api/contracts/knowledge/connectors' -import { - defineInternalJsonRoute, - internalRateLimits, - internalSessionAuth, -} from '@/lib/api/server/routes' -import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { readSearchSourceOverview } from '@/lib/knowledge/application/search-source-overview' - -export const GET = defineInternalJsonRoute({ - contract: readSearchSourceOverviewContract, - auth: internalSessionAuth, - operation: knowledgeOperations.readSearchSourceOverview, - rateLimit: internalRateLimits.none({ - reason: 'Bounded provider existence probes for source setup and indexing progress', - }), - errorPolicy: internalKnowledgeErrorPolicies.connectors, - mapInput: ({ query }) => query, - useCase: readSearchSourceOverview, - present: (overview) => ({ success: true as const, data: overview }), - staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, -}) diff --git a/apps/sim/app/api/knowledge/sim-search/sources/progress/route.ts b/apps/sim/app/api/knowledge/sim-search/sources/progress/route.ts deleted file mode 100644 index a6a793a28af..00000000000 --- a/apps/sim/app/api/knowledge/sim-search/sources/progress/route.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { readSearchSourceProgressContract } from '@/lib/api/contracts/knowledge/connectors' -import { - defineInternalJsonRoute, - internalRateLimits, - internalSessionAuth, -} from '@/lib/api/server/routes' -import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { readSearchSourceProgress } from '@/lib/knowledge/application/search-source-progress' - -export const POST = defineInternalJsonRoute({ - contract: readSearchSourceProgressContract, - auth: internalSessionAuth, - operation: knowledgeOperations.readSearchSourceProgress, - rateLimit: internalRateLimits.none({ - reason: 'Bounded viewer-authorized indexing progress polling', - }), - errorPolicy: internalKnowledgeErrorPolicies.connectors, - mapInput: ({ body }) => body, - useCase: readSearchSourceProgress, - present: ({ sources }) => ({ success: true as const, data: sources }), - staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, -}) diff --git a/apps/sim/app/api/knowledge/sim-search/sources/route.test.ts b/apps/sim/app/api/knowledge/sim-search/sources/route.test.ts index 4b2f0c54243..d255e78bcf6 100644 --- a/apps/sim/app/api/knowledge/sim-search/sources/route.test.ts +++ b/apps/sim/app/api/knowledge/sim-search/sources/route.test.ts @@ -5,32 +5,16 @@ import type { SearchSourceSummary } from '@/lib/api/contracts/knowledge/connecto const mocks = vi.hoisted(() => ({ execute: vi.fn(), - overview: vi.fn(), - adminOverview: vi.fn(), -})) -vi.mock('@/lib/knowledge/application/organization-search-overview', () => ({ - readOrganizationSearchOverview: { - operation: { id: 'knowledge.search.integrations.overview' }, - execute: mocks.adminOverview, - }, })) vi.mock('@/lib/knowledge/application/search-sources', () => ({ listSearchSources: { operation: { id: 'knowledge.search.sources.list' }, execute: mocks.execute }, })) -vi.mock('@/lib/knowledge/application/search-source-overview', () => ({ - readSearchSourceOverview: { - operation: { id: 'knowledge.search.sources.overview' }, - execute: mocks.overview, - }, -})) vi.mock('@/lib/knowledge/application/search', () => knowledgeSearchUseCaseMock) vi.mock('@/lib/knowledge/application/upload-sessions', () => ({ KnowledgeDocumentUnsupportedMediaTypeError: class extends Error {}, })) import { NoWorkspaceAccessError } from '@/lib/core/application/workspace-authorization' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { GET as getAdminOverview } from '@/app/api/knowledge/sim-search/integrations/overview/route' import { GET } from '@/app/api/knowledge/sim-search/sources/route' const WORKSPACE_ID = '7d28e5e2-fb03-4118-9c52-4ab77ccff369' @@ -42,15 +26,7 @@ const source = { accessMode: 'admin', availability: 'available', enabled: true, - isSyncing: false, - lastSyncAt: null, - hasSyncError: false, - hasViewerDocuments: false, - viewerFailedDocumentCount: 0, - viewerEmailVerified: true, - viewerAccounts: [], - connectionRequired: false, - viewerMembership: null, + isGitHubInstallation: false, } satisfies SearchSourceSummary beforeEach(() => { @@ -112,21 +88,3 @@ describe('Search pagination boundary', () => { expect(mocks.execute).not.toHaveBeenCalled() }) }) - -describe('organization administration overview boundary', () => { - it('preserves a role refusal without exposing health data', async () => { - mocks.adminOverview.mockRejectedValue( - new OrchestrationError('forbidden', 'Organization administrator access is required') - ) - const response = await getAdminOverview( - createMockRequest( - 'GET', - undefined, - {}, - `http://localhost/api/knowledge/sim-search/integrations/overview?organizationId=${WORKSPACE_ID}` - ) - ) - expect(response.status).toBe(403) - expect(await response.json()).not.toHaveProperty('data') - }) -}) diff --git a/apps/sim/app/api/knowledge/sim-search/stats/route.ts b/apps/sim/app/api/knowledge/sim-search/stats/route.ts deleted file mode 100644 index e7ae7eb9777..00000000000 --- a/apps/sim/app/api/knowledge/sim-search/stats/route.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { readOrganizationSearchStatsContract } from '@/lib/api/contracts/knowledge/search-stats' -import { - defineInternalJsonRoute, - internalRateLimits, - internalSessionAuth, -} from '@/lib/api/server/routes' -import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { readOrganizationSearchStats } from '@/lib/knowledge/application/organization-search-stats' - -export const GET = defineInternalJsonRoute({ - contract: readOrganizationSearchStatsContract, - auth: internalSessionAuth, - operation: knowledgeOperations.readOrganizationSearchStats, - rateLimit: internalRateLimits.user({ - bucketName: 'organization-search-stats', - config: { maxTokens: 30, refillRate: 30, refillIntervalMs: 60_000 }, - }), - errorPolicy: internalKnowledgeErrorPolicies.connectors, - mapInput: ({ query }) => query, - useCase: readOrganizationSearchStats, - present: (data) => ({ success: true as const, data }), - staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, -}) diff --git a/apps/sim/app/api/organizations/[id]/connected-accounts/indexing/route.ts b/apps/sim/app/api/organizations/[id]/connected-accounts/indexing/route.ts deleted file mode 100644 index d79a286c00f..00000000000 --- a/apps/sim/app/api/organizations/[id]/connected-accounts/indexing/route.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { updateOrganizationAccountIndexingContract } from '@/lib/api/contracts/organization-accounts' -import { - defineInternalJsonRoute, - internalOrchestrationErrorPolicy, - internalRateLimits, - internalSessionAuth, -} from '@/lib/api/server/routes' -import { - updateOrganizationAccountIndexing, - updateOrganizationAccountIndexingOperation, -} from '@/lib/credential-groups/application/organization-account-indexing' - -export const PUT = defineInternalJsonRoute({ - contract: updateOrganizationAccountIndexingContract, - auth: internalSessionAuth, - operation: updateOrganizationAccountIndexingOperation, - rateLimit: internalRateLimits.user({ bucketName: 'organization-account-indexing' }), - errorPolicy: internalOrchestrationErrorPolicy, - mapInput: ({ params, body }) => ({ organizationId: params.id, ...body }), - useCase: updateOrganizationAccountIndexing, - present: ({ enabled, knowledgeBaseIds }) => ({ enabled, knowledgeBaseIds }), -}) diff --git a/apps/sim/app/api/v1/knowledge/search/route.ts b/apps/sim/app/api/v1/knowledge/search/route.ts index 2a1b97b5b3e..91c1a56a91c 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.ts @@ -24,7 +24,6 @@ import { import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service' import { buildUndefinedTagsError, validateTagValue } from '@/lib/knowledge/tags/utils' import type { StructuredFilter } from '@/lib/knowledge/types' -import { usesIndexedRetrieval } from '@/lib/sim-search/indexed/gate' import { checkKnowledgeBaseAccess, type KnowledgeBaseAccessResult } from '@/app/api/knowledge/utils' import { handleError, resolveV1KnowledgeReadAccess } from '@/app/api/v1/knowledge/utils' import { @@ -251,7 +250,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { accessProvider, searchMode, boostRecency, - indexedRetrieval: usesIndexedRetrieval(accessibleKbs), structuredFilters, }) } else if (hasQuery) { @@ -268,7 +266,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { accessProvider, searchMode, boostRecency, - indexedRetrieval: usesIndexedRetrieval(accessibleKbs), query, queryVector: { vector: JSON.stringify(queryEmbeddingResult.embedding), diff --git a/apps/sim/app/o/[organizationId]/home/components/composer/composer.test.tsx b/apps/sim/app/o/[organizationId]/home/components/composer/composer.test.tsx index 72e85c1c61c..8e365f58e08 100644 --- a/apps/sim/app/o/[organizationId]/home/components/composer/composer.test.tsx +++ b/apps/sim/app/o/[organizationId]/home/components/composer/composer.test.tsx @@ -18,7 +18,6 @@ import type { useSpeechToText } from '@/hooks/use-speech-to-text' import { useMothershipEffortStore } from '@/stores/mothership-effort/store' const mocks = vi.hoisted(() => ({ - live: false, plan: false, advanced: false, speech: vi.fn(), @@ -90,10 +89,9 @@ import type { ChatRequestMode } from '@/app/workspace/[workspaceId]/home/types' import { FeatureFlagsProvider } from '@/app/workspace/[workspaceId]/providers/feature-flags-provider' import { useFileAttachments } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-file-attachments' -const liveShape = () => - createMockDeploymentShape({ features: { liveEnterpriseSearch: mocks.live } }) -deploymentShapeMockFns.mockUseDeploymentShape.mockImplementation(liveShape) -deploymentShapeMockFns.mockGetDeploymentShape.mockImplementation(liveShape) +const deploymentShape = () => createMockDeploymentShape() +deploymentShapeMockFns.mockUseDeploymentShape.mockImplementation(deploymentShape) +deploymentShapeMockFns.mockGetDeploymentShape.mockImplementation(deploymentShape) organizationProviderMockFns.mockUseOrganizationContext.mockReturnValue({ organization: { id: 'organization-a' }, }) @@ -109,7 +107,6 @@ beforeEach(() => { modelSelection: { model: 'gpt-6-astra', fastMode: false }, }) mocks.plan = false - mocks.live = false vi.clearAllMocks() mocks.workspaces = [ { diff --git a/apps/sim/app/o/[organizationId]/home/components/get-started/get-started.tsx b/apps/sim/app/o/[organizationId]/home/components/get-started/get-started.tsx index 90d272b5883..ac6e75ecd0b 100644 --- a/apps/sim/app/o/[organizationId]/home/components/get-started/get-started.tsx +++ b/apps/sim/app/o/[organizationId]/home/components/get-started/get-started.tsx @@ -6,11 +6,10 @@ import { ArrowRight } from '@sim/emcn/icons' import Link from 'next/link' import { HomeSection } from '@/components/home/home-section' import { OAUTH_SEARCH_READ_SCOPE, oauthScopeSatisfies } from '@/lib/auth/oauth-provider' -import type { ResourceScope } from '@/lib/core/resource-scope' import { organizationRoutes } from '@/lib/navigation/paths' import { useOrganizationContext } from '@/app/o/[organizationId]/providers/organization-provider' -import { useSearchSourceOverview } from '@/hooks/queries/kb/connectors' import { useAuthorizedApps } from '@/hooks/queries/oauth-provider' +import { useOrganizationAccounts } from '@/hooks/queries/organization-accounts' type StepId = 'connect-integration' | 'connect-sim-search' @@ -67,14 +66,12 @@ function StepMark({ complete }: { complete: boolean }) { * The organization home's onboarding list under the composer. Same chrome as * the workspace home's suggested actions: a hover-revealed disclosure header * over hairline-separated rows. Each step leads to the page that completes it, - * and reads as done from the organization's real state: a source the viewer can - * search and an OAuth app authorized to use Search. + * and reads as done from the organization's real state: a connected account and an OAuth app authorized to use Search. */ export function GetStarted() { const { organization, viewer } = useOrganizationContext() const routes = organizationRoutes(organization.id) - const scope: ResourceScope = { kind: 'organization', organizationId: organization.id } - const { data: overview } = useSearchSourceOverview(scope) + const { data: accounts } = useOrganizationAccounts(organization.id) const { data: authorizedApps, fetchNextPage, @@ -94,7 +91,10 @@ export function GetStarted() { 'connect-sim-search': routes.settingsSection('search-mcp'), } const completed: Record = { - 'connect-integration': overview?.hasSearchableDocuments === true, + 'connect-integration': Boolean( + accounts?.viewerAccounts?.some((account) => account.status === 'active') || + accounts?.viewerMcpAccounts?.some((account) => account.status === 'active') + ), 'connect-sim-search': hasSearchAuthorization, } const steps = STEPS.filter((step) => step.id !== 'connect-sim-search' || viewer.canUseSearchMcp) diff --git a/apps/sim/app/o/[organizationId]/home/organization-home.test.tsx b/apps/sim/app/o/[organizationId]/home/organization-home.test.tsx index f7da9cd270f..aa3b1c5a65a 100644 --- a/apps/sim/app/o/[organizationId]/home/organization-home.test.tsx +++ b/apps/sim/app/o/[organizationId]/home/organization-home.test.tsx @@ -8,10 +8,7 @@ import { deploymentShapeMockFns, } from '@sim/testing/mocks/deployment-shape.mock' import { integrationMatcherMock } from '@sim/testing/mocks/integration-matcher.mock' -import { - kbConnectorsQueriesMock, - kbConnectorsQueriesMockFns, -} from '@sim/testing/mocks/kb-connectors-queries.mock' +import { kbConnectorsQueriesMock } from '@sim/testing/mocks/kb-connectors-queries.mock' import { nextNavigationMock } from '@sim/testing/mocks/next-navigation.mock' import { organizationProviderMock, @@ -26,7 +23,6 @@ import { useMothershipDraftsStore } from '@/stores/mothership-drafts/store' import { useOrganizationChatModeStore } from '@/stores/organization-chat-mode/store' const mocks = vi.hoisted(() => ({ - live: false, plan: false, resourcePanel: vi.fn(), chat: vi.fn(), @@ -99,18 +95,15 @@ import { OrganizationHome } from '@/app/o/[organizationId]/home/organization-hom const mockSession = authClientMockFns.mockUseSession const mockContext = organizationProviderMockFns.mockUseOrganizationContext -const mockSources = kbConnectorsQueriesMockFns.mockUseSearchSourceOverview -const liveShape = () => - createMockDeploymentShape({ features: { liveEnterpriseSearch: mocks.live } }) -deploymentShapeMockFns.mockUseDeploymentShape.mockImplementation(liveShape) -deploymentShapeMockFns.mockGetDeploymentShape.mockImplementation(liveShape) +const deploymentShape = () => createMockDeploymentShape() +deploymentShapeMockFns.mockUseDeploymentShape.mockImplementation(deploymentShape) +deploymentShapeMockFns.mockGetDeploymentShape.mockImplementation(deploymentShape) let root: Root let container: HTMLDivElement beforeEach(() => { useMothershipDraftsStore.setState({ drafts: {} }) mocks.plan = false - mocks.live = false mocks.activeResource = null mockSession.mockReturnValue({ data: { user: { id: 'reader' } } }) useOrganizationChatModeStore.setState({ modes: {}, assistantSearchLevels: {} }) @@ -131,7 +124,6 @@ beforeEach(() => { canBuild: true, viewer: { isAdmin: false, canUseSearchMcp: true }, }) - mockSources.mockReturnValue({ data: { providers: [], hasSearchableDocuments: false } }) mocks.apiKeys.mockReturnValue({ data: { personalKeys: [] } }) mockAuthorizedApps([{ apps: [], nextCursor: null }]) mocks.chat.mockReturnValue({ diff --git a/apps/sim/app/o/[organizationId]/home/organization-home.tsx b/apps/sim/app/o/[organizationId]/home/organization-home.tsx index 7baabe357df..b8619c5057c 100644 --- a/apps/sim/app/o/[organizationId]/home/organization-home.tsx +++ b/apps/sim/app/o/[organizationId]/home/organization-home.tsx @@ -9,7 +9,6 @@ import { requestJson } from '@/lib/api/client/request' import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge' import { getWorkspaceHostContextContract } from '@/lib/api/contracts/workspaces' import { useSession } from '@/lib/auth/auth-client' -import { getDeploymentShape } from '@/lib/core/config/deployment-shape' import { MothershipHandoffStorage } from '@/lib/core/utils/browser-storage' import { getMothershipAttachmentPreviewUrl, @@ -139,7 +138,6 @@ function OrganizationHomeContent({ const hasChat = Boolean(chatId || chat.messages.length) const canSelectMode = !hasChat && mothershipAvailable && canBuild && (searchAccess.memberScoped || planEnabled) - const liveSearch = getDeploymentShape().features.liveEnterpriseSearch === true const assistantSearchLevel = 'fast' const panel = useChatResourcePanel(chat, controller) const addResource = panel.addResourceFromUser @@ -257,7 +255,7 @@ function OrganizationHomeContent({ ...(handoff.assistantSearch ? { assistantSearch: handoff.assistantSearch } : {}), }) } - }, [chatId, organization.id, requestMode, sendMessage, assistantSearchLevel, liveSearch]) + }, [chatId, organization.id, requestMode, sendMessage, assistantSearchLevel]) const send = ( message: string, @@ -390,11 +388,7 @@ function OrganizationHomeContent({ chatId={chat.resolvedChatId} composer={composer} onWorkspaceResourceSelect={requestMode !== 'assistant' ? selectResource : undefined} - initialScrollBlocked={ - (requestMode !== 'assistant' || liveSearch) && - chat.resources.length > 0 && - panel.isResourceCollapsed - } + initialScrollBlocked={chat.resources.length > 0 && panel.isResourceCollapsed} /> ) : ( - canConnect: boolean -} - -/** A member authorizes GitHub once, independently of the repositories added by admins. */ -export function GitHubMemberIntegration({ - organizationId, - inventory, - canConnect, -}: GitHubMemberIntegrationProps) { - const connect = useConnectOrganizationAccount() - const reconnect = useReconnectPersonalOrganizationAccount() - const accounts = - inventory.data?.viewerAccounts?.filter( - (account) => account.providerId === 'github-repositories' - ) ?? [] - const account = accounts.find((entry) => entry.status === 'needs_reauth') ?? accounts[0] - const option = - inventory.data?.credentialGroup?.status === 'active' - ? inventory.data.credentialGroup.options.find( - (entry) => entry.provider === 'github-repositories' && entry.status === 'active' - ) - : undefined - const loading = inventory.isPending && !inventory.data - const failed = inventory.isError - const meta = CONNECTOR_META_REGISTRY.github - const onError = (error: Error) => toast.error(error.message) - const description = account - ? `${accounts.map((entry) => entry.displayName).join(', ')} · ${account.status === 'needs_reauth' ? 'Reconnect required' : 'Connected'}` - : loading - ? 'Loading connection' - : failed - ? 'Could not load connection' - : option - ? 'Connect once to search the repositories your admin adds' - : 'An admin needs to reconnect GitHub' - - return ( - : undefined} - title='GitHub' - description={description} - trailing={ -
- - {failed ? ( - void inventory.refetch()}> - {inventory.isFetching ? 'Retrying' : 'Retry'} - - ) : account?.status === 'needs_reauth' && option?.id === account.optionId ? ( - reconnect.mutate(account.credentialId, { onError })} - > - Reconnect - - ) : !account && !loading && canConnect && option ? ( - connect.mutate({ organizationId, optionId: option.id }, { onError })} - > - Connect - - ) : null} -
- } - /> - ) -} diff --git a/apps/sim/app/o/[organizationId]/integrations/indexed/index.ts b/apps/sim/app/o/[organizationId]/integrations/indexed/index.ts deleted file mode 100644 index 9b1281fd906..00000000000 --- a/apps/sim/app/o/[organizationId]/integrations/indexed/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { MemberIntegrationsList } from '@/app/o/[organizationId]/integrations/indexed/member-integrations-list' diff --git a/apps/sim/app/o/[organizationId]/integrations/indexed/member-integration-row.tsx b/apps/sim/app/o/[organizationId]/integrations/indexed/member-integration-row.tsx deleted file mode 100644 index 6619a285d71..00000000000 --- a/apps/sim/app/o/[organizationId]/integrations/indexed/member-integration-row.tsx +++ /dev/null @@ -1,223 +0,0 @@ -'use client' - -import { Chip, ChipLink } from '@sim/emcn' -import { organizationRoutes } from '@/lib/navigation/paths' -import { connectorDisplayName } from '@/lib/sim-search/connectors' -import { DisconnectAccountMenu } from '@/app/o/[organizationId]/integrations/disconnect-account-menu' -import { getSearchSourceStatus } from '@/app/o/[organizationId]/integrations/indexed/source-status' -import { - CONNECTABLE_MEMBERSHIPS, - enrollmentActionLabel, - type useMemberEnrollment, -} from '@/app/o/[organizationId]/integrations/indexed/use-member-enrollment' -import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' -import type { RowAction } from '@/app/workspace/[workspaceId]/settings/components/row-actions-menu' -import { SettingsResourceRow } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' -import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' -import type { useSearchSources } from '@/hooks/queries/kb/connectors' - -interface MemberIntegrationRowProps { - organizationId: string - connectorType: string - configured: boolean - sources: ReturnType - enrollment: ReturnType - memberAccessAvailable: boolean - mirroredAccessAvailable: boolean - onCreate?: () => void - addLabel?: string -} - -/** One flat account row per integration, independent of how many content scopes it indexes. */ -export function MemberIntegrationRow({ - organizationId, - connectorType, - configured, - sources, - enrollment, - memberAccessAvailable, - mirroredAccessAvailable, - onCreate, - addLabel, -}: MemberIntegrationRowProps) { - const name = connectorDisplayName(connectorType) - const meta = CONNECTOR_META_REGISTRY[connectorType] - const rows = sources.data ?? [] - const accounts = [ - ...new Map( - rows - .flatMap((source) => source.viewerAccounts) - .map((account) => [account.credentialId, account]) - ).values(), - ] - const accountLabels = new Map() - for (const account of accounts) { - if (accounts.filter((other) => other.displayName === account.displayName).length < 2) continue - const descriptions = [ - ...new Set( - rows - .filter((source) => - source.viewerAccounts.some((other) => other.credentialId === account.credentialId) - ) - .map((source) => source.sourceDescription) - .filter(Boolean) - ), - ] - const context = [ - account.status === 'needs_reauth' ? 'Reconnect required' : undefined, - descriptions.length > 1 ? `${descriptions[0]} +${descriptions.length - 1}` : descriptions[0], - ].filter(Boolean) - accountLabels.set(account.credentialId, [...context, account.displayName].join(' · ')) - } - for (const label of new Set(accountLabels.values())) { - const matching = accounts.filter((account) => accountLabels.get(account.credentialId) === label) - if (matching.length < 2) continue - matching.forEach((account, index) => - accountLabels.set(account.credentialId, `Connection ${index + 1} · ${label}`) - ) - } - const isUsable = (source: (typeof rows)[number]) => - source.availability === 'available' && - (source.accessMode === 'members' - ? memberAccessAvailable - : mirroredAccessAvailable && (!source.connectionRequired || memberAccessAvailable)) - const eligible = rows.filter( - (source) => - source.connectionRequired && - source.viewerEmailVerified && - source.enabled && - source.approved !== false && - isUsable(source) && - source.viewerMembership !== null && - CONNECTABLE_MEMBERSHIPS.has(source.viewerMembership) - ) - const target = - eligible.find((source) => source.viewerMembership === 'needs_reauth') ?? eligible[0] - const hasLoadError = configured && sources.isError && !sources.isFetchNextPageError - const ready = !configured || (!sources.isPending && !hasLoadError) - const allCentral = - rows.length > 0 && !sources.hasNextPage && rows.every((source) => !source.connectionRequired) - const needsEmailVerification = rows.some( - (source) => - source.enabled && source.approved !== false && isUsable(source) && !source.viewerEmailVerified - ) - const waiting = target - ? enrollment.isAwaiting(target.connectorId) - : enrollment.isAwaitingSource(connectorType) - const status = (source: (typeof rows)[number]) => - getSearchSourceStatus({ - source, - scopeKind: 'organization', - supported: meta?.search === true, - usable: isUsable(source), - connectable: eligible.includes(source), - waiting: enrollment.isAwaiting(source.connectorId), - }) - function description() { - if (!configured) return waiting ? 'Finish connecting in the other tab' : 'Not connected' - if (hasLoadError) return 'Could not load connection' - if (sources.isPending) return 'Loading connection' - if (target) { - if (waiting) return 'Finish connecting in the other tab' - if (target.viewerMembership === 'needs_reauth') return 'Reconnect your account' - const hasConnectedContent = rows.some( - (source) => - source.enabled && - source.approved !== false && - isUsable(source) && - (!source.connectionRequired || source.viewerMembership === 'connected') - ) - return hasConnectedContent ? 'Additional connection required' : 'Not connected' - } - if (rows.length === 1 && !sources.hasNextPage) return status(rows[0]) - if (needsEmailVerification) return 'Verify your email' - if ( - rows.some( - (source) => - !source.enabled || - source.approved === false || - !isUsable(source) || - source.viewerMembership === 'revoked' || - (source.connectionRequired && source.viewerMembership === null) - ) - ) - return 'Some connections need attention' - if (rows.some((source) => source.hasSyncError || source.viewerFailedDocumentCount > 0)) - return 'Sync needs attention' - if (rows.some((source) => source.isSyncing)) return 'Indexing' - if (sources.hasNextPage) - return sources.isFetchNextPageError - ? 'Could not check remaining connections' - : 'More connections to check' - if (allCentral) return 'Connected by your organization' - return accounts.length ? 'Connected' : 'No connected content' - } - const actions: RowAction[] = [] - if (configured && ready && !sources.hasNextPage && !allCentral && addLabel && onCreate) - actions.push({ label: addLabel, onSelect: onCreate, disabled: enrollment.isPending }) - const canCheckConnections = - configured && ready && sources.hasNextPage && !target && !needsEmailVerification - const canConnect = ready && (target || (!configured && onCreate)) - - return ( - : undefined} - title={name} - description={description()} - trailing={ -
- - {ready && needsEmailVerification && ( - - Verify email - - )} - {hasLoadError && ( - void sources.refetch()}> - {sources.isFetching ? 'Retrying' : 'Retry'} - - )} - {canCheckConnections && ( - void sources.fetchNextPage({ cancelRefetch: false })} - > - {sources.isFetchingNextPage - ? 'Checking' - : sources.isFetchNextPageError - ? 'Retry' - : 'Check connections'} - - )} - {canConnect && ( - - target - ? enrollment.connect(target.knowledgeBaseId, target.connectorId) - : onCreate?.() - } - > - {target?.viewerMembership - ? enrollmentActionLabel(target.viewerMembership, waiting) - : waiting - ? 'Open again' - : 'Connect'} - - )} -
- } - /> - ) -} diff --git a/apps/sim/app/o/[organizationId]/integrations/indexed/member-integrations-list.tsx b/apps/sim/app/o/[organizationId]/integrations/indexed/member-integrations-list.tsx deleted file mode 100644 index bcb1692d8eb..00000000000 --- a/apps/sim/app/o/[organizationId]/integrations/indexed/member-integrations-list.tsx +++ /dev/null @@ -1,268 +0,0 @@ -'use client' - -import { useMemo } from 'react' -import { toast } from '@sim/emcn' -import type { ResourceScope } from '@/lib/core/resource-scope' -import { getSearchConnectionLabels } from '@/lib/sim-search/connection-labels' -import { - getConnectorAccessAvailability, - SEARCH_CONNECTORS, - SEARCH_SOURCE_TYPES, - type SearchConnector, -} from '@/lib/sim-search/connectors' -import { GitHubMemberIntegration } from '@/app/o/[organizationId]/integrations/indexed/github-member-integration' -import { MemberIntegrationRow } from '@/app/o/[organizationId]/integrations/indexed/member-integration-row' -import { useMemberEnrollment } from '@/app/o/[organizationId]/integrations/indexed/use-member-enrollment' -import { useOrganizationContext } from '@/app/o/[organizationId]/providers/organization-provider' -import { SourceSetupModal } from '@/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal' -import { - SettingsEmptyState, - SettingsQueryErrorState, -} from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' -import { RESOURCE_LIST_STACK } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' -import { useSearchSourceOverview, useSearchSources } from '@/hooks/queries/kb/connectors' -import { - organizationAccountsKeys, - useOrganizationAccounts, -} from '@/hooks/queries/organization-accounts' -import { usePersonalSearchIntegrations } from '@/hooks/queries/personal-search-integrations' -import { useSearchIntegrations } from '@/hooks/queries/search-integrations' -import { searchSourceKeys } from '@/hooks/queries/utils/search-source-keys' -import { usePermissionConfig } from '@/hooks/use-permission-config' - -interface MemberIntegrationsListProps { - search?: string - showEmpty?: boolean -} - -/** Provider existence comes from the complete overview; account status uses bounded source pages. */ -export function MemberIntegrationsList({ - search = '', - showEmpty = true, -}: MemberIntegrationsListProps = {}) { - const { organization, searchAccess } = useOrganizationContext() - const scope: ResourceScope = { kind: 'organization', organizationId: organization.id } - const overview = useSearchSourceOverview(scope) - const integrations = useSearchIntegrations(organization.id) - const organizationAccounts = useOrganizationAccounts(organization.id) - const githubAccounts = - organizationAccounts.data?.viewerAccounts?.filter( - (account) => account.providerId === 'github-repositories' - ) ?? [] - const usesGitHubInventory = - organizationAccounts.isPending || - organizationAccounts.isError || - organizationAccounts.data?.viewerAccounts !== undefined - const slackInventory = usePersonalSearchIntegrations({ - organizationId: organization.id, - connectorType: 'slack', - }) - const canConnectSharedSlack = - !slackInventory.isError && - slackInventory.data?.available.some( - (entry) => entry.target.connectorType === 'slack' && !entry.target.connectorId - ) === true - const availability = usePermissionConfig() - const configured = new Map( - overview.data?.providers.map((provider) => [provider.connectorType, provider]) - ) - const approved = new Set( - integrations.data - ?.filter((integration) => integration.approved) - .map((integration) => integration.connectorType) - ) - const providers = SEARCH_SOURCE_TYPES.flatMap(([type, meta]) => { - const connector = SEARCH_CONNECTORS.find((entry) => entry.type === type) - const canCreate = Boolean( - connector && - (type !== 'slack' || canConnectSharedSlack) && - approved.has(type) && - getConnectorAccessAvailability(meta, availability.integrationAvailability, { - memberAccessAvailable: searchAccess.memberScoped, - mirroredAccessAvailable: searchAccess.sourceMirrored, - oauthServiceAvailability: availability.oauthServiceAvailability, - isIntegrationAvailabilityReady: availability.isIntegrationAvailabilityReady, - }).members - ) - const hasGitHubAccount = - type === 'github' && - (githubAccounts.length > 0 || organizationAccounts.isPending || organizationAccounts.isError) - return configured.has(type) || canCreate || hasGitHubAccount - ? [{ type, meta, connector, canCreate, configured: configured.has(type) }] - : [] - }) - const failedQuery = overview.isError ? overview : integrations.isError ? integrations : null - const query = search.trim().toLowerCase() - const showSlackSetupError = - slackInventory.isError && approved.has('slack') && 'slack'.includes(query) - const visible = providers.filter( - (provider) => - provider.meta.name.toLowerCase().includes(query) || - (provider.type === 'github' && - githubAccounts.some((account) => account.displayName.toLowerCase().includes(query))) - ) - const githubProvider = visible.find((provider) => provider.type === 'github') - const githubRow = - githubProvider && usesGitHubInventory ? ( - - ) : null - - return ( - <> -
- {failedQuery ? ( - <> - void failedQuery.refetch()} - variant='inline' - /> - {githubRow} - - ) : overview.isPending || integrations.isPending ? ( - <> - Loading integrations - {githubRow} - - ) : ( - <> - {showSlackSetupError && ( - void slackInventory.refetch()} - variant='inline' - /> - )} - {availability.integrationAvailabilityError && ( - void availability.refetchIntegrationAvailability()} - variant='inline' - /> - )} - {providers.map((provider) => ( - - ))} - {showEmpty && - visible.length === 0 && - !availability.integrationAvailabilityError && - !showSlackSetupError && ( - - {!availability.isIntegrationAvailabilityReady || - (approved.has('slack') && 'slack'.includes(query) && slackInventory.isPending) - ? 'Loading integrations' - : search - ? 'No matching integrations.' - : 'No integrations are available to connect.'} - - )} - - )} -
- - ) -} - -interface MemberIntegrationProps { - scope: ResourceScope & { kind: 'organization' } - connectorType: string - connector?: SearchConnector - configured: boolean - canCreate: boolean - memberAccessAvailable: boolean - mirroredAccessAvailable: boolean -} - -/** Each provider loads one bounded page; additional content is loaded explicitly. */ -function MemberIntegration({ - scope, - connectorType, - connector, - configured, - canCreate, - memberAccessAvailable, - mirroredAccessAvailable, -}: MemberIntegrationProps) { - const sources = useSearchSources(scope, { connectorType, enabled: configured }) - const membershipQueryKeys = useMemo( - () => [searchSourceKeys.list(scope), organizationAccountsKeys.detail(scope.organizationId)], - [scope.organizationId] - ) - const connectedConnectorIds = useMemo( - () => - new Set( - sources.data - ?.filter((source) => source.viewerMembership === 'connected') - .map((source) => source.connectorId) - ), - [sources.data] - ) - const enrollment = useMemberEnrollment({ - membershipQueryKeys, - connectedConnectorIds, - directOAuth: true, - onConnectionError: toast.error, - }) - return ( - <> - enrollment.connectSearchSource(scope, connector) - : undefined - } - addLabel={ - connector?.setupFields.length - ? getSearchConnectionLabels(connectorType, 'members').add - : undefined - } - /> - {enrollment.setupConnector && ( - - enrollment.connectSource(scope, enrollment.setupConnector!.type, config) - } - /> - )} - - ) -} diff --git a/apps/sim/app/o/[organizationId]/integrations/indexed/source-status.ts b/apps/sim/app/o/[organizationId]/integrations/indexed/source-status.ts deleted file mode 100644 index 77769aea4a6..00000000000 --- a/apps/sim/app/o/[organizationId]/integrations/indexed/source-status.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { SearchSourceSummary } from '@/lib/api/contracts/knowledge/connectors' -import type { ResourceScope } from '@/lib/core/resource-scope' - -interface SearchSourceStatusInput { - source: SearchSourceSummary - scopeKind: ResourceScope['kind'] - supported: boolean - usable: boolean - connectable: boolean - waiting: boolean -} - -/** Source and integration rows share one member-facing status priority. */ -export function getSearchSourceStatus({ - source, - scopeKind, - supported, - usable, - connectable, - waiting, -}: SearchSourceStatusInput): string { - const membership = source.viewerMembership - let status: string - if (!supported) status = 'Available in its knowledge base' - else if (source.approved === false) status = 'Deactivated by an organization admin' - else if (!usable) status = `Not available in this ${scopeKind}` - else if (!source.enabled) status = 'Syncing is paused' - else if (!source.viewerEmailVerified || membership === 'unverified_email') - status = 'Verify your email to search this source' - else if (membership === 'revoked') status = 'Your access was removed by an admin' - else if (source.connectionRequired && membership === null) status = 'Needs admin attention' - else if (connectable) - status = waiting - ? 'Finish connecting in the other tab' - : membership === 'needs_reauth' - ? 'Your account needs to be reconnected' - : 'Connect your account to search this source' - else if (source.hasSyncError || source.viewerFailedDocumentCount > 0) - status = 'Sync needs attention' - else if (source.isSyncing) status = 'Indexing' - else if (source.hasViewerDocuments) status = 'Ready to search' - else status = source.lastSyncAt ? 'No searchable documents yet' : 'Waiting for the first sync' - return status -} diff --git a/apps/sim/app/o/[organizationId]/integrations/indexed/use-member-enrollment.test.tsx b/apps/sim/app/o/[organizationId]/integrations/indexed/use-member-enrollment.test.tsx deleted file mode 100644 index 7bba56306ae..00000000000 --- a/apps/sim/app/o/[organizationId]/integrations/indexed/use-member-enrollment.test.tsx +++ /dev/null @@ -1,358 +0,0 @@ -/** - * @vitest-environment jsdom - */ -import { act } from 'react' -import { - kbConnectorsQueriesMock, - kbConnectorsQueriesMockFns, -} from '@sim/testing/mocks/kb-connectors-queries.mock' -import { reactQueryMock } from '@sim/testing/mocks/react-query.mock' -import { createRoot, type Root } from 'react-dom/client' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const mocks = vi.hoisted(() => ({ - enrollmentMutate: vi.fn(), - sourceConnectionMutate: vi.fn(), - connectionError: vi.fn(), - channels: [] as Array<{ - name: string - onmessage: ((event: MessageEvent) => void) | null - close: ReturnType - }>, -})) - -vi.mock('@tanstack/react-query', () => reactQueryMock) -vi.mock('@/hooks/queries/kb/connectors', () => kbConnectorsQueriesMock) - -import { useMemberEnrollment } from '@/app/o/[organizationId]/integrations/indexed/use-member-enrollment' - -type Enrollment = ReturnType - -let latest: Enrollment | null = null -let root: Root | null = null -let container: HTMLDivElement | null = null -let enrollmentTab: { location: { href: string }; closed: boolean; close: () => void } - -function Harness({ - connected, - directOAuth, - onConnectionError, -}: { - connected: ReadonlySet - directOAuth?: boolean - onConnectionError?: (message: string) => void -}) { - latest = useMemberEnrollment({ - membershipQueryKeys: [['test-memberships']], - connectedConnectorIds: connected, - directOAuth, - onConnectionError, - }) - return null -} - -function mount( - connected: ReadonlySet = new Set(), - directOAuth = false, - onConnectionError?: (message: string) => void -) { - ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true - container = document.createElement('div') - document.body.appendChild(container) - root = createRoot(container) - act(() => - root?.render( - - ) - ) -} - -function enrollment(): Enrollment { - if (!latest) throw new Error('Hook did not render') - return latest -} - -beforeEach(() => { - kbConnectorsQueriesMockFns.mockUseStartConnectorMemberEnrollment.mockReturnValue({ - mutate: mocks.enrollmentMutate, - submittedAt: 0, - isPending: false, - error: null, - }) - kbConnectorsQueriesMockFns.mockUseConnectSimSearchConnector.mockReturnValue({ - mutate: mocks.sourceConnectionMutate, - submittedAt: 0, - isPending: false, - error: null, - }) - vi.useFakeTimers() - mocks.channels.length = 0 - vi.stubGlobal( - 'BroadcastChannel', - class { - onmessage: ((event: MessageEvent) => void) | null = null - close = vi.fn() - constructor(public name: string) { - mocks.channels.push(this) - } - } - ) - enrollmentTab = { - location: { href: '' }, - closed: false, - close: vi.fn(), - } - vi.spyOn(window, 'open').mockReturnValue(enrollmentTab as unknown as Window) -}) - -afterEach(() => { - if (root) act(() => root?.unmount()) - container?.remove() - root = null - container = null - latest = null - vi.useRealTimers() -}) - -describe('useMemberEnrollment', () => { - it('reports an OAuth failure once per attempt and allows the same error on a later retry', () => { - mount(new Set(), true, mocks.connectionError) - for (let index = 0; index < 2; index += 1) { - act(() => enrollment().connect('kb-1', 'connector-1')) - act(() => - mocks.enrollmentMutate.mock.calls[index][1].onSuccess({ - url: 'https://provider.test/authorize', - }) - ) - act(() => - mocks.channels[index].onmessage?.( - new MessageEvent('message', { data: 'permissions_required' }) - ) - ) - act(() => - mocks.channels[index].onmessage?.( - new MessageEvent('message', { data: 'permissions_required' }) - ) - ) - expect(mocks.connectionError).toHaveBeenCalledTimes(index + 1) - expect(enrollment().isAwaiting('connector-1')).toBe(false) - } - expect(mocks.connectionError).toHaveBeenLastCalledWith( - 'All requested permissions are required to connect this account.' - ) - act(() => vi.advanceTimersByTime(10 * 60_000)) - expect(mocks.connectionError).toHaveBeenCalledTimes(2) - }) - - it.each(['existing', 'new'] as const)( - 'does not expire a superseded %s source authorization after its retry connects', - (source) => { - mount(new Set(), true, mocks.connectionError) - const mutation = source === 'existing' ? mocks.enrollmentMutate : mocks.sourceConnectionMutate - for (let index = 0; index < 2; index += 1) { - act(() => { - if (source === 'existing') enrollment().connect('kb-1', 'connector-1') - else enrollment().connectSource('workspace-1', 'jira') - }) - act(() => - mutation.mock.calls[index][1].onSuccess({ - url: `https://provider.test/attempt-${index}`, - connectorId: 'connector-1', - }) - ) - } - act(() => mocks.channels[1].onmessage?.(new MessageEvent('message', { data: 'connected' }))) - act(() => vi.advanceTimersByTime(10 * 60_000)) - act(() => - mocks.channels[0].onmessage?.(new MessageEvent('message', { data: 'permissions_required' })) - ) - expect(mocks.connectionError).not.toHaveBeenCalled() - expect(enrollment().error).toBeNull() - expect(enrollment().isAwaiting('connector-1')).toBe(false) - expect(mocks.channels[0].close).toHaveBeenCalledOnce() - expect(mocks.channels[1].close).toHaveBeenCalledOnce() - } - ) - - it.each([ - ['existing', 'permissions_required'], - ['existing', 'denied'], - ['existing', 'expired'], - ['new', 'permissions_required'], - ['new', 'denied'], - ['new', 'expired'], - ] as const)( - 'ignores the previous %s source’s %s while its retry request is pending', - (source, failure) => { - mount(new Set(), true, mocks.connectionError) - const mutation = source === 'existing' ? mocks.enrollmentMutate : mocks.sourceConnectionMutate - const connect = () => { - if (source === 'existing') enrollment().connect('kb-1', 'connector-1') - else enrollment().connectSource('workspace-1', 'jira', { projectKey: 'ENG' }) - } - act(connect) - act(() => - mutation.mock.calls[0][1].onSuccess({ - url: 'https://provider.test/previous', - connectorId: 'connector-1', - }) - ) - act(() => vi.advanceTimersByTime(9 * 60_000)) - act(connect) - if (failure !== 'expired') { - act(() => mocks.channels[0].onmessage?.(new MessageEvent('message', { data: failure }))) - } - act(() => vi.advanceTimersByTime(60_000)) - expect(mocks.connectionError).not.toHaveBeenCalled() - expect(enrollment().error).toBeNull() - act(() => - mutation.mock.calls[1][1].onSuccess({ - url: 'https://provider.test/retry', - connectorId: 'connector-1', - }) - ) - expect(enrollment().isAwaiting('connector-1')).toBe(true) - expect(mocks.channels[1].close).not.toHaveBeenCalled() - } - ) - - it.each([ - ['existing', 'success'], - ['existing', 'failure'], - ['new', 'success'], - ['new', 'failure'], - ] as const)('ignores a superseded %s source request’s late %s', (source, outcome) => { - mount(new Set(), true, mocks.connectionError) - const mutation = source === 'existing' ? mocks.enrollmentMutate : mocks.sourceConnectionMutate - const retryTab = { location: { href: '' }, closed: false, close: vi.fn() } - vi.mocked(window.open) - .mockReturnValueOnce(enrollmentTab as unknown as Window) - .mockReturnValueOnce(retryTab as unknown as Window) - for (let index = 0; index < 2; index += 1) { - act(() => { - if (source === 'existing') enrollment().connect('kb-1', 'connector-1') - else enrollment().connectSource('workspace-1', 'jira', { projectKey: 'ENG' }) - }) - } - act(() => - mutation.mock.calls[1][1].onSuccess({ - url: 'https://provider.test/retry', - connectorId: 'connector-1', - }) - ) - act(() => { - if (outcome === 'failure') { - mutation.mock.calls[0][1].onError(new Error('Previous request failed')) - } else { - mutation.mock.calls[0][1].onSuccess({ - url: 'https://provider.test/previous', - connectorId: 'connector-1', - }) - } - }) - expect(enrollmentTab.location.href).toBe('') - expect(retryTab.location.href).toBe('https://provider.test/retry') - expect(retryTab.close).not.toHaveBeenCalled() - expect(enrollment().isAwaiting('connector-1')).toBe(true) - expect(mocks.channels[1].close).not.toHaveBeenCalled() - expect(mocks.connectionError).not.toHaveBeenCalled() - expect(enrollment().error).toBeNull() - }) - - it('does not let a delayed first-source response replace its newer connector authorization', () => { - mount(new Set(), true, mocks.connectionError) - act(() => enrollment().connectSource('workspace-1', 'jira', { projectKey: 'ENG' })) - act(() => enrollment().connect('kb-1', 'connector-1')) - act(() => - mocks.enrollmentMutate.mock.calls[0][1].onSuccess({ url: 'https://provider.test/retry' }) - ) - act(() => - mocks.sourceConnectionMutate.mock.calls[0][1].onSuccess({ - url: 'https://provider.test/previous', - connectorId: 'connector-1', - }) - ) - expect(enrollmentTab.location.href).toBe('https://provider.test/retry') - expect(mocks.channels[1].close).not.toHaveBeenCalled() - expect(enrollment().isAwaiting('connector-1')).toBe(true) - }) - - it('ignores first-source success while a newer request for its connector is still pending', () => { - mount(new Set(), true, mocks.connectionError) - const retryTab = { location: { href: '' }, closed: false, close: vi.fn() } - vi.mocked(window.open) - .mockReturnValueOnce(enrollmentTab as unknown as Window) - .mockReturnValueOnce(retryTab as unknown as Window) - act(() => enrollment().connectSource('workspace-1', 'jira', { projectKey: 'ENG' })) - act(() => enrollment().connect('kb-1', 'connector-1')) - act(() => - mocks.sourceConnectionMutate.mock.calls[0][1].onSuccess({ - url: 'https://provider.test/previous', - connectorId: 'connector-1', - }) - ) - expect(enrollmentTab.location.href).toBe('') - expect(enrollment().isAwaiting('connector-1')).toBe(false) - act(() => mocks.channels[0].onmessage?.(new MessageEvent('message', { data: 'denied' }))) - expect(mocks.connectionError).not.toHaveBeenCalled() - act(() => - mocks.enrollmentMutate.mock.calls[0][1].onSuccess({ url: 'https://provider.test/retry' }) - ) - expect(retryTab.location.href).toBe('https://provider.test/retry') - expect(enrollment().isAwaiting('connector-1')).toBe(true) - expect(mocks.channels[1].close).not.toHaveBeenCalled() - }) - - it('keeps overlapping provider authorizations separate and reports a rejected one on the original page', () => { - mount(new Set(), true) - act(() => enrollment().connect('kb-1', 'connector-1')) - act(() => - mocks.enrollmentMutate.mock.calls[0][1].onSuccess({ url: 'https://provider.test/one' }) - ) - act(() => enrollment().connect('kb-1', 'connector-2')) - act(() => - mocks.enrollmentMutate.mock.calls[1][1].onSuccess({ url: 'https://provider.test/two' }) - ) - expect(mocks.channels[0].name).not.toBe(mocks.channels[1].name) - act(() => mocks.channels[0].onmessage?.(new MessageEvent('message', { data: 'denied' }))) - expect(enrollment().isAwaiting('connector-1')).toBe(false) - expect(enrollment().isAwaiting('connector-2')).toBe(true) - expect(enrollment().error).toContain('Authorization was canceled') - act(() => mocks.channels[1].onmessage?.(new MessageEvent('message', { data: 'unrecognized' }))) - expect(enrollment().isAwaiting('connector-2')).toBe(true) - }) - - it('passes direct authorization correlation through first-source setup and cleans it up on failure', () => { - mount(new Set(), true) - act(() => - enrollment().connectSource({ kind: 'organization', organizationId: 'org-1' }, 'gmail') - ) - const [input, handlers] = mocks.sourceConnectionMutate.mock.calls[0] - expect(input).toMatchObject({ - organizationId: 'org-1', - connectorType: 'gmail', - oauthCompletionId: expect.any(String), - }) - act(() => handlers.onError(new Error('Unavailable'))) - expect(mocks.channels[0].close).toHaveBeenCalledOnce() - expect(enrollmentTab.close).toHaveBeenCalledOnce() - }) - - it('does not navigate or await a tab closed before the enrollment request completes', () => { - mount() - act(() => enrollment().connectSource('workspace-1', 'google_drive')) - enrollmentTab.closed = true - const [, handlers] = mocks.sourceConnectionMutate.mock.calls[0] - act(() => - handlers.onSuccess({ url: 'https://example.test/enroll', connectorId: 'connector-1' }) - ) - - expect(enrollmentTab.location.href).toBe('') - expect(enrollment().isAwaiting('connector-1')).toBe(false) - expect(enrollment().isAwaitingSource('google_drive')).toBe(false) - }) -}) diff --git a/apps/sim/app/o/[organizationId]/integrations/indexed/use-member-enrollment.ts b/apps/sim/app/o/[organizationId]/integrations/indexed/use-member-enrollment.ts deleted file mode 100644 index 76faa926cc8..00000000000 --- a/apps/sim/app/o/[organizationId]/integrations/indexed/use-member-enrollment.ts +++ /dev/null @@ -1,416 +0,0 @@ -'use client' - -import { useCallback, useEffect, useRef, useState } from 'react' -import { createLogger } from '@sim/logger' -import { generateId } from '@sim/utils/id' -import { type QueryKey, useMutation, useQueryClient } from '@tanstack/react-query' -import type { DesktopSourceRequest } from '@/lib/api/contracts/desktop-source-connect' -import { - type ResourceScope, - resourceScopeFields, - resourceScopeKey, -} from '@/lib/core/resource-scope' -import { - CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES, - credentialGroupOAuthCompletionChannel, - isCredentialGroupOAuthFailure, -} from '@/lib/credential-groups/oauth-completion' -import { isDesktopApp } from '@/lib/desktop' -import { connectDesktopSource } from '@/lib/desktop/source-connect' -import type { SearchConnector } from '@/lib/sim-search/connectors' -import { - useConnectSimSearchConnector, - useStartConnectorMemberEnrollment, - type ViewerConnectorMembership, -} from '@/hooks/queries/kb/connectors' - -const logger = createLogger('MemberEnrollment') - -/** How often the membership queries are refreshed while a member connects in another tab. */ -const AWAITING_CONNECTION_POLL_MS = 4_000 -/** How long a connection is awaited before the surface stops refreshing on its own. */ -const AWAITING_CONNECTION_TIMEOUT_MS = 10 * 60_000 -const POPUP_BLOCKED_MESSAGE = 'Allow pop-ups for this site to connect your account.' - -/** Memberships the viewer can act on themselves. */ -export const CONNECTABLE_MEMBERSHIPS: ReadonlySet = new Set([ - 'needs_reauth', - 'invited', - 'not_enrolled', -]) - -/** The label of the one action a connectable membership offers. */ -export function enrollmentActionLabel( - membership: ViewerConnectorMembership, - waiting: boolean -): string { - if (waiting) return 'Open again' - return membership === 'needs_reauth' ? 'Reconnect' : 'Connect' -} - -/** An enrollment tab this surface opened that has not connected yet. */ -interface AwaitingEnrollment { - since: number - tab: Window - /** - * The Sim Search source whose connect created the connector, so the source - * can be told it is awaited before its membership row exists to look it up by. - */ - connectorType: string | null - oauthCompletionId?: string -} - -interface UseMemberEnrollmentProps { - /** Queries this surface reads memberships from, refreshed while a connection is awaited. */ - membershipQueryKeys: readonly QueryKey[] - /** Connector ids the viewer is now connected to; awaiting stops for them. */ - connectedConnectorIds: ReadonlySet - /** Main Integrations skips the invitation page; invitation-based surfaces keep their flow. */ - directOAuth?: boolean - onConnectionError?: (message: string) => void -} - -/** - * Lets the viewer connect their own account to a per-member connector, by - * connector or by Sim Search source. Enrollment opens in a new tab, and the - * membership queries are polled meanwhile so the surface that started it - * updates on its own once the account is connected. - * - * The tab is opened in the click itself, before the enrollment link is - * minted, because a tab opened after a network round trip is outside the - * click's activation window and popup blockers swallow it. - */ -export function useMemberEnrollment({ - membershipQueryKeys, - connectedConnectorIds, - directOAuth = false, - onConnectionError, -}: UseMemberEnrollmentProps) { - const connectedRef = useRef(connectedConnectorIds) - const oauthPopups = useRef( - new Map< - string, - { - channel: BroadcastChannel - timer: ReturnType - attemptKey: string - connectorId?: string - } - >() - ) - const queryClient = useQueryClient() - const nativeAbort = useRef(null) - useEffect(() => () => nativeAbort.current?.abort(), []) - const nativeConnection = useMutation({ - mutationFn: async (request: DesktopSourceRequest) => { - nativeAbort.current?.abort() - const controller = new AbortController() - nativeAbort.current = controller - return connectDesktopSource(request, controller.signal) - }, - onSettled: () => - Promise.all( - membershipQueryKeys.map((queryKey) => queryClient.invalidateQueries({ queryKey })) - ), - onError: (error) => onConnectionError?.(error.message), - onSuccess: () => setSetupConnector(null), - }) - const enrollment = useStartConnectorMemberEnrollment() - const sourceConnection = useConnectSimSearchConnector() - const [awaitingSince, setAwaitingSince] = useState>( - () => new Map() - ) - const [popupBlocked, setPopupBlocked] = useState(false) - const [oauthError, setOAuthError] = useState(null) - - const refreshMemberships = useCallback(() => { - for (const queryKey of membershipQueryKeys) { - void queryClient.invalidateQueries({ queryKey }) - } - }, [membershipQueryKeys, queryClient]) - - const clearOAuth = (completionId: string) => { - const popup = oauthPopups.current.get(completionId) - if (!popup) return false - clearTimeout(popup.timer) - popup.channel.close() - oauthPopups.current.delete(completionId) - setAwaitingSince( - (current) => - new Map([...current].filter(([, entry]) => entry.oauthCompletionId !== completionId)) - ) - return true - } - - const finishOAuth = (completionId: string, error: string | null) => { - if (!clearOAuth(completionId)) return - setOAuthError(error) - if (error) onConnectionError?.(error) - refreshMemberships() - } - - useEffect(() => { - const popups = oauthPopups.current - return () => { - for (const popup of popups.values()) { - clearTimeout(popup.timer) - popup.channel.close() - } - popups.clear() - } - }, []) - - useEffect(() => { - connectedRef.current = connectedConnectorIds - }, [connectedConnectorIds]) - - /** - * Polls while any connection is awaited, and once more after the last one - * connects: that tick drops the connected ids, so a token that later needs - * reauthorization is not mistaken for a connection still being awaited. - * Direct OAuth waits for its completion message: provider window isolation - * can report a closed handle while authorization is still in progress. - */ - const awaiting = awaitingSince.size > 0 - useEffect(() => { - if (!awaiting) return - const timer = setInterval(() => { - const now = Date.now() - setAwaitingSince((current) => { - const next = new Map( - [...current].filter( - ([id, { since, tab, oauthCompletionId }]) => - (Boolean(oauthCompletionId) || !tab.closed) && - (Boolean(oauthCompletionId) || !connectedRef.current.has(id)) && - now - since < AWAITING_CONNECTION_TIMEOUT_MS - ) - ) - return next.size === current.size ? current : next - }) - refreshMemberships() - }, AWAITING_CONNECTION_POLL_MS) - return () => clearInterval(timer) - }, [awaiting, refreshMemberships]) - - /** Opens the tab inside the click, then sends it wherever `start` mints. */ - const openEnrollment = ( - attemptKey: string, - start: (handlers: { - oauthCompletionId?: string - onSuccess: (url: string, connectorId: string, connectorType?: string) => boolean - onError: () => boolean - }) => void - ) => { - const tab = window.open('about:blank', '_blank') - if (!tab) { - setPopupBlocked(true) - onConnectionError?.(POPUP_BLOCKED_MESSAGE) - return - } - tab.opener = null - setPopupBlocked(false) - setOAuthError(null) - const oauthCompletionId = directOAuth ? generateId() : undefined - if (oauthCompletionId) { - for (const [previousId, previous] of oauthPopups.current) { - if ( - previous.attemptKey === attemptKey || - (previous.connectorId && `connector:${previous.connectorId}` === attemptKey) - ) { - clearOAuth(previousId) - } - } - const channel = new BroadcastChannel(credentialGroupOAuthCompletionChannel(oauthCompletionId)) - channel.onmessage = ({ data }: MessageEvent) => { - if (data === 'connected') finishOAuth(oauthCompletionId, null) - else if (isCredentialGroupOAuthFailure(data)) - finishOAuth(oauthCompletionId, CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES[data]) - } - const timer = setTimeout(() => { - finishOAuth(oauthCompletionId, CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES.expired) - }, AWAITING_CONNECTION_TIMEOUT_MS) - oauthPopups.current.set(oauthCompletionId, { channel, timer, attemptKey }) - } - start({ - ...(oauthCompletionId ? { oauthCompletionId } : {}), - onSuccess: (url, connectorId, connectorType) => { - if (tab.closed || (oauthCompletionId && !oauthPopups.current.has(oauthCompletionId))) { - if (oauthCompletionId) - finishOAuth(oauthCompletionId, CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES.denied) - return false - } - if (oauthCompletionId) { - const popup = oauthPopups.current.get(oauthCompletionId)! - const connectorAttemptKey = `connector:${connectorId}` - const latestAttempt = [...oauthPopups.current] - .reverse() - .find( - ([id, entry]) => - id === oauthCompletionId || - entry.connectorId === connectorId || - entry.attemptKey === connectorAttemptKey - ) - if (latestAttempt?.[0] !== oauthCompletionId) { - clearOAuth(oauthCompletionId) - return false - } - for (const [previousId, previous] of oauthPopups.current) { - if ( - previousId === oauthCompletionId || - (previous.connectorId !== connectorId && previous.attemptKey !== connectorAttemptKey) - ) - continue - clearOAuth(previousId) - } - popup.connectorId = connectorId - } - tab.location.href = url - setAwaitingSince((current) => - new Map(current).set(connectorId, { - since: Date.now(), - tab, - connectorType: connectorType ?? null, - ...(oauthCompletionId ? { oauthCompletionId } : {}), - }) - ) - return true - }, - onError: () => { - const active = !oauthCompletionId || oauthPopups.current.has(oauthCompletionId) - if (oauthCompletionId) finishOAuth(oauthCompletionId, null) - tab.close() - return active - }, - }) - } - - const connect = (knowledgeBaseId: string, connectorId: string) => { - if (isDesktopApp()) { - nativeConnection.mutate({ - kind: 'member-enrollment', - params: { id: knowledgeBaseId, connectorId }, - ...(directOAuth ? { completionId: generateId() } : {}), - }) - return - } - openEnrollment(`connector:${connectorId}`, ({ onSuccess, onError, oauthCompletionId }) => { - enrollment.mutate( - { knowledgeBaseId, connectorId, ...(oauthCompletionId ? { oauthCompletionId } : {}) }, - { - onSuccess: ({ url }) => onSuccess(url, connectorId), - onError: (err) => { - if (!onError()) return - onConnectionError?.(err.message) - logger.error('Failed to start member enrollment', { error: err.message }) - }, - } - ) - }) - } - - /** - * Connects a Sim Search source: its per-member connector exists afterwards, - * and the viewer enrolls. The setup fields are read only when this connect - * creates the connector. - */ - const connectSource = ( - owner: string | ResourceScope, - connectorType: string, - sourceConfig?: Record - ) => { - const scope = - typeof owner === 'string' ? { kind: 'workspace' as const, workspaceId: owner } : owner - if (isDesktopApp()) { - nativeConnection.mutate({ - kind: 'search-source', - body: { ...resourceScopeFields(scope), connectorType, sourceConfig }, - ...(directOAuth ? { completionId: generateId() } : {}), - }) - return - } - const configKey = JSON.stringify( - Object.entries(sourceConfig ?? {}).sort(([left], [right]) => left.localeCompare(right)) - ) - openEnrollment( - `source:${resourceScopeKey(scope)}:${connectorType}:${configKey}`, - ({ onSuccess, onError, oauthCompletionId }) => { - sourceConnection.mutate( - { - ...resourceScopeFields(scope), - connectorType, - sourceConfig, - ...(oauthCompletionId ? { oauthCompletionId } : {}), - }, - { - onSuccess: ({ url, connectorId }) => { - if (onSuccess(url, connectorId, connectorType)) setSetupConnector(null) - }, - onError: (err) => { - if (!onError()) return - onConnectionError?.(err.message) - logger.error('Failed to connect a Sim Search source', { error: err.message }) - }, - } - ) - } - ) - } - - const [setupConnector, setSetupConnector] = useState(null) - - /** - * One click on a new Sim Search source: ask for its setup fields when it - * needs them, and otherwise create it and enroll in one step. - */ - const connectSearchSource = (owner: string | ResourceScope, connector: SearchConnector) => { - if (connector.setupFields.length > 0) { - setSetupConnector(connector) - return - } - connectSource(owner, connector.type) - } - - const isAwaiting = (connectorId: string) => - (nativeConnection.isPending && - nativeConnection.variables?.kind === 'member-enrollment' && - nativeConnection.variables.params.connectorId === connectorId) || - (awaitingSince.has(connectorId) && - (Boolean(awaitingSince.get(connectorId)?.oauthCompletionId) || - !connectedConnectorIds.has(connectorId))) - - /** - * Whether a Sim Search source is awaited by the connect that created its - * connector: the membership list has no row for it until it refetches, so - * the source cannot be looked up by connector id yet. - */ - const isAwaitingSource = (connectorType: string) => - (nativeConnection.isPending && - nativeConnection.variables?.kind === 'search-source' && - nativeConnection.variables.body.connectorType === connectorType) || - [...awaitingSince].some( - ([id, awaiting]) => - awaiting.connectorType === connectorType && - (Boolean(awaiting.oauthCompletionId) || !connectedConnectorIds.has(id)) - ) - - /** The surface reports the latest attempt, whichever path made it. */ - const latest = - enrollment.submittedAt >= sourceConnection.submittedAt ? enrollment : sourceConnection - return { - connect, - connectSource, - connectSearchSource, - setupConnector, - closeSetup: () => { - nativeAbort.current?.abort() - nativeAbort.current = null - setSetupConnector(null) - }, - isAwaiting, - isAwaitingSource, - isPending: nativeConnection.isPending || enrollment.isPending || sourceConnection.isPending, - error: popupBlocked - ? POPUP_BLOCKED_MESSAGE - : (nativeConnection.error?.message ?? oauthError ?? latest.error?.message ?? null), - } -} diff --git a/apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx b/apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx deleted file mode 100644 index 243d053ed17..00000000000 --- a/apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx +++ /dev/null @@ -1,1041 +0,0 @@ -/** @vitest-environment jsdom */ -import { act, type ReactNode } from 'react' -import { toast } from '@sim/emcn' -import { - createMockDeploymentShape, - deploymentShapeMock, - deploymentShapeMockFns, -} from '@sim/testing/mocks/deployment-shape.mock' -import { - kbConnectorsQueriesMock, - kbConnectorsQueriesMockFns, -} from '@sim/testing/mocks/kb-connectors-queries.mock' -import { - organizationAccountsQueriesMock, - organizationAccountsQueriesMockFns, -} from '@sim/testing/mocks/organization-accounts-queries.mock' -import { - organizationProviderMock, - organizationProviderMockFns, -} from '@sim/testing/mocks/organization-provider.mock' -import { NuqsTestingAdapter } from 'nuqs/adapters/testing' -import { createRoot, type Root } from 'react-dom/client' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import type { SearchSourceSummary } from '@/lib/api/contracts/knowledge/connectors' -import type { SearchConnector } from '@/lib/sim-search/connectors' - -const mocks = vi.hoisted(() => ({ - live: false, - integrations: vi.fn(), - slackInventory: vi.fn(), - filters: vi.fn(), - connect: vi.fn(), - connectSearchSource: vi.fn(), - availability: vi.fn(), - refetch: vi.fn(), - nextPage: vi.fn(), - enrollment: vi.fn(), - accountMenu: vi.fn(), - request: vi.fn(), - updateUrl: vi.fn(), - setupConnector: null as SearchConnector | null, - connectOrganizationAccount: vi.fn(), - reconnectOrganizationAccount: vi.fn(), - refetchAccounts: vi.fn(), -})) -vi.mock('@/lib/core/config/deployment-shape', () => deploymentShapeMock) -vi.mock('@/hooks/queries/organization-secrets', () => ({ - useOrganizationSecretSource: () => ({ data: { source: null } }), -})) -vi.mock('@/hooks/queries/organization-accounts', () => organizationAccountsQueriesMock) -vi.mock('@/app/o/[organizationId]/integrations/slack-search-actions', () => ({ - SlackSearchActions: () => Return to Slack, -})) -vi.mock( - '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/search-integration-connection', - () => ({ - SearchIntegrationConnection: (props: unknown) => { - mocks.request(props) - return Requested connection - }, - }) -) -vi.mock('@/hooks/queries/personal-search-integrations', () => ({ - usePersonalSearchIntegrations: mocks.slackInventory, -})) -vi.mock('@/hooks/queries/search-integrations', () => ({ - useSearchIntegrations: mocks.integrations, -})) -vi.mock('@/hooks/use-permission-config', () => ({ usePermissionConfig: mocks.availability })) -vi.mock('@/app/o/[organizationId]/components/organization-page', () => ({ - OrganizationPage: ({ action, children }: { action?: ReactNode; children?: ReactNode }) => ( - <> - {action} - {children} - - ), -})) -vi.mock( - '@/app/o/[organizationId]/components/organization-page/use-organization-page-filters', - () => ({ useOrganizationPageFilters: mocks.filters }) -) -vi.mock('@/app/o/[organizationId]/providers/organization-provider', () => organizationProviderMock) -vi.mock('@/app/workspace/[workspaceId]/integrations/components/integrations-showcase', () => ({ - IntegrationTile: () => null, -})) -vi.mock('@/app/o/[organizationId]/integrations/disconnect-account-menu', () => ({ - DisconnectAccountMenu: (props: { - integrationName: string - accounts: { credentialId: string }[] - actions?: RowAction[] - }) => { - mocks.accountMenu(props) - const actions = [ - ...(props.actions ?? []), - ...props.accounts.map((account) => ({ - label: `Disconnect ${account.credentialId}`, - onSelect: vi.fn(), - })), - ] - return actions.length ? ( - - ) : null - }, -})) -vi.mock('@/hooks/queries/kb/connectors', () => kbConnectorsQueriesMock) -vi.mock('@/app/o/[organizationId]/integrations/indexed/use-member-enrollment', () => ({ - enrollmentActionLabel: (membership: string, waiting: boolean) => - waiting ? 'Open again' : membership === 'needs_reauth' ? 'Reconnect' : 'Connect', - CONNECTABLE_MEMBERSHIPS: new Set(['invited', 'not_enrolled', 'needs_reauth']), - useMemberEnrollment: (options: unknown) => { - mocks.enrollment(options) - return { - connect: mocks.connect, - connectSearchSource: mocks.connectSearchSource, - isAwaiting: () => false, - isAwaitingSource: () => false, - isPending: false, - setupConnector: mocks.setupConnector, - closeSetup: vi.fn(), - } - }, -})) -vi.mock('@/hooks/use-oauth-return', () => ({ - useDesktopOAuthConnectListener: () => undefined, - useOAuthReturnRouter: () => undefined, -})) - -import { MemberIntegrationsList } from '@/app/o/[organizationId]/integrations/indexed' -import { OrganizationIntegrations } from '@/app/o/[organizationId]/integrations/integrations' -import { - type RowAction, - RowActionsMenu, -} from '@/app/workspace/[workspaceId]/settings/components/row-actions-menu' -import { organizationAccountsKeys } from '@/hooks/queries/organization-accounts' - -deploymentShapeMockFns.mockUseDeploymentShape.mockImplementation(() => - createMockDeploymentShape({ features: { liveEnterpriseSearch: mocks.live } }) -) - -const mockUseOrganizationContext = organizationProviderMockFns.mockUseOrganizationContext -const mockUseOrganizationAccounts = organizationAccountsQueriesMockFns.mockUseOrganizationAccounts -const mockUseSearchSources = kbConnectorsQueriesMockFns.mockUseSearchSources -const mockUseSearchSourceOverview = kbConnectorsQueriesMockFns.mockUseSearchSourceOverview - -const scope = { kind: 'organization', organizationId: 'organization-a' } as const -const account = { - credentialId: 'account', - displayName: 'My work account', - status: 'active' as const, -} -const memberSource: SearchSourceSummary = { - knowledgeBaseId: 'search-index', - connectorId: 'source-a', - connectorType: 'gmail', - sourceDescription: 'Inbox', - accessMode: 'members', - availability: 'available', - enabled: true, - isSyncing: false, - lastSyncAt: null, - hasSyncError: false, - hasViewerDocuments: false, - viewerFailedDocumentCount: 0, - viewerEmailVerified: true, - viewerAccounts: [], - connectionRequired: true, - viewerMembership: 'not_enrolled', - approved: true, -} -const centralSource: SearchSourceSummary = { - ...memberSource, - connectorId: 'central', - connectorType: 'google_drive', - sourceDescription: 'Shared Drive', - accessMode: 'admin', - connectionRequired: false, - viewerMembership: null, -} - -let root: Root -let container: HTMLDivElement -let rows: SearchSourceSummary[] -let queryOverrides: Record -beforeEach(() => { - mocks.live = false - vi.spyOn(toast, 'error').mockReturnValue('toast') - vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) - mocks.setupConnector = null - organizationAccountsQueriesMockFns.mockUseConnectOrganizationAccount.mockReturnValue({ - mutate: mocks.connectOrganizationAccount, - isPending: false, - }) - organizationAccountsQueriesMockFns.mockUseReconnectPersonalOrganizationAccount.mockReturnValue({ - mutate: mocks.reconnectOrganizationAccount, - isPending: false, - }) - mockUseOrganizationAccounts.mockReturnValue({ - data: { credentialGroup: null, viewerAccounts: [] }, - isPending: false, - isError: false, - refetch: mocks.refetchAccounts, - }) - rows = [memberSource] - queryOverrides = {} - mockUseOrganizationContext.mockReturnValue({ - organization: { id: scope.organizationId }, - viewer: { isAdmin: false }, - searchAccess: { memberScoped: true, sourceMirrored: true }, - }) - mocks.filters.mockReturnValue({ search: '' }) - mocks.slackInventory.mockReturnValue({ - data: { available: [] }, - isPending: false, - isError: false, - }) - mocks.integrations.mockReturnValue({ - data: [{ connectorType: 'gmail', approved: true }], - isPending: false, - }) - mockUseSearchSourceOverview.mockReturnValue({ - data: { - providers: [{ connectorType: 'gmail', isSyncing: false }], - hasSearchableDocuments: false, - }, - isPending: false, - }) - mocks.availability.mockReturnValue({ - integrationAvailability: new Map(), - oauthServiceAvailability: new Map([ - ['google-email', true], - ['confluence', true], - ['jira', true], - ]), - isIntegrationAvailabilityReady: true, - integrationAvailabilityError: null, - }) - mockUseSearchSources.mockImplementation( - (_scope: unknown, options: { enabled: boolean; connectorType?: string }) => ({ - data: options.enabled - ? rows.filter((row) => row.connectorType === options.connectorType) - : undefined, - isPending: false, - isError: false, - isFetching: false, - isFetchNextPageError: false, - hasNextPage: false, - fetchNextPage: mocks.nextPage, - refetch: mocks.refetch, - ...queryOverrides, - }) - ) - container = document.createElement('div') - document.body.appendChild(container) - root = createRoot(container) -}) -afterEach(async () => { - await act(async () => root.unmount()) - container.remove() -}) -async function render(searchParams = '', element: ReactNode = ) { - await act(async () => - root.render( - - {element} - - ) - ) -} -function buttons(label: string) { - return Array.from(document.querySelectorAll('button')).filter( - (button) => button.textContent?.trim() === label - ) -} - -async function openMenu(name: string) { - const trigger = document.querySelector( - `[aria-label="${name} integration actions"]` - )! - await act(async () => - trigger.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) - ) -} -function menuItem(label: string) { - return [...document.querySelectorAll('[role="menuitem"]')].find( - (item) => item.textContent === label - )! -} - -describe('GitHub member account inventory', () => { - const githubAccount = { - credentialId: 'github-account', - providerId: 'github-repositories', - groupId: 'accounts-group', - optionId: 'github-option', - displayName: 'My GitHub', - status: 'active' as const, - } - const githubGroup = { - id: 'accounts-group', - status: 'active', - options: [{ id: 'github-option', provider: 'github-repositories', status: 'active' }], - } - - beforeEach(() => { - rows = ['repo-one', 'repo-two'].map((connectorId) => ({ - ...memberSource, - connectorId, - connectorType: 'github', - sourceDescription: connectorId, - })) - mockUseSearchSourceOverview.mockReturnValue({ - data: { providers: [{ connectorType: 'github' }] }, - isPending: false, - }) - mocks.integrations.mockReturnValue({ - data: [{ connectorType: 'github', approved: true }], - isPending: false, - }) - mocks.availability.mockReturnValue({ - integrationAvailability: new Map(), - oauthServiceAvailability: new Map([['github-repositories', true]]), - isIntegrationAvailabilityReady: true, - }) - mockUseOrganizationAccounts.mockReturnValue({ - data: { credentialGroup: githubGroup, viewerAccounts: [] }, - isPending: false, - isError: false, - refetch: mocks.refetchAccounts, - }) - }) - - it('connects once through the account operation', async () => { - await render() - expect(buttons('Connect')).toHaveLength(1) - expect(container.textContent).toContain('Connect once') - await act(async () => buttons('Connect')[0].click()) - expect(mocks.connectOrganizationAccount).toHaveBeenCalledExactlyOnceWith( - { organizationId: scope.organizationId, optionId: 'github-option' }, - expect.any(Object) - ) - expect(mockUseSearchSources).not.toHaveBeenCalled() - expect(mocks.connect).not.toHaveBeenCalled() - expect(mocks.connectSearchSource).not.toHaveBeenCalled() - expect(document.querySelector('[role="dialog"]')).toBeNull() - expect(container.textContent).not.toContain('repo-one') - }) - - it('keeps one account row when an admin adds another repository', async () => { - mockUseOrganizationAccounts.mockReturnValue({ - data: { credentialGroup: githubGroup, viewerAccounts: [githubAccount] }, - isPending: false, - }) - await render() - rows.push({ - ...rows[0], - connectorId: 'future-repository', - sourceDescription: 'future-repository', - }) - await render() - expect(document.querySelectorAll('[aria-label="GitHub integration actions"]')).toHaveLength(1) - expect(mocks.accountMenu).toHaveBeenLastCalledWith( - expect.objectContaining({ accounts: [githubAccount] }) - ) - expect(buttons('Connect')).toHaveLength(0) - expect(buttons('Reconnect')).toHaveLength(0) - expect(mockUseSearchSources).not.toHaveBeenCalled() - expect(container.textContent).not.toContain('future-repository') - }) - - it('keeps an owned account visible before any repository source exists', async () => { - mockUseSearchSourceOverview.mockReturnValue({ data: { providers: [] }, isPending: false }) - mocks.integrations.mockReturnValue({ data: [], isPending: false }) - mockUseOrganizationAccounts.mockReturnValue({ - data: { credentialGroup: githubGroup, viewerAccounts: [githubAccount] }, - isPending: false, - }) - await render('', ) - expect(container.textContent).toContain('My GitHub · Connected') - expect(document.querySelectorAll('[aria-label="GitHub integration actions"]')).toHaveLength(1) - expect(mockUseSearchSources).not.toHaveBeenCalled() - expect(buttons('Connect')).toHaveLength(0) - expect(container.textContent).not.toContain('No integrations are available') - }) - - it.each(['group', 'option'] as const)( - 'keeps Disconnect but hides Reconnect when the canonical %s is disabled', - async (disabled) => { - const expired = { ...githubAccount, status: 'needs_reauth' } - mockUseOrganizationAccounts.mockReturnValue({ - data: { - credentialGroup: { - ...githubGroup, - status: disabled === 'group' ? 'disabled' : 'active', - options: [ - { ...githubGroup.options[0], status: disabled === 'option' ? 'disabled' : 'active' }, - ], - }, - viewerAccounts: [expired], - }, - isPending: false, - }) - await render() - expect(buttons('Reconnect')).toHaveLength(0) - expect(mocks.accountMenu).toHaveBeenLastCalledWith( - expect.objectContaining({ accounts: [expired] }) - ) - await openMenu('GitHub') - expect(menuItem('Disconnect github-account')).toBeDefined() - } - ) - - it('allows personal reauthorization while Search is disabled', async () => { - mockUseSearchSourceOverview.mockReturnValue({ data: { providers: [] }, isPending: false }) - mocks.integrations.mockReturnValue({ - data: [{ connectorType: 'github', approved: false }], - isPending: false, - }) - mockUseOrganizationContext.mockReturnValue({ - organization: { id: scope.organizationId }, - searchAccess: { memberScoped: false, sourceMirrored: false }, - }) - mockUseOrganizationAccounts.mockReturnValue({ - data: { - credentialGroup: githubGroup, - viewerAccounts: [{ ...githubAccount, status: 'needs_reauth' }], - }, - isPending: false, - }) - await render() - expect(buttons('Reconnect')).toHaveLength(1) - await act(async () => buttons('Reconnect')[0].click()) - expect(mocks.reconnectOrganizationAccount).toHaveBeenCalledExactlyOnceWith( - 'github-account', - expect.any(Object) - ) - expect(mocks.connect).not.toHaveBeenCalled() - }) - - it('does not reconnect an account through a different active option', async () => { - mockUseOrganizationAccounts.mockReturnValue({ - data: { - credentialGroup: githubGroup, - viewerAccounts: [{ ...githubAccount, optionId: 'other-option', status: 'needs_reauth' }], - }, - isPending: false, - }) - await render() - expect(buttons('Reconnect')).toHaveLength(0) - expect(document.querySelector('[aria-label="GitHub integration actions"]')).not.toBeNull() - }) - - it.each(['pending', 'error'] as const)( - 'does not fall through to repository setup when the inventory is %s', - async (state) => { - mockUseOrganizationAccounts.mockReturnValue({ - data: undefined, - isPending: state === 'pending', - isError: state === 'error', - error: state === 'error' ? new Error('Could not load accounts') : null, - isFetching: false, - refetch: mocks.refetchAccounts, - }) - await render() - expect(container.textContent).toContain('GitHub') - expect(mockUseSearchSources).not.toHaveBeenCalled() - expect(buttons('Connect')).toHaveLength(0) - expect(mocks.connectSearchSource).not.toHaveBeenCalled() - if (state === 'error') { - await act(async () => buttons('Retry')[0].click()) - expect(mocks.refetchAccounts).toHaveBeenCalledOnce() - } - } - ) - - it('retains legacy account management only after a successful response omits the inventory', async () => { - mockUseOrganizationAccounts.mockReturnValue({ - data: { credentialGroup: githubGroup }, - isPending: false, - isError: false, - }) - rows = rows.map((source) => ({ - ...source, - viewerMembership: 'connected', - viewerAccounts: [githubAccount], - })) - await render() - expect(mockUseSearchSources).toHaveBeenCalledWith(scope, { - connectorType: 'github', - enabled: true, - }) - expect(document.querySelectorAll('[aria-label="GitHub integration actions"]')).toHaveLength(1) - expect(mocks.accountMenu).toHaveBeenLastCalledWith( - expect.objectContaining({ accounts: [githubAccount] }) - ) - expect(buttons('Connect')).toHaveLength(0) - }) -}) - -describe('grouped member integrations', () => { - it('renders one provider row and loads bounded pages per configured provider', async () => { - mockUseSearchSourceOverview.mockReturnValue({ - data: { providers: [{ connectorType: 'gmail' }, { connectorType: 'google_drive' }] }, - isPending: false, - }) - rows = Array.from({ length: 25 }, (_, index) => ({ - ...memberSource, - connectorId: `gmail-${index}`, - })) - await render() - expect(buttons('Connect')).toHaveLength(1) - expect(container.textContent).toContain('Google Drive') - expect(container.textContent).not.toContain('Inbox') - expect(mockUseSearchSources).toHaveBeenCalledWith(scope, { - connectorType: 'gmail', - enabled: true, - }) - expect(mockUseSearchSources).toHaveBeenCalledWith(scope, { - connectorType: 'google_drive', - enabled: true, - }) - expect(document.querySelector('[role="dialog"]')).toBeNull() - }) - it('keeps the list flat even when an old details URL is opened', async () => { - rows = [ - memberSource, - { ...memberSource, connectorId: 'source-b', sourceDescription: 'Archive' }, - ] - await render('?integration=gmail') - expect(document.querySelector('[role="region"]')).toBeNull() - expect(document.querySelector('[role="dialog"]')).toBeNull() - expect(container.textContent).not.toContain('Inbox') - expect(container.textContent).not.toContain('Archive') - expect(document.querySelector('[aria-label="Gmail integration actions"]')).toBeNull() - expect(buttons('Connect')).toHaveLength(1) - }) - it('connects one configured target even with many same-provider content scopes', async () => { - rows = [ - memberSource, - { ...memberSource, connectorId: 'source-b', sourceDescription: 'Archive' }, - ] - await render('?integration=gmail') - expect(buttons('Connect')).toHaveLength(1) - await act(async () => buttons('Connect')[0].click()) - expect(mocks.connect).toHaveBeenCalledExactlyOnceWith('search-index', 'source-a') - }) - it('deduplicates the same account across scopes', async () => { - rows = [memberSource, { ...memberSource, connectorId: 'source-b' }].map((source) => ({ - ...source, - viewerMembership: 'connected', - viewerAccounts: [account], - })) - await render('?integration=gmail') - expect(mocks.accountMenu).toHaveBeenCalledWith(expect.objectContaining({ accounts: [account] })) - expect(buttons('Connect')).toHaveLength(0) - expect(buttons('Reconnect')).toHaveLength(0) - expect(container.textContent).toContain('Connected') - }) - - it('distinguishes same-name accounts by content and renewal state only when needed', async () => { - rows = [ - { ...memberSource, sourceDescription: 'Engineering', viewerAccounts: [account] }, - { - ...memberSource, - connectorId: 'source-b', - sourceDescription: 'Handbook', - viewerAccounts: [{ ...account, credentialId: 'expired', status: 'needs_reauth' }], - }, - ] - await render() - expect(mocks.accountMenu.mock.calls.at(-1)?.[0].accountLabels).toEqual( - new Map([ - ['account', 'Engineering · My work account'], - ['expired', 'Reconnect required · Handbook · My work account'], - ]) - ) - }) - - it('gives otherwise identical accounts distinct connection labels', async () => { - rows = [ - { - ...memberSource, - viewerAccounts: [account, { ...account, credentialId: 'second' }], - }, - ] - await render() - expect(mocks.accountMenu.mock.calls.at(-1)?.[0].accountLabels).toEqual( - new Map([ - ['account', 'Connection 1 · Inbox · My work account'], - ['second', 'Connection 2 · Inbox · My work account'], - ]) - ) - }) - it.each(['personal', 'central'] as const)( - 'keeps existing %s content connected when another source needs authorization', - async (kind) => { - const connected: SearchSourceSummary = - kind === 'personal' - ? { ...memberSource, viewerMembership: 'connected', viewerAccounts: [account] } - : { ...centralSource, connectorType: 'gmail' } - rows = [connected, { ...memberSource, connectorId: 'additional-source' }] - await render() - expect(container.textContent).toContain('Additional connection required') - expect(container.textContent).not.toContain('Not connected') - expect(buttons('Connect')).toHaveLength(1) - expect(buttons('Reconnect')).toHaveLength(0) - await act(async () => buttons('Connect')[0].click()) - expect(mocks.connect).toHaveBeenCalledExactlyOnceWith('search-index', 'additional-source') - } - ) - it('does not ask for authorization again when connected content fails to sync', async () => { - rows = [memberSource, { ...memberSource, connectorId: 'source-b' }].map((source) => ({ - ...source, - viewerMembership: 'connected', - viewerAccounts: [account], - hasSyncError: true, - })) - await render() - expect(container.textContent).toContain('Sync needs attention') - expect(buttons('Connect')).toHaveLength(0) - expect(buttons('Reconnect')).toHaveLength(0) - }) - it('reconnects the expired target before any connected account', async () => { - rows = [ - { ...memberSource, viewerMembership: 'connected', viewerAccounts: [account] }, - { - ...memberSource, - connectorId: 'expired-source', - viewerMembership: 'needs_reauth', - viewerAccounts: [{ ...account, credentialId: 'expired-account', status: 'needs_reauth' }], - }, - { ...memberSource, connectorId: 'unconnected-source' }, - ] - await render('?integration=gmail') - await act(async () => buttons('Reconnect')[0].click()) - expect(mocks.connect).toHaveBeenCalledExactlyOnceWith('search-index', 'expired-source') - }) - it('explains central connections without asking for a personal account', async () => { - mockUseSearchSourceOverview.mockReturnValue({ - data: { providers: [{ connectorType: 'google_drive', isSyncing: false }] }, - isPending: false, - }) - mocks.integrations.mockReturnValue({ - data: [{ connectorType: 'google_drive', approved: true }], - isPending: false, - }) - rows = [centralSource] - await render('?integration=google_drive') - expect(container.textContent).toContain('Google Drive') - expect(buttons('Connect')).toHaveLength(0) - }) - it.each([{ enabled: false }, { approved: false }, { availability: 'unavailable' as const }])( - 'retains own-account removal when a source cannot connect: %o', - async (override) => { - rows = [ - { - ...memberSource, - viewerMembership: 'needs_reauth', - viewerAccounts: [account], - ...override, - }, - ] - await render('?integration=gmail') - expect(mocks.accountMenu).toHaveBeenCalledWith( - expect.objectContaining({ accounts: [account] }) - ) - expect(buttons('Reconnect')).toHaveLength(0) - } - ) - it.each(['members', 'admin'] as const)( - 'does not claim connected when multi-scope %s access is disabled', - async (accessMode) => { - mockUseOrganizationContext.mockReturnValue({ - organization: { id: scope.organizationId }, - searchAccess: { memberScoped: false, sourceMirrored: false }, - }) - rows = [memberSource, { ...memberSource, connectorId: 'source-b' }].map((source) => ({ - ...source, - accessMode, - isSyncing: true, - viewerMembership: 'connected', - viewerAccounts: [account], - })) - await render() - expect(container.textContent).toContain('Some connections need attention') - expect(container.textContent).not.toContain('Indexing') - expect(buttons('Connect')).toHaveLength(0) - expect(mocks.accountMenu).toHaveBeenCalledWith( - expect.objectContaining({ accounts: [account] }) - ) - } - ) - it('keeps email verification as account recovery and returns to the integration list', async () => { - rows = [{ ...memberSource, viewerEmailVerified: false, viewerMembership: 'unverified_email' }] - await render() - expect(container.textContent).toContain('Verify your email') - expect(buttons('Connect')).toHaveLength(0) - const recovery = container.querySelector('a[href^="/verify"]')! - expect(new URL(recovery.href).searchParams.get('redirectAfter')).toBe( - '/o/organization-a/integrations' - ) - }) - it.each([{ enabled: false }, { approved: false }, { availability: 'unavailable' as const }])( - 'does not offer verification for blocked content: %o', - async (override) => { - rows = [memberSource, { ...memberSource, connectorId: 'source-b' }].map((source) => ({ - ...source, - ...override, - viewerEmailVerified: false, - viewerMembership: 'unverified_email', - })) - await render() - expect(container.querySelector('a[href^="/verify"]')).toBeNull() - expect(container.textContent).toContain('Some connections need attention') - expect(buttons('Connect')).toHaveLength(0) - } - ) - it('preserves explicit pagination instead of pretending loaded scope counts are complete', async () => { - queryOverrides = { hasNextPage: true } - rows = [] - await render('?integration=gmail') - expect(container.textContent).toContain('More connections to check') - await act(async () => buttons('Check connections')[0].click()) - expect(mocks.nextPage).toHaveBeenCalledOnce() - expect(buttons('Connect')).toHaveLength(0) - }) - it.each(['needs_reauth', 'not_enrolled'] as const)( - 'exposes an older %s source without marking a partial inventory connected', - async (membership) => { - rows = Array.from({ length: 25 }, (_, index) => ({ - ...memberSource, - connectorId: `source-${index}`, - viewerMembership: 'connected', - viewerAccounts: [account], - })) - queryOverrides = { hasNextPage: true } - await render() - expect(container.textContent).toContain('More connections to check') - expect(container.textContent).not.toContain('Connected') - expect(buttons('Connect')).toHaveLength(0) - await act(async () => buttons('Check connections')[0].click()) - expect(mocks.nextPage).toHaveBeenCalledOnce() - queryOverrides = { hasNextPage: true, isFetchingNextPage: true, isFetching: true } - await render() - expect(buttons('Checking')[0]).toBeDisabled() - rows = [ - ...rows, - { ...memberSource, connectorId: 'older-source', viewerMembership: membership }, - ] - queryOverrides = { hasNextPage: false } - await render() - expect(buttons('Check connections')).toHaveLength(0) - const action = membership === 'needs_reauth' ? 'Reconnect' : 'Connect' - await act(async () => buttons(action)[0].click()) - expect(mocks.connect).toHaveBeenCalledExactlyOnceWith('search-index', 'older-source') - } - ) - it('retries a failed connection read directly from its flat row', async () => { - queryOverrides = { isError: true, error: new Error('Could not load') } - await render() - expect(buttons('Connect')).toHaveLength(0) - expect(document.querySelector('[role="region"]')).toBeNull() - await act(async () => buttons('Retry')[0].click()) - expect(mocks.refetch).toHaveBeenCalledOnce() - }) - it('keeps later connection pages retryable without an expanded section', async () => { - queryOverrides = { hasNextPage: true, isError: true, isFetchNextPageError: true } - rows = [] - await render() - expect(container.textContent).toContain('Could not check remaining connections') - await act(async () => buttons('Retry')[0].click()) - expect(mocks.nextPage).toHaveBeenCalledOnce() - expect(document.querySelector('[role="region"]')).toBeNull() - }) - it('offers direct Connect only for a new eligible provider, with no duplicate scope row', async () => { - mockUseSearchSourceOverview.mockReturnValue({ data: { providers: [] }, isPending: false }) - await render() - expect(buttons('Connect')).toHaveLength(1) - await act(async () => buttons('Connect')[0].click()) - expect(mocks.connectSearchSource).toHaveBeenCalledWith( - scope, - expect.objectContaining({ type: 'gmail' }) - ) - expect(document.querySelector('[role="dialog"]')).toBeNull() - }) - it.each([ - { data: undefined, isPending: true, isError: false }, - { data: undefined, isPending: false, isError: true, error: new Error('Slack unavailable') }, - ])( - 'keeps other integrations usable when Slack inventory is unavailable: %o', - async (inventory) => { - mocks.slackInventory.mockReturnValue(inventory) - await render() - expect(buttons('Connect')).toHaveLength(1) - await act(async () => buttons('Connect')[0].click()) - expect(mocks.connect).toHaveBeenCalledExactlyOnceWith('search-index', 'source-a') - expect(container.textContent).toContain('Gmail') - } - ) - it.each([false, true])( - 'preserves shared Slack onboarding without a duplicate row (configured: %s)', - async (configured) => { - mockUseSearchSourceOverview.mockReturnValue({ - data: { providers: configured ? [{ connectorType: 'slack' }] : [] }, - isPending: false, - }) - mocks.integrations.mockReturnValue({ - data: [{ connectorType: 'slack', approved: true }], - isPending: false, - }) - mocks.availability.mockReturnValue({ - integrationAvailability: new Map([['slack_v2', { state: 'ready', oauthAvailable: true }]]), - oauthServiceAvailability: new Map([['slack', true]]), - isIntegrationAvailabilityReady: true, - }) - mocks.slackInventory.mockReturnValue({ - data: { available: [{ target: { connectorType: 'slack' } }] }, - isPending: false, - isError: false, - }) - rows = configured ? [{ ...memberSource, connectorType: 'slack' }] : [] - await render() - expect(buttons('Connect')).toHaveLength(1) - expect(mocks.slackInventory).toHaveBeenCalledWith({ - organizationId: scope.organizationId, - connectorType: 'slack', - }) - await act(async () => buttons('Connect')[0].click()) - if (configured) { - expect(mocks.connect).toHaveBeenCalledExactlyOnceWith('search-index', 'source-a') - expect(mocks.connectSearchSource).not.toHaveBeenCalled() - } else { - expect(mocks.connectSearchSource).toHaveBeenCalledExactlyOnceWith( - scope, - expect.objectContaining({ type: 'slack' }) - ) - expect(mocks.connect).not.toHaveBeenCalled() - } - expect(document.querySelector('[role="dialog"]')).toBeNull() - } - ) - it('keeps Slack setup errors relevant to the selected integration filter', async () => { - mocks.integrations.mockReturnValue({ - data: [ - { connectorType: 'gmail', approved: true }, - { connectorType: 'slack', approved: true }, - ], - isPending: false, - }) - mocks.slackInventory.mockReturnValue({ - isPending: false, - isError: true, - error: new Error('Could not load Slack setup'), - }) - await render('', ) - expect(container.textContent).not.toContain('Could not load Slack setup') - expect(buttons('Connect')).toHaveLength(1) - await render('', ) - expect(container.textContent).toContain('Could not load Slack setup') - expect(container.textContent).not.toContain('No integrations are available to connect') - expect(container.textContent).not.toContain('No matching integrations') - }) - it.each([ - { data: { available: [] }, isPending: false, isError: false }, - { - data: { available: [{ target: { connectorType: 'slack', connectorId: 'existing' } }] }, - isPending: false, - isError: false, - }, - { data: undefined, isPending: true, isError: false }, - { data: undefined, isPending: false, isError: true, error: new Error('Could not load Slack') }, - ])('withholds new Slack setup without a ready shared-app target: %o', async (inventory) => { - mockUseSearchSourceOverview.mockReturnValue({ data: { providers: [] }, isPending: false }) - mocks.integrations.mockReturnValue({ - data: [{ connectorType: 'slack', approved: true }], - isPending: false, - }) - mocks.availability.mockReturnValue({ - integrationAvailability: new Map([['slack_v2', { state: 'ready', oauthAvailable: true }]]), - oauthServiceAvailability: new Map([['slack', true]]), - isIntegrationAvailabilityReady: true, - }) - mocks.slackInventory.mockReturnValue(inventory) - await render() - expect(buttons('Connect')).toHaveLength(0) - expect(mocks.connectSearchSource).not.toHaveBeenCalled() - }) - it('withholds new setup on failed/incomplete provider data', async () => { - mockUseSearchSourceOverview.mockReturnValue({ - data: { providers: [{ connectorType: 'confluence', isSyncing: false }] }, - isPending: false, - }) - mocks.integrations.mockReturnValue({ - data: [{ connectorType: 'confluence', approved: true }], - isPending: false, - }) - rows = [{ ...memberSource, connectorType: 'confluence' }] - queryOverrides = { isError: true, error: new Error('Could not load'), hasNextPage: false } - await render('?integration=confluence') - expect( - mocks.accountMenu.mock.calls.at(-1)?.[0].actions.map((action: RowAction) => action.label) - ).toEqual([]) - queryOverrides = { hasNextPage: true } - await render('?integration=confluence') - expect( - mocks.accountMenu.mock.calls.at(-1)?.[0].actions.map((action: RowAction) => action.label) - ).toEqual([]) - }) - it('keeps the integration menu open during background indexing refreshes', async () => { - mockUseSearchSourceOverview.mockReturnValue({ - data: { providers: [{ connectorType: 'confluence', isSyncing: true }] }, - isPending: false, - }) - mocks.integrations.mockReturnValue({ - data: [{ connectorType: 'confluence', approved: true }], - isPending: false, - }) - rows = [{ ...memberSource, connectorType: 'confluence' }] - await render('?integration=confluence') - const trigger = document.querySelector( - '[aria-label="Confluence integration actions"]' - )! - await act(async () => - trigger.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true })) - ) - expect(document.querySelector('[role="menu"]')).not.toBeNull() - queryOverrides = { isFetching: true } - await render('?integration=confluence') - expect(document.querySelector('[aria-label="Confluence integration actions"]')).toBe(trigger) - expect(document.querySelector('[role="menu"]')).not.toBeNull() - }) - it('retains typed connection requests and Slack onboarding on the main page', async () => { - const connectionRequest = { - userId: 'person', - target: { - type: 'link' as const, - provider: 'google-email', - connectorType: 'gmail', - connectorId: 'source-a', - }, - } - await render( - '', - - ) - expect(mocks.request).toHaveBeenCalledWith( - expect.objectContaining({ ...connectionRequest, organizationId: scope.organizationId }) - ) - expect(container.textContent).toContain('Return to Slack') - }) - it('keeps scoped enrollment invalidations and toast error handling', async () => { - await render('?integration=gmail') - const options = mocks.enrollment.mock.calls.at(-1)?.[0] as { - membershipQueryKeys: unknown[] - onConnectionError: (message: string) => void - } - expect(options.membershipQueryKeys).toContainEqual( - organizationAccountsKeys.detail(scope.organizationId) - ) - options.onConnectionError('Choose the matching account') - expect(toast.error).toHaveBeenCalledExactlyOnceWith('Choose the matching account') - expect(document.body.textContent).not.toContain('Choose the matching account') - }) -}) - -describe('live integrations backend selection', () => { - it('connects through existing OAuth enrollment without loading indexed sources', async () => { - mocks.live = true - mocks.integrations.mockReturnValue({ - data: [{ connectorType: 'google_drive', approved: true }], - }) - mockUseOrganizationAccounts.mockReturnValue({ - data: { - availableMcpConnectors: [], - credentialGroup: { - status: 'active', - mcpServers: [], - options: [ - { - id: 'drive', - provider: 'google-drive', - label: 'Google Drive', - status: 'active', - configurationStatus: 'ready', - }, - ], - }, - viewerAccounts: [], - }, - isError: false, - }) - await render('', ) - await act(async () => buttons('Connect')[0].click()) - expect(mocks.connectOrganizationAccount).toHaveBeenCalledWith( - { organizationId: scope.organizationId, optionId: 'drive' }, - expect.any(Object) - ) - expect(mockUseSearchSources).not.toHaveBeenCalled() - expect(mockUseSearchSourceOverview).not.toHaveBeenCalled() - expect(mocks.integrations).toHaveBeenCalledWith(scope.organizationId) - expect(mocks.connectSearchSource).not.toHaveBeenCalled() - }) - it('offers reconnect for an existing personal grant', async () => { - mocks.live = true - mocks.integrations.mockReturnValue({ data: [{ connectorType: 'slack', approved: true }] }) - mockUseOrganizationAccounts.mockReturnValue({ - data: { - availableMcpConnectors: [], - credentialGroup: { - status: 'active', - mcpServers: [], - options: [ - { - id: 'slack', - provider: 'slack', - label: 'Slack', - status: 'active', - configurationStatus: 'ready', - }, - ], - }, - viewerAccounts: [ - { - credentialId: 'my-slack', - providerId: 'slack', - optionId: 'slack', - displayName: 'My Slack', - status: 'needs_reauth', - }, - ], - }, - isError: false, - }) - await render('', ) - await act(async () => buttons('Reconnect')[0].click()) - expect(mocks.reconnectOrganizationAccount).toHaveBeenCalledWith('my-slack', expect.any(Object)) - expect(container.textContent).toContain('Reconnect needed') - }) -}) diff --git a/apps/sim/app/o/[organizationId]/integrations/integrations.tsx b/apps/sim/app/o/[organizationId]/integrations/integrations.tsx index 5a92f870368..0869432d66d 100644 --- a/apps/sim/app/o/[organizationId]/integrations/integrations.tsx +++ b/apps/sim/app/o/[organizationId]/integrations/integrations.tsx @@ -1,11 +1,9 @@ 'use client' -import { useDeploymentShape } from '@/lib/core/config/deployment-shape' import type { SearchConnectionTarget } from '@/lib/knowledge/search/connection-target' import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import { OrganizationPage } from '@/app/o/[organizationId]/components/organization-page' import { useOrganizationPageFilters } from '@/app/o/[organizationId]/components/organization-page/use-organization-page-filters' -import { MemberIntegrationsList } from '@/app/o/[organizationId]/integrations/indexed' import { LiveMemberIntegrations } from '@/app/o/[organizationId]/integrations/live-member-integrations' import { SlackSearchActions } from '@/app/o/[organizationId]/integrations/slack-search-actions' import { useOrganizationContext } from '@/app/o/[organizationId]/providers/organization-provider' @@ -25,16 +23,12 @@ export function OrganizationIntegrations({ useOAuthReturnRouter() useDesktopOAuthConnectListener() const { organization } = useOrganizationContext() - const { features } = useDeploymentShape() const { search } = useOrganizationPageFilters() const sourceSearch = useDebounce(search.trim(), SEARCH_DEBOUNCE_MS) return ( )} - {features.liveEnterpriseSearch ? ( - - ) : ( - - )} + ) } diff --git a/apps/sim/app/o/[organizationId]/knowledge/[knowledgeBaseId]/[documentId]/page.tsx b/apps/sim/app/o/[organizationId]/knowledge/[knowledgeBaseId]/[documentId]/page.tsx deleted file mode 100644 index 35ad5bdf2f7..00000000000 --- a/apps/sim/app/o/[organizationId]/knowledge/[knowledgeBaseId]/[documentId]/page.tsx +++ /dev/null @@ -1,91 +0,0 @@ -import { ChipLink } from '@sim/emcn' -import { notFound, redirect } from 'next/navigation' -import type { SearchParams } from 'nuqs/server' -import { readSearchDocumentResultSchema } from '@/lib/api/contracts/knowledge/documents' -import { getSession } from '@/lib/auth' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { readSearchDocument } from '@/lib/sim-search/indexed' -import { isIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' -import { buildAuthCrossLink } from '@/app/(auth)/auth-redirect' -import { - loadDocumentReadParams, - serializeDocumentReadParams, -} from '@/app/o/[organizationId]/knowledge/[knowledgeBaseId]/[documentId]/search-params' -import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' -import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' - -interface OrganizationDocumentPageProps { - params: Promise<{ organizationId: string; knowledgeBaseId: string; documentId: string }> - searchParams: Promise -} - -export default async function OrganizationDocumentPage({ - params, - searchParams, -}: OrganizationDocumentPageProps) { - if (!isIndexedOrgSearchEnabled()) notFound() - const { organizationId, knowledgeBaseId, documentId } = await params - const position = await loadDocumentReadParams(searchParams, { strict: true }).catch(() => - notFound() - ) - const href = `/o/${encodeURIComponent(organizationId)}/knowledge/${encodeURIComponent(knowledgeBaseId)}/${encodeURIComponent(documentId)}` - const session = await getSession() - if (!session?.user) { - redirect( - buildAuthCrossLink('/login', { - callbackUrl: serializeDocumentReadParams(href, position), - isInviteFlow: false, - }) - ) - } - const registry = new ResolvedSecretTraceRegistry() - let result: Awaited> - try { - result = await readSearchDocument.execute({ - principal: { kind: 'session', userId: session.user.id, sessionId: session.session.id }, - input: { - documentId, - assertedOrganizationId: organizationId, - ...position, - limit: 3, - resultSecretRegistry: registry, - }, - }) - } catch (error) { - if ( - error instanceof OrchestrationError && - (error.code === 'not_found' || error.code === 'forbidden' || error.code === 'validation') - ) - notFound() - throw error - } - if (result.knowledgeBaseId !== knowledgeBaseId) notFound() - const projected = projectResolvedSecretModelContent(result, registry, 1024 * 1024) - if (!projected.safe) return

This document cannot be displayed safely.

- const document = readSearchDocumentResultSchema.parse(projected.value) - return ( -
-
-

- {document.documentName ?? 'Document'} -

- {document.chunks.map((chunk) => ( -

- {chunk.content} -

- ))} - -
-
- ) -} diff --git a/apps/sim/app/o/[organizationId]/knowledge/[knowledgeBaseId]/[documentId]/search-params.ts b/apps/sim/app/o/[organizationId]/knowledge/[knowledgeBaseId]/[documentId]/search-params.ts deleted file mode 100644 index 42f9dd224e8..00000000000 --- a/apps/sim/app/o/[organizationId]/knowledge/[knowledgeBaseId]/[documentId]/search-params.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { createLoader, createParser, createSerializer } from 'nuqs/server' - -const parseAsDocumentPosition = createParser({ - parse: (value) => { - if (!/^\d+$/.test(value)) return null - const position = Number(value) - return Number.isSafeInteger(position) && position <= 2147483647 ? position : null - }, - serialize: String, -}).withDefault(0) - -export const documentReadParams = { - startChunkIndex: parseAsDocumentPosition, - startOffset: parseAsDocumentPosition, -} - -const documentReadUrlKeys = { - urlKeys: { - startChunkIndex: 'start-chunk-index', - startOffset: 'start-offset', - }, -} as const - -export const loadDocumentReadParams = createLoader(documentReadParams, documentReadUrlKeys) -export const serializeDocumentReadParams = createSerializer(documentReadParams, documentReadUrlKeys) diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/index.ts b/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/index.ts deleted file mode 100644 index 658a877fbc1..00000000000 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { IndexedOrganizationIntegrationsSettings } from '@/app/o/[organizationId]/settings/components/integrations/indexed/indexed-organization-integrations-settings' diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/indexed-organization-integrations-settings.tsx b/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/indexed-organization-integrations-settings.tsx deleted file mode 100644 index 714b6b164ec..00000000000 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/indexed-organization-integrations-settings.tsx +++ /dev/null @@ -1,139 +0,0 @@ -'use client' - -import { useState } from 'react' -import { Chip, ChipConfirmModal, ChipModalError, ChipSwitch, toast } from '@sim/emcn' -import { useQueryStates } from 'nuqs' -import { getOrganizationAccountUpdateOptions } from '@/lib/credential-groups/organization-account-options' -import { useOrganizationContext } from '@/app/o/[organizationId]/providers/organization-provider' -import { OrganizationIntegrationsSetup } from '@/app/o/[organizationId]/settings/components/integrations/indexed/organization-integrations-setup' -import { OrganizationSourcePeople } from '@/app/o/[organizationId]/settings/components/integrations/indexed/organization-source-people' -import { OrganizationSourceStats } from '@/app/o/[organizationId]/settings/components/integrations/indexed/organization-source-stats' -import { - organizationIntegrationsTabParam, - organizationPeopleIntegrationParam, -} from '@/app/o/[organizationId]/settings/components/integrations/search-params' -import { RowActionsMenu } from '@/app/workspace/[workspaceId]/settings/components/row-actions-menu' -import { - SettingsEmptyState, - SettingsQueryErrorState, -} from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' -import { - useOrganizationAccounts, - useUpdateOrganizationAccounts, -} from '@/hooks/queries/organization-accounts' - -/** - * Organization Integrations settings for indexed organization search: the Sources, People, and - * Stats tabs over the connectors that crawl the search index. Rendered only while the deployment - * serves indexed search (`features.liveEnterpriseSearch === false`). - */ -export function IndexedOrganizationIntegrationsSettings() { - const { organization, viewer } = useOrganizationContext() - const [{ tab }, setNavigation] = useQueryStates({ - [organizationIntegrationsTabParam.key]: organizationIntegrationsTabParam.parser, - [organizationPeopleIntegrationParam.key]: organizationPeopleIntegrationParam.parser, - }) - const accounts = useOrganizationAccounts(viewer.isAdmin ? organization.id : undefined) - const update = useUpdateOrganizationAccounts() - const [refreshOpen, setRefreshOpen] = useState(false) - const group = accounts.data?.credentialGroup - const refreshConnections = () => { - if (!group || update.isPending) return - update.mutate( - { - organizationId: organization.id, - groupId: group.id, - update: { options: getOrganizationAccountUpdateOptions(group) }, - }, - { - onSuccess: () => { - setRefreshOpen(false) - toast.success('Sign-in settings updated') - }, - } - ) - } - if (!viewer.isAdmin) return null - - const tabs = ( - void setNavigation({ tab: value, integration: null })} - options={[ - { value: 'providers', label: 'Sources' }, - { value: 'people', label: 'People' }, - { value: 'stats', label: 'Stats' }, - ]} - /> - ) - - return ( -
- {tab === 'providers' && ( -
- {tabs} - {!accounts.error && group && group.options.length > 0 && ( - { - update.reset() - setRefreshOpen(true) - }, - }, - ]} - /> - )} -
- )} - { - if (!update.isPending) setRefreshOpen(open) - }} - title='Update sign-in settings?' - text='Apply Sim’s current OAuth app and permission settings to member connections. People whose settings changed must reconnect. This does not sync content.' - confirm={{ label: 'Update', pending: update.isPending, onClick: refreshConnections }} - > - {update.error?.message} - - {tab === 'providers' && } - {tab === 'stats' && } - {tab === 'people' && ( - void accounts.refetch()} - variant='inline' - /> - ) : !accounts.data ? ( - Loading connected accounts - ) : !accounts.data.credentialGroup ? ( -
- - Add a source that uses member accounts before requesting connections. - - void setNavigation({ tab: 'providers', integration: null })}> - View sources - -
- ) : undefined - } - /> - )} -
- ) -} diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-integrations-setup.tsx b/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-integrations-setup.tsx deleted file mode 100644 index 83fd54b8354..00000000000 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-integrations-setup.tsx +++ /dev/null @@ -1,188 +0,0 @@ -'use client' - -import { toast } from '@sim/emcn' -import { Plus } from '@sim/emcn/icons' -import { useQueryStates } from 'nuqs' -import { SettingsPanel } from '@/components/settings/settings-panel' -import { organizationRoutes } from '@/lib/navigation/paths' -import { getConnectorAccessAvailability, SEARCH_SOURCE_TYPES } from '@/lib/sim-search/connectors' -import { searchSetupAccessParam, searchSetupParam } from '@/lib/sim-search/search-params' -import { useOrganizationContext } from '@/app/o/[organizationId]/providers/organization-provider' -import { AddOrganizationSourceModal } from '@/app/o/[organizationId]/settings/components/integrations/add-organization-source-modal' -import { organizationSearchStatusLabel } from '@/app/o/[organizationId]/settings/components/integrations/organization-search-status' -import { SearchSourceSetup } from '@/app/o/[organizationId]/settings/components/integrations/search-source-setup' -import { OrganizationSlackAccountSetup } from '@/app/o/[organizationId]/settings/components/integrations/slack-account-setup' -import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' -import { - SettingsEmptyState, - SettingsQueryErrorState, -} from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' -import { - RESOURCE_LIST_STACK, - SettingsResourceRow, -} from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' -import { useSettingsSearch } from '@/app/workspace/[workspaceId]/settings/components/use-settings-search' -import { useOrganizationSearchOverview } from '@/hooks/queries/kb/connectors' -import { useUpdateSearchIntegration } from '@/hooks/queries/search-integrations' -import { usePermissionConfig } from '@/hooks/use-permission-config' - -export function OrganizationIntegrationsSetup() { - const { organization, viewer, searchAccess } = useOrganizationContext() - const [search, setSearch] = useSettingsSearch() - const [setup, setSetup] = useQueryStates( - { - [searchSetupParam.key]: searchSetupParam.parser, - [searchSetupAccessParam.key]: searchSetupAccessParam.parser, - }, - { history: 'replace' } - ) - const overview = useOrganizationSearchOverview(organization.id, { enabled: viewer.isAdmin }) - const availability = usePermissionConfig() - const approval = useUpdateSearchIntegration() - const providers = new Map( - overview.data?.providers.map((provider) => [provider.connectorType, provider]) - ) - const sources = SEARCH_SOURCE_TYPES.map(([type, meta]) => ({ - type, - meta, - access: getConnectorAccessAvailability(meta, availability.integrationAvailability, { - memberAccessAvailable: searchAccess.memberScoped, - mirroredAccessAvailable: searchAccess.sourceMirrored, - oauthServiceAvailability: availability.oauthServiceAvailability, - isIntegrationAvailabilityReady: availability.isIntegrationAvailabilityReady, - }), - })) - const query = search.trim().toLowerCase() - const visible = sources.flatMap((source) => { - const provider = providers.get(source.type) - return provider && - (provider.approved || provider.sourceCount > 0) && - source.meta.name.toLowerCase().includes(query) - ? [{ ...source, provider }] - : [] - }) - const ready = Boolean( - !overview.isPending && - !overview.isError && - availability.isIntegrationAvailabilityReady && - !availability.integrationAvailabilityError - ) - const closePicker = () => { - if (!approval.isPending) void setSetup({ addConnector: null, 'source-access': null }) - } - const selectSource = (type: string, accessMode: 'admin' | 'members') => { - const selectedType = searchSetupParam.parser.parse(type) - if (!selectedType || !ready || approval.isPending) return - const startSetup = () => - void setSetup({ - addConnector: selectedType, - 'source-access': accessMode === 'members' ? 'members' : null, - }) - if (providers.get(type)?.approved) { - startSetup() - return - } - approval.mutate( - { organizationId: organization.id, connectorType: type, approved: true }, - { onSuccess: startSetup, onError: (error) => toast.error(error.message) } - ) - } - if (!viewer.isAdmin) return null - if (!searchAccess.memberScoped && !searchAccess.sourceMirrored) - return ( - - Search sources are not enabled for this organization. - - ) - - const feedback = overview.isError ? ( - void overview.refetch()} - variant='inline' - /> - ) : availability.integrationAvailabilityError ? ( - void availability.refetchIntegrationAvailability()} - variant='inline' - /> - ) : null - - return ( - <> - void setSetup({ addConnector: '', 'source-access': null }), - }, - ]} - search={{ value: search, onChange: setSearch, placeholder: 'Search sources' }} - > - {setup.addConnector !== '' && feedback} -
- {overview.isError ? null : overview.isPending ? ( - Loading sources - ) : visible.length === 0 ? ( - - {query ? 'No matching sources' : 'No sources yet. Add a source to get started.'} - - ) : ( - visible.map(({ type, meta, access, provider }) => { - const available = access.admin || access.members - const status = - provider.approved && ready && !available - ? 'Unavailable in this deployment' - : organizationSearchStatusLabel(provider) - return ( - } - title={meta.name} - description={[ - status, - provider.sourceCount > 0 - ? `${provider.sourceCount} ${provider.sourceCount === 1 ? 'connection' : 'connections'}` - : undefined, - ] - .filter(Boolean) - .join(' · ')} - href={organizationRoutes(organization.id).searchProvider(type)} - clickLabel={`Manage ${meta.name}`} - navigable - /> - ) - }) - )} -
- - {setup.addConnector === '' ? ( - - ) : ( - - )} - - - ) -} diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-search-stats-period.tsx b/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-search-stats-period.tsx deleted file mode 100644 index cff874b2785..00000000000 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-search-stats-period.tsx +++ /dev/null @@ -1,89 +0,0 @@ -'use client' - -import { useRef, useState } from 'react' -import { Calendar, ChipCombobox, Popover, PopoverAnchor, PopoverContent, toast } from '@sim/emcn' -import { formatDateShort } from '@/lib/core/utils/date-display' -import { getSearchStatsRangeError, type SEARCH_STATS_PERIODS } from '@/lib/knowledge/search/stats' - -const PERIOD_OPTIONS = [ - { value: 'today', label: 'Today' }, - { value: '3d', label: 'Past 3 days' }, - { value: '7d', label: 'Past 7 days' }, - { value: '14d', label: 'Past 14 days' }, - { value: '30d', label: 'Past 30 days' }, - { value: '90d', label: 'Past 90 days' }, - { value: 'custom', label: 'Custom range' }, -] satisfies { value: (typeof SEARCH_STATS_PERIODS)[number]; label: string }[] - -interface SearchStatsPeriodSelection { - period: (typeof SEARCH_STATS_PERIODS)[number] - startDate: string | null - endDate: string | null -} - -interface OrganizationSearchStatsPeriodProps extends SearchStatsPeriodSelection { - onChange: (selection: SearchStatsPeriodSelection) => void -} - -export function OrganizationSearchStatsPeriod({ - period, - startDate, - endDate, - onChange, -}: OrganizationSearchStatsPeriodProps) { - const triggerContainerRef = useRef(null) - const calendarRef = useRef(null) - const [calendarOpen, setCalendarOpen] = useState(false) - const label = - period === 'custom' && startDate && endDate && !getSearchStatsRangeError({ startDate, endDate }) - ? `${formatDateShort(startDate)} – ${formatDateShort(endDate)}` - : PERIOD_OPTIONS.find((option) => option.value === period)?.label - - return ( -
- { - const selected = PERIOD_OPTIONS.find((option) => option.value === value) - if (!selected) return - if (selected.value === 'custom') setCalendarOpen(true) - else onChange({ period: selected.value, startDate: null, endDate: null }) - }} - /> - - - calendarRef.current?.focus()} - onCloseAutoFocus={() => - triggerContainerRef.current?.querySelector('[role="combobox"]')?.focus() - } - > - setCalendarOpen(false)} - onRangeChange={(start, end) => { - const error = getSearchStatsRangeError({ startDate: start, endDate: end }) - if (error) { - toast.error(error) - return - } - onChange({ period: 'custom', startDate: start, endDate: end }) - setCalendarOpen(false) - }} - /> - - -
- ) -} diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-source-people.tsx b/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-source-people.tsx deleted file mode 100644 index 338ba91c5d9..00000000000 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-source-people.tsx +++ /dev/null @@ -1,79 +0,0 @@ -'use client' - -import type { ComponentProps, ReactNode } from 'react' -import { ChipSelect } from '@sim/emcn' -import { useQueryState } from 'nuqs' -import type { CredentialGroupOption } from '@/lib/api/contracts/credential-groups' -import { getCredentialGroupIndexingConnector } from '@/lib/credential-groups/indexing' -import { organizationPeopleIntegrationParam } from '@/app/o/[organizationId]/settings/components/integrations/search-params' -import { OrganizationAccountPeople } from '@/ee/credential-groups/components/organization-account-people' - -interface OrganizationSourcePeopleProps - extends Omit, 'searchConnection' | 'filters'> { - options: CredentialGroupOption[] - tabs: ReactNode -} - -export function OrganizationSourcePeople({ - options, - tabs, - ...props -}: OrganizationSourcePeopleProps) { - const [integration, setIntegration] = useQueryState( - organizationPeopleIntegrationParam.key, - organizationPeopleIntegrationParam.parser - ) - const integrations = options - .flatMap((option) => { - const connector = getCredentialGroupIndexingConnector(option.provider) - return option.status === 'active' && connector - ? [ - { - optionId: option.id, - type: connector.type, - name: connector.meta.name, - icon: connector.meta.icon, - needsSetup: option.provider === 'slack' && option.configurationStatus !== 'ready', - }, - ] - : [] - }) - .sort((a, b) => a.name.localeCompare(b.name)) - const selected = integrations.find((item) => item.type === integration) - - return ( - -
- {tabs} - void setIntegration(value === 'all' ? null : value)} - disabled={props.enabled === false || Boolean(props.setupFallback)} - options={[ - { value: 'all', label: 'All integrations' }, - ...integrations.map((item) => ({ - value: item.type, - label: item.name, - icon: item.icon, - })), - ]} - /> -
- {selected?.needsSetup && ( -

- Update the Slack app from Sources before requesting connections. -

- )} - - } - /> - ) -} diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-source-stats.tsx b/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-source-stats.tsx deleted file mode 100644 index 0048318bfaa..00000000000 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/indexed/organization-source-stats.tsx +++ /dev/null @@ -1,226 +0,0 @@ -'use client' - -import { type ReactNode, useMemo } from 'react' -import { BarChart, Chip, ChipSelect, Tooltip } from '@sim/emcn' -import { CircleInfo } from '@sim/emcn/icons' -import { useQueryStates } from 'nuqs' -import { - SEARCH_STATS_PEOPLE_LIMIT, - SEARCH_STATS_SURFACE_LABELS, - SEARCH_STATS_SURFACES, -} from '@/lib/knowledge/search/stats' -import { OrganizationSearchStatsPeriod } from '@/app/o/[organizationId]/settings/components/integrations/indexed/organization-search-stats-period' -import { SettingsEmptyState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' -import { SettingsPanel } from '@/app/workspace/[workspaceId]/settings/components/settings-panel' -import { - RESOURCE_LIST_STACK, - SettingsResourceRow, -} from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' -import { SettingsSection } from '@/app/workspace/[workspaceId]/settings/components/settings-section/settings-section' -import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' -import { - organizationSearchStatsParsers, - organizationSearchStatsUrlOptions, -} from '@/ee/organization-search-stats/search-params' -import { useOrganizationSearchStats } from '@/hooks/queries/organization-search-stats' - -const SURFACE_OPTIONS = [ - { value: 'all', label: 'All surfaces' }, - ...SEARCH_STATS_SURFACES.map((surface) => ({ - value: surface, - label: SEARCH_STATS_SURFACE_LABELS[surface], - })), -] - -interface OrganizationSourceStatsProps { - organizationId: string - tabs?: ReactNode -} - -function sourceLabel(sourceType: string) { - return ( - CONNECTOR_META_REGISTRY[sourceType]?.name ?? (sourceType === 'uploads' ? 'Uploads' : sourceType) - ) -} - -export function OrganizationSourceStats({ organizationId, tabs }: OrganizationSourceStatsProps) { - const [{ period, surface, startDate, endDate }, setFilters] = useQueryStates( - organizationSearchStatsParsers, - organizationSearchStatsUrlOptions - ) - const stats = useOrganizationSearchStats({ - organizationId, - period, - surface: surface ?? undefined, - ...(period === 'custom' - ? { startDate: startDate ?? undefined, endDate: endDate ?? undefined } - : {}), - }) - const series = useMemo( - () => - stats.data?.series.map((point) => ({ - timestamp: point.timestamp, - value: point.invocations, - })) ?? [], - [stats.data?.series] - ) - const data = stats.data - const totals = data?.totals - const metrics = totals - ? [ - { label: 'Search invocations', value: totals.invocations.toLocaleString() }, - { label: 'Active people', value: totals.activePeople.toLocaleString() }, - { label: 'Results returned', value: totals.results.toLocaleString() }, - ] - : [] - - return ( - -
- {tabs} -
- - void setFilters({ surface: organizationSearchStatsParsers.surface.parse(value) }) - } - /> - void setFilters(selection)} - /> -
-
- {stats.isError ? ( - - Couldn’t load Search stats. void stats.refetch()}>Try again - - ) : !data || !totals ? ( - Loading Search stats - ) : ( - <> -
- {metrics.map((metric) => ( -
-
{metric.label}
-
{metric.value}
-
- ))} -
- - - - - - Successful Search requests since tracking was enabled. Assistant and MCP counts - are Search tool calls. Results count each document once per request. One request - can return multiple sources. - - - } - action={ - - {period === 'custom' ? 'UTC' : 'UTC · Includes today'} - - } - > - - - {!surface && totals.invocations > 0 && ( - -
- {data.surfaces.map((row) => ( - - {row.invocations.toLocaleString()} - - } - /> - ))} -
- - )} - - Invocations returning this source - - } - > - {data.sources.length ? ( -
- {data.sources.map((row) => { - const Icon = CONNECTOR_META_REGISTRY[row.sourceType]?.icon - return ( - : undefined} - title={sourceLabel(row.sourceType)} - badge={ - - {row.invocations.toLocaleString()} - - } - /> - ) - })} -
- ) : ( - - No sources returned in this period. - - )} - - - Top {SEARCH_STATS_PEOPLE_LIMIT} · Invocations - - } - > - {data.people.length ? ( -
- {data.people.map((person) => ( - - {person.invocations.toLocaleString()} - - } - /> - ))} -
- ) : ( - - No active people in this period. - - )} - - - )} - - ) -} diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-integrations-settings.test.tsx b/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-integrations-settings.test.tsx deleted file mode 100644 index d5c61f0370a..00000000000 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-integrations-settings.test.tsx +++ /dev/null @@ -1,456 +0,0 @@ -/** @vitest-environment jsdom */ -import { act } from 'react' -import { toast } from '@sim/emcn' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' -import { - organizationAccountsQueriesMock, - organizationAccountsQueriesMockFns, -} from '@sim/testing/mocks/organization-accounts-queries.mock' -import { - organizationProviderMock, - organizationProviderMockFns, -} from '@sim/testing/mocks/organization-provider.mock' -import { NuqsTestingAdapter } from 'nuqs/adapters/testing' -import { createRoot, type Root } from 'react-dom/client' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const mocks = vi.hoisted(() => ({ - invite: vi.fn(), - refetch: vi.fn(), - update: vi.fn(), - updatePending: false, - updateError: null as Error | null, - resetUpdate: vi.fn(), -})) -vi.mock('@/app/o/[organizationId]/providers/organization-provider', () => organizationProviderMock) -vi.mock( - '@/app/o/[organizationId]/settings/components/integrations/indexed/organization-integrations-setup', - () => ({ OrganizationIntegrationsSetup: () =>
Provider setup
}) -) -vi.mock( - '@/app/o/[organizationId]/settings/components/integrations/indexed/organization-source-stats', - () => ({ - OrganizationSourceStats: ({ organizationId }: { organizationId: string }) => ( -
Stats for {organizationId}
- ), - }) -) -vi.mock('@/hooks/queries/organization-accounts', () => organizationAccountsQueriesMock) - -import { SettingsHeaderProvider, SettingsHeaderShell } from '@/components/settings/settings-header' -import { resetDeploymentShape } from '@/lib/core/config/deployment-shape' -import { OrganizationIntegrationsSettings } from '@/app/o/[organizationId]/settings/components/integrations/organization-integrations-settings' - -const mockContext = organizationProviderMockFns.mockUseOrganizationContext -const { mockUseOrganizationAccounts: mockAccounts, mockUseOrganizationAccountPeople: mockPeople } = - organizationAccountsQueriesMockFns -organizationAccountsQueriesMockFns.mockUseUpdateOrganizationAccounts.mockImplementation(() => ({ - mutate: mocks.update, - isPending: mocks.updatePending, - error: mocks.updateError, - reset: mocks.resetUpdate, -})) -organizationAccountsQueriesMockFns.mockUseInviteOrganizationAccountPeople.mockImplementation( - () => ({ - mutateAsync: mocks.invite, - reset: vi.fn(), - }) -) -organizationAccountsQueriesMockFns.mockUseResendOrganizationAccountInvitation.mockImplementation( - () => ({}) -) -organizationAccountsQueriesMockFns.mockUseRevokeOrganizationAccountEnrollment.mockImplementation( - () => ({}) -) - -describe('organization integration invitations', () => { - let root: Root - let container: HTMLDivElement - - beforeEach(() => { - /** These cover the indexed organization Integrations settings. */ - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) - resetDeploymentShape() - vi.spyOn(toast, 'success').mockReturnValue('toast-id') - vi.spyOn(toast, 'error').mockReturnValue('toast-id') - mocks.updatePending = false - mocks.updateError = null - vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) - mockContext.mockReturnValue({ organization: { id: 'org-a' }, viewer: { isAdmin: true } }) - mockAccounts.mockReturnValue({ - isSuccess: true, - data: { credentialGroup: { id: 'group-a', options: [] } }, - error: null, - refetch: mocks.refetch, - }) - mockPeople.mockReturnValue({ data: { pages: [{ enrollments: [] }] } }) - mocks.invite.mockResolvedValue({ - sentCount: 2, - results: [ - { email: 'one@example.com', success: true }, - { email: 'two@example.com', success: true }, - ], - }) - container = document.createElement('div') - document.body.appendChild(container) - root = createRoot(container) - }) - - afterEach(async () => { - await act(async () => root.unmount()) - container.remove() - resetEnvFlagsMock() - resetDeploymentShape() - }) - - async function render(searchParams = '') { - await act(async () => - root.render( - - - - - - - - ) - ) - } - - function findButton(label: string) { - const button = Array.from(document.querySelectorAll('button')).find( - (element) => element.textContent === label || element.getAttribute('aria-label') === label - ) - if (!button) throw new Error(`Missing ${label} button`) - return button - } - - async function click(label: string) { - await act(async () => findButton(label).click()) - } - - async function openRefresh() { - expect(container.textContent).not.toContain('Update configurations') - await act(async () => - findButton('More source actions').dispatchEvent( - new MouseEvent('pointerdown', { bubbles: true, button: 0 }) - ) - ) - const item = document.querySelector('[role="menuitem"]') - expect(item?.textContent).toBe('Update sign-in settings') - await act(async () => item?.click()) - expect(document.body.textContent).toContain('People whose settings changed must reconnect.') - } - - it('keeps provider setup as the default and sends manual invitations from People to this org', async () => { - await render() - expect(container.textContent).toContain('Provider setup') - expect(mockAccounts).toHaveBeenLastCalledWith('org-a') - expect(mockPeople).not.toHaveBeenCalled() - - await click('People') - expect(container.textContent).not.toContain('Provider setup') - expect(mockAccounts).toHaveBeenLastCalledWith('org-a') - expect(mockPeople).toHaveBeenLastCalledWith('org-a', '', { enabled: true }) - expect(container.querySelector('input[placeholder="Search people..."]')).not.toBeNull() - expect(mocks.invite).not.toHaveBeenCalled() - - await click('Request connections') - const input = document.querySelector('input[placeholder="Enter emails"]') - if (!input) throw new Error('Missing invitation email input') - const paste = new Event('paste', { bubbles: true, cancelable: true }) - Object.defineProperty(paste, 'clipboardData', { - value: { getData: () => 'one@example.com two@example.com' }, - }) - await act(async () => input.dispatchEvent(paste)) - await click('Send requests') - expect(mocks.invite).toHaveBeenCalledExactlyOnceWith({ - organizationId: 'org-a', - emails: ['one@example.com', 'two@example.com'], - }) - expect(document.querySelector('[role="dialog"]')).toBeNull() - }) - - it('refreshes saved provider identities only after choosing the maintenance action and confirming', async () => { - mockAccounts.mockReturnValue({ - isSuccess: true, - data: { - credentialGroup: { - id: 'group-a', - options: [ - { - id: 'github-option', - provider: 'github-repositories', - label: 'Engineering', - required: true, - }, - { - id: 'slack-option', - provider: 'slack', - label: 'Slack', - required: false, - slackBotCredentialId: 'slack-bot', - requiredScopes: ['search:read'], - }, - ], - }, - }, - error: null, - }) - mocks.update.mockImplementationOnce((_input, { onSuccess }) => onSuccess()) - await render() - await openRefresh() - await click('Update') - expect(mocks.update).toHaveBeenCalledWith( - { - organizationId: 'org-a', - groupId: 'group-a', - update: { - options: [ - { - id: 'github-option', - provider: 'github-repositories', - label: 'Engineering', - required: true, - }, - { - id: 'slack-option', - provider: 'slack', - label: 'Slack', - required: false, - slackBotCredentialId: 'slack-bot', - }, - ], - }, - }, - expect.any(Object) - ) - expect(toast.success).toHaveBeenCalledWith('Sign-in settings updated') - - expect(document.querySelector('[role="dialog"]')).toBeNull() - }) - - it('keeps failed refreshes open for retry and blocks duplicate submissions', async () => { - mockAccounts.mockReturnValue({ - isSuccess: true, - data: { credentialGroup: { id: 'group-a', options: [{ provider: 'gmail' }] } }, - error: null, - }) - await render() - await openRefresh() - await click('Update') - mocks.updateError = new Error('Update denied') - await render() - expect(document.body.textContent).toContain('Update denied') - expect(document.querySelector('[role="dialog"]')).not.toBeNull() - mocks.updatePending = true - await render() - expect(findButton('Update')).toBeDisabled() - expect(mocks.update).toHaveBeenCalledOnce() - }) - - it('does not offer maintenance without saved providers', async () => { - await render() - expect(container.querySelector('[aria-label="More source actions"]')).toBeNull() - }) - - it('opens People directly from the saved URL', async () => { - await render('?tab=people') - expect(container.textContent).toContain('Request connections') - expect(container.textContent).not.toContain('Provider setup') - expect(mockPeople).toHaveBeenLastCalledWith('org-a', '', { enabled: true }) - }) - - it('waits for integration options before loading filtered people or allowing invitations', async () => { - mockAccounts.mockReturnValue({ - isSuccess: false, - data: undefined, - error: null, - isPending: true, - }) - await render('?tab=people&integration=jira') - expect(mockAccounts).toHaveBeenLastCalledWith('org-a') - expect(mockPeople).toHaveBeenLastCalledWith('org-a', '', { enabled: false }) - expect(container.textContent).toContain('Loading connected accounts') - expect(container.textContent).not.toContain('No people invited yet') - expect(findButton('Request connections')).toBeDisabled() - await click('Request connections') - expect(document.querySelector('[role="dialog"]')).toBeNull() - - mockAccounts.mockReturnValue({ - isSuccess: true, - data: { credentialGroup: { id: 'group-a', options: [] } }, - error: null, - }) - await render('?tab=people&integration=jira') - expect(container.textContent).not.toContain('Loading connected accounts') - expect(findButton('Request connections')).not.toBeDisabled() - await click('Request connections') - expect(document.querySelector('[role="dialog"]')).not.toBeNull() - expect(mocks.invite).not.toHaveBeenCalled() - }) - - it('stops the people query when setup resolves without a pool and preserves the setup action', async () => { - mockAccounts.mockReturnValue({ - isSuccess: false, - data: undefined, - error: null, - isPending: true, - }) - mockPeople.mockReturnValue({ error: new Error('Organization accounts not configured') }) - await render('?tab=people&integration=jira') - expect(mockPeople).toHaveBeenLastCalledWith('org-a', '', { enabled: false }) - expect(container.textContent).not.toContain('Organization accounts not configured') - - mockAccounts.mockReturnValue({ - isSuccess: true, - data: { credentialGroup: null }, - error: null, - }) - await render('?tab=people&integration=jira') - expect(mockPeople).toHaveBeenLastCalledWith('org-a', '', { enabled: false }) - expect(container.textContent).toContain('before requesting connections') - expect(container.textContent).not.toContain('Organization accounts not configured') - expect(findButton('Request connections')).toBeDisabled() - }) - - it('sends an org without a credential group back to provider setup before invitations', async () => { - mockAccounts.mockReturnValue({ - isSuccess: true, - data: { credentialGroup: null }, - error: null, - }) - await render('?tab=people') - expect(container.textContent).toContain('before requesting connections') - expect(mockPeople).toHaveBeenLastCalledWith('org-a', '', { enabled: false }) - expect(findButton('Request connections')).toBeDisabled() - await click('View sources') - expect(container.textContent).toContain('Provider setup') - expect(mocks.invite).not.toHaveBeenCalled() - }) - - it('surfaces account lookup errors instead of treating them as missing setup', async () => { - mockAccounts.mockReturnValue({ - isSuccess: true, - error: new Error('Account access denied'), - refetch: mocks.refetch, - }) - await render('?tab=people') - expect(container.textContent).toContain('Account access denied') - expect(container.textContent).not.toContain('View sources') - expect(mockPeople).toHaveBeenLastCalledWith('org-a', '', { enabled: false }) - expect(findButton('Request connections')).toBeDisabled() - await click('Try again') - expect(mocks.refetch).toHaveBeenCalledOnce() - }) - - it('does not load admin account data or expose invitations to an ordinary member', async () => { - mockContext.mockReturnValue({ organization: { id: 'org-a' }, viewer: { isAdmin: false } }) - await render('?tab=people') - expect(container.textContent).toBe('') - expect(mockAccounts).toHaveBeenLastCalledWith(undefined) - expect(mockPeople).not.toHaveBeenCalled() - expect(mocks.invite).not.toHaveBeenCalled() - }) - it('opens organization stats without loading people', async () => { - await render() - await click('Stats') - expect(container.textContent).toContain('Stats for org-a') - expect(mockPeople).not.toHaveBeenCalled() - }) - - it('filters connection summaries and requests to the selected integration, then returns to All', async () => { - mockAccounts.mockReturnValue({ - isSuccess: true, - data: { - credentialGroup: { - id: 'group-a', - options: [ - { id: 'jira-option', provider: 'jira', status: 'active' }, - { id: 'gmail-option', provider: 'gmail', status: 'active' }, - { id: 'old-option', provider: 'confluence', status: 'revoked' }, - ], - }, - }, - }) - await render('?tab=people&integration=jira&credential-group-people=alex') - expect(mockPeople).toHaveBeenLastCalledWith('org-a', 'alex', { - enabled: true, - optionId: 'jira-option', - }) - expect(findButton('Filter people by integration').textContent).toContain('Jira') - await click('Request connections') - expect(document.querySelector('[role="dialog"]')?.textContent).toContain( - 'Request Jira connections' - ) - await click('Cancel') - await act(async () => - findButton('Filter people by integration').dispatchEvent( - new MouseEvent('pointerdown', { bubbles: true, button: 0 }) - ) - ) - const all = Array.from(document.querySelectorAll('[role="menuitem"]')).find( - (item) => item.textContent === 'All integrations' - ) - expect(all).toBeDefined() - expect(document.querySelector('[role="menu"]')?.textContent).not.toContain('Confluence') - await act(async () => all?.click()) - await vi.waitFor(() => - expect(mockPeople).toHaveBeenLastCalledWith('org-a', 'alex', { enabled: true }) - ) - expect(container.querySelector('input[placeholder="Search people..."]')).toHaveValue('alex') - }) - - it.each(['', '&integration=gmail'])( - 'defaults to All on navigation with one integration and initial filter %s', - async (filter) => { - mockAccounts.mockReturnValue({ - isSuccess: true, - data: { - credentialGroup: { - id: 'group-a', - options: [{ id: 'gmail-option', provider: 'gmail', status: 'active' }], - }, - }, - }) - await render(`?tab=people${filter}`) - expect(findButton('Filter people by integration').textContent).toContain( - filter ? 'Gmail' : 'All integrations' - ) - expect(mockPeople).toHaveBeenLastCalledWith('org-a', '', { - enabled: true, - ...(filter ? { optionId: 'gmail-option' } : {}), - }) - await click('Sources') - await click('People') - expect(findButton('Filter people by integration').textContent).toContain('All integrations') - expect(mockPeople).toHaveBeenLastCalledWith('org-a', '', { enabled: true }) - } - ) - - it('preserves Slack setup recovery in People without hiding existing connections', async () => { - mockAccounts.mockReturnValue({ - isSuccess: true, - data: { - credentialGroup: { - id: 'group-a', - options: [ - { - id: 'slack-option', - provider: 'slack', - status: 'active', - configurationStatus: 'needs_update', - }, - ], - }, - }, - }) - await render('?tab=people&integration=slack') - expect(mockPeople).toHaveBeenLastCalledWith('org-a', '', { - enabled: true, - optionId: 'slack-option', - }) - expect(findButton('Request connections')).toBeDisabled() - expect(container.textContent).toContain('Update the Slack app from Sources') - }) -}) diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-integrations-settings.tsx b/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-integrations-settings.tsx index 58942c418ad..035c95cb41c 100644 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-integrations-settings.tsx +++ b/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-integrations-settings.tsx @@ -1,14 +1,7 @@ 'use client' -import { useDeploymentShape } from '@/lib/core/config/deployment-shape' -import { IndexedOrganizationIntegrationsSettings } from '@/app/o/[organizationId]/settings/components/integrations/indexed' import { LiveSearchSettings } from '@/app/o/[organizationId]/settings/components/integrations/live-search-settings' export function OrganizationIntegrationsSettings() { - const { features } = useDeploymentShape() - return features.liveEnterpriseSearch ? ( - - ) : ( - - ) + return } diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-search-status.ts b/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-search-status.ts deleted file mode 100644 index d4eed45ee37..00000000000 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/organization-search-status.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { OrganizationSearchProviderSummary } from '@/lib/api/contracts/knowledge/connectors' - -const STATUS_LABELS: Record = { - needs_setup: 'Setup required', - waiting_for_connections: 'Waiting for connections', - indexing: 'Indexing', - needs_attention: 'Sync failed', - paused: 'Paused', - active: 'Ready to search', -} - -export function organizationSearchStatusLabel(provider: OrganizationSearchProviderSummary): string { - if (!provider.approved) return 'Deactivated' - if (provider.status === 'needs_setup' && provider.sourceCount > 0) return 'Waiting for first sync' - if (provider.status === 'needs_attention') { - const error = - provider.issue === 'account_sync_incomplete' - ? 'Some accounts are not up to date' - : provider.issue === 'permission_sync_incomplete' - ? 'Some permissions could not be verified' - : provider.issue === 'document_indexing_failed' - ? 'Some documents failed to index' - : 'Sync failed' - return provider.isSyncing ? `Indexing · ${error}` : error - } - return STATUS_LABELS[provider.status] -} diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.test.tsx b/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.test.tsx index e228074fefb..f06463ec193 100644 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.test.tsx +++ b/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.test.tsx @@ -4,7 +4,6 @@ import { act, cloneElement, type ReactNode } from 'react' import { authClientMock, authClientMockFns } from '@sim/testing/mocks/auth-client.mock' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' import { kbConnectorsQueriesMock, kbConnectorsQueriesMockFns, @@ -406,16 +405,9 @@ afterEach(async () => { container?.remove() root = null container = null - resetEnvFlagsMock() resetDeploymentShape() }) -/** Selects the indexed backend, whose arms of these dialogs index and mirror sources. */ -function selectIndexedSearch() { - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) - resetDeploymentShape() -} - describe('Search source setup with real connector dialogs', () => { it.each([ ['source-one', '/o/org-1/settings/integrations/sources/source-one'], @@ -439,25 +431,6 @@ describe('Search source setup with real connector dialogs', () => { expect(document.querySelector('[role="dialog"]')).toBeNull() } ) - - it('prepares organization connected-account indexing in members mode even when central access is available', async () => { - selectIndexedSearch() - mocks.bases = [] - await render( - , - '?addConnector=slack' - ) - expect(mocks.prepare).toHaveBeenCalledWith({ - organizationId: 'org-1', - connectorType: 'slack', - accessMode: 'members', - }) - }) }) describe('member content credentials in real add and edit dialogs', () => { @@ -742,43 +715,6 @@ describe('administrator source prerequisites in real connector dialogs', () => { mocks.credentials = [driveCredential] }) - it.each(['admin', 'members'] as const)( - 'shows and saves Gmail’s Search default date window in %s mode', - async (accessMode) => { - selectIndexedSearch() - mocks.credentials = [ - { - id: 'gmail-service', - name: 'Gmail indexing', - provider: 'google-email', - type: 'service_account', - }, - ] - await render( - - ) - expect(document.body.textContent).toContain('Last 6 months') - expect(document.body.textContent).not.toContain('All time (default)') - if (accessMode === 'admin') await fill(adminEmailPlaceholder, 'admin@example.com') - await click(button(accessMode === 'admin' ? 'Connect & Sync' : 'Create & Invite')) - expect(mocks.create).toHaveBeenCalledWith( - expect.objectContaining({ - accessMode, - connectorType: 'gmail', - sourceConfig: expect.objectContaining({ dateRange: '6m' }), - }), - expect.any(Object) - ) - } - ) - it('preserves a deliberate Gmail date-range draft and keeps general KB defaults separate', async () => { const key = 'gmail-all-time' useConnectorSetupStore.getState().saveDraft(key, { @@ -929,14 +865,12 @@ describe('administrator source prerequisites in real connector dialogs', () => { ])( 'requires the Directory administrator email in $type administrator mode and refuses empty or blank subjects', async ({ type, provider }) => { - selectIndexedSearch() mocks.credentials = [{ ...driveCredential, provider }] await render( @@ -970,7 +904,6 @@ describe('administrator source prerequisites in real connector dialogs', () => { ])( 'excludes personal OAuth accounts and stale OAuth drafts from $type administrator setup', async ({ type, provider, name }) => { - selectIndexedSearch() const oauthCredential = { id: 'drive-personal', name: 'Personal Drive account', @@ -992,8 +925,7 @@ describe('administrator source prerequisites in real connector dialogs', () => { @@ -1097,42 +1029,6 @@ describe('administrator source prerequisites in real connector dialogs', () => { } ) - it('does not let an administrator erase the crawl subject from an existing mirrored Drive source', async () => { - selectIndexedSearch() - await render( - - ) - expect(document.body.textContent).toContain('Directory administrator email*') - await fill(adminEmailPlaceholder, '') - expect(button('Save')).toBeDisabled() - await click(button('Save')) - expect(mocks.update).not.toHaveBeenCalled() - await fill(adminEmailPlaceholder, 'replacement@example.com') - expect(button('Save')).toBeEnabled() - await click(button('Save')) - expect(mocks.update.mock.calls[0][0]).toMatchObject({ - connectorId: 'connector-1', - updates: { - sourceConfig: { - adminEmail: 'replacement@example.com', - fileType: 'documents', - }, - }, - }) - expect(mocks.applyAccess).not.toHaveBeenCalled() - }) - it('guides a general knowledge-base member source back to saving its crawl subject without losing drafts or combining mutations', async () => { const existing = connector({ connectorType: 'google_drive', @@ -1338,48 +1234,6 @@ describe('canonical Search connector safety', () => { ) }) - it('defaults an OAuth source to member accounts and never offers workspace-wide access', async () => { - selectIndexedSearch() - await render( - {}} - knowledgeBaseId='kb-search' - isSearchIndex - initialConnectorType='google_drive' - /> - ) - expect(button('Member accounts')).toHaveAttribute('aria-checked', 'true') - expect( - Array.from(document.querySelectorAll('button')).some( - (node) => node.textContent === 'Workspace' - ) - ).toBe(false) - expect(document.body.textContent).not.toContain('Everyone in this workspace') - await click(button('Choose another source')) - const gitlab = Array.from(document.querySelectorAll('button')).find( - (node) => node.getAttribute('aria-label') === 'GitLab' - ) - expect(gitlab).toBeDefined() - await click(gitlab!) - expect(button('Administrator token')).toHaveAttribute('aria-checked', 'true') - expect(button('Non-admin token')).toHaveAttribute('aria-checked', 'false') - expect(document.body.textContent).not.toContain('Connection method') - expect( - Array.from(document.querySelectorAll('button')).some( - (node) => node.textContent === 'Workspace' - ) - ).toBe(false) - await fill('Enter your GitLab PAT', 'fixture-pat') - await fill('gitlab.example.com', 'gitlab.example.test') - await fill('group/project or numeric ID', 'engineering/search') - await click(button('Connect & Sync')) - expect(mocks.create).toHaveBeenCalledWith( - expect.objectContaining({ accessMode: 'admin', connectorType: 'gitlab' }), - expect.any(Object) - ) - }) - it('keeps an existing Search member source out of workspace-wide mode', async () => { await render( - type === 'github' || (!liveSearch && (setup['source-access'] === 'members' || type === 'slack')) - ? ('members' as const) - : ('admin' as const) + type === 'github' ? ('members' as const) : ('admin' as const) const selectedAccessMode = selectedType ? initialMode(selectedType) : undefined const selectedAvailability = selectedMeta @@ -199,10 +195,7 @@ export function SearchSourceSetup({ ) { if (selectedType && session?.user?.id) { const accessMode = initialMode(selectedType) - const setupMode = - liveSearch || !(selectedMeta?.mirrorsSourceAcls && selectedMeta.auth.mode === 'oauth') - ? accessMode - : 'choose' + const setupMode = accessMode return ( void setSelectedType(type !== null ? searchSetupParam.parser.parse(type) : null) @@ -224,7 +217,7 @@ export function SearchSourceSetup({ onCreated={async (type, connector) => { await setSelectedType(null) const destination = organizationRoutes(scope.organizationId).searchSource(connector.id) - if (liveSearch && accessMode === 'admin' && type !== 'gitlab') { + if (accessMode === 'admin' && type !== 'gitlab') { updateSearchIntegration.mutate( { organizationId: scope.organizationId, @@ -366,11 +359,7 @@ export function SearchSourceSetup({ isIntegrationAvailabilityReady, } ) - const available = - type === 'github' || - (!liveSearch && (setup['source-access'] === 'members' || type === 'slack')) - ? members - : central + const available = type === 'github' ? members : central return ( ({ authorize: vi.fn() })) vi.mock('@/lib/settings/application/organization-section-access', () => ({ @@ -27,16 +26,13 @@ import OrganizationProviderPage from '@/app/o/[organizationId]/settings/integrat const mockRedirect = nextNavigationMockFns.mockRedirect const mockGetSession = authMockFns.mockGetSession -/** Member providers such as Jira keep a provider page only under indexed organization search. */ beforeEach(() => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) mockGetSession.mockResolvedValue({ user: { id: 'admin-1' } }) mocks.authorize.mockResolvedValue(true) }) -afterEach(resetEnvFlagsMock) it.each(['jira', 'confluence'])( - 'moves legacy %s Accounts links to filtered People and preserves the search', + 'moves legacy %s Accounts links to current Sources settings', async (connectorType) => { await expect( OrganizationProviderPage({ @@ -49,10 +45,6 @@ it.each(['jira', 'confluence'])( ).rejects.toThrow('NEXT_REDIRECT') const url = new URL(mockRedirect.mock.lastCall![0], 'https://example.com') expect(url.pathname).toBe('/o/org-1/settings/integrations') - expect(url.searchParams.get('tab')).toBe('people') - expect(url.searchParams.get('integration')).toBe(connectorType) - expect(url.searchParams.get('credential-group-people')).toBe('alex+qa@example.com') - expect(url.searchParams.has('view')).toBe(false) } ) @@ -66,14 +58,3 @@ it('authorizes organization settings before redirecting a legacy link', async () ).rejects.toThrow('NEXT_NOT_FOUND') expect(mockRedirect).not.toHaveBeenCalled() }) - -it.each(['jira', ''])( - 'preserves an active setup in a legacy Accounts link (%s)', - async (addConnector) => { - await OrganizationProviderPage({ - params: Promise.resolve({ organizationId: 'org-1', connectorType: 'jira' }), - searchParams: Promise.resolve({ view: 'accounts', addConnector, 'source-access': 'members' }), - }) - expect(mockRedirect).not.toHaveBeenCalled() - } -) diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/page.tsx b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/page.tsx index f0782f40b20..6ecec2b7578 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/page.tsx +++ b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/page.tsx @@ -2,7 +2,6 @@ import { Suspense } from 'react' import type { Metadata } from 'next' import { notFound, redirect } from 'next/navigation' import { getSession } from '@/lib/auth' -import { isLiveEnterpriseSearchEnabled } from '@/lib/core/config/env-flags' import { organizationRoutes } from '@/lib/navigation/paths' import { authorizeOrganizationSettingsSection } from '@/lib/settings/application/organization-section-access' import { SEARCH_SOURCE_TYPES } from '@/lib/sim-search/connectors' @@ -49,7 +48,7 @@ export default async function OrganizationProviderPage({ })) ) notFound() - if (isLiveEnterpriseSearchEnabled && !LIVE_SEARCH_SERVICE_PROVIDERS.includes(connectorType)) + if (!LIVE_SEARCH_SERVICE_PROVIDERS.includes(connectorType)) redirect(organizationRoutes(organizationId).settingsSection('integrations')) const query = await searchParams const activeSetup = diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.tsx b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.tsx index 84142d6f837..0c300e32171 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.tsx +++ b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.tsx @@ -1,21 +1,17 @@ 'use client' import { useState } from 'react' -import { ChipConfirmModal, ChipModalError } from '@sim/emcn' import { ArrowLeft, Plus } from '@sim/emcn/icons' -import { format } from 'date-fns' import { useRouter } from 'next/navigation' import { useQueryState, useQueryStates } from 'nuqs' import type { SettingsAction } from '@/components/settings/settings-header' import { SettingsPanel } from '@/components/settings/settings-panel' -import { useDeploymentShape } from '@/lib/core/config/deployment-shape' import { organizationRoutes } from '@/lib/navigation/paths' import { getSearchConnectionLabels } from '@/lib/sim-search/connection-labels' import { getConnectorAccessAvailability } from '@/lib/sim-search/connectors' import { searchSetupAccessParam, searchSetupParam } from '@/lib/sim-search/search-params' import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import { useOrganizationContext } from '@/app/o/[organizationId]/providers/organization-provider' -import { organizationSearchStatusLabel } from '@/app/o/[organizationId]/settings/components/integrations/organization-search-status' import { connectedAccountsParam } from '@/app/o/[organizationId]/settings/components/integrations/search-params' import { SearchSourcePagination } from '@/app/o/[organizationId]/settings/components/integrations/search-source-pagination' import { SearchSourceSetup } from '@/app/o/[organizationId]/settings/components/integrations/search-source-setup' @@ -31,9 +27,9 @@ import { } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' import { useSettingsSearch } from '@/app/workspace/[workspaceId]/settings/components/use-settings-search' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' -import { useOrganizationSearchOverview, useSearchSources } from '@/hooks/queries/kb/connectors' +import { useSearchSources } from '@/hooks/queries/kb/connectors' import { useOrganizationAccounts } from '@/hooks/queries/organization-accounts' -import { useUpdateSearchIntegration } from '@/hooks/queries/search-integrations' +import { useSearchIntegrations } from '@/hooks/queries/search-integrations' import { useDebounce } from '@/hooks/use-debounce' import { usePermissionConfig } from '@/hooks/use-permission-config' @@ -44,21 +40,18 @@ interface OrganizationProviderDetailProps { export function OrganizationProviderDetail({ connectorType }: OrganizationProviderDetailProps) { const { organization, viewer, searchAccess } = useOrganizationContext() const router = useRouter() - const liveSearch = useDeploymentShape().features.liveEnterpriseSearch const meta = CONNECTOR_META_REGISTRY[connectorType] const [search, setSearch] = useSettingsSearch() const sourceSearch = useDebounce(search.trim(), SEARCH_DEBOUNCE_MS) - const [deactivating, setDeactivating] = useState(false) const [removingSlackAccounts, setRemovingSlackAccounts] = useState(false) const scope = { kind: 'organization', organizationId: organization.id } as const - const overview = useOrganizationSearchOverview(organization.id, { enabled: viewer.isAdmin }) + const overview = useSearchIntegrations(organization.id) const sources = useSearchSources(scope, { connectorType, search: sourceSearch, enabled: viewer.isAdmin, }) const availability = usePermissionConfig() - const approval = useUpdateSearchIntegration() const accounts = useOrganizationAccounts( viewer.isAdmin && connectorType === 'slack' ? organization.id : undefined ) @@ -73,7 +66,7 @@ export function OrganizationProviderDetail({ connectorType }: OrganizationProvid connectedAccountsParam.key, connectedAccountsParam.parser ) - const provider = overview.data?.providers.find((item) => item.connectorType === connectorType) + const provider = overview.data?.find((item) => item.connectorType === connectorType) const approved = provider?.approved === true const back = { text: 'Sources', @@ -103,13 +96,11 @@ export function OrganizationProviderDetail({ connectorType }: OrganizationProvid approved && unavailable ? 'Unavailable in this deployment' : provider - ? liveSearch - ? connectorType === 'gitlab' - ? 'Projects and permissions' - : connectorType === 'github' - ? 'GitHub App repositories' - : 'Service account connections' - : organizationSearchStatusLabel(provider) + ? connectorType === 'gitlab' + ? 'Projects and permissions' + : connectorType === 'github' + ? 'GitHub App repositories' + : 'Service account connections' : undefined, docsLink: meta.searchDocsUrl, search: searchField, @@ -135,30 +126,22 @@ export function OrganizationProviderDetail({ connectorType }: OrganizationProvid connectorType === 'slack' && (option?.provider !== 'slack' || option.configurationStatus !== 'ready') const pending = - overview.isPending || - overview.isError || - approval.isPending || - !availability.isIntegrationAvailabilityReady + overview.isPending || overview.isError || !availability.isIntegrationAvailabilityReady const startSource = () => void setSetup({ addConnector: searchSetupParam.parser.parse(connectorType), 'source-access': access.admin ? null : 'members', }) - const activate = () => - approval.mutate({ organizationId: organization.id, connectorType, approved: true }) const actions: SettingsAction[] = approved ? [ - ...(needsSlackSetup || - access.admin || - (connectorType === 'github' && access.members) || - (!liveSearch && access.members) + ...(needsSlackSetup || access.admin || (connectorType === 'github' && access.members) ? [ { text: needsSlackSetup ? 'Set up Slack app' - : liveSearch && connectorType === 'github' + : connectorType === 'github' ? 'Add repository' - : liveSearch && connectorType !== 'gitlab' + : connectorType !== 'gitlab' ? 'Add service account' : getSearchConnectionLabels(connectorType, access.admin ? 'admin' : 'members') .add, @@ -171,28 +154,15 @@ export function OrganizationProviderDetail({ connectorType }: OrganizationProvid }, ] : []), - ...(!liveSearch - ? [ - { - text: 'Deactivate', - disabled: approval.isPending, - onSelect: () => { - approval.reset() - setDeactivating(true) - }, - }, - ] - : []), ] : [ { - text: liveSearch ? 'View sources' : provider ? 'Activate' : 'Add integration', + text: 'View sources', variant: 'primary', disabled: pending || (!access.admin && !access.members), tooltip: unavailable ? 'This integration is unavailable in this deployment.' : undefined, - onSelect: liveSearch - ? () => router.push(organizationRoutes(organization.id).settingsSection('integrations')) - : activate, + onSelect: () => + router.push(organizationRoutes(organization.id).settingsSection('integrations')), }, ] actions.push(...removalActions) @@ -217,11 +187,6 @@ export function OrganizationProviderDetail({ connectorType }: OrganizationProvid const renderSources = () => ( - {approval.error && ( - - {approval.error.message} - - )} {availability.integrationAvailabilityError && ( - !liveSearch || source.accessMode === 'admin' || (connectorType === 'github' && source.isGitHubInstallation) ) @@ -263,41 +227,7 @@ export function OrganizationProviderDetail({ connectorType }: OrganizationProvid 0 - ? `${source.viewerFailedDocumentCount} ${source.viewerFailedDocumentCount === 1 ? 'document' : 'documents'} failed to index` - : source.isSyncing - ? 'Indexing' - : source.lastSyncAt - ? `Last synced ${format(new Date(source.lastSyncAt), 'MMM d, h:mm a')}` - : 'Waiting for the first sync', - ] - .filter(Boolean) - .join(' · ') - } + description={!approved ? 'Unavailable' : !source.enabled ? 'Paused' : undefined} href={organizationRoutes(organization.id).searchSource(source.connectorId)} clickLabel={`Open ${source.sourceDescription || meta.name}`} navigable @@ -305,7 +235,6 @@ export function OrganizationProviderDetail({ connectorType }: OrganizationProvid ))} {!sources.data?.some( (source) => - !liveSearch || source.accessMode === 'admin' || (connectorType === 'github' && source.isGitHubInstallation) ) && @@ -343,26 +272,6 @@ export function OrganizationProviderDetail({ connectorType }: OrganizationProvid onRemoved={() => setRemovingSlackAccounts(false)} /> )} - { - if (!approval.isPending) setDeactivating(open) - }} - title={`Deactivate ${meta.name}?`} - text='Its content will be unavailable in Search, Assistant, and MCP. Connections and accounts are preserved.' - confirm={{ - label: 'Deactivate', - variant: 'destructive', - pending: approval.isPending, - onClick: () => - approval.mutate( - { organizationId: organization.id, connectorType, approved: false }, - { onSuccess: () => setDeactivating(false) } - ), - }} - > - {approval.error?.message} - ) } diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/search-params.ts b/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/search-params.ts deleted file mode 100644 index b4182204393..00000000000 --- a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/search-params.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { parseAsStringLiteral } from 'nuqs/server' -import { connectorDocumentFilterSchema } from '@/lib/api/contracts/knowledge/connectors' - -export const sourceViewParam = { - key: 'view', - parser: parseAsStringLiteral(['documents', 'settings', 'history']).withDefault('documents'), -} as const - -export const sourceDocumentFilterParam = { - key: 'document-filter', - parser: parseAsStringLiteral(connectorDocumentFilterSchema.options).withDefault('active'), -} as const - -export type SourceView = NonNullable> diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.test.tsx b/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.test.tsx index 7abbbdeb7e3..e7d844179da 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.test.tsx +++ b/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.test.tsx @@ -1,11 +1,6 @@ /** @vitest-environment jsdom */ import { act } from 'react' -import { - createMockDeploymentShape, - deploymentShapeMock, - deploymentShapeMockFns, -} from '@sim/testing/mocks/deployment-shape.mock' import { kbConnectorsQueriesMock, kbConnectorsQueriesMockFns, @@ -15,7 +10,6 @@ import { organizationProviderMock, organizationProviderMockFns, } from '@sim/testing/mocks/organization-provider.mock' -import { NuqsTestingAdapter } from 'nuqs/adapters/testing' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { ApiClientError } from '@/lib/api/client/errors' @@ -23,19 +17,14 @@ import type { ConnectorData } from '@/lib/api/contracts/knowledge/connectors' import type { ConnectorActionsOptions } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/use-connector-actions' const mocks = vi.hoisted(() => ({ - live: false, admin: true, integrations: vi.fn(), - documents: vi.fn(), actions: vi.fn(), - recovery: vi.fn(), - history: vi.fn(), form: vi.fn(), dirty: false, saving: false, save: vi.fn(), })) -vi.mock('@/lib/core/config/deployment-shape', () => deploymentShapeMock) vi.mock('next/navigation', () => nextNavigationMock) vi.mock('@/app/o/[organizationId]/providers/organization-provider', () => organizationProviderMock) vi.mock('@/hooks/use-oauth-return', () => ({ useOAuthReturnForKBConnectors: vi.fn() })) @@ -52,25 +41,6 @@ vi.mock('@/connectors/registry', () => ({ }, }, })) -vi.mock( - '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-documents/connector-documents', - () => ({ - ConnectorDocuments: (props: unknown) => { - mocks.documents(props) - return

Source documents

- }, - }) -) -vi.mock('@/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section', () => ({ - ConnectorRecovery: (props: { onEdit?: () => void }) => { - mocks.recovery(props) - return props.onEdit ? : null - }, - ConnectorSyncHistory: () => { - mocks.history() - return

Source sync history

- }, -})) vi.mock( '@/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/use-connector-actions', () => ({ @@ -101,9 +71,6 @@ nextNavigationMockFns.mockUsePathname.mockReturnValue( ) const mockIndex = kbConnectorsQueriesMockFns.mockUseSearchIndex const mockDetail = kbConnectorsQueriesMockFns.mockUseConnectorDetail -deploymentShapeMockFns.mockUseDeploymentShape.mockImplementation(() => - createMockDeploymentShape({ features: { liveEnterpriseSearch: mocks.live } }) -) organizationProviderMockFns.mockUseOrganizationContext.mockImplementation(() => ({ organization: { id: 'org-one' }, viewer: { isAdmin: mocks.admin }, @@ -143,7 +110,6 @@ describe('organization source detail navigation', () => { beforeEach(() => { vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) mocks.admin = true - mocks.live = false mocks.dirty = false mocks.saving = false mockIndex.mockReturnValue({ data: { knowledgeBaseId: 'index-one' }, isPending: false }) @@ -180,16 +146,14 @@ describe('organization source detail navigation', () => { await act(async () => root.unmount()) container.remove() }) - async function render(searchParams = '') { + async function render() { await act(async () => root.render( - - - - - - - + + + + + ) ) } @@ -214,7 +178,7 @@ describe('organization source detail navigation', () => { }) it('hides cached source data after access is revoked, even if the index also failed', async () => { - await render('?view=settings') + await render() mockIndex.mockReturnValue({ data: { knowledgeBaseId: 'index-one' }, isError: true, @@ -227,36 +191,36 @@ describe('organization source detail navigation', () => { error: new ApiClientError({ status: 403, message: 'Access denied', body: null }), refetch: vi.fn(), }) - await render('?view=settings') + await render() expect(container.textContent).toContain('Access denied') expect(container.textContent).not.toContain('Source configuration') }) it('hides cached source settings when integration status reports revoked access', async () => { - await render('?view=settings') + await render() mocks.integrations.mockReturnValue({ data: [{ connectorType: 'google_drive', approved: true }], isError: true, error: new ApiClientError({ status: 403, message: 'Access denied', body: null }), refetch: vi.fn(), }) - await render('?view=settings') + await render() expect(container.textContent).toContain('Access denied') expect(container.textContent).not.toContain('Source configuration') }) it('preserves the editable baseline across background connector updates', async () => { - await render('?view=settings') + await render() mockDetail.mockReturnValue({ data: { ...connector, status: 'syncing', sourceConfig: { folderId: 'changed-remotely' } }, }) - await render('?view=settings') + await render() expect(mocks.form).toHaveBeenLastCalledWith(expect.objectContaining({ connector })) }) it('uses the canonical saved row as the new settings baseline without leaving the source', async () => { mocks.dirty = true - await render('?view=settings') + await render() await click('Save') expect(mocks.save).toHaveBeenCalledOnce() @@ -267,16 +231,16 @@ describe('organization source detail navigation', () => { expect(container.textContent).toContain('Source configuration') mockDetail.mockReturnValue({ data: { ...connector, status: 'syncing' } }) - await render('?view=settings') + await render() expect(mocks.form).toHaveBeenLastCalledWith(expect.objectContaining({ connector: saved })) }) it('discards to the latest server settings only when explicitly requested', async () => { mocks.dirty = true - await render('?view=settings') + await render() const refreshed = { ...connector, sourceConfig: { folderId: 'latest-server-folder' } } mockDetail.mockReturnValue({ data: refreshed }) - await render('?view=settings') + await render() expect(mocks.form).toHaveBeenLastCalledWith(expect.objectContaining({ connector })) await click('Discard') diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.tsx b/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.tsx index ba0b0812e79..f84cf589856 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.tsx +++ b/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.tsx @@ -1,34 +1,20 @@ 'use client' import { type ReactNode, useState } from 'react' -import { ChipLink, ChipModalTabs } from '@sim/emcn' +import { ChipLink } from '@sim/emcn' import { ArrowLeft } from '@sim/emcn/icons' import { useRouter } from 'next/navigation' -import { useQueryState } from 'nuqs' import { saveDiscardActions } from '@/components/settings/save-discard-actions' import type { SettingsAction, SettingsBackAction } from '@/components/settings/settings-header' import { SettingsPanel } from '@/components/settings/settings-panel' import { useSettingsUnsavedGuard } from '@/components/settings/use-settings-unsaved-guard' import { isApiClientError } from '@/lib/api/client/errors' import type { ConnectorData, ConnectorDetailData } from '@/lib/api/contracts/knowledge/connectors' -import { useDeploymentShape } from '@/lib/core/config/deployment-shape' import type { ResourceScope } from '@/lib/core/resource-scope' -import { SOURCE_PERMISSION_ERROR } from '@/lib/knowledge/connectors/sync-limits' import { organizationRoutes } from '@/lib/navigation/paths' import { describeSearchSource } from '@/lib/sim-search/source-identity' -import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import { useOrganizationContext } from '@/app/o/[organizationId]/providers/organization-provider' -import { - type SourceView, - sourceDocumentFilterParam, - sourceViewParam, -} from '@/app/o/[organizationId]/settings/integrations/sources/[connectorId]/search-params' import { UnsavedChangesModal } from '@/app/workspace/[workspaceId]/components/credential-detail/components/unsaved-changes-modal' -import { ConnectorDocuments } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-documents/connector-documents' -import { - ConnectorRecovery, - ConnectorSyncHistory, -} from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section' import { ConnectorActionFeedback } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connector-actions' import { getConnectorSyncState } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connector-sync-state' import { useConnectorActions } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/use-connector-actions' @@ -39,7 +25,6 @@ import { SettingsQueryErrorState, } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' import { SettingsResourceRow } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' -import { useSettingsSearch } from '@/app/workspace/[workspaceId]/settings/components/use-settings-search' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' import { isConnectorSyncingOrPending, @@ -47,15 +32,8 @@ import { useSearchIndex, } from '@/hooks/queries/kb/connectors' import { useSearchIntegrations } from '@/hooks/queries/search-integrations' -import { useDebounce } from '@/hooks/use-debounce' import { useOAuthReturnForKBConnectors } from '@/hooks/use-oauth-return' -const SOURCE_VIEWS = [ - { value: 'documents', label: 'Documents' }, - { value: 'settings', label: 'Settings' }, - { value: 'history', label: 'Sync history' }, -] as const - interface OrganizationSourceDetailProps { connectorId: string } @@ -63,7 +41,6 @@ interface OrganizationSourceDetailProps { export function OrganizationSourceDetail({ connectorId }: OrganizationSourceDetailProps) { const { organization, viewer } = useOrganizationContext() const router = useRouter() - const liveSearch = useDeploymentShape().features.liveEnterpriseSearch const scope: ResourceScope = { kind: 'organization', organizationId: organization.id } const backHref = organizationRoutes(organization.id).settingsSection('integrations') const index = useSearchIndex(scope, { enabled: viewer.isAdmin }) @@ -128,7 +105,6 @@ export function OrganizationSourceDetail({ connectorId }: OrganizationSourceDeta ) if ( - liveSearch && detail.data.accessMode === 'members' && !(detail.data.connectorType === 'github' && detail.data.sourceConfig.githubRepositoryId) ) @@ -188,47 +164,22 @@ function SourceDetailContent({ const { organization } = useOrganizationContext() const integrations = useSearchIntegrations(organization.id) const router = useRouter() - const liveSearch = useDeploymentShape().features.liveEnterpriseSearch - const [view, setView] = useQueryState( - sourceViewParam.key, - sourceViewParam.parser.withOptions({ history: 'replace' }) - ) - const [filter, setFilter] = useQueryState( - sourceDocumentFilterParam.key, - sourceDocumentFilterParam.parser - ) - const [search, setSearch] = useSettingsSearch() - const documentSearch = useDebounce(search.trim(), SEARCH_DEBOUNCE_MS) const meta = CONNECTOR_META_REGISTRY[connector.connectorType] const title = meta ? describeSearchSource(meta, connector.sourceConfig) || meta.name : 'Connection' - const { effectiveStatus, lastSyncError } = getConnectorSyncState(connector) - const permissionsIncomplete = - connector.lastSyncError?.split('\n').includes(SOURCE_PERMISSION_ERROR) ?? false + const { effectiveStatus } = getConnectorSyncState(connector) const status = effectiveStatus === 'paused' - ? liveSearch - ? 'Search paused' - : 'Sync paused' + ? 'Search paused' : effectiveStatus === 'disabled' - ? liveSearch - ? 'Search disabled' - : 'Sync disabled' - : effectiveStatus === 'error' - ? liveSearch - ? undefined - : 'Sync failed' - : undefined + ? 'Search disabled' + : undefined const description = [title === meta?.name ? undefined : meta?.name, status].filter(Boolean).join(' · ') || undefined const onBack = () => router.push(backHref) const onRemoved = () => router.replace(organizationRoutes(organization.id).settingsSection('integrations')) - const onViewChange = (value: string) => { - const next = sourceViewParam.parser.parse(value) - if (next) void setView(next) - } if ( integrations.isError && isApiClientError(integrations.error) && @@ -271,95 +222,18 @@ function SourceDetailContent({ )} ) - if (liveSearch || view === 'settings') - return ( - - ) return ( - - {integrationFeedback} - - {lastSyncError && - (effectiveStatus === 'active' || - (permissionsIncomplete && - (effectiveStatus === 'pending' || effectiveStatus === 'syncing'))) && ( - - )} - onViewChange('settings')} - /> - {view === 'documents' ? ( - void setFilter(next)} - progressScope={scope} - isSearchIndex - syncing={isConnectorSyncingOrPending(connector)} - /> - ) : ( - - )} - - ) -} - -interface SourceNavigationProps { - view: SourceView - onViewChange: (view: string) => void -} - -function SourceNavigation({ view, onViewChange }: SourceNavigationProps) { - return ( -
- -
+ /> ) } @@ -383,7 +257,6 @@ function SourcePanel({ children, ...panel }: SourcePanelProps) { - const liveSearch = useDeploymentShape().features.liveEnterpriseSearch const lifecycle = useConnectorActions({ connector, knowledgeBaseId: connector.knowledgeBaseId, @@ -397,9 +270,9 @@ function SourcePanel({ {...panel} actions={[ ...lifecycle.actions - .filter((action) => !liveSearch || action.id !== 'sync') + .filter((action) => action.id !== 'sync') .map((action) => - liveSearch && action.id === 'pause' + action.id === 'pause' ? { ...action, text: connector.status === 'paused' ? 'Resume search' : 'Pause search', @@ -424,7 +297,6 @@ interface SourceSettingsEditorProps { backText: string onBack: () => void onRemoved: () => void - onViewChange: (view: string) => void } function SourceSettingsEditor(props: SourceSettingsEditorProps) { @@ -458,11 +330,9 @@ function SourceSettingsForm({ backText, onBack, onRemoved, - onViewChange, onSaved, onDiscard, }: SourceSettingsFormProps) { - const liveSearch = useDeploymentShape().features.liveEnterpriseSearch const form = useConnectorSettingsForm({ connector: baseline, syncing: isConnectorSyncingOrPending(connector), @@ -491,15 +361,7 @@ function SourceSettingsForm({ })} > {queryError} - {!liveSearch && ( - { - if (next !== 'settings') guard.guardBack(() => onViewChange(next)) - }} - /> - )} - {liveSearch && connector.connectorType === 'gitlab' && ( + {connector.connectorType === 'gitlab' && ( (null) - const { - data: index, - isPending: basesPending, - isError: basesFailed, - isFetching: basesFetching, - refetch: refetchIndex, - } = useSearchIndex(scope) - const [filters] = useQueryStates(searchFilterParsers, resourceUrlKeys) - const custom = filters.updated === 'custom' - const pageFilters = useMemo( - () => searchFiltersFromParams(filters, searchedAt), - [filters.source, filters.updated, filters.from, filters.to, searchedAt] - ) - const searchFilters = suppliedFilters ?? pageFilters - const scopeId = scope.kind === 'organization' ? scope.organizationId : scope.workspaceId - useEffect(() => { - onSearchChange?.({ scope, query, filters: searchFilters, ...(topK ? { topK } : {}) }) - }, [scope.kind, scopeId, query, searchFilters, topK, onSearchChange]) - const filtersKey = JSON.stringify(searchFilters) - const expanded = expandedFor === filtersKey - /** A custom window is two-ended: until both days are chosen, nothing is searched. */ - const awaitingRange = !suppliedFilters && custom && !(filters.from && filters.to) - const { - data: search, - isPending, - isFetching, - isPlaceholderData, - isError: searchFailed, - refetch: refetchSearch, - } = useWorkspaceKnowledgeSearch( - scope, - awaitingRange ? '' : query, - searchFilters, - topK ?? - (expanded - ? WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.expanded - : WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.initial), - { retainAcrossLimits: topK === undefined } - ) - /** A full first page may collapse to few cards, yet more documents may still match. */ - const mayHaveMore = - topK === undefined && - !expanded && - (search?.results.length ?? 0) >= WORKSPACE_KNOWLEDGE_SEARCH_LIMITS.initial - const { data: overview } = useSearchSourceOverview(scope) - const indexing = (overview?.providers ?? []) - .filter((provider) => provider.isSyncing) - .map((provider) => connectorDisplayName(provider.connectorType)) - const documents = groupResultsByDocument(search?.results ?? []) - const sourceTypes = [ - ...new Set([ - ...(filters.source ? [filters.source] : []), - ...(overview?.providers.map((provider) => provider.connectorType) ?? []), - UPLOAD_SOURCE, - ]), - ].sort((left, right) => connectorDisplayName(left).localeCompare(connectorDisplayName(right))) - const failed = basesFailed || searchFailed - const pending = basesPending || isPending - const fetching = basesFetching || isFetching - const noSources = !basesPending && !basesFailed && !index?.knowledgeBaseId - const partial = search?.retrieval.status === 'partial' - const documentCount = documents.length === 1 ? '1 document' : `${documents.length} documents` - - const indexingNote = - indexing.length > 0 - ? `Still indexing ${indexing.join(', ')}; results grow as documents land.` - : null - - const showResults = !noSources && !failed && !basesPending && documents.length > 0 - /** A custom window waiting for its days must show the filters, or the picker is unreachable. */ - const showFilters = - hasShownFilters || - showResults || - awaitingRange || - (!noSources && !pending && !failed && !!search && !partial) - if (showFilters && !hasShownFilters) setHasShownFilters(true) - - return noSources ? ( -
-

No sources are set up yet.

- - View sources - -
- ) : ( -
-
-
- {awaitingRange ? ( -

- Choose the days to search. -

- ) : fetching ? ( - - ) : pending && !failed ? null : ( -

- {failed - ? 'Search couldn’t run.' - : partial - ? documents.length === 0 - ? 'Search timed out.' - : `${documentCount} · some results may be missing.` - : documents.length === 0 - ? 'Search found no results.' - : `${documentCount} · searched as you`} -

- )} - {indexingNote && !failed && !partial && ( -

{indexingNote}

- )} -
- {(failed || partial) && ( - void (basesFailed ? refetchIndex() : refetchSearch())} - > - {fetching ? 'Retrying' : 'Try again'} - - )} -
- {suppliedFilters === undefined && showFilters && } - {showResults && ( -
- {documents.map((result) => { - const source = toSource(result, query, scope) - return ( - - onSummarize(`Summarize "${cited.title ?? cited.url}"`, { - ...searchFilters, - documentIds: [result.documentId], - }) - } - /> - ) - })} - {mayHaveMore && ( -
- setExpandedFor(filtersKey)} - > - Show more - -
- )} -
- )} -
- ) -} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx index 0505039e054..060e04932ff 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -6,9 +6,7 @@ import { useQueryStates } from 'nuqs' import { ActivityStatus } from '@/components/ui/activity-status' import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge' import { useSession } from '@/lib/auth/auth-client' -import { useDeploymentShape } from '@/lib/core/config/deployment-shape' import { type ResourceScope, resourceScopeKey } from '@/lib/core/resource-scope' -import { IndexedSearchResults } from '@/app/workspace/[workspaceId]/home/components/knowledge-search-results/indexed' import { SearchFilters } from '@/app/workspace/[workspaceId]/home/components/knowledge-search-results/search-filters' import { groupResultsByDocument, @@ -48,8 +46,7 @@ export function KnowledgeSearchResults({ const scope: ResourceScope = suppliedScope ?? { kind: 'workspace', workspaceId: workspaceId! } const { data: session } = useSession() const trimmed = query.trim() - const { features } = useDeploymentShape() - const Results = features.liveEnterpriseSearch ? LiveSearchResults : IndexedSearchResults + const Results = LiveSearchResults return ( ({ data: { knowledgeBaseId: 'index' }, isPending: false, })) -kbConnectorsQueriesMockFns.mockUseSearchSourceOverview.mockImplementation(() => ({ - data: { - providers: [ - { connectorType: 'slack', isSyncing: false }, - { connectorType: 'gmail', isSyncing: false }, - ], - }, -})) const mockRequestJson = apiClientRequestMockFns.mockRequestJson @@ -235,48 +226,12 @@ async function complete( }) } -describe('search refinement with the real query cache and URL state', () => { - /** The source refinement chips belong to the indexed search results. */ - beforeEach(() => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) - resetDeploymentShape() - }) - afterEach(() => { - resetEnvFlagsMock() - resetDeploymentShape() - }) - - it('does not restore cleared access data as a placeholder', async () => { - await render() - await complete(0) - await act(async () => { - void client.resetQueries({ queryKey: knowledgeKeys.searches() }) - await vi.advanceTimersByTimeAsync(1) - }) - expect(container.textContent).not.toContain('Release plan') - await click('Gmail') - expect(container.textContent).not.toContain('Release plan') - }) - - it('clears displayed placeholder data when access is reset during a refinement', async () => { - await render() - await complete(0) - await click('Gmail') - expect(container.textContent).toContain('Release plan') - await act(async () => { - void client.resetQueries({ queryKey: knowledgeKeys.searches() }) - await vi.advanceTimersByTimeAsync(1) - }) - expect(container.textContent).not.toContain('Release plan') - }) -}) - describe('live search submission feedback', () => { it.each(['launch', 'edited draft', ''])( 'cancels with draft %j, ignores late results, and allows a fresh submission', async (draft) => { const shape = resolveDeploymentShape() - seedDeploymentShape({ ...shape, features: { ...shape.features, liveEnterpriseSearch: true } }) + seedDeploymentShape({ ...shape, features: shape.features }) await render({ organizationPage: true, params: '?q=launch' }) await act(async () => { await vi.advanceTimersByTimeAsync(1) @@ -328,7 +283,7 @@ describe('live search submission feedback', () => { it('acknowledges the submitted query before exposing refinement controls', async () => { const shape = resolveDeploymentShape() - seedDeploymentShape({ ...shape, features: { ...shape.features, liveEnterpriseSearch: true } }) + seedDeploymentShape({ ...shape, features: shape.features }) await render({ organizationPage: true, params: '?q=launch' }) await act(async () => { await vi.advanceTimersByTimeAsync(1) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/search-integration-connection.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/search-integration-connection.tsx index 8e7e408da0f..29658e390a9 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/search-integration-connection.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/search-integration-connection.tsx @@ -1,6 +1,5 @@ 'use client' -import { useState } from 'react' import { Chip } from '@sim/emcn' import { Check } from '@sim/emcn/icons' import type { SearchConnectionTarget } from '@/lib/knowledge/search/connection-target' @@ -9,7 +8,6 @@ import { InteractionCard, InteractionCardActionRow, } from '@/app/workspace/[workspaceId]/home/components/message-content/components/interaction-card' -import { SourceSetupModal } from '@/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal' import { BrandIcon } from '@/blocks/brand-icon' import { useSearchIntegrationConnection } from '@/hooks/use-search-integration-connection' @@ -42,7 +40,6 @@ function SearchIntegrationConnectionControl({ divided, onConnected, }: SearchIntegrationConnectionProps) { - const [setupOpen, setSetupOpen] = useState(false) const connector = SEARCH_CONNECTORS.find((entry) => entry.type === target.connectorType) const connection = useSearchIntegrationConnection({ organizationId, @@ -63,17 +60,7 @@ function SearchIntegrationConnectionControl({ : !connection.available ? `${name} connection is no longer available` : `${action} ${name}` - const handleConnect = () => { - if ( - target.connectionMode !== 'live' && - connector && - !connection.connectorId && - connector.setupFields.length && - !connection.pending - ) - setSetupOpen(true) - else void connection.connect() - } + const handleConnect = () => void connection.connect() const content = ( <> )} - {setupOpen && connector && ( - setSetupOpen(false)} - isPending={connection.isStarting} - error={connection.error} - onConnect={(config) => { - void connection.connect(config).then((started) => { - if (started) setSetupOpen(false) - }) - }} - /> - )} ) return embedded ? content : {content} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/atlassian-source-setup-modal.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/atlassian-source-setup-modal.tsx deleted file mode 100644 index 92515383b1b..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/atlassian-source-setup-modal.tsx +++ /dev/null @@ -1,254 +0,0 @@ -'use client' - -import { useState } from 'react' -import { - Button, - Chip, - ChipCombobox, - ChipInput, - ChipModal, - ChipModalBody, - ChipModalField, - ChipModalFooter, - ChipModalHeader, - Tooltip, - toast, -} from '@sim/emcn' -import { ArrowLeftRight, Plus } from '@sim/emcn/icons' -import { getErrorMessage } from '@sim/utils/errors' -import type { PersonalSourceSetupQuery } from '@/lib/api/contracts/knowledge/personal-source-setup' -import type { SearchConnector } from '@/lib/sim-search/connectors' -import { MAX_PERSONAL_SOURCE_SETUP_KEYS } from '@/lib/sim-search/personal-source-setup' -import { ConnectorSelectorField } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field' -import { useConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' -import { useConnectPersonalSourceSetup } from '@/hooks/queries/personal-source-setup' -import { usePersonalSourceAccount } from '@/hooks/use-personal-source-account' - -interface AtlassianSourceSetupModalProps { - organizationId: string - connector: SearchConnector - connectorType: PersonalSourceSetupQuery['connectorType'] - onClose: () => void - onConnected?: (connection: { connectorId: string; credentialId: string }) => void -} - -export function AtlassianSourceSetupModal({ - organizationId, - connector, - connectorType, - onClose, - onConnected, -}: AtlassianSourceSetupModalProps) { - const account = usePersonalSourceAccount({ - organizationId, - connectorType, - onConnected: (id) => { - setSelectedAccount(id) - config.setSourceConfig((previous) => ({ domain: previous.domain ?? '' })) - }, - }) - const { mutateAsync: connect, isPending } = useConnectPersonalSourceSetup() - const config = useConnectorConfigFields({ - connectorConfig: connector.meta, - accessMode: 'members', - }) - const [selectedAccount, setSelectedAccount] = useState() - const accounts = account.accounts.data?.accounts ?? [] - const requestedAccount = - selectedAccount ?? - account.accounts.data?.completedCredentialId ?? - (accounts.length === 1 ? accounts[0].id : undefined) - const credentialId = accounts.find((item) => item.id === requestedAccount)?.id ?? null - const canonicalId = connectorType === 'jira' ? 'projectKey' : 'spaceKey' - const picker = connector.meta.configFields.find( - (field) => field.canonicalParamId === canonicalId && field.type === 'selector' - )! - const manual = connector.meta.configFields.find((field) => field.id === canonicalId)! - const advanced = config.canonicalModes[canonicalId] === 'advanced' - const domain = typeof config.sourceConfig.domain === 'string' ? config.sourceConfig.domain : '' - const resolved = config.resolveSourceConfig()[canonicalId] - const manualValue = config.sourceConfig[canonicalId] - const keys = Array.isArray(resolved) - ? resolved - .filter((key): key is string => typeof key === 'string' && Boolean(key.trim())) - .map((key) => key.trim()) - : [] - const pending = isPending || account.pending - const close = () => { - if (!isPending) onClose() - } - const chooseAccount = (id: string) => { - if (id === credentialId) return - setSelectedAccount(id) - config.setSourceConfig({ domain }) - } - const addAccount = () => { - void account.connect() - } - const submit = async () => { - if (!credentialId || !domain.trim() || !keys.length || pending) return - if (keys.length > MAX_PERSONAL_SOURCE_SETUP_KEYS) { - toast.error('Choose no more than 1,000 projects or spaces per source.') - return - } - try { - const result = await connect({ - action: 'connect', - organizationId, - connectorType, - credentialId, - domain: domain.trim(), - keys, - }) - onConnected?.({ connectorId: result.connectorId, credentialId }) - onClose() - } catch (error) { - toast.error(getErrorMessage(error, 'Could not connect the source')) - } - } - - return ( - { - if (!open) close() - }} - srTitle={`Connect ${connector.meta.name}`} - > - Connect {connector.meta.name} - - - {(aria) => ( - <> - ({ value: item.id, label: item.name })), - { - value: '__connect_new__', - label: `Connect ${connector.meta.name} account`, - icon: Plus, - onSelect: addAccount, - }, - ]} - placeholder={ - account.pending - ? 'Waiting for authorization' - : account.accounts.isPending - ? 'Loading accounts' - : 'Select your account' - } - disabled={pending || account.accounts.isPending} - /> - {account.pending && Cancel authorization} - {account.accounts.isError && ( - void account.accounts.refetch()}>Retry loading accounts - )} - - )} - - {credentialId && !account.pending && ( - <> - config.handleFieldChange('domain', value)} - placeholder='yoursite.atlassian.net' - autoComplete='off' - required - disabled={isPending} - /> - - - - - - {advanced ? 'Switch to selector' : 'Switch to manual input'} - - - } - > - {(aria) => ( - <> - {advanced ? ( - - config.handleFieldChange(canonicalId, event.target.value) - } - placeholder={manual.placeholder} - disabled={isPending} - /> - ) : picker.selectorKey ? ( - - config.handleFieldChange(picker.id, value, labels) - } - credentialId={credentialId} - sourceConfig={config.sourceConfig} - configFields={connector.meta.configFields} - canonicalModes={config.canonicalModes} - selectedLabels={config.selectionLabels[canonicalId]} - disabled={isPending} - /> - ) : null} - - )} - - - )} - - - window.open(connector.meta.searchDocsUrl, '_blank', 'noopener,noreferrer'), - }, - ] - : undefined - } - primaryAction={{ - label: isPending ? 'Connecting' : 'Connect & Sync', - onClick: () => void submit(), - disabled: !credentialId || !domain.trim() || !keys.length || pending, - }} - /> - - ) -} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx deleted file mode 100644 index 5570dd8917f..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx +++ /dev/null @@ -1,131 +0,0 @@ -'use client' - -import { useState } from 'react' -import { - ChipModal, - ChipModalBody, - ChipModalField, - ChipModalFooter, - ChipModalHeader, -} from '@sim/emcn' -import type { SearchConnector } from '@/lib/sim-search/connectors' -import { AtlassianSourceSetupModal } from '@/app/workspace/[workspaceId]/home/components/search-sources/atlassian-source-setup-modal' - -interface SourceSetupModalProps { - organizationId?: string - onConnected?: (connection: { connectorId: string; credentialId: string }) => void - connector: SearchConnector - onClose: () => void - isPending?: boolean - error?: string | null - /** Connects the source with the filled-in fields; the caller opens the OAuth tab in this click. */ - onConnect: (sourceConfig: Record) => void -} - -/** - * The few fields a source needs before its first connect, such as a site and - * a space. Everyone after the first person clicks straight through. - */ -export function SourceSetupModal(props: SourceSetupModalProps) { - if ( - props.organizationId && - (props.connector.type === 'jira' || props.connector.type === 'confluence') - ) { - return ( - - ) - } - return -} - -function ManualSourceSetupModal({ - connector, - onClose, - onConnect, - isPending = false, - error, -}: SourceSetupModalProps) { - const docsUrl = connector.meta.searchDocsUrl - const fields = connector.setupFields - const [values, setValues] = useState>({}) - const complete = fields.every((field) => values[field.id]?.trim()) - - const submit = () => { - if (!complete || isPending) return - onConnect(Object.fromEntries(fields.map((field) => [field.id, values[field.id]?.trim() ?? '']))) - } - - return ( - { - if (!open) onClose() - }} - srTitle={`Connect ${connector.meta.name}`} - > - Connect {connector.meta.name} - - {fields.map((field) => - field.type === 'dropdown' ? ( - setValues((current) => ({ ...current, [field.id]: value }))} - options={(field.options ?? []).map((option) => ({ - value: option.id, - label: option.label, - }))} - placeholder={field.placeholder} - hint={field.description} - required - /> - ) : ( - setValues((current) => ({ ...current, [field.id]: value }))} - placeholder={field.placeholder} - hint={field.description} - autoComplete='off' - required - /> - ) - )} - {error && ( -

- {error} -

- )} - - window.open(docsUrl, '_blank', 'noopener,noreferrer'), - }, - ] - : undefined - } - primaryAction={{ - label: isPending ? 'Connecting' : 'Connect', - onClick: submit, - disabled: !complete || isPending, - }} - /> - - ) -} diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx index acd0efa6d15..1781894479c 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.dom.test.tsx @@ -511,7 +511,7 @@ describe('useChat remount send recovery', () => { const shape = resolveDeploymentShape() seedDeploymentShape({ ...shape, - features: { ...shape.features, liveEnterpriseSearch: true }, + features: shape.features, }) const history: MothershipChatHistory = { id: 'chat-cited-search', @@ -632,7 +632,7 @@ describe('useChat remount send recovery', () => { it('restores an explicitly selected Search tab while reconnecting an active turn', async () => { const shape = resolveDeploymentShape() - seedDeploymentShape({ ...shape, features: { ...shape.features, liveEnterpriseSearch: true } }) + seedDeploymentShape({ ...shape, features: shape.features }) const history: MothershipChatHistory = { id: 'chat-reconnecting-search', title: 'Search', diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts index 805b8fb6604..c1b1c76c40d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts @@ -30,7 +30,6 @@ import type { MothershipTableViewContext } from '@/lib/api/contracts/mothership- import { useSession } from '@/lib/auth/auth-client' import { buildResourceAttachments } from '@/lib/browser-agent/attachments' import { cancelActiveBrowserTools, initBrowserAgentTransport } from '@/lib/browser-agent/transport' -import { getDeploymentShape } from '@/lib/core/config/deployment-shape' import { MothershipHandoffStorage } from '@/lib/core/utils/browser-storage' import { withinDeadline } from '@/lib/core/utils/deadline' import { readSSELines } from '@/lib/core/utils/sse' @@ -1922,7 +1921,6 @@ export function useChat( ) /** Recovery discards interim search tabs without taking an already visible panel away. */ if ( - getDeploymentShape().features.liveEnterpriseSearch && requestModeRef.current === 'assistant' && !sendingRef.current && (!activeStreamId || isTerminalStreamStatus(chatHistory.streamSnapshot?.status)) @@ -1951,7 +1949,6 @@ export function useChat( ? (reorderStoredChatResources(updatedResources, pendingOrder) ?? updatedResources) : updatedResources const keepSearchPanelStable = - getDeploymentShape().features.liveEnterpriseSearch && requestModeRef.current === 'assistant' && (sendingRef.current || (activeStreamId && !isTerminalStreamStatus(chatHistory.streamSnapshot?.status))) @@ -2168,9 +2165,7 @@ export function useChat( } const clearStreamResourceActivity = () => clearResourceActivity(activityTracker, true) const ctx = createStreamLoopContext({ - citedSourcesEnabled: - getDeploymentShape().features.liveEnterpriseSearch && - requestModeRef.current === 'assistant', + citedSourcesEnabled: requestModeRef.current === 'assistant', refreshRoute: () => router.refresh(), viewerId, workspaceId, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx index 003454c9e36..d119f9d2a3c 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx @@ -18,7 +18,6 @@ import { } from '@sim/emcn' import { ArrowLeft, ChevronDown, ChevronRight, Plus, Search } from '@sim/emcn/icons' import type { ConnectorData } from '@/lib/api/contracts/knowledge/connectors' -import { useDeploymentShape } from '@/lib/core/config/deployment-shape' import { type ResourceScope, resourceScopeFields } from '@/lib/core/resource-scope' import { asServiceAccountProviderId } from '@/lib/credentials/service-account-provider-ids' import { getIntegrationsForCredentialProvider } from '@/lib/integrations/credential-display' @@ -180,7 +179,7 @@ export function AddConnectorModal({ ) const { mutate: createConnector, isPending: isCreating } = useCreateConnector() - const liveSearch = useDeploymentShape().features.liveEnterpriseSearch && isSearchIndex + const liveSearch = isSearchIndex const canSetUpGitHubInstallation = canAdmin && isSearchIndex && selectedType === 'github' && scope.kind === 'organization' const connectorConfig = liveSearchSourceMeta( diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-documents/connector-documents.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-documents/connector-documents.tsx index 7323a54c0ac..0bebc964de0 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-documents/connector-documents.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-documents/connector-documents.tsx @@ -3,7 +3,6 @@ import { Chip, ChipInput, ChipLink, Skeleton } from '@sim/emcn' import { RefreshCw, Search, SquareArrowUpRight } from '@sim/emcn/icons' import type { ConnectorDocumentFilter } from '@/lib/api/contracts/knowledge/connectors' -import type { ResourceScope } from '@/lib/core/resource-scope' import { getDocumentIndexingStatus } from '@/lib/knowledge/documents/types' import { ConnectorDocumentStatusFilter } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-documents/connector-document-status-filter' import { @@ -27,9 +26,6 @@ interface ConnectorDocumentsProps { search?: string searchControl?: { value: string; onChange: (value: string) => void } showToolbar?: boolean - progressScope?: ResourceScope - isSearchIndex?: boolean - syncing?: boolean filter: ConnectorDocumentFilter onFilterChange: (filter: ConnectorDocumentFilter) => void } @@ -41,16 +37,11 @@ export function ConnectorDocuments({ search, searchControl, showToolbar = true, - progressScope, - isSearchIndex = false, - syncing, onFilterChange, }: ConnectorDocumentsProps) { const query = useConnectorDocuments(knowledgeBaseId, connectorId, { filter, search, - progressScope, - syncing, }) const { data, hasNextPage, isFetchingNextPage, fetchNextPage } = query const isLoading = query.isLoading || query.isPlaceholderData @@ -79,9 +70,6 @@ export function ConnectorDocuments({ return ( <>
- {isSearchIndex && ( -

Documents you can access

- )} {showToolbar && (
{searchControl && ( diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.test.tsx index ed1f9c3e86a..5463e820c43 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.test.tsx @@ -93,54 +93,3 @@ it.each([ } } ) - -it('keeps the prior selection when all personal setup options exceed its source limit', async () => { - mocks.loadAll.mockResolvedValue({ - status: 'complete', - options: Array.from({ length: 1001 }, (_, index) => ({ - id: `P${index}`, - label: `Project ${index}`, - })), - }) - const field: ConnectorConfigField & { selectorKey: 'jira.projectKeys' } = { - id: 'projects', - title: 'Projects', - type: 'selector', - selectorKey: 'jira.projectKeys', - multi: true, - allowSelectAll: true, - } - const container = document.createElement('div') - const root = createRoot(container) - try { - await act(async () => - root.render( - - ) - ) - const all = mocks.combobox.mock.lastCall![0].options.find((item) => item.label === 'All') - expect(container.textContent).not.toContain('Select all') - await act(async () => all?.onSelect?.()) - expect(mocks.loadAll).toHaveBeenCalledTimes(1) - expect(mocks.change).not.toHaveBeenCalled() - expect(container.querySelector('[role="alert"]')?.textContent).toContain( - 'Select up to 1,000 items' - ) - } finally { - await act(async () => root.unmount()) - } -}) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx index eae49ed34e5..1fd83e71c37 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-selector-field/connector-selector-field.tsx @@ -7,8 +7,7 @@ import { useParams } from 'next/navigation' import { type ResourceScope, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { projectSelectorContext } from '@/lib/selectors/context' import { getSelectorManifestEntry, type SelectorKey } from '@/lib/selectors/manifest' -import type { SelectorContext, SelectorSurface } from '@/lib/selectors/types' -import { MAX_PERSONAL_SOURCE_SETUP_KEYS } from '@/lib/sim-search/personal-source-setup' +import type { SelectorContext } from '@/lib/selectors/types' import type { SourceSelectionLabel } from '@/lib/sim-search/source-identity' import { SEARCH_DEBOUNCE_MS } from '@/lib/url-state' import { getDependsOnFields } from '@/lib/workflows/subblocks/dependencies' @@ -27,7 +26,6 @@ import { useDebounce } from '@/hooks/use-debounce' interface ConnectorSelectorFieldProps { controlAria?: ChipModalFieldAria scope?: ResourceScope - selectorSurface?: SelectorSurface field: ConnectorConfigField & { selectorKey: SelectorKey } value: ConfigFieldValue onChange: (value: ConfigFieldValue, selectedOptions?: SourceSelectionLabel[]) => void @@ -43,7 +41,6 @@ interface ConnectorSelectorFieldProps { export function ConnectorSelectorField({ controlAria, scope: explicitScope, - selectorSurface, field, value, onChange, @@ -112,9 +109,7 @@ export function ConnectorSelectorField({ }, [field.dependsOn, sourceConfig, configFields, canonicalModes]) const isEnabled = !disabled && !!credentialId && depsResolved - const missingDependencyMessage = selectorSurface - ? 'Enter your Atlassian site first' - : `Select ${getDependencyLabel(field, configFields)} first` + const missingDependencyMessage = `Select ${getDependencyLabel(field, configFields)} first` const debouncedSearch = useDebounce(searchTerm.trim(), SEARCH_DEBOUNCE_MS) const { data: options = [], @@ -131,7 +126,6 @@ export function ConnectorSelectorField({ } = useSelectorOptions(field.selectorKey, { context, scope, - surface: selectorSurface, search: debouncedSearch, enabled: isEnabled, surfaceId: `connector:${field.id}`, @@ -155,7 +149,6 @@ export function ConnectorSelectorField({ { context, scope, - surface: selectorSurface, detailIds: isEnabled ? missingSelectedIds : [], surfaceId: `connector:${field.id}`, } @@ -171,7 +164,6 @@ export function ConnectorSelectorField({ const { data: searchedOption } = useSelectorOptionDetail(field.selectorKey, { context, scope, - surface: selectorSurface, detailId: resolvesUnknownIds && isEnabled && debouncedSearch.length > 0 ? debouncedSearch : undefined, surfaceId: `connector:${field.id}`, @@ -254,16 +246,6 @@ export function ConnectorSelectorField({ }) return } - if ( - selectorSurface?.kind === 'personal-search-setup' && - result.options.length > MAX_PERSONAL_SOURCE_SETUP_KEYS - ) { - setBulkError({ - context, - message: `Select up to ${MAX_PERSONAL_SOURCE_SETUP_KEYS.toLocaleString()} items. Choose a smaller set to continue.`, - }) - return - } onChange( result.options.map((option) => option.id), result.options.map((option) => ({ id: option.id, label: option.label })) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx index 3e07a27b34c..0142cfa7513 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx @@ -11,7 +11,6 @@ import { } from '@sim/emcn' import { ChevronDown, ChevronRight, Plus } from '@sim/emcn/icons' import type { ConnectorAccessMode } from '@/lib/api/contracts/knowledge/connectors' -import { useDeploymentShape } from '@/lib/core/config/deployment-shape' import { type ResourceScope, resourceScopeFields } from '@/lib/core/resource-scope' import { asServiceAccountProviderId } from '@/lib/credentials/service-account-provider-ids' import { @@ -159,7 +158,7 @@ export function ConnectorSettingsFields({ onContentCredentialChange, onWorkspaceCredentialChange, }: ConnectorSettingsFieldsProps) { - const liveSearch = useDeploymentShape().features.liveEnterpriseSearch && isSearchIndex + const liveSearch = isSearchIndex const connectorConfig = liveSearchSourceMeta(originalConfig, Boolean(liveSearch), { githubInstallation: usesGitHubInstallation && scope.kind === 'organization', }) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.test.tsx index 4be5a4f9a53..0113ffce7b4 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.test.tsx @@ -2,11 +2,7 @@ * @vitest-environment jsdom */ import { act } from 'react' -import { - createMockDeploymentShape, - deploymentShapeMock, - deploymentShapeMockFns, -} from '@sim/testing/mocks/deployment-shape.mock' +import { deploymentShapeMock } from '@sim/testing/mocks/deployment-shape.mock' import { kbConnectorsQueriesMock, kbConnectorsQueriesMockFns, @@ -16,7 +12,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { ConnectorData } from '@/lib/api/contracts/knowledge/connectors' const mocks = vi.hoisted(() => ({ - live: false, update: vi.fn(), applyAccess: vi.fn(), settingsPending: false, @@ -54,9 +49,6 @@ vi.mock('@/hooks/use-permission-config', () => ({ import { useConnectorSettingsForm } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form' -deploymentShapeMockFns.mockUseDeploymentShape.mockImplementation(() => - createMockDeploymentShape({ features: { liveEnterpriseSearch: mocks.live } }) -) kbConnectorsQueriesMockFns.mockUseUpdateConnector.mockImplementation(() => ({ mutate: mocks.update, isPending: mocks.settingsPending, @@ -120,7 +112,6 @@ describe('shared connector settings form', () => { } beforeEach(() => { - mocks.live = false mocks.settingsPending = false mocks.accessPending = false ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true @@ -209,7 +200,7 @@ describe('shared connector settings form', () => { }) it('preserves the GitHub repository and pending connection after an incompatible replacement is refused', () => { - const sourceConfig = { repository: 'acme/platform', branch: 'main' } + const sourceConfig = { repository: 'acme/platform' } render( connector({ connectorType: 'github', diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts index 32696d582b4..300366d0e7a 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts @@ -4,7 +4,6 @@ import { useCallback, useMemo, useState } from 'react' import { createLogger } from '@sim/logger' import { isEqual } from 'es-toolkit' import type { UpdateConnectorBody } from '@/lib/api/contracts/knowledge/connectors' -import { useDeploymentShape } from '@/lib/core/config/deployment-shape' import type { ResourceScope } from '@/lib/core/resource-scope' import { isContentEngineAccessMode } from '@/lib/knowledge/connectors/access-modes' import { getConnectorAccessAvailability } from '@/lib/sim-search/connectors' @@ -137,7 +136,7 @@ export function useConnectorSettingsForm({ onSaved, syncing = isConnectorSyncingOrPending(connector), }: UseConnectorSettingsFormOptions) { - const liveSearch = useDeploymentShape().features.liveEnterpriseSearch && isSearchIndex + const liveSearch = isSearchIndex const connectorConfig = liveSearchSourceMeta( CONNECTOR_META_REGISTRY[connector.connectorType] ?? null, Boolean(liveSearch), diff --git a/apps/sim/bootstrap.ts b/apps/sim/bootstrap.ts index 80574b20329..e5e08c65b52 100644 --- a/apps/sim/bootstrap.ts +++ b/apps/sim/bootstrap.ts @@ -9,8 +9,6 @@ await loadRuntimeSecrets() // Explicit runtime configuration wins over deployment defaults. process.env.MSHIP_PLAN_MODE ??= process.env.MSHIP_PLAN_MODE_DEFAULT ?? (process.env.NODE_ENV === 'development' ? 'true' : 'false') -process.env.SIM_SEARCH_LIVE ??= process.env.SIM_SEARCH_LIVE_DEFAULT ?? 'true' -process.env.NEXT_PUBLIC_SIM_SEARCH_LIVE = process.env.SIM_SEARCH_LIVE // `server.js` is the Next standalone build artifact, a sibling of this file in // the image; it does not exist at type-check time, so the specifier is held in a // variable to keep it out of static module resolution. diff --git a/apps/sim/connectors/coda/README.md b/apps/sim/connectors/coda/README.md index b0f2d04d92d..adf4015a5ee 100644 --- a/apps/sim/connectors/coda/README.md +++ b/apps/sim/connectors/coda/README.md @@ -1,6 +1,6 @@ # Coda indexed connector decisions and verification -This document covers ordinary knowledge-base ingestion and the explicitly selected legacy Search backend (`SIM_SEARCH_LIVE=false`). Its historical verification notes refer to that indexed path. Default live Search uses personal Coda MCP authorization plus optional service-source verification; see [live Search](../../lib/sim-search/live/README.md) and the [Coda Search guide](../../../docs/content/docs/search/coda.mdx). Background Coda content/ACL/directory builds are not part of live Search. +This document covers ordinary knowledge-base ingestion. Historical organization-indexing verification notes describe the retired indexed Search path. Default live Search uses personal Coda MCP authorization plus optional service-source verification; see [live Search](../../lib/sim-search/live/README.md) and the [Coda Search guide](../../../docs/content/docs/search/coda.mdx). Background Coda content/ACL/directory builds are not part of live Search. ## Precedent and authentication diff --git a/apps/sim/ee/organization-search-stats/search-params.ts b/apps/sim/ee/organization-search-stats/search-params.ts deleted file mode 100644 index 0514a128245..00000000000 --- a/apps/sim/ee/organization-search-stats/search-params.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { parseAsString, parseAsStringLiteral } from 'nuqs/server' -import { SEARCH_STATS_PERIODS, SEARCH_STATS_SURFACES } from '@/lib/knowledge/search/stats' - -export const organizationSearchStatsParsers = { - period: parseAsStringLiteral(SEARCH_STATS_PERIODS).withDefault('30d'), - /** An absent surface includes every Search entry point. */ - surface: parseAsStringLiteral(SEARCH_STATS_SURFACES), - startDate: parseAsString, - endDate: parseAsString, -} - -export const organizationSearchStatsUrlOptions = { - history: 'replace', - shallow: true, - clearOnDefault: true, - urlKeys: { - period: 'stats-period', - surface: 'stats-surface', - startDate: 'stats-start', - endDate: 'stats-end', - }, -} as const diff --git a/apps/sim/executor/utils/credential-token.ts b/apps/sim/executor/utils/credential-token.ts index 074e5361ca1..78e7e66cc33 100644 --- a/apps/sim/executor/utils/credential-token.ts +++ b/apps/sim/executor/utils/credential-token.ts @@ -1,6 +1,5 @@ import { createLogger } from '@sim/logger' import { AuthType } from '@/lib/auth/hybrid' -import { isLiveEnterpriseSearchEnabled } from '@/lib/core/config/env-flags' import { createCopilotManagedOAuthPrincipal } from '@/lib/credentials/application/copilot-managed-oauth-delegation' import { bindExecutorManagedOAuthDelegation } from '@/lib/credentials/application/managed-oauth-delegation' import { authorizePersonalCredential } from '@/lib/credentials/application/personal-credentials' @@ -64,9 +63,7 @@ export async function resolveExecutorCredentialToken( throw new Error('Assistant credential use requires the authenticated person for this turn.') } const original = toolId ? getToolMetadata(toolId) : undefined - const tool = original - ? projectAssistantConnectedAccountTool(original, isLiveEnterpriseSearchEnabled) - : undefined + const tool = original ? projectAssistantConnectedAccountTool(original) : undefined if ( !tool?.oauth?.required || (!copilotExecutionContext.workspaceId && !copilotExecutionContext.organizationId) || diff --git a/apps/sim/hooks/queries/kb/connectors.test.ts b/apps/sim/hooks/queries/kb/connectors.test.ts index e6bfc7d9c97..054e6388420 100644 --- a/apps/sim/hooks/queries/kb/connectors.test.ts +++ b/apps/sim/hooks/queries/kb/connectors.test.ts @@ -287,7 +287,7 @@ describe('useSearchSources', () => { expect(sources.queryKey).not.toEqual(searchSourceKeys.list('scope-1')) await sources.queryFn({ signal }) expect(mockRequestJson).toHaveBeenLastCalledWith(listSearchSourcesContract, { - query: { organizationId: 'scope-1', search: '', mine: false }, + query: { organizationId: 'scope-1', search: '' }, signal, }) }) diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index 6347604ac6c..6674d38ff01 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -1,4 +1,3 @@ -import { useEffect } from 'react' import { keepPreviousData, type QueryClient, @@ -13,8 +12,6 @@ import { type ConnectorDetailData, type ConnectorDocumentsData, type ConnectorMemberSummary, - type ConnectSimSearchConnectorBody, - connectSimSearchConnectorContract, createKnowledgeConnectorContract, deleteKnowledgeConnectorContract, getKnowledgeConnectorContract, @@ -41,13 +38,9 @@ import { type ConnectorDocumentsQuery, type PrepareSearchSourceBody, prepareSearchSourceContract, - readOrganizationSearchOverviewContract, readSearchIndexContract, - readSearchSourceOverviewContract, - readSearchSourceProgressContract, type SearchConnectionOAuthQuery, type SearchSourcePage, - type SearchSourceProgress, } from '@/lib/api/contracts/knowledge/connectors' import { type ResourceScope, @@ -55,10 +48,7 @@ import { resourceScopeFromOwner, resourceScopeKey, } from '@/lib/core/resource-scope' -import { - MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, - MAX_SEARCH_SOURCE_PROGRESS_ITEMS, -} from '@/lib/knowledge/constants' +import { MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE } from '@/lib/knowledge/constants' import { organizationAccountsKeys } from '@/hooks/queries/organization-accounts' import { credentialGroupKeys } from '@/hooks/queries/utils/credential-group-queries' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' @@ -91,13 +81,6 @@ export const connectorKeys = { details: (knowledgeBaseId?: string) => [...connectorKeys.all(knowledgeBaseId), 'detail'] as const, detail: (knowledgeBaseId?: string, connectorId?: string) => [...connectorKeys.details(knowledgeBaseId), connectorId ?? ''] as const, - progress: (knowledgeBaseId?: string, connectorId?: string, scope?: ResourceScope) => - [ - ...connectorKeys.progresses(knowledgeBaseId, connectorId), - scope ? resourceScopeKey(scope) : '', - ] as const, - progresses: (knowledgeBaseId?: string, connectorId?: string) => - [...connectorKeys.detail(knowledgeBaseId, connectorId), 'progress'] as const, } async function fetchConnectors( @@ -437,80 +420,30 @@ export function useSearchIndex(scope: ResourceScope, options?: { enabled?: boole }) } -/** Full viewer counts update less often than bounded progress probes. */ -const SEARCH_SOURCE_SUMMARY_POLL_MS = 30_000 - -export function useSearchSourceOverview(scope: ResourceScope, options?: { enabled?: boolean }) { - return useQuery({ - queryKey: searchSourceKeys.overview(scope), - queryFn: async ({ signal }) => - ( - await requestJson(readSearchSourceOverviewContract, { - query: resourceScopeFields(scope), - signal, - }) - ).data, - enabled: options?.enabled ?? true, - staleTime: CONNECTOR_LIST_STALE_TIME, - refetchInterval: (query) => - query.state.data?.providers.some((provider) => provider.isSyncing) - ? SEARCH_SOURCE_SUMMARY_POLL_MS - : false, - }) -} - -/** Administrative health is independent of the viewer's account and document permissions. */ -export function useOrganizationSearchOverview( - organizationId: string, - options?: { enabled?: boolean } -) { - return useQuery({ - queryKey: searchSourceKeys.organizationOverview(organizationId), - queryFn: async ({ signal }) => - ( - await requestJson(readOrganizationSearchOverviewContract, { - query: { organizationId }, - signal, - }) - ).data, - enabled: Boolean(organizationId) && (options?.enabled ?? true), - staleTime: CONNECTOR_LIST_STALE_TIME, - refetchInterval: (query) => - query.state.data?.providers.some((provider) => provider.isSyncing || provider.hasPendingSync) - ? SEARCH_SOURCE_SUMMARY_POLL_MS - : false, - }) -} - export function useSearchSources( owner?: string | ResourceScope, options?: { enabled?: boolean search?: string - mine?: boolean connectorType?: string excludeConnectorType?: string } ) { - const queryClient = useQueryClient() const scope = typeof owner === 'string' ? owner ? { kind: 'workspace' as const, workspaceId: owner } : undefined : owner - const workspaceId = scope?.kind === 'workspace' ? scope.workspaceId : undefined - const organizationId = scope?.kind === 'organization' ? scope.organizationId : undefined const enabled = Boolean(scope) && (options?.enabled ?? true) const filters = { search: options?.search?.trim().toLowerCase() ?? '', - mine: options?.mine ?? false, ...(options?.connectorType?.trim() ? { connectorType: options.connectorType.trim() } : {}), ...(options?.excludeConnectorType?.trim() ? { excludeConnectorType: options.excludeConnectorType.trim() } : {}), } - const summary = useInfiniteQuery({ + return useInfiniteQuery({ queryKey: searchSourceKeys.pages(scope, filters), initialPageParam: null as string | null, getNextPageParam: (page: SearchSourcePage) => page.nextCursor, @@ -528,73 +461,7 @@ export function useSearchSources( ).data, enabled, staleTime: CONNECTOR_LIST_STALE_TIME, - refetchInterval: (query) => - query.state.data?.pages.some((page) => page.sources.some((source) => source.isSyncing)) - ? SEARCH_SOURCE_SUMMARY_POLL_MS - : false, }) - const activeIds = (summary.data ?? []) - .filter((source) => source.isSyncing) - .map((source) => source.connectorId) - .sort() - const progress = useQuery({ - queryKey: searchSourceKeys.progress(scope, activeIds), - queryFn: async ({ signal }) => { - if (!scope) throw new Error('A Search source scope is required') - const sources: SearchSourceProgress[] = [] - for (let offset = 0; offset < activeIds.length; offset += MAX_SEARCH_SOURCE_PROGRESS_ITEMS) { - const response = await requestJson(readSearchSourceProgressContract, { - body: { - ...resourceScopeFields(scope), - connectorIds: activeIds.slice(offset, offset + MAX_SEARCH_SOURCE_PROGRESS_ITEMS), - }, - signal, - }) - sources.push(...response.data) - } - return sources - }, - enabled: enabled && activeIds.length > 0, - staleTime: CONNECTOR_SYNC_POLL_INTERVAL_MS, - refetchInterval: (query) => - query.state.dataUpdateCount < 20 ? CONNECTOR_SYNC_POLL_INTERVAL_MS : 15_000, - }) - - /** Reconcile exact viewer counts when the cheaper probe observes a state transition. */ - useEffect(() => { - if (!enabled || !progress.data || progress.dataUpdatedAt <= summary.dataUpdatedAt) return - const states = new Map(progress.data.map((source) => [source.connectorId, source])) - const changed = summary.data?.some((source) => { - if (!source.isSyncing) return false - const state = states.get(source.connectorId) - return ( - !state || - state.isSyncing !== source.isSyncing || - state.hasSyncError !== source.hasSyncError || - state.hasIndexingError !== source.viewerFailedDocumentCount > 0 - ) - }) - if (changed) { - const owner = organizationId - ? { kind: 'organization' as const, organizationId } - : { kind: 'workspace' as const, workspaceId: workspaceId! } - queryClient.invalidateQueries( - { queryKey: searchSourceKeys.list(owner) }, - { cancelRefetch: false } - ) - } - }, [ - enabled, - progress.data, - progress.dataUpdatedAt, - summary.data, - summary.dataUpdatedAt, - queryClient, - workspaceId, - organizationId, - ]) - - return summary } /** Mints the viewer's enrollment link for a per-member connector; the caller navigates to it. */ @@ -751,11 +618,7 @@ export function useTriggerSync() { queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) } }, - onSuccess: (_data, { knowledgeBaseId, connectorId }) => { - queryClient.invalidateQueries({ - queryKey: connectorKeys.progresses(knowledgeBaseId, connectorId), - }) - }, + /** An early poll can read idle before dispatch marks pending; reconcile after the request settles. */ onSettled: (_data, error, { knowledgeBaseId, connectorId }) => Promise.all([ @@ -807,9 +670,8 @@ async function fetchConnectorDocuments( export function useConnectorDocuments( knowledgeBaseId?: string, connectorId?: string, - options?: ConnectorDocumentListOptions & { progressScope?: ResourceScope; syncing?: boolean } + options?: ConnectorDocumentListOptions ) { - const queryClient = useQueryClient() const query = { includeExcluded: options?.filter ? undefined : (options?.includeExcluded ?? false), failedOnly: options?.filter ? undefined : (options?.failedOnly ?? false), @@ -838,54 +700,6 @@ export function useConnectorDocuments( staleTime: CONNECTOR_DOCUMENT_LIST_STALE_TIME, placeholderData: keepPreviousData, }) - const scope = options?.progressScope - const hasProgressScope = Boolean(scope) - const syncing = options?.syncing ?? false - const progress = useQuery({ - queryKey: connectorKeys.progress(knowledgeBaseId, connectorId, scope), - queryFn: async ({ signal }) => { - if (!scope || !connectorId) - throw new Error('A Search source scope and connector are required') - return ( - await requestJson(readSearchSourceProgressContract, { - body: { ...resourceScopeFields(scope), connectorIds: [connectorId] }, - signal, - }) - ).data - }, - enabled: Boolean(scope && knowledgeBaseId && connectorId), - staleTime: CONNECTOR_SYNC_POLL_INTERVAL_MS, - refetchInterval: (query) => - syncing || query.state.data?.some((source) => source.isSyncing) - ? query.state.dataUpdateCount < 20 - ? CONNECTOR_SYNC_POLL_INTERVAL_MS - : 15_000 - : false, - }) - - /** Refresh document pages once indexing settles, rather than polling every loaded page. */ - useEffect(() => { - if ( - !hasProgressScope || - syncing || - !progress.data || - progress.data.some((source) => source.isSyncing) || - progress.dataUpdatedAt <= documents.dataUpdatedAt - ) - return - void queryClient.invalidateQueries({ - queryKey: connectorDocumentKeys.lists(knowledgeBaseId, connectorId), - }) - }, [ - hasProgressScope, - syncing, - progress.data, - progress.dataUpdatedAt, - documents.dataUpdatedAt, - queryClient, - knowledgeBaseId, - connectorId, - ]) return documents } @@ -976,34 +790,6 @@ export function useRestoreConnectorDocument() { }) } -async function connectSimSearchConnector(body: ConnectSimSearchConnectorBody) { - const result = await requestJson(connectSimSearchConnectorContract, { body }) - return result.data -} - -/** - * One click on a Sim Search source: the source's per-member connector exists - * afterwards and the viewer has their enrollment link. The member list and the - * base list both gain a row on a first connect. - */ -export function useConnectSimSearchConnector() { - const queryClient = useQueryClient() - return useMutation({ - mutationFn: connectSimSearchConnector, - onSuccess: (data) => { - /** A first connect added a connector to the base; its own list is open on the settings page. */ - queryClient.invalidateQueries({ queryKey: connectorKeys.all(data.knowledgeBaseId) }) - }, - onSettled: () => { - queryClient.invalidateQueries({ queryKey: searchIndexKeys.details() }) - queryClient.invalidateQueries({ queryKey: searchSourceKeys.lists() }) - queryClient.invalidateQueries({ queryKey: searchIntegrationKeys.lists() }) - queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }) - void invalidateConnectorAccounts(queryClient) - }, - }) -} - export function usePrepareSearchSource() { const queryClient = useQueryClient() return useMutation({ diff --git a/apps/sim/hooks/queries/kb/knowledge.test.ts b/apps/sim/hooks/queries/kb/knowledge.test.ts index b14daea048f..a3d9329521e 100644 --- a/apps/sim/hooks/queries/kb/knowledge.test.ts +++ b/apps/sim/hooks/queries/kb/knowledge.test.ts @@ -1,10 +1,6 @@ import { apiClientRequestMock } from '@sim/testing/mocks/api-client-request.mock' import { authClientMock, authClientMockFns } from '@sim/testing/mocks/auth-client.mock' -import { - createMockDeploymentShape, - deploymentShapeMock, - deploymentShapeMockFns, -} from '@sim/testing/mocks/deployment-shape.mock' +import { deploymentShapeMock } from '@sim/testing/mocks/deployment-shape.mock' import { emcnMock } from '@sim/testing/mocks/emcn.mock' import { reactQueryMock, reactQueryMockFns } from '@sim/testing/mocks/react-query.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -36,9 +32,6 @@ const mocks = { getQueryData: reactQueryMockFns.mockQueryClient.getQueryData, } authClientMockFns.mockUseSession.mockReturnValue({ data: { user: { id: 'reader' } } }) -deploymentShapeMockFns.mockUseDeploymentShape.mockImplementation(() => - createMockDeploymentShape({ features: { liveEnterpriseSearch: mocks.live } }) -) interface CapturedQuery { queryKey: readonly unknown[] @@ -101,49 +94,13 @@ describe('knowledge query placeholder scope', () => { ).toBeUndefined() }) - it('retains successful refinements only for the same reader, query, scope, and result limit', () => { - const query = captureQuery(() => - useWorkspaceKnowledgeSearch('workspace-1', 'release', { source: 'slack' }, 5) - ) - const previous = { results: [{ documentId: 'private-document' }] } - mocks.getQueryData.mockReturnValue(previous) - const placeholder = (scope: string, text: string, topK: number, userId: string) => - query.placeholderData?.(previous, { - queryKey: knowledgeKeys.search(scope, text, {}, topK, userId), - state: { status: 'success', isInvalidated: false }, - }) - expect(placeholder('workspace-1', 'release', 5, 'reader')).toBe(previous) - expect(placeholder('workspace-1', 'release', 20, 'reader')).toBeUndefined() - expect(placeholder('workspace-1', 'release', 5, 'other')).toBeUndefined() - expect(placeholder('workspace-2', 'release', 5, 'reader')).toBeUndefined() - expect(placeholder('workspace-1', 'different', 5, 'reader')).toBeUndefined() - }) - - it('retains a page-owned search across result limits only for the same reader', () => { - const query = captureQuery(() => - useWorkspaceKnowledgeSearch('workspace-1', 'release', { source: 'slack' }, 50, { - retainAcrossLimits: true, - }) - ) - const previous = { results: [{ documentId: 'private-document' }] } - mocks.getQueryData.mockReturnValue(previous) - const placeholder = (topK: number, userId: string) => - query.placeholderData?.(previous, { - queryKey: knowledgeKeys.search('workspace-1', 'release', {}, topK, userId), - state: { status: 'success', isInvalidated: false }, - }) - expect(placeholder(20, 'reader')).toBe(previous) - expect(placeholder(50, 'reader')).toBe(previous) - expect(placeholder(20, 'other')).toBeUndefined() - }) - it('partitions search cache entries by filter and reader', () => { const query = captureQuery(() => useWorkspaceKnowledgeSearch('workspace-1', 'new query', { source: 'slack' }) ) expect(query.queryKey).toEqual([ ...knowledgeKeys.search('workspace-1', 'new query', { source: 'slack' }, 20, 'reader'), - 'indexed', + 'live', ]) expect(knowledgeKeys.search('workspace-1', 'query', { source: 'slack' })).not.toEqual( knowledgeKeys.search('workspace-1', 'query', { source: 'gitlab' }) diff --git a/apps/sim/hooks/queries/kb/knowledge.ts b/apps/sim/hooks/queries/kb/knowledge.ts index ce2833fd81c..8a9a4e1d349 100644 --- a/apps/sim/hooks/queries/kb/knowledge.ts +++ b/apps/sim/hooks/queries/kb/knowledge.ts @@ -58,7 +58,6 @@ import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge/searc import type { NativeSearchQuery } from '@/lib/api/contracts/mothership-assistant-tools' import { useSession } from '@/lib/auth/auth-client' import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types' -import { useDeploymentShape } from '@/lib/core/config/deployment-shape' import { type ResourceScope, resourceScopeFields, @@ -1214,15 +1213,9 @@ async function searchWorkspaceKnowledge( interface WorkspaceKnowledgeSearchOptions { nativeQueries?: NativeSearchQuery[] reuseFreshResult?: boolean - /** - * Keeps the previous result painted when only the result limit changes. Set it when the - * surface owns the limit (Show more widens the same search); leave it off when the limit is - * part of what was asked for, so a new limit is a new search that never shows the old one. - */ - retainAcrossLimits?: boolean } -/** Searches the canonical index under the signed-in person's ACLs. */ +/** Searches connected providers with the signed-in person's access. */ export function useWorkspaceKnowledgeSearch( owner: string | ResourceScope | undefined, query: string, @@ -1230,10 +1223,7 @@ export function useWorkspaceKnowledgeSearch( topK = 20, options?: WorkspaceKnowledgeSearchOptions ) { - const { features } = useDeploymentShape() - const live = features.liveEnterpriseSearch === true const { data: session } = useSession() - const queryClient = useQueryClient() const userId = session?.user?.id const trimmed = query.trim() const scope = @@ -1247,7 +1237,7 @@ export function useWorkspaceKnowledgeSearch( return useQuery({ queryKey: [ ...knowledgeKeys.search(scopeKey, trimmed, filters, topK, userId, options?.nativeQueries), - live ? 'live' : 'indexed', + 'live', ], queryFn: ({ signal }) => searchWorkspaceKnowledge( @@ -1256,7 +1246,7 @@ export function useWorkspaceKnowledgeSearch( query: trimmed, filters, topK, - ...(live && options?.nativeQueries ? { nativeQueries: options.nativeQueries } : {}), + ...(options?.nativeQueries ? { nativeQueries: options.nativeQueries } : {}), }, signal ), @@ -1269,21 +1259,7 @@ export function useWorkspaceKnowledgeSearch( filters?.modifiedAfter || filters?.modifiedBefore ), - staleTime: live - ? options?.reuseFreshResult - ? 60_000 - : 0 - : WORKSPACE_KNOWLEDGE_SEARCH_STALE_TIME, + staleTime: options?.reuseFreshResult ? WORKSPACE_KNOWLEDGE_SEARCH_STALE_TIME : 0, retry: false, - placeholderData: (previous, previousQuery) => { - if (live || !userId || previousQuery?.state.status !== 'success') return undefined - if (previousQuery.state.isInvalidated) return undefined - const prefix = knowledgeKeys.searchQuery(scopeKey, trimmed, userId) - if (!prefix.every((part, index) => previousQuery.queryKey[index] === part)) return undefined - /** `search()` appends filters, then the limit, after the reader/query prefix. */ - const previousTopK = previousQuery.queryKey[prefix.length + 1] - if (!options?.retainAcrossLimits && previousTopK !== topK) return undefined - return queryClient.getQueryData(previousQuery.queryKey) === previous ? previous : undefined - }, }) } diff --git a/apps/sim/hooks/queries/organization-accounts.ts b/apps/sim/hooks/queries/organization-accounts.ts index 9ea8c72b571..571895a5a25 100644 --- a/apps/sim/hooks/queries/organization-accounts.ts +++ b/apps/sim/hooks/queries/organization-accounts.ts @@ -48,7 +48,6 @@ import { connectDesktopSource } from '@/lib/desktop/source-connect' import { personalCredentialKeys } from '@/hooks/queries/personal-credentials' import { mcpKeys } from '@/hooks/queries/utils/mcp-keys' import { resetOrganizationSearchAccess } from '@/hooks/queries/utils/reset-organization-search-access' -import { searchSourceKeys } from '@/hooks/queries/utils/search-source-keys' import { invalidateSelectorQueries } from '@/hooks/queries/utils/selector-keys' import { slackSearchKeys } from '@/hooks/queries/utils/slack-search-keys' @@ -242,9 +241,6 @@ export function useUpdateOrganizationAccounts() { queryClient.invalidateQueries({ queryKey: slackSearchKeys.organizationManifests(organizationId), }), - queryClient.invalidateQueries({ - queryKey: searchSourceKeys.organizationOverview(organizationId), - }), ]), }) } diff --git a/apps/sim/hooks/queries/organization-search-stats.ts b/apps/sim/hooks/queries/organization-search-stats.ts deleted file mode 100644 index a11c471103b..00000000000 --- a/apps/sim/hooks/queries/organization-search-stats.ts +++ /dev/null @@ -1,28 +0,0 @@ -'use client' - -import { useQuery } from '@tanstack/react-query' -import { requestJson } from '@/lib/api/client/request' -import { - type OrganizationSearchStatsQuery, - readOrganizationSearchStatsContract, -} from '@/lib/api/contracts/knowledge/search-stats' - -export const ORGANIZATION_SEARCH_STATS_STALE_TIME = 60_000 - -export const organizationSearchStatsKeys = { - all: ['organization-search-stats'] as const, - summaries: () => [...organizationSearchStatsKeys.all, 'summary'] as const, - summary: (query: OrganizationSearchStatsQuery) => - [...organizationSearchStatsKeys.summaries(), query] as const, -} - -export function useOrganizationSearchStats(query: OrganizationSearchStatsQuery) { - return useQuery({ - queryKey: organizationSearchStatsKeys.summary(query), - queryFn: async ({ signal }) => { - const result = await requestJson(readOrganizationSearchStatsContract, { query, signal }) - return result.data - }, - staleTime: ORGANIZATION_SEARCH_STATS_STALE_TIME, - }) -} diff --git a/apps/sim/hooks/queries/personal-source-setup.ts b/apps/sim/hooks/queries/personal-source-setup.ts deleted file mode 100644 index 645b8bd8c97..00000000000 --- a/apps/sim/hooks/queries/personal-source-setup.ts +++ /dev/null @@ -1,67 +0,0 @@ -'use client' - -import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' -import { requestJson } from '@/lib/api/client/request' -import { - listPersonalSourceSetupAccountsContract, - type PersonalSourceSetupBody, - type PersonalSourceSetupQuery, - personalSourceSetupContract, -} from '@/lib/api/contracts/knowledge/personal-source-setup' -import { organizationAccountsKeys } from '@/hooks/queries/organization-accounts' -import { personalSearchIntegrationKeys } from '@/hooks/queries/personal-search-integrations' -import { searchSourceKeys } from '@/hooks/queries/utils/search-source-keys' - -export const PERSONAL_SOURCE_SETUP_STALE_TIME = 15_000 - -export const personalSourceSetupKeys = { - all: ['personal-source-setup'] as const, - lists: () => [...personalSourceSetupKeys.all, 'list'] as const, - list: (query: PersonalSourceSetupQuery) => [...personalSourceSetupKeys.lists(), query] as const, -} - -export function usePersonalSourceSetupAccounts(query: PersonalSourceSetupQuery) { - return useQuery({ - queryKey: personalSourceSetupKeys.list(query), - queryFn: async ({ signal }) => - (await requestJson(listPersonalSourceSetupAccountsContract, { query, signal })).data, - staleTime: PERSONAL_SOURCE_SETUP_STALE_TIME, - refetchInterval: (state) => - query.completionId && !state.state.data?.completedCredentialId ? 1_500 : false, - }) -} - -export function useAuthorizePersonalSourceSetup() { - return useMutation({ - mutationFn: async (body: Extract) => { - const { data } = await requestJson(personalSourceSetupContract, { body }) - if (data.kind !== 'authorization') throw new Error('Could not start account authorization') - return data - }, - }) -} - -export function useConnectPersonalSourceSetup() { - const client = useQueryClient() - return useMutation({ - mutationFn: async (body: Extract) => { - const { data } = await requestJson(personalSourceSetupContract, { body }) - if (data.kind !== 'connected') throw new Error('Could not connect the source') - return data - }, - onSuccess: (_data, body) => - Promise.all([ - client.invalidateQueries({ queryKey: personalSourceSetupKeys.lists() }), - client.invalidateQueries({ queryKey: personalSearchIntegrationKeys.lists() }), - client.invalidateQueries({ - queryKey: searchSourceKeys.list({ - kind: 'organization', - organizationId: body.organizationId, - }), - }), - client.invalidateQueries({ - queryKey: organizationAccountsKeys.detail(body.organizationId), - }), - ]), - }) -} diff --git a/apps/sim/hooks/queries/selectors.test.tsx b/apps/sim/hooks/queries/selectors.test.tsx index 5b6a5170813..f766668d124 100644 --- a/apps/sim/hooks/queries/selectors.test.tsx +++ b/apps/sim/hooks/queries/selectors.test.tsx @@ -3,10 +3,6 @@ */ import { act } from 'react' -import { - apiClientRequestMock, - apiClientRequestMockFns, -} from '@sim/testing/mocks/api-client-request.mock' import { QueryClient, QueryClientProvider } from '@tanstack/react-query' import { createRoot, type Root } from 'react-dom/client' import { afterEach, describe, expect, it, vi } from 'vitest' @@ -19,12 +15,8 @@ vi.mock('@/lib/selectors/client/execute-selector', () => ({ executeSelectorRequest: mockExecuteSelectorRequest, })) -vi.mock('@/lib/api/client/request', () => apiClientRequestMock) - import { useSelectorOptionDetail, useSelectorOptions } from '@/hooks/queries/selectors' -const mockRequestJson = apiClientRequestMockFns.mockRequestJson - interface HookHarness { getResult: () => T queryClient: QueryClient @@ -104,76 +96,6 @@ afterEach(() => { }) describe('generic selector queries', () => { - it('uses the dedicated personal setup contract and isolates it from ordinary browsing', async () => { - const personalItems = [{ id: 'PERSONAL', label: 'Personal project' }] - mockRequestJson.mockResolvedValue({ - success: true, - data: { kind: 'list', items: personalItems }, - }) - mockExecuteSelectorRequest.mockResolvedValue({ - kind: 'list', - items: [{ id: 'ADMIN', label: 'Admin project' }], - }) - const hook = renderHookWithClient(() => - useSelectorOptions('jira.projectKeys', { - context: { oauthCredential: 'credential-1', domain: 'example.atlassian.net' }, - scope: { kind: 'organization', organizationId: 'org-1' }, - surface: { kind: 'personal-search-setup', organizationId: 'org-1', connectorType: 'jira' }, - surfaceId: 'projects', - }) - ) - await waitFor(() => expect(hook.getResult().data).toEqual(personalItems)) - expect(mockExecuteSelectorRequest).not.toHaveBeenCalled() - expect(mockRequestJson).toHaveBeenCalledWith( - expect.objectContaining({ path: '/api/knowledge/sim-search/personal-source-setup' }), - expect.objectContaining({ - body: { - action: 'options', - organizationId: 'org-1', - connectorType: 'jira', - credentialId: 'credential-1', - domain: 'example.atlassian.net', - request: { kind: 'list' }, - }, - signal: expect.any(AbortSignal), - }) - ) - hook.rerender(() => - useSelectorOptions('jira.projectKeys', { - context: { oauthCredential: 'credential-1', domain: 'example.atlassian.net' }, - scope: { kind: 'organization', organizationId: 'org-1' }, - surfaceId: 'projects', - }) - ) - await waitFor(() => - expect(hook.getResult().data).toEqual([{ id: 'ADMIN', label: 'Admin project' }]) - ) - expect(mockExecuteSelectorRequest).toHaveBeenCalledTimes(1) - }) - - it.each(['selector', 'organization'] as const)( - 'rejects a mismatched personal setup %s before sending a request', - async (mismatch) => { - const hook = renderHookWithClient(() => - useSelectorOptions(mismatch === 'selector' ? 'confluence.spaces' : 'jira.projectKeys', { - context: { oauthCredential: 'credential-1', domain: 'example.atlassian.net' }, - scope: { - kind: 'organization', - organizationId: mismatch === 'organization' ? 'org-2' : 'org-1', - }, - surface: { - kind: 'personal-search-setup', - organizationId: 'org-1', - connectorType: 'jira', - }, - }) - ) - await waitFor(() => expect(hook.getResult().error).not.toBeNull()) - expect(mockRequestJson).not.toHaveBeenCalled() - expect(mockExecuteSelectorRequest).not.toHaveBeenCalled() - } - ) - it('transports supported search and keeps context and request plaintext out of query keys', async () => { const credentialReference = '{{SHARED_GOOGLE_CREDENTIAL}}' const search = 'private search phrase' diff --git a/apps/sim/hooks/queries/selectors.ts b/apps/sim/hooks/queries/selectors.ts index a92dcf537c1..1dd37f1995c 100644 --- a/apps/sim/hooks/queries/selectors.ts +++ b/apps/sim/hooks/queries/selectors.ts @@ -2,12 +2,7 @@ import { useCallback, useEffect, useId, useMemo, useRef, useState } from 'react' import { useInfiniteQuery, useQueries, useQuery } from '@tanstack/react-query' -import { requestJson } from '@/lib/api/client/request' -import { personalSourceSetupContract } from '@/lib/api/contracts/knowledge/personal-source-setup' -import { - type ExecuteSelectorClientInput, - executeSelectorRequest, -} from '@/lib/selectors/client/execute-selector' +import { executeSelectorRequest } from '@/lib/selectors/client/execute-selector' import { projectSelectorContext } from '@/lib/selectors/context' import { MAX_SELECTOR_OPTIONS, MAX_SELECTOR_PAGES } from '@/lib/selectors/limits' import { @@ -21,7 +16,6 @@ import type { SelectorOption, SelectorPage, SelectorScope, - SelectorSurface, } from '@/lib/selectors/types' import { selectorKeys } from '@/hooks/queries/utils/selector-keys' @@ -40,38 +34,6 @@ interface SelectorHookArgs { search?: string enabled?: boolean surfaceId?: string - surface?: SelectorSurface -} - -async function executeForSurface( - input: ExecuteSelectorClientInput, - surface?: SelectorSurface -): Promise { - if (!surface) return executeSelectorRequest(input) - const expectedKey = surface.connectorType === 'jira' ? 'jira.projectKeys' : 'confluence.spaces' - if ( - input.selectorKey !== expectedKey || - input.scope?.kind !== 'organization' || - input.scope.organizationId !== surface.organizationId || - !input.context.oauthCredential || - !input.context.domain - ) - throw new Error('This selector is not available during personal source setup') - const result = await requestJson(personalSourceSetupContract, { - body: { - action: 'options', - organizationId: surface.organizationId, - connectorType: surface.connectorType, - credentialId: input.context.oauthCredential, - domain: input.context.domain, - request: input.request, - }, - signal: input.signal, - }) - if (result.data.kind !== 'list' && result.data.kind !== 'detail') { - throw new Error('Personal source setup returned an unexpected selector result') - } - return result.data } export interface SelectorOptionsResult { @@ -181,13 +143,7 @@ function usePreparedSelector( const context = projectSelectorContext(key, args.context) const scope = selectorScopeFromContext(args.context, args.scope) const contextValues = manifest.context.allowed.map((field) => context[field]) - const revision = useOpaqueRevision([ - ...contextValues, - ...requestValues, - args.surface?.kind, - args.surface?.organizationId, - args.surface?.connectorType, - ]) + const revision = useOpaqueRevision([...contextValues, ...requestValues]) const ready = args.enabled !== false && isSelectorReady(key, context) && @@ -199,7 +155,6 @@ function usePreparedSelector( revision, ready, surfaceId: args.surfaceId ?? generatedSurfaceId, - surface: args.surface, } } @@ -222,19 +177,16 @@ export function useSelectorOptions( // rq-lint-allow: context and search are represented by an opaque privacy revision. queryKey: baseKey, queryFn: async ({ signal }) => { - const result = await executeForSurface( - { - selectorKey: key, - scope: prepared.scope, - context: prepared.context, - request: { - kind: 'list', - ...(effectiveSearch !== undefined ? { search: effectiveSearch } : {}), - }, - signal, + const result = await executeSelectorRequest({ + selectorKey: key, + scope: prepared.scope, + context: prepared.context, + request: { + kind: 'list', + ...(effectiveSearch !== undefined ? { search: effectiveSearch } : {}), }, - prepared.surface - ) + signal, + }) if (result.kind !== 'list') throw new Error('Selector returned an unexpected detail result') return result }, @@ -247,20 +199,17 @@ export function useSelectorOptions( // rq-lint-allow: context and search are represented by an opaque privacy revision. queryKey: [...baseKey, 'paged'], queryFn: async ({ pageParam, signal }) => { - const result = await executeForSurface( - { - selectorKey: key, - scope: prepared.scope, - context: prepared.context, - request: { - kind: 'list', - ...(effectiveSearch !== undefined ? { search: effectiveSearch } : {}), - ...(typeof pageParam === 'string' ? { cursor: pageParam } : {}), - }, - signal, + const result = await executeSelectorRequest({ + selectorKey: key, + scope: prepared.scope, + context: prepared.context, + request: { + kind: 'list', + ...(effectiveSearch !== undefined ? { search: effectiveSearch } : {}), + ...(typeof pageParam === 'string' ? { cursor: pageParam } : {}), }, - prepared.surface - ) + signal, + }) if (result.kind !== 'list') throw new Error('Selector returned an unexpected detail result') return result }, @@ -441,16 +390,13 @@ export function useSelectorOptionDetail( prepared.revision ), queryFn: async ({ signal }) => { - const result = await executeForSurface( - { - selectorKey: key, - scope: prepared.scope, - context: prepared.context, - request: { kind: 'detail', id: args.detailId! }, - signal, - }, - prepared.surface - ) + const result = await executeSelectorRequest({ + selectorKey: key, + scope: prepared.scope, + context: prepared.context, + request: { kind: 'detail', id: args.detailId! }, + signal, + }) if (result.kind !== 'detail') throw new Error('Selector returned an unexpected list result') return result.item }, @@ -478,16 +424,13 @@ export function useSelectorOptionDetails( ordinal ), queryFn: async ({ signal }: { signal: AbortSignal }) => { - const result = await executeForSurface( - { - selectorKey: key, - scope: prepared.scope, - context: prepared.context, - request: { kind: 'detail', id: detailId }, - signal, - }, - prepared.surface - ) + const result = await executeSelectorRequest({ + selectorKey: key, + scope: prepared.scope, + context: prepared.context, + request: { kind: 'detail', id: detailId }, + signal, + }) if (result.kind !== 'detail') throw new Error('Selector returned an unexpected list result') return result.item }, diff --git a/apps/sim/hooks/queries/utils/reset-organization-search-access.test.ts b/apps/sim/hooks/queries/utils/reset-organization-search-access.test.ts index 3c44a253310..acf3fa91296 100644 --- a/apps/sim/hooks/queries/utils/reset-organization-search-access.test.ts +++ b/apps/sim/hooks/queries/utils/reset-organization-search-access.test.ts @@ -1,62 +1,9 @@ -import { QueryClient, QueryObserver } from '@tanstack/react-query' +import { QueryClient } from '@tanstack/react-query' import { expect, it, vi } from 'vitest' import type { WorkspaceKnowledgeSearchResult } from '@/lib/api/contracts/knowledge/search' import { resourceScopeKey } from '@/lib/core/resource-scope' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' import { resetOrganizationSearchAccess } from '@/hooks/queries/utils/reset-organization-search-access' -import { searchSourceKeys } from '@/hooks/queries/utils/search-source-keys' - -it.each([true, false])( - 'keeps administrative rows visible while revalidating access, refresh success=%s', - async (success) => { - const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) - const scope = { kind: 'organization', organizationId: 'org-1' } as const - const adminKey = searchSourceKeys.organizationOverview(scope.organizationId) - const otherKey = searchSourceKeys.organizationOverview('org-2') - const viewerKeys = [ - searchSourceKeys.list(scope), - searchSourceKeys.overview(scope), - searchSourceKeys.pages(scope, { search: '', mine: false }), - ] - const before = { providers: [{ connectorType: 'gmail', approved: true }] } - const after = { providers: [{ connectorType: 'gmail', approved: false }] } - const response = Promise.withResolvers() - const fetchOverview = vi.fn(() => response.promise) - client.setQueryData(adminKey, before) - client.setQueryData(otherKey, before) - for (const key of viewerKeys) client.setQueryData(key, { privateContent: 'previous access' }) - const observer = new QueryObserver(client, { - queryKey: adminKey, - queryFn: fetchOverview, - staleTime: Number.POSITIVE_INFINITY, - }) - const observed = vi.fn() - const unsubscribe = observer.subscribe(observed) - try { - const refreshing = resetOrganizationSearchAccess(client, scope.organizationId) - expect(fetchOverview).toHaveBeenCalledOnce() - expect(observer.getCurrentResult()).toMatchObject({ data: before, isPending: false }) - for (const key of viewerKeys) expect(client.getQueryData(key)).toBeUndefined() - expect(client.getQueryState(otherKey)?.isInvalidated).toBe(false) - - if (success) response.resolve(after) - else response.reject(new Error('Could not refresh sources')) - await refreshing - - expect(observer.getCurrentResult()).toMatchObject({ - data: success ? after : before, - isError: !success, - isFetching: false, - }) - expect(observed.mock.calls.every(([result]) => result.data && !result.isPending)).toBe(true) - expect(client.getQueryData(otherKey)).toEqual(before) - } finally { - response.resolve(after) - unsubscribe() - client.clear() - } - } -) it.each([ { name: 'document', key: knowledgeKeys.document('kb-direct', 'document-direct') }, diff --git a/apps/sim/hooks/queries/utils/reset-organization-search-access.ts b/apps/sim/hooks/queries/utils/reset-organization-search-access.ts index 53f7b3a5cfb..f97837d750c 100644 --- a/apps/sim/hooks/queries/utils/reset-organization-search-access.ts +++ b/apps/sim/hooks/queries/utils/reset-organization-search-access.ts @@ -1,4 +1,4 @@ -import { matchQuery, type QueryClient } from '@tanstack/react-query' +import type { QueryClient } from '@tanstack/react-query' import { resourceScopeKey } from '@/lib/core/resource-scope' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' import { searchSourceKeys } from '@/hooks/queries/utils/search-source-keys' @@ -9,10 +9,6 @@ export async function resetOrganizationSearchAccess( organizationId: string ) { const scope = { kind: 'organization', organizationId } as const - const adminOverview = { - queryKey: searchSourceKeys.organizationOverview(organizationId), - exact: true, - } await Promise.all([ queryClient.resetQueries({ queryKey: [...knowledgeKeys.searches(), resourceScopeKey(scope)], @@ -21,8 +17,6 @@ export async function resetOrganizationSearchAccess( queryClient.resetQueries({ queryKey: knowledgeKeys.details() }), queryClient.resetQueries({ queryKey: searchSourceKeys.list(scope), - predicate: (query) => !matchQuery(adminOverview, query), }), - queryClient.invalidateQueries(adminOverview), ]) } diff --git a/apps/sim/hooks/queries/utils/search-source-keys.ts b/apps/sim/hooks/queries/utils/search-source-keys.ts index ee508d3fb1b..1ceb82e5bf2 100644 --- a/apps/sim/hooks/queries/utils/search-source-keys.ts +++ b/apps/sim/hooks/queries/utils/search-source-keys.ts @@ -3,26 +3,14 @@ import { type ResourceScope, resourceScopeKey } from '@/lib/core/resource-scope' export const searchSourceKeys = { all: ['search-sources'] as const, lists: () => [...searchSourceKeys.all, 'list'] as const, - progress: (scope: ResourceScope | undefined, connectorIds: string[]) => - [ - ...searchSourceKeys.all, - 'progress', - scope ? resourceScopeKey(scope) : '', - connectorIds, - ] as const, pages: ( scope: string | ResourceScope | undefined, filters: { search: string - mine: boolean connectorType?: string excludeConnectorType?: string } ) => [...searchSourceKeys.list(scope), 'pages', filters] as const, - overview: (scope?: string | ResourceScope) => - [...searchSourceKeys.list(scope), 'overview'] as const, - organizationOverview: (organizationId: string) => - [...searchSourceKeys.list({ kind: 'organization', organizationId }), 'admin-overview'] as const, list: (scope?: string | ResourceScope) => [ ...searchSourceKeys.lists(), diff --git a/apps/sim/hooks/use-personal-source-account.test.tsx b/apps/sim/hooks/use-personal-source-account.test.tsx deleted file mode 100644 index 20e9aba58a1..00000000000 --- a/apps/sim/hooks/use-personal-source-account.test.tsx +++ /dev/null @@ -1,130 +0,0 @@ -/** - * @vitest-environment jsdom - */ -import { act } from 'react' -import { emcnMock, emcnMockFns } from '@sim/testing/mocks/emcn.mock' -import { reactQueryMock, reactQueryMockFns } from '@sim/testing/mocks/react-query.mock' -import { createRoot, type Root } from 'react-dom/client' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const mocks = vi.hoisted(() => ({ - authorize: vi.fn(), - refetch: vi.fn(), - completed: null as string | null, - connected: vi.fn(), -})) -vi.mock('@tanstack/react-query', () => reactQueryMock) -vi.mock('@sim/emcn', () => emcnMock) -vi.mock('@/hooks/queries/personal-source-setup', () => ({ - personalSourceSetupKeys: { list: (query: unknown) => ['personal-source-setup', query] }, - useAuthorizePersonalSourceSetup: () => ({ mutateAsync: mocks.authorize, isPending: false }), - usePersonalSourceSetupAccounts: () => ({ - data: { accounts: [], completedCredentialId: mocks.completed }, - refetch: mocks.refetch, - }), -})) - -import { usePersonalSourceAccount } from '@/hooks/use-personal-source-account' - -const mockToastError = emcnMockFns.mockToast.error -const mockSetQueryData = reactQueryMockFns.mockQueryClient.setQueryData - -describe('personal source account authorization', () => { - let root: Root - let container: HTMLDivElement - let current: ReturnType - let channels: Array<{ - onmessage: ((event: MessageEvent) => void) | null - close: ReturnType - }> - let tab: { - opener: unknown - location: { href: string } - focus: ReturnType - close: ReturnType - } - function Probe() { - current = usePersonalSourceAccount({ - organizationId: 'org-1', - connectorType: 'jira', - onConnected: mocks.connected, - }) - return null - } - beforeEach(() => { - vi.useFakeTimers() - mocks.completed = null - mocks.authorize.mockResolvedValue({ - kind: 'authorization', - url: 'https://auth.atlassian.com/authorize', - }) - channels = [] - vi.stubGlobal( - 'BroadcastChannel', - class { - onmessage = null - close = vi.fn() - constructor() { - channels.push(this) - } - } - ) - tab = { opener: {}, location: { href: 'about:blank' }, focus: vi.fn(), close: vi.fn() } - vi.spyOn(window, 'open').mockReturnValue(tab as unknown as Window) - ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true - container = document.createElement('div') - document.body.appendChild(container) - root = createRoot(container) - act(() => root.render()) - }) - afterEach(() => { - act(() => root.unmount()) - container.remove() - vi.useRealTimers() - }) - it('authorizes before a source exists and requires server confirmation of completion', async () => { - await act(async () => current.connect()) - expect(tab.opener).toBeNull() - expect(tab.location.href).toBe('https://auth.atlassian.com/authorize') - expect(mocks.authorize).toHaveBeenCalledWith( - expect.objectContaining({ - action: 'authorize', - organizationId: 'org-1', - connectorType: 'jira', - oauthCompletionId: expect.any(String), - }) - ) - expect(current.pending).toBe(true) - act(() => channels[0].onmessage?.({ data: 'connected' } as MessageEvent)) - expect(mocks.refetch).toHaveBeenCalledOnce() - expect(current.pending).toBe(true) - expect(tab.close).not.toHaveBeenCalled() - mocks.completed = 'my-account' - act(() => root.render()) - expect(current.pending).toBe(false) - expect(tab.close).toHaveBeenCalledOnce() - expect(mocks.connected).toHaveBeenCalledWith('my-account') - mocks.completed = null - act(() => root.render()) - expect(current.pending).toBe(false) - }) - it('ignores a late authorization response after cancellation', async () => { - let resolve!: (value: { kind: string; url: string }) => void - mocks.authorize.mockReturnValue( - new Promise((done) => { - resolve = done - }) - ) - let connecting!: Promise - act(() => { - connecting = current.connect() - }) - act(() => current.cancel()) - await act(async () => { - resolve({ kind: 'authorization', url: 'https://auth.atlassian.com/authorize' }) - await connecting - }) - expect(tab.location.href).toBe('about:blank') - expect(current.pending).toBe(false) - }) -}) diff --git a/apps/sim/hooks/use-personal-source-account.ts b/apps/sim/hooks/use-personal-source-account.ts deleted file mode 100644 index 3ac8447127d..00000000000 --- a/apps/sim/hooks/use-personal-source-account.ts +++ /dev/null @@ -1,135 +0,0 @@ -'use client' - -import { useCallback, useEffect, useRef, useState } from 'react' -import { toast } from '@sim/emcn' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { useQueryClient } from '@tanstack/react-query' -import type { PersonalSourceSetupQuery } from '@/lib/api/contracts/knowledge/personal-source-setup' -import { - CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES, - credentialGroupOAuthCompletionChannel, - isCredentialGroupOAuthFailure, -} from '@/lib/credential-groups/oauth-completion' -import { - personalSourceSetupKeys, - useAuthorizePersonalSourceSetup, - usePersonalSourceSetupAccounts, -} from '@/hooks/queries/personal-source-setup' - -interface PersonalSourceAccountProps { - organizationId: string - connectorType: PersonalSourceSetupQuery['connectorType'] - onConnected?: (credentialId: string) => void -} - -/** A popup message only refreshes the account inventory; the server proves completion. */ -export function usePersonalSourceAccount({ - organizationId, - connectorType, - onConnected, -}: PersonalSourceAccountProps) { - const [completionId, setCompletionId] = useState() - const attempt = useRef<{ id: string; tab: Window } | null>(null) - const accounts = usePersonalSourceSetupAccounts({ organizationId, connectorType, completionId }) - const { mutateAsync: authorize } = useAuthorizePersonalSourceSetup() - const queryClient = useQueryClient() - const onConnectedRef = useRef(onConnected) - const completed = accounts.data?.completedCredentialId - const { refetch } = accounts - - useEffect(() => { - onConnectedRef.current = onConnected - }, [onConnected]) - - useEffect(() => { - return () => { - attempt.current?.tab.close() - attempt.current = null - } - }, []) - - useEffect(() => { - if (!completionId || completed) return - const fail = (message: string) => { - if (attempt.current?.id !== completionId) return - attempt.current.tab.close() - attempt.current = null - setCompletionId(undefined) - toast.error(message) - } - const channel = new BroadcastChannel(credentialGroupOAuthCompletionChannel(completionId)) - channel.onmessage = ({ data }: MessageEvent) => { - if (isCredentialGroupOAuthFailure(data)) fail(CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES[data]) - else if (data === 'connected') void refetch() - } - const timer = window.setTimeout( - () => fail(CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES.expired), - 10 * 60_000 - ) - return () => { - channel.close() - window.clearTimeout(timer) - } - }, [completionId, completed, refetch]) - - useEffect(() => { - if (completed && attempt.current && attempt.current.id === completionId) { - attempt.current.tab.close() - attempt.current = null - setCompletionId(undefined) - queryClient.setQueryData(personalSourceSetupKeys.list({ organizationId, connectorType }), { - ...accounts.data, - completedCredentialId: null, - }) - onConnectedRef.current?.(completed) - } - }, [completed, completionId, accounts.data, queryClient, organizationId, connectorType]) - - const connect = useCallback(async () => { - if (attempt.current) { - attempt.current.tab.focus() - return - } - const tab = window.open('about:blank', '_blank', 'width=600,height=700') - if (!tab) { - toast.error('Allow pop-ups for this site to connect your account.') - return - } - tab.opener = null - const id = generateId() - attempt.current = { id, tab } - setCompletionId(id) - try { - const result = await authorize({ - action: 'authorize', - organizationId, - connectorType, - oauthCompletionId: id, - }) - if (attempt.current?.id !== id) return - const url = new URL(result.url) - if ( - url.protocol !== 'https:' && - !(url.protocol === 'http:' && url.origin === window.location.origin) - ) { - throw new Error('The provider authorization URL is invalid') - } - tab.location.href = url.href - } catch (error) { - if (attempt.current?.id !== id) return - tab.close() - attempt.current = null - setCompletionId(undefined) - toast.error(getErrorMessage(error, 'Could not connect your account')) - } - }, [authorize, organizationId, connectorType]) - - const cancel = useCallback(() => { - attempt.current?.tab.close() - attempt.current = null - setCompletionId(undefined) - }, []) - - return { accounts, connect, cancel, pending: Boolean(completionId) } -} diff --git a/apps/sim/hooks/use-search-integration-connection.test.tsx b/apps/sim/hooks/use-search-integration-connection.test.tsx index 0f7c87a0a8d..078f1555e89 100644 --- a/apps/sim/hooks/use-search-integration-connection.test.tsx +++ b/apps/sim/hooks/use-search-integration-connection.test.tsx @@ -22,9 +22,8 @@ const m = vi.hoisted(() => ({ type: 'link' provider: string connectorType: string - connectorId?: string - connectionMode?: 'live' - optionId?: string + connectionMode: 'live' + optionId: string } | undefined, })) @@ -32,7 +31,8 @@ const target = { type: 'link', provider: 'gmail', connectorType: 'gmail', - connectorId: 'source', + connectionMode: 'live', + optionId: 'gmail-option', } as const vi.mock('@tanstack/react-query', () => reactQueryMock) vi.mock('@/hooks/queries/personal-search-integrations', () => ({ @@ -41,9 +41,7 @@ vi.mock('@/hooks/queries/personal-search-integrations', () => ({ usePersonalSearchIntegrations: (query: { completionId?: string }) => ({ data: { connections: [{ accounts: m.accounts }], - available: [ - { target: m.requestedTarget?.connectionMode === 'live' ? m.requestedTarget : target }, - ], + available: [{ target: m.requestedTarget ?? target }], completedCredentialId: m.receipts.get(query.completionId ?? '') ?? null, }, isSuccess: !m.queryError, @@ -123,8 +121,6 @@ beforeEach(() => { m.refetch.mockResolvedValue({ isSuccess: true, data: { connections: [] } }) m.mutate.mockResolvedValue({ url: 'https://provider.test/authorize', - connectorId: 'source', - knowledgeBaseId: 'kb', }) container = document.createElement('div') document.body.append(container) @@ -138,20 +134,6 @@ afterEach(() => { }) describe('Search connection card lifecycle', () => { - it('confirms account-first setup against inventory and reflects later revocation', () => { - m.requestedTarget = { type: 'link', provider: 'jira', connectorType: 'jira' } - render() - act(() => connection().completeSetup({ connectorId: 'new-source', credentialId: 'mine' })) - expect(connection().connectorId).toBe('new-source') - expect(connection().connected).toBe(false) - expect(m.mutate).not.toHaveBeenCalled() - m.accounts = [{ credentialId: 'mine', status: 'connected' }] - render() - expect(connection().connected).toBe(true) - m.accounts = [{ credentialId: 'mine', status: 'reconnect_needed' }] - render() - expect(connection().connected).toBe(false) - }) it('starts OAuth only on click and completes only when its receipt and current account agree', async () => { render() expect(m.mutate).not.toHaveBeenCalled() @@ -246,24 +228,6 @@ describe('Search connection card lifecycle', () => { expect(connection().connected).toBe(true) expect(windows[1].close).toHaveBeenCalled() }) - it('retries the exact source created by the first attempt after cancellation or reload', async () => { - m.requestedTarget = { type: 'link', provider: 'gmail', connectorType: 'gmail' } - render() - await act(async () => { - await connection().connect() - }) - expect(m.mutate.mock.calls[0][0].target.connectorId).toBeUndefined() - act(() => connection().cancel()) - act(() => root.unmount()) - root = createRoot(container) - render() - expect(connection().available).toBe(true) - await act(async () => { - await connection().connect() - }) - expect(m.mutate.mock.calls[1][0].target.connectorId).toBe('source') - expect(m.mutate.mock.calls[1][0].target.credentialId).toBeUndefined() - }) it('keeps failed starts actionable and rejects stale data after authorization errors', async () => { render() m.mutate.mockRejectedValueOnce(new Error('Source no longer available')) diff --git a/apps/sim/hooks/use-search-integration-connection.ts b/apps/sim/hooks/use-search-integration-connection.ts index ed7ba243072..3596fd7c0dc 100644 --- a/apps/sim/hooks/use-search-integration-connection.ts +++ b/apps/sim/hooks/use-search-integration-connection.ts @@ -61,13 +61,10 @@ export function useSearchIntegrationConnection({ const client = useQueryClient() const { mutateAsync, isPending } = useConnectPersonalSearchIntegration() const pending = attempt?.status === 'pending' - const connectorId = target.connectorId ?? attempt?.connectorId - const effectiveTarget = connectorId ? { ...target, connectorId } : target const query = usePersonalSearchIntegrations( { organizationId, connectorType: target.connectorType, - connectorId, completionId: attempt?.completionId, }, { pending } @@ -93,9 +90,7 @@ export function useSearchIntegrationConnection({ ] const available = query.isSuccess && - availableTargets.some( - (candidate) => JSON.stringify(candidate) === JSON.stringify(effectiveTarget) - ) + availableTargets.some((candidate) => JSON.stringify(candidate) === JSON.stringify(target)) useEffect(() => { const refresh = () => setAttempt(readSearchConnectionAttempt(key)) @@ -152,72 +147,66 @@ export function useSearchIntegrationConnection({ }, [connected, attempt, key, query.data?.completedCredentialId]) const { refetch } = query - const connect = useCallback( - async (sourceConfig?: Record) => { - if (starting.current || isPending || connected) return - if (pending && popup.current && !popup.current.closed) { - popup.current.focus() - return - } - const desktop = isDesktopApp() - if (desktop && pending) return - const tab = desktop ? null : window.open('about:blank', '_blank', 'width=600,height=700') - if (!desktop && !tab) { - setLocalError('Allow pop-ups for this site to connect your account.') - return + const connect = useCallback(async () => { + if (starting.current || isPending || connected) return + if (pending && popup.current && !popup.current.closed) { + popup.current.focus() + return + } + const desktop = isDesktopApp() + if (desktop && pending) return + const tab = desktop ? null : window.open('about:blank', '_blank', 'width=600,height=700') + if (!desktop && !tab) { + setLocalError('Allow pop-ups for this site to connect your account.') + return + } + if (tab) tab.opener = null + popup.current = tab + starting.current = true + const controller = new AbortController() + nativeAbort.current = controller + setLocalError(null) + let next: SearchConnectionAttempt | undefined + try { + if (!desktop) { + const fresh = await refetch() + if (!fresh.isSuccess) throw fresh.error } - if (tab) tab.opener = null - popup.current = tab - starting.current = true - const controller = new AbortController() - nativeAbort.current = controller - setLocalError(null) - let next: SearchConnectionAttempt | undefined - try { - if (!desktop) { - const fresh = await refetch() - if (!fresh.isSuccess) throw fresh.error - } - controller.signal.throwIfAborted() - next = { - completionId: generateId(), - requestedAt: Date.now(), - connectorId, - status: 'pending', - error: null, - } - writeSearchConnectionAttempt(key, next) - const result = await mutateAsync({ - organizationId, - target: connectorId ? { ...target, connectorId } : target, - sourceConfig, - oauthCompletionId: next.completionId, - signal: controller.signal, - }) - if (!result || !tab) return true - const url = new URL(result.url) - if ( - url.protocol !== 'https:' && - !(url.protocol === 'http:' && url.origin === window.location.origin) - ) - throw new Error('The provider authorization URL is invalid') - writeSearchConnectionAttempt(key, { ...next, connectorId: result.connectorId }) - tab.location.href = url.href - return true - } catch (error) { - tab?.close() - const message = getErrorMessage(error, 'Could not start the connection') - const current = readSearchConnectionAttempt(key) - if (next && current?.completionId === next.completionId && current.status === 'pending') - writeSearchConnectionAttempt(key, { ...current, status: 'failed', error: message }) - if (!controller.signal.aborted) setLocalError(message) - return false - } finally { - starting.current = false + controller.signal.throwIfAborted() + next = { + completionId: generateId(), + requestedAt: Date.now(), + status: 'pending', + error: null, } - }, - [isPending, connected, pending, refetch, mutateAsync, organizationId, target, connectorId, key] - ) + writeSearchConnectionAttempt(key, next) + const result = await mutateAsync({ + organizationId, + target, + oauthCompletionId: next.completionId, + signal: controller.signal, + }) + if (!result || !tab) return true + const url = new URL(result.url) + if ( + url.protocol !== 'https:' && + !(url.protocol === 'http:' && url.origin === window.location.origin) + ) + throw new Error('The provider authorization URL is invalid') + tab.location.href = url.href + return true + } catch (error) { + tab?.close() + const message = getErrorMessage(error, 'Could not start the connection') + const current = readSearchConnectionAttempt(key) + if (next && current?.completionId === next.completionId && current.status === 'pending') + writeSearchConnectionAttempt(key, { ...current, status: 'failed', error: message }) + if (!controller.signal.aborted) setLocalError(message) + return false + } finally { + starting.current = false + } + }, [isPending, connected, pending, refetch, mutateAsync, organizationId, target, key]) const cancel = useCallback(() => { nativeAbort.current?.abort() popup.current?.close() @@ -228,26 +217,11 @@ export function useSearchIntegrationConnection({ error: 'Connection canceled. You can try again.', }) }, [attempt, key]) - const completeSetup = useCallback( - (result: { connectorId: string; credentialId: string }) => { - writeSearchConnectionAttempt(key, { - completionId: generateId(), - requestedAt: Date.now(), - connectorId: result.connectorId, - credentialId: result.credentialId, - status: 'connected', - error: null, - }) - }, - [key] - ) return { - completeSetup, connect, cancel, inventoryError: query.error?.message, connected, - connectorId, pending, available, isStarting: isPending, diff --git a/apps/sim/lib/api/contracts/desktop-source-connect.ts b/apps/sim/lib/api/contracts/desktop-source-connect.ts index b18e749bbf2..c383303e362 100644 --- a/apps/sim/lib/api/contracts/desktop-source-connect.ts +++ b/apps/sim/lib/api/contracts/desktop-source-connect.ts @@ -1,6 +1,5 @@ import { z } from 'zod' import { startSlackCredentialGroupConfigurationBodySchema } from '@/lib/api/contracts/credential-groups' -import { connectSimSearchConnectorBodySchema } from '@/lib/api/contracts/knowledge/connectors' import { startGitHubSearchSetupBodySchema } from '@/lib/api/contracts/knowledge/github-setup' import { connectPersonalSearchIntegrationBodySchema } from '@/lib/api/contracts/knowledge/personal-integrations' import { knowledgeConnectorParamsSchema } from '@/lib/api/contracts/knowledge/shared' @@ -31,11 +30,6 @@ export const desktopSourceRequestSchema = z.discriminatedUnion('kind', [ params: knowledgeConnectorParamsSchema, completionId: z.string().uuid().optional(), }), - z.object({ - kind: z.literal('search-source'), - body: connectSimSearchConnectorBodySchema, - completionId: z.string().uuid().optional(), - }), z.object({ kind: z.literal('slack-managed-users'), owner: resourceOwnerSchema, diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index 962d101a731..91ac509ac86 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -8,11 +8,7 @@ import { knowledgeConnectorParamsSchema, successResponseSchema, } from '@/lib/api/contracts/knowledge/shared' -import { - booleanQueryFlagSchema, - organizationIdSchema, - resourceOwnerSchema, -} from '@/lib/api/contracts/primitives' +import { booleanQueryFlagSchema, resourceOwnerSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' import { CONNECTOR_ACCESS_MODES } from '@/lib/knowledge/connectors/access-modes' import { @@ -20,9 +16,6 @@ import { MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_MUTATION_ITEMS, MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_SEARCH_LENGTH, - MAX_SEARCH_SOURCE_PROGRESS_ITEMS, - MAX_SEARCH_SOURCE_PROVIDER_TYPES, - SEARCH_SOURCE_CANDIDATE_PAGE_SIZE, SEARCH_SOURCE_PAGE_SIZE, } from '@/lib/knowledge/constants' import { MEMBER_SYNC_STATUSES } from '@/lib/knowledge/types' @@ -350,41 +343,15 @@ const searchSourceSummaryFields = { availability: z.enum(['available', 'unavailable']), enabled: z.boolean(), approved: z.boolean().optional(), - isSyncing: z.boolean(), - lastSyncAt: z.string().datetime().nullable(), - hasSyncError: z.boolean(), - /** - * Whether the viewer can search at least one indexed document from this source. An - * existence flag rather than a count: counting means access-checking every visible document. - */ - hasViewerDocuments: z.boolean(), - viewerFailedDocumentCount: z.number().int().nonnegative().default(0), - viewerEmailVerified: z.boolean(), - viewerAccounts: z - .array( - z.object({ - credentialId: z.string().min(1).max(128), - displayName: z.string(), - status: z.enum(['active', 'needs_reauth']).optional(), - }) - ) - .max(SEARCH_SOURCE_CANDIDATE_PAGE_SIZE), } -export const searchSourceSummarySchema = z.discriminatedUnion('connectionRequired', [ - z.object({ - ...searchSourceSummaryFields, - connectionRequired: z.literal(true), - viewerMembership: viewerConnectorMembershipSchema.nullable(), - }), - z.object({ - ...searchSourceSummaryFields, - connectionRequired: z.literal(false), - viewerMembership: z.null(), - }), -]) +export const searchSourceSummarySchema = z.object(searchSourceSummaryFields) export type SearchSourceSummary = z.output -export type ViewerSearchSourceAccount = SearchSourceSummary['viewerAccounts'][number] +export interface ViewerSearchSourceAccount { + credentialId: string + displayName: string + status?: 'active' | 'needs_reauth' +} export const searchSourceCursorSchema = z.object({ createdAt: z.string().datetime(), @@ -402,7 +369,6 @@ export const listSearchSourcesQuerySchema = resourceOwnerSchema.safeExtend({ .max(100) .optional(), search: z.string().trim().max(200).optional(), - mine: booleanQueryFlagSchema.optional(), }) export type ListSearchSourcesQuery = z.input @@ -419,112 +385,6 @@ export const listSearchSourcesContract = defineRouteContract({ response: { mode: 'json', schema: successResponseSchema(searchSourcePageSchema) }, }) -export const searchSourceOverviewSchema = z.object({ - providers: z - .array( - z.object({ - connectorType: z.string().min(1).max(100), - isSyncing: z.boolean(), - }) - ) - .max(MAX_SEARCH_SOURCE_PROVIDER_TYPES), - hasSearchableDocuments: z.boolean(), -}) -export type SearchSourceOverview = z.output - -export const readSearchSourceOverviewContract = defineRouteContract({ - method: 'GET', - path: '/api/knowledge/sim-search/sources/overview', - query: resourceOwnerSchema, - response: { mode: 'json', schema: successResponseSchema(searchSourceOverviewSchema) }, -}) - -export const organizationSearchProviderStatusSchema = z.enum([ - 'needs_setup', - 'waiting_for_connections', - 'indexing', - 'needs_attention', - 'paused', - 'active', -]) -export type OrganizationSearchProviderStatus = z.output< - typeof organizationSearchProviderStatusSchema -> - -export const organizationSearchProviderSummarySchema = z.object({ - connectorType: z.string().min(1).max(100), - approved: z.boolean(), - sourceCount: z.number().int().nonnegative(), - status: organizationSearchProviderStatusSchema, - issue: z - .enum([ - 'sync_failed', - 'account_sync_incomplete', - 'document_indexing_failed', - 'permission_sync_incomplete', - ]) - .nullable(), - isSyncing: z.boolean(), - /** Older servers omit the continuation signal during a rolling deployment. */ - hasPendingSync: z.boolean().optional(), -}) -export type OrganizationSearchProviderSummary = z.output< - typeof organizationSearchProviderSummarySchema -> - -export const organizationSearchOverviewSchema = z.object({ - providers: z.array(organizationSearchProviderSummarySchema).max(MAX_SEARCH_SOURCE_PROVIDER_TYPES), -}) -export type OrganizationSearchOverview = z.output - -export const readOrganizationSearchOverviewQuerySchema = z.object({ - organizationId: organizationIdSchema, -}) -export type ReadOrganizationSearchOverviewQuery = z.input< - typeof readOrganizationSearchOverviewQuerySchema -> - -export const readOrganizationSearchOverviewContract = defineRouteContract({ - method: 'GET', - path: '/api/knowledge/sim-search/integrations/overview', - query: readOrganizationSearchOverviewQuerySchema, - response: { mode: 'json', schema: successResponseSchema(organizationSearchOverviewSchema) }, -}) - -export const searchSourceProgressSchema = z.object({ - connectorId: knowledgeConnectorParamsSchema.shape.connectorId, - isSyncing: z.boolean(), - hasSyncError: z.boolean(), - hasIndexingError: z.boolean(), -}) -export type SearchSourceProgress = z.output - -export const readSearchSourceProgressContract = defineRouteContract({ - method: 'POST', - path: '/api/knowledge/sim-search/sources/progress', - body: resourceOwnerSchema.safeExtend({ - connectorIds: z - .array(knowledgeConnectorParamsSchema.shape.connectorId.max(255)) - .min(1) - .max(MAX_SEARCH_SOURCE_PROGRESS_ITEMS), - }), - response: { - mode: 'json', - schema: successResponseSchema( - z.array(searchSourceProgressSchema).max(MAX_SEARCH_SOURCE_PROGRESS_ITEMS) - ), - }, -}) - -export const connectSimSearchConnectorBodySchema = resourceOwnerSchema.safeExtend({ - connectorType: z.string().min(1, 'connectorType cannot be empty').max(100), - connectorId: knowledgeConnectorParamsSchema.shape.connectorId.max(255).optional(), - /** Settings identify a compatible source, or assert the configuration of a selected source. */ - sourceConfig: z.record(z.string(), z.string().max(500)).optional(), - oauthCompletionId: searchConnectionOAuthQuerySchema.shape.oauthCompletionId, -}) -export type ConnectSimSearchConnectorBody = z.input - export const prepareSearchSourceBodySchema = resourceOwnerSchema.safeExtend({ connectorType: z.string().min(1, 'connectorType cannot be empty').max(100), accessMode: z.enum(['admin', 'members']).optional().default('admin'), @@ -547,28 +407,6 @@ export const prepareSearchSourceContract = defineRouteContract({ }, }) -/** - * One click on a Sim Search source: the workspace's Sim Search knowledge base - * and per-member connector exist afterwards, and the caller gets the link that - * connects their own account. - */ -export const connectSimSearchConnectorContract = defineRouteContract({ - method: 'POST', - path: '/api/knowledge/sim-search/connect', - body: connectSimSearchConnectorBodySchema, - response: { - mode: 'json', - schema: z.object({ - success: z.literal(true), - data: z.object({ - knowledgeBaseId: z.string(), - connectorId: z.string(), - url: z.string().url(), - }), - }), - }, -}) - export const deleteKnowledgeConnectorContract = defineRouteContract({ method: 'DELETE', path: '/api/knowledge/[id]/connectors/[connectorId]', diff --git a/apps/sim/lib/api/contracts/knowledge/mcp.ts b/apps/sim/lib/api/contracts/knowledge/mcp.ts index 285595da2ac..9a3d279959a 100644 --- a/apps/sim/lib/api/contracts/knowledge/mcp.ts +++ b/apps/sim/lib/api/contracts/knowledge/mcp.ts @@ -16,59 +16,11 @@ export const organizationKnowledgeMcpContract = defineRouteContract({ response: { mode: 'json', schema: mcpJsonRpcMessageSchema }, }) -const documentIdSchema = z.string().min(1, 'Document ID is required').max(255) - export const liveSearchMcpSchema = searchWorkspaceInputSchema.strict() export const readLiveDocumentMcpSchema = readDocumentInputSchema.strict() -export const searchMcpSchema = workspaceSearchFiltersSchema - .extend({ - query: z.string().trim().min(1, 'Search query is required').max(8192), - topK: z.number().int().min(1).max(50).default(10), - }) - .strict() - -export const readDocumentMcpSchema = z - .object({ - documentId: documentIdSchema.optional(), - url: z - .string() - .trim() - .url('Provide the original document URL') - .max(8192) - .refine((value) => { - if (!URL.canParse(value)) return false - const url = new URL(value) - return ['http:', 'https:'].includes(url.protocol) && !url.username && !url.password - }, 'Document URL must use HTTP or HTTPS without credentials') - .optional(), - limit: z.number().int().min(1).max(50).default(20), - offset: z.number().int().min(0).max(1_000_000).optional(), - aroundChunkIndex: z.number().int().min(0).max(1_000_000).optional(), - }) - .strict() - .superRefine((input, ctx) => { - if (Boolean(input.documentId) === Boolean(input.url)) { - ctx.addIssue({ - code: 'custom', - path: ['documentId'], - message: 'Provide either a document ID or its original URL', - }) - } - if (input.offset !== undefined && input.aroundChunkIndex !== undefined) { - ctx.addIssue({ - code: 'custom', - path: ['aroundChunkIndex'], - message: 'Use either an offset or a matching chunk index', - }) - } - }) - export const chatSearchMcpSchema = workspaceSearchFiltersSchema .extend({ query: z.string().trim().min(1, 'A question is required').max(8192), }) .strict() - -export type SearchMcpInput = z.input -export type ReadDocumentMcpInput = z.input diff --git a/apps/sim/lib/api/contracts/knowledge/personal-integrations.ts b/apps/sim/lib/api/contracts/knowledge/personal-integrations.ts index 1b2fa761bb7..16de7537189 100644 --- a/apps/sim/lib/api/contracts/knowledge/personal-integrations.ts +++ b/apps/sim/lib/api/contracts/knowledge/personal-integrations.ts @@ -8,8 +8,6 @@ export const personalSearchIntegrationSchema = z.object({ name: z.string().max(200), providerId: z.string().min(1).max(100), connectorType: z.string().min(1).max(100), - connectorId: z.string().min(1).max(200).optional(), - knowledgeBaseId: z.string().min(1).max(200).optional(), description: z.string().max(240), accounts: z .array( @@ -22,9 +20,6 @@ export const personalSearchIntegrationSchema = z.object({ ) .max(100), connectionStatus: z.enum(['connected', 'reconnect_needed', 'not_connected', 'unavailable']), - indexingStatus: z - .enum(['indexing', 'indexed', 'not_indexed', 'sync_failed', 'paused']) - .optional(), action: searchConnectionTargetSchema.nullable(), }) @@ -49,8 +44,6 @@ export const personalSearchIntegrationsQuerySchema = z.object({ completionId: z.string().uuid().optional(), organizationId: organizationIdSchema, connectorType: z.string().trim().min(1).max(100).optional(), - connectorId: z.string().min(1).max(200).optional(), - cursor: z.string().min(1).max(1024).optional(), }) export type PersonalSearchIntegrationsQuery = z.input @@ -65,10 +58,6 @@ export const connectPersonalSearchIntegrationBodySchema = z.object({ organizationId: organizationIdSchema, target: searchConnectionTargetSchema, oauthCompletionId: z.string().uuid(), - sourceConfig: z - .record(z.string().min(1).max(100), z.string().max(2000)) - .refine((config) => Object.keys(config).length <= 30, 'Too many source configuration fields') - .optional(), }) export type ConnectPersonalSearchIntegrationBody = z.input< typeof connectPersonalSearchIntegrationBodySchema @@ -82,8 +71,6 @@ export const connectPersonalSearchIntegrationContract = defineRouteContract({ schema: successResponseSchema( z.object({ url: z.string().url(), - connectorId: z.string().min(1).max(200).optional(), - knowledgeBaseId: z.string().min(1).max(200).optional(), }) ), }, diff --git a/apps/sim/lib/api/contracts/knowledge/personal-source-setup.test.ts b/apps/sim/lib/api/contracts/knowledge/personal-source-setup.test.ts deleted file mode 100644 index b0fcd945edf..00000000000 --- a/apps/sim/lib/api/contracts/knowledge/personal-source-setup.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { personalSourceSetupBodySchema } from '@/lib/api/contracts/knowledge/personal-source-setup' -import { executeSelectorBodySchema } from '@/lib/api/contracts/selectors/execute' - -const source = { - action: 'connect', - organizationId: 'organization-1', - connectorType: 'jira', - credentialId: 'account-1', - domain: 'example.atlassian.net', - keys: ['PROJECT'], -} - -describe('personal Search setup contracts', () => { - it('rejects arbitrary credential kinds, scope overrides and provider keys', () => { - for (const extra of [ - { workspaceId: 'workspace-1' }, - { selectorKey: 'jira.issues' }, - { accessMode: 'admin' }, - { connectorType: 'slack' }, - ]) { - expect(personalSourceSetupBodySchema.safeParse({ ...source, ...extra }).success).toBe(false) - } - }) - it('does not expose the trusted personal browsing marker through the generic selector API', () => { - expect( - executeSelectorBodySchema.safeParse({ - selectorKey: 'jira.projectKeys', - scope: { kind: 'organization', organizationId: 'organization-1' }, - context: { oauthCredential: 'account-1', domain: 'example.atlassian.net' }, - request: { kind: 'list' }, - personalSearchSetup: 'jira', - }).success - ).toBe(false) - }) -}) diff --git a/apps/sim/lib/api/contracts/knowledge/personal-source-setup.ts b/apps/sim/lib/api/contracts/knowledge/personal-source-setup.ts deleted file mode 100644 index c3e64253078..00000000000 --- a/apps/sim/lib/api/contracts/knowledge/personal-source-setup.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { z } from 'zod' -import { defineRouteContract } from '@/lib/api/contracts' -import { successResponseSchema } from '@/lib/api/contracts/knowledge/shared' -import { organizationIdSchema } from '@/lib/api/contracts/primitives' -import { - executeSelectorResponseSchema, - selectorRequestSchema, -} from '@/lib/api/contracts/selectors/execute' -import { MAX_PERSONAL_SOURCE_SETUP_KEYS } from '@/lib/sim-search/personal-source-setup' - -const setupOwnerSchema = z.object({ - organizationId: organizationIdSchema, - connectorType: z.enum(['jira', 'confluence']), -}) -const setupCredentialSchema = setupOwnerSchema.extend({ - credentialId: z.string().min(1).max(128), - domain: z.string().trim().min(1, 'Enter your Atlassian site').max(253), -}) - -export const personalSourceSetupQuerySchema = setupOwnerSchema.extend({ - completionId: z.string().uuid().optional(), -}) -export type PersonalSourceSetupQuery = z.input - -export const personalSourceSetupAccountsSchema = z.object({ - accounts: z - .array( - z.object({ - id: z.string().min(1).max(128), - name: z.string().max(512), - provider: z.enum(['jira', 'confluence']), - type: z.literal('managed_oauth'), - scopes: z.array(z.string().max(200)).max(200), - }) - ) - .max(1000), - completedCredentialId: z.string().min(1).max(128).nullable(), -}) -export type PersonalSourceSetupAccounts = z.output - -export const personalSourceSetupBodySchema = z.discriminatedUnion('action', [ - setupOwnerSchema - .extend({ action: z.literal('authorize'), oauthCompletionId: z.string().uuid() }) - .strict(), - setupCredentialSchema - .extend({ - action: z.literal('connect'), - keys: z - .array(z.string().trim().min(1).max(255)) - .min(1, 'Select at least one project or space') - .max( - MAX_PERSONAL_SOURCE_SETUP_KEYS, - 'Choose no more than 1,000 projects or spaces per source' - ), - }) - .strict(), - setupCredentialSchema - .extend({ action: z.literal('options'), request: selectorRequestSchema }) - .strict(), -]) -export type PersonalSourceSetupBody = z.input - -export const personalSourceSetupResultSchema = z.discriminatedUnion('kind', [ - z.object({ kind: z.literal('authorization'), url: z.string().url() }), - z.object({ - kind: z.literal('connected'), - knowledgeBaseId: z.string().min(1).max(200), - connectorId: z.string().min(1).max(200), - }), - ...executeSelectorResponseSchema.options, -]) -export type PersonalSourceSetupResult = z.output - -export const listPersonalSourceSetupAccountsContract = defineRouteContract({ - method: 'GET', - path: '/api/knowledge/sim-search/personal-source-setup', - query: personalSourceSetupQuerySchema, - response: { mode: 'json', schema: successResponseSchema(personalSourceSetupAccountsSchema) }, -}) - -export const personalSourceSetupContract = defineRouteContract({ - method: 'POST', - path: '/api/knowledge/sim-search/personal-source-setup', - body: personalSourceSetupBodySchema, - response: { mode: 'json', schema: successResponseSchema(personalSourceSetupResultSchema) }, -}) diff --git a/apps/sim/lib/api/contracts/knowledge/search-stats.ts b/apps/sim/lib/api/contracts/knowledge/search-stats.ts deleted file mode 100644 index 63f30e2ec8f..00000000000 --- a/apps/sim/lib/api/contracts/knowledge/search-stats.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { z } from 'zod' -import { organizationIdSchema } from '@/lib/api/contracts/primitives' -import { defineRouteContract } from '@/lib/api/contracts/types' -import { - getSearchStatsRangeError, - SEARCH_STATS_MAX_DAYS, - SEARCH_STATS_PEOPLE_LIMIT, - SEARCH_STATS_PERIODS, - SEARCH_STATS_SOURCE_LIMIT, - SEARCH_STATS_SURFACES, -} from '@/lib/knowledge/search/stats' - -export const organizationSearchStatsQuerySchema = z - .object({ - organizationId: organizationIdSchema, - period: z.enum(SEARCH_STATS_PERIODS).default('30d'), - surface: z.enum(SEARCH_STATS_SURFACES).optional(), - startDate: z.string().max(10).optional(), - endDate: z.string().max(10).optional(), - }) - .superRefine((query, context) => { - if (query.period === 'custom') { - const error = getSearchStatsRangeError(query) - if (error) context.addIssue({ code: 'custom', path: ['startDate'], message: error }) - } else if (query.startDate !== undefined || query.endDate !== undefined) { - context.addIssue({ - code: 'custom', - path: ['period'], - message: 'Choose Custom range to use start and end dates.', - }) - } - }) -export type OrganizationSearchStatsQuery = z.input - -const countSchema = z.number().int().nonnegative() -const sourceTypeSchema = z.string().min(1).max(100) -export const organizationSearchStatsSchema = z.object({ - start: z.string().datetime(), - end: z.string().datetime(), - totals: z.object({ - invocations: countSchema, - activePeople: countSchema, - results: countSchema, - }), - series: z - .array(z.object({ timestamp: z.string().datetime(), invocations: countSchema })) - .max(SEARCH_STATS_MAX_DAYS), - surfaces: z - .array(z.object({ surface: z.enum(SEARCH_STATS_SURFACES), invocations: countSchema })) - .max(7), - sources: z - .array(z.object({ sourceType: sourceTypeSchema, invocations: countSchema })) - .max(SEARCH_STATS_SOURCE_LIMIT), - people: z - .array( - z.object({ - userId: z.string().nullable(), - name: z.string().nullable(), - email: z.string().nullable(), - invocations: countSchema, - sourceTypes: z.array(sourceTypeSchema).max(SEARCH_STATS_SOURCE_LIMIT), - }) - ) - .max(SEARCH_STATS_PEOPLE_LIMIT), -}) -export type OrganizationSearchStats = z.output - -export const readOrganizationSearchStatsContract = defineRouteContract({ - method: 'GET', - path: '/api/knowledge/sim-search/stats', - query: organizationSearchStatsQuerySchema, - response: { - mode: 'json', - schema: z.object({ success: z.literal(true), data: organizationSearchStatsSchema }), - }, -}) diff --git a/apps/sim/lib/api/contracts/mothership-search-sources.ts b/apps/sim/lib/api/contracts/mothership-search-sources.ts index 573cb1d3179..40bbdd611a0 100644 --- a/apps/sim/lib/api/contracts/mothership-search-sources.ts +++ b/apps/sim/lib/api/contracts/mothership-search-sources.ts @@ -17,7 +17,6 @@ export const organizationSearchSourcesInputSchema = z.discriminatedUnion('action cursor: z.string().min(1).max(1024).optional(), connectorType: connectorTypeSchema.optional(), search: z.string().trim().max(200).optional(), - mine: z.boolean().optional(), }), z.strictObject({ action: z.literal('get'), connectorId: z.string().min(1).max(255) }), z.strictObject({ action: z.literal('providers') }), diff --git a/apps/sim/lib/api/contracts/organization-accounts.ts b/apps/sim/lib/api/contracts/organization-accounts.ts index 8fd59a09d5b..e97824502f5 100644 --- a/apps/sim/lib/api/contracts/organization-accounts.ts +++ b/apps/sim/lib/api/contracts/organization-accounts.ts @@ -18,7 +18,6 @@ import { organizationIdSchema, workspaceIdSchema } from '@/lib/api/contracts/pri import { defineRouteContract } from '@/lib/api/contracts/types' import { ORGANIZATION_CREDENTIAL_TYPES } from '@/lib/credential-groups/credential-types' import { - ORGANIZATION_ACCOUNT_INDEXING_SOURCE_LIMIT, ORGANIZATION_ACCOUNT_WORKSPACE_LIMIT, ORGANIZATION_VIEWER_ACCOUNT_LIMIT, } from '@/lib/credential-groups/limits' @@ -140,31 +139,6 @@ export type OrganizationAccountsSettings = z.output< typeof getOrganizationAccountsContract.response.schema > -export const updateOrganizationAccountIndexingBodySchema = z - .object({ - optionId: z.string().min(1, 'Provider option is required').max(128), - enabled: z.boolean(), - }) - .strict() - -export const updateOrganizationAccountIndexingContract = defineRouteContract({ - method: 'PUT', - path: '/api/organizations/[id]/connected-accounts/indexing', - params: organizationAccountsParamsSchema, - body: updateOrganizationAccountIndexingBodySchema, - response: { - mode: 'json', - schema: z.object({ - enabled: z.boolean(), - knowledgeBaseIds: z - .array(z.string().min(1).max(128)) - .max(ORGANIZATION_ACCOUNT_INDEXING_SOURCE_LIMIT), - }), - }, -}) -export type UpdateOrganizationAccountIndexingBody = z.input< - NonNullable -> export type EnsureOrganizationAccountsBody = z.input< NonNullable > diff --git a/apps/sim/lib/api/contracts/workspaces.ts b/apps/sim/lib/api/contracts/workspaces.ts index 1ddf9880495..b7f03ef2826 100644 --- a/apps/sim/lib/api/contracts/workspaces.ts +++ b/apps/sim/lib/api/contracts/workspaces.ts @@ -214,7 +214,6 @@ export type WorkspaceOwnerBilling = z.output * these only off-hosted, where no subscription plan exists to decide entitlement. */ export const deploymentFeaturesSchema = z.object({ - liveEnterpriseSearch: z.boolean().optional(), accessControl: z.boolean(), auditLogs: z.boolean(), customBlocks: z.boolean(), diff --git a/apps/sim/lib/core/config/deployment-shape.ts b/apps/sim/lib/core/config/deployment-shape.ts index 0d31ab553d1..5d415b26020 100644 --- a/apps/sim/lib/core/config/deployment-shape.ts +++ b/apps/sim/lib/core/config/deployment-shape.ts @@ -13,7 +13,6 @@ import { isDataRetentionEnabled, isHosted, isInboxEnabled, - isLiveEnterpriseSearchEnabled, isSandboxesEnabled, isScimEnabled, isSessionPoliciesEnabled, @@ -89,7 +88,6 @@ export function resolveDeploymentShape(): DeploymentShape { azureConfigured: isAzureConfigured, cohereConfigured: isCohereConfigured, features: { - liveEnterpriseSearch: isLiveEnterpriseSearchEnabled, accessControl: isAccessControlEnabled, auditLogs: isAuditLogsEnabled, customBlocks: isCustomBlocksEnabled, diff --git a/apps/sim/lib/core/config/env-flags.ts b/apps/sim/lib/core/config/env-flags.ts index c44c19adb93..9ad4f597f0d 100644 --- a/apps/sim/lib/core/config/env-flags.ts +++ b/apps/sim/lib/core/config/env-flags.ts @@ -722,7 +722,3 @@ export function getCostMultiplier(): number { } /** Backend selector. Kept independent of enterprise entitlement overrides. */ -const liveEnterpriseSearchSetting = - typeof window === 'undefined' ? env.SIM_SEARCH_LIVE : getEnv('NEXT_PUBLIC_SIM_SEARCH_LIVE') -export const isLiveEnterpriseSearchEnabled = - liveEnterpriseSearchSetting === undefined || isTruthy(liveEnterpriseSearchSetting) diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index eb3975a08f8..7f232d483bb 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -600,7 +600,6 @@ export const env = createEnv({ MSHIP_PLAN_MODE: z.boolean().optional(), DASHBOARDS: z.boolean().optional(), MSHIP_MODEL_SELECTOR: z.boolean().optional(), - SIM_SEARCH_LIVE: z.boolean().optional(), // Query connected providers directly; false preserves indexed search INBOX_ENABLED: z.boolean().optional(), // Enable inbox (Sim Mailer) on self-hosted (bypasses hosted requirements) SANDBOXES_ENABLED: z.boolean().optional(), // Enable custom sandboxes on self-hosted (bypasses hosted requirements) @@ -766,7 +765,6 @@ export const env = createEnv({ NEXT_PUBLIC_ORGANIZATIONS_ENABLED: z.boolean().optional(), // Enable organizations on self-hosted (bypasses plan requirements) NEXT_PUBLIC_DISABLE_INVITATIONS: z.boolean().optional(), // Disable workspace invitations globally (for self-hosted deployments) NEXT_PUBLIC_DISABLE_PUBLIC_API: z.boolean().optional(), // Disable public API access UI toggle globally - NEXT_PUBLIC_SIM_SEARCH_LIVE: z.boolean().optional(), NEXT_PUBLIC_INBOX_ENABLED: z.boolean().optional(), // Enable inbox (Sim Mailer) on self-hosted NEXT_PUBLIC_CHAT_DISABLED: z.boolean().optional(), // Hide the Chat module (Chat is shown when unset) NEXT_PUBLIC_STATUS_NOTICE_PREVIEW: z.boolean().optional(), // Force the sidebar service-status notice into its critical preview state @@ -814,7 +812,6 @@ export const env = createEnv({ NEXT_PUBLIC_ORGANIZATIONS_ENABLED: process.env.NEXT_PUBLIC_ORGANIZATIONS_ENABLED, NEXT_PUBLIC_DISABLE_INVITATIONS: process.env.NEXT_PUBLIC_DISABLE_INVITATIONS, NEXT_PUBLIC_DISABLE_PUBLIC_API: process.env.NEXT_PUBLIC_DISABLE_PUBLIC_API, - NEXT_PUBLIC_SIM_SEARCH_LIVE: process.env.NEXT_PUBLIC_SIM_SEARCH_LIVE, NEXT_PUBLIC_INBOX_ENABLED: process.env.NEXT_PUBLIC_INBOX_ENABLED, NEXT_PUBLIC_CHAT_DISABLED: process.env.NEXT_PUBLIC_CHAT_DISABLED, NEXT_PUBLIC_STATUS_NOTICE_PREVIEW: process.env.NEXT_PUBLIC_STATUS_NOTICE_PREVIEW, diff --git a/apps/sim/lib/credential-groups/README.md b/apps/sim/lib/credential-groups/README.md index 5c46c85faf0..4f9eba0719c 100644 --- a/apps/sim/lib/credential-groups/README.md +++ b/apps/sim/lib/credential-groups/README.md @@ -37,7 +37,7 @@ The Providers tab lists only added providers. **Add provider** opens a searchabl - Databricks: **Add** collects and validates the organization's tenant MCP URL and registered OAuth client before creating an enabled provider in one transaction. Cancelling leaves nothing added. Organization owners and admins create it through `POST /api/organizations/[id]/connected-accounts/mcp-providers` and read or edit settings through `GET` / `PUT /api/organizations/[id]/connected-accounts/databricks`; no workspace configuration is used. Unfinished entries from the earlier flow appear in the Add catalog until their configuration is saved. The form never reads stored client secrets, and leaving the secret blank when editing preserves it. Endpoint/client identity changes invalidate affected grants and pending attempts, requiring people to reconnect. - Slack personal OAuth: the org admin supplies App ID, Slack workspace ID, client ID, and client secret, then verifies authorization. The org has one configured app/workspace. Existing workspace bots and their triggers remain separate. -Search availability and mode are managed through **Organization settings → Sources**. Member mode has no resource filters; service mode uses the configured source’s resource boundary. Live search/read adapters are registered under `lib/sim-search/live/`; they resolve only the acting member’s current grants. OAuth completion may invoke the shared dispatch helper, but live Search sources are rejected by queue and worker guards before indexing. Ordinary workspace KB sources and the explicit `SIM_SEARCH_LIVE=false` backend retain ingestion behavior. +Search availability and mode are managed through **Organization settings → Sources**. Member mode has no resource filters; service mode uses the configured source’s resource boundary. Live search/read adapters are registered under `lib/sim-search/live/`; they resolve only the acting member’s current grants. OAuth completion may invoke the shared dispatch helper, but live Search sources are rejected by queue and worker guards before indexing. Ordinary workspace KB sources retain ingestion behavior. Search requires both `CREDENTIAL_GROUPS` and `KNOWLEDGE_MEMBER_ACCESS` locally; Credential Groups alone requires only its own flag. Hosted deployments additionally enforce the routed org's feature rules and Enterprise availability. Owners and admins manage Search sources; existing Knowledge permission-group rules still apply. Managed MCP account connections remain available for live tool calls only and have no indexing switch. Separate API-key KB connectors for Fireflies, Granola, and Databricks do not consume these managed MCP connections. @@ -48,7 +48,7 @@ Search requires both `CREDENTIAL_GROUPS` and `KNOWLEDGE_MEMBER_ACCESS` locally; | Credential Groups settings page | `credential-groups` enabled, independently of Search | | Provider setup APIs, personal contributions, workspace access to the pool | `credential-groups` | | Organization Home/Assistant and chat pages, Sources, member Integrations, Search MCP settings | `credential-groups` and `knowledge-member-access` | -| Legacy Search content/member sync and persisted directory maintenance | Search features above and explicit `SIM_SEARCH_LIVE=false` | + | Organization Search MCP endpoint and organization knowledge search through internal/public APIs or trusted tools | `credential-groups` and `knowledge-member-access`, checked after current authorization | The Search gate uses the persisted knowledge base owner or the authenticated route's target organization. User, platform-admin, and workspace targeting cannot opt a different organization into Search. Disabled organizations receive `403 Search is not enabled for this organization` before index lookup or model execution; hiding navigation is not the authorization boundary. Organization Home, Search, and chat URLs open full settings in the viewer's most recent accessible workspace when Search is disabled. Default app entry uses that same destination, and Home, Integrations, chat history, and Assistant loading UI are hidden. Connected accounts settings and Workspaces remain available. Settings and source-setup URLs also enforce their gates. Ordinary workspace knowledge search keeps its existing behavior. The legacy indexed surface retains its pause controls when that backend is selected; live Sources does not expose indexing controls. diff --git a/apps/sim/lib/credential-groups/application/organization-account-indexing.test.ts b/apps/sim/lib/credential-groups/application/organization-account-indexing.test.ts deleted file mode 100644 index ea2d7468736..00000000000 --- a/apps/sim/lib/credential-groups/application/organization-account-indexing.test.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { auditMock, auditMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' -import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' -import { - credentialGroupsAvailabilityMock, - credentialGroupsAvailabilityMockFns, -} from '@sim/testing/mocks/credential-groups-availability.mock' -import { - credentialGroupsCredentialsMock, - credentialGroupsCredentialsMockFns, -} from '@sim/testing/mocks/credential-groups-credentials.mock' -import { credentialGroupsOrganizationSetupMock } from '@sim/testing/mocks/credential-groups-organization-setup.mock' -import { credentialGroupsSelfEnrollmentMock } from '@sim/testing/mocks/credential-groups-self-enrollment.mock' -import { credentialGroupsServiceMock } from '@sim/testing/mocks/credential-groups-service.mock' -import { - knowledgeAvailabilityMock, - knowledgeAvailabilityMockFns, -} from '@sim/testing/mocks/knowledge-availability.mock' -import { - knowledgeMemberQueueMock, - knowledgeMemberQueueMockFns, -} from '@sim/testing/mocks/knowledge-member-queue.mock' -import { permissionGroupsResolveMock } from '@sim/testing/mocks/permission-groups-resolve.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const hoisted = vi.hoisted(() => ({ - setIndexing: vi.fn(), -})) -vi.mock('@sim/audit', () => auditMock) -vi.mock('@/lib/credential-groups/scoped-availability', () => credentialGroupsAvailabilityMock) -vi.mock('@/lib/credential-groups/credentials', () => credentialGroupsCredentialsMock) -vi.mock('@/lib/credential-groups/organization-setup', () => credentialGroupsOrganizationSetupMock) -vi.mock('@/lib/permission-groups/resolve.server', () => permissionGroupsResolveMock) -vi.mock('@/lib/credential-groups/service', () => credentialGroupsServiceMock) -vi.mock('@/lib/credential-groups/provider-availability', () => ({ - listConfiguredCredentialGroupProviders: vi.fn(), -})) -vi.mock('@/lib/credential-groups/self-enrollment', () => credentialGroupsSelfEnrollmentMock) -vi.mock('@/lib/credential-groups/managed-mcp-service', () => ({ - ManagedMcpConnectorError: class extends Error {}, -})) -vi.mock('@/lib/knowledge/access/availability', () => knowledgeAvailabilityMock) -vi.mock('@/lib/knowledge/connectors/organization-account-indexing', () => ({ - setOrganizationAccountIndexing: hoisted.setIndexing, -})) -vi.mock('@/lib/knowledge/connectors/member-queue', () => knowledgeMemberQueueMock) - -import { updateOrganizationAccountIndexing } from '@/lib/credential-groups/application/organization-account-indexing' - -const mocks = { - ...hoisted, - available: credentialGroupsAvailabilityMockFns.mockIsScopedCredentialGroupsAvailable, - group: credentialGroupsCredentialsMockFns.mockLoadScopedAccountsCredentialListContext, - dispatch: knowledgeMemberQueueMockFns.mockDispatchMemberSyncsForCredentialOption, -} - -const feature = knowledgeAvailabilityMockFns.mockRequireKnowledgeMemberAccessAvailable -const principal = createSessionPrincipal({ userId: 'admin-1' }) -const input = { organizationId: 'org-1', optionId: 'option-1', enabled: true } - -describe('organization account indexing authorization', () => { - beforeEach(() => { - resetDbChainMock() - mocks.available.mockResolvedValue(true) - mocks.group.mockResolvedValue({ credentialGroupId: 'group-1' }) - feature.mockResolvedValue(undefined) - mocks.setIndexing.mockResolvedValue({ - enabled: true, - changed: true, - providerName: 'Gmail', - knowledgeBaseIds: ['kb-1'], - }) - }) - it.each(['member', null])('denies a %s before reading account data', async (role) => { - queueTableRows(schemaMock.member, role ? [{ role }] : []) - await expect(updateOrganizationAccountIndexing.execute({ principal, input })).rejects.toThrow() - expect(mocks.group).not.toHaveBeenCalled() - expect(mocks.setIndexing).not.toHaveBeenCalled() - }) - it('allows an org admin and dispatches only that organization option', async () => { - queueTableRows(schemaMock.member, [{ role: 'admin' }]) - await updateOrganizationAccountIndexing.execute({ principal, input }) - expect(mocks.setIndexing).toHaveBeenCalledWith({ ...input, credentialGroupId: 'group-1' }) - expect(mocks.dispatch).toHaveBeenCalledWith({ - organizationId: 'org-1', - credentialGroupOptionId: 'option-1', - }) - expect(auditMockFns.mockRecordAudit).toHaveBeenCalledWith( - expect.objectContaining({ - actorId: 'admin-1', - metadata: { - organizationId: 'org-1', - operation: 'organization_accounts.indexing.update', - actor: { kind: 'session', userId: 'admin-1' }, - }, - }) - ) - }) - it('requires indexing availability to enable a source', async () => { - queueTableRows(schemaMock.member, [{ role: 'admin' }]) - feature.mockRejectedValue(new Error('Search is not enabled')) - await expect(updateOrganizationAccountIndexing.execute({ principal, input })).rejects.toThrow( - 'Search is not enabled' - ) - expect(mocks.setIndexing).not.toHaveBeenCalled() - expect(mocks.dispatch).not.toHaveBeenCalled() - }) - it('allows pausing when the Search feature is off and does not dispatch', async () => { - queueTableRows(schemaMock.member, [{ role: 'admin' }]) - mocks.setIndexing.mockResolvedValue({ - enabled: false, - changed: true, - providerName: 'Gmail', - knowledgeBaseIds: ['kb-1'], - }) - await updateOrganizationAccountIndexing.execute({ - principal, - input: { ...input, enabled: false }, - }) - expect(feature).not.toHaveBeenCalled() - expect(mocks.dispatch).not.toHaveBeenCalled() - }) - it('does not audit or redispatch an unchanged setting', async () => { - queueTableRows(schemaMock.member, [{ role: 'admin' }]) - mocks.setIndexing.mockResolvedValue({ - enabled: true, - changed: false, - providerName: 'Gmail', - knowledgeBaseIds: ['kb-1'], - }) - await updateOrganizationAccountIndexing.execute({ principal, input }) - expect(auditMockFns.mockRecordAudit).not.toHaveBeenCalled() - expect(mocks.dispatch).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/credential-groups/application/organization-account-indexing.ts b/apps/sim/lib/credential-groups/application/organization-account-indexing.ts deleted file mode 100644 index fb346f46e83..00000000000 --- a/apps/sim/lib/credential-groups/application/organization-account-indexing.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { OrganizationMembershipContext } from '@/lib/core/application/organization-authorization' -import { defineOrganizationOperation } from '@/lib/core/application/organization-operation' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { defineOrganizationAccountsUseCase } from '@/lib/credential-groups/application/organization-accounts' -import { loadScopedAccountsCredentialListContext } from '@/lib/credential-groups/credentials' -import { requireKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' -import { dispatchMemberSyncsForCredentialOption } from '@/lib/knowledge/connectors/member-queue' -import { setOrganizationAccountIndexing } from '@/lib/knowledge/connectors/organization-account-indexing' - -export const updateOrganizationAccountIndexingOperation = defineOrganizationOperation({ - id: 'organization_accounts.indexing.update', - minimumRole: 'admin', - principalKinds: ['session', 'organization_delegated'], - delegationAudience: 'sim:settings', - delegatedServices: ['copilot'], - capability: 'knowledge.use', -}) - -export const updateOrganizationAccountIndexing = defineOrganizationAccountsUseCase({ - operation: updateOrganizationAccountIndexingOperation, - async execute({ - input, - context, - }: { - input: { organizationId: string; optionId: string; enabled: boolean } - context: OrganizationMembershipContext - }) { - if (input.enabled) - await requireKnowledgeMemberAccessAvailable({ organizationId: context.organizationId }) - const group = await loadScopedAccountsCredentialListContext({ - kind: 'organization', - organizationId: context.organizationId, - }) - if (!group) - throw new OrchestrationError( - 'not_found', - 'Organization connected accounts are not configured' - ) - const result = await setOrganizationAccountIndexing({ - organizationId: context.organizationId, - credentialGroupId: group.credentialGroupId, - optionId: input.optionId, - enabled: input.enabled, - }) - return { ...result, credentialGroupId: group.credentialGroupId, optionId: input.optionId } - }, - projectAudit: (result) => - result.changed - ? { - resourceId: result.credentialGroupId, - resourceName: 'Connected accounts', - description: `${result.enabled ? 'Enabled' : 'Paused'} ${result.providerName} indexing`, - } - : null, - async afterSuccess({ context, result }) { - if (result.enabled && result.changed) - await dispatchMemberSyncsForCredentialOption({ - organizationId: context.organizationId, - credentialGroupOptionId: result.optionId, - }) - }, -}) diff --git a/apps/sim/lib/credential-groups/application/organization-settings-delegation.test.ts b/apps/sim/lib/credential-groups/application/organization-settings-delegation.test.ts index 2ea84440626..76abae115ad 100644 --- a/apps/sim/lib/credential-groups/application/organization-settings-delegation.test.ts +++ b/apps/sim/lib/credential-groups/application/organization-settings-delegation.test.ts @@ -11,7 +11,6 @@ vi.mock('@/lib/permission-groups/resolve.server', () => permissionGroupsResolveM import { authorizeOrganizationOperation } from '@/lib/core/application/organization-authorization' import { organizationAccountAccessOperations } from '@/lib/credential-groups/application/organization-access' -import { updateOrganizationAccountIndexingOperation } from '@/lib/credential-groups/application/organization-account-indexing' import { organizationAccountManagementOperations } from '@/lib/credential-groups/application/organization-account-management' import { organizationAccountOperations } from '@/lib/credential-groups/application/organization-accounts' @@ -19,7 +18,6 @@ const operations = [ ...Object.values(organizationAccountOperations), ...Object.values(organizationAccountAccessOperations), ...Object.values(organizationAccountManagementOperations), - updateOrganizationAccountIndexingOperation, ] const principal: OrganizationDelegatedPrincipal = { kind: 'organization_delegated', diff --git a/apps/sim/lib/credential-groups/self-enrollment-oauth.ts b/apps/sim/lib/credential-groups/self-enrollment-oauth.ts index 79e474798c0..fe3265b7306 100644 --- a/apps/sim/lib/credential-groups/self-enrollment-oauth.ts +++ b/apps/sim/lib/credential-groups/self-enrollment-oauth.ts @@ -8,7 +8,7 @@ import { startCredentialGroupOAuth } from '@/lib/credential-groups/oauth' import type { CredentialGroupConnectionIntent } from '@/lib/credential-groups/oauth-intent' import { createViewerCredentialGroupEnrollment } from '@/lib/credential-groups/self-enrollment' -/** Starts a user-initiated connection with the same scoped receipt for live and indexed Search. */ +/** Starts a user-initiated connection with a scoped receipt for Search and knowledge-base enrollment. */ export async function startViewerCredentialGroupOAuth(input: { userId: string organizationId?: string diff --git a/apps/sim/lib/credential-groups/slack-provider.test.ts b/apps/sim/lib/credential-groups/slack-provider.test.ts index 51da5603c64..e8d0a6b0b23 100644 --- a/apps/sim/lib/credential-groups/slack-provider.test.ts +++ b/apps/sim/lib/credential-groups/slack-provider.test.ts @@ -1,5 +1,4 @@ import type { CredentialGroupOptionConfig } from '@sim/db/schema' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' import { resetUrlsMock, urlsMockFns } from '@sim/testing/mocks/urls.mock' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' @@ -34,7 +33,6 @@ afterAll(resetUrlsMock) describe('Slack member scope policy', () => { beforeEach(() => { - resetEnvFlagsMock() mocks.configuration.mockResolvedValue({ slackBotCredentialId: 'bot-1', clientId: 'client', @@ -80,45 +78,32 @@ describe('Slack member scope policy', () => { } } - it('requests RTS consent only with live search enabled while retaining the stored policy', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) - const current = context(SLACK_SEARCH_USER_SCOPES) - const policy = await adapter.getPolicy(current.option, { - workspaceId: current.workspaceId, - credentialGroupId: current.credentialGroupId, - }) - const authorization = await adapter.prepareAuthorization(current, policy) - const url = new URL( - await authorization.buildAuthorizationUrl({ state: 'state', nonce: 'nonce' }) - ) - expect(url.searchParams.get('user_scope')?.split(',')).toEqual( - expect.arrayContaining([ - 'search:read.public', - 'search:read.private', - 'search:read.im', - 'search:read.mpim', - 'search:read.files', - 'files:read', - ]) - ) - expect(policy.requiredScopes).toEqual([...SLACK_SEARCH_USER_SCOPES]) - }) - - it('uses the option policy for enrollment instead of widening it', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) - const scopes = SLACK_MANAGED_USER_SCOPES - const current = context(scopes) - const policy = await adapter.getPolicy(current.option, { - workspaceId: current.workspaceId, - credentialGroupId: current.credentialGroupId, - }) - expect(policy.requiredScopes).toEqual([...scopes]) - const authorization = await adapter.prepareAuthorization(current, policy) - const url = new URL( - await authorization.buildAuthorizationUrl({ state: 'state', nonce: 'nonce' }) - ) - expect(url.searchParams.get('user_scope')?.split(',')).toEqual([...scopes]) - }) + it.each([{ scopes: SLACK_SEARCH_USER_SCOPES }, { scopes: SLACK_MANAGED_USER_SCOPES }])( + 'requests RTS consent while retaining the stored option policy', + async ({ scopes }) => { + const current = context(scopes) + const policy = await adapter.getPolicy(current.option, { + workspaceId: current.workspaceId, + credentialGroupId: current.credentialGroupId, + }) + const authorization = await adapter.prepareAuthorization(current, policy) + const url = new URL( + await authorization.buildAuthorizationUrl({ state: 'state', nonce: 'nonce' }) + ) + expect(url.searchParams.get('user_scope')?.split(',')).toEqual( + expect.arrayContaining([ + ...scopes, + 'search:read.public', + 'search:read.private', + 'search:read.im', + 'search:read.mpim', + 'search:read.files', + 'files:read', + ]) + ) + expect(policy.requiredScopes).toEqual([...scopes]) + } + ) it('accepts a different provider email', async () => { const scopes = SLACK_SEARCH_USER_SCOPES diff --git a/apps/sim/lib/credential-groups/slack-provider.ts b/apps/sim/lib/credential-groups/slack-provider.ts index 51b0405af9b..8acc6001224 100644 --- a/apps/sim/lib/credential-groups/slack-provider.ts +++ b/apps/sim/lib/credential-groups/slack-provider.ts @@ -2,7 +2,6 @@ import { db } from '@sim/db' import { credentialGroup } from '@sim/db/schema' import { normalizeEmail } from '@sim/utils/string' import { and, eq } from 'drizzle-orm' -import { isLiveEnterpriseSearchEnabled } from '@/lib/core/config/env-flags' import { resourceScopeColumns, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { getBaseUrl } from '@/lib/core/utils/urls' @@ -183,12 +182,7 @@ export const slackCredentialGroupProviderAdapter: CredentialGroupProviderAdapter authorizationUrl.searchParams.set('client_id', currentPolicy.clientId) authorizationUrl.searchParams.set( 'user_scope', - [ - ...new Set([ - ...policy.requiredScopes, - ...(isLiveEnterpriseSearchEnabled ? SLACK_RTS_USER_SCOPES : []), - ]), - ].join(',') + [...new Set([...policy.requiredScopes, ...SLACK_RTS_USER_SCOPES])].join(',') ) authorizationUrl.searchParams.set('redirect_uri', redirectUri) authorizationUrl.searchParams.set('state', state) diff --git a/apps/sim/lib/credentials/application/resolve-organization-personal-token.test.ts b/apps/sim/lib/credentials/application/resolve-organization-personal-token.test.ts index 4cf02aca8b1..1dc2ae6dfba 100644 --- a/apps/sim/lib/credentials/application/resolve-organization-personal-token.test.ts +++ b/apps/sim/lib/credentials/application/resolve-organization-personal-token.test.ts @@ -11,7 +11,7 @@ import { credentialsManagedOauthMock, credentialsManagedOauthMockFns, } from '@sim/testing/mocks/credentials-managed-oauth.mock' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' +import { resetEnvFlagsMock } from '@sim/testing/mocks/env-flags.mock' import { knowledgeSearchIntegrationPolicyMock, knowledgeSearchIntegrationPolicyMockFns, @@ -55,12 +55,6 @@ vi.mock('@/lib/sim-search/connectors', () => ({ ], })) vi.mock('@/lib/oauth/utils', () => oauthUtilsMock) -/** The barrel's other use cases need the application layer mocked above; ownership is exercised as is. */ -vi.mock('@/lib/sim-search/indexed', async () => ({ - ownsIndexedPersonalSearchAccount: ( - await import('@/lib/sim-search/indexed/integrations/personal-account-ownership') - ).ownsIndexedPersonalSearchAccount, -})) import { prepareOrganizationPersonalConnection, @@ -125,7 +119,6 @@ describe('organization personal token authorization', () => { }) it('uses current personal OAuth inventory in live mode without consulting indexed sources', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) mocks.liveAccounts.mockResolvedValue([ { id: 'own', providerId: 'google-drive', type: 'managed_oauth' }, ]) @@ -136,7 +129,6 @@ describe('organization personal token authorization', () => { expect(mocks.liveAccounts).toHaveBeenCalledWith({ organizationId: 'org' }, 'person') }) it('does not let direct integration tools bypass current organization scope restrictions', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) mocks.liveAccounts.mockResolvedValue([ { id: 'own', providerId: 'google-drive', type: 'managed_oauth' }, ]) @@ -156,7 +148,6 @@ describe('organization personal token authorization', () => { it.each(['service_account'])( 'never substitutes a %s for a personal OAuth account', async (type) => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) mocks.liveAccounts.mockResolvedValue([{ id: 'own', providerId: 'google-drive', type }]) await expect(resolveOrganizationPersonalToken.execute({ principal, input })).rejects.toThrow( 'own connected account' @@ -166,14 +157,15 @@ describe('organization personal token authorization', () => { } ) it('observes a revoked live account without falling back to old indexing membership', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) await expect(resolveOrganizationPersonalToken.execute({ principal, input })).rejects.toThrow( 'own connected account' ) expect(mocks.token).not.toHaveBeenCalled() }) it('uses the authenticated person inventory and organization token scope without a workspace', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) + mocks.liveAccounts.mockResolvedValue([ + { id: 'own', providerId: 'google-drive', type: 'managed_oauth' }, + ]) await expect(resolveOrganizationPersonalToken.execute({ principal, input })).resolves.toEqual({ accessToken: 'secret', refreshed: false, @@ -219,21 +211,7 @@ describe('organization personal token authorization', () => { expect(mocks.token).not.toHaveBeenCalled() }) - it('does not confuse paused indexing with account authorization', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) - mocks.inventory.mockResolvedValue({ - connections: [ - { indexingStatus: 'paused', accounts: [{ credentialId: 'own', status: 'connected' }] }, - ], - nextCursor: null, - }) - await expect( - resolveOrganizationPersonalToken.execute({ principal, input }) - ).resolves.toHaveProperty('credentialType', 'managed_oauth') - }) - it('returns the live account target for a connection request without an indexed source', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) const target = { type: 'link', provider: 'google-drive', @@ -254,28 +232,4 @@ describe('organization personal token authorization', () => { expect(result).not.toHaveProperty('settingsPath') expect(mocks.liveAccounts).not.toHaveBeenCalled() }) - - it('finds an exact reconnect control on a later inventory page', async () => { - const target = { - type: 'link', - provider: 'google-drive', - connectorType: 'drive', - connectorId: 'connector', - credentialId: 'own', - } - mocks.inventory.mockResolvedValueOnce({ connections: [], available: [], nextCursor: 'next' }) - mocks.inventory.mockResolvedValueOnce({ - connections: [{ accounts: [{ credentialId: 'own', action: target }] }], - available: [], - nextCursor: null, - }) - await expect( - prepareOrganizationPersonalConnection.execute({ - principal, - input: { organizationId: 'org', providerName: 'Google Drive', credentialId: 'own' }, - }) - ).resolves.toEqual({ provider: 'Google Drive', providerId: 'google-drive', target }) - expect(mocks.inventory.mock.calls[1][0].input.cursor).toBe('next') - expect(mocks.token).not.toHaveBeenCalled() - }) }) diff --git a/apps/sim/lib/credentials/application/resolve-organization-personal-token.ts b/apps/sim/lib/credentials/application/resolve-organization-personal-token.ts index a1c7f8f8b29..2fc9b72a254 100644 --- a/apps/sim/lib/credentials/application/resolve-organization-personal-token.ts +++ b/apps/sim/lib/credentials/application/resolve-organization-personal-token.ts @@ -11,12 +11,11 @@ import { } from '@/lib/credential-groups/credentials' import { resolveManagedOAuthToken } from '@/lib/credentials/managed-oauth' import { projectIntegrationToolsForViewer } from '@/lib/integrations/tool-projection' -import { personalSearchIntegrationPages } from '@/lib/knowledge/application/personal-search-integration-pages' +import { listPersonalSearchIntegrations } from '@/lib/knowledge/application/personal-search-integrations' import { requireOrganizationSearchApproval } from '@/lib/knowledge/search/integration-policy' import { providerIdsForService } from '@/lib/oauth/utils' import { getUserPermissionConfigForOrganization } from '@/lib/permission-groups/resolve.server' import { SEARCH_CONNECTORS } from '@/lib/sim-search/connectors' -import { isIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' import { listLiveAccounts } from '@/lib/sim-search/live/accounts' import { requiresScopedRetrieval } from '@/lib/sim-search/live/policy-schema' import { livePolicyFor, loadLiveSearchPolicies } from '@/lib/sim-search/live/policy-store' @@ -84,36 +83,25 @@ export const resolveOrganizationPersonalToken = { ) { throw new OrchestrationError('forbidden', 'This integration operation is unavailable.') } - const indexed = isIndexedOrgSearchEnabled() - /** Loaded lazily: this resolver is on the executor's credential path, which never needs it otherwise. */ - const owned = indexed - ? await (await import('@/lib/sim-search/indexed')).ownsIndexedPersonalSearchAccount( - principal, - { - organizationId: context.organizationId, - connectorType: connector.type, - credentialId: input.credentialId, - } - ) - : (await listLiveAccounts({ organizationId: context.organizationId }, context.userId)).some( - (account) => - account.id === input.credentialId && - account.type === 'managed_oauth' && - account.providerId === binding.providerId - ) + const owned = ( + await listLiveAccounts({ organizationId: context.organizationId }, context.userId) + ).some( + (account) => + account.id === input.credentialId && + account.type === 'managed_oauth' && + account.providerId === binding.providerId + ) if (!owned) throw new OrchestrationError( 'forbidden', 'Assistant can only use your own connected account for this integration.' ) - if (!indexed) { - const policies = await loadLiveSearchPolicies({ organizationId: context.organizationId }) - if (requiresScopedRetrieval(connector.type, livePolicyFor(policies, connector.type))) - throw new OrchestrationError( - 'forbidden', - 'This organization restricts search scope for this app. Use search_workspace with nativeQueries and read_document so those restrictions are enforced.' - ) - } + const policies = await loadLiveSearchPolicies({ organizationId: context.organizationId }) + if (requiresScopedRetrieval(connector.type, livePolicyFor(policies, connector.type))) + throw new OrchestrationError( + 'forbidden', + 'This organization restricts search scope for this app. Use search_workspace with nativeQueries and read_document so those restrictions are enforced.' + ) const token = await resolveManagedOAuthToken({ ...input, expectedProviderId: binding.providerId, @@ -177,17 +165,16 @@ export const prepareOrganizationPersonalConnection = { ) if (!connector) throw new OrchestrationError('validation', 'This integration is unavailable in Search.') - for await (const inventory of personalSearchIntegrationPages({ + const inventory = await listPersonalSearchIntegrations.execute({ principal, input: { organizationId: context.organizationId, connectorType: connector.type }, - })) { - const target = input.credentialId - ? inventory.connections - .flatMap((connection) => connection.accounts) - .find((account) => account.credentialId === input.credentialId)?.action - : inventory.available[0]?.target - if (target) return { provider: connector.meta.name, providerId: connector.providerId, target } - } + }) + const target = input.credentialId + ? inventory.connections + .flatMap((connection) => connection.accounts) + .find((account) => account.credentialId === input.credentialId)?.action + : inventory.available[0]?.target + if (target) return { provider: connector.meta.name, providerId: connector.providerId, target } throw new OrchestrationError( 'validation', 'No connection action is currently available. Check your integration connection status.' diff --git a/apps/sim/lib/desktop/source-browser.ts b/apps/sim/lib/desktop/source-browser.ts index 3494632bc43..ffe71f0c111 100644 --- a/apps/sim/lib/desktop/source-browser.ts +++ b/apps/sim/lib/desktop/source-browser.ts @@ -5,10 +5,7 @@ import { consumeDesktopSourceRequestContract, type DesktopSourceRequest, } from '@/lib/api/contracts/desktop-source-connect' -import { - connectSimSearchConnectorContract, - startKnowledgeConnectorMemberEnrollmentContract, -} from '@/lib/api/contracts/knowledge/connectors' +import { startKnowledgeConnectorMemberEnrollmentContract } from '@/lib/api/contracts/knowledge/connectors' import { gitHubSearchSetupScopeSchema, readGitHubSearchSetupContract, @@ -120,17 +117,6 @@ async function startRequest( : enrollmentMatch(result.data.url), } } - case 'search-source': { - const result = await requestJson(connectSimSearchConnectorContract, { - body: { ...request.body, oauthCompletionId: request.completionId }, - }) - return { - url: result.data.url, - match: request.completionId - ? { kind: 'completion', id: request.completionId } - : enrollmentMatch(result.data.url), - } - } case 'slack-managed-users': { const { owner, body, credentialGroupId } = request const result = owner.organizationId diff --git a/apps/sim/lib/knowledge/__integration__/coda-live.integration.ts b/apps/sim/lib/knowledge/__integration__/coda-live.integration.ts index ec084696138..e40cdb28090 100644 --- a/apps/sim/lib/knowledge/__integration__/coda-live.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/coda-live.integration.ts @@ -11,9 +11,7 @@ import { db } from '@sim/db' import { credential, document, - knowledgeBase, knowledgeConnector, - member, organization, session, user, @@ -24,14 +22,6 @@ import { generateId } from '@sim/utils/id' import { serializeSignedCookie } from 'better-call' import { eq } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' - -/** Its search-index knowledge bases are read through indexed organization search, dormant unless Live Search is off. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) - import { z } from 'zod' const metrics = vi.hoisted(() => ({ embeddingCalls: 0 })) @@ -51,12 +41,8 @@ vi.mock('@/lib/embeddings', async () => ({ }, })) -import { - resolveBillingAttribution, - resolveOrganizationBillingAttribution, -} from '@/lib/billing/core/billing-attribution' +import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' import { encryptSecret } from '@/lib/core/security/encryption' -import { createOrganizationCredential } from '@/lib/credentials/application/organization-credentials' import { assertCodaLiveFixture } from '@/lib/knowledge/__integration__/coda-live-fixture' import { seedKnowledgeAclFixture } from '@/lib/knowledge/__integration__/seed-source-access-fixture' import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' @@ -73,7 +59,6 @@ const tokenPath = process.env.CODA_CONNECTOR_LIVE_TOKEN_FILE const fixturePath = process.env.CODA_CONNECTOR_LIVE_FIXTURE_FILE const secondEmail = process.env.CODA_CONNECTOR_LIVE_SECOND_EMAIL const allowSharing = process.env.CODA_CONNECTOR_LIVE_ALLOW_SHARING !== 'false' -const organizationScope = process.env.CODA_CONNECTOR_LIVE_SCOPE === 'organization' const uiFixturePath = process.env.CODA_CONNECTOR_LIVE_UI_FIXTURE_FILE const fixtureSchema = z.object({ docId: z.string(), @@ -97,7 +82,7 @@ describe.skipIf(!tokenPath || !fixturePath || !secondEmail)( let token: string let fixture: z.infer let fixtureValidated = false - let credentialId = generateId() + const credentialId = generateId() const principal = (userId: string): Principal => ({ kind: 'session', userId, @@ -139,9 +124,7 @@ describe.skipIf(!tokenPath || !fixturePath || !secondEmail)( const result = await searchKnowledge.execute({ principal: as, input: { - ...(organizationScope - ? { organizationId: ids.organizationId } - : { workspaceId: ids.workspaceId }), + workspaceId: ids.workspaceId, knowledgeBaseIds: [ids.knowledgeBaseId], query: fixture.marker, searchMode: 'hybrid', @@ -154,15 +137,10 @@ describe.skipIf(!tokenPath || !fixturePath || !secondEmail)( async function sync() { const result = await executeSync(connectorId, { fullSync: false, - billingAttribution: organizationScope - ? await resolveOrganizationBillingAttribution({ - actorUserId: ids.aliceId, - organizationId: ids.organizationId, - }) - : await resolveBillingAttribution({ - actorUserId: ids.aliceId, - workspaceId: ids.workspaceId, - }), + billingAttribution: await resolveBillingAttribution({ + actorUserId: ids.aliceId, + workspaceId: ids.workspaceId, + }), }) expect(result.error).toBeUndefined() expect(result.skipReason).toBeUndefined() @@ -185,63 +163,28 @@ describe.skipIf(!tokenPath || !fixturePath || !secondEmail)( ids = await seedKnowledgeAclFixture() await db.update(user).set({ email: source.owner }).where(eq(user.id, ids.aliceId)) await db.update(user).set({ email: secondEmail! }).where(eq(user.id, ids.bobId)) - if (organizationScope) { - await db.insert(member).values([ - { - id: generateId(), - organizationId: ids.organizationId, - userId: ids.aliceId, - role: 'owner', - createdAt: new Date(), - }, - { - id: generateId(), - organizationId: ids.organizationId, - userId: ids.bobId, - role: 'member', - createdAt: new Date(), - }, - ]) - await db - .update(knowledgeBase) - .set({ workspaceId: null, organizationId: ids.organizationId, isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) - const createdCredential = await createOrganizationCredential.execute({ - principal: principal(ids.aliceId), - input: { - organizationId: ids.organizationId, - type: 'service_account', - providerId: 'coda-service-account', - displayName: 'Disposable Coda live fixture', - apiToken: token, - }, - }) - credentialId = createdCredential.credential.id - } else - await db.insert(credential).values({ - id: credentialId, - workspaceId: ids.workspaceId, - createdBy: ids.aliceId, - type: 'service_account', - providerId: 'coda-service-account', - displayName: 'Disposable Coda live fixture', - encryptedServiceAccountKey: ( - await encryptSecret( - JSON.stringify({ - type: 'token_service_account', - providerId: 'coda-service-account', - apiToken: token, - }) - ) - ).encrypted, - }) + await db.insert(credential).values({ + id: credentialId, + workspaceId: ids.workspaceId, + createdBy: ids.aliceId, + type: 'service_account', + providerId: 'coda-service-account', + displayName: 'Disposable Coda live fixture', + encryptedServiceAccountKey: ( + await encryptSecret( + JSON.stringify({ + type: 'token_service_account', + providerId: 'coda-service-account', + apiToken: token, + }) + ) + ).encrypted, + }) const created = await createKnowledgeConnector.execute({ principal: principal(ids.aliceId), input: { knowledgeBaseId: ids.knowledgeBaseId, - ...(organizationScope - ? { assertedOrganizationId: ids.organizationId } - : { assertedWorkspaceId: ids.workspaceId }), + assertedWorkspaceId: ids.workspaceId, connectorType: 'coda', credentialId, accessMode: 'admin', @@ -321,8 +264,7 @@ describe.skipIf(!tokenPath || !fixturePath || !secondEmail)( workspaceId: ids.workspaceId, keyId: 'fixture', }) - if (organizationScope) await expect(keySearch).rejects.toThrow() - else expect(await keySearch).not.toContain(documentId) + expect(await keySearch).not.toContain(documentId) const chunks = await listKnowledgeChunks.execute({ principal: principal(ids.aliceId), input: { knowledgeBaseId: ids.knowledgeBaseId, documentId }, diff --git a/apps/sim/lib/knowledge/__integration__/dormant-processing-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/dormant-processing-recovery.integration.ts index fea881f3e4b..2ccf293562c 100644 --- a/apps/sim/lib/knowledge/__integration__/dormant-processing-recovery.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/dormant-processing-recovery.integration.ts @@ -24,11 +24,6 @@ import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' const fixture = vi.hoisted(() => ({ root: '' })) vi.mock('@/lib/core/config/trigger-runtime', () => ({ isInsideTriggerRun: () => false })) -/** Pinned to Live Search, whatever `SIM_SEARCH_LIVE` the run was started with. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => ({ - ...(await importOriginal>()), - isLiveEnterpriseSearchEnabled: true, -})) vi.mock('@/lib/uploads/core/setup.server', () => ({ get UPLOAD_DIR_SERVER() { return fixture.root diff --git a/apps/sim/lib/knowledge/__integration__/dormant-search-processing.integration.ts b/apps/sim/lib/knowledge/__integration__/dormant-search-processing.integration.ts index 9da06ae10c6..0ed419014ed 100644 --- a/apps/sim/lib/knowledge/__integration__/dormant-search-processing.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/dormant-search-processing.integration.ts @@ -2,7 +2,7 @@ import { db } from '@sim/db' import { document, knowledgeBase, organization, user, workspace } from '@sim/db/schema' import { generateId } from '@sim/utils/id' import { eq, inArray } from 'drizzle-orm' -import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' import { createKnowledgeAclFixtureIds, seedKnowledgeAclFixture, @@ -14,11 +14,6 @@ import { processDocumentsWithQueue, } from '@/lib/knowledge/documents/service' -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => ({ - ...(await importOriginal>()), - isLiveEnterpriseSearchEnabled: true, -})) - /** Queued work cannot revive dormant Search before retirement reaches its documents. */ describe('dormant Search document processing', () => { const ids = createKnowledgeAclFixtureIds() diff --git a/apps/sim/lib/knowledge/__integration__/embedding-insert-batches.integration.ts b/apps/sim/lib/knowledge/__integration__/embedding-insert-batches.integration.ts index 25a060da5fb..7c47768857e 100644 --- a/apps/sim/lib/knowledge/__integration__/embedding-insert-batches.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/embedding-insert-batches.integration.ts @@ -17,13 +17,6 @@ import { generateId } from '@sim/utils/id' import { eq, inArray } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' -/** These transaction checks exercise indexed Search, which Live Search normally disables. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) - const fixtures = vi.hoisted(() => ({ root: '', process: vi.fn(), embeddings: vi.fn() })) vi.mock('@/lib/uploads/core/setup.server', () => ({ get UPLOAD_DIR_SERVER() { @@ -56,11 +49,6 @@ describe('bounded embedding insert transactions', () => { beforeAll(async () => { fixtures.root = mkdtempSync(path.join(tmpdir(), 'sim-embedding-batches-')) await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) - /** A search index, the only kind of base whose chunks the keyword projection holds. */ - await db - .update(knowledgeBase) - .set({ isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) vi.spyOn(embeddingClient, 'assertKnowledgeEmbeddingCapacity').mockResolvedValue(undefined) }) @@ -163,7 +151,7 @@ describe('bounded embedding insert transactions', () => { .from(embedding) .where(eq(embedding.documentId, file.documentId)) ).toEqual([{ id: previousId }]) - for (const table of ['embedding_search', 'embedding_keyword_search']) { + for (const table of ['embedding_search']) { expect( await db.$client.unsafe(`SELECT id FROM ${table} WHERE document_id = $1`, [file.documentId]) ).toEqual([{ id: previousId }]) @@ -187,7 +175,7 @@ describe('bounded embedding insert transactions', () => { expect(await db.select().from(document).where(eq(document.id, file.documentId))).toMatchObject([ { processingStatus: 'completed', chunkCount: 205, processingError: null }, ]) - for (const table of ['embedding', 'embedding_search', 'embedding_keyword_search']) { + for (const table of ['embedding', 'embedding_search']) { expect( await db.$client.unsafe( `SELECT count(*)::int AS count FROM ${table} WHERE document_id = $1`, diff --git a/apps/sim/lib/knowledge/__integration__/excluded-member-documents.integration.ts b/apps/sim/lib/knowledge/__integration__/excluded-member-documents.integration.ts index 451f96df73b..79e4b2a111e 100644 --- a/apps/sim/lib/knowledge/__integration__/excluded-member-documents.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/excluded-member-documents.integration.ts @@ -3,7 +3,6 @@ import { db } from '@sim/db' import { document, embedding, - knowledgeBase, knowledgeConnector, knowledgeConnectorMember, knowledgeDocumentObservation, @@ -17,12 +16,6 @@ import { eq, inArray } from 'drizzle-orm' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' const provider = vi.hoisted(() => ({ list: vi.fn(), get: vi.fn(), changes: vi.fn() })) -/** This suite covers indexed organization search, which is dormant unless Live Search is off. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: { google_drive: { @@ -139,10 +132,6 @@ describe('excluded member documents retain current source authorization', () => }, ids.aliceId ) - await db - .update(knowledgeBase) - .set({ isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) await db .update(knowledgeConnector) .set({ status: 'active', memberSyncStatus: 'idle', memberSyncLockToken: null }) diff --git a/apps/sim/lib/knowledge/__integration__/github-member.integration.ts b/apps/sim/lib/knowledge/__integration__/github-member.integration.ts index b680357e2f6..7829bd571eb 100644 --- a/apps/sim/lib/knowledge/__integration__/github-member.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/github-member.integration.ts @@ -32,12 +32,6 @@ import { generateId } from '@sim/utils/id' import { and, eq, inArray, isNull, sql } from 'drizzle-orm' import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -/** This suite covers indexed organization search, which is dormant unless Live Search is off. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) vi.mock('@/lib/embeddings', async () => ({ ...(await import('@/lib/embeddings/client')), assertKnowledgeEmbeddingCapacity: async () => {}, @@ -81,7 +75,6 @@ import { seedKnowledgeAclFixture, seedKnowledgeMemberFixture, } from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { GITHUB_READ_SOURCE_TIMEOUT_MS } from '@/lib/knowledge/access/github-installation' import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' import { subjectToken } from '@/lib/knowledge/access/tokens' import { KnowledgeDocumentNotReadyError } from '@/lib/knowledge/application/chunk-errors' @@ -92,8 +85,6 @@ import { } from '@/lib/knowledge/application/connectors' import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' import { searchKnowledge } from '@/lib/knowledge/application/search' -import { readSearchSourceOverview } from '@/lib/knowledge/application/search-source-overview' -import { listSearchSources } from '@/lib/knowledge/application/search-sources' import { grantKnowledgeConnectorCredentialAccess } from '@/lib/knowledge/connectors/member-access' import * as memberSyncEngine from '@/lib/knowledge/connectors/member-sync-engine' import { @@ -101,11 +92,8 @@ import { MEMBER_TOMBSTONE_PURGE_DAYS, } from '@/lib/knowledge/connectors/sync-limits' import { getDocuments } from '@/lib/knowledge/documents/service' -import { getTagUsageStats } from '@/lib/knowledge/tags/service' -import { readIndexedKnowledgeDocument } from '@/lib/sim-search/indexed/documents/read-indexed-document' import { deleteFile } from '@/lib/uploads/core/storage-service' import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' -import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const redisUrl = readTestRedisUrl() @@ -144,7 +132,6 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { const { privateKey, publicKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }) let organizationSource = false let installationSuspended = false - let referenceObserved: ((repository: string) => void) | undefined const installation = () => ({ id: 42, app_id: 1, @@ -349,7 +336,6 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { : Response.json({ message: 'Not Found' }, { status: 404 }) } if (match[2].startsWith('/git/ref/heads/')) { - referenceObserved?.(match[1]) if (source.stallRef) return new Promise((_resolve, reject) => { request.signal.addEventListener('abort', () => reject(request.signal.reason), { @@ -444,7 +430,6 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { refreshedUsers.clear() organizationSource = false installationSuspended = false - referenceObserved = undefined oauthStateKey = undefined oauthVerification = undefined Object.assign(env, { @@ -453,10 +438,6 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { }) ids = createKnowledgeAclFixtureIds() await seedKnowledgeAclFixture(ids) - await db - .update(knowledgeBase) - .set({ isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) enrolled = await seedKnowledgeMemberFixture(ids) const policy = await getCredentialGroupProviderAdapter('github-repositories').getPolicy( undefined, @@ -682,7 +663,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { ]) await db .update(knowledgeBase) - .set({ workspaceId: null, organizationId: ids.organizationId }) + .set({ workspaceId: null, organizationId: ids.organizationId, isSearchIndex: true }) .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) await db .update(credentialGroup) @@ -760,10 +741,8 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { return installationCredentialId } - it('reuses connected organization members for a later installation source without another enrollment', async () => { + it('creates a live installation source without indexing or replacing connected member accounts', async () => { const installationCredentialId = await useOrganizationInstallation() - expect((await sync()).error).toBeUndefined() - const [shared] = await rows() const enrollmentsBefore = await db .select({ id: credentialGroupEnrollment.id, @@ -795,17 +774,10 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { await expect( createKnowledgeConnector.execute({ principal: actor(ids.bobId), input }) ).rejects.toThrow() - const dispatchedSync = vi.spyOn(memberSyncEngine, 'executeMemberSync') const { connector } = await createKnowledgeConnector.execute({ principal: actor(ids.aliceId), input, }) - try { - expect(dispatchedSync).toHaveBeenCalledExactlyOnceWith(connector.id, expect.any(Object)) - expect((await dispatchedSync.mock.results[0].value).error).toBeUndefined() - } finally { - dispatchedSync.mockRestore() - } const connectorId = connector.id expect(connector).toMatchObject({ credentialGroupId: enrolled.groupId, @@ -816,16 +788,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { .select() .from(knowledgeConnector) .where(eq(knowledgeConnector.id, connectorId)) - expect(current).toMatchObject({ memberSyncStatus: 'idle' }) - expect(current.lastMemberSyncAt).not.toBeNull() - const memberships = await db - .select() - .from(knowledgeConnectorMember) - .where(eq(knowledgeConnectorMember.connectorId, connectorId)) - expect(memberships).toHaveLength(2) - expect(memberships.map((row) => row.credentialId).sort()).toEqual( - credentialsBefore.map((row) => row.id).sort() - ) + expect(current).toMatchObject({ lastMemberSyncAt: null, nextMemberSyncAt: null }) expect( await db .select({ @@ -848,54 +811,17 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { .where(eq(credential.credentialGroupOptionId, enrolled.optionId)) .orderBy(credential.id) ).toEqual(credentialsBefore) - const [indexed] = await rows(connectorId) - expect(await search(actor(ids.aliceId))).toEqual([shared.id, indexed.id].sort()) - expect(await search(actor(ids.bobId))).toEqual([shared.id]) - await assertAccess(actor(ids.aliceId), indexed, true) - await assertAccess(actor(ids.bobId), indexed, false) - const indexedRead = (userId: string) => - readIndexedKnowledgeDocument.execute({ - principal: actor(userId), - input: { - organizationId: ids.organizationId, - target: { kind: 'id', documentId: indexed.id }, - limit: 10, - resultSecretRegistry: new ResolvedSecretTraceRegistry(), - }, - }) + expect(await rows(connectorId)).toEqual([]) expect( - (await indexedRead(ids.aliceId)).chunks?.map((chunk) => chunk.content).join('\n') - ).toContain('Orion later') - await expect(indexedRead(ids.bobId)).rejects.toThrow('Document not found') - /** Organization cache bytes are internal; members read through the authorized Search operation. */ - await expect( - downloadFileFromUrl(indexed.fileUrl, { userId: ids.aliceId, knowledgeAccess: 'user' }) - ).rejects.toThrow('Access denied') - await expect( - downloadFileFromUrl(indexed.fileUrl, { userId: ids.bobId, knowledgeAccess: 'user' }) - ).rejects.toThrow('Access denied') - for (const userId of [ids.aliceId, ids.bobId]) { - const { sources } = await listSearchSources.execute({ - principal: actor(userId), - input: { organizationId: ids.organizationId, connectorId }, - }) - expect(sources).toMatchObject([ - { - connectorId, - viewerMembership: 'connected', - hasViewerDocuments: userId === ids.aliceId, - }, - ]) - } - expect( - requests - .filter((entry) => entry.path.startsWith('/repos/fixture/later/git/blobs/')) - .map((entry) => entry.userId) - ).toEqual(['installation']) + await db + .select({ id: embedding.id }) + .from(embedding) + .where(eq(embedding.knowledgeBaseId, ids.knowledgeBaseId)) + ).toEqual([]) + expect(requests.filter((entry) => entry.path.includes('/git/blobs/'))).toEqual([]) }) - it('keeps actual skips, legacy skips, and provider failures distinct in authorized lists and source counts', async () => { - await useOrganizationInstallation() + it('keeps actual skips, legacy skips, and provider failures distinct in authorized document lists', async () => { const source = repositories.get('shared')! source.readers.delete(ids.bobId) source.files.set('empty.txt', '') @@ -908,18 +834,6 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { expect(binary).toMatchObject({ processingStatus: 'failed', storageKey: null }) expect(empty.contentHash).not.toBeNull() await db.update(document).set({ processingStatus: 'failed' }).where(eq(document.id, empty.id)) - const summary = async (userId: string) => - ( - await listSearchSources.execute({ - principal: actor(userId), - input: { organizationId: ids.organizationId, connectorId: enrolled.connectorId }, - }) - ).sources[0] - expect(await summary(ids.aliceId)).toMatchObject({ - hasViewerDocuments: true, - viewerFailedDocumentCount: 0, - hasSyncError: false, - }) source.files.set('unavailable.txt', 'Orion content whose blob cannot be fetched.') source.failedBlobs.add(shaFor(source.files.get('unavailable.txt')!)) await sync() @@ -932,7 +846,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { }) for (const userId of [ids.aliceId, ids.bobId]) { const provider = createKnowledgeAccessProvider(actor(userId), { - organizationId: ids.organizationId, + workspaceId: ids.workspaceId, knowledgeBaseIds: [ids.knowledgeBaseId], }) const listed = await getDocuments(ids.knowledgeBaseId, {}, 'github-skip-outcomes', provider) @@ -973,192 +887,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { : [] ) } - expect(await summary(userId)).toMatchObject({ - hasViewerDocuments: userId === ids.aliceId, - viewerFailedDocumentCount: userId === ids.aliceId ? 1 : 0, - }) - } - }) - - it('indexes an organization installation once and denies live user, app, and org revocations before search or reads', async () => { - const installationCredentialId = await useOrganizationInstallation() - const unrelatedSources = Array.from({ length: 105 }, () => generateId()) - await db.insert(knowledgeConnector).values( - unrelatedSources.map((id) => ({ - id, - knowledgeBaseId: ids.knowledgeBaseId, - connectorType: 'github', - accessMode: 'members', - credentialId: installationCredentialId, - credentialGroupId: enrolled.groupId, - credentialGroupOptionId: enrolled.optionId, - sourceConfig: { repository: 'fixture/shared', githubRepositoryId: '9001' }, - })) - ) - await db.insert(knowledgeConnectorMember).values( - unrelatedSources.map((connectorId) => ({ - id: generateId(), - organizationId: ids.organizationId, - connectorId, - credentialId: enrolled.members[0].credentialId, - subjectToken: enrolled.members[0].subjectToken, - })) - ) - const result = await sync() - expect(result.error).toBeUndefined() - expect(result.docsHydratedOnce).toBe(1) - const [indexed] = await rows() - expect(indexed).toBeDefined() - const provider = (userId: string) => - createKnowledgeAccessProvider(actor(userId), { - organizationId: ids.organizationId, - knowledgeBaseIds: [ids.knowledgeBaseId], - }) - const page = (userId: string, offset = 0) => - getDocuments( - ids.knowledgeBaseId, - { limit: 1, offset, sortBy: 'filename', sortOrder: 'asc' }, - 'github-candidate-regression', - provider(userId) - ) - expect(await page(ids.aliceId)).toMatchObject({ - documents: [{ id: indexed.id }], - pagination: { total: 1 }, - }) - expect( - ( - await readSearchSourceOverview.execute({ - principal: actor(ids.aliceId), - input: { organizationId: ids.organizationId }, - }) - ).hasSearchableDocuments - ).toBe(true) - await db.update(document).set({ tag1: 'fixture' }).where(eq(document.id, indexed.id)) - await db.update(embedding).set({ tag1: 'fixture' }).where(eq(embedding.documentId, indexed.id)) - expect( - await getTagUsageStats(ids.knowledgeBaseId, provider(ids.aliceId), 'github-tag-regression') - ).toEqual( - expect.arrayContaining([ - expect.objectContaining({ tagSlot: 'tag1', documentCount: 1, chunkCount: 1 }), - ]) - ) - expect( - ( - await readIndexedKnowledgeDocument.execute({ - principal: actor(ids.aliceId), - input: { - organizationId: ids.organizationId, - target: { kind: 'url', url: indexed.sourceUrl! }, - limit: 1, - resultSecretRegistry: new ResolvedSecretTraceRegistry(), - }, - }) - ).documentId - ).toBe(indexed.id) - expect( - requests.filter((entry) => entry.path.includes('/git/blobs/')).map((entry) => entry.userId) - ).toEqual(['installation']) - expect(await search(actor(ids.aliceId))).toEqual([indexed.id]) - expect(await search(actor(ids.bobId))).toEqual([indexed.id]) - await assertAccess(actor(ids.bobId), indexed, true) - const source = repositories.get('shared')! - source.readers.delete(ids.bobId) - expect(await page(ids.bobId)).toMatchObject({ documents: [], pagination: { total: 0 } }) - expect(await search(actor(ids.bobId))).toEqual([]) - await assertAccess(actor(ids.bobId), indexed, false) - expect(await search(actor(ids.aliceId))).toEqual([indexed.id]) - expect( - await db - .select() - .from(knowledgeDocumentObservation) - .where(eq(knowledgeDocumentObservation.documentId, indexed.id)) - ).toHaveLength(2) - source.readers.add(ids.bobId) - source.public = true - source.installed = false - expect(await search(actor(ids.aliceId))).toEqual([]) - await assertAccess(actor(ids.aliceId), indexed, false) - source.installed = true - installationSuspended = true - expect(await search(actor(ids.aliceId))).toEqual([]) - installationSuspended = false - expect(await search(actor(ids.aliceId))).toEqual([indexed.id]) - const slowRepository = repository('slow') - const slowSourceId = generateId() - await db.insert(knowledgeConnector).values({ - id: slowSourceId, - knowledgeBaseId: ids.knowledgeBaseId, - connectorType: 'github', - accessMode: 'members', - credentialId: installationCredentialId, - credentialGroupId: enrolled.groupId, - credentialGroupOptionId: enrolled.optionId, - sourceConfig: { repository: 'fixture/slow', githubRepositoryId: String(slowRepository.id) }, - }) - expect((await sync(slowSourceId)).error).toBeUndefined() - const [slowDocument] = await rows(slowSourceId) - expect(slowDocument).toBeDefined() - const deniedRepository = repository('denied-paging', [ids.aliceId]) - const deniedSourceId = generateId() - await db.insert(knowledgeConnector).values({ - id: deniedSourceId, - knowledgeBaseId: ids.knowledgeBaseId, - connectorType: 'github', - accessMode: 'members', - credentialId: installationCredentialId, - credentialGroupId: enrolled.groupId, - credentialGroupOptionId: enrolled.optionId, - sourceConfig: { - repository: 'fixture/denied-paging', - githubRepositoryId: String(deniedRepository.id), - }, - }) - expect((await sync(deniedSourceId)).error).toBeUndefined() - const [deniedDocument] = await rows(deniedSourceId) - deniedRepository.readers.delete(ids.aliceId) - for (const [id, filename] of [ - [indexed.id, 'alpha'], - [deniedDocument.id, 'beta'], - [slowDocument.id, 'gamma'], - ]) - await db.update(document).set({ filename }).where(eq(document.id, id)) - expect(await page(ids.aliceId, 1)).toMatchObject({ - documents: [{ id: slowDocument.id }], - pagination: { total: 2, offset: 1 }, - }) - slowRepository.stallRef = true - const sourceTimers: AbortController[] = [] - const nativeTimeout = AbortSignal.timeout.bind(AbortSignal) - const timerSpy = vi.spyOn(AbortSignal, 'timeout').mockImplementation((duration) => { - if (duration !== GITHUB_READ_SOURCE_TIMEOUT_MS) return nativeTimeout(duration) - const controller = new AbortController() - sourceTimers.push(controller) - return controller.signal - }) - try { - const observed = new Set() - const candidatesStarted = new Promise((resolve) => { - referenceObserved = (name) => { - observed.add(name) - if (observed.has('shared') && observed.has('slow')) resolve() - } - }) - const pending = search(actor(ids.aliceId), 'vector') - await candidatesStarted - /** Complete the fast response's microtasks before expiring the stalled candidate. */ - for (let turn = 0; turn < 20; turn++) await Promise.resolve() - for (const timer of sourceTimers) timer.abort(new Error('fixture source timeout')) - expect(await pending).toEqual([indexed.id]) - } finally { - timerSpy.mockRestore() - referenceObserved = undefined - slowRepository.stallRef = false } - await db - .delete(member) - .where(and(eq(member.organizationId, ids.organizationId), eq(member.userId, ids.bobId))) - await expect(search(actor(ids.bobId))).rejects.toThrow() - await assertAccess(actor(ids.bobId), indexed, false) }) it.runIf(Boolean(redisUrl))( @@ -1316,24 +1045,6 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { expect(privateFile.acl).toEqual([enrolled.members[0].subjectToken]) expect(await search(actor(ids.aliceId))).toEqual([shared.id, privateFile.id].sort()) expect(await search(actor(ids.bobId))).toEqual([shared.id]) - for (const userId of [ids.aliceId, ids.bobId]) { - const summaries = await listSearchSources.execute({ - principal: actor(userId), - input: { workspaceId: ids.workspaceId }, - }) - expect( - summaries.sources.map((source) => ({ - connectorId: source.connectorId, - hasViewerDocuments: source.hasViewerDocuments, - })) - ).toEqual( - expect.arrayContaining([ - { connectorId: enrolled.connectorId, hasViewerDocuments: true }, - { connectorId: privateId, hasViewerDocuments: userId === ids.aliceId }, - { connectorId: blockedId, hasViewerDocuments: false }, - ]) - ) - } expect(await search(workspaceKey())).toEqual([]) await assertAccess(actor(ids.bobId), shared, true) await assertAccess(actor(ids.bobId), privateFile, false) diff --git a/apps/sim/lib/knowledge/__integration__/gitlab-live.integration.ts b/apps/sim/lib/knowledge/__integration__/gitlab-live.integration.ts index 774b7d00846..974768e2295 100644 --- a/apps/sim/lib/knowledge/__integration__/gitlab-live.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/gitlab-live.integration.ts @@ -20,9 +20,7 @@ import { knowledgeConnector, knowledgeConnectorPermissionGrant, knowledgeConnectorPermissionSnapshot, - member, organization, - organizationSearchIntegration, permissions, session, user, @@ -36,14 +34,6 @@ import { serializeSignedCookie } from 'better-call' import { and, eq, inArray, isNull, or } from 'drizzle-orm' import { NextRequest } from 'next/server' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' - -/** Its search-index knowledge bases are read through indexed organization search, dormant unless Live Search is off. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) - import type { EmbedOptions } from '@/lib/embeddings/types' const fixture = vi.hoisted(() => ({ embeddingCalls: 0 })) @@ -85,10 +75,7 @@ import type { UpdateConnectorBody, } from '@/lib/api/contracts/knowledge/connectors' import { decryptApiKey, encryptApiKey } from '@/lib/api-key/crypto' -import { - resolveBillingAttribution, - resolveOrganizationBillingAttribution, -} from '@/lib/billing/core/billing-attribution' +import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' import { seedKnowledgeAclFixture } from '@/lib/knowledge/__integration__/seed-source-access-fixture' import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' import { updateKnowledgeConnectorAccess } from '@/lib/knowledge/application/connector-access' @@ -96,13 +83,11 @@ import { updateKnowledgeConnector } from '@/lib/knowledge/application/connectors import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' import { searchKnowledge } from '@/lib/knowledge/application/search' import { executeSync } from '@/lib/knowledge/connectors/sync-engine' -import { readSearchDocument } from '@/lib/sim-search/indexed/documents/read-search-document' import * as storage from '@/lib/uploads/core/storage-service' import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' import { PATCH as updateConnectorRoute } from '@/app/api/knowledge/[id]/connectors/[connectorId]/route' import { POST as createConnectorRoute } from '@/app/api/knowledge/[id]/connectors/route' import { gitlabConnector } from '@/connectors/gitlab/gitlab' -import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' interface GitLabPerson { id: number @@ -700,336 +685,297 @@ describe.skipIf(!fixtureFile)('live self-hosted GitLab ingestion and permission throw new Error('Disposable connector sync did not finish within one minute') } - it.each([false, true])( - 'enforces non-admin CSV permissions through authenticated API, indexing and every read surface (Search=%s)', - async (isSearchIndex) => { - await api(`/projects/${projectId}`, 'PUT', { visibility: 'private' }) - await api(`/projects/${projectId}/issues/${issueIid}`, 'PUT', { confidential: false }) - const owner = isSearchIndex - ? { organizationId: ids.organizationId } - : { workspaceId: ids.workspaceId } - if (isSearchIndex) { - await db.insert(member).values( - Object.values(people).map((person) => ({ - id: generateId(), - userId: person.simId, - organizationId: ids.organizationId, - role: person.simId === ids.aliceId ? 'owner' : 'member', - createdAt: new Date(), - })) - ) - await db - .insert(organizationSearchIntegration) - .values({ organizationId: ids.organizationId, connectorType: 'gitlab', approved: true }) - } - const attribution = isSearchIndex - ? await resolveOrganizationBillingAttribution({ - actorUserId: ids.aliceId, - organizationId: ids.organizationId, - }) - : await resolveBillingAttribution({ - actorUserId: ids.aliceId, - workspaceId: ids.workspaceId, - }) - const knowledgeBaseId = generateId() - await db.insert(knowledgeBase).values({ - id: knowledgeBaseId, - userId: ids.aliceId, - ...owner, - name: `CSV live fixture ${isSearchIndex ? 'Search' : 'KB'}`, - isSearchIndex, - chunkingConfig: { maxSize: 1024, minSize: 1, overlap: 20 }, - }) - const mapping = { - filename: 'users.csv', - content: `user_id,email\n${people.reporter.id},${people.reporter.email}\n${people.guest.id},${people.guest.email}\n`, - } - const projectPermissions = (userId: number) => ({ - filename: 'permissions.csv', - content: `project_path,user_id\n${config.project},${userId}\nunrelated/project,${people.guest.id}\n${config.project},999999999\n`, - }) - if (!auditorToken) await waitForProjectAccess(people.reporter, 200) - const indexingToken = auditorToken ?? people.reporter.token - const createBody: CreateConnectorBody = { - connectorType: 'gitlab', - accessMode: 'admin', - apiKey: indexingToken, - sourceConfig: { ...config, contentTypes: 'issues' }, - syncIntervalMinutes: 60, - permissionConfig: { - provider: 'gitlab', - mode: 'csv', - userMapping: mapping, - projectPermissions: projectPermissions(people.reporter.id), - }, - } - const denied = await connectorRequest(knowledgeBaseId, createBody, undefined, readerCookie) - expect(denied.status).toBe(403) - const unsupported = await connectorRequest(knowledgeBaseId, { - ...createBody, - accessMode: 'workspace', - }) - expect(unsupported.status).toBe(400) - const inaccessibleProject = await connectorRequest(knowledgeBaseId, { - ...createBody, - sourceConfig: { ...createBody.sourceConfig, project: 'missing-fixture-project' }, - }) - expect(inaccessibleProject.status).toBe(400) - const createdResponse = await connectorRequest(knowledgeBaseId, createBody) - const created = await createdResponse.json() - expect(createdResponse.status, JSON.stringify(created)).toBe(201) - let connector = created.data as ConnectorData - expect(connector.permissionConfig).toMatchObject({ + it('enforces non-admin CSV permissions through authenticated API, indexing and every knowledge-base read surface', async () => { + await api(`/projects/${projectId}`, 'PUT', { visibility: 'private' }) + await api(`/projects/${projectId}/issues/${issueIid}`, 'PUT', { confidential: false }) + const owner = { workspaceId: ids.workspaceId } + const attribution = await resolveBillingAttribution({ + actorUserId: ids.aliceId, + workspaceId: ids.workspaceId, + }) + const knowledgeBaseId = generateId() + await db.insert(knowledgeBase).values({ + id: knowledgeBaseId, + userId: ids.aliceId, + ...owner, + name: 'CSV live fixture KB', + chunkingConfig: { maxSize: 1024, minSize: 1, overlap: 20 }, + }) + const mapping = { + filename: 'users.csv', + content: `user_id,email\n${people.reporter.id},${people.reporter.email}\n${people.guest.id},${people.guest.email}\n`, + } + const projectPermissions = (userId: number) => ({ + filename: 'permissions.csv', + content: `project_path,user_id\n${config.project},${userId}\nunrelated/project,${people.guest.id}\n${config.project},999999999\n`, + }) + if (!auditorToken) await waitForProjectAccess(people.reporter, 200) + const indexingToken = auditorToken ?? people.reporter.token + const createBody: CreateConnectorBody = { + connectorType: 'gitlab', + accessMode: 'admin', + apiKey: indexingToken, + sourceConfig: { ...config, contentTypes: 'issues' }, + syncIntervalMinutes: 60, + permissionConfig: { provider: 'gitlab', mode: 'csv', - revision: 1, - userMapping: { rowCount: 2 }, - }) - expect(JSON.stringify(created)).not.toContain(indexingToken) - expect(JSON.stringify(created)).not.toContain(people.reporter.email) - expect(connector.sourceConfig).not.toHaveProperty('permissionConfig') - const storedConnector = await waitForSync(connector.id) - expect(storedConnector.encryptedApiKey).not.toBe(indexingToken) - expect( - (await decryptApiKey(storedConnector.encryptedApiKey!)).decrypted === indexingToken - ).toBe(true) - const docs = await db.select().from(document).where(eq(document.connectorId, connector.id)) - const ordinary = docs.find((doc) => doc.externalId === `issue:${issueIid}`)! - expect(ordinary.processingStatus).toBe('completed') - const excluded = docs.find((doc) => doc.externalId === `issue:${confidentialIid}`) - if (excluded) { - expect(excluded.acl).toEqual([]) - expect(excluded.storageKey).toBeNull() - } - const searchFor = async (person: GitLabPerson) => - ( - await searchKnowledge.execute({ + userMapping: mapping, + projectPermissions: projectPermissions(people.reporter.id), + }, + } + const denied = await connectorRequest(knowledgeBaseId, createBody, undefined, readerCookie) + expect(denied.status).toBe(403) + const unsupported = await connectorRequest(knowledgeBaseId, { + ...createBody, + accessMode: 'workspace', + }) + expect(unsupported.status).toBe(400) + const inaccessibleProject = await connectorRequest(knowledgeBaseId, { + ...createBody, + sourceConfig: { ...createBody.sourceConfig, project: 'missing-fixture-project' }, + }) + expect(inaccessibleProject.status).toBe(400) + const createdResponse = await connectorRequest(knowledgeBaseId, createBody) + const created = await createdResponse.json() + expect(createdResponse.status, JSON.stringify(created)).toBe(201) + let connector = created.data as ConnectorData + expect(connector.permissionConfig).toMatchObject({ + provider: 'gitlab', + mode: 'csv', + revision: 1, + userMapping: { rowCount: 2 }, + }) + expect(JSON.stringify(created)).not.toContain(indexingToken) + expect(JSON.stringify(created)).not.toContain(people.reporter.email) + expect(connector.sourceConfig).not.toHaveProperty('permissionConfig') + const storedConnector = await waitForSync(connector.id) + expect(storedConnector.encryptedApiKey).not.toBe(indexingToken) + expect( + (await decryptApiKey(storedConnector.encryptedApiKey!)).decrypted === indexingToken + ).toBe(true) + const docs = await db.select().from(document).where(eq(document.connectorId, connector.id)) + const ordinary = docs.find((doc) => doc.externalId === `issue:${issueIid}`)! + expect(ordinary.processingStatus).toBe('completed') + const excluded = docs.find((doc) => doc.externalId === `issue:${confidentialIid}`) + if (excluded) { + expect(excluded.acl).toEqual([]) + expect(excluded.storageKey).toBeNull() + } + const searchFor = async (person: GitLabPerson) => + ( + await searchKnowledge.execute({ + principal: principal(person), + input: { + ...owner, + knowledgeBaseIds: [knowledgeBaseId], + query: 'Orion', + topK: 100, + }, + }) + ).results.map((row) => row.documentId) + const assertReads = async (person: GitLabPerson, allowed: boolean) => { + expect((await searchFor(person)).includes(ordinary.id)).toBe(allowed) + const operations: Array<() => Promise> = [ + () => + readKnowledgeDocument.execute({ principal: principal(person), - input: { - ...owner, - knowledgeBaseIds: [knowledgeBaseId], - query: 'Orion', - topK: 100, - }, - }) - ).results.map((row) => row.documentId) - const assertReads = async (person: GitLabPerson, allowed: boolean) => { - expect((await searchFor(person)).includes(ordinary.id)).toBe(allowed) - const operations: Array<() => Promise> = [ - () => - readKnowledgeDocument.execute({ - principal: principal(person), - input: { knowledgeBaseId, documentId: ordinary.id }, - }), - () => - listKnowledgeChunks.execute({ - principal: principal(person), - input: { knowledgeBaseId, documentId: ordinary.id, limit: 10, offset: 0 }, - }), - ] - const download = () => - downloadFileFromUrl(ordinary.fileUrl!, { userId: person.simId, knowledgeAccess: 'user' }) - if (isSearchIndex) { - await expect(download()).rejects.toThrow('Access denied') - operations.push(() => - readSearchDocument.execute({ - principal: principal(person), - input: { - documentId: ordinary.id, - assertedOrganizationId: ids.organizationId, - limit: 3, - resultSecretRegistry: new ResolvedSecretTraceRegistry(), - }, - }) - ) - } else operations.push(download) - for (const operation of operations) { - if (allowed) await expect(operation()).resolves.toBeDefined() - else await expect(operation()).rejects.toBeDefined() - } + input: { knowledgeBaseId, documentId: ordinary.id }, + }), + () => + listKnowledgeChunks.execute({ + principal: principal(person), + input: { knowledgeBaseId, documentId: ordinary.id, limit: 10, offset: 0 }, + }), + ] + const download = () => + downloadFileFromUrl(ordinary.fileUrl!, { userId: person.simId, knowledgeAccess: 'user' }) + operations.push(download) + for (const operation of operations) { + if (allowed) await expect(operation()).resolves.toBeDefined() + else await expect(operation()).rejects.toBeDefined() } - await assertReads(people.reporter, true) - await assertReads(people.guest, false) - await assertReads(people.outsider, false) - const vectors = await db - .select({ content: embedding.content }) - .from(embedding) - .where(eq(embedding.documentId, ordinary.id)) - expect(vectors.map((row) => row.content).join('\n')).not.toContain( - 'INTERNAL_FIXTURE_NOTE_MUST_NOT_BE_INDEXED' - ) - const before = fixture.embeddingCalls - const tooLarge = await connectorRequest( - knowledgeBaseId, - { - permissionConfig: { - provider: 'gitlab', - mode: 'csv', - expectedRevision: 1, - userMapping: { filename: 'large.csv', content: 'x'.repeat(4 * 1024 * 1024 + 1) }, - }, - }, - connector.id - ) - expect(tooLarge.status).toBe(400) - const overRequestLimit = await connectorRequest( - knowledgeBaseId, - { - sourceConfig: { description: 'x'.repeat(10 * 1024 * 1024) }, + } + await assertReads(people.reporter, true) + await assertReads(people.guest, false) + await assertReads(people.outsider, false) + const vectors = await db + .select({ content: embedding.content }) + .from(embedding) + .where(eq(embedding.documentId, ordinary.id)) + expect(vectors.map((row) => row.content).join('\n')).not.toContain( + 'INTERNAL_FIXTURE_NOTE_MUST_NOT_BE_INDEXED' + ) + const before = fixture.embeddingCalls + const tooLarge = await connectorRequest( + knowledgeBaseId, + { + permissionConfig: { + provider: 'gitlab', + mode: 'csv', + expectedRevision: 1, + userMapping: { filename: 'large.csv', content: 'x'.repeat(4 * 1024 * 1024 + 1) }, }, - connector.id - ) - expect(overRequestLimit.status).toBe(413) - const invalid = await connectorRequest( - knowledgeBaseId, - { - permissionConfig: { - provider: 'gitlab', - mode: 'csv', - expectedRevision: 1, - userMapping: { filename: 'bad.csv', content: '1,not-an-email' }, - }, + }, + connector.id + ) + expect(tooLarge.status).toBe(400) + const overRequestLimit = await connectorRequest( + knowledgeBaseId, + { + sourceConfig: { description: 'x'.repeat(10 * 1024 * 1024) }, + }, + connector.id + ) + expect(overRequestLimit.status).toBe(413) + const invalid = await connectorRequest( + knowledgeBaseId, + { + permissionConfig: { + provider: 'gitlab', + mode: 'csv', + expectedRevision: 1, + userMapping: { filename: 'bad.csv', content: '1,not-an-email' }, }, - connector.id - ) - expect(invalid.status).toBe(400) - await assertReads(people.reporter, true) - const replace = await connectorRequest( - knowledgeBaseId, - { - permissionConfig: { - provider: 'gitlab', - mode: 'csv', - expectedRevision: 1, - projectPermissions: projectPermissions(people.guest.id), - }, + }, + connector.id + ) + expect(invalid.status).toBe(400) + await assertReads(people.reporter, true) + const replace = await connectorRequest( + knowledgeBaseId, + { + permissionConfig: { + provider: 'gitlab', + mode: 'csv', + expectedRevision: 1, + projectPermissions: projectPermissions(people.guest.id), }, - connector.id - ) - expect(replace.status).toBe(200) - connector = (await replace.json()).data - expect(connector.permissionConfig?.revision).toBe(2) - await assertReads(people.reporter, false) - await assertReads(people.guest, true) - expect(fixture.embeddingCalls).toBe(before) - const stale = await connectorRequest( - knowledgeBaseId, - { - permissionConfig: { - provider: 'gitlab', - mode: 'csv', - expectedRevision: 1, - projectPermissions: projectPermissions(people.reporter.id), - }, + }, + connector.id + ) + expect(replace.status).toBe(200) + connector = (await replace.json()).data + expect(connector.permissionConfig?.revision).toBe(2) + await assertReads(people.reporter, false) + await assertReads(people.guest, true) + expect(fixture.embeddingCalls).toBe(before) + const stale = await connectorRequest( + knowledgeBaseId, + { + permissionConfig: { + provider: 'gitlab', + mode: 'csv', + expectedRevision: 1, + projectPermissions: projectPermissions(people.reporter.id), }, - connector.id - ) - expect(stale.status).toBe(409) - const concurrent = await Promise.all( - [people.reporter, people.guest].map((person) => - connectorRequest( - knowledgeBaseId, - { - permissionConfig: { - provider: 'gitlab', - mode: 'csv', - expectedRevision: 2, - projectPermissions: projectPermissions(person.id), - }, - }, - connector.id - ) - ) - ) - expect(concurrent.map((result) => result.status).sort()).toEqual([200, 409]) - const grants = await db - .select() - .from(knowledgeConnectorPermissionGrant) - .where(eq(knowledgeConnectorPermissionGrant.connectorId, connector.id)) - expect(grants).toHaveLength(1) - let winner = - grants[0].subjectToken === `u:${people.reporter.email}` ? people.reporter : people.guest - await assertReads(winner, true) - expect(fixture.embeddingCalls).toBe(before) - await db.update(user).set({ emailVerified: false }).where(eq(user.id, winner.simId)) - await assertReads(winner, false) - await db.update(user).set({ emailVerified: true }).where(eq(user.id, winner.simId)) - const removedMember = winner - const newlyMappedMember = winner === people.reporter ? people.guest : people.reporter - const remapped = await connectorRequest( - knowledgeBaseId, - { - permissionConfig: { - provider: 'gitlab', - mode: 'csv', - expectedRevision: 3, - userMapping: { - filename: 'replacement-users.csv', - content: `${winner.id},${newlyMappedMember.email}`, + }, + connector.id + ) + expect(stale.status).toBe(409) + const concurrent = await Promise.all( + [people.reporter, people.guest].map((person) => + connectorRequest( + knowledgeBaseId, + { + permissionConfig: { + provider: 'gitlab', + mode: 'csv', + expectedRevision: 2, + projectPermissions: projectPermissions(person.id), }, }, - }, - connector.id + connector.id + ) ) - expect(remapped.status).toBe(200) - await assertReads(removedMember, false) - winner = newlyMappedMember - await assertReads(winner, true) - expect(fixture.embeddingCalls).toBe(before) - const crossOwner = await connectorRequest( - ids.knowledgeBaseId, - { - permissionConfig: { - provider: 'gitlab', - mode: 'csv', - expectedRevision: 3, - projectPermissions: projectPermissions(people.reporter.id), + ) + expect(concurrent.map((result) => result.status).sort()).toEqual([200, 409]) + const grants = await db + .select() + .from(knowledgeConnectorPermissionGrant) + .where(eq(knowledgeConnectorPermissionGrant.connectorId, connector.id)) + expect(grants).toHaveLength(1) + let winner = + grants[0].subjectToken === `u:${people.reporter.email}` ? people.reporter : people.guest + await assertReads(winner, true) + expect(fixture.embeddingCalls).toBe(before) + await db.update(user).set({ emailVerified: false }).where(eq(user.id, winner.simId)) + await assertReads(winner, false) + await db.update(user).set({ emailVerified: true }).where(eq(user.id, winner.simId)) + const removedMember = winner + const newlyMappedMember = winner === people.reporter ? people.guest : people.reporter + const remapped = await connectorRequest( + knowledgeBaseId, + { + permissionConfig: { + provider: 'gitlab', + mode: 'csv', + expectedRevision: 3, + userMapping: { + filename: 'replacement-users.csv', + content: `${winner.id},${newlyMappedMember.email}`, }, }, - connector.id - ) - expect(crossOwner.status).toBe(404) - await api(`/projects/${projectId}/issues/${issueIid}`, 'PUT', { confidential: true }) - const syncResult = await executeSync(connector.id, { - billingAttribution: attribution, - fullSync: true, - }) - expect(syncResult.docsFailed).toBe(0) - await assertReads(winner, false) - const [removed] = await db.select().from(document).where(eq(document.id, ordinary.id)) - expect(removed.storageKey).toBeNull() - expect(removed.acl).toEqual([]) - expect( - await db.select().from(embedding).where(eq(embedding.documentId, ordinary.id)) - ).toEqual([]) - const [snapshot] = await db - .select() - .from(knowledgeConnectorPermissionSnapshot) - .where(eq(knowledgeConnectorPermissionSnapshot.connectorId, connector.id)) - expect(snapshot.revision).toBe(4) - await connectorRequest(knowledgeBaseId, { status: 'paused' }, connector.id) - const switched = await connectorRequest( - knowledgeBaseId, - { - apiKey: adminToken, - permissionConfig: { provider: 'gitlab', mode: 'administrator', expectedRevision: 4 }, + }, + connector.id + ) + expect(remapped.status).toBe(200) + await assertReads(removedMember, false) + winner = newlyMappedMember + await assertReads(winner, true) + expect(fixture.embeddingCalls).toBe(before) + const crossOwner = await connectorRequest( + ids.knowledgeBaseId, + { + permissionConfig: { + provider: 'gitlab', + mode: 'csv', + expectedRevision: 3, + projectPermissions: projectPermissions(people.reporter.id), }, - connector.id - ) - expect(switched.status).toBe(200) - expect((await switched.json()).data.accessRewritePending).toBe(true) - await assertReads(winner, false) - await connectorRequest(knowledgeBaseId, { status: 'active' }, connector.id) - await executeSync(connector.id, { - billingAttribution: attribution, - fullSync: true, - }) - const [restored] = await db - .select() - .from(knowledgeConnector) - .where(eq(knowledgeConnector.id, connector.id)) - expect(restored.accessRewritePending).toBe(false) - }, - 180000 - ) + }, + connector.id + ) + expect(crossOwner.status).toBe(404) + await api(`/projects/${projectId}/issues/${issueIid}`, 'PUT', { confidential: true }) + const syncResult = await executeSync(connector.id, { + billingAttribution: attribution, + fullSync: true, + }) + expect(syncResult.docsFailed).toBe(0) + await assertReads(winner, false) + const [removed] = await db.select().from(document).where(eq(document.id, ordinary.id)) + expect(removed.storageKey).toBeNull() + expect(removed.acl).toEqual([]) + expect(await db.select().from(embedding).where(eq(embedding.documentId, ordinary.id))).toEqual( + [] + ) + const [snapshot] = await db + .select() + .from(knowledgeConnectorPermissionSnapshot) + .where(eq(knowledgeConnectorPermissionSnapshot.connectorId, connector.id)) + expect(snapshot.revision).toBe(4) + await connectorRequest(knowledgeBaseId, { status: 'paused' }, connector.id) + const switched = await connectorRequest( + knowledgeBaseId, + { + apiKey: adminToken, + permissionConfig: { provider: 'gitlab', mode: 'administrator', expectedRevision: 4 }, + }, + connector.id + ) + expect(switched.status).toBe(200) + expect((await switched.json()).data.accessRewritePending).toBe(true) + await assertReads(winner, false) + await connectorRequest(knowledgeBaseId, { status: 'active' }, connector.id) + await executeSync(connector.id, { + billingAttribution: attribution, + fullSync: true, + }) + const [restored] = await db + .select() + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, connector.id)) + expect(restored.accessRewritePending).toBe(false) + }, 180000) }) diff --git a/apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts b/apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts index 716af7b1b66..af955d4df2b 100644 --- a/apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts @@ -11,7 +11,6 @@ import { credentialGroupEnrollment, document, embedding, - knowledgeBase, knowledgeConnector, knowledgeConnectorMember, knowledgeDocumentObservation, @@ -24,12 +23,6 @@ import { eq, inArray } from 'drizzle-orm' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' const counters = vi.hoisted(() => ({ embeddedTexts: 0 })) -/** This suite covers indexed organization search, which is dormant unless Live Search is off. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) vi.mock('@/lib/embeddings', async () => ({ ...(await import('@/lib/embeddings/client')), assertKnowledgeEmbeddingCapacity: async () => {}, @@ -224,10 +217,6 @@ describe('Gmail member ingestion and ACLs in PostgreSQL (provider fixtures)', () credentialGroupId: fixture.groupId, credentialGroupOptionId: fixture.optionId, }) - await db - .update(knowledgeBase) - .set({ isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) await db .update(credentialGroup) .set({ diff --git a/apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts b/apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts index 2a66fdbf151..4d406455565 100644 --- a/apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts @@ -12,7 +12,6 @@ import { credentialGroup, credentialGroupEnrollment, document, - knowledgeBase, knowledgeConnector, knowledgeConnectorMember, knowledgeDocumentObservation, @@ -24,12 +23,6 @@ import { generateId } from '@sim/utils/id' import { and, eq, inArray, isNull } from 'drizzle-orm' import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' -/** This suite covers indexed organization search, which is dormant unless Live Search is off. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) vi.mock('@/lib/embeddings', async () => ({ ...(await import('@/lib/embeddings/client')), assertKnowledgeEmbeddingCapacity: async () => {}, @@ -208,10 +201,6 @@ describe('Google Calendar member indexing and authorization in PostgreSQL', () = }) }) await seedKnowledgeAclFixture(ids) - await db - .update(knowledgeBase) - .set({ isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) const policy = await getCredentialGroupProviderAdapter('google-calendar').getPolicy(undefined, { workspaceId: ids.workspaceId, }) diff --git a/apps/sim/lib/knowledge/__integration__/jira-member.integration.ts b/apps/sim/lib/knowledge/__integration__/jira-member.integration.ts index e9d76507746..b3ae46cd07f 100644 --- a/apps/sim/lib/knowledge/__integration__/jira-member.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/jira-member.integration.ts @@ -13,7 +13,6 @@ import { credentialGroupEnrollment, document, embedding, - knowledgeBase, knowledgeConnector, knowledgeConnectorMember, knowledgeDocumentObservation, @@ -25,12 +24,6 @@ import { generateId } from '@sim/utils/id' import { and, eq, inArray, isNull } from 'drizzle-orm' import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' -/** This suite covers indexed organization search, which is dormant unless Live Search is off. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) vi.mock('@/lib/embeddings', async () => ({ ...(await import('@/lib/embeddings/client')), assertKnowledgeEmbeddingCapacity: async () => {}, @@ -188,10 +181,6 @@ describe('Jira member indexing and authorization in PostgreSQL', () => { }) }) await seedKnowledgeAclFixture(ids) - await db - .update(knowledgeBase) - .set({ isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) const policy = await getCredentialGroupProviderAdapter('jira').getPolicy(undefined, { workspaceId: ids.workspaceId, }) diff --git a/apps/sim/lib/knowledge/__integration__/kb-block-search.integration.ts b/apps/sim/lib/knowledge/__integration__/kb-block-search.integration.ts index 9eff697046f..7ebef980cfb 100644 --- a/apps/sim/lib/knowledge/__integration__/kb-block-search.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/kb-block-search.integration.ts @@ -3,19 +3,15 @@ import type { Principal } from '@sim/auth/principal' import { db } from '@sim/db' import { document, embedding, knowledgeBase, organization, user, workspace } from '@sim/db/schema' import { generateId } from '@sim/utils/id' -import { eq, inArray, sql } from 'drizzle-orm' +import { eq, inArray } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, it } from 'vitest' import { createKnowledgeAclFixtureIds, seedKnowledgeAclFixture, } from '@/lib/knowledge/__integration__/seed-source-access-fixture' import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' -import { VECTOR_PROBE_DOCUMENT_LIMIT } from '@/lib/knowledge/search/candidates' import { retrieveKnowledgeSearch } from '@/lib/knowledge/search/queries' import { embeddingVectorValues } from '@/lib/knowledge/vector-columns' -import { resolveSearchAccessPlan } from '@/lib/sim-search/indexed/retrieval/access-plan' -import { resolvePermittedDocuments } from '@/lib/sim-search/indexed/retrieval/permitted' -import { forgetProjectionFilled } from '@/lib/sim-search/indexed/retrieval/projection-fill' describe('API-key KB block fan-out', () => { const ids = createKnowledgeAclFixtureIds() @@ -92,8 +88,6 @@ describe('API-key KB block fan-out', () => { it.each([false, true])( 'completes 18 concurrent KB searches with access checks intact (tag filter: %s)', async (withTags) => { - /** A cold process must also skip the global readiness probe for ordinary KBs. */ - forgetProjectionFilled() const previousDebug = db.$client.options.debug const statements: string[] = [] db.$client.options.debug = (_connection, query) => { @@ -159,37 +153,4 @@ describe('API-key KB block fan-out', () => { } } ) - - it('bounds the permitted set by the requested bases, not by what the tokens reach elsewhere', async () => { - const crowded = generateId() - await db.insert(knowledgeBase).values({ - id: crowded, - userId: ids.aliceId, - workspaceId: ids.workspaceId, - name: 'Crowded neighbour', - }) - try { - /** Baseline tokens are shared by every tenant, so another base can hold more than the limit. */ - await db.execute(sql` - INSERT INTO ${document} (id, knowledge_base_id, filename, file_url, file_size, mime_type, - processing_status, acl) - SELECT 'crowded-' || n, ${crowded}, 'crowded', 'https://fixture.invalid/crowded', 1, - 'text/plain', 'completed', ARRAY['ws']::text[] - FROM generate_series(1, ${VECTOR_PROBE_DOCUMENT_LIMIT + 1}) AS n - `) - const access = { kind: 'user' as const, userId: ids.bobId, tokens: ['pub', 'ws'] } - const permitted = await resolvePermittedDocuments({ - knowledgeBaseIds: [bases[0].id], - access, - accessPlan: await resolveSearchAccessPlan([bases[0].id], access), - filtered: false, - }) - expect(permitted).toEqual({ - kind: 'bounded', - documents: [{ id: bases[0].visible, connectorId: null }], - }) - } finally { - await db.delete(knowledgeBase).where(eq(knowledgeBase.id, crowded)) - } - }) }) diff --git a/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts b/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts index 56ff0027e65..3b9f4f8a174 100644 --- a/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/knowledge-projection.integration.ts @@ -1,232 +1,43 @@ -/** - * The knowledge projector and the readers that must stay correct while it lags. A GitHub member - * source's document in a search index is changed by a writer in either projection mode — - * synchronous, as every writer now is, or deferred, as writers of earlier releases could be, - * leaving only a mark — and search is checked before the - * projector runs: a revoked member is refused and a granted one is served, on the vector and - * keyword legs and under the source filter, and a disabled or deleted chunk is gone at once. The - * projector's own contract follows: it converges the rows and removes the mark, keeps a mark that - * a write bumped during its pass, survives a document deleted under it, writes in pages bounded by - * chunk rows, releases workspace marks with nothing to project without a pass, and a deferred - * commit writes no projection row at all. - */ -import { createHash } from 'node:crypto' +/** Real PostgreSQL proof that legacy deferred vector repair survives indexed Search retirement. */ import { db } from '@sim/db' import { hasKnowledgeProjectionWork, - type KnowledgeProjection, releaseSettledMarks, runKnowledgeProjection, } from '@sim/db/knowledge-projection' import { - credential, - credentialGroup, - credentialGroupEnrollment, document, embedding, embeddingKeywordSearch, embeddingKeywordTin, embeddingSearch, knowledgeBase, - knowledgeConnector, - knowledgeConnectorMember, - knowledgeDocumentObservation, knowledgeProjectionDirty, organization, user, workspace, } from '@sim/db/schema' -import { sleep } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' -import { and, eq, inArray, sql } from 'drizzle-orm' +import { eq, inArray, sql } from 'drizzle-orm' import postgres from 'postgres' import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' - -/** This suite covers indexed organization search, which is dormant unless Live Search is off. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) -/** The TINQL `resolveTinKeywordQuery` renders for `fixture`: its `english` stem, quoted. */ -vi.mock('@/lib/sim-search/indexed/retrieval/tin-keyword', () => ({ - resolveTinKeywordQuery: async () => '"fixtur"', -})) - -vi.mock('@/lib/core/config/feature-flags', () => ({ isFeatureEnabled: async () => false })) - import { createKnowledgeAclFixtureIds, seedKnowledgeAclFixture, } from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import type { - GitHubInstallationReadGrant, - KnowledgeAccessProvider, - UserAccessScope, -} from '@/lib/knowledge/access/types' -import { leaseTransaction } from '@/lib/knowledge/connectors/sync-lock' -import { liveSourceAccessForConnectors } from '@/lib/knowledge/search/candidates' -import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' -import type { SearchAccessPlan } from '@/lib/sim-search/indexed/retrieval/access-plan' -import { executeIndexedKeywordSearch } from '@/lib/sim-search/indexed/retrieval/keyword' -import type { IndexedRetrievalContext } from '@/lib/sim-search/indexed/retrieval/permitted' -import { projectionCandidateAccessCondition } from '@/lib/sim-search/indexed/retrieval/projection-access' -import { forgetProjectionFilled } from '@/lib/sim-search/indexed/retrieval/projection-fill' -import { selectIndexedVectorResults } from '@/lib/sim-search/indexed/retrieval/vector' const ids = createKnowledgeAclFixtureIds() -const connectorId = generateId() -const otherConnectorId = generateId() -const contentCredentialId = generateId() -const groupId = generateId() -const optionId = generateId() +const searchBaseId = generateId() const documentId = generateId() -const chunkId = generateId() -const repositoryId = '4242' -const members = { - alice: { id: generateId(), subject: 'alice-gh', credentialId: generateId() }, - bob: { id: generateId(), subject: 'bob-gh', credentialId: generateId() }, -} -const subjectToken = (subject: string) => `s:github-repositories:-:${subject}` -const aclOf = (...who: Array<'alice' | 'bob'>) => - who.map((name) => subjectToken(members[name].subject)).sort() -/** - * A direction no other integration file writes, so no row of theirs ties with this file's chunks. - * Inside the first 512 dimensions, the only ones the candidate projection keeps. - */ +const searchDocumentId = generateId() const vector = Array.from({ length: 1536 }, (_, index) => (index === 511 ? 1 : 0)) -const queryVector = { - vector: JSON.stringify(vector), - dimensions: 1536 as const, - model: 'text-embedding-3-small', -} - -/** Alice holds the installation grant, so what she reads is decided by the document's ACL alone. */ -const scope: UserAccessScope = { - kind: 'user', - userId: ids.aliceId, - tokens: [ - 'pub', - subjectToken(members.alice.subject), - `u:${ids.aliceId}@fixture.test`, - 'ws', - ].sort(), -} -const plan: SearchAccessPlan = { - connectors: { - workspace: [], - admin: [], - members: [connectorId], - liveProofRequired: [connectorId], - }, - observers: { confirmed: [{ id: members.alice.id, connectorId }], observed: [] }, - memberSources: [connectorId], - connectorTypes: new Map([[connectorId, 'github']]), - uploads: false, -} -const grant: GitHubInstallationReadGrant = { - connectorId, - contentCredentialId, - readerCredentialId: members.alice.credentialId, - readerSubjectToken: subjectToken(members.alice.subject), - repositoryId, -} - -const searchInputs = { - knowledgeBaseIds: [ids.knowledgeBaseId], - topK: 20, - access: scope, - queryVector, -} - -/** A narrow reader of the search index, who proves the installation grant live. */ -function searchContext(): IndexedRetrievalContext { - const granted = { ...scope, githubInstallationGrants: [grant] } - const accessProvider: KnowledgeAccessProvider = { - get: async () => scope, - getForConnectors: async () => granted, - getForDocuments: async () => granted, - liveSourceConnectorCondition: async () => null, - } - return { - access: scope, - accessPlan: plan, - filtered: false, - permitted: { kind: 'unbounded', broad: false }, - liveSourceAccess: liveSourceAccessForConnectors( - plan.connectors.liveProofRequired, - accessProvider - ), - } -} - -const keywordIds = async () => - (await executeIndexedKeywordSearch({ ...searchInputs, query: 'fixture' }, searchContext())) - .map((row) => row.id) - .sort() - -/** - * Ranked exactly on the row, under the same visibility predicate the graph walk applies. The walk - * is approximate: in a graph the other files sharing this database crowd with degenerate vectors, - * a chunk can be pruned from every neighbour list and never be reached, however far the walk goes. - */ -const vectorIds = async () => - (await selectIndexedVectorResults({ ...searchInputs, distanceThreshold: 2 }, searchContext())) - .map((row) => row.id) - .sort() - -/** The chunks of the fixture document each ranking projection admits for Alice. */ -async function admitted(): Promise> { - const [vectorRows, keywordRows] = await Promise.all([ - db - .select({ id: embeddingSearch.id }) - .from(embeddingSearch) - .where( - and( - eq(embeddingSearch.documentId, documentId), - projectionCandidateAccessCondition(embeddingSearch, scope, plan, { filled: true }) - ) - ), - db - .select({ id: embeddingKeywordTin.id }) - .from(embeddingKeywordTin) - .where( - and( - eq(embeddingKeywordTin.documentId, documentId), - projectionCandidateAccessCondition(embeddingKeywordTin, scope, plan, { filled: true }) - ) - ), - ]) - return { - vector: vectorRows.map((row) => row.id).sort(), - keyword: keywordRows.map((row) => row.id).sort(), - } -} - -type Mode = 'sync' | 'async' - -/** - * What a writer of a release that deferred its projection selected first: the setting that skips - * the synchronous projection triggers, which the database still honours. - */ -const DEFER_PROJECTION = `SELECT set_config('sim.projection_mode', 'async', true)` - -/** Runs a write in a transaction of the given projection mode, as a knowledge writer would. */ -function write( - mode: Mode, - work: (tx: Parameters[0]>[0]) => Promise -) { - return db.transaction(async (tx) => { - if (mode === 'async') await tx.execute(sql.raw(DEFER_PROJECTION)) - await work(tx) - }) -} +let projector: postgres.Sql -function chunkRow(id: string, chunkIndex: number) { +function chunkRow(id: string, chunkIndex: number, search = false) { return { id, - documentId, - knowledgeBaseId: ids.knowledgeBaseId, + documentId: search ? searchDocumentId : documentId, + knowledgeBaseId: search ? searchBaseId : ids.knowledgeBaseId, chunkIndex, chunkHash: `fixture-hash-${chunkIndex}`, content: 'fixture readme', @@ -239,737 +50,311 @@ function chunkRow(id: string, chunkIndex: number) { } } -const markOf = async () => { - const [row] = await db - .select() - .from(knowledgeProjectionDirty) - .where(eq(knowledgeProjectionDirty.documentId, documentId)) - return row +/** Reproduces pending writes from the older release that deferred its projections. */ +async function defer( + work: (tx: Parameters[0]>[0]) => Promise +) { + await db.transaction(async (tx) => { + await tx.execute(sql`SELECT set_config('sim.projection_mode', 'async', true)`) + await work(tx) + }) } -const rowAcl = async (table: typeof embeddingSearch | typeof embeddingKeywordTin, id = chunkId) => { +const markOf = async (id = documentId) => { const [row] = await db - .select({ acl: table.acl, connectorId: table.connectorId, enabled: table.enabled }) - .from(table) - .where(eq(table.id, id)) + .select() + .from(knowledgeProjectionDirty) + .where(eq(knowledgeProjectionDirty.documentId, id)) return row } -/** The projector's connection: a pass holds its per-document advisory locks on it. */ -let projector: postgres.Sql -const project = (options: Partial[1]> = {}) => - runKnowledgeProjection(projector, { searchIndexes: true, ...options }) - -/** Shims for the Tin extension, which the test database does not carry. */ -let createdTinShims = false +const vectorsOf = (id = documentId) => + db + .select({ + id: embeddingSearch.id, + enabled: embeddingSearch.enabled, + vector512: embeddingSearch.vector512, + acl: embeddingSearch.acl, + connectorId: embeddingSearch.connectorId, + }) + .from(embeddingSearch) + .where(eq(embeddingSearch.documentId, id)) beforeAll(async () => { - projector = postgres(process.env.DATABASE_URL!, { max: 1, onnotice: () => undefined }) await seedKnowledgeAclFixture(ids) - const now = new Date() - await db - .update(knowledgeBase) - .set({ isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) - await db.insert(credential).values({ - id: contentCredentialId, - workspaceId: ids.workspaceId, - type: 'service_account', - displayName: 'Fixture GitHub installation', - createdBy: ids.aliceId, - providerId: GITHUB_INSTALLATION_PROVIDER_ID, - }) - await db.insert(credentialGroup).values({ - id: groupId, + projector = postgres(process.env.DATABASE_URL!, { max: 1, onnotice: () => undefined }) + await db.insert(knowledgeBase).values({ + id: searchBaseId, + userId: ids.aliceId, workspaceId: ids.workspaceId, - publicId: generateId(), - name: 'GitHub readers', - options: [ - { - id: optionId, - provider: 'github-repositories', - label: 'GitHub fixture', - authorizationAppId: 'fixture-app', - requiredScopes: ['repo'], - scopeVersion: 1, - required: false, - status: 'active', - }, - ], - } as typeof credentialGroup.$inferInsert) - for (const who of ['alice', 'bob'] as const) { - const userId = who === 'alice' ? ids.aliceId : ids.bobId - const [enrollment] = await db - .insert(credentialGroupEnrollment) - .values({ - id: generateId(), - credentialGroupId: groupId, - userId, - email: `${userId}@fixture.test`, - status: 'completed', - invitationTokenHash: createHash('sha256').update(generateId()).digest('hex'), - invitationExpiresAt: new Date(Date.now() + 60 * 60 * 1000), - invitedAt: now, - }) - .returning({ id: credentialGroupEnrollment.id }) - await db.insert(credential).values({ - id: members[who].credentialId, - workspaceId: ids.workspaceId, - type: 'managed_oauth', - displayName: 'Fixture GitHub reader', - providerId: 'github-repositories', - authorizationAppId: 'fixture-app', - credentialGroupEnrollmentId: enrollment!.id, - credentialGroupOptionId: optionId, - managedOauthScopeVersion: 1, - providerSubjectId: members[who].subject, - providerTenantId: '', - managedOauthStatus: 'active', - grantedScopes: ['repo'], - encryptedOauthTokenSet: 'fixture-not-an-oauth-token', - grantedAt: now, - createdBy: userId, - }) - } - await db.insert(knowledgeConnector).values( - [connectorId, otherConnectorId].map((id) => ({ - id, - knowledgeBaseId: ids.knowledgeBaseId, - connectorType: 'github', - sourceConfig: { githubRepositoryId: repositoryId }, - accessMode: 'members', - status: 'active', - credentialId: contentCredentialId, - credentialGroupId: groupId, - credentialGroupOptionId: optionId, - })) - ) - await db.insert(knowledgeConnectorMember).values( - (['alice', 'bob'] as const).map((who) => ({ - id: members[who].id, - workspaceId: ids.workspaceId, - connectorId, - credentialId: members[who].credentialId, - subjectToken: subjectToken(members[who].subject), - status: 'active', - memberSyncedThrough: now, - })) - ) - await db.insert(document).values({ - id: documentId, - connectorId, - knowledgeBaseId: ids.knowledgeBaseId, - externalId: 'fixture-file', - filename: 'readme.md', - fileUrl: 'https://fixture.test/readme', - fileSize: 12, - mimeType: 'text/plain', - processingStatus: 'completed', - acl: aclOf('alice', 'bob'), + name: 'Retired projection fixture', + chunkingConfig: { maxSize: 1024, minSize: 1, overlap: 20 }, }) - await db.insert(knowledgeDocumentObservation).values( - (['alice', 'bob'] as const).map((who) => ({ - documentId, - memberId: members[who].id, - lastSeenAt: now, - runId: generateId(), +}) + +beforeEach(async () => { + await db + .delete(document) + .where(inArray(document.knowledgeBaseId, [ids.knowledgeBaseId, searchBaseId])) + await db + .update(knowledgeBase) + .set({ isSearchIndex: false }) + .where(inArray(knowledgeBase.id, [ids.knowledgeBaseId, searchBaseId])) + await db.insert(document).values( + [ + { id: documentId, knowledgeBaseId: ids.knowledgeBaseId }, + { id: searchDocumentId, knowledgeBaseId: searchBaseId }, + ].map((row) => ({ + ...row, + filename: 'repair.md', + fileUrl: 'https://fixture.test/repair', + fileSize: 12, + mimeType: 'text/plain', + processingStatus: 'completed', })) ) - const [tin] = await db.execute<{ present: boolean }>( - sql`SELECT to_regnamespace('tin') IS NOT NULL AS present` - ) - if (!tin?.present) { - createdTinShims = true - await db.execute( - sql.raw(`CREATE SCHEMA tin; - CREATE FUNCTION tin.full_score(tid) RETURNS double precision LANGUAGE sql IMMUTABLE AS 'SELECT 1.0::float8'; - CREATE FUNCTION knowledge_tin_base_token(text) RETURNS text LANGUAGE sql IMMUTABLE AS $$SELECT 'kb'$$; - CREATE FUNCTION knowledge_tin_stream(vector tsvector) RETURNS text LANGUAGE sql IMMUTABLE AS $$ - SELECT coalesce(string_agg(entry.lexeme, ' ' ORDER BY position), '') - FROM unnest(vector) AS entry(lexeme, positions, weights), unnest(entry.positions) AS position - $$; - CREATE FUNCTION tin_fixture_match(text, text) RETURNS boolean LANGUAGE sql IMMUTABLE AS 'SELECT true'; - CREATE OPERATOR ==> (LEFTARG = text, RIGHTARG = text, FUNCTION = tin_fixture_match);`) - ) - } -}, 60_000) +}) afterAll(async () => { - if (createdTinShims) { - await db.execute( - sql.raw(`DROP OPERATOR IF EXISTS ==> (text, text); - DROP FUNCTION IF EXISTS tin_fixture_match(text, text); - DROP FUNCTION IF EXISTS knowledge_tin_base_token(text); - DROP FUNCTION IF EXISTS knowledge_tin_stream(tsvector); - DROP SCHEMA IF EXISTS tin CASCADE;`) - ) - } await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) - await db.delete(credentialGroup).where(eq(credentialGroup.id, groupId)) await db.delete(organization).where(eq(organization.id, ids.organizationId)) await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) await projector?.end() - forgetProjectionFilled() -}) - -/** - * Every test starts from one converged chunk readable by Alice and Bob: whatever the last test - * left is removed, the chunk is written again, and the projector converges it. It is written - * deferred, so the projector writes all three projections: this database has no Tin extension, - * so no synchronous trigger would write the Tin row. - */ -beforeEach(async () => { - await db.delete(embedding).where(eq(embedding.documentId, documentId)) - await db - .update(document) - .set({ acl: aclOf('alice', 'bob'), connectorId }) - .where(eq(document.id, documentId)) - await write('async', (tx) => tx.insert(embedding).values(chunkRow(chunkId, 0))) - await project() - forgetProjectionFilled() }) -describe.each(['sync', 'async'] as const)('a %s writer', (mode) => { - it('revokes a grant before the projector runs, on both rankings', async () => { - await write(mode, (tx) => - tx - .update(document) - .set({ acl: aclOf('bob') }) - .where(eq(document.id, documentId)) - ) - expect(await markOf()).toMatchObject({ content: false }) - if (mode === 'async') { - /** The rows still name Alice: only the mark keeps her out. */ - expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('alice', 'bob')) - expect((await rowAcl(embeddingKeywordTin))?.acl).toEqual(aclOf('alice', 'bob')) - } - expect(await admitted()).toEqual({ vector: [], keyword: [] }) - expect(await vectorIds()).toEqual([]) - expect(await keywordIds()).toEqual([]) - - await project() +describe('ordinary KB vector repair after indexed Search retirement', () => { + it('repairs deferred vectors in bounded pages without creating copied ACL or keyword rows', async () => { + const chunks = Array.from({ length: 5 }, (_, index) => chunkRow(generateId(), index)) + await defer((tx) => tx.insert(embedding).values(chunks)) + expect(await vectorsOf()).toEqual([]) + expect(await markOf()).toMatchObject({ content: true }) + const writes: number[] = [] + await runKnowledgeProjection(projector, { + pageSize: 2, + onPage: (page) => { + if (page.documentId === documentId) writes.push(page.written) + }, + }) + expect(writes).toEqual([2, 2, 1]) + const rows = await vectorsOf() + expect(rows.map((row) => row.id).sort()).toEqual(chunks.map((row) => row.id).sort()) + expect( + rows.every( + (row) => row.vector512?.[511] === 1 && row.acl === null && row.connectorId === null + ) + ).toBe(true) + expect( + await db + .select() + .from(embeddingKeywordSearch) + .where(eq(embeddingKeywordSearch.documentId, documentId)) + ).toEqual([]) + expect( + await db + .select() + .from(embeddingKeywordTin) + .where(eq(embeddingKeywordTin.documentId, documentId)) + ).toEqual([]) expect(await markOf()).toBeUndefined() - expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('bob')) - expect((await rowAcl(embeddingKeywordTin))?.acl).toEqual(aclOf('bob')) - expect(await admitted()).toEqual({ vector: [], keyword: [] }) }) - it('serves a new grant before the projector runs, on both rankings', async () => { + it('settles retired Search content marks without recreating any candidate projection', async () => { await db - .update(document) - .set({ acl: aclOf('bob') }) - .where(eq(document.id, documentId)) - await project() - expect(await vectorIds()).toEqual([]) - - await write(mode, (tx) => - tx - .update(document) - .set({ acl: aclOf('alice', 'bob') }) - .where(eq(document.id, documentId)) - ) - if (mode === 'async') expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('bob')) - expect(await admitted()).toEqual({ vector: [chunkId], keyword: [chunkId] }) - expect(await vectorIds()).toEqual([chunkId]) - expect(await keywordIds()).toEqual([chunkId]) - }) - - it('decides a document that moved to a source outside the plan on the document', async () => { - await write(mode, (tx) => - tx.update(document).set({ connectorId: otherConnectorId }).where(eq(document.id, documentId)) - ) - if (mode === 'async') expect((await rowAcl(embeddingSearch))?.connectorId).toBe(connectorId) - expect(await admitted()).toEqual({ vector: [], keyword: [] }) - expect(await vectorIds()).toEqual([]) - expect(await keywordIds()).toEqual([]) - await project() - expect((await rowAcl(embeddingSearch))?.connectorId).toBe(otherConnectorId) - }) - - it('hides a disabled chunk at once', async () => { - await write(mode, (tx) => - tx.update(embedding).set({ enabled: false }).where(eq(embedding.id, chunkId)) - ) - expect(await markOf()).toMatchObject({ content: mode === 'async' }) - expect(await vectorIds()).toEqual([]) - expect(await keywordIds()).toEqual([]) - await project() - expect((await rowAcl(embeddingSearch))?.enabled).toBe(false) - }) - - it('removes a deleted chunk from every projection in the deleting statement', async () => { - await write(mode, (tx) => tx.delete(embedding).where(eq(embedding.id, chunkId))) - expect(await rowAcl(embeddingSearch)).toBeUndefined() - expect(await rowAcl(embeddingKeywordTin)).toBeUndefined() - expect(await vectorIds()).toEqual([]) + .update(knowledgeBase) + .set({ isSearchIndex: true }) + .where(eq(knowledgeBase.id, searchBaseId)) + await defer((tx) => tx.insert(embedding).values(chunkRow(generateId(), 0, true))) + expect(await markOf(searchDocumentId)).toMatchObject({ content: true }) + await runKnowledgeProjection(projector, {}) + expect(await vectorsOf(searchDocumentId)).toEqual([]) + expect( + await db + .select() + .from(embeddingKeywordSearch) + .where(eq(embeddingKeywordSearch.documentId, searchDocumentId)) + ).toEqual([]) + expect( + await db + .select() + .from(embeddingKeywordTin) + .where(eq(embeddingKeywordTin.documentId, searchDocumentId)) + ).toEqual([]) + expect(await markOf(searchDocumentId)).toBeUndefined() }) - it('makes a new chunk searchable, at once or once the projector runs', async () => { - const added = generateId() - await write(mode, (tx) => tx.insert(embedding).values(chunkRow(added, 1))) - expect(await markOf()).toMatchObject({ content: mode === 'async' }) - const expected = [chunkId, added].sort() - if (mode === 'sync') { - expect(await vectorIds()).toEqual(expected) - } else { - expect(await rowAcl(embeddingSearch, added)).toBeUndefined() - expect(await vectorIds()).toEqual([chunkId]) - } - await project() + it('settles permission-only marks without rewriting vector rows or canonical document access', async () => { + const chunk = chunkRow(generateId(), 0) + await db.insert(embedding).values(chunk) + const [before] = + await projector`SELECT xmin::text AS version FROM embedding_search WHERE id = ${chunk.id}` + await defer((tx) => tx.update(document).set({ acl: [] }).where(eq(document.id, documentId))) + expect(await markOf()).toMatchObject({ content: false }) + await runKnowledgeProjection(projector, {}) + const [after] = + await projector`SELECT xmin::text AS version FROM embedding_search WHERE id = ${chunk.id}` + expect(after.version).toBe(before.version) + const [canonical] = await db + .select({ acl: document.acl }) + .from(document) + .where(eq(document.id, documentId)) + expect(canonical.acl).toEqual([]) expect(await markOf()).toBeUndefined() - expect(await vectorIds()).toEqual(expected) - /** A synchronous writer's Tin row is its trigger's, which only a database with Tin has. */ - if (mode === 'async') expect(await keywordIds()).toEqual(expected) }) -}) -describe('the projector', () => { - it('keeps a mark that a write bumped during its pass, and settles it on the next', async () => { - await db - .update(document) - .set({ acl: aclOf('bob') }) - .where(eq(document.id, documentId)) + it('keeps a concurrent content change for the next pass instead of settling its newer generation', async () => { + const chunks = [chunkRow(generateId(), 0), chunkRow(generateId(), 1)] + await defer((tx) => tx.insert(embedding).values(chunks)) const before = await markOf() - let bumped = false - await project({ - onPage: async () => { - if (bumped) return - bumped = true - await db - .update(document) - .set({ acl: aclOf('alice') }) - .where(eq(document.id, documentId)) + let changed = false + await runKnowledgeProjection(projector, { + pageSize: 1, + onPage: async (page) => { + if (page.documentId !== documentId || changed) return + changed = true + await defer((tx) => + tx.update(embedding).set({ enabled: false }).where(eq(embedding.id, chunks[0].id)) + ) }, }) - expect(bumped).toBe(true) - const after = await markOf() - expect(after?.generation).toBe((before?.generation ?? 0) + 1) - await project() + expect((await markOf())?.generation).toBe((before?.generation ?? 0) + 1) + expect((await vectorsOf()).find((row) => row.id === chunks[0].id)?.enabled).toBe(true) + await runKnowledgeProjection(projector, {}) + expect((await vectorsOf()).find((row) => row.id === chunks[0].id)?.enabled).toBe(false) expect(await markOf()).toBeUndefined() - expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('alice')) }) - it('keeps the mark when a synchronous ACL change commits under a page that then writes stale values', async () => { - /** A pending revocation of Bob: the rows still name both until a pass. */ - await write('async', (tx) => + it('stops repairing a base marked as Search between committed pages', async () => { + await defer((tx) => tx - .update(document) - .set({ acl: aclOf('alice') }) - .where(eq(document.id, documentId)) + .insert(embedding) + .values(Array.from({ length: 3 }, (_, index) => chunkRow(generateId(), index))) ) - const before = await markOf() - const [{ pid }] = await projector>`SELECT pg_backend_pid() AS pid` - /** - * A synchronous writer revokes Alice too and holds its transaction open: its fan-out has - * locked the projection rows and its mark bump is not yet visible. - */ - const writer = postgres(process.env.DATABASE_URL!, { max: 1, onnotice: () => undefined }) - let release: () => void = () => {} - const held = new Promise((resolve) => { - release = resolve - }) - let locked: () => void = () => {} - const fannedOut = new Promise((resolve) => { - locked = resolve + let changed = false + await runKnowledgeProjection(projector, { + pageSize: 1, + onPage: async (page) => { + if (page.documentId !== documentId || changed) return + changed = true + await db + .update(knowledgeBase) + .set({ isSearchIndex: true }) + .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) + }, }) - try { - const writing = writer.begin(async (tx) => { - await tx`UPDATE document SET acl = ${aclOf('bob')} WHERE id = ${documentId}` - locked() - await held - }) - await fannedOut - /** - * The pass reads the committed document, Alice alone, and its page blocks on the rows the - * writer holds. Once the writer commits, the page rewrites those rows from what it read. - */ - const pass = project() - await vi.waitFor(async () => { - const [activity] = await db.execute<{ waiting: boolean }>( - sql`SELECT wait_event_type = 'Lock' AS waiting FROM pg_stat_activity WHERE pid = ${pid}` - ) - expect(activity?.waiting).toBe(true) - }) - release() - await writing - await pass - } finally { - release() - await writer.end() - } - /** The page wrote the value it read over the writer's newer one. */ - expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('alice')) - /** The writer's bump is what the pass could not settle over: the rows stay decided on the document. */ - expect((await markOf())?.generation).toBe((before?.generation ?? 0) + 1) - expect(await admitted()).toEqual({ vector: [], keyword: [] }) - expect(await vectorIds()).toEqual([]) - await project() + expect(await vectorsOf()).toHaveLength(1) expect(await markOf()).toBeUndefined() - expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('bob')) - expect((await rowAcl(embeddingKeywordTin))?.acl).toEqual(aclOf('bob')) - expect(await admitted()).toEqual({ vector: [], keyword: [] }) - }) - - it('passes over a document deleted while it is being projected', async () => { - const doomed = generateId() - await db.insert(document).values({ - id: doomed, - connectorId, - knowledgeBaseId: ids.knowledgeBaseId, - externalId: 'doomed-file', - filename: 'doomed.md', - fileUrl: 'https://fixture.test/doomed', - fileSize: 12, - mimeType: 'text/plain', - processingStatus: 'completed', - acl: aclOf('alice'), - }) - await write('async', (tx) => - tx.insert(embedding).values( - Array.from({ length: 3 }, (_, index) => ({ - ...chunkRow(generateId(), index), - documentId: doomed, - })) - ) - ) - await expect( - project({ - pageSize: 1, - onPage: async (page) => { - if (page.documentId === doomed) await db.delete(document).where(eq(document.id, doomed)) - }, - }) - ).resolves.toMatchObject({ remaining: false }) - const [left] = await db - .select({ count: sql`count(*)::int` }) - .from(embeddingSearch) - .where(eq(embeddingSearch.documentId, doomed)) - expect(left?.count).toBe(0) - expect( - await db - .select() - .from(knowledgeProjectionDirty) - .where(eq(knowledgeProjectionDirty.documentId, doomed)) - ).toEqual([]) }) - it("rewrites a connector ACL page's projection rows in the page's own statement", async () => { - await leaseTransaction(connectorId)((tx) => + it('does not resurrect a document deleted after the first page commits', async () => { + await defer((tx) => tx - .update(document) - .set({ acl: aclOf('bob') }) - .where(eq(document.id, documentId)) + .insert(embedding) + .values(Array.from({ length: 3 }, (_, index) => chunkRow(generateId(), index))) ) - expect(await markOf()).toMatchObject({ content: false }) - expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('bob')) - expect(await admitted()).toEqual({ vector: [], keyword: [] }) - await project() + await runKnowledgeProjection(projector, { + pageSize: 1, + onPage: async (page) => { + if (page.documentId === documentId) + await db.delete(document).where(eq(document.id, documentId)) + }, + }) + expect(await vectorsOf()).toEqual([]) expect(await markOf()).toBeUndefined() }) - it('splits the marks between passes that start together', async () => { - const documents = await Promise.all( - Array.from({ length: 6 }, async (_, index) => { - const id = generateId() - await db.insert(document).values({ - id, - connectorId, - knowledgeBaseId: ids.knowledgeBaseId, - externalId: `split-${index}`, - filename: `split-${index}.md`, - fileUrl: `https://fixture.test/split-${index}`, - fileSize: 12, - mimeType: 'text/plain', - processingStatus: 'completed', - acl: aclOf('alice'), - }) - return id - }) - ) - await write('async', (tx) => - tx - .insert(embedding) - .values( - documents.map((id, index) => ({ ...chunkRow(generateId(), index), documentId: id })) - ) - ) - const second = postgres(process.env.DATABASE_URL!, { max: 1, onnotice: () => undefined }) + it('preserves work held by another pass until its advisory lock is released', async () => { + await defer((tx) => tx.insert(embedding).values(chunkRow(generateId(), 0))) + const other = postgres(process.env.DATABASE_URL!, { max: 1, onnotice: () => undefined }) try { - /** Each page lingers, so neither pass can finish the batch before the other starts. */ - const lingering = { onPage: () => sleep(25), searchIndexes: true } - const passes = await Promise.all([ - project(lingering), - runKnowledgeProjection(second, lingering), - ]) - expect(passes.every((pass) => pass.settled > 0)).toBe(true) - expect(passes[0].settled + passes[1].settled).toBe(documents.length) + await other`SELECT pg_advisory_lock(hashtextextended('knowledge_projection:' || ${documentId}, 0))` + await runKnowledgeProjection(projector, {}) + expect(await vectorsOf()).toEqual([]) + expect(await markOf()).toMatchObject({ content: true }) + await other`SELECT pg_advisory_unlock(hashtextextended('knowledge_projection:' || ${documentId}, 0))` + await runKnowledgeProjection(projector, {}) + expect(await vectorsOf()).toHaveLength(1) + expect(await markOf()).toBeUndefined() } finally { - await second.end() + await other.end() } - expect( - await db - .select() - .from(knowledgeProjectionDirty) - .where(inArray(knowledgeProjectionDirty.documentId, documents)) - ).toEqual([]) - await db.delete(document).where(inArray(document.id, documents)) }) - it('writes in pages bounded by chunk rows', async () => { - await write('async', (tx) => - tx - .insert(embedding) - .values(Array.from({ length: 4 }, (_, index) => chunkRow(generateId(), index + 1))) - ) - const pages: Array<{ projection: KnowledgeProjection; written: number }> = [] - await project({ pageSize: 2, onPage: (page) => void pages.push(page) }) - const vectorPages = pages.filter((page) => page.projection === 'embedding_search') - expect(vectorPages.map((page) => page.written)).toEqual([1, 2, 1]) - expect(pages.every((page) => page.written <= 2)).toBe(true) - expect(await markOf()).toBeUndefined() - }) - - it('writes keyword rows for search-index knowledge bases only', async () => { - const workspaceBaseId = generateId() - const workspaceDocument = generateId() - const workspaceChunk = generateId() - await db.insert(knowledgeBase).values({ - id: workspaceBaseId, - userId: ids.aliceId, - workspaceId: ids.workspaceId, - name: 'Workspace keyword fixture', - chunkingConfig: { maxSize: 1024, minSize: 1, overlap: 20 }, - }) - try { - await db.insert(document).values({ - id: workspaceDocument, - knowledgeBaseId: workspaceBaseId, - filename: 'keyword.md', - fileUrl: 'https://fixture.test/keyword', + it('yields after one thousand deferred documents without losing any pending marks', async () => { + const documentIds = Array.from({ length: 1_001 }, () => generateId()) + await db.insert(document).values( + documentIds.map((id) => ({ + id, + knowledgeBaseId: ids.knowledgeBaseId, + filename: 'deferred.md', + fileUrl: 'https://fixture.test/deferred', fileSize: 12, mimeType: 'text/plain', processingStatus: 'completed', + })) + ) + await db + .insert(knowledgeProjectionDirty) + .values(documentIds.map((documentId) => ({ documentId, content: true }))) + const other = postgres(process.env.DATABASE_URL!, { max: 1, onnotice: () => undefined }) + try { + await other` + SELECT pg_advisory_lock(hashtextextended('knowledge_projection:' || id, 0)) + FROM unnest(${documentIds}::text[]) AS locked(id)` + expect(await runKnowledgeProjection(projector, {})).toMatchObject({ + deferred: 1_000, + written: 0, + remaining: true, }) - await write('async', (tx) => - tx.insert(embedding).values({ - ...chunkRow(workspaceChunk, 0), - documentId: workspaceDocument, - knowledgeBaseId: workspaceBaseId, - }) - ) - await project() - const keywordRows = await db - .select({ id: embeddingKeywordSearch.id }) - .from(embeddingKeywordSearch) - .where(inArray(embeddingKeywordSearch.id, [chunkId, workspaceChunk])) - expect(keywordRows).toEqual([{ id: chunkId }]) - expect(await rowAcl(embeddingSearch, workspaceChunk)).toBeDefined() + const [pending] = await db + .select({ count: sql`count(*)::int` }) + .from(knowledgeProjectionDirty) + .where(inArray(knowledgeProjectionDirty.documentId, documentIds)) + expect(pending.count).toBe(1_001) } finally { - await db.delete(knowledgeBase).where(eq(knowledgeBase.id, workspaceBaseId)) + await other.end() } }) - it.each(['sync', 'async'] as const)( - 'writes %s projection rows from a chunk commit only when the writer did not defer them', - async (mode) => { - /** - * The projector's single connection makes the write, so its own statistics can be flushed - * and read back without waiting on the collector's interval. - */ - const inserted = async () => { - await projector`SELECT pg_stat_force_next_flush()` - await projector`SELECT pg_stat_clear_snapshot()` - const [row] = await projector>` - SELECT n_tup_ins AS inserted FROM pg_stat_user_tables WHERE relname = 'embedding_search'` - return Number(row?.inserted ?? 0) - } - const chunks = Array.from({ length: 50 }, (_, index) => chunkRow(generateId(), index + 1)) - const before = await inserted() - await projector.begin(async (tx) => { - if (mode === 'async') await tx`SELECT set_config('sim.projection_mode', 'async', true)` - for (const chunk of chunks) { - await tx`INSERT INTO embedding (id, document_id, knowledge_base_id, chunk_index, chunk_hash, - content, content_length, token_count, start_offset, end_offset, embedding_model, embedding) - VALUES (${chunk.id}, ${documentId}, ${ids.knowledgeBaseId}, ${chunk.chunkIndex}, - ${chunk.chunkHash}, ${chunk.content}, 14, 2, 0, 14, ${chunk.embeddingModel}, - ${JSON.stringify(vector)}::vector)` - } - }) - expect((await inserted()) - before).toBe(mode === 'async' ? 0 : chunks.length) - expect(await markOf()).toMatchObject({ content: mode === 'async' }) - } - ) - - it('owes a pass for content and, while indexed search is on, search-index marks, asking only for content behind an undrained release', async () => { - const workspaceBaseId = generateId() - const [workspaceDocument, contentDocument] = [generateId(), generateId()] - await db.insert(knowledgeBase).values({ - id: workspaceBaseId, - userId: ids.aliceId, - workspaceId: ids.workspaceId, - name: 'Workspace work fixture', - chunkingConfig: { maxSize: 1024, minSize: 1, overlap: 20 }, - }) - /** Thrown to roll the transaction back, so the marks of other suites are never touched for good. */ - const rollback = new Error('rollback') + it('resumes a bounded pass without rewriting an already current page', async () => { + const chunks = Array.from({ length: 3 }, (_, index) => chunkRow(generateId(), index)) + await defer((tx) => tx.insert(embedding).values(chunks)) + const startedAt = Date.now() + const clock = vi.spyOn(Date, 'now').mockReturnValue(startedAt) try { - await db.insert(document).values( - [workspaceDocument, contentDocument].map((id, index) => ({ - id, - knowledgeBaseId: workspaceBaseId, - filename: `work-${index}.md`, - fileUrl: `https://fixture.test/work-${index}`, - fileSize: 12, - mimeType: 'text/plain', - processingStatus: 'completed' as const, - })) - ) - const answers: Record = {} - const indexed = { searchIndexes: true } - const answer = async (tx: postgres.TransactionSql, label: string) => { - answers[label] = { - drained: await hasKnowledgeProjectionWork(tx, { drained: true }, indexed), - undrained: await hasKnowledgeProjectionWork(tx, { drained: false }, indexed), - dormant: await hasKnowledgeProjectionWork( - tx, - { drained: true }, - { searchIndexes: false } - ), - } - } - await projector - .begin(async (tx) => { - await tx`DELETE FROM knowledge_projection_dirty` - await tx`SELECT mark_knowledge_projection(ARRAY[${workspaceDocument}]::text[], false)` - await answer(tx, 'workspace') - await tx`SELECT mark_knowledge_projection(ARRAY[${documentId}]::text[], false)` - await answer(tx, 'search index') - await tx`SELECT mark_knowledge_projection(ARRAY[${contentDocument}]::text[], true)` - await answer(tx, 'content') - throw rollback - }) - .catch((error) => { - if (error !== rollback) throw error - }) - expect(answers).toEqual({ - workspace: { drained: false, undrained: false, dormant: false }, - 'search index': { drained: true, undrained: false, dormant: false }, - content: { drained: true, undrained: true, dormant: true }, + await runKnowledgeProjection(projector, { + pageSize: 1, + budgetMs: 1_000, + onPage: (page) => { + if (page.documentId === documentId) clock.mockReturnValue(startedAt + 1_000) + }, }) } finally { - await db.delete(knowledgeBase).where(eq(knowledgeBase.id, workspaceBaseId)) + clock.mockRestore() } + expect(await vectorsOf()).toHaveLength(1) + expect(await markOf()).toMatchObject({ content: true }) + const [before] = + await projector`SELECT xmin::text AS version FROM embedding_search WHERE id = ${chunks[0].id}` + await runKnowledgeProjection(projector, { pageSize: 1 }) + const [after] = + await projector`SELECT xmin::text AS version FROM embedding_search WHERE id = ${chunks[0].id}` + expect(after.version).toBe(before.version) + expect(await vectorsOf()).toHaveLength(3) + expect(await markOf()).toBeUndefined() }) - it('releases marks with nothing to project, keeping search-index marks for a pass only while indexed search is on', async () => { - const workspaceBaseId = generateId() - const [synced, deferred, held] = [generateId(), generateId(), generateId()] - await db.insert(knowledgeBase).values({ - id: workspaceBaseId, - userId: ids.aliceId, - workspaceId: ids.workspaceId, - name: 'Workspace fixture', - chunkingConfig: { maxSize: 1024, minSize: 1, overlap: 20 }, - }) - try { - await db.insert(document).values( - [synced, deferred, held].map((id, index) => ({ - id, - knowledgeBaseId: workspaceBaseId, - filename: `workspace-${index}.md`, - fileUrl: `https://fixture.test/workspace-${index}`, - fileSize: 12, - mimeType: 'text/plain', - processingStatus: 'completed' as const, - })) - ) - const workspaceChunk = (id: string, documentId: string) => ({ - ...chunkRow(id, 0), - documentId, - knowledgeBaseId: workspaceBaseId, - }) - const deferredChunk = generateId() - await write('sync', (tx) => - tx - .insert(embedding) - .values([workspaceChunk(generateId(), synced), workspaceChunk(generateId(), held)]) - ) - await write('async', (tx) => - tx.insert(embedding).values(workspaceChunk(deferredChunk, deferred)) - ) - /** A search-index mark with nothing but a source and ACL change is still a pass's. */ - await db - .update(document) - .set({ acl: aclOf('bob') }) - .where(eq(document.id, documentId)) - const marked = async () => - ( - await db - .select({ documentId: knowledgeProjectionDirty.documentId }) - .from(knowledgeProjectionDirty) - .where( - inArray(knowledgeProjectionDirty.documentId, [synced, deferred, held, documentId]) - ) - ) - .map((row) => row.documentId) - .sort() - - /** A writer re-marking `held` holds its mark until it commits; the release passes it over. */ - const writer = postgres(process.env.DATABASE_URL!, { max: 1, onnotice: () => undefined }) - let release: () => void = () => {} - const holding = new Promise((resolve) => { - release = resolve - }) - let locked: () => void = () => {} - const marking = new Promise((resolve) => { - locked = resolve - }) - try { - const writing = writer.begin(async (tx) => { - await tx`SELECT mark_knowledge_projection(ARRAY[${held}]::text[], false)` - locked() - await holding - }) - await marking - expect( - (await releaseSettledMarks(projector, Number.POSITIVE_INFINITY, { searchIndexes: true })) - .released - ).toBeGreaterThanOrEqual(1) - expect(await marked()).toEqual([deferred, held, documentId].sort()) - release() - await writing - } finally { - release() - await writer.end() - } - expect( - (await releaseSettledMarks(projector, Number.POSITIVE_INFINITY, { searchIndexes: true })) - .released - ).toBeGreaterThanOrEqual(1) - expect(await marked()).toEqual([deferred, documentId].sort()) - - /** The deferred chunk has no row until a pass writes it, and the pass settles both marks. */ - const deferredRow = async () => - db - .select({ id: embeddingSearch.id }) - .from(embeddingSearch) - .where(eq(embeddingSearch.id, deferredChunk)) - expect(await deferredRow()).toEqual([]) - await project() - expect(await deferredRow()).toEqual([{ id: deferredChunk }]) - expect(await marked()).toEqual([]) - expect((await rowAcl(embeddingSearch))?.acl).toEqual(aclOf('bob')) - - /** While indexed search is dormant, nothing reads a search-index mark, so it is released too. */ - await db - .update(document) - .set({ acl: aclOf('alice') }) - .where(eq(document.id, documentId)) - expect(await marked()).toEqual([documentId]) - await releaseSettledMarks(projector, Number.POSITIVE_INFINITY, { searchIndexes: false }) - expect(await marked()).toEqual([]) - } finally { - await db.delete(knowledgeBase).where(eq(knowledgeBase.id, workspaceBaseId)) - } + it('releases retired Search marks while retaining ordinary content repair for admission', async () => { + await db + .update(knowledgeBase) + .set({ isSearchIndex: true }) + .where(eq(knowledgeBase.id, searchBaseId)) + await defer((tx) => + tx.insert(embedding).values([chunkRow(generateId(), 0), chunkRow(generateId(), 0, true)]) + ) + await releaseSettledMarks(projector, Date.now() + 10_000) + expect(await markOf(searchDocumentId)).toBeUndefined() + expect(await markOf()).toMatchObject({ content: true }) + expect(await hasKnowledgeProjectionWork(projector)).toBe(true) + expect(await vectorsOf(searchDocumentId)).toEqual([]) }) }) diff --git a/apps/sim/lib/knowledge/__integration__/organization-mcp-search.integration.ts b/apps/sim/lib/knowledge/__integration__/organization-mcp-search.integration.ts deleted file mode 100644 index 7b2d0f2a111..00000000000 --- a/apps/sim/lib/knowledge/__integration__/organization-mcp-search.integration.ts +++ /dev/null @@ -1,988 +0,0 @@ -/** - * Real organization-owned ingestion, Postgres, API-key authentication, application - * authorization, source ACLs, and MCP SDK transport. Only embedding vectors are - * deterministic and storage is temporary; no provider or live credential is used. - */ -import { mkdtempSync } from 'node:fs' -import { rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import path from 'node:path' -import { Client } from '@modelcontextprotocol/sdk/client/index.js' -import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js' -import { CallToolResultSchema } from '@modelcontextprotocol/sdk/types.js' -import type { Principal } from '@sim/auth/principal' -import { db } from '@sim/db' -import { - apiKey, - document, - embedding, - embeddingSecretProvenance, - knowledgeBase, - knowledgeExternalGroup, - knowledgeExternalGroupMember, - member, - oauthAccessToken, - oauthClient, - oauthConsent, - organization, - organizationSearchIntegration, - organizationSearchMcpInvocation, - rateLimitBucket, - user, - workspace, -} from '@sim/db/schema' -import { sha256Hex } from '@sim/security/hash' -import { generateId } from '@sim/utils/id' -import { isPlainRecord } from '@sim/utils/object' -import { and, eq, inArray } from 'drizzle-orm' -import { NextRequest } from 'next/server' -import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' - -const fixtures = vi.hoisted(() => ({ - storageRoot: '', - afterResponse: [] as Array<() => Promise>, -})) -/** This ingestion suite covers the explicit rollback backend; live Search has its own suites. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) -vi.mock('@/lib/core/utils/after-response', () => ({ - afterResponse: (task: () => Promise) => fixtures.afterResponse.push(task), -})) - -async function flushAfterResponse() { - for (const task of fixtures.afterResponse.splice(0)) await task() -} -vi.mock('@/lib/uploads/core/setup.server', () => ({ - get UPLOAD_DIR_SERVER() { - return fixtures.storageRoot - }, -})) -vi.mock('@/lib/embeddings', async () => ({ - ...(await import('@/lib/embeddings/client')), - assertKnowledgeEmbeddingCapacity: async () => {}, - embedKnowledge: async (texts: string[]) => ({ - embeddings: texts.map(() => [1, ...Array(1535).fill(0)]), - totalTokens: texts.length, - billableTokens: 0, - isBYOK: true, - modelName: 'text-embedding-3-small', - pricingId: 'text-embedding-3-small', - }), -})) - -import { hashApiKey } from '@/lib/api-key/crypto' -import { hashOAuthToken } from '@/lib/auth/oauth-access-token' -import { OAUTH_ACCESS_TOKEN_PREFIX } from '@/lib/auth/oauth-provider' -import { resolveOrganizationBillingAttribution } from '@/lib/billing/core/billing-attribution' -import { encryptSecret } from '@/lib/core/security/encryption' -import { - createKnowledgeAclFixtureIds, - seedKnowledgeAclFixture, -} from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { confluencePageAcl } from '@/lib/knowledge/access/confluence-permissions' -import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' -import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' -import { listKnowledgeBases } from '@/lib/knowledge/application/knowledge-bases' -import { prepareSearchSource } from '@/lib/knowledge/application/sim-search' -import { createContentSyncLease } from '@/lib/knowledge/connectors/sync-lock' -import { addDocument, persistDocumentAcls } from '@/lib/knowledge/connectors/sync-persistence' -import { processDocumentAsync } from '@/lib/knowledge/documents/service' -import { getSearchMcpUrl } from '@/lib/knowledge/mcp/urls' -import { replaceKnowledgeEmbeddingSecretProvenanceInTx } from '@/lib/knowledge/secret-provenance' -import { readIndexedKnowledgeDocument } from '@/lib/sim-search/indexed/documents/read-indexed-document' -import { searchScopedKnowledge } from '@/lib/sim-search/indexed/search/scoped-search' -import { DELETE, GET, POST } from '@/app/api/mcp/search/organizations/[organizationId]/route' -import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' - -describe('organization Search MCP rollback backend with real ingestion and current access', () => { - const ids = createKnowledgeAclFixtureIds() - const { - aliceId, - bobId, - workspaceId, - organizationId, - knowledgeBaseId, - connectorId, - lockId, - groupIds, - } = ids - const otherOrganizationId = generateId() - const workspaceKnowledgeBaseId = generateId() - const outsiderId = generateId() - const otherAdminId = generateId() - const bobMembershipId = generateId() - const tokens = { - alice: generateId(), - bob: generateId(), - outsider: generateId(), - workspace: generateId(), - expired: generateId(), - } - const oauthClientId = generateId() - const oauthTokens = { alice: generateId(), bob: generateId() } - const clients: Client[] = [] - const alicePrincipal: Principal = { kind: 'session', userId: aliceId, sessionId: generateId() } - const bobPrincipal: Principal = { kind: 'session', userId: bobId, sessionId: generateId() } - const otherAdminPrincipal: Principal = { - kind: 'session', - userId: otherAdminId, - sessionId: generateId(), - } - const bobSourceMembership = { - groupId: groupIds[2], - subjectToken: `u:${bobId}@fixture.test`, - } - let otherKnowledgeBaseId: string - let documentId: string - let alice: Client - let bob: Client - let aliceOAuth: Client - let bobOAuth: Client - - async function request( - token?: string, - target = organizationId, - options: { - method?: 'GET' | 'POST' | 'DELETE' - body?: unknown - raw?: string - headers?: Record - } = {} - ) { - const method = options.method ?? 'POST' - return { GET, POST, DELETE }[method]( - new NextRequest(`http://localhost:3000/api/mcp/search/organizations/${target}`, { - method, - headers: { - 'x-forwarded-for': '127.0.0.1', - 'content-type': 'application/json', - accept: 'application/json, text/event-stream', - ...(token ? { 'x-api-key': token } : {}), - ...options.headers, - }, - ...(method === 'POST' - ? { - body: - options.raw ?? - JSON.stringify(options.body ?? { jsonrpc: '2.0', id: 1, method: 'tools/list' }), - } - : {}), - }), - { params: Promise.resolve({ organizationId: target }) } - ) - } - - async function connect(token: string, bearer = false) { - const client = new Client({ name: 'Organization ACL fixture', version: '1.0.0' }) - clients.push(client) - await client.connect( - new StreamableHTTPClientTransport( - new URL(`http://localhost:3000/api/mcp/search/organizations/${organizationId}`), - { - requestInit: { - headers: bearer ? { authorization: `Bearer ${token}` } : { 'x-api-key': token }, - }, - fetch: async (url, init) => { - const req = new NextRequest(url instanceof Request ? url : String(url), { - ...init, - signal: init?.signal ?? undefined, - }) - req.headers.set('x-forwarded-for', '127.0.0.1') - const context = { params: Promise.resolve({ organizationId }) } - return req.method === 'GET' - ? GET(req, context) - : req.method === 'DELETE' - ? DELETE(req, context) - : POST(req, context) - }, - } - ) - ) - return client - } - - async function call(client: Client, name: string, args: Record) { - return CallToolResultSchema.parse(await client.callTool({ name, arguments: args })) - } - - async function value(client: Client, name: string, args: Record) { - const result = await call(client, name, args) - expect(result.isError).not.toBe(true) - const first = result.content[0] - if (first?.type !== 'text') throw new Error('Expected JSON text tool result') - const parsed: unknown = JSON.parse(first.text) - if (!isPlainRecord(parsed)) throw new Error('Expected JSON object tool result') - return parsed - } - - async function search(client: Client) { - const result = await value(client, 'search', { query: 'Orion', topK: 50 }) - if (!Array.isArray(result.results) || !result.results.every(isPlainRecord)) { - throw new Error('Expected search result objects') - } - return result.results - } - - async function applicationSearch(principal: Principal) { - const result = await searchScopedKnowledge.execute({ - principal, - input: { organizationId, query: 'Orion', searchMode: 'hybrid', topK: 50 }, - }) - return result.results - } - - async function expectDocumentHidden(client: Client) { - expect(await search(client)).toEqual([]) - expect(await call(client, 'read_document', { documentId })).toMatchObject({ - isError: true, - content: [{ type: 'text', text: 'Document not found' }], - }) - } - - async function revokeBobSourceAccess() { - await db - .delete(knowledgeExternalGroupMember) - .where( - and( - eq(knowledgeExternalGroupMember.groupId, bobSourceMembership.groupId), - eq(knowledgeExternalGroupMember.subjectToken, bobSourceMembership.subjectToken) - ) - ) - } - - beforeAll(async () => { - vi.stubGlobal('fetch', async () => { - throw new Error('Unexpected outbound organization MCP fixture request') - }) - fixtures.storageRoot = mkdtempSync(path.join(tmpdir(), 'sim-organization-mcp-integration-')) - await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) - await db.insert(user).values( - [outsiderId, otherAdminId].map((id) => ({ - id, - name: 'Other organization fixture', - email: `${id}@fixture.test`, - emailVerified: true, - createdAt: new Date(), - updatedAt: new Date(), - })) - ) - await db.insert(organization).values({ - id: otherOrganizationId, - name: 'Other organization MCP fixture', - slug: otherOrganizationId, - createdAt: new Date(), - }) - await db.insert(member).values([ - { id: generateId(), userId: aliceId, organizationId, role: 'owner' }, - { id: bobMembershipId, userId: bobId, organizationId, role: 'member' }, - { id: generateId(), userId: outsiderId, organizationId: otherOrganizationId, role: 'owner' }, - { - id: generateId(), - userId: otherAdminId, - organizationId: otherOrganizationId, - role: 'admin', - }, - ]) - /** Reuse source identities, but establish exclusive organization ownership before ingestion. */ - await db - .update(knowledgeBase) - .set({ - workspaceId: null, - organizationId, - isSearchIndex: true, - name: 'Renamed org index', - chunkingConfig: { maxSize: 256, minSize: 1, overlap: 20 }, - }) - .where(eq(knowledgeBase.id, knowledgeBaseId)) - await db - .update(knowledgeExternalGroup) - .set({ workspaceId: null, organizationId }) - .where(inArray(knowledgeExternalGroup.id, groupIds)) - const prepared = await prepareSearchSource.execute({ - principal: otherAdminPrincipal, - input: { organizationId: otherOrganizationId, connectorType: 'gitlab' }, - }) - otherKnowledgeBaseId = prepared.knowledgeBaseId - await db.insert(knowledgeBase).values({ - id: workspaceKnowledgeBaseId, - userId: bobId, - workspaceId, - name: 'Workspace documents', - }) - await db.insert(organizationSearchIntegration).values({ - organizationId, - connectorType: 'google_drive', - approved: true, - }) - await db.insert(apiKey).values( - Object.entries(tokens).map(([name, token]) => ({ - id: generateId(), - userId: name === 'bob' ? bobId : name === 'outsider' ? outsiderId : aliceId, - name, - key: `fixture-${generateId()}`, - keyHash: hashApiKey(token), - type: name === 'workspace' ? 'workspace' : 'personal', - workspaceId: name === 'workspace' ? workspaceId : null, - expiresAt: name === 'expired' ? new Date(0) : null, - })) - ) - await db.insert(oauthClient).values({ - id: oauthClientId, - clientId: oauthClientId, - name: 'Search MCP OAuth fixture', - public: true, - requirePKCE: true, - redirectUris: ['http://127.0.0.1/callback'], - tokenEndpointAuthMethod: 'none', - scopes: ['search:read'], - grantTypes: ['authorization_code'], - responseTypes: ['code'], - }) - for (const [userId, token] of [ - [aliceId, oauthTokens.alice], - [bobId, oauthTokens.bob], - ]) { - await db.insert(oauthConsent).values({ - id: generateId(), - clientId: oauthClientId, - userId, - scopes: ['search:read'], - createdAt: new Date(), - updatedAt: new Date(), - }) - await db.insert(oauthAccessToken).values({ - id: generateId(), - clientId: oauthClientId, - userId, - token: hashOAuthToken(token), - scopes: ['search:read'], - resource: getSearchMcpUrl(organizationId), - createdAt: new Date(), - expiresAt: new Date(Date.now() + 3600_000), - }) - } - const doc = await addDocument( - knowledgeBaseId, - connectorId, - 'google_drive', - { - externalId: 'organization-mcp-page', - mimeType: 'text/plain', - title: 'Orion organization project', - content: Array.from( - { length: 30 }, - (_, index) => - `Orion section ${index}: engineers approved the release checklist and documented the customer migration dependencies.` - ).join('\n\n'), - contentHash: 'fixture-organization-orion', - sourceUrl: 'https://fixture.atlassian.net/wiki/pages/organization-mcp', - }, - { userId: aliceId, workspaceId: null, organizationId }, - undefined, - 'admin', - createContentSyncLease(connectorId, lockId) - ) - documentId = doc.documentId - await processDocumentAsync( - knowledgeBaseId, - documentId, - doc, - {}, - await resolveOrganizationBillingAttribution({ actorUserId: aliceId, organizationId }) - ) - const [persisted] = await db - .select({ status: document.processingStatus, error: document.processingError }) - .from(document) - .where(eq(document.id, documentId)) - expect(persisted).toEqual({ status: 'completed', error: null }) - await persistDocumentAcls( - connectorId, - new Map([ - [ - 'organization-mcp-page', - confluencePageAcl({ - providerId: 'google-drive', - tenantId: 'fixture-tenant', - spacePrincipals: [{ kind: 'group', id: 'space' }], - restrictionChain: [[{ kind: 'group', id: 'page' }], [{ kind: 'group', id: 'parent' }]], - }), - ], - ]) - ) - alice = await connect(tokens.alice) - bob = await connect(tokens.bob, true) - aliceOAuth = await connect(OAUTH_ACCESS_TOKEN_PREFIX + oauthTokens.alice, true) - bobOAuth = await connect(OAUTH_ACCESS_TOKEN_PREFIX + oauthTokens.bob, true) - }) - - afterEach(flushAfterResponse) - - afterAll(async () => { - await Promise.all(clients.map((client) => client.close())) - await db.delete(oauthClient).where(eq(oauthClient.clientId, oauthClientId)) - await db - .delete(organization) - .where(inArray(organization.id, [organizationId, otherOrganizationId])) - await db.delete(workspace).where(eq(workspace.id, workspaceId)) - await db.delete(user).where(inArray(user.id, [aliceId, bobId, outsiderId, otherAdminId])) - if (fixtures.storageRoot) await rm(fixtures.storageRoot, { recursive: true, force: true }) - await db.$client.end() - vi.unstubAllGlobals() - }) - - it('creates an organization-only index, keeps it out of the workspace catalog, and separates actor from payer', async () => { - const input = { organizationId: otherOrganizationId, connectorType: 'gitlab' } - const results = await Promise.all([ - prepareSearchSource.execute({ principal: otherAdminPrincipal, input }), - prepareSearchSource.execute({ principal: otherAdminPrincipal, input }), - ]) - expect(results.map((result) => result.knowledgeBaseId)).toEqual([ - otherKnowledgeBaseId, - otherKnowledgeBaseId, - ]) - const indexes = await db - .select() - .from(knowledgeBase) - .where(eq(knowledgeBase.organizationId, otherOrganizationId)) - expect(indexes).toEqual([ - expect.objectContaining({ - id: otherKnowledgeBaseId, - workspaceId: null, - organizationId: otherOrganizationId, - isSearchIndex: true, - userId: otherAdminId, - }), - ]) - const catalog = await listKnowledgeBases.execute({ - principal: alicePrincipal, - input: { workspaceId }, - }) - expect(catalog.knowledgeBases.map(({ knowledgeBase }) => knowledgeBase.id)).toEqual([ - workspaceKnowledgeBaseId, - ]) - await expect( - resolveOrganizationBillingAttribution({ - actorUserId: otherAdminId, - organizationId: otherOrganizationId, - }) - ).resolves.toMatchObject({ - actorUserId: otherAdminId, - workspaceId: null, - organizationId: otherOrganizationId, - billedAccountUserId: outsiderId, - billingEntity: { type: 'organization', id: otherOrganizationId }, - }) - }) - - it('finds the canonical org-owned index and applies each current member’s source ACL to all tools', async () => { - const [owner] = await db - .select({ - workspaceId: knowledgeBase.workspaceId, - organizationId: knowledgeBase.organizationId, - }) - .from(knowledgeBase) - .where(eq(knowledgeBase.id, knowledgeBaseId)) - expect(owner).toEqual({ workspaceId: null, organizationId }) - expect((await alice.listTools()).tools.map((tool) => tool.name)).toEqual([ - 'search', - 'read_document', - 'chat', - ]) - const rows = await search(alice) - expect(rows.length).toBeGreaterThan(1) - expect(rows.every((row) => row.documentId === documentId && !('knowledgeBaseId' in row))).toBe( - true - ) - expect((await applicationSearch(alicePrincipal)).map((row) => row.documentId)).toEqual( - rows.map((row) => row.documentId) - ) - expect(await value(alice, 'read_document', { documentId })).toMatchObject({ - title: 'Orion organization project', - documentId, - chunks: expect.arrayContaining([ - expect.objectContaining({ chunkIndex: 0, content: expect.stringContaining('Orion') }), - ]), - pagination: expect.objectContaining({ limit: 20, offset: 0 }), - }) - const page = await value(alice, 'read_document', { - documentId, - limit: 1, - }) - expect(page.chunks).toEqual([ - expect.objectContaining({ chunkIndex: 0, content: expect.stringContaining('Orion') }), - ]) - expect(page.pagination).toMatchObject({ limit: 1, offset: 0, hasMore: true }) - const nextPage = await value(alice, 'read_document', { - documentId, - limit: 1, - offset: 1, - }) - expect(nextPage.chunks).toEqual([expect.objectContaining({ chunkIndex: 1 })]) - expect(nextPage.chunks).not.toEqual(page.chunks) - expect(nextPage.pagination).toMatchObject({ limit: 1, offset: 1 }) - await expectDocumentHidden(bob) - expect(await applicationSearch(bobPrincipal)).toEqual([]) - }) - - it('persists content-free per-client tool outcomes separately from search counters', async () => { - await db - .delete(organizationSearchMcpInvocation) - .where(eq(organizationSearchMcpInvocation.organizationId, organizationId)) - const clientName = 'MCP fixture client'.repeat(20) - await db - .update(oauthClient) - .set({ name: clientName }) - .where(eq(oauthClient.clientId, oauthClientId)) - try { - await aliceOAuth.listTools() - expect(fixtures.afterResponse).toHaveLength(0) - await search(aliceOAuth) - await value(aliceOAuth, 'read_document', { documentId }) - expect((await call(bob, 'read_document', { documentId })).isError).toBe(true) - expect(fixtures.afterResponse).toHaveLength(3) - await db - .update(oauthClient) - .set({ name: 'Renamed client' }) - .where(eq(oauthClient.clientId, oauthClientId)) - await flushAfterResponse() - const rows = await db - .select() - .from(organizationSearchMcpInvocation) - .where(eq(organizationSearchMcpInvocation.organizationId, organizationId)) - .orderBy(organizationSearchMcpInvocation.createdAt) - .limit(10) - expect(rows).toHaveLength(3) - expect(rows).toMatchObject([ - { - organizationId, - userId: aliceId, - authKind: 'oauth_access_token', - oauthClientId, - clientName: clientName.slice(0, 256), - toolName: 'search', - outcome: 'success', - }, - { - organizationId, - userId: aliceId, - authKind: 'oauth_access_token', - oauthClientId, - clientName: clientName.slice(0, 256), - toolName: 'read_document', - outcome: 'success', - }, - { - organizationId, - userId: bobId, - authKind: 'personal_api_key', - oauthClientId: null, - clientName: null, - toolName: 'read_document', - outcome: 'error', - }, - ]) - expect(rows.every((row) => row.durationMs >= 0)).toBe(true) - expect(JSON.stringify(rows)).not.toContain(documentId) - expect(JSON.stringify(rows)).not.toContain(oauthTokens.alice) - } finally { - await db - .update(oauthClient) - .set({ name: 'Search MCP OAuth fixture' }) - .where(eq(oauthClient.clientId, oauthClientId)) - } - }) - - it('enforces current document and organization access on Search OAuth clients', async () => { - expect((await aliceOAuth.listTools()).tools).toHaveLength(3) - expect(await search(aliceOAuth)).toEqual(await search(alice)) - expect(await value(aliceOAuth, 'read_document', { documentId })).toMatchObject({ - documentId, - chunks: expect.arrayContaining([ - expect.objectContaining({ content: expect.stringContaining('Orion') }), - ]), - }) - expect(await value(aliceOAuth, 'read_document', { documentId })).toHaveProperty('chunks') - await expectDocumentHidden(bobOAuth) - await db.insert(knowledgeExternalGroupMember).values(bobSourceMembership) - try { - expect((await search(bobOAuth)).length).toBeGreaterThan(0) - await db.delete(member).where(eq(member.id, bobMembershipId)) - await expect(bobOAuth.listTools()).rejects.toThrow() - await expect(search(bobOAuth)).rejects.toThrow() - } finally { - await db - .insert(member) - .values({ id: bobMembershipId, userId: bobId, organizationId, role: 'member' }) - .onConflictDoNothing() - await revokeBobSourceAccess() - } - await expectDocumentHidden(bobOAuth) - await db - .delete(oauthAccessToken) - .where(eq(oauthAccessToken.token, hashOAuthToken(oauthTokens.bob))) - await expect(bobOAuth.listTools()).rejects.toThrow() - }) - - it('resolves provider URLs in the organization index and focuses OAuth document reads', async () => { - const url = 'https://fixture.atlassian.net/wiki/pages/organization-mcp' - expect( - await value(aliceOAuth, 'read_document', { url, aroundChunkIndex: 1, limit: 1 }) - ).toMatchObject({ - documentId, - citationUrl: url, - chunks: [expect.objectContaining({ chunkIndex: 1 })], - pagination: { offset: 1, limit: 1 }, - }) - expect((await call(bob, 'read_document', { url })).isError).toBe(true) - expect( - (await call(alice, 'read_document', { url, knowledgeBaseId: generateId() })).isError - ).toBe(true) - await expect( - readIndexedKnowledgeDocument.execute({ - principal: alicePrincipal, - input: { - organizationId: otherOrganizationId, - target: { kind: 'url', url }, - limit: 20, - resultSecretRegistry: new ResolvedSecretTraceRegistry(), - }, - }) - ).rejects.toMatchObject({ code: 'not_found' }) - expect((await call(alice, 'read_document', { url, documentId })).isError).toBe(true) - expect( - (await call(alice, 'read_document', { url, offset: 0, aroundChunkIndex: 1 })).isError - ).toBe(true) - }) - - it('applies organization source, modified-date, and document filters through search', async () => { - const [original] = await db - .select({ sourceModifiedAt: document.sourceModifiedAt }) - .from(document) - .where(eq(document.id, documentId)) - await db - .update(document) - .set({ sourceModifiedAt: new Date('2026-01-02T00:00:00Z') }) - .where(eq(document.id, documentId)) - try { - const included = await value(alice, 'search', { - query: 'Orion', - source: 'google_drive', - modifiedAfter: '2026-01-01T00:00:00Z', - documentIds: [documentId], - }) - expect(included.results).toEqual( - expect.arrayContaining([expect.objectContaining({ documentId })]) - ) - for (const filters of [ - { source: 'slack' }, - { modifiedAfter: '2026-01-03T00:00:00Z' }, - { documentIds: [generateId()] }, - ]) { - expect(await value(alice, 'search', { query: 'Orion', ...filters })).toMatchObject({ - results: [], - }) - } - } finally { - await db.update(document).set(original).where(eq(document.id, documentId)) - } - }) - - it('denies nonmembers, other organizations and same-organization workspace keys before discovery', async () => { - expect((await request(tokens.outsider)).status).toBe(404) - expect((await request(tokens.alice, otherOrganizationId)).status).toBe(404) - expect((await request(tokens.workspace)).status).toBe(403) - expect( - ( - await call(alice, 'search', { - query: 'Orion', - knowledgeBaseIds: [otherKnowledgeBaseId], - }) - ).isError - ).toBe(true) - expect( - (await call(alice, 'read_document', { knowledgeBaseId: otherKnowledgeBaseId, documentId })) - .isError - ).toBe(true) - }) - - it('grants and revokes source ACL membership on existing clients', async () => { - await db.insert(knowledgeExternalGroupMember).values(bobSourceMembership) - try { - expect((await search(bob)).length).toBeGreaterThan(0) - expect(await value(bob, 'read_document', { documentId })).toHaveProperty( - 'documentId', - documentId - ) - expect(await value(bob, 'read_document', { documentId })).toHaveProperty('chunks') - } finally { - await revokeBobSourceAccess() - } - await expectDocumentHidden(bob) - }) - - it('hides source content immediately when organization approval is disabled', async () => { - const approval = and( - eq(organizationSearchIntegration.organizationId, organizationId), - eq(organizationSearchIntegration.connectorType, 'google_drive') - ) - await db.update(organizationSearchIntegration).set({ approved: false }).where(approval) - try { - await expectDocumentHidden(alice) - expect(await applicationSearch(alicePrincipal)).toEqual([]) - } finally { - await db.update(organizationSearchIntegration).set({ approved: true }).where(approval) - } - expect((await search(alice)).length).toBeGreaterThan(0) - }) - - it('excludes disabled documents from Search and MCP while preserving admin management previews', async () => { - const chunks = await db - .select({ enabled: embedding.enabled }) - .from(embedding) - .where(eq(embedding.documentId, documentId)) - expect(chunks.length).toBeGreaterThan(1) - expect(chunks.every((chunk) => chunk.enabled)).toBe(true) - await db.update(document).set({ enabled: false }).where(eq(document.id, documentId)) - try { - await expectDocumentHidden(alice) - expect(await applicationSearch(alicePrincipal)).toEqual([]) - const input = { knowledgeBaseId, documentId, assertedOrganizationId: organizationId } - const metadata = await readKnowledgeDocument.execute({ principal: alicePrincipal, input }) - expect(metadata.document.enabled).toBe(false) - const preview = await listKnowledgeChunks.execute({ principal: alicePrincipal, input }) - expect(preview.chunks.length).toBeGreaterThan(0) - } finally { - await db.update(document).set({ enabled: true }).where(eq(document.id, documentId)) - } - expect((await search(alice)).length).toBeGreaterThan(0) - }) - - it('revokes an existing client after current organization membership is removed despite retained source grants', async () => { - await db.insert(knowledgeExternalGroupMember).values(bobSourceMembership) - try { - expect((await search(bob)).length).toBeGreaterThan(0) - await db.delete(member).where(eq(member.id, bobMembershipId)) - expect((await request(tokens.bob)).status).toBe(404) - await expect(bob.listTools()).rejects.toThrow() - await expect(search(bob)).rejects.toThrow() - await expect(call(bob, 'read_document', { documentId })).rejects.toThrow() - await expect(applicationSearch(bobPrincipal)).rejects.toMatchObject({ code: 'not_found' }) - } finally { - await db - .insert(member) - .values({ id: bobMembershipId, userId: bobId, organizationId, role: 'member' }) - .onConflictDoNothing() - await revokeBobSourceAccess() - } - await expectDocumentHidden(bob) - }) - it('authenticates before protocol parsing and rejects unsupported or untrusted requests', async () => { - for (const method of ['initialize', 'tools/list', 'unknown']) { - for (const token of [undefined, 'invalid-fixture-key', tokens.expired]) { - expect( - (await request(token, organizationId, { body: { jsonrpc: '2.0', id: 1, method } })).status - ).toBe(401) - } - } - expect((await request(undefined, organizationId, { raw: '{' })).status).toBe(401) - for (const method of ['GET', 'DELETE'] as const) { - expect((await request(undefined, organizationId, { method })).status).toBe(401) - expect((await request(tokens.alice, organizationId, { method })).status).toBe(405) - } - expect( - ( - await request(tokens.alice, organizationId, { - headers: { authorization: `Bearer ${tokens.bob}` }, - }) - ).status - ).toBe(401) - expect( - ( - await request(tokens.alice, organizationId, { - headers: { origin: 'https://untrusted.example' }, - }) - ).status - ).toBe(403) - expect( - ( - await request(tokens.alice, organizationId, { - body: { jsonrpc: '2.0', id: 1, method: 'tools/list', ignored: 'x'.repeat(64 * 1024) }, - }) - ).status - ).toBe(413) - }) - - it('exposes no mutation tools and rejects invalid page bounds', async () => { - expect((await call(alice, 'delete_document', { documentId })).isError).toBe(true) - for (const page of [{ limit: 51 }, { offset: -1 }, { aroundChunkIndex: 1_000_001 }]) { - expect((await call(alice, 'read_document', { documentId, ...page })).isError).toBe(true) - } - expect((await call(alice, 'search', { query: 'Orion', topK: 51 })).isError).toBe(true) - }) - - it('returns an actionable empty index without creating one or accepting an alternate knowledge base', async () => { - await db - .update(knowledgeBase) - .set({ deletedAt: new Date() }) - .where(eq(knowledgeBase.id, knowledgeBaseId)) - try { - const empty = await value(alice, 'search', { query: 'Orion' }) - expect(empty.results).toEqual([]) - expect(empty.message).toContain('No Search index') - expect((await call(alice, 'read_document', { documentId })).isError).toBe(true) - expect( - (await call(alice, 'search', { query: 'Orion', knowledgeBaseIds: [knowledgeBaseId] })) - .isError - ).toBe(true) - } finally { - await db - .update(knowledgeBase) - .set({ deletedAt: null }) - .where(eq(knowledgeBase.id, knowledgeBaseId)) - } - }) - - it.each(['pending', 'processing', 'failed'])( - 'returns metadata without text when indexing status is %s', - async (processingStatus) => { - await db.update(document).set({ processingStatus }).where(eq(document.id, documentId)) - try { - const result = await value(alice, 'read_document', { documentId }) - expect(result).toMatchObject({ - documentId, - processingStatus, - title: 'Orion organization project', - }) - expect(result).not.toHaveProperty('chunks') - expect(result).not.toHaveProperty('pagination') - expect((await call(bob, 'read_document', { documentId })).isError).toBe(true) - } finally { - await db - .update(document) - .set({ processingStatus: 'completed' }) - .where(eq(document.id, documentId)) - } - } - ) - - it('rate limits discovery and reads through the existing shared buckets', async () => { - const metadataBucket = `v2:knowledge.search.index.read:user:${aliceId}` - const searchBucket = `v2:knowledge.search:user:${aliceId}` - await db - .update(rateLimitBucket) - .set({ tokens: '0', lastRefillAt: new Date() }) - .where(eq(rateLimitBucket.key, metadataBucket)) - try { - const denied = await request(tokens.alice) - expect(denied.status).toBe(429) - expect(denied.headers.get('retry-after')).not.toBeNull() - } finally { - await db.delete(rateLimitBucket).where(eq(rateLimitBucket.key, metadataBucket)) - } - await db - .update(rateLimitBucket) - .set({ tokens: '0', lastRefillAt: new Date() }) - .where(eq(rateLimitBucket.key, searchBucket)) - try { - expect((await call(alice, 'search', { query: 'Orion' })).isError).toBe(true) - expect((await alice.listTools()).tools).toHaveLength(3) - } finally { - await db.delete(rateLimitBucket).where(eq(rateLimitBucket.key, searchBucket)) - } - expect((await search(alice)).length).toBeGreaterThan(0) - }) - - it('bounds returned text and redacts source secrets without exposing their owner’s secret names', async () => { - const [original] = await db - .select({ - id: embedding.id, - content: embedding.content, - chunkHash: embedding.chunkHash, - secretProvenanceVersion: embedding.secretProvenanceVersion, - }) - .from(embedding) - .where(eq(embedding.documentId, documentId)) - .orderBy(embedding.chunkIndex) - .limit(1) - const [sidecar] = await db - .select() - .from(embeddingSecretProvenance) - .where(eq(embeddingSecretProvenance.embeddingId, original.id)) - try { - await db - .update(embedding) - .set({ content: 'Orion '.repeat(180_000), secretProvenanceVersion: null }) - .where(eq(embedding.id, original.id)) - const oversized = await call(alice, 'read_document', { - documentId, - limit: 1, - }) - expect(oversized.isError).toBe(true) - expect(JSON.stringify(oversized).length).toBeLessThan(1024) - - const secret = `fixture-secret-${generateId()}` - const text = `Orion source secret: ${secret}` - const encrypted = await encryptSecret(secret) - await db.transaction(async (tx) => { - await tx - .update(embedding) - .set({ content: text, chunkHash: sha256Hex(text) }) - .where(eq(embedding.id, original.id)) - await replaceKnowledgeEmbeddingSecretProvenanceInTx(tx, original.id, text, { - status: 'exact', - entries: [ - { - encryptedValue: encrypted.encrypted, - name: 'FIXTURE_PRIVATE_SECRET_NAME', - sourceUserId: aliceId, - }, - ], - }) - }) - for (const result of [ - await value(alice, 'read_document', { documentId, limit: 1 }), - await value(alice, 'search', { query: 'Orion' }), - ]) { - expect(JSON.stringify(result)).not.toContain(secret) - expect(JSON.stringify(result)).not.toContain('FIXTURE_PRIVATE_SECRET_NAME') - } - await db - .update(embeddingSecretProvenance) - .set({ entries: [{ encryptedValue: 'invalid-fixture-cipher', sourceUserId: aliceId }] }) - .where(eq(embeddingSecretProvenance.embeddingId, original.id)) - expect((await call(alice, 'read_document', { documentId, limit: 1 })).isError).toBe(true) - } finally { - await db - .update(embedding) - .set({ - content: original.content, - chunkHash: original.chunkHash, - secretProvenanceVersion: original.secretProvenanceVersion, - }) - .where(eq(embedding.id, original.id)) - if (sidecar) - await db - .insert(embeddingSecretProvenance) - .values(sidecar) - .onConflictDoUpdate({ target: embeddingSecretProvenance.embeddingId, set: sidecar }) - else - await db - .delete(embeddingSecretProvenance) - .where(eq(embeddingSecretProvenance.embeddingId, original.id)) - } - }) - - it('handles client cancellation without retaining or poisoning a session', async () => { - const controller = new AbortController() - controller.abort() - await expect( - alice.callTool({ name: 'search', arguments: { query: 'Orion' } }, undefined, { - signal: controller.signal, - }) - ).rejects.toThrow() - expect((await search(alice)).length).toBeGreaterThan(0) - }) -}) diff --git a/apps/sim/lib/knowledge/__integration__/organization-search-overview.integration.ts b/apps/sim/lib/knowledge/__integration__/organization-search-overview.integration.ts deleted file mode 100644 index f9565f1097e..00000000000 --- a/apps/sim/lib/knowledge/__integration__/organization-search-overview.integration.ts +++ /dev/null @@ -1,432 +0,0 @@ -import { db } from '@sim/db' -import { - account, - credential, - document, - knowledgeBase, - knowledgeConnector, - knowledgeConnectorMember, - knowledgeConnectorMemberSyncLog, - knowledgeConnectorSyncLog, - member, - organization, - organizationSearchIntegration, - user, - workspace, -} from '@sim/db/schema' -import { generateId } from '@sim/utils/id' -import { eq, inArray, sql } from 'drizzle-orm' -import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' -import { - createKnowledgeAclFixtureIds, - seedKnowledgeAclFixture, -} from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { readOrganizationSearchOverview } from '@/lib/knowledge/application/organization-search-overview' -import { listSearchSources } from '@/lib/knowledge/application/search-sources' -import { SOURCE_PERMISSION_ERROR } from '@/lib/knowledge/connectors/sync-limits' - -const ids = createKnowledgeAclFixtureIds() -const indexId = generateId() -const driveId = generateId() -const pausedDriveId = generateId() -const gmailId = generateId() -const credentialId = generateId() -const accountId = generateId() -const memberId = generateId() -const documentId = generateId() -const sourceIds = [driveId, pausedDriveId, gmailId] -const principal = { kind: 'session', userId: ids.aliceId, sessionId: 'overview-admin' } as const -const input = { organizationId: ids.organizationId } - -beforeAll(async () => { - await seedKnowledgeAclFixture(ids) - await db.insert(member).values([ - { id: generateId(), organizationId: ids.organizationId, userId: ids.aliceId, role: 'admin' }, - { id: generateId(), organizationId: ids.organizationId, userId: ids.bobId, role: 'member' }, - ]) - await db.insert(knowledgeBase).values({ - id: indexId, - userId: ids.aliceId, - organizationId: ids.organizationId, - name: 'Overview fixture', - isSearchIndex: true, - }) - await db.insert(knowledgeConnector).values([ - { - id: driveId, - knowledgeBaseId: indexId, - connectorType: 'google_drive', - accessMode: 'admin', - sourceConfig: {}, - }, - { - id: pausedDriveId, - knowledgeBaseId: indexId, - connectorType: 'google_drive', - accessMode: 'admin', - sourceConfig: {}, - }, - { - id: gmailId, - knowledgeBaseId: indexId, - connectorType: 'gmail', - accessMode: 'members', - sourceConfig: {}, - }, - ]) - await db.insert(account).values({ - id: accountId, - accountId: ids.bobId, - userId: ids.bobId, - providerId: 'google-email', - createdAt: new Date(), - updatedAt: new Date(), - }) - await db.insert(credential).values({ - id: credentialId, - organizationId: ids.organizationId, - type: 'oauth', - displayName: 'Fixture connection', - providerId: 'google-email', - accountId, - createdBy: ids.bobId, - }) - await db.insert(knowledgeConnectorMember).values({ - id: memberId, - organizationId: ids.organizationId, - connectorId: gmailId, - credentialId, - subjectToken: `s:google-email:fixture:${ids.bobId}`, - }) - await db.insert(document).values({ - id: documentId, - knowledgeBaseId: indexId, - connectorId: driveId, - externalId: 'private-fixture', - filename: 'private-title.txt', - fileUrl: 'https://fixture.test/private', - fileSize: 5, - mimeType: 'text/plain', - processingStatus: 'completed', - acl: ['u:someone-else@fixture.test'], - aclVerifiedAt: new Date(), - }) -}) - -beforeEach(async () => { - await db - .delete(organizationSearchIntegration) - .where(eq(organizationSearchIntegration.organizationId, ids.organizationId)) - await db - .delete(knowledgeConnectorMemberSyncLog) - .where(inArray(knowledgeConnectorMemberSyncLog.connectorId, sourceIds)) - await db - .delete(knowledgeConnectorSyncLog) - .where(inArray(knowledgeConnectorSyncLog.connectorId, sourceIds)) - await db - .update(knowledgeConnector) - .set({ - status: 'active', - memberSyncStatus: 'idle', - lastSyncAt: new Date(), - lastMemberSyncAt: new Date(), - lastSyncError: null, - lastMemberSyncError: null, - listingCheckpoint: null, - directoryCheckpoint: null, - nextMemberSyncAt: null, - }) - .where(inArray(knowledgeConnector.id, sourceIds)) - await db - .update(knowledgeConnector) - .set({ status: 'paused' }) - .where(eq(knowledgeConnector.id, pausedDriveId)) - await db - .update(knowledgeConnectorMember) - .set({ - status: 'active', - lastCompleteListingAt: new Date(), - memberSyncedThrough: new Date(), - lastError: null, - consecutiveFailures: 0, - listingCheckpoint: null, - }) - .where(eq(knowledgeConnectorMember.id, memberId)) - await db - .update(document) - .set({ - processingStatus: 'completed', - enabled: true, - userExcluded: false, - contentHash: null, - storageKey: null, - fileUrl: 'https://fixture.test/private', - }) - .where(eq(document.id, documentId)) -}) - -afterAll(async () => { - await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) - await db.delete(organization).where(eq(organization.id, ids.organizationId)) - await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) -}) - -async function provider(connectorType: string) { - return (await readOrganizationSearchOverview.execute({ principal, input })).providers.find( - (item) => item.connectorType === connectorType - ) -} - -describe('organization operational overview with real SQL', () => { - it.each([ - SOURCE_PERMISSION_ERROR, - `Directory refresh incomplete: fixture\n${SOURCE_PERMISSION_ERROR}`, - `${SOURCE_PERMISSION_ERROR}\nSource listing failed for a fixture account`, - `Directory refresh incomplete: fixture\n${SOURCE_PERMISSION_ERROR}\nSource listing failed for a fixture account`, - ])( - 'recognizes a complete permission notice within composed diagnostics: %s', - async (lastSyncError) => { - await db - .update(knowledgeConnector) - .set({ lastSyncError }) - .where(eq(knowledgeConnector.id, driveId)) - expect(await provider('google_drive')).toMatchObject({ - status: 'needs_attention', - issue: 'permission_sync_incomplete', - }) - } - ) - - it('does not classify a provider message containing the permission text as its own notice', async () => { - await db - .update(knowledgeConnector) - .set({ lastSyncError: `Provider message: ${SOURCE_PERMISSION_ERROR}` }) - .where(eq(knowledgeConnector.id, driveId)) - expect(await provider('google_drive')).toMatchObject({ - status: 'needs_attention', - issue: 'sync_failed', - }) - }) - - it('ignores permission notices on paused sources', async () => { - await db - .update(knowledgeConnector) - .set({ lastSyncError: `Directory refresh incomplete: fixture\n${SOURCE_PERMISSION_ERROR}` }) - .where(eq(knowledgeConnector.id, pausedDriveId)) - expect(await provider('google_drive')).toMatchObject({ status: 'active', issue: null }) - }) - - it('counts configured sources independently of viewer ACLs, and excludes workspace and untouched providers', async () => { - const result = await readOrganizationSearchOverview.execute({ principal, input }) - expect(result.providers).toEqual( - expect.arrayContaining([ - { - connectorType: 'google_drive', - sourceCount: 2, - approved: true, - status: 'active', - issue: null, - isSyncing: false, - hasPendingSync: false, - }, - { - connectorType: 'gmail', - sourceCount: 1, - approved: true, - status: 'active', - issue: null, - isSyncing: false, - hasPendingSync: false, - }, - ]) - ) - expect(result.providers).toHaveLength(2) - expect(JSON.stringify(result)).not.toMatch(/private-title|someone-else|fixture connection/i) - const visible = await listSearchSources.execute({ - principal, - input: { ...input, connectorType: 'google_drive' }, - }) - expect(visible.sources).toHaveLength(2) - expect(visible.sources.every((source) => !source.hasViewerDocuments)).toBe(true) - }) - it('keeps explicit approvals and deactivations visible before source creation', async () => { - await db.insert(organizationSearchIntegration).values([ - { organizationId: ids.organizationId, connectorType: 'github', approved: true }, - { organizationId: ids.organizationId, connectorType: 'confluence', approved: false }, - { organizationId: ids.organizationId, connectorType: 'notion', approved: true }, - ]) - expect(await provider('github')).toMatchObject({ - sourceCount: 0, - approved: true, - status: 'needs_setup', - }) - expect(await provider('confluence')).toMatchObject({ - sourceCount: 0, - approved: false, - status: 'paused', - }) - expect(await provider('notion')).toBeUndefined() - }) - it('reports waiting accounts despite an empty member run having a completion timestamp', async () => { - await db - .update(knowledgeConnectorMember) - .set({ status: 'disabled' }) - .where(eq(knowledgeConnectorMember.id, memberId)) - await db - .update(knowledgeConnector) - .set({ memberSyncStatus: 'pending' }) - .where(eq(knowledgeConnector.id, gmailId)) - expect(await provider('gmail')).toMatchObject({ status: 'waiting_for_connections' }) - }) - it('distinguishes queued member continuation from active indexing and partial failure', async () => { - await db - .update(knowledgeConnectorMember) - .set({ listingCheckpoint: { cursor: 'fixture' } }) - .where(eq(knowledgeConnectorMember.id, memberId)) - const logId = generateId() - await db.insert(knowledgeConnectorMemberSyncLog).values({ - id: logId, - connectorId: gmailId, - status: 'partial', - membersIncomplete: 1, - docsFailed: 0, - processingDispatchFailed: 0, - completedAt: new Date(), - }) - expect(await provider('gmail')).toMatchObject({ - status: 'active', - isSyncing: false, - hasPendingSync: true, - }) - await db - .update(knowledgeConnectorMemberSyncLog) - .set({ membersFailed: 1 }) - .where(eq(knowledgeConnectorMemberSyncLog.id, logId)) - expect(await provider('gmail')).toMatchObject({ status: 'needs_attention' }) - await db - .update(knowledgeConnectorMemberSyncLog) - .set({ membersFailed: 0 }) - .where(eq(knowledgeConnectorMemberSyncLog.id, logId)) - await db - .update(knowledgeConnectorMember) - .set({ listingCheckpoint: null }) - .where(eq(knowledgeConnectorMember.id, memberId)) - expect(await provider('gmail')).toMatchObject({ status: 'needs_attention' }) - await db - .update(knowledgeConnector) - .set({ nextMemberSyncAt: sql`statement_timestamp() - interval '1 second'` }) - .where(eq(knowledgeConnector.id, gmailId)) - expect(await provider('gmail')).toMatchObject({ - status: 'active', - isSyncing: false, - hasPendingSync: true, - }) - await db - .update(knowledgeConnector) - .set({ nextMemberSyncAt: null }) - .where(eq(knowledgeConnector.id, gmailId)) - await db.insert(knowledgeConnectorMemberSyncLog).values({ - id: generateId(), - connectorId: gmailId, - status: 'completed', - docsFailed: 0, - processingDispatchFailed: 0, - startedAt: new Date(Date.now() + 1000), - completedAt: new Date(), - }) - expect(await provider('gmail')).toMatchObject({ status: 'active' }) - }) - it('ignores intentional skips while reporting inaccessible source and indexing failures', async () => { - await db - .update(document) - .set({ processingStatus: 'failed', contentHash: 'immutable-sha', fileUrl: '' }) - .where(eq(document.id, documentId)) - expect(await provider('google_drive')).toMatchObject({ - status: 'active', - issue: null, - isSyncing: false, - }) - await db - .update(knowledgeConnector) - .set({ lastSyncError: 'previous sync failed' }) - .where(eq(knowledgeConnector.id, driveId)) - expect(await provider('google_drive')).toMatchObject({ - status: 'needs_attention', - issue: 'sync_failed', - isSyncing: false, - }) - await db - .update(knowledgeConnector) - .set({ lastSyncError: null }) - .where(eq(knowledgeConnector.id, driveId)) - await db.update(document).set({ contentHash: null }).where(eq(document.id, documentId)) - expect(await provider('google_drive')).toMatchObject({ - status: 'needs_attention', - issue: 'document_indexing_failed', - isSyncing: false, - }) - await db - .update(document) - .set({ contentHash: 'immutable-sha', storageKey: 'fixture-retained-artifact' }) - .where(eq(document.id, documentId)) - expect(await provider('google_drive')).toMatchObject({ - status: 'needs_attention', - issue: 'document_indexing_failed', - isSyncing: false, - }) - await db.update(document).set({ userExcluded: true }).where(eq(document.id, documentId)) - expect(await provider('google_drive')).toMatchObject({ status: 'active' }) - await db - .update(document) - .set({ userExcluded: false, processingStatus: 'processing' }) - .where(eq(document.id, documentId)) - expect(await provider('google_drive')).toMatchObject({ status: 'indexing' }) - await db - .update(knowledgeConnector) - .set({ lastSyncError: 'previous sync failed' }) - .where(eq(knowledgeConnector.id, driveId)) - expect(await provider('google_drive')).toMatchObject({ - status: 'needs_attention', - isSyncing: true, - }) - }) - it('surfaces retained source errors and stale member permissions, while pause and deactivation take precedence', async () => { - await db - .update(knowledgeConnector) - .set({ lastSyncError: 'private-provider-error' }) - .where(eq(knowledgeConnector.id, driveId)) - expect(await provider('google_drive')).toMatchObject({ status: 'needs_attention' }) - await db - .update(knowledgeConnectorMember) - .set({ - memberSyncedThrough: new Date(Date.now() - 3 * 86_400_000), - lastCompleteListingAt: new Date(Date.now() - 3 * 86_400_000), - }) - .where(eq(knowledgeConnectorMember.id, memberId)) - expect(await provider('gmail')).toMatchObject({ status: 'needs_attention' }) - await db - .update(knowledgeConnector) - .set({ status: 'paused' }) - .where(eq(knowledgeConnector.id, driveId)) - expect(await provider('google_drive')).toMatchObject({ status: 'paused' }) - await db - .insert(organizationSearchIntegration) - .values({ organizationId: ids.organizationId, connectorType: 'gmail', approved: false }) - expect(await provider('gmail')).toMatchObject({ - approved: false, - status: 'paused', - isSyncing: false, - }) - }) - it('requires a current organization admin even for a workspace administrator', async () => { - await expect( - readOrganizationSearchOverview.execute({ - principal: { ...principal, userId: ids.bobId }, - input, - }) - ).rejects.toMatchObject({ code: 'forbidden' }) - await expect( - readOrganizationSearchOverview.execute({ principal, input: { organizationId: generateId() } }) - ).rejects.toMatchObject({ code: 'not_found' }) - }) -}) diff --git a/apps/sim/lib/knowledge/__integration__/processing-lock-scope.integration.ts b/apps/sim/lib/knowledge/__integration__/processing-lock-scope.integration.ts index 427f1e758de..fb793b32330 100644 --- a/apps/sim/lib/knowledge/__integration__/processing-lock-scope.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/processing-lock-scope.integration.ts @@ -13,13 +13,6 @@ import { eq, inArray } from 'drizzle-orm' import postgres from 'postgres' import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' -/** These transaction checks exercise indexed Search, which Live Search normally disables. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) - const fixtures = vi.hoisted(() => ({ root: '', process: vi.fn(), embeddings: vi.fn() })) vi.mock('@/lib/uploads/core/setup.server', () => ({ get UPLOAD_DIR_SERVER() { @@ -61,11 +54,6 @@ describe('document processing commit lock scope', () => { beforeAll(async () => { fixtures.root = mkdtempSync(path.join(tmpdir(), 'sim-processing-lock-scope-')) await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) - /** A search index, the only kind of base whose chunks the keyword projection holds. */ - await db - .update(knowledgeBase) - .set({ isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) vi.spyOn(embeddingClient, 'assertKnowledgeEmbeddingCapacity').mockResolvedValue(undefined) fixtures.process.mockResolvedValue({ chunks, @@ -164,7 +152,7 @@ describe('document processing commit lock scope', () => { expect(await db.select().from(document).where(eq(document.id, file.documentId))).toMatchObject([ { processingStatus: 'completed', chunkCount: 3 }, ]) - for (const projection of ['embedding_search', 'embedding_keyword_search']) { + for (const projection of ['embedding_search']) { expect( await db.$client.unsafe( `SELECT count(*)::int AS count FROM ${projection} WHERE document_id = $1`, @@ -228,7 +216,7 @@ describe('document processing commit lock scope', () => { .from(embedding) .where(eq(embedding.documentId, file.documentId)) ).toEqual([]) - for (const projection of ['embedding_search', 'embedding_keyword_search']) { + for (const projection of ['embedding_search']) { expect( await db.$client.unsafe(`SELECT id FROM ${projection} WHERE document_id = $1`, [ file.documentId, diff --git a/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts index 2003f25e31e..a3b567598f1 100644 --- a/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts @@ -11,7 +11,6 @@ import { document, embedding, knowledgeBase, - member, organization, outboxEvent, rateLimitBucket, @@ -25,22 +24,13 @@ import { and, eq, inArray, sql } from 'drizzle-orm' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' const fixtureStorage = vi.hoisted(() => ({ root: '' })) -/** This suite covers indexed organization search, which is dormant unless Live Search is off. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) vi.mock('@/lib/uploads/core/setup.server', () => ({ get UPLOAD_DIR_SERVER() { return fixtureStorage.root }, })) -import { - resolveBillingAttribution, - resolveOrganizationBillingAttribution, -} from '@/lib/billing/core/billing-attribution' +import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' import { env } from '@/lib/core/config/env' import { processOutboxEventById } from '@/lib/core/outbox/service' import * as egress from '@/lib/core/security/input-validation.server' @@ -60,13 +50,12 @@ import { recordMemberObservations, } from '@/lib/knowledge/connectors/member-observations' import { createContentSyncLease, createMemberSyncLease } from '@/lib/knowledge/connectors/sync-lock' -import { addDocument, persistDocumentAcls } from '@/lib/knowledge/connectors/sync-persistence' +import { addDocument } from '@/lib/knowledge/connectors/sync-persistence' import { KNOWLEDGE_DOCUMENT_CONTINUATION_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-continuation-dispatch' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' import { assertDocumentProcessingPayload } from '@/lib/knowledge/documents/processing-payload' import * as providerContinuation from '@/lib/knowledge/documents/processing-provider-continuation' import { processDocumentsWithQueue } from '@/lib/knowledge/documents/service' -import { searchScopedKnowledge } from '@/lib/sim-search/indexed/search/scoped-search' const PNG = Buffer.from( 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=', @@ -114,7 +103,7 @@ describe('provider throttling resumes the shared indexing pipeline', () => { await db.$client.end() }) - it.each(['regular KB', 'member source', 'organization Search'] as const)( + it.each(['regular KB', 'member source'] as const)( 'recovers a %s after Mistral 429 without burning dispatches or charging twice', async (scope) => { vi.useRealTimers() @@ -129,28 +118,6 @@ describe('provider throttling resumes the shared indexing pipeline', () => { connectorId = memberFixture.connectorId lease = createMemberSyncLease(connectorId, memberFixture.runId) } - const orgOwned = scope === 'organization Search' - if (orgOwned) { - await db.insert(member).values([ - { - id: generateId(), - organizationId: ids.organizationId, - userId: ids.aliceId, - role: 'owner', - }, - { - id: generateId(), - organizationId: ids.organizationId, - userId: ids.bobId, - role: 'member', - }, - ]) - await db - .update(knowledgeBase) - .set({ workspaceId: null, organizationId: ids.organizationId, isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) - } - const file = await addDocument( ids.knowledgeBaseId, connectorId, @@ -163,11 +130,9 @@ describe('provider throttling resumes the shared indexing pipeline', () => { contentHash: 'synthetic-scan-v1', sourceFile: { bytes: PNG, fileName: 'Orion scan.png', mimeType: 'image/png' }, }, - orgOwned - ? { userId: ids.aliceId, workspaceId: null, organizationId: ids.organizationId } - : { userId: ids.aliceId, workspaceId: ids.workspaceId }, + { userId: ids.aliceId, workspaceId: ids.workspaceId }, undefined, - scope === 'member source' ? 'members' : orgOwned ? 'admin' : 'workspace', + scope === 'member source' ? 'members' : 'workspace', lease ) if (scope === 'regular KB') { @@ -180,11 +145,6 @@ describe('provider throttling resumes the shared indexing pipeline', () => { memberFixture.runId ) await materializeDocumentAcls(connectorId, [file.documentId]) - } else { - await persistDocumentAcls( - connectorId, - new Map([['orion-scan', [`u:${ids.aliceId}@fixture.test`]]]) - ) } let ocrRequests = 0 @@ -260,15 +220,10 @@ describe('provider throttling resumes the shared indexing pipeline', () => { arrayBuffer: () => response.arrayBuffer(), } }) - const billing = orgOwned - ? await resolveOrganizationBillingAttribution({ - actorUserId: ids.aliceId, - organizationId: ids.organizationId, - }) - : await resolveBillingAttribution({ - actorUserId: ids.aliceId, - workspaceId: ids.workspaceId, - }) + const billing = await resolveBillingAttribution({ + actorUserId: ids.aliceId, + workspaceId: ids.workspaceId, + }) const requestId = generateId() const startedAt = Date.now() const holdParentHandoff = scope === 'regular KB' @@ -431,26 +386,16 @@ describe('provider throttling resumes the shared indexing pipeline', () => { expect(ocrRequests).toBe(2) expect(await charges()).toHaveLength(1) const principal = { kind: 'session' as const, userId: ids.aliceId, sessionId: generateId() } - const result = orgOwned - ? await searchScopedKnowledge.execute({ - principal, - input: { - organizationId: ids.organizationId, - query: 'Orion', - topK: 3, - searchMode: 'hybrid', - }, - }) - : await searchKnowledge.execute({ - principal, - input: { - workspaceId: ids.workspaceId, - knowledgeBaseIds: [ids.knowledgeBaseId], - query: 'Orion', - topK: 3, - searchMode: 'hybrid', - }, - }) + const result = await searchKnowledge.execute({ + principal, + input: { + workspaceId: ids.workspaceId, + knowledgeBaseIds: [ids.knowledgeBaseId], + query: 'Orion', + topK: 3, + searchMode: 'hybrid', + }, + }) expect(result.results.map((row) => row.documentId)).toContain(file.documentId) if (scope === 'member source') { const hidden = await searchKnowledge.execute({ diff --git a/apps/sim/lib/knowledge/__integration__/read-indexed-document.integration.ts b/apps/sim/lib/knowledge/__integration__/read-indexed-document.integration.ts deleted file mode 100644 index 9eea5b6cc4a..00000000000 --- a/apps/sim/lib/knowledge/__integration__/read-indexed-document.integration.ts +++ /dev/null @@ -1,343 +0,0 @@ -/** Real scoped document resolution, ACLs, ingestion, pagination, and provenance on disposable Postgres. */ -import { mkdtempSync } from 'node:fs' -import { rm } from 'node:fs/promises' -import { tmpdir } from 'node:os' -import path from 'node:path' -import type { Principal } from '@sim/auth/principal' -import { db } from '@sim/db' -import { - document, - embedding, - knowledgeBase, - knowledgeExternalGroup, - member, - organization, - organizationSearchIntegration, - user, - workspace, -} from '@sim/db/schema' -import { generateId } from '@sim/utils/id' -import { eq, inArray } from 'drizzle-orm' -import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' - -const fixtures = vi.hoisted(() => ({ storageRoot: '' })) -/** This suite covers indexed organization search, which is dormant unless Live Search is off. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) -vi.mock('@/lib/uploads/core/setup.server', () => ({ - get UPLOAD_DIR_SERVER() { - return fixtures.storageRoot - }, -})) -vi.mock('@/lib/embeddings', async () => ({ - ...(await import('@/lib/embeddings/client')), - assertKnowledgeEmbeddingCapacity: async () => {}, - embedKnowledge: async (texts: string[]) => ({ - embeddings: texts.map(() => [1, ...Array(1535).fill(0)]), - totalTokens: texts.length, - billableTokens: 0, - isBYOK: true, - modelName: 'text-embedding-3-small', - pricingId: 'text-embedding-3-small', - }), -})) - -import { resolveOrganizationBillingAttribution } from '@/lib/billing/core/billing-attribution' -import { - createKnowledgeAclFixtureIds, - seedKnowledgeAclFixture, -} from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { confluencePageAcl } from '@/lib/knowledge/access/confluence-permissions' -import { createContentSyncLease } from '@/lib/knowledge/connectors/sync-lock' -import { addDocument, persistDocumentAcls } from '@/lib/knowledge/connectors/sync-persistence' -import { processDocumentAsync } from '@/lib/knowledge/documents/service' -import { - type ReadIndexedKnowledgeDocumentInput, - readIndexedKnowledgeDocument, -} from '@/lib/sim-search/indexed/documents/read-indexed-document' -import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' - -describe('indexed document references', () => { - const ids = createKnowledgeAclFixtureIds() - const other = createKnowledgeAclFixtureIds() - const alice: Principal = { kind: 'session', userId: ids.aliceId, sessionId: 'fixture-alice' } - const bob: Principal = { kind: 'session', userId: ids.bobId, sessionId: 'fixture-bob' } - const sourceUrl = 'https://fixture.atlassian.net/wiki/pages/target?view=all#section' - const content = Array.from( - { length: 240 }, - (_, index) => - `Document section ${index}: the customer setup checklist includes secure source connections, indexing, search, and citations. ` - ).join('\n\n') - let documentId: string - let hiddenDocumentId: string - - async function ingest(externalId: string) { - const doc = await addDocument( - ids.knowledgeBaseId, - ids.connectorId, - 'google_drive', - { - externalId, - mimeType: 'text/plain', - title: externalId, - content, - contentHash: externalId, - sourceUrl, - }, - { userId: ids.aliceId, workspaceId: null, organizationId: ids.organizationId }, - undefined, - 'admin', - createContentSyncLease(ids.connectorId, ids.lockId) - ) - await processDocumentAsync( - ids.knowledgeBaseId, - doc.documentId, - doc, - {}, - await resolveOrganizationBillingAttribution({ - actorUserId: ids.aliceId, - organizationId: ids.organizationId, - }) - ) - return doc.documentId - } - - function read(principal: Principal, input: Partial = {}) { - return readIndexedKnowledgeDocument.execute({ - principal, - input: { - organizationId: ids.organizationId, - target: { kind: 'url', url: sourceUrl }, - limit: 20, - resultSecretRegistry: new ResolvedSecretTraceRegistry(), - ...input, - }, - }) - } - - beforeAll(async () => { - fixtures.storageRoot = mkdtempSync(path.join(tmpdir(), 'sim-indexed-document-')) - vi.stubGlobal('fetch', async () => { - throw new Error('Indexed document reads must never fetch provider URLs') - }) - await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) - await seedKnowledgeAclFixture(other, { connectorType: 'google_drive' }) - await db - .update(knowledgeBase) - .set({ workspaceId: null, organizationId: ids.organizationId, isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) - await db.insert(member).values([ - { id: generateId(), userId: ids.aliceId, organizationId: ids.organizationId, role: 'owner' }, - { id: generateId(), userId: ids.bobId, organizationId: ids.organizationId, role: 'member' }, - ]) - await db - .update(knowledgeExternalGroup) - .set({ workspaceId: null, organizationId: ids.organizationId }) - .where(inArray(knowledgeExternalGroup.id, ids.groupIds)) - await db - .insert(organizationSearchIntegration) - .values({ organizationId: ids.organizationId, connectorType: 'google_drive', approved: true }) - documentId = await ingest('visible-target') - hiddenDocumentId = await ingest('hidden-target') - const sourceAcl = confluencePageAcl({ - providerId: 'google-drive', - tenantId: 'fixture-tenant', - spacePrincipals: [{ kind: 'group', id: 'space' }], - restrictionChain: [[{ kind: 'group', id: 'page' }]], - }) - await persistDocumentAcls(ids.connectorId, new Map([['visible-target', sourceAcl]])) - }) - - afterAll(async () => { - for (const fixture of [ids, other]) { - await db.delete(workspace).where(eq(workspace.id, fixture.workspaceId)) - await db.delete(organization).where(eq(organization.id, fixture.organizationId)) - await db.delete(user).where(eq(user.id, fixture.aliceId)) - await db.delete(user).where(eq(user.id, fixture.bobId)) - } - await rm(fixtures.storageRoot, { recursive: true, force: true }) - vi.unstubAllGlobals() - await db.$client.end() - }) - - it('reads an exact URL without treating inaccessible duplicates as ambiguous', async () => { - const result = await read(alice) - expect(result.documentId).toBe(documentId) - expect(result.sourceUrl).toBe(sourceUrl) - expect(result.chunks?.length).toBeGreaterThan(3) - expect(result.chunks?.[0].content).toContain('Document section 0') - }) - - it('conceals URLs from a member without the provider permissions', async () => { - await expect(read(bob)).rejects.toThrow('Document not found') - }) - - it('rejects ambiguous accessible URLs and still accepts explicit document identifiers', async () => { - const [original] = await db.select().from(document).where(eq(document.id, documentId)) - await db - .update(document) - .set({ - acl: original.acl, - aclRequirements: original.aclRequirements, - aclVerifiedAt: original.aclVerifiedAt, - }) - .where(eq(document.id, hiddenDocumentId)) - try { - await expect(read(alice)).rejects.toThrow('Multiple accessible documents use this URL') - await expect(read(alice, { target: { kind: 'id', documentId } })).resolves.toMatchObject({ - documentId, - }) - } finally { - await db - .update(document) - .set({ acl: [], aclRequirements: [] }) - .where(eq(document.id, hiddenDocumentId)) - } - }) - - it('requires the canonical organization index for both URL and ID reads', async () => { - await db - .update(knowledgeBase) - .set({ deletedAt: new Date() }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) - try { - await expect(read(alice)).rejects.toThrow('Document not found') - await expect(read(alice, { target: { kind: 'id', documentId } })).rejects.toThrow( - 'Document not found' - ) - } finally { - await db - .update(knowledgeBase) - .set({ deletedAt: null }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) - } - }) - - it('does not resolve document URLs across organizations', async () => { - await expect(read(alice, { organizationId: other.organizationId })).rejects.toMatchObject({ - code: 'not_found', - }) - }) - - it('rejects caller kinds and revoked owner access before resolving the URL', async () => { - await expect( - read({ kind: 'workspace_api_key', workspaceId: other.workspaceId, keyId: 'foreign' }) - ).rejects.toMatchObject({ code: 'forbidden' }) - await expect( - read({ kind: 'session', userId: other.aliceId, sessionId: 'outsider' }) - ).rejects.toMatchObject({ code: 'not_found' }) - }) - - it.each([ - 'javascript:alert(1)', - 'https://user:secret@source.test/doc', - 'https:source.test/doc', - '/path', - 'https://source.test/a\nb', - ])('rejects malformed or unsafe provider URL %s', async (url) => { - await expect(read(alice, { target: { kind: 'url', url } })).rejects.toThrow( - 'HTTP or HTTPS document URL' - ) - }) - - it('does not rewrite provider queries or fragment identity during exact lookup', async () => { - await expect( - read(alice, { target: { kind: 'url', url: sourceUrl.replace('#section', '') } }) - ).rejects.toThrow('Document not found') - }) - - it.each(['enabled', 'userExcluded', 'archivedAt', 'deletedAt'] as const)( - 'omits documents when %s removes them from reading', - async (field) => { - const value = field === 'enabled' ? false : field === 'userExcluded' ? true : new Date() - await db - .update(document) - .set({ [field]: value }) - .where(eq(document.id, documentId)) - try { - await expect(read(alice)).rejects.toThrow('Document not found') - const target = { kind: 'id', documentId } as const - for (const page of [{}, { offset: 1 }, { aroundChunkIndex: 1 }]) { - await expect(read(alice, { target, ...page })).rejects.toThrow('Document not found') - } - } finally { - await db - .update(document) - .set({ [field]: field === 'enabled' ? true : field === 'userExcluded' ? false : null }) - .where(eq(document.id, documentId)) - } - } - ) - - it('centers enabled context around the actual matching chunk and resumes from the returned offset', async () => { - const all = await read(alice) - const first = all.chunks![0] - const target = all.chunks![4] - await db.update(embedding).set({ enabled: false }).where(eq(embedding.id, first.id)) - try { - const result = await read(alice, { aroundChunkIndex: target.chunkIndex, limit: 3 }) - expect(result.pagination?.offset).toBe(1) - expect(result.chunks?.map((chunk) => chunk.id)).toEqual( - all.chunks!.slice(2, 5).map((chunk) => chunk.id) - ) - const next = await read(alice, { - offset: result.pagination!.offset + result.chunks!.length, - limit: 3, - }) - expect(next.chunks?.[0].id).toBe(all.chunks![5].id) - const single = await read(alice, { aroundChunkIndex: target.chunkIndex, limit: 1 }) - expect(single.chunks?.map((chunk) => chunk.id)).toEqual([target.id]) - await expect(read(alice, { aroundChunkIndex: first.chunkIndex })).rejects.toThrow( - 'Document chunk not found' - ) - await expect(read(alice, { aroundChunkIndex: 999999 })).rejects.toThrow( - 'Document chunk not found' - ) - } finally { - await db.update(embedding).set({ enabled: true }).where(eq(embedding.id, first.id)) - } - }) - - it('bounds pagination and forbids ambiguous page coordinates', async () => { - for (const input of [ - { limit: 0 }, - { limit: 51 }, - { offset: -1 }, - { offset: 1_000_001 }, - { aroundChunkIndex: -1 }, - { aroundChunkIndex: 1.5 }, - ]) { - await expect(read(alice, input)).rejects.toThrow('Invalid document page bounds') - } - await expect(read(alice, { offset: 0, aroundChunkIndex: 1 })).rejects.toThrow( - 'Use offset or aroundChunkIndex, not both' - ) - }) - - it('retains metadata-only reads until indexing completes', async () => { - await db - .update(document) - .set({ processingStatus: 'processing' }) - .where(eq(document.id, documentId)) - try { - const result = await read(alice) - expect(result.processingStatus).toBe('processing') - expect(result).not.toHaveProperty('chunks') - expect(result).not.toHaveProperty('pagination') - } finally { - await db - .update(document) - .set({ processingStatus: 'completed' }) - .where(eq(document.id, documentId)) - } - }) - - it('stops cancelled reads', async () => { - const controller = new AbortController() - controller.abort() - await expect(read(alice, { signal: controller.signal })).rejects.toThrow() - }) -}) diff --git a/apps/sim/lib/knowledge/__integration__/search-latency.integration.ts b/apps/sim/lib/knowledge/__integration__/search-latency.integration.ts index 5eb6e667a39..687a872fa56 100644 --- a/apps/sim/lib/knowledge/__integration__/search-latency.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-latency.integration.ts @@ -3,37 +3,22 @@ import { readFileSync, statSync, writeFileSync } from 'node:fs' import type { Principal } from '@sim/auth/principal' import { db } from '@sim/db' import { - copilotChats, credential, credentialGroup, document, - embedding, - embeddingSearch, knowledgeBase, knowledgeConnector, knowledgeConnectorMember, knowledgeDocumentObservation, - member, organization, user, workspace, } from '@sim/db/schema' import { createLogger, Logger } from '@sim/logger' import { generateId } from '@sim/utils/id' -import { and, eq, inArray, type SQL, sql } from 'drizzle-orm' -import { NextRequest } from 'next/server' +import { eq, inArray, sql } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, it, type MockInstance, vi } from 'vitest' - -/** Turns indexed organization search on: its search-index knowledge bases are read through it. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) - import { z } from 'zod' -import { workspaceKnowledgeSearchDataSchema } from '@/lib/api/contracts/knowledge/search' -import { internalSessionAuth } from '@/lib/api/server/routes' import { seedSearchReaderFixture } from '@/lib/knowledge/__integration__/seed-search-reader-fixture' import { createKnowledgeAclFixtureIds, @@ -41,7 +26,6 @@ import { seedKnowledgeMemberFixture, } from '@/lib/knowledge/__integration__/seed-source-access-fixture' import { type KnowledgeSearchTagFilter, searchKnowledge } from '@/lib/knowledge/application/search' -import type { KbEmbeddingDimensions } from '@/lib/knowledge/embedding-models' import { SearchBudget, SearchDeadlineError, @@ -49,18 +33,6 @@ import { } from '@/lib/knowledge/search/budget' import type { SearchStage } from '@/lib/knowledge/search/diagnostics' import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters' -import { embeddingCandidateDistance } from '@/lib/knowledge/vector-columns' -import type { - MothershipStreamV1CheckpointPausePayload, - MothershipStreamV1ToolCallDescriptor, -} from '@/lib/mothership/generated/mothership-stream-v1' -import { isContractStreamEventEnvelope } from '@/lib/mothership/request/session/contract' -import { - readDocumentServerTool, - searchWorkspaceServerTool, -} from '@/lib/mothership/tools/server/knowledge/workspace-search' -import { POST as searchRoute } from '@/app/api/knowledge/search/route' -import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' /** Initialize controlled provider configuration before the real application modules load. */ vi.hoisted(() => { @@ -74,7 +46,6 @@ vi.hoisted(() => { } }) -const externalFetch = globalThis.fetch const enabled = process.env.KNOWLEDGE_SEARCH_PERFORMANCE_TEST === 'true' const batchSize = 1000 const MIN_CHUNK_COUNT = 5000 @@ -86,7 +57,6 @@ const unrelatedChunkCount = Number( const evictSharedBuffers = process.env.KNOWLEDGE_SEARCH_PERFORMANCE_EVICT_BUFFERS === 'true' const dimensions = 1536 const candidateDimensions = 512 -const HYBRID_CANDIDATE_LIMIT = 1600 const chunksPerDocument = 4 const logger = createLogger('SearchLatencyIntegration') const fixtureSchema = z.object({ @@ -119,7 +89,6 @@ const ids = reused?.fixture ?? createKnowledgeAclFixtureIds() const unrelated = reused?.unrelatedFixture ?? createKnowledgeAclFixtureIds() const fullWidthFixture = reused?.fullWidthFixture ?? createKnowledgeAclFixtureIds() const FULL_WIDTH_CHUNK_COUNT = 5000 -const organizationChatId = generateId() function topicVector(topic = 0) { const vector = Array.from({ length: dimensions }, (_, index) => Math.sin( @@ -137,20 +106,6 @@ const queryVector = topicVector() * The exact nearest chunks on the projection's stored halfvec, which is what the page's order * is measured against: the walk ranks on that column, and nothing rescores it. */ -async function exactProjectionNeighbors(vector: number[], limit: number, readerClause?: SQL) { - const distance = embeddingCandidateDistance( - dimensions as KbEmbeddingDimensions, - JSON.stringify(vector), - 'text-embedding-3-small' - ) - /** Unaliased: the distance expression qualifies its column with the table's own name. */ - return db.execute<{ id: string }>(sql`SELECT ${embeddingSearch.id} AS id FROM ${embeddingSearch} - INNER JOIN document d ON d.id = ${embeddingSearch.documentId} - WHERE ${embeddingSearch.knowledgeBaseId} = ${ids.knowledgeBaseId} - AND ${embeddingSearch.enabled} ${readerClause ?? sql``} - ORDER BY (${distance}) + 0, ${embeddingSearch.id} - LIMIT ${limit}`) -} const captured: CapturedQuery[] = [] const report: Record = { fixture: ids, @@ -170,7 +125,7 @@ const report: Record = { vectors: 'Normalized 512-dimensional topic/noise geometry with permuted copies across 1536 dimensions; verifies prefix candidate ranking, not semantic embedding quality', cache: evictSharedBuffers - ? 'Selected workspace and organization samples evict PostgreSQL shared buffers; operating-system cache is not cleared' + ? 'Selected workspace samples evict PostgreSQL shared buffers; operating-system cache is not cleared' : 'First and repeated samples; no claim of a cold operating-system cache', layout: reused ? 'Reused fixture; physical layout is inherited from its original report' @@ -283,19 +238,6 @@ function assertIndexedCandidates( expect(traversed['Actual Rows']).toBeLessThanOrEqual(candidateLimit) } -/** Small scopes must seek chunk metadata by document without reading the full vector projection. */ -function assertIndexedChunkProbe(node: ExplainNode): number { - let lookups = 0 - if (node['Relation Name'] === 'embedding_search') { - expect(['Index Scan', 'Index Only Scan']).toContain(node['Node Type']) - expect(node['Index Name']).toBe('embedding_search_document_lookup_idx') - expect((node.Output ?? []).join(' ')).not.toMatch(/(?:embedding_search\.)?(?:vector|binary)/) - lookups = node['Actual Loops'] - } - for (const child of node.Plans ?? []) lookups += assertIndexedChunkProbe(child) - return lookups -} - /** Keyword sort memory must scale with identities and scores, not the matched document text. */ function assertScalarKeywordSorts(node: ExplainNode) { if (node['Node Type'] === 'Sort') { @@ -310,7 +252,7 @@ function saveReport() { } /** Only the disposable fixture may evict shared buffers; the operating-system cache stays intact. */ -async function prepareOrganizationSample(label: string) { +async function prepareSample(label: string) { if (!evictSharedBuffers) return const [eviction] = await db.execute<{ buffers: number; evicted: number }>(sql` WITH cached AS MATERIALIZED ( @@ -354,6 +296,9 @@ const diagnosticSchema = z const resultSchema = z.object({ success: z.literal(true), data: z.object({ + retrieval: z + .object({ status: z.enum(['complete', 'partial']), timedOutLegs: z.array(z.string()) }) + .optional(), results: z.array( z.object({ documentId: z.string(), @@ -368,63 +313,22 @@ const resultSchema = z.object({ async function search( userId = ids.aliceId, query = 'Orion deployment', - filters: WorkspaceSearchFilters = {}, - organizationScope = false, - topK = 15 + filters: WorkspaceSearchFilters = {} ) { - return resultSchema.parse( - await searchWorkspaceServerTool.execute( - { query, topK, ...filters }, - { - userId, - ...(organizationScope - ? { organizationId: ids.organizationId, chatId: organizationChatId } - : { workspaceId: ids.workspaceId }), - requestMode: 'assistant', - toolCallId: generateId(), - copilotToolExecution: true, - resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry([], { - userId, - ...(organizationScope ? {} : { workspaceId: ids.workspaceId }), - }), - } - ) - ) -} - -async function searchDashboard( - query = 'Orion deployment', - userId = ids.aliceId, - organizationScope = false, - topK = 15 -) { - const authenticate = vi.spyOn(internalSessionAuth, 'authenticate').mockResolvedValue({ - kind: 'session', - userId, - sessionId: 'fixture-dashboard', + const data = await searchKnowledge.execute({ + principal: { kind: 'session', userId, sessionId: 'fixture-knowledge-search' }, + input: { + workspaceId: ids.workspaceId, + knowledgeBaseIds: [ids.knowledgeBaseId], + query, + topK: 15, + filters, + searchMode: 'hybrid', + allowPartialResults: true, + surface: 'api', + }, }) - try { - const response = await searchRoute( - new NextRequest('http://localhost/api/knowledge/search', { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ - ...(organizationScope - ? { organizationId: ids.organizationId } - : { workspaceId: ids.workspaceId }), - query, - topK, - }), - }) - ) - expect(response.status).toBe(200) - return { - success: true as const, - data: workspaceKnowledgeSearchDataSchema.parse((await response.json()).data), - } - } finally { - authenticate.mockRestore() - } + return resultSchema.parse({ success: true, data }) } async function searchWorkspaceKb( @@ -489,15 +393,6 @@ async function sample( const diagnostics = diagnosticSchema.parse(completed[0][1]) expect(diagnostics.stages.embedding.count).toBe(1) expect(diagnostics.stages.retrieval.count).toBe(1) - if (diagnostics.surface === 'copilot') { - const passageBytes = result.data.results.map((row) => Buffer.byteLength(row.content)) - expect(diagnostics.passageBytes).toBe(passageBytes.reduce((total, bytes) => total + bytes, 0)) - expect(diagnostics.maxPassageBytes).toBe(Math.max(0, ...passageBytes)) - expect(diagnostics.uniqueDocumentCount).toBe( - new Set(result.data.results.map((row) => row.documentId)).size - ) - expect(diagnostics.toolResultBytes).toBeGreaterThan(diagnostics.passageBytes!) - } expect(captured.length).toBeLessThan(300) const searches = captured.filter( (item) => @@ -685,6 +580,7 @@ describe.skipIf(!enabled)('Knowledge search latency on a realistic indexed corpu workspaceId: ids.workspaceId, organizationId: null, embeddingModel: 'text-embedding-3-small', + isSearchIndex: false, }) .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) await db @@ -693,17 +589,6 @@ describe.skipIf(!enabled)('Knowledge search latency on a realistic indexed corpu .where(eq(knowledgeConnector.id, ids.connectorId)) await db.delete(credential).where(eq(credential.workspaceId, ids.workspaceId)) await db.delete(credentialGroup).where(eq(credentialGroup.workspaceId, ids.workspaceId)) - await db - .delete(copilotChats) - .where( - and( - eq(copilotChats.organizationId, ids.organizationId), - eq(copilotChats.userId, ids.aliceId) - ) - ) - await db - .delete(member) - .where(and(eq(member.organizationId, ids.organizationId), eq(member.userId, ids.aliceId))) await db.execute( sql`UPDATE document SET acl = ARRAY[${`u:${ids.aliceId}@fixture.test`}], user_excluded = false, acl_verified_at = statement_timestamp() WHERE knowledge_base_id = ${ids.knowledgeBaseId}` ) @@ -715,10 +600,6 @@ describe.skipIf(!enabled)('Knowledge search latency on a realistic indexed corpu .update(knowledgeBase) .set({ embeddingModel: 'text-embedding-3-small' }) .where(inArray(knowledgeBase.id, [ids.knowledgeBaseId, unrelated.knowledgeBaseId])) - await db - .update(knowledgeBase) - .set({ isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) const indexes = await db.execute<{ indexname: string indexdef: string @@ -904,7 +785,7 @@ describe.skipIf(!enabled)('Knowledge search latency on a realistic indexed corpu }) it.each(['vector', 'keyword', 'both'] as const)( - 'keeps the Assistant budget when %s SQL branches are delayed', + 'keeps knowledge-search deadlines when %s SQL branches are delayed', async (delayedLegs) => { diagnosticLog?.mockClear() let vectorDelayed = false @@ -927,19 +808,7 @@ describe.skipIf(!enabled)('Knowledge search latency on a realistic indexed corpu }) as Promise }) try { - const result = await searchWorkspaceServerTool.execute( - { query: 'Orion deployment', topK: 15 }, - { - userId: ids.aliceId, - workspaceId: ids.workspaceId, - toolCallId: generateId(), - copilotToolExecution: true, - resolvedSecretTraceRegistry: new ResolvedSecretTraceRegistry([], { - userId: ids.aliceId, - workspaceId: ids.workspaceId, - }), - } - ) + const result = await search() const completed = diagnosticLog?.mock.calls.find( ([message]) => message === 'Knowledge search completed' ) @@ -971,52 +840,7 @@ describe.skipIf(!enabled)('Knowledge search latency on a realistic indexed corpu expect( parsed.data.results.every((row) => row.knowledgeBaseId === ids.knowledgeBaseId) ).toBe(true) - report[`assistant.deadline.${delayedLegs}`] = { resultCount: parsed.data.results.length } - } finally { - delayed.mockRestore() - } - }, - 30_000 - ) - - it.each(['vector', 'both'] as const)( - 'returns incomplete dashboard coverage when %s SQL branches exceed their deadline', - async (delayedLegs) => { - diagnosticLog?.mockClear() - const query = SearchBudget.prototype.query - const delayed = vi.spyOn(SearchBudget.prototype, 'query').mockImplementation(function ( - this: SearchBudget, - stage: SearchStage, - run: (executor: SearchExecutor) => PromiseLike - ): Promise { - return query.call(this, stage, async (tx) => { - if (delayedLegs === 'both' || this.leg === 'vector') - await tx.execute(sql`SELECT pg_sleep(${delayedLegs === 'both' ? 9 : 4})`) - return run(tx) - }) as Promise - }) - try { - const { data } = await searchDashboard() - const completed = diagnosticLog?.mock.calls.find( - ([message]) => message === 'Knowledge search completed' - ) - const diagnostics = diagnosticSchema.parse(completed?.[1]) - expect(diagnostics.outcome).toBe('partial') - expect(diagnostics.vectorBudgetMs).toBe(3000) - expect(diagnostics.stages.vector.totalMs).toBeGreaterThan(2500) - expect(diagnostics.stages.vector.totalMs).toBeLessThan(4000) - expect(data.retrieval).toEqual({ - status: 'partial', - timedOutLegs: delayedLegs === 'both' ? ['vector', 'keyword'] : ['vector'], - }) - if (delayedLegs === 'both') expect(data.results).toEqual([]) - else { - expect(data.results.length).toBeGreaterThan(0) - expect( - data.results.every((result) => result.knowledgeBaseId === ids.knowledgeBaseId) - ).toBe(true) - } - report[`dashboard.deadline.${delayedLegs}`] = { resultCount: data.results.length } + report[`knowledge.deadline.${delayedLegs}`] = { resultCount: parsed.data.results.length } } finally { delayed.mockRestore() } @@ -1024,216 +848,11 @@ describe.skipIf(!enabled)('Knowledge search latency on a realistic indexed corpu 30_000 ) - it('records first and repeated application searches with the actual SQL plans', async () => { - const before = embeddingCalls - for (let iteration = 0; iteration < 2; iteration++) { - const { result, plans, diagnostics } = await sample(`broad.${iteration}`, () => search()) - expectCompleteVectorSearch(diagnostics) - expect(result.data.results).toHaveLength(15) - expect(result.data.results.every((row) => row.knowledgeBaseId === ids.knowledgeBaseId)).toBe( - true - ) - expect(plans.length).toBeGreaterThanOrEqual(2) - const vectorPlans = plans.filter((plan) => plan.kind === 'vector') - expect(vectorPlans).toHaveLength(1) - expect(vectorPlans[0].plan[0].Plan['Actual Rows']).toBeGreaterThan(0) - assertCompactCandidates(vectorPlans[0].plan[0].Plan) - expect(plans.some((plan) => plan.kind === 'page')).toBe(true) - const page = plans.find((plan) => plan.kind === 'page')! - const actual = await db.$client.unsafe(page.query, page.parameters).values() - const expected = await exactProjectionNeighbors(queryVector, actual.length) - const expectedIds = new Set(expected.map(({ id }) => id)) - const recall = actual.filter(([id]) => expectedIds.has(id)).length / expected.length - expect(recall).toBeGreaterThanOrEqual(0.95) - report[`recall.${iteration}`] = { neighbors: expected.length, recall } - saveReport() - } - expect(embeddingCalls - before).toBe(2) - }, 180_000) - - it('preserves exact-neighbor recall across different query vectors', async () => { - for (const topic of [3, 11, 23]) { - const { plans, diagnostics } = await sample(`topic.${topic}`, () => - search(ids.aliceId, `Topic ${topic} deployment`) - ) - expectCompleteVectorSearch(diagnostics) - const candidates = plans.find((plan) => plan.kind === 'vector')! - assertCompactCandidates(candidates.plan[0].Plan) - const page = plans.find((plan) => plan.kind === 'page')! - const actual = await db.$client.unsafe(page.query, page.parameters).values() - const expected = await exactProjectionNeighbors(topicVector(topic), actual.length) - const expectedIds = new Set(expected.map(({ id }) => id)) - const recall = actual.filter(([id]) => expectedIds.has(id)).length / expected.length - expect(recall).toBeGreaterThanOrEqual(0.95) - report[`recall.topic.${topic}`] = { neighbors: expected.length, recall } - saveReport() - } - }, 180_000) - - it('compares the Search tab and Assistant with the same person, query and index', async () => { - const dashboard = await sample('dashboard', searchDashboard) - const assistant = await sample('assistant.comparison', () => search()) - expectCompleteVectorSearch(dashboard.diagnostics) - expectCompleteVectorSearch(assistant.diagnostics) - expect(dashboard.diagnostics.surface).toBe('dashboard') - expect(assistant.diagnostics.surface).toBe('copilot') - expect(dashboard.diagnostics.stages.result_provenance).toBeUndefined() - expect(assistant.diagnostics.stages.result_provenance.count).toBe(1) - expect(dashboard.result.data.results).toHaveLength(15) - expect(assistant.result.data.results).toHaveLength(15) - const dashboardVector = dashboard.plans.filter((plan) => plan.kind === 'vector') - const assistantVector = assistant.plans.filter((plan) => plan.kind === 'vector') - expect(dashboardVector).toHaveLength(1) - expect(assistantVector).toHaveLength(1) - expect(dashboardVector[0].query).toBe(assistantVector[0].query) - expect(dashboardVector[0].parameters).toEqual(assistantVector[0].parameters) - expect(dashboardVector[0].plan[0].Plan['Actual Rows']).toBeGreaterThan(0) - }, 180_000) - it('keeps inaccessible content out of an otherwise identical search', async () => { const { result } = await sample('denied', () => search(ids.bobId)) expect(result.data.results).toEqual([]) }, 180_000) - it('preserves recall when the nearest topic is mostly inaccessible within a broad permission scope', async () => { - const reader = `u:${ids.aliceId}@fixture.test` - /** Four consecutive topics share each document; hide 99% of the query's topic cluster. */ - await db.execute(sql`UPDATE document - SET acl = ARRAY[${`u:${ids.bobId}@fixture.test`}] - WHERE knowledge_base_id = ${ids.knowledgeBaseId} - AND external_id::int % 8 = 0 AND external_id::int % 800 <> 0`) - try { - for (const surface of ['copilot', 'dashboard'] as const) { - const { result, plans, diagnostics } = await sample( - `filtered-neighborhood.${surface}`, - surface === 'copilot' ? () => search() : searchDashboard - ) - expectCompleteVectorSearch(diagnostics) - expect(result.data.results).toHaveLength(15) - const page = plans.find((plan) => plan.kind === 'page')! - expect(page).toBeDefined() - const actual = await db.$client.unsafe(page.query, page.parameters).values() - const expected = await exactProjectionNeighbors( - queryVector, - actual.length, - sql`AND d.acl @> ARRAY[${reader}]::text[]` - ) - expect(expected.length).toBeGreaterThan(0) - const expectedIds = new Set(expected.map(({ id }) => id)) - const recall = actual.filter(([id]) => expectedIds.has(id)).length / expected.length - expect(recall).toBeGreaterThanOrEqual(0.95) - report[`recall.filtered-neighborhood.${surface}`] = { neighbors: expected.length, recall } - saveReport() - } - } finally { - await db - .update(document) - .set({ acl: [reader] }) - .where(eq(document.knowledgeBaseId, ids.knowledgeBaseId)) - } - }, 180_000) - - it('ranks a small permission scope by its bounded IDs without a corpus-wide vector probe', async () => { - const lastTopicDocument = Math.floor((chunkCount / chunksPerDocument - 1) / 8) * 8 - const documentIds = [0, 8, 16].map( - (offset) => `${ids.workspaceId}-doc-${lastTopicDocument - offset}` - ) - await db - .update(document) - .set({ acl: [`u:${ids.aliceId}@fixture.test`, `u:${ids.bobId}@fixture.test`] }) - .where(inArray(document.id, documentIds)) - try { - for (const surface of ['copilot', 'dashboard'] as const) { - const { result, plans, diagnostics } = await sample(`small-scope.${surface}`, () => - surface === 'copilot' ? search(ids.bobId) : searchDashboard('Orion deployment', ids.bobId) - ) - expectCompleteVectorSearch(diagnostics) - expect(result.data.results.length).toBeGreaterThan(0) - expect(result.data.results.every((row) => documentIds.includes(row.documentId))).toBe(true) - const probe = plans.filter((plan) => plan.kind === 'probe') - expect(probe).toHaveLength(1) - expect(probe[0].query).not.toContain('<=>') - expect(probe[0].plan[0].Plan['Actual Rows']).toBe(12) - expect(assertIndexedChunkProbe(probe[0].plan[0].Plan)).toBe(documentIds.length) - /** The page reads the bounded ranking's identities from the projection, never the original vectors. */ - const page = plans.filter((plan) => plan.kind === 'page') - expect(page).toHaveLength(1) - expect(page[0].query).not.toContain('"embedding"."embedding"') - } - } finally { - await db - .update(document) - .set({ acl: [`u:${ids.aliceId}@fixture.test`] }) - .where(inArray(document.id, documentIds)) - } - }, 180_000) - - it.each([200, 1000, 1596, 1600, 2000])( - 'keeps a selective scope of %s chunks within both retrieval budgets', - async (count) => { - const documentCount = count / chunksPerDocument - const documentIds = Array.from( - { length: documentCount }, - (_, index) => `${ids.workspaceId}-doc-${chunkCount / chunksPerDocument - 1 - index}` - ) - await db - .update(document) - .set({ acl: [`u:${ids.aliceId}@fixture.test`, `u:${ids.bobId}@fixture.test`] }) - .where(inArray(document.id, documentIds)) - try { - for (const surface of ['copilot', 'dashboard'] as const) { - const { result, plans, diagnostics } = await sample( - `selective-${count}.${surface}`, - () => - surface === 'copilot' - ? search(ids.bobId) - : searchDashboard('Orion deployment', ids.bobId) - ) - expectCompleteVectorSearch(diagnostics) - expect(result.data.results.length).toBeGreaterThan(0) - expect(result.data.results.every((row) => documentIds.includes(row.documentId))).toBe( - true - ) - const probe = plans.find((plan) => plan.kind === 'probe')! - expect(probe.plan[0].Plan['Actual Rows']).toBe(Math.min(count, HYBRID_CANDIDATE_LIMIT)) - expect(assertIndexedChunkProbe(probe.plan[0].Plan)).toBe( - Math.min(documentCount, HYBRID_CANDIDATE_LIMIT / chunksPerDocument) - ) - expect(plans.filter((plan) => plan.kind === 'vector')).toHaveLength( - count < HYBRID_CANDIDATE_LIMIT ? 0 : 1 - ) - if (count > HYBRID_CANDIDATE_LIMIT) { - const page = plans.find((plan) => plan.kind === 'page')! - const actual = await db.$client.unsafe(page.query, page.parameters).values() - const expected = await exactProjectionNeighbors( - queryVector, - actual.length, - sql`AND ${embeddingSearch.documentId} IN (${sql.join( - documentIds.map((id) => sql`${id}`), - sql`, ` - )})` - ) - const expectedIds = new Set(expected.map(({ id }) => id)) - const recall = actual.filter(([id]) => expectedIds.has(id)).length / expected.length - expect(recall).toBeGreaterThanOrEqual(0.95) - report[`recall.selective-${count}.${surface}`] = { - neighbors: expected.length, - recall, - candidateScan: diagnostics.vectorCandidateScan, - } - saveReport() - } - } - } finally { - await db - .update(document) - .set({ acl: [`u:${ids.aliceId}@fixture.test`] }) - .where(inArray(document.id, documentIds)) - } - }, - 180_000 - ) - it('bounds member-observation searches and rejects suspended readers with current ACL checks', async () => { const fixture = await seedKnowledgeMemberFixture(ids) const [alice, bob] = fixture.members @@ -1263,28 +882,13 @@ describe.skipIf(!enabled)('Knowledge search latency on a realistic indexed corpu .where(inArray(document.id, documentIds)) await db.execute(sql`ANALYZE document`) await db.execute(sql`ANALYZE knowledge_document_observation`) - for (const surface of ['copilot', 'dashboard'] as const) { - const broad = await sample(`member-broad.${surface}`, () => - surface === 'copilot' ? search() : searchDashboard() - ) - expectCompleteVectorSearch(broad.diagnostics) - expect(broad.result.data.results).toHaveLength(15) - const broadProbe = broad.plans.find((plan) => plan.kind === 'probe')! - expect(broadProbe.plan[0].Plan['Actual Rows']).toBe(HYBRID_CANDIDATE_LIMIT) - expect(assertIndexedChunkProbe(broadProbe.plan[0].Plan)).toBe( - HYBRID_CANDIDATE_LIMIT / chunksPerDocument - ) - const { result, plans, diagnostics } = await sample(`member-scope.${surface}`, () => - surface === 'copilot' ? search(ids.bobId) : searchDashboard('Orion deployment', ids.bobId) - ) - expectCompleteVectorSearch(diagnostics) - expect(result.data.results.length).toBeGreaterThan(0) - expect(result.data.results.every((row) => documentIds.includes(row.documentId))).toBe(true) - const probe = plans.find((plan) => plan.kind === 'probe')! - expect(probe).toBeDefined() - expect(probe.query).toContain('knowledge_document_observation') - expect(assertIndexedChunkProbe(probe.plan[0].Plan)).toBe(documentIds.length) - } + const broad = await sample('member-broad', () => search()) + expectCompleteVectorSearch(broad.diagnostics) + expect(broad.result.data.results).toHaveLength(15) + const { result, diagnostics } = await sample('member-scope', () => search(ids.bobId)) + expectCompleteVectorSearch(diagnostics) + expect(result.data.results.length).toBeGreaterThan(0) + expect(result.data.results.every((row) => documentIds.includes(row.documentId))).toBe(true) await db .update(knowledgeConnectorMember) .set({ status: 'suspended' }) @@ -1328,21 +932,15 @@ describe.skipIf(!enabled)('Knowledge search latency on a realistic indexed corpu } }, 180_000) - it.each(['copilot', 'dashboard'] as const)( - 'keeps %s searches for common keyword terms complete', - async (surface) => { - const { result, diagnostics } = await sample(`common-keyword.${surface}`, () => - surface === 'dashboard' - ? searchDashboard('Engineering operations') - : search(ids.aliceId, 'Engineering operations') - ) - expectCompleteVectorSearch(diagnostics) - expect(result.data.results).toHaveLength(15) - }, - 180_000 - ) + it('keeps common-keyword searches complete', async () => { + const { result, diagnostics } = await sample('common-keyword', () => + search(ids.aliceId, 'Engineering operations') + ) + expectCompleteVectorSearch(diagnostics) + expect(result.data.results).toHaveLength(15) + }, 180_000) - it('runs two independent Assistant searches concurrently', async () => { + it('runs two independent knowledge-base searches concurrently', async () => { diagnosticLog?.mockClear() const start = performance.now() const results = await Promise.all([search(), search(ids.aliceId, 'Topic 11 deployment')]) @@ -1374,7 +972,7 @@ describe.skipIf(!enabled)('Knowledge search latency on a realistic indexed corpu WHERE knowledge_base_id = ${ids.knowledgeBaseId}`) for (const topic of [0, 11, 23]) { const label = `workspace-kb.topic.${topic}` - await prepareOrganizationSample(label) + await prepareSample(label) const { result, plans, diagnostics } = await sample(label, () => searchWorkspaceKb(`Topic ${topic} deployment`) ) @@ -1441,7 +1039,7 @@ describe.skipIf(!enabled)('Knowledge search latency on a realistic indexed corpu for (const concurrency of [2, 8]) { const label = `workspace-kb.concurrent.${concurrency}` - await prepareOrganizationSample(label) + await prepareSample(label) diagnosticLog?.mockClear() const started = performance.now() const results = await Promise.all( @@ -1531,285 +1129,4 @@ describe.skipIf(!enabled)('Knowledge search latency on a realistic indexed corpu const restored = await sample('live.restored', () => search()) expect(restored.result.data.results).toHaveLength(15) }, 180_000) - - it('keeps organization searches complete with stale ACL estimates and concurrent requests', async () => { - await db.insert(member).values({ - id: generateId(), - organizationId: ids.organizationId, - userId: ids.aliceId, - role: 'owner', - }) - await db.insert(copilotChats).values({ - id: organizationChatId, - organizationId: ids.organizationId, - userId: ids.aliceId, - type: 'mothership', - }) - await db - .update(knowledgeBase) - .set({ workspaceId: null, organizationId: ids.organizationId }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) - await db - .update(knowledgeConnector) - .set({ connectorType: 'google_drive', credentialId: null, sourceConfig: {} }) - .where(eq(knowledgeConnector.id, ids.connectorId)) - /** Keep the deliberate tenfold visibility underestimate until the measured requests finish. */ - await db.execute(sql`ALTER TABLE document SET (autovacuum_enabled = false)`) - try { - await db.execute(sql`UPDATE document - SET acl = ARRAY[CASE WHEN external_id::int % 10 = 0 - THEN ${`u:${ids.aliceId}@fixture.test`} ELSE ${`u:${ids.bobId}@fixture.test`} END] - WHERE knowledge_base_id = ${ids.knowledgeBaseId}`) - await db.execute(sql`ANALYZE document`) - await db.execute(sql`UPDATE document SET acl = ARRAY[${`u:${ids.aliceId}@fixture.test`}] - WHERE knowledge_base_id = ${ids.knowledgeBaseId}`) - report.organizationVisibility = { - analyzedVisibleDocuments: chunkCount / chunksPerDocument / 10, - actualVisibleDocuments: chunkCount / chunksPerDocument, - unrelatedChunks: unrelatedChunkCount, - } - /** Capture latency samples before EXPLAIN ANALYZE can warm the candidate paths. */ - for (const surface of ['dashboard', 'copilot'] as const) { - const label = `organization.${surface}` - await prepareOrganizationSample(label) - const { result, diagnostics } = await sample( - label, - () => - surface === 'dashboard' - ? searchDashboard('Orion deployment', ids.aliceId, true, 20) - : search(ids.aliceId, 'Orion deployment', {}, true, 20), - { explain: false } - ) - expectCompleteVectorSearch(diagnostics) - expect(result.data.results).toHaveLength(20) - expect( - result.data.results.every((row) => row.knowledgeBaseId === ids.knowledgeBaseId) - ).toBe(true) - } - await prepareOrganizationSample('organization.concurrent') - diagnosticLog?.mockClear() - const started = performance.now() - const results = await Promise.all([ - search(ids.aliceId, 'Orion deployment', {}, true, 20), - search(ids.aliceId, 'Topic 11 deployment', {}, true, 20), - ]) - const diagnostics = diagnosticLog!.mock.calls - .filter(([message]) => message === 'Knowledge search completed') - .map(([, metadata]) => diagnosticSchema.parse(metadata)) - report['organization.concurrent'] = { - milliseconds: performance.now() - started, - resultCounts: results.map((result) => result.data.results.length), - diagnostics, - } - saveReport() - expect(diagnostics).toHaveLength(2) - for (const item of diagnostics) expectCompleteVectorSearch(item) - for (const result of results) { - expect(result.data.results).toHaveLength(20) - expect( - result.data.results.every((row) => row.knowledgeBaseId === ids.knowledgeBaseId) - ).toBe(true) - } - const planned = await sample('organization.plans', () => - search(ids.aliceId, 'Orion deployment', {}, true, 20) - ) - expectCompleteVectorSearch(planned.diagnostics) - expect(planned.plans.filter((plan) => plan.kind === 'vector')).toHaveLength(1) - } finally { - await db.execute(sql`ALTER TABLE document RESET (autovacuum_enabled)`) - await db.execute(sql`ANALYZE document`) - } - }, 180_000) - /** Opt in with local Sim and Go URLs; uses the real configured provider, billing adapter, and async resume protocol. */ - it.skipIf(!process.env.KNOWLEDGE_SEARCH_ASSISTANT_URL)( - 'recovers quietly from incomplete search through local Go Assistant, then reads and cites evidence', - async () => { - const assistantUrl = new URL(process.env.KNOWLEDGE_SEARCH_ASSISTANT_URL!) - const simUrl = new URL(process.env.KNOWLEDGE_SEARCH_SIM_URL!) - for (const url of [assistantUrl, simUrl]) { - if (!['127.0.0.1', 'localhost'].includes(url.hostname)) - throw new Error( - 'Assistant integration requires local servers using the disposable test databases' - ) - } - const apiKey = process.env.KNOWLEDGE_SEARCH_ASSISTANT_API_KEY! - const internalKey = process.env.KNOWLEDGE_SEARCH_SIM_INTERNAL_KEY! - expect(apiKey).toBeTruthy() - expect(internalKey).toBeTruthy() - const chunkId = `${ids.workspaceId}-chunk-0` - const [original] = await db - .select({ - content: embedding.content, - contentLength: embedding.contentLength, - tokenCount: embedding.tokenCount, - }) - .from(embedding) - .where(eq(embedding.id, chunkId)) - .limit(1) - const longContent = - 'Orion deployment guide. The activation phrase and final checksum appear at the end.\n' + - 'Review the deployment stages in order. Preserve the rollback procedure.\n'.repeat(320) + - '\nActivation phrase: SILVER COMET\nFinal checksum: K7M2-84\n' - const registry = new ResolvedSecretTraceRegistry([], { userId: ids.aliceId }) - const calls: Array<{ - name: string - arguments: unknown - milliseconds: number - bytes: number - }> = [] - let answer = '' - let incompleteSearch = true - const query = SearchBudget.prototype.query - const delayed = vi.spyOn(SearchBudget.prototype, 'query').mockImplementation(function ( - this: SearchBudget, - stage: SearchStage, - run: (executor: SearchExecutor) => PromiseLike - ): Promise { - return query.call(this, stage, async (tx) => { - if (incompleteSearch) await tx.execute(sql`SELECT pg_sleep(9)`) - return run(tx) - }) as Promise - }) - const started = performance.now() - try { - await db - .update(embedding) - .set({ - content: longContent, - contentLength: longContent.length, - tokenCount: Math.ceil(longContent.length / 4), - }) - .where(eq(embedding.id, chunkId)) - const admission = await externalFetch(new URL('/api/copilot/api-keys/validate', simUrl), { - method: 'POST', - headers: { - 'content-type': 'application/json', - 'x-api-key': internalKey, - 'x-sim-billing-protocol': 'legacy-v0', - }, - body: JSON.stringify({ - userId: ids.aliceId, - organizationId: ids.organizationId, - chatId: organizationChatId, - }), - }) - expect(admission.status, await admission.text()).toBe(200) - let path = '/api/mothership' - let body: Record = { - message: - 'Search for the Orion deployment guide that mentions an activation phrase and final checksum. Read enough of that document to report both values and cite it.', - version: '3.0.0', - mode: 'assistant', - userId: ids.aliceId, - organizationId: ids.organizationId, - chatId: organizationChatId, - } - for (let round = 0; round < 8; round++) { - const response = await externalFetch(new URL(path, assistantUrl), { - method: 'POST', - headers: { - 'content-type': 'application/json', - 'x-api-key': apiKey, - 'x-sim-billing-protocol': 'legacy-v0', - }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(120000), - }) - const wire = await response.text() - expect(response.status, wire.slice(0, 2000)).toBe(200) - expect(Buffer.byteLength(wire)).toBeLessThan(2 * 1024 * 1024) - const pending = new Map() - let checkpoint: MothershipStreamV1CheckpointPausePayload | undefined - let streamId = '' - for (const line of wire.split('\n')) { - if (!line.startsWith('data:')) continue - const raw = line.slice(5).trim() - if (!raw || raw === '[DONE]') continue - const event: unknown = JSON.parse(raw) - if (!isContractStreamEventEnvelope(event)) - throw new Error('Assistant returned an invalid generated stream envelope') - streamId = event.stream.streamId - if (event.type === 'error') throw new Error(JSON.stringify(event.payload)) - if (event.type === 'text') answer += event.payload.text - if ( - event.type === 'tool' && - event.payload.phase === 'call' && - !event.payload.partial && - event.payload.arguments - ) - pending.set(event.payload.toolCallId, event.payload) - if (event.type === 'run' && event.payload.kind === 'checkpoint_pause') - checkpoint = event.payload - } - if (!checkpoint) break - const results = await Promise.all( - checkpoint.pendingToolCallIds.map(async (callId) => { - const call = pending.get(callId) - if (!call) throw new Error('Checkpoint referenced an absent tool call') - const tool = - call.toolName === 'search_workspace' - ? searchWorkspaceServerTool - : call.toolName === 'read_document' - ? readDocumentServerTool - : undefined - if (!tool) throw new Error(`Unexpected Assistant tool: ${call.toolName}`) - const toolStarted = performance.now() - const result = await tool.execute(call.arguments, { - userId: ids.aliceId, - organizationId: ids.organizationId, - chatId: organizationChatId, - toolCallId: callId, - copilotToolExecution: true, - requestMode: 'assistant', - resolvedSecretTraceRegistry: registry, - }) - calls.push({ - name: call.toolName, - arguments: call.arguments, - milliseconds: performance.now() - toolStarted, - bytes: Buffer.byteLength(JSON.stringify(result)), - }) - const { success } = z.object({ success: z.boolean() }).parse(result) - expect(success).toBe(true) - if (incompleteSearch) { - expect(call.toolName).toBe('search_workspace') - expect(result).toMatchObject({ - data: { - retrieval: { status: 'partial', timedOutLegs: ['vector', 'keyword'] }, - results: [], - }, - }) - } - return { callId, name: call.toolName, success, data: result } - }) - ) - incompleteSearch = false - path = '/api/tools/resume' - body = { - checkpointId: checkpoint.checkpointId, - streamId, - userId: ids.aliceId, - organizationId: ids.organizationId, - chatId: organizationChatId, - results, - } - report['assistant.live.progress'] = { rounds: round + 1, calls } - saveReport() - } - report['assistant.live'] = { milliseconds: performance.now() - started, calls, answer } - saveReport() - expect(answer).toContain('SILVER COMET') - expect(answer).toContain('K7M2-84') - expect(answer).toContain('') - expect(answer).not.toMatch(/timed?\s*out|timeout|internal retr(?:y|ies)/i) - expect(calls.some((call) => call.name === 'read_document')).toBe(true) - expect(calls.filter((call) => call.name === 'search_workspace').length).toBeGreaterThan(1) - expect(calls.every((call) => call.bytes < 40000)).toBe(true) - } finally { - delayed.mockRestore() - await db.update(embedding).set(original).where(eq(embedding.id, chunkId)) - } - }, - 10 * 60_000 - ) }) diff --git a/apps/sim/lib/knowledge/__integration__/search-source-pagination.integration.ts b/apps/sim/lib/knowledge/__integration__/search-source-pagination.integration.ts index 4bc7a62c3f6..42e7b124f87 100644 --- a/apps/sim/lib/knowledge/__integration__/search-source-pagination.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-source-pagination.integration.ts @@ -1,7 +1,5 @@ import { db } from '@sim/db' import { - document, - embedding, knowledgeBase, knowledgeConnector, member, @@ -17,19 +15,14 @@ import { createKnowledgeAclFixtureIds, seedKnowledgeAclFixture, } from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { readSearchSourceOverview } from '@/lib/knowledge/application/search-source-overview' -import { readSearchSourceProgress } from '@/lib/knowledge/application/search-source-progress' import { listSearchSources } from '@/lib/knowledge/application/search-sources' const ids = createKnowledgeAclFixtureIds() const alice = { kind: 'session' as const, userId: ids.aliceId, sessionId: 'fixture-alice' } -const bob = { kind: 'session' as const, userId: ids.bobId, sessionId: 'fixture-bob' } const sourceIds = Array.from({ length: 105 }, () => generateId()) .sort() .reverse() const olderSourceId = generateId() -const documentId = generateId() -const embeddingId = generateId() const input = { workspaceId: ids.workspaceId } beforeAll(async () => { @@ -65,33 +58,6 @@ beforeAll(async () => { status: 'active', createdAt: new Date('2025-12-31T00:00:00Z'), }) - await db.insert(document).values({ - id: documentId, - connectorId: olderSourceId, - knowledgeBaseId: ids.knowledgeBaseId, - externalId: 'fixture', - filename: 'readable.txt', - fileUrl: 'https://fixture.test/readable', - fileSize: 12, - mimeType: 'text/plain', - processingStatus: 'completed', - acl: [`u:${ids.aliceId}@fixture.test`], - aclVerifiedAt: new Date(), - }) - await db.insert(embedding).values({ - id: embeddingId, - documentId, - knowledgeBaseId: ids.knowledgeBaseId, - chunkIndex: 0, - chunkHash: 'fixture-hash', - content: 'Fixture text', - contentLength: 12, - tokenCount: 3, - startOffset: 0, - endOffset: 12, - embeddingModel: 'text-embedding-3-small', - embedding: [1, ...Array(1535).fill(0)], - }) }) afterAll(async () => { @@ -101,7 +67,7 @@ afterAll(async () => { await db.delete(user).where(eq(user.id, ids.bobId)) }) -describe('bounded source pagination and provider overview', () => { +describe('bounded live Search source configuration pagination', () => { it('finds a provider beyond the unfiltered candidate bound on its first filtered page', async () => { const result = await listSearchSources.execute({ principal: alice, @@ -141,31 +107,6 @@ describe('bounded source pagination and provider overview', () => { expect(second.sources.map((source) => source.connectorId)).toEqual([olderSourceId]) expect(second.nextCursor).toBeNull() }) - it('includes providers beyond the loaded page and requires viewer-readable indexed content', async () => { - const first = await listSearchSources.execute({ principal: alice, input }) - expect(first.sources.every((source) => source.connectorType === 'confluence')).toBe(true) - const [aliceOverview, bobOverview] = await Promise.all( - [alice, bob].map((principal) => readSearchSourceOverview.execute({ principal, input })) - ) - expect(aliceOverview.providers).toEqual( - expect.arrayContaining([{ connectorType: 'google_drive', isSyncing: false }]) - ) - expect(aliceOverview.hasSearchableDocuments).toBe(true) - expect(bobOverview.hasSearchableDocuments).toBe(false) - }) - it('keeps paused but readable sources complete and ignores disabled chunks', async () => { - await db - .update(knowledgeConnector) - .set({ status: 'paused' }) - .where(eq(knowledgeConnector.id, olderSourceId)) - const paused = await readSearchSourceOverview.execute({ principal: alice, input }) - expect(paused.hasSearchableDocuments).toBe(true) - expect(paused.providers.every((provider) => !provider.isSyncing)).toBe(true) - await db.update(embedding).set({ enabled: false }).where(eq(embedding.id, embeddingId)) - expect( - (await readSearchSourceOverview.execute({ principal: alice, input })).hasSearchableDocuments - ).toBe(false) - }) it('shows a newly created source on the first-page refresh so enrollment can observe completion', async () => { const before = await listSearchSources.execute({ principal: alice, input }) expect(before.sources[0].connectorId).toBe(sourceIds[0]) @@ -179,7 +120,7 @@ describe('bounded source pagination and provider overview', () => { status: 'pending', }) const refreshed = await listSearchSources.execute({ principal: alice, input }) - expect(refreshed.sources[0]).toMatchObject({ connectorId: newSourceId, isSyncing: true }) + expect(refreshed.sources[0]).toMatchObject({ connectorId: newSourceId }) expect(refreshed.sources.map((source) => source.connectorId)).toEqual([ newSourceId, ...sourceIds.slice(0, 24), @@ -190,7 +131,7 @@ describe('bounded source pagination and provider overview', () => { }) expect(next.sources.map((source) => source.connectorId)).toEqual(sourceIds.slice(24, 49)) }) - it('does not report deactivated pending sources as indexing in any read model', async () => { + it('preserves organization deactivation when listing source configuration', async () => { await db.insert(member).values({ id: generateId(), organizationId: ids.organizationId, @@ -217,22 +158,10 @@ describe('bounded source pagination and provider overview', () => { approved: false, }) const owner = { organizationId: ids.organizationId } - const [progress, overview, summary] = await Promise.all([ - readSearchSourceProgress.execute({ - principal: alice, - input: { ...owner, connectorIds: [approvalSourceId] }, - }), - readSearchSourceOverview.execute({ principal: alice, input: owner }), - listSearchSources.execute({ principal: alice, input: owner }), - ]) - expect(progress.sources[0].isSyncing).toBe(false) - expect( - overview.providers.find((provider) => provider.connectorType === 'google_drive')?.isSyncing - ).toBe(false) + const summary = await listSearchSources.execute({ principal: alice, input: owner }) expect(summary.sources[0]).toMatchObject({ connectorId: approvalSourceId, approved: false, - isSyncing: false, }) }) }) diff --git a/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts b/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts index e47cb75cfc6..34ca5cf96c8 100644 --- a/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts @@ -28,8 +28,6 @@ import { readKnowledgeDocument, updateKnowledgeDocument, } from '@/lib/knowledge/application/documents' -import { readSearchSourceProgress } from '@/lib/knowledge/application/search-source-progress' -import { listSearchSources } from '@/lib/knowledge/application/search-sources' import { KNOWLEDGE_CONNECTOR_DETACH_EVENT } from '@/lib/knowledge/connectors/detachment' import { createContentSyncLease } from '@/lib/knowledge/connectors/sync-lock' import { persistSkippedDocuments } from '@/lib/knowledge/connectors/sync-persistence' @@ -41,15 +39,10 @@ const alice = { kind: 'session' as const, userId: ids.aliceId, sessionId: 'fixtu const bob = { kind: 'session' as const, userId: ids.bobId, sessionId: 'fixture-bob' } const failedId = generateId() const pendingId = generateId() -const input = { workspaceId: ids.workspaceId, connectorIds: [ids.connectorId] } -/** Drive models the mirrored email grants exercised by these provider-independent progress tests. */ +/** Drive models the mirrored email grants exercised by these document recovery tests. */ beforeAll(async () => { await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) - await db - .update(knowledgeBase) - .set({ isSearchIndex: true }) - .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) await db .update(knowledgeConnector) .set({ status: 'active', syncLockToken: null }) @@ -80,32 +73,7 @@ afterAll(async () => { await db.delete(user).where(eq(user.id, ids.bobId)) }) -describe('viewer-isolated indexing progress and recovery lists', () => { - it('keeps failed and pending state scoped to the viewer, including admins', async () => { - expect((await readSearchSourceProgress.execute({ principal: alice, input })).sources).toEqual([ - { - connectorId: ids.connectorId, - isSyncing: false, - hasSyncError: false, - hasIndexingError: true, - }, - ]) - expect((await readSearchSourceProgress.execute({ principal: bob, input })).sources).toEqual([ - { - connectorId: ids.connectorId, - isSyncing: true, - hasSyncError: false, - hasIndexingError: false, - }, - ]) - const [aliceSources, bobSources] = await Promise.all( - [alice, bob].map((principal) => - listSearchSources.execute({ principal, input: { workspaceId: ids.workspaceId } }) - ) - ) - expect(aliceSources.sources[0].viewerFailedDocumentCount).toBe(1) - expect(bobSources.sources[0].viewerFailedDocumentCount).toBe(0) - }) +describe('viewer-isolated knowledge-base recovery lists', () => { it('only lists accessible failures, with authoritative filtered pagination', async () => { const read = (principal: typeof alice) => listKnowledgeConnectorDocuments.execute({ @@ -123,29 +91,6 @@ describe('viewer-isolated indexing progress and recovery lists', () => { expect(result.hasMore).toBe(false) expect((await read(bob)).documents).toEqual([]) }) - it('does not report excluded or deleted failures as actionable', async () => { - await db.update(document).set({ userExcluded: true }).where(eq(document.id, failedId)) - expect( - (await readSearchSourceProgress.execute({ principal: alice, input })).sources[0] - .hasIndexingError - ).toBe(false) - await db - .update(document) - .set({ userExcluded: false, deletedAt: new Date() }) - .where(eq(document.id, failedId)) - expect( - (await readSearchSourceProgress.execute({ principal: alice, input })).sources[0] - .hasIndexingError - ).toBe(false) - }) - it('rechecks membership before showing progress', async () => { - await db - .delete(permissions) - .where(and(eq(permissions.entityId, ids.workspaceId), eq(permissions.userId, ids.bobId))) - await expect(readSearchSourceProgress.execute({ principal: bob, input })).rejects.toThrow( - 'Insufficient workspace permissions' - ) - }) }) describe('connector document filename search and document sets', () => { @@ -322,10 +267,6 @@ describe('intentional skips and genuine failures across document reads', () => { beforeAll(async () => { await seedKnowledgeAclFixture(fixture, { connectorType: 'google_drive' }) - await db - .update(knowledgeBase) - .set({ isSearchIndex: true }) - .where(eq(knowledgeBase.id, fixture.knowledgeBaseId)) const rows: Array & { id: string; filename: string }> = [ { id: legacySkipId, @@ -453,29 +394,7 @@ describe('intentional skips and genuine failures across document reads', () => { expect(legacyFailures.documents.map((row) => row.id)).toEqual(failureIds) }) - it('does not turn another viewer’s skips into indexing errors or expose their documents', async () => { - for (const [principal, failedCount] of [ - [viewer, 2], - [otherViewer, 0], - ] as const) { - const sources = await listSearchSources.execute({ - principal, - input: { workspaceId: fixture.workspaceId }, - }) - expect(sources.sources[0].viewerFailedDocumentCount).toBe(failedCount) - const progress = await readSearchSourceProgress.execute({ - principal, - input: { workspaceId: fixture.workspaceId, connectorIds: [fixture.connectorId] }, - }) - expect(progress.sources).toEqual([ - { - connectorId: fixture.connectorId, - isSyncing: false, - hasSyncError: false, - hasIndexingError: failedCount > 0, - }, - ]) - } + it('does not expose another viewer’s skipped documents or failures', async () => { for (const filter of ['active', 'skipped', 'failed'] as const) { const result = await listKnowledgeConnectorDocuments.execute({ principal: otherViewer, @@ -614,10 +533,6 @@ describe('intentional skips and genuine failures across document reads', () => { input: { ...scope, deleteDocuments: false }, }) ).rejects.toThrow('cannot be kept') - await db - .update(knowledgeBase) - .set({ isSearchIndex: false }) - .where(eq(knowledgeBase.id, fixture.knowledgeBaseId)) await db .update(knowledgeConnector) .set({ accessMode: 'workspace' }) diff --git a/apps/sim/lib/knowledge/__integration__/search-source-setup.integration.ts b/apps/sim/lib/knowledge/__integration__/search-source-setup.integration.ts index a339cd91cbe..ef488c24b48 100644 --- a/apps/sim/lib/knowledge/__integration__/search-source-setup.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-source-setup.integration.ts @@ -14,13 +14,6 @@ import { generateId } from '@sim/utils/id' import { and, eq, isNull } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' -/** A Search source crawls into a search index, which only indexed organization search reads. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) - const fixture = vi.hoisted(() => ({ dispatch: vi.fn() })) vi.mock('@/lib/credential-groups/provider-registry', () => ({ getCredentialGroupProviderAdapter: (provider: string) => ({ @@ -36,13 +29,6 @@ vi.mock('@/lib/credential-groups/provider-registry', () => ({ }), })) vi.mock('@/lib/knowledge/connectors/member-queue', () => ({ dispatchMemberSync: fixture.dispatch })) -vi.mock('@/lib/knowledge/application/connector-access', () => ({ - startKnowledgeConnectorMemberEnrollment: { - execute: async ({ input }: { input: { connectorId: string } }) => ({ - url: `https://fixture.test/enroll/${input.connectorId}`, - }), - }, -})) vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: { google_drive: { @@ -67,11 +53,7 @@ import { seedKnowledgeMemberFixture, } from '@/lib/knowledge/__integration__/seed-source-access-fixture' import { createKnowledgeConnector } from '@/lib/knowledge/application/connectors' -import { - connectSimSearchConnector, - prepareSearchSource, - readSearchIndex, -} from '@/lib/knowledge/application/sim-search' +import { prepareSearchSource, readSearchIndex } from '@/lib/knowledge/application/sim-search' import { performCreateKnowledgeConnector } from '@/lib/knowledge/orchestration/connectors' import { createWorkspaceInTransaction } from '@/lib/workspaces/create' @@ -124,6 +106,7 @@ describe('Search source identity and concurrent creation', () => { id: ids.knowledgeBaseId, name: 'Renamed company index', workspaceId: ids.workspaceId, + isSearchIndex: true, }, connectorType: 'google_drive', sourceConfig: { folderId }, @@ -295,7 +278,7 @@ describe('Search source identity and concurrent creation', () => { const successful = results.filter((result) => result.success) expect(new Set(successful.map((result) => result.connector.id)).size).toBe(1) expect(successful.filter((result) => result.reused)).toHaveLength(1) - expect(fixture.dispatch).toHaveBeenCalledTimes(1) + expect(fixture.dispatch).not.toHaveBeenCalled() sourceIds.push(successful[0]!.connector.id) const [policy] = await db .select() @@ -316,57 +299,12 @@ describe('Search source identity and concurrent creation', () => { expect(rows.filter((row) => row.id !== member.connectorId)).toHaveLength(1) }) - it('keeps distinct source settings separate and allows readers to select the exact source', async () => { + it('keeps distinct source settings separate', async () => { const second = await createSource('other-folder') expect(second.success).toBe(true) if (!second.success) throw new Error(second.error) sourceIds.push(second.connector.id) expect(second.connector.id).not.toBe(sourceIds[0]) - const result = await connectSimSearchConnector.execute({ - principal: { kind: 'session', userId: ids.bobId, sessionId: 'fixture-reader' }, - input: { - workspaceId: ids.workspaceId, - connectorType: 'google_drive', - connectorId: second.connector.id, - }, - }) - expect(result.connectorId).toBe(second.connector.id) - expect(result.knowledgeBaseId).toBe(ids.knowledgeBaseId) - }) - - it('rejects stale settings, different providers, foreign sources, and noncanonical knowledge bases', async () => { - const noncanonical = generateId() - const extraSource = generateId() - await db.insert(knowledgeBase).values({ - id: noncanonical, - name: 'Ordinary base', - userId: ids.aliceId, - workspaceId: ids.workspaceId, - }) - await db.insert(knowledgeConnector).values({ - id: extraSource, - knowledgeBaseId: noncanonical, - connectorType: 'google_drive', - sourceConfig: {}, - accessMode: 'members', - }) - for (const input of [ - { - connectorType: 'google_drive', - connectorId: sourceIds[0], - sourceConfig: { folderId: 'changed-folder' }, - }, - { connectorType: 'confluence', connectorId: sourceIds[0] }, - { connectorType: 'google_drive', connectorId: other.connectorId }, - { connectorType: 'google_drive', connectorId: extraSource }, - ]) { - await expect( - connectSimSearchConnector.execute({ - principal: { kind: 'session', userId: ids.bobId, sessionId: 'fixture-reader' }, - input: { workspaceId: ids.workspaceId, ...input }, - }) - ).rejects.toMatchObject({ code: 'not_found' }) - } }) it('provisions one workspace container with optional provider options under concurrent setup', async () => { const [previousPolicy] = await db @@ -492,32 +430,6 @@ describe('Search source identity and concurrent creation', () => { }) }) - it('uses the same accounts option through actual first-source application setup', async () => { - const results = await Promise.all( - [1, 2].map(() => - connectSimSearchConnector.execute({ - principal: { kind: 'session', userId: ids.aliceId, sessionId: 'fixture-admin' }, - input: { - workspaceId: ids.workspaceId, - connectorType: 'google_drive', - sourceConfig: { folderId: 'search-account-folder' }, - }, - }) - ) - ) - expect(results[0]!.connectorId).toBe(results[1]!.connectorId) - const [source] = await db - .select() - .from(knowledgeConnector) - .where(eq(knowledgeConnector.id, results[0]!.connectorId)) - expect(source!.credentialGroupId).toBe(searchGroupId) - const groups = await db - .select() - .from(credentialGroup) - .where(eq(credentialGroup.workspaceId, ids.workspaceId)) - expect(groups).toHaveLength(1) - }) - it('refuses automatic provider expansion when a concurrent workflow grant commits first', async () => { let releaseGrant!: () => void let acquiredLock!: () => void diff --git a/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts index 4a50b7a74d0..50b29de0fc2 100644 --- a/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts @@ -9,7 +9,6 @@ import { embedding, knowledgeBase, knowledgeConnector, - member, organization, outboxEvent, user, @@ -26,12 +25,6 @@ const fixture = vi.hoisted(() => ({ listRuns: vi.fn(), batchTrigger: vi.fn(), })) -/** This suite covers indexed organization search, which is dormant unless Live Search is off. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) vi.mock('@/lib/core/config/trigger-runtime', () => ({ isInsideTriggerRun: () => fixture.useTrigger, })) @@ -105,7 +98,6 @@ import { retryDocumentProcessing, } from '@/lib/knowledge/documents/service' import { MAX_PROCESSING_ATTEMPTS, QUEUED_DISPATCH_GRACE_MS } from '@/lib/knowledge/documents/types' -import { searchScopedKnowledge } from '@/lib/sim-search/indexed/search/scoped-search' import type { SyncResult } from '@/connectors/types' interface QueryPlan { @@ -132,10 +124,7 @@ async function eventsFor(ids: ReturnType) { ) ) } -async function failedFile( - ids: ReturnType, - organizationOwned = false -) { +async function failedFile(ids: ReturnType) { const file = await addDocument( ids.knowledgeBaseId, ids.connectorId, @@ -147,9 +136,7 @@ async function failedFile( mimeType: 'text/plain', contentHash: 'fixture-retained-v1', }, - organizationOwned - ? { userId: ids.aliceId, workspaceId: null, organizationId: ids.organizationId } - : { userId: ids.aliceId, workspaceId: ids.workspaceId }, + { userId: ids.aliceId, workspaceId: ids.workspaceId }, undefined, 'admin', createContentSyncLease(ids.connectorId, ids.lockId) @@ -628,45 +615,34 @@ describe('independent recovery of retained connector documents', () => { .where(eq(knowledgeConnector.id, ids.connectorId)) }) - it('uses the organization owner and preserves Search visibility during source backoff', async () => { + it('does not restart retired Search indexing when a source leaves provider backoff', async () => { const ids = await seed() - await db.insert(member).values({ - id: generateId(), - organizationId: ids.organizationId, - userId: ids.aliceId, - role: 'owner', - }) + const file = await failedFile(ids) await db .update(knowledgeBase) .set({ workspaceId: null, organizationId: ids.organizationId, isSearchIndex: true }) .where(eq(knowledgeBase.id, ids.knowledgeBaseId)) - const file = await failedFile(ids, true) - await db - .update(knowledgeConnector) - .set({ status: 'error', nextSyncAt: new Date(Date.now() + 3_600_000) }) - .where(eq(knowledgeConnector.id, ids.connectorId)) - expect(await recoverKnowledgeDocumentProcessing()).toBe(1) - const [event] = await eventsFor(ids) - expect(event.payload).toMatchObject({ - billingScope: 'organization', - workspaceId: null, - organizationId: ids.organizationId, + const before = fixture.embeddingCalls + for (const nextSyncAt of [new Date(Date.now() + 3_600_000), old()]) { + await db + .update(knowledgeConnector) + .set({ status: 'error', nextSyncAt }) + .where(eq(knowledgeConnector.id, ids.connectorId)) + await recoverKnowledgeDocumentProcessing() + expect(await eventsFor(ids)).toEqual([]) + } + const [retained] = await db.select().from(document).where(eq(document.id, file.documentId)) + expect(retained).toMatchObject({ + processingStatus: 'failed', + processingAttempts: 1, + processingQueueToken: 'old-fixture-generation', }) + expect(fixture.embeddingCalls).toBe(before) expect( - await outbox.processOutboxEventById(event.id, knowledgeDocumentProcessingOutboxHandlers) - ).toBe('completed') - const [indexed] = await db.select().from(document).where(eq(document.id, file.documentId)) - expect(indexed.processingStatus, indexed.processingError ?? undefined).toBe('completed') - const result = await searchScopedKnowledge.execute({ - principal: { kind: 'session', userId: ids.aliceId, sessionId: 'fixture-session' }, - input: { - organizationId: ids.organizationId, - query: 'Orion', - topK: 3, - }, - }) - expect(result.results.some((row) => row.documentId === file.documentId)).toBe(true) + await db.select().from(embedding).where(eq(embedding.documentId, file.documentId)) + ).toEqual([]) }) + it('recovers while the source is deferred, fences its old worker, and indexes exactly once', async () => { const ids = await seed() const file = await failedFile(ids) diff --git a/apps/sim/lib/knowledge/__integration__/unfilled-projection-source.integration.ts b/apps/sim/lib/knowledge/__integration__/unfilled-projection-source.integration.ts deleted file mode 100644 index 0cb20102915..00000000000 --- a/apps/sim/lib/knowledge/__integration__/unfilled-projection-source.integration.ts +++ /dev/null @@ -1,402 +0,0 @@ -/** - * A search candidate's source decides whether the caller's live source proof is resolved before - * its content is read. These fixtures put a GitHub installation source's chunks on projection rows - * the source and ACL fill has not reached (`acl` and `connector_id` NULL), and check that such a - * chunk still reaches a member who holds the installation grant, stays hidden from one who does - * not, and is left out of a page once its source is known to be denied. - */ -import { createHash } from 'node:crypto' -import { db } from '@sim/db' -import { - credential, - credentialGroup, - credentialGroupEnrollment, - document, - embedding, - embeddingKeywordTin, - embeddingSearch, - knowledgeConnector, - knowledgeConnectorMember, - knowledgeDocumentObservation, - organization, - user, - workspace, -} from '@sim/db/schema' -import { generateId } from '@sim/utils/id' -import { isRecordLike } from '@sim/utils/object' -import { eq, inArray, sql } from 'drizzle-orm' -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' - -/** This suite covers indexed organization search, which is dormant unless Live Search is off. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags( - importOriginal - ) -) -/** The TINQL `resolveTinKeywordQuery` renders for `fixture`: its `english` stem, quoted. */ -vi.mock('@/lib/sim-search/indexed/retrieval/tin-keyword', () => ({ - resolveTinKeywordQuery: async () => '"fixtur"', -})) - -import { - createKnowledgeAclFixtureIds, - seedKnowledgeAclFixture, -} from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import type { - GitHubInstallationReadGrant, - KnowledgeAccessProvider, - UserAccessScope, -} from '@/lib/knowledge/access/types' -import { liveSourceAccessForConnectors } from '@/lib/knowledge/search/candidates' -import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' -import type { SearchAccessPlan } from '@/lib/sim-search/indexed/retrieval/access-plan' -import { executeIndexedKeywordSearch } from '@/lib/sim-search/indexed/retrieval/keyword' -import type { IndexedRetrievalContext } from '@/lib/sim-search/indexed/retrieval/permitted' -import { forgetProjectionFilled } from '@/lib/sim-search/indexed/retrieval/projection-fill' -import { selectIndexedVectorResults } from '@/lib/sim-search/indexed/retrieval/vector' - -const ids = createKnowledgeAclFixtureIds() -const connectorId = generateId() -const contentCredentialId = generateId() -const groupId = generateId() -const optionId = generateId() -const documentId = generateId() -const embeddingId = generateId() -const repositoryId = '4242' -const members = { - alice: { id: generateId(), subject: 'alice-gh', credentialId: generateId() }, - bob: { id: generateId(), subject: 'bob-gh', credentialId: generateId() }, -} -const userIdOf = (who: 'alice' | 'bob') => (who === 'alice' ? ids.aliceId : ids.bobId) -const subjectToken = (subject: string) => `s:github-repositories:-:${subject}` -const queryVector = { - vector: JSON.stringify([1, ...Array(1535).fill(0)]), - dimensions: 1536 as const, - model: 'text-embedding-3-small', -} - -/** The shims stand in for the Tin extension, which the test database does not carry. */ -let createdTinShims = false - -const scopeFor = (who: 'alice' | 'bob'): UserAccessScope => ({ - kind: 'user', - userId: userIdOf(who), - tokens: [ - 'pub', - subjectToken(members[who].subject), - `u:${userIdOf(who)}@fixture.test`, - 'ws', - ].sort(), -}) - -const planFor = (who: 'alice' | 'bob'): SearchAccessPlan => ({ - connectors: { - workspace: [], - admin: [], - members: [connectorId], - liveProofRequired: [connectorId], - }, - observers: { confirmed: [{ id: members[who].id, connectorId }], observed: [] }, - memberSources: [connectorId], - connectorTypes: new Map([[connectorId, 'github']]), - uploads: false, -}) - -/** Alice's reader credential backs a real installation grant; Bob holds none. */ -const aliceGrant: GitHubInstallationReadGrant = { - connectorId, - contentCredentialId, - readerCredentialId: members.alice.credentialId, - readerSubjectToken: subjectToken(members.alice.subject), - repositoryId, -} - -const searchInputs = (who: 'alice' | 'bob') => ({ - knowledgeBaseIds: [ids.knowledgeBaseId], - topK: 5, - access: scopeFor(who), - queryVector, -}) - -/** A narrow reader of the search index, whose live installation grant is resolved on demand. */ -function searchContext(who: 'alice' | 'bob'): IndexedRetrievalContext { - const access = scopeFor(who) - const accessPlan = planFor(who) - const granted = who === 'alice' ? { ...access, githubInstallationGrants: [aliceGrant] } : access - const accessProvider: KnowledgeAccessProvider = { - get: async () => access, - getForConnectors: async () => granted, - getForDocuments: async () => granted, - liveSourceConnectorCondition: async () => null, - } - return { - access, - accessPlan, - filtered: false, - permitted: { kind: 'unbounded', broad: false }, - liveSourceAccess: liveSourceAccessForConnectors( - accessPlan.connectors.liveProofRequired, - accessProvider - ), - } -} - -const keywordIds = async (who: 'alice' | 'bob') => - ( - await executeIndexedKeywordSearch( - { ...searchInputs(who), query: 'fixture' }, - searchContext(who) - ) - ).map((row) => row.id) - -/** - * Ranked exactly on the row, under the same visibility predicate the graph walk applies. The walk - * is approximate: in a graph the other files sharing this database crowd with degenerate vectors, - * a chunk can be pruned from every neighbour list and never be reached, however far the walk goes. - */ -const vectorIds = async (who: 'alice' | 'bob') => - ( - await selectIndexedVectorResults( - { ...searchInputs(who), distanceThreshold: 2 }, - searchContext(who) - ) - ).map((row) => row.id) - -async function setProjection(state: 'filled' | 'unfilled') { - for (const table of [embeddingSearch, embeddingKeywordTin]) { - await db - .update(table) - .set( - state === 'filled' - ? { - connectorId, - acl: [subjectToken(members.alice.subject), subjectToken(members.bob.subject)].sort(), - } - : { connectorId: null, acl: null } - ) - .where(eq(table.id, embeddingId)) - } - forgetProjectionFilled() -} - -beforeAll(async () => { - await seedKnowledgeAclFixture(ids) - const now = new Date() - await db - .update(user) - .set({ emailVerified: true }) - .where(inArray(user.id, [ids.aliceId, ids.bobId])) - await db.insert(credential).values({ - id: contentCredentialId, - workspaceId: ids.workspaceId, - type: 'service_account', - displayName: 'Fixture GitHub installation', - createdBy: ids.aliceId, - providerId: GITHUB_INSTALLATION_PROVIDER_ID, - }) - await db.insert(credentialGroup).values({ - id: groupId, - workspaceId: ids.workspaceId, - publicId: generateId(), - name: 'GitHub readers', - options: [ - { - id: optionId, - provider: 'github-repositories', - label: 'GitHub fixture', - authorizationAppId: 'fixture-app', - requiredScopes: ['repo'], - scopeVersion: 1, - required: false, - status: 'active', - }, - ], - } as typeof credentialGroup.$inferInsert) - for (const who of ['alice', 'bob'] as const) { - const [enrollment] = await db - .insert(credentialGroupEnrollment) - .values({ - id: generateId(), - credentialGroupId: groupId, - userId: userIdOf(who), - email: `${userIdOf(who)}@fixture.test`, - status: 'completed', - invitationTokenHash: createHash('sha256').update(generateId()).digest('hex'), - invitationExpiresAt: new Date(Date.now() + 60 * 60 * 1000), - invitedAt: now, - }) - .returning({ id: credentialGroupEnrollment.id }) - await db.insert(credential).values({ - id: members[who].credentialId, - workspaceId: ids.workspaceId, - type: 'managed_oauth', - displayName: 'Fixture GitHub reader', - providerId: 'github-repositories', - authorizationAppId: 'fixture-app', - credentialGroupEnrollmentId: enrollment!.id, - credentialGroupOptionId: optionId, - managedOauthScopeVersion: 1, - providerSubjectId: members[who].subject, - providerTenantId: '', - managedOauthStatus: 'active', - grantedScopes: ['repo'], - encryptedOauthTokenSet: 'fixture-not-an-oauth-token', - grantedAt: now, - createdBy: userIdOf(who), - }) - } - await db.insert(knowledgeConnector).values({ - id: connectorId, - knowledgeBaseId: ids.knowledgeBaseId, - connectorType: 'github', - sourceConfig: { githubRepositoryId: repositoryId }, - accessMode: 'members', - status: 'active', - credentialId: contentCredentialId, - credentialGroupId: groupId, - credentialGroupOptionId: optionId, - }) - await db.insert(knowledgeConnectorMember).values( - (['alice', 'bob'] as const).map((who) => ({ - id: members[who].id, - workspaceId: ids.workspaceId, - connectorId, - credentialId: members[who].credentialId, - subjectToken: subjectToken(members[who].subject), - status: 'active', - memberSyncedThrough: now, - })) - ) - await db.insert(document).values({ - id: documentId, - connectorId, - knowledgeBaseId: ids.knowledgeBaseId, - externalId: 'fixture-file', - filename: 'readme.md', - fileUrl: 'https://fixture.test/readme', - fileSize: 12, - mimeType: 'text/plain', - processingStatus: 'completed', - acl: [subjectToken(members.alice.subject), subjectToken(members.bob.subject)].sort(), - }) - await db.insert(knowledgeDocumentObservation).values( - (['alice', 'bob'] as const).map((who) => ({ - documentId, - memberId: members[who].id, - lastSeenAt: now, - runId: generateId(), - })) - ) - await db.insert(embedding).values({ - id: embeddingId, - documentId, - knowledgeBaseId: ids.knowledgeBaseId, - chunkIndex: 0, - chunkHash: 'fixture-hash', - content: 'fixture readme', - contentLength: 14, - tokenCount: 2, - startOffset: 0, - endOffset: 14, - embeddingModel: 'text-embedding-3-small', - embedding: [1, ...Array(1535).fill(0)], - }) - const [tin] = await db.execute<{ present: boolean }>( - sql`SELECT to_regnamespace('tin') IS NOT NULL AS present` - ) - if (!tin?.present) { - createdTinShims = true - await db.execute( - sql.raw(`CREATE SCHEMA tin; - CREATE FUNCTION tin.full_score(tid) RETURNS double precision LANGUAGE sql IMMUTABLE AS 'SELECT 1.0::float8'; - CREATE FUNCTION knowledge_tin_base_token(text) RETURNS text LANGUAGE sql IMMUTABLE AS $$SELECT 'kb'$$; - CREATE FUNCTION knowledge_tin_stream(vector tsvector) RETURNS text LANGUAGE sql IMMUTABLE AS $$ - SELECT coalesce(string_agg(entry.lexeme, ' ' ORDER BY position), '') - FROM unnest(vector) AS entry(lexeme, positions, weights), unnest(entry.positions) AS position - $$; - CREATE FUNCTION tin_fixture_match(text, text) RETURNS boolean LANGUAGE sql IMMUTABLE AS 'SELECT true'; - CREATE OPERATOR ==> (LEFTARG = text, RIGHTARG = text, FUNCTION = tin_fixture_match);`) - ) - } - /** Written as the projection trigger writes it, so real Tin scopes the row to its base. */ - await db.execute(sql` - INSERT INTO ${embeddingKeywordTin} (id, knowledge_base_id, document_id, enabled, content) - SELECT id, knowledge_base_id, document_id, enabled, - knowledge_tin_base_token(knowledge_base_id) || ' ' || knowledge_tin_stream(content_tsv) - FROM ${embedding} WHERE id = ${embeddingId} - ON CONFLICT (id) DO UPDATE SET content = EXCLUDED.content`) -}) - -afterAll(async () => { - if (createdTinShims) { - await db.execute( - sql.raw(`DROP OPERATOR IF EXISTS ==> (text, text); - DROP FUNCTION IF EXISTS tin_fixture_match(text, text); - DROP FUNCTION IF EXISTS knowledge_tin_base_token(text); - DROP FUNCTION IF EXISTS knowledge_tin_stream(tsvector); - DROP SCHEMA IF EXISTS tin CASCADE;`) - ) - } - await db.delete(embeddingKeywordTin).where(eq(embeddingKeywordTin.id, embeddingId)) - await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) - await db.delete(credentialGroup).where(eq(credentialGroup.id, groupId)) - await db.delete(organization).where(eq(organization.id, ids.organizationId)) - await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) - forgetProjectionFilled() -}) - -describe('a chunk whose projection row the fill has not reached', () => { - beforeEach(() => setProjection('unfilled')) - - it('reaches the member holding the installation grant through the keyword ranking', async () => { - expect(await keywordIds('alice')).toEqual([embeddingId]) - }) - - it('reaches the member holding the installation grant through the vector ranking', async () => { - expect(await vectorIds('alice')).toEqual([embeddingId]) - }) - - it('stays hidden from a member without the grant', async () => { - expect(await keywordIds('bob')).toEqual([]) - expect(await vectorIds('bob')).toEqual([]) - }) - - describe('once its source is known to be denied', () => { - /** Each Tin ranking statement's page of candidates, in the order the search read them. */ - const pages: Array<{ candidates: unknown[] }> = [] - beforeEach(() => { - pages.length = 0 - const execute = db.execute.bind(db) - vi.spyOn(db, 'execute').mockImplementation((async (query: Parameters[0]) => { - const rows = await execute(query) - const [row] = Array.from(rows) - if (isRecordLike(row) && 'ranked' in row && Array.isArray(row.candidates)) - pages.push({ candidates: row.candidates }) - return rows - }) as typeof db.execute) - }) - afterEach(() => vi.restoreAllMocks()) - - it('carries the source read from its document and is left out of the rebuilt keyword page', async () => { - expect(await keywordIds('bob')).toEqual([]) - expect(pages.length).toBeGreaterThanOrEqual(2) - expect(pages[0]!.candidates).toEqual([{ id: embeddingId, documentId, connectorId }]) - expect(pages.at(-1)!.candidates).toEqual([]) - }) - }) -}) - -describe('a chunk whose projection row is filled', () => { - beforeEach(() => setProjection('filled')) - - it('ranks on the row as before and is read only by the member holding the grant', async () => { - const [{ unfilled }] = await db.execute<{ unfilled: boolean }>( - sql`SELECT EXISTS (SELECT 1 FROM ${embeddingKeywordTin} WHERE ${embeddingKeywordTin.acl} IS NULL) AS unfilled` - ) - expect(unfilled).toBe(false) - expect(await keywordIds('alice')).toEqual([embeddingId]) - expect(await keywordIds('bob')).toEqual([]) - expect(await vectorIds('alice')).toEqual([embeddingId]) - expect(await vectorIds('bob')).toEqual([]) - }) -}) diff --git a/apps/sim/lib/knowledge/__integration__/workspace-kb-document-access.integration.ts b/apps/sim/lib/knowledge/__integration__/workspace-kb-document-access.integration.ts index c0fc6cfd66b..85be5ca3113 100644 --- a/apps/sim/lib/knowledge/__integration__/workspace-kb-document-access.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/workspace-kb-document-access.integration.ts @@ -5,8 +5,8 @@ * the keyword projection — so projection rows that are stale, unfilled, or missing change * nothing about what a workspace search returns. A signed-in reader's own grants widen what they * read, and a source whose reader must be proven live admits a candidate only once the proof - * holds, with the readable rows below it filling the page when it does not. A search index named - * by id while indexed organization search is dormant is searched the same way. + * holds, with the readable rows below it filling the page when it does not. A Search-marked knowledge base + * named explicitly by id is searched the same way. */ import { createHash } from 'node:crypto' import type { Principal } from '@sim/auth/principal' @@ -29,14 +29,7 @@ import { } from '@sim/db/schema' import { generateId } from '@sim/utils/id' import { eq, inArray } from 'drizzle-orm' -import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' - -/** Pinned to Live Search, so indexed organization search is dormant whatever the run's environment. */ -vi.mock('@/lib/core/config/env-flags', async (importOriginal) => ({ - ...(await importOriginal>()), - isLiveEnterpriseSearchEnabled: true, -})) - +import { afterAll, beforeAll, describe, expect, it } from 'vitest' import { createKnowledgeAclFixtureIds, seedKnowledgeAclFixture, @@ -50,7 +43,6 @@ import type { import { type KnowledgeSearchMode, retrieveKnowledgeSearch } from '@/lib/knowledge/search/queries' import { embeddingVectorValues } from '@/lib/knowledge/vector-columns' import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' -import { usesIndexedRetrieval } from '@/lib/sim-search/indexed/gate' afterAll(async () => { await db.$client.end() @@ -165,7 +157,6 @@ describe.each([ access: await accessProvider.get(), accessProvider, searchMode, - indexedRetrieval: usesIndexedRetrieval([{ isSearchIndex }]), query: 'handbook onboarding', queryVector, }) diff --git a/apps/sim/lib/knowledge/access/predicate.integration.ts b/apps/sim/lib/knowledge/access/predicate.integration.ts index c42d9108c8b..d00e3f08dcf 100644 --- a/apps/sim/lib/knowledge/access/predicate.integration.ts +++ b/apps/sim/lib/knowledge/access/predicate.integration.ts @@ -1,6 +1,6 @@ import { readFile } from 'node:fs/promises' import { readTestDatabaseUrl } from '@sim/db/testing/test-infrastructure' -import { type SQL, sql } from 'drizzle-orm' +import type { SQL } from 'drizzle-orm' import type postgres from 'postgres' import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' import { createEnterpriseSearchMigrationFixture } from '@/lib/knowledge/__integration__/migration-fixture' @@ -26,9 +26,6 @@ const { PgDialect } = await import('drizzle-orm/pg-core') const { knowledgeAccessCondition, knowledgeMetadataCandidateAccessCondition } = await import( '@/lib/knowledge/access/predicate' ) -const { restrictSearchAccessPlan } = await import('@/lib/sim-search/indexed/retrieval/access-plan') -const { knowledgeCandidateAccessConditionForConnectors, projectionCandidateAccessCondition } = - await import('@/lib/sim-search/indexed/retrieval/projection-access') const { confluencePageAcl } = await import('@/lib/knowledge/access/confluence-permissions') /** Every table and index belongs to an isolated disposable schema. */ @@ -511,209 +508,6 @@ describe('knowledge ACLs in PostgreSQL', () => { expect(await readable(['ws'], 'upload')).toBe(true) }) - it('admits the same documents whether connector state is proven per row or resolved per query', async () => { - const scope = { kind: 'user' as const, userId: 'reader', tokens: [alice, 'u:alice@corp.com'] } - await connection.unsafe( - `INSERT INTO knowledge_connector(id, access_mode, deleted_at, archived_at) VALUES - ('gone', 'admin', statement_timestamp(), NULL), ('shelved', 'admin', NULL, statement_timestamp())` - ) - await connection.unsafe( - "INSERT INTO knowledge_connector(id, access_mode) VALUES ('ws-mode', 'workspace')" - ) - await connection.unsafe( - `INSERT INTO knowledge_connector_member(id, workspace_id, connector_id, subject_token, status) - VALUES ('m-alice', 'workspace', 'members', $1, 'active')`, - [alice] - ) - const cases: Array<[string, string, string[]]> = [ - ['admin-current', 'admin', ['u:alice@corp.com']], - ['members-current', 'members', [alice]], - ['workspace-doc', 'ws-mode', ['ws']], - ['deleted-connector', 'gone', ['u:alice@corp.com']], - ['archived-connector', 'shelved', ['u:alice@corp.com']], - ] - for (const [id, connectorId, acl] of cases) { - await connection.unsafe( - `INSERT INTO document(id, connector_id, acl, acl_verified_at) - VALUES ($1, $2, string_to_array($3, E'\n'), statement_timestamp())`, - [id, connectorId, acl.join('\n')] - ) - } - await connection.unsafe( - "INSERT INTO knowledge_document_observation VALUES ('members-current', 'm-alice', statement_timestamp())" - ) - await connection.unsafe("INSERT INTO document(id) VALUES ('upload-doc')") - /** What `resolveConnectorEligibility` returns for this base: the connectors it admits, by mode. */ - const eligibility = { - workspace: ['ws-mode'], - admin: ['admin'], - members: ['members'], - liveProofRequired: [], - } - /** What `resolveSearchAccessPlan` resolves for this caller: their member identity, confirmed. */ - const observers = { confirmed: [{ id: 'm-alice', connectorId: 'members' }], observed: [] } - const perRow = knowledgeMetadataCandidateAccessCondition(scope) - const plan = { - connectors: eligibility, - observers, - memberSources: ['members'], - connectorTypes: new Map([ - ['ws-mode', 'slack'], - ['admin', 'google_drive'], - ['members', 'slack'], - ]), - uploads: true, - } - const perQuery = knowledgeCandidateAccessConditionForConnectors(scope, plan) - for (const id of [...cases.map(([documentId]) => documentId), 'upload-doc']) { - expect([id, await admits(perQuery, id)]).toEqual([id, await admits(perRow, id)]) - } - /** - * A document that changed hands keeps its old observations. The caller's member observed it - * while its old connector held it; under its new connector, of which the caller is no member, - * neither predicate may carry it on that observation. - */ - await connection.unsafe( - "INSERT INTO knowledge_connector(id, access_mode) VALUES ('members-elsewhere', 'members')" - ) - await connection.unsafe( - `INSERT INTO document(id, connector_id, acl, acl_verified_at) - VALUES ('moved-doc', 'members-elsewhere', ARRAY[$1], statement_timestamp())`, - [alice] - ) - await connection.unsafe( - "INSERT INTO knowledge_document_observation VALUES ('moved-doc', 'm-alice', statement_timestamp())" - ) - const movedPlan = { - ...plan, - connectors: { ...eligibility, members: [...eligibility.members, 'members-elsewhere'] }, - } - expect(await admits(perRow, 'moved-doc')).toBe(false) - expect( - await admits(knowledgeCandidateAccessConditionForConnectors(scope, movedPlan), 'moved-doc') - ).toBe(false) - /** - * The projection predicate decides on the ranking row alone, from the source and ACL mirrored - * there. It must never refuse a document the per-row predicate admits: whatever it admits beyond - * that is refused at hydration, under the full predicate, before content is returned. - */ - await connection.unsafe(`CREATE TABLE IF NOT EXISTS embedding_search( - id text PRIMARY KEY, document_id text, connector_id text, acl text[])`) - await connection.unsafe( - 'CREATE TABLE IF NOT EXISTS knowledge_projection_dirty(document_id text PRIMARY KEY)' - ) - await connection.unsafe('DELETE FROM embedding_search') - await connection.unsafe( - "INSERT INTO embedding_search SELECT id || '-chunk', id, connector_id, acl FROM document" - ) - const onRow = new PgDialect().sqlToQuery( - projectionCandidateAccessCondition(schema.embeddingSearch, scope, plan) - ) - const onRowAdmits = async (id: string) => { - const rows = await connection.unsafe( - `SELECT 1 FROM embedding_search WHERE ${onRow.sql} AND document_id = $${onRow.params.length + 1}`, - [...(onRow.params as string[]), id] - ) - return rows.length > 0 - } - for (const id of [...cases.map(([documentId]) => documentId), 'upload-doc']) { - if (await admits(perRow, id)) expect([id, await onRowAdmits(id)]).toEqual([id, true]) - } - /** - * A source the caller is a member of still holds documents that name other members only; the - * mirrored ACL keeps those out of the ranking rather than leaving them for hydration to drop. - */ - await connection.unsafe( - "INSERT INTO document(id, connector_id, acl, acl_verified_at) VALUES ('members-other', 'members', ARRAY['s:slack:-:bob'], statement_timestamp())" - ) - await connection.unsafe( - "INSERT INTO embedding_search SELECT id || '-chunk', id, connector_id, acl FROM document WHERE id = 'members-other'" - ) - expect(await admits(perRow, 'members-other')).toBe(false) - expect(await onRowAdmits('members-other')).toBe(false) - /** - * A chunk the backfill has not reached carries no source or ACL yet and is decided on its - * document, as every candidate was before the columns existed: search must not depend on the - * backfill, must not lose a document to it, and must not rank an unreadable one because of it. - */ - await connection.unsafe( - "INSERT INTO document(id, connector_id, acl, acl_verified_at) VALUES ('unfilled-other', 'members', ARRAY['s:slack:-:bob'], statement_timestamp())" - ) - await connection.unsafe(`INSERT INTO embedding_search(id, document_id) VALUES - ('unfilled-other-chunk', 'unfilled-other'), ('unfilled-admin-chunk', 'admin-current'), - ('unfilled-members-chunk', 'members-current'), ('unfilled-gone-chunk', 'deleted-connector')`) - await connection.unsafe( - "DELETE FROM embedding_search WHERE id IN ('admin-current-chunk', 'members-current-chunk', 'deleted-connector-chunk')" - ) - expect(await onRowAdmits('unfilled-other')).toBe(false) - expect(await onRowAdmits('admin-current')).toBe(true) - expect(await onRowAdmits('members-current')).toBe(true) - expect(await onRowAdmits('deleted-connector')).toBe(false) - /** - * Candidate ranking defers the live source proof, as the per-row candidate predicate does: a - * caller holds those grants only after authorization, so applying the clause during ranking - * would drop every candidate of a gated source before it could be proven. - */ - const gated = { ...plan, connectors: { ...eligibility, liveProofRequired: ['admin'] } } - expect( - await admits(knowledgeCandidateAccessConditionForConnectors(scope, gated), 'admin-current') - ).toBe(true) - expect( - await admits( - knowledgeCandidateAccessConditionForConnectors(scope, gated, sql`false`), - 'admin-current' - ) - ).toBe(false) - /** - * A plan confined to one kind of source admits that kind alone, on the document and on the - * row, and a plan confined to uploads admits only documents without a source. - */ - const drive = restrictSearchAccessPlan(plan, 'google_drive') - const driveOnRow = new PgDialect().sqlToQuery( - projectionCandidateAccessCondition(schema.embeddingSearch, scope, drive) - ) - const driveOnRowAdmits = async (id: string) => { - const rows = await connection.unsafe( - `SELECT 1 FROM embedding_search WHERE ${driveOnRow.sql} AND document_id = $${driveOnRow.params.length + 1}`, - [...(driveOnRow.params as string[]), id] - ) - return rows.length > 0 - } - const drivePerQuery = knowledgeCandidateAccessConditionForConnectors(scope, drive) - for (const [id, expected] of [ - ['admin-current', true], - ['workspace-doc', false], - ['members-current', false], - ['upload-doc', false], - ] as const) { - expect([id, await admits(drivePerQuery, id)]).toEqual([id, expected]) - expect([id, await driveOnRowAdmits(id)]).toEqual([id, expected]) - } - /** Uploads carry the workspace ACL, so a caller with that token reads them and nothing sourced. */ - await connection.unsafe("INSERT INTO document(id, acl) VALUES ('upload-mine', ARRAY['ws'])") - const wsScope = { ...scope, tokens: [...scope.tokens, 'ws'] } - const uploadsPerQuery = knowledgeCandidateAccessConditionForConnectors( - wsScope, - restrictSearchAccessPlan(plan, 'upload') - ) - expect(await admits(knowledgeMetadataCandidateAccessCondition(wsScope), 'upload-mine')).toBe( - true - ) - expect(await admits(uploadsPerQuery, 'upload-mine')).toBe(true) - expect(await admits(uploadsPerQuery, 'workspace-doc')).toBe(false) - expect(await admits(uploadsPerQuery, 'admin-current')).toBe(false) - /** A connector left out of the resolution is refused, however current its documents are. */ - expect( - await admits( - knowledgeCandidateAccessConditionForConnectors(scope, { - ...plan, - connectors: { ...eligibility, admin: [] }, - }), - 'admin-current' - ) - ).toBe(false) - }) - it('does not let one member refresh another member’s stale observation', async () => { await connection.unsafe( "INSERT INTO document(id, connector_id, acl) VALUES ('shared', 'members', string_to_array($1, ','))", diff --git a/apps/sim/lib/knowledge/access/predicate.test.ts b/apps/sim/lib/knowledge/access/predicate.test.ts index 06b4e248481..b2ef8ba4d37 100644 --- a/apps/sim/lib/knowledge/access/predicate.test.ts +++ b/apps/sim/lib/knowledge/access/predicate.test.ts @@ -13,80 +13,13 @@ vi.unmock('@sim/db/schema') process.env.DATABASE_URL ??= 'postgresql://user:pass@localhost:5432/test' const { PgDialect } = await import('drizzle-orm/pg-core') -const { embeddingSearch } = await import('@sim/db/schema') -const { sql: rawSql } = await import('drizzle-orm') -const sqlColumn = (name: string) => rawSql.raw(`"row"."${name}"`) const { knowledgeAccessCondition } = await import('@/lib/knowledge/access/predicate') -const { restrictSearchAccessPlan } = await import('@/lib/sim-search/indexed/retrieval/access-plan') -const { knowledgeCandidateAccessConditionForConnectors, projectionCandidateAccessCondition } = - await import('@/lib/sim-search/indexed/retrieval/projection-access') const { SYSTEM_ACCESS_SCOPE } = await import('@/lib/knowledge/access/types') function render(condition: ReturnType) { return new PgDialect().sqlToQuery(condition) } -describe('projectionCandidateAccessCondition', () => { - const plan = { - connectors: { workspace: ['ws-src'], admin: [], members: [], liveProofRequired: [] }, - observers: { confirmed: [], observed: [] }, - memberSources: [], - connectorTypes: new Map(), - uploads: true, - } - - const pending = - '(EXISTS (SELECT 1 FROM "knowledge_projection_dirty" WHERE "knowledge_projection_dirty"."document_id" = "embedding_search"."document_id"))' - const onDocument = - 'AND (SELECT "document"."id" FROM "document"\n WHERE "document"."id" = "embedding_search"."document_id"\n AND (' - - it('decides a filled row on its mirrored columns and an unfilled or marked row on its document', () => { - const { sql, params } = render( - projectionCandidateAccessCondition( - embeddingSearch, - { kind: 'user', userId: 'user-1', tokens: ['ws', 'u:alice'] }, - plan - ) - ) - expect(sql).toContain(`(("embedding_search"."acl" IS NULL OR ${pending}) ${onDocument}`) - expect(sql).toContain('"document"."acl" && ARRAY[$1, $2]::text[]') - expect(sql).toContain('OR (("embedding_search"."acl" && ARRAY[') - expect(sql).toContain( - 'AND ("embedding_search"."connector_id" IS NULL OR "embedding_search"."connector_id" = ANY(ARRAY[' - ) - expect(sql).toContain(`AND NOT ${pending})`) - expect(params.slice(0, 2)).toEqual(['ws', 'u:alice']) - expect(params.slice(-3)).toEqual(['ws', 'u:alice', 'ws-src']) - for (const param of params) expect(Array.isArray(param)).toBe(false) - }) - - it('still decides a marked row on its document once the projection is filled', () => { - const { sql } = render( - projectionCandidateAccessCondition( - embeddingSearch, - { kind: 'user', userId: 'user-1', tokens: ['ws', 'u:alice'] }, - plan, - { filled: true } - ) - ) - expect(sql).toContain(`((${pending} ${onDocument}`) - expect(sql).not.toContain('"embedding_search"."acl" IS NULL') - expect(sql).toContain(`AND NOT ${pending})`) - }) - - it('still denies everything for an empty token set', () => { - expect( - render( - projectionCandidateAccessCondition( - embeddingSearch, - { kind: 'user', userId: 'user-1', tokens: [] }, - plan - ) - ).sql - ).toBe('false') - }) -}) - describe('knowledgeAccessCondition', () => { it('overlaps the ACL with the tokens as a literal array of scalar binds', () => { const { sql, params } = render( @@ -134,52 +67,3 @@ describe('knowledgeAccessCondition', () => { expect(sql).not.toContain('"document"."acl"') }) }) - -describe('restrictSearchAccessPlan', () => { - const plan = { - connectors: { - workspace: ['slack-ws'], - admin: ['drive-admin', 'confluence-admin'], - members: ['slack-members'], - liveProofRequired: ['confluence-admin'], - }, - observers: { - confirmed: [{ id: 'm-1', connectorId: 'slack-members' }], - observed: [{ id: 'm-2', connectorId: 'drive-admin' }], - }, - memberSources: ['slack-members'], - connectorTypes: new Map([ - ['slack-ws', 'slack'], - ['slack-members', 'slack'], - ['drive-admin', 'google_drive'], - ['confluence-admin', 'confluence'], - ]), - uploads: true, - } - const reader = { kind: 'user' as const, userId: 'u', tokens: ['u:reader@example.com'] } - - it('drops source-less rows from both predicates once uploads are out of scope', () => { - const rowSql = (restricted: typeof plan) => - render( - projectionCandidateAccessCondition( - { - connectorId: sqlColumn('connector_id'), - acl: sqlColumn('acl'), - documentId: sqlColumn('document_id'), - }, - reader, - restricted - ) - ).sql - expect(rowSql(plan)).toContain('IS NULL OR') - expect(rowSql(restrictSearchAccessPlan(plan, 'slack'))).not.toContain( - '"row"."connector_id" IS NULL' - ) - const documentSql = (restricted: typeof plan) => - render(knowledgeCandidateAccessConditionForConnectors(reader, restricted)).sql - expect(documentSql(plan)).toContain('"document"."connector_id" IS NULL OR') - expect(documentSql(restrictSearchAccessPlan(plan, 'slack'))).not.toContain( - '"document"."connector_id" IS NULL' - ) - }) -}) diff --git a/apps/sim/lib/knowledge/api/route-policies.ts b/apps/sim/lib/knowledge/api/route-policies.ts index 27f3b4c36f8..80102c19bec 100644 --- a/apps/sim/lib/knowledge/api/route-policies.ts +++ b/apps/sim/lib/knowledge/api/route-policies.ts @@ -22,7 +22,6 @@ import { KnowledgeDocumentNotReadyError } from '@/lib/knowledge/application/chun import { KnowledgeSearchProvenanceUnavailableError } from '@/lib/knowledge/application/search' import { KnowledgeDocumentUnsupportedMediaTypeError } from '@/lib/knowledge/application/upload-sessions' import { SearchDeadlineError } from '@/lib/knowledge/search/budget' -import { SearchIndexDormantError } from '@/lib/sim-search/indexed/gate' import { v2Error } from '@/app/api/v2/lib/response' function internalKnowledgeErrorPolicy(unhandledMessage: string): InternalErrorPolicy { @@ -89,18 +88,6 @@ export const internalKnowledgeSessionOrExecutorAuth = createInternalSessionOrExe export const KNOWLEDGE_BASE_NOT_FOUND_MESSAGE = 'Knowledge base not found' -/** - * Answers an indexed-only surface refused while indexed organization search is dormant with a - * `409`: the request is well formed and authorized, and the deployment's state is what refuses it. - */ -function refuseDormantSearchIndex(base: InternalErrorPolicy): InternalErrorPolicy { - return extendInternalErrorPolicy(base, (error) => - error instanceof SearchIndexDormantError - ? internalErrorResponse(409, { error: error.message }) - : null - ) -} - /** * Conceals a knowledge-base-scoped internal policy the way the v2 knowledge * routes conceal theirs. The workspace-level `list` and `create` policies are @@ -154,12 +141,8 @@ export const internalKnowledgeErrorPolicies = { tags: concealKnowledgeBase( internalKnowledgeErrorPolicy('Failed to process knowledge tag request') ), - connectors: concealKnowledgeBase( - refuseDormantSearchIndex(internalKnowledgeErrorPolicy('Internal server error')) - ), - connectAccount: concealKnowledgeBase( - refuseDormantSearchIndex(internalPersonalCredentialConnectionErrorPolicy) - ), + connectors: concealKnowledgeBase(internalKnowledgeErrorPolicy('Internal server error')), + connectAccount: concealKnowledgeBase(internalPersonalCredentialConnectionErrorPolicy), uploads: concealKnowledgeBase(internalKnowledgeUploadErrorPolicy), } as const diff --git a/apps/sim/lib/knowledge/application/connect-personal-search-integration.test.ts b/apps/sim/lib/knowledge/application/connect-personal-search-integration.test.ts index fc2c7726945..67f5b55117e 100644 --- a/apps/sim/lib/knowledge/application/connect-personal-search-integration.test.ts +++ b/apps/sim/lib/knowledge/application/connect-personal-search-integration.test.ts @@ -17,7 +17,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const hoisted = vi.hoisted(() => ({ resolve: vi.fn(), - indexed: vi.fn(), oauth: vi.fn(), })) vi.mock('@/lib/core/application/organization-authorization', () => organizationAuthorizationMock) @@ -25,9 +24,6 @@ vi.mock('@/lib/knowledge/application/contexts', () => knowledgeContextsMock) vi.mock('@/lib/knowledge/application/personal-search-integrations', () => ({ resolvePersonalSearchConnection: { execute: hoisted.resolve }, })) -vi.mock('@/lib/knowledge/application/sim-search', () => ({ - connectSimSearchConnector: { execute: hoisted.indexed }, -})) vi.mock('@/lib/credential-groups/service', () => credentialGroupsServiceMock) vi.mock('@/lib/credential-groups/scoped-availability', () => credentialGroupsAvailabilityMock) vi.mock('@/lib/credential-groups/credentials', () => credentialGroupsCredentialsMock) @@ -90,7 +86,6 @@ describe('personal live Search connection', () => { input: { ...input, target: selected }, }) expect(result.url).toBe('https://provider.test/oauth') - expect(result.connectorId).toBeUndefined() expect( connectPersonalSearchIntegrationContract.response.schema.safeParse({ success: true, @@ -105,7 +100,6 @@ describe('personal live Search connection', () => { completionId: input.oauthCompletionId, connectionIntent: credentialId ? { kind: 'reconnect', credentialId } : { kind: 'create' }, }) - expect(m.indexed).not.toHaveBeenCalled() expect( organizationAuthorizationMockFns.mockAuthorizeOrganizationOperation ).toHaveBeenCalledWith( @@ -125,7 +119,6 @@ describe('personal live Search connection', () => { 'no longer available' ) expect(m.oauth).not.toHaveBeenCalled() - expect(m.indexed).not.toHaveBeenCalled() }) it('rechecks disabled account options after resolving the card', async () => { @@ -139,33 +132,4 @@ describe('personal live Search connection', () => { ) expect(m.oauth).not.toHaveBeenCalled() }) - - it('continues to use indexed enrollment for indexed controls', async () => { - const indexed = { - type: 'link', - provider: 'slack', - connectorType: 'slack', - connectorId: 'source', - } as const - m.resolve.mockResolvedValue({ target: indexed }) - m.indexed.mockResolvedValue({ - url: 'https://provider.test/oauth', - connectorId: 'source', - knowledgeBaseId: 'kb', - }) - await connectPersonalSearchIntegration.execute({ - principal, - input: { ...input, target: indexed }, - }) - expect(m.indexed).toHaveBeenCalledWith( - expect.objectContaining({ - principal, - input: expect.objectContaining({ - connectorId: 'source', - oauthCompletionId: input.oauthCompletionId, - }), - }) - ) - expect(m.oauth).not.toHaveBeenCalled() - }) }) diff --git a/apps/sim/lib/knowledge/application/connect-personal-search-integration.ts b/apps/sim/lib/knowledge/application/connect-personal-search-integration.ts index 8dbfaf8051a..bf24747149d 100644 --- a/apps/sim/lib/knowledge/application/connect-personal-search-integration.ts +++ b/apps/sim/lib/knowledge/application/connect-personal-search-integration.ts @@ -3,14 +3,12 @@ import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/au import { resolveKnowledgeOrganizationContext } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { resolvePersonalSearchConnection } from '@/lib/knowledge/application/personal-search-integrations' -import { connectSimSearchConnector } from '@/lib/knowledge/application/sim-search' import type { SearchConnectionTarget } from '@/lib/knowledge/search/connection-target' interface ConnectPersonalSearchIntegrationInput { organizationId: string target: SearchConnectionTarget oauthCompletionId: string - sourceConfig?: Record } /** A user click validates the requested control before starting the ordinary source enrollment. */ @@ -20,38 +18,20 @@ export const connectPersonalSearchIntegration = defineAuthorizedKnowledgeUseCase resolveKnowledgeOrganizationContext(input), async execute({ principal, input, request }) { const { target } = await resolvePersonalSearchConnection.execute({ principal, input }) - if (target.connectionMode === 'live' && target.optionId) { - const result = await startOrganizationAccountConnection.execute({ - principal, - request, - input: { - organizationId: input.organizationId, - optionId: target.optionId, - oauthCompletionId: input.oauthCompletionId, - connectionIntent: target.credentialId - ? { kind: 'reconnect', credentialId: target.credentialId } - : { kind: 'create' }, - }, - }) - return { - url: result.authorizationUrl ?? result.invitationLink, - connectorId: undefined, - knowledgeBaseId: undefined, - } - } - return connectSimSearchConnector.execute({ + const result = await startOrganizationAccountConnection.execute({ principal, request, input: { organizationId: input.organizationId, - connectorType: target.connectorType, - connectorId: target.connectorId, - sourceConfig: input.sourceConfig, + optionId: target.optionId, oauthCompletionId: input.oauthCompletionId, connectionIntent: target.credentialId ? { kind: 'reconnect', credentialId: target.credentialId } : { kind: 'create' }, }, }) + return { + url: result.authorizationUrl ?? result.invitationLink, + } }, }) diff --git a/apps/sim/lib/knowledge/application/connectors.test.ts b/apps/sim/lib/knowledge/application/connectors.test.ts index 0763cf0c38f..630893c5e51 100644 --- a/apps/sim/lib/knowledge/application/connectors.test.ts +++ b/apps/sim/lib/knowledge/application/connectors.test.ts @@ -17,10 +17,6 @@ import { knowledgeContextsMock, knowledgeContextsMockFns, } from '@sim/testing/mocks/knowledge-contexts.mock' -import { - knowledgeSearchIntegrationPolicyMock, - knowledgeSearchIntegrationPolicyMockFns, -} from '@sim/testing/mocks/knowledge-search-integration-policy.mock' import { permissionGroupsResolveMock, permissionGroupsResolveMockFns, @@ -44,8 +40,6 @@ const hoisted = vi.hoisted(() => ({ vi.mock('@sim/audit', () => auditMock) -vi.mock('@/lib/knowledge/search/integration-policy', () => knowledgeSearchIntegrationPolicyMock) - vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) vi.mock('@/lib/knowledge/application/contexts', () => knowledgeContextsMock) @@ -120,7 +114,6 @@ import { internalOrchestrationErrorPolicy } from '@/lib/api/server/routes/intern import { OrchestrationError } from '@/lib/core/orchestration/types' import * as encryption from '@/lib/core/security/encryption' import { - createApprovedSearchSource, createKnowledgeConnector, deleteKnowledgeConnector, resolveConnectorCredentialAccessToken, @@ -140,7 +133,6 @@ import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta' const mocks = { ...hoisted, - requireApproval: knowledgeSearchIntegrationPolicyMockFns.mockRequireOrganizationSearchApproval, getCredentialActorContext: credentialsAccessMockFns.mockGetCredentialActorContext, canUseCredential: credentialsAccessMockFns.mockCanUseCredential, resolveTokenIdentity: credentialsAccessMockFns.mockResolveCredentialTokenIdentity, @@ -937,89 +929,6 @@ describe('members-mode connector creation', () => { }) }) -describe('approved organization member source creation', () => { - const principal = createSessionPrincipal({ userId: 'actor', sessionId: 'session' }) - const input = { - knowledgeBaseId: 'org-index', - assertedOrganizationId: 'org', - connectorType: 'google_drive', - sourceConfig: {}, - } - beforeEach(() => { - resetDbChainMock() - queueTableRows(member, [{ role: 'member' }]) - permissionGroupsResolveMockFns.mockGetUserPermissionConfig.mockResolvedValue(null) - mocks.requireApproval.mockResolvedValue(undefined) - knowledgeContextsMockFns.mockResolveActiveKnowledgeBaseContext.mockResolvedValue({ - organizationId: 'org', - knowledgeBaseId: 'org-index', - knowledgeBase: { id: 'org-index', name: 'Search', isSearchIndex: true }, - }) - mocks.resolveMembersBinding.mockResolvedValue({ - organizationId: 'org', - credentialGroupId: 'group', - credentialGroupOptionId: 'option', - }) - mocks.createConnector.mockResolvedValue({ - success: true, - connector: { id: 'connector', connectorType: 'google_drive', accessMode: 'members' }, - }) - }) - - it('uses the member actor and ignores attempts to supply credentials or broader access', async () => { - const maliciousInput = { - ...input, - credentialId: 'other-person', - apiKey: 'injected', - accessMode: 'admin', - } - await createApprovedSearchSource.execute({ principal, input: maliciousInput }) - expect(mocks.requireApproval).toHaveBeenCalledWith('org', 'google_drive') - expect(mocks.resolveMembersBinding).toHaveBeenCalledWith( - expect.objectContaining({ actingUserId: 'actor', organizationId: 'org' }) - ) - expect(mocks.createConnector).toHaveBeenCalledWith( - expect.objectContaining({ - userId: 'actor', - accessMode: 'members', - credentialId: undefined, - apiKey: undefined, - }) - ) - }) - - it('refuses deactivated integrations before provisioning a credential group', async () => { - mocks.requireApproval.mockRejectedValue(new Error('Integration is deactivated')) - await expect(createApprovedSearchSource.execute({ principal, input })).rejects.toThrow( - 'deactivated' - ) - expect(mocks.resolveMembersBinding).not.toHaveBeenCalled() - expect(mocks.createConnector).not.toHaveBeenCalled() - }) - - it('refuses custom configuration outside the personal setup fields', async () => { - await expect( - createApprovedSearchSource.execute({ - principal, - input: { ...input, sourceConfig: { adminEmail: 'other-person@fixture.test' } }, - }) - ).rejects.toMatchObject({ code: 'validation' }) - expect(mocks.createConnector).not.toHaveBeenCalled() - }) - - it('refuses a knowledge base that is not the organization Search index', async () => { - knowledgeContextsMockFns.mockResolveActiveKnowledgeBaseContext.mockResolvedValue({ - organizationId: 'org', - knowledgeBaseId: 'org-index', - knowledgeBase: { isSearchIndex: false }, - }) - await expect(createApprovedSearchSource.execute({ principal, input })).rejects.toMatchObject({ - code: 'forbidden', - }) - expect(mocks.createConnector).not.toHaveBeenCalled() - }) -}) - describe('organization connector credential authorization', () => { const principal = createSessionPrincipal({ userId: 'org-admin', sessionId: 'session' }) const credential = { diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index c9ef81eb794..fe3be8719f5 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -105,19 +105,12 @@ import type { KnowledgeOperationSource, KnowledgeOrchestrationResult, } from '@/lib/knowledge/orchestration/shared' -import { requireOrganizationSearchApproval } from '@/lib/knowledge/search/integration-policy' import { escapeLikePattern } from '@/lib/knowledge/tags/utils' import { credentialProviderMatchesService, type ServiceProviderIdentity } from '@/lib/oauth' import { ServiceAccountTokenError } from '@/lib/oauth/credential-service' import { CAPABILITY_RULES, refuseCapability } from '@/lib/permission-groups/capabilities' import { resolvePermissionGroupConfig } from '@/lib/permission-groups/config-scope.server' import { getUserPermissionConfigForOrganization } from '@/lib/permission-groups/resolve.server' -import { - canConnectPersonally, - personalSourceConfigFieldIds, - withSearchSourceDefaults, -} from '@/lib/sim-search/connectors' -import { SIM_SEARCH_SYNC_INTERVAL_MINUTES } from '@/lib/sim-search/constants' import { getConnectorApiKeyConfig, isConnectorCredentialTypeAllowed } from '@/connectors/auth' import { getConnectorMeta } from '@/connectors/registry' import type { ConnectorAuthConfig } from '@/connectors/types' @@ -698,20 +691,17 @@ async function resolveConnectorApiKey( return value } -async function executeCreateKnowledgeConnector( - { - principal, - input, - context, - request, - }: { - principal: Principal - input: CreateKnowledgeConnectorInput - context: ActiveKnowledgeResourceBaseContext - request?: OrchestrationRequestContext - }, - approvedMemberSetup = false -) { +async function executeCreateKnowledgeConnector({ + principal, + input, + context, + request, +}: { + principal: Principal + input: CreateKnowledgeConnectorInput + context: ActiveKnowledgeResourceBaseContext + request?: OrchestrationRequestContext +}) { const requestId = generateRequestId() const scope = resourceScopeFromOwner(context) const owner = resourceScopeFields(scope) @@ -769,9 +759,7 @@ async function executeCreateKnowledgeConnector( if (!subjectUserId) { throw new OrchestrationError('forbidden', 'Permission-scoped access needs a signed-in admin') } - if (approvedMemberSetup && context.organizationId) { - await requireOrganizationSearchApproval(context.organizationId, input.connectorType) - } else if (context.organizationId) + if (context.organizationId) await requireOrganizationMembership( principal, context.organizationId, @@ -899,76 +887,6 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ }, }) -/** Members may create only a personal-account source in an approved organization index. */ -export const createApprovedSearchSource = defineAuthorizedKnowledgeUseCase({ - operation: knowledgeOperations.createApprovedSearchSource, - resolveContext: ({ - principal, - input, - }: { - principal: Principal - input: { - knowledgeBaseId: string - assertedOrganizationId: string - connectorType: string - sourceConfig: Record - } - }) => resolveActiveKnowledgeResourceContext(input, principal), - async execute({ principal, input, context, request }) { - const meta = getConnectorMeta(input.connectorType) - if ( - !context.organizationId || - !context.knowledgeBase.isSearchIndex || - !meta || - !canConnectPersonally(meta) - ) { - throw new OrchestrationError( - 'forbidden', - 'Only approved personal Search sources may be connected' - ) - } - const allowedFields = personalSourceConfigFieldIds(meta) - if (Object.keys(input.sourceConfig).some((field) => !allowedFields.has(field))) { - throw new OrchestrationError( - 'validation', - 'Only the personal connection settings may be supplied' - ) - } - return executeCreateKnowledgeConnector( - { - principal, - context, - request, - input: { - knowledgeBaseId: input.knowledgeBaseId, - assertedOrganizationId: input.assertedOrganizationId, - connectorType: input.connectorType, - sourceConfig: withSearchSourceDefaults(meta, input.sourceConfig), - accessMode: 'members', - syncIntervalMinutes: SIM_SEARCH_SYNC_INTERVAL_MINUTES, - reuseSearchSource: true, - source: 'ui', - }, - }, - true - ) - }, - projectAudit: ({ result, context }) => - result.reused - ? [] - : { - action: AuditAction.CONNECTOR_CREATED, - resourceType: AuditResourceType.CONNECTOR, - resourceId: result.connector.id, - resourceName: result.connector.connectorType, - metadata: { - knowledgeBaseId: context.knowledgeBaseId, - accessMode: 'members', - approvedMemberSetup: true, - }, - }, -}) - /** * Deliberately not gated by `knowledge.connectors`: an update may change the * source config, sync interval or status, never the connector type. The diff --git a/apps/sim/lib/knowledge/application/operations.ts b/apps/sim/lib/knowledge/application/operations.ts index c341f2cb6be..c8f1e7768d6 100644 --- a/apps/sim/lib/knowledge/application/operations.ts +++ b/apps/sim/lib/knowledge/application/operations.ts @@ -767,25 +767,6 @@ export const knowledgeOperations = { delegatedServices: ['copilot'], }) ), - readSearchSourceOverview: defineKnowledgeOperation( - defineWorkspaceOperation({ - id: 'knowledge.search.sources.overview', - minimumRole: 'read', - workspaceApiKey: 'deny', - capability: 'knowledge.use', - principalKinds: ['session', 'delegated'], - delegatedServices: ['copilot'], - }) - ), - readSearchSourceProgress: defineKnowledgeOperation( - defineWorkspaceOperation({ - id: 'knowledge.search.sources.progress', - minimumRole: 'read', - workspaceApiKey: 'deny', - capability: 'knowledge.use', - principalKinds: ['session'], - }) - ), listSearchIntegrations: defineKnowledgeOperation( defineWorkspaceOperation({ id: 'knowledge.search.integrations.list', @@ -796,24 +777,6 @@ export const knowledgeOperations = { }), { organizationDelegation: 'allow' } ), - readOrganizationSearchOverview: defineKnowledgeOperation( - defineWorkspaceOperation({ - id: 'knowledge.search.integrations.overview', - minimumRole: 'admin', - workspaceApiKey: 'deny', - capability: 'knowledge.use', - principalKinds: ['session'], - }) - ), - readOrganizationSearchStats: defineKnowledgeOperation( - defineWorkspaceOperation({ - id: 'knowledge.search.stats.read', - minimumRole: 'admin', - workspaceApiKey: 'deny', - capability: 'knowledge.use', - principalKinds: ['session'], - }) - ), approveSearchIntegration: defineKnowledgeOperation( defineWorkspaceOperation({ id: 'knowledge.search.integrations.approve', @@ -837,48 +800,6 @@ export const knowledgeOperations = { principalKinds: ['session'], }) ), - /** - * Connecting a Sim Search source: any reader may connect their own account. - * The first connect of a source also creates its knowledge base and - * connector, which the use case reserves for an admin and refuses to anyone - * else with the way forward (ask an admin to connect the source first). - */ - simSearchConnect: defineKnowledgeOperation( - defineWorkspaceOperation({ - id: 'knowledge.simSearch.connect', - minimumRole: 'read', - workspaceApiKey: 'deny', - capability: 'knowledge.use', - principalKinds: ['session'], - }) - ), - listPersonalSourceSetupAccounts: defineKnowledgeOperation( - defineWorkspaceOperation({ - id: 'knowledge.search.personalSetup.accounts.list', - minimumRole: 'read', - workspaceApiKey: 'deny', - capability: 'knowledge.use', - principalKinds: ['session'], - }) - ), - personalSourceSetup: defineKnowledgeOperation( - defineWorkspaceOperation({ - id: 'knowledge.search.personalSetup', - minimumRole: 'read', - workspaceApiKey: 'deny', - capability: 'knowledge.use', - principalKinds: ['session'], - }) - ), - createApprovedSearchSource: defineKnowledgeOperation( - defineWorkspaceOperation({ - id: 'knowledge.search.sources.connectApproved', - minimumRole: 'read', - workspaceApiKey: 'deny', - capability: 'knowledge.use', - principalKinds: ['session'], - }) - ), readSearchIndex: defineKnowledgeOperation( defineWorkspaceOperation({ id: 'knowledge.search.index.read', diff --git a/apps/sim/lib/knowledge/application/organization-search-overview.test.ts b/apps/sim/lib/knowledge/application/organization-search-overview.test.ts deleted file mode 100644 index bdf895c32cb..00000000000 --- a/apps/sim/lib/knowledge/application/organization-search-overview.test.ts +++ /dev/null @@ -1,181 +0,0 @@ -import { knowledgeConnector, member, organizationSearchIntegration } from '@sim/db/schema' -import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' -import { - createSessionPrincipal, - createWorkspaceApiKeyPrincipal, -} from '@sim/testing/factories/principal.factory' -import { - knowledgeAvailabilityMock, - knowledgeAvailabilityMockFns, -} from '@sim/testing/mocks/knowledge-availability.mock' -import { - knowledgeContextsMock, - knowledgeContextsMockFns, -} from '@sim/testing/mocks/knowledge-contexts.mock' -import { - permissionGroupsResolveMock, - permissionGroupsResolveMockFns, -} from '@sim/testing/mocks/permission-groups-resolve.mock' -import { workspaceAuthzMock } from '@sim/testing/mocks/workspace-authz.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/knowledge/application/contexts', () => knowledgeContextsMock) -vi.mock('@/lib/permission-groups/resolve.server', () => permissionGroupsResolveMock) -vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) -vi.mock('@/lib/knowledge/access/availability', () => knowledgeAvailabilityMock) -vi.mock('@/lib/sim-search/connectors', () => ({ - canConnectWithDefaults: (meta: { id: string }) => ['google_drive', 'gmail'].includes(meta.id), - SEARCH_SOURCE_TYPES: [ - ['google_drive', { id: 'google_drive', mirrorsSourceAcls: true, permissionScopedListing: {} }], - ['gmail', { id: 'gmail', permissionScopedListing: {} }], - ['github', { id: 'github', permissionScopedListing: {} }], - ['gitlab', { id: 'gitlab', mirrorsSourceAcls: true }], - ], -})) - -import { readOrganizationSearchOverview } from '@/lib/knowledge/application/organization-search-overview' -import { SOURCE_CONTENT_ERROR } from '@/lib/knowledge/connectors/sync-limits' - -/** The global drizzle mock nests fragments as params; flatten one for inspection. */ -function renderFragment(fragment: unknown): { sql: string; params: unknown[] } { - if (!fragment || typeof fragment !== 'object') return { sql: '', params: [] } - if ('conditions' in fragment && Array.isArray(fragment.conditions)) { - const parts = fragment.conditions.map(renderFragment) - return { - sql: parts.map((part) => part.sql).join(' '), - params: parts.flatMap((part) => part.params), - } - } - const rendered = (fragment as { toSQL?: () => { sql: string; params: unknown[] } }).toSQL?.() - if (!rendered) return { sql: '', params: [] } - const params: unknown[] = [] - let sqlText = rendered.sql - for (const param of rendered.params) { - if (param && typeof param === 'object' && 'toSQL' in param) { - const nested = renderFragment(param) - sqlText += ` ${nested.sql}` - params.push(...nested.params) - } else params.push(param) - } - return { sql: sqlText, params } -} - -const principal = createSessionPrincipal({ userId: 'admin', sessionId: 'session' }) -const input = { organizationId: 'organization' } -const health = { - connectorType: 'google_drive', - sourceCount: 4, - pausedCount: 0, - hasError: false, - hasAccountError: false, - hasDocumentError: false, - hasPermissionError: false, - hasIndexing: false, - hasPendingSync: false, - hasWaiting: false, - hasUnstarted: false, -} - -beforeEach(() => { - resetDbChainMock() - knowledgeContextsMockFns.mockResolveKnowledgeOwnerContext.mockResolvedValue(input) - permissionGroupsResolveMockFns.mockGetUserPermissionConfigForOrganization.mockResolvedValue(null) - knowledgeAvailabilityMockFns.mockResolveKnowledgeAccessAvailability.mockResolvedValue({ - memberScoped: true, - sourceMirrored: true, - }) -}) - -describe('organization Search administration overview', () => { - it.each([ - { - hasPermissionError: true, - hasAccountError: false, - hasDocumentError: false, - issue: 'permission_sync_incomplete', - }, - { hasAccountError: true, hasDocumentError: false, issue: 'account_sync_incomplete' }, - { hasAccountError: false, hasDocumentError: true, issue: 'document_indexing_failed' }, - { hasAccountError: false, hasDocumentError: false, issue: 'sync_failed' }, - ])('identifies $issue without exposing error details', async ({ issue, ...errors }) => { - queueTableRows(member, [{ role: 'admin' }]) - queueTableRows(knowledgeConnector, [ - { ...health, ...errors, hasError: true, rawError: 'private provider response' }, - ]) - const result = await readOrganizationSearchOverview.execute({ principal, input }) - expect(result.providers[0]).toMatchObject({ status: 'needs_attention', issue }) - expect(JSON.stringify(result)).not.toContain('private provider response') - }) - it('does not count the per-document relisting marker as a member account error', async () => { - queueTableRows(member, [{ role: 'admin' }]) - queueTableRows(knowledgeConnector, [{ ...health }]) - await readOrganizationSearchOverview.execute({ principal, input }) - const rendered = dbChainMockFns.where.mock.calls.flatMap((call) => call.map(renderFragment)) - const memberErrorClause = rendered.find((fragment) => fragment.sql.includes("'suspended'")) - expect(memberErrorClause).toBeDefined() - expect(memberErrorClause?.sql).toContain('IS NOT NULL AND ? <> ?') - expect(memberErrorClause?.params).toContain(SOURCE_CONTENT_ERROR) - }) - it.each([ - { rows: [{ role: 'member' }], code: 'forbidden' }, - { rows: [], code: 'not_found' }, - ])('refuses unauthorized reads with $code before health queries', async ({ rows, code }) => { - queueTableRows(member, rows) - await expect( - readOrganizationSearchOverview.execute({ principal, input }) - ).rejects.toMatchObject({ code }) - expect(dbChainMockFns.from).not.toHaveBeenCalledWith(knowledgeConnector) - expect(dbChainMockFns.from).not.toHaveBeenCalledWith(organizationSearchIntegration) - }) - it('rejects a workspace key before canonical loading', async () => { - await expect( - readOrganizationSearchOverview.execute({ - principal: createWorkspaceApiKeyPrincipal({ workspaceId: 'workspace', keyId: 'key' }), - input, - }) - ).rejects.toThrow() - expect(knowledgeContextsMockFns.mockResolveKnowledgeOwnerContext).not.toHaveBeenCalled() - }) - it('does not mistake infrastructure failure for an empty integration list', async () => { - queueTableRows(member, [{ role: 'admin' }]) - permissionGroupsResolveMockFns.mockGetUserPermissionConfigForOrganization.mockRejectedValue( - new Error('Database unavailable') - ) - await expect(readOrganizationSearchOverview.execute({ principal, input })).rejects.toThrow( - 'Database unavailable' - ) - }) - it('does not report indexing when the owner-scoped Search gate is disabled', async () => { - queueTableRows(member, [{ role: 'admin' }]) - queueTableRows(knowledgeConnector, [{ ...health, hasIndexing: true }]) - queueTableRows(organizationSearchIntegration, [{ connectorType: 'gmail', approved: true }]) - knowledgeAvailabilityMockFns.mockResolveKnowledgeAccessAvailability.mockResolvedValue({ - memberScoped: false, - sourceMirrored: false, - }) - const result = await readOrganizationSearchOverview.execute({ principal, input }) - expect( - knowledgeAvailabilityMockFns.mockResolveKnowledgeAccessAvailability - ).toHaveBeenCalledWith(input) - expect(result.providers).toEqual([ - { - connectorType: 'google_drive', - sourceCount: 4, - approved: true, - status: 'paused', - issue: null, - isSyncing: false, - hasPendingSync: false, - }, - { - connectorType: 'gmail', - sourceCount: 0, - approved: true, - status: 'paused', - issue: null, - isSyncing: false, - hasPendingSync: false, - }, - ]) - }) -}) diff --git a/apps/sim/lib/knowledge/application/organization-search-overview.ts b/apps/sim/lib/knowledge/application/organization-search-overview.ts deleted file mode 100644 index f1d0b114158..00000000000 --- a/apps/sim/lib/knowledge/application/organization-search-overview.ts +++ /dev/null @@ -1,307 +0,0 @@ -import { db } from '@sim/db' -import { - document, - knowledgeBase, - knowledgeConnector, - knowledgeConnectorMember, - knowledgeConnectorMemberSyncLog, - knowledgeConnectorSyncLog, - organizationSearchIntegration, -} from '@sim/db/schema' -import { and, eq, exists, inArray, isNotNull, isNull, type SQL, sql } from 'drizzle-orm' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { resolveKnowledgeAccessAvailability } from '@/lib/knowledge/access/availability' -import { sourceAclFreshnessCutoff } from '@/lib/knowledge/access/predicate' -import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' -import { resolveKnowledgeOwnerContext } from '@/lib/knowledge/application/contexts' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { - SOURCE_CONTENT_ERROR, - SOURCE_PERMISSION_ERROR, -} from '@/lib/knowledge/connectors/sync-limits' -import { MAX_SEARCH_SOURCE_PROVIDER_TYPES } from '@/lib/knowledge/constants' -import { failedDocumentCondition } from '@/lib/knowledge/documents/processing-status' -import { canConnectWithDefaults, SEARCH_SOURCE_TYPES } from '@/lib/sim-search/connectors' - -interface OrganizationSearchOverviewInput { - organizationId: string -} - -interface ProviderHealth { - sourceCount: number - pausedCount: number - hasError: boolean - hasAccountError: boolean - hasDocumentError: boolean - hasPermissionError: boolean - hasIndexing: boolean - hasPendingSync: boolean - hasWaiting: boolean - hasUnstarted: boolean -} - -/** A successful empty crawl is active; neither this state nor its count describes readable documents. */ -function organizationSearchProviderStatus( - health: ProviderHealth | undefined, - approved: boolean, - automaticSetup: boolean, - available: boolean -) { - if (!approved || !available) return 'paused' as const - if (!health?.sourceCount) - return automaticSetup ? ('waiting_for_connections' as const) : ('needs_setup' as const) - if (health.pausedCount === health.sourceCount) return 'paused' as const - if (health.hasError) return 'needs_attention' as const - if (health.hasIndexing) return 'indexing' as const - if (health.hasWaiting) return 'waiting_for_connections' as const - if (health.hasUnstarted) return 'needs_setup' as const - return 'active' as const -} - -/** Organization admins receive aggregate operational facts, never documents, account identities or raw errors. */ -export const readOrganizationSearchOverview = defineAuthorizedKnowledgeUseCase({ - operation: knowledgeOperations.readOrganizationSearchOverview, - resolveContext: ({ input }: { input: OrganizationSearchOverviewInput }) => - resolveKnowledgeOwnerContext({ organizationId: input.organizationId }), - async execute({ context }) { - if (!context.organizationId) - throw new OrchestrationError('validation', 'Organization is required') - const providerTypes = SEARCH_SOURCE_TYPES.map(([connectorType]) => connectorType) - if (providerTypes.length > MAX_SEARCH_SOURCE_PROVIDER_TYPES) { - throw new Error('Search provider catalog exceeds the overview bound') - } - const availability = await resolveKnowledgeAccessAvailability(context) - const identityRequiredTypes = SEARCH_SOURCE_TYPES.filter( - ([, meta]) => meta.requiresMemberIdentity - ).map(([connectorType]) => connectorType) - - const hasActiveMembers = exists( - db - .select({ id: knowledgeConnectorMember.id }) - .from(knowledgeConnectorMember) - .where( - and( - eq(knowledgeConnectorMember.connectorId, knowledgeConnector.id), - eq(knowledgeConnectorMember.status, 'active') - ) - ) - ) - const hasMemberContinuation = exists( - db - .select({ id: knowledgeConnectorMember.id }) - .from(knowledgeConnectorMember) - .where( - and( - eq(knowledgeConnectorMember.connectorId, knowledgeConnector.id), - eq(knowledgeConnectorMember.status, 'active'), - isNotNull(knowledgeConnectorMember.listingCheckpoint) - ) - ) - ) - const continuing = sql`( - ${knowledgeConnector.listingCheckpoint} IS NOT NULL - OR (${knowledgeConnector.accessMode} = 'members' AND ( - ${knowledgeConnector.directoryCheckpoint} IS NOT NULL - OR ${hasMemberContinuation} - OR coalesce(${knowledgeConnector.nextMemberSyncAt} <= statement_timestamp(), false) - )) - )` - const cutoff = sourceAclFreshnessCutoff() - /** - * A member whose last run only had per-document content failures carries - * {@link SOURCE_CONTENT_ERROR} as a marker so its next run lists fully; the - * member itself is healthy and the documents are reported separately. - */ - const hasMemberError = exists( - db - .select({ id: knowledgeConnectorMember.id }) - .from(knowledgeConnectorMember) - .where( - and( - eq(knowledgeConnectorMember.connectorId, knowledgeConnector.id), - sql`( - ${knowledgeConnectorMember.status} = 'suspended' - OR (${knowledgeConnectorMember.status} = 'active' AND ( - (${knowledgeConnectorMember.lastError} IS NOT NULL AND ${knowledgeConnectorMember.lastError} <> ${SOURCE_CONTENT_ERROR}) - OR ${knowledgeConnectorMember.consecutiveFailures} > 0 - OR coalesce(${knowledgeConnectorMember.memberSyncedThrough}, ${knowledgeConnectorMember.lastCompleteListingAt}, ${knowledgeConnectorMember.createdAt}) < ${cutoff} - )) - )` - ) - ) - ) - const hasMemberFirstListing = exists( - db - .select({ id: knowledgeConnectorMember.id }) - .from(knowledgeConnectorMember) - .where( - and( - eq(knowledgeConnectorMember.connectorId, knowledgeConnector.id), - eq(knowledgeConnectorMember.status, 'active'), - isNull(knowledgeConnectorMember.lastCompleteListingAt) - ) - ) - ) - const hasDocumentsInState = (condition: SQL) => - exists( - db - .select({ id: document.id }) - .from(document) - .where( - and( - eq(document.connectorId, knowledgeConnector.id), - eq(document.knowledgeBaseId, knowledgeConnector.knowledgeBaseId), - eq(document.enabled, true), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - condition - ) - ) - ) - /** Partial runs can be normal continuations; completed timestamps alone do not prove a complete crawl. */ - const latestMemberRunHasError = sql`coalesce(( - SELECT ${knowledgeConnectorMemberSyncLog.status} = 'failed' - OR (${knowledgeConnectorMemberSyncLog.status} = 'partial' AND ( - ${knowledgeConnectorMemberSyncLog.membersFailed} > 0 - OR ${knowledgeConnectorMemberSyncLog.docsFailed} > 0 - OR ${knowledgeConnectorMemberSyncLog.processingDispatchFailed} > 0 - OR NOT ${continuing} - )) - FROM ${knowledgeConnectorMemberSyncLog} - WHERE ${knowledgeConnectorMemberSyncLog.connectorId} = ${knowledgeConnector.id} - AND ${knowledgeConnectorMemberSyncLog.status} <> 'started' - ORDER BY ${knowledgeConnectorMemberSyncLog.startedAt} DESC, ${knowledgeConnectorMemberSyncLog.id} DESC - LIMIT 1 - ), false)` - const latestCentralRunHasError = sql`coalesce(( - SELECT ${knowledgeConnectorSyncLog.status} = 'failed' - OR (${knowledgeConnectorSyncLog.status} = 'partial' AND ( - ${knowledgeConnectorSyncLog.docsFailed} > 0 OR NOT ${continuing} - )) - FROM ${knowledgeConnectorSyncLog} - WHERE ${knowledgeConnectorSyncLog.connectorId} = ${knowledgeConnector.id} - AND ${knowledgeConnectorSyncLog.status} <> 'started' - ORDER BY ${knowledgeConnectorSyncLog.startedAt} DESC, ${knowledgeConnectorSyncLog.id} DESC - LIMIT 1 - ), false)` - const paused = sql`( - ${knowledgeConnector.status} IN ('paused', 'disabled') - OR (${knowledgeConnector.accessMode} = 'members' AND ${knowledgeConnector.memberSyncStatus} = 'disabled') - OR (${knowledgeConnector.accessMode} = 'members' AND ${!availability.memberScoped}) - OR (${knowledgeConnector.accessMode} = 'admin' AND ( - ${!availability.sourceMirrored} - OR (${!availability.memberScoped} AND ${inArray(knowledgeConnector.connectorType, identityRequiredTypes)}) - )) - )` - const [health, decisions] = await Promise.all([ - db - .select({ - connectorType: knowledgeConnector.connectorType, - sourceCount: sql`count(*)::int`, - pausedCount: sql`count(*) FILTER (WHERE ${paused})::int`, - hasError: sql`bool_or(NOT ${paused} AND ( - ${knowledgeConnector.status} = 'error' - OR ${knowledgeConnector.lastSyncError} IS NOT NULL - OR ${hasDocumentsInState(failedDocumentCondition())} - OR (${knowledgeConnector.accessMode} = 'admin' AND ${latestCentralRunHasError}) - OR (${knowledgeConnector.accessMode} = 'members' AND ( - ${knowledgeConnector.memberSyncStatus} = 'error' - OR ${knowledgeConnector.lastMemberSyncError} IS NOT NULL - OR ${hasMemberError} OR ${latestMemberRunHasError} - )) - ))`, - hasAccountError: sql`bool_or(NOT ${paused} AND ${knowledgeConnector.accessMode} = 'members' AND ${hasMemberError})`, - hasDocumentError: sql`bool_or(NOT ${paused} AND ${hasDocumentsInState(failedDocumentCondition())})`, - hasPermissionError: sql`bool_or(NOT ${paused} AND ${SOURCE_PERMISSION_ERROR} = ANY(string_to_array(${knowledgeConnector.lastSyncError}, ${'\n'})))`, - hasIndexing: sql`bool_or(NOT ${paused} - AND (${knowledgeConnector.accessMode} <> 'members' OR ${hasActiveMembers} OR ${knowledgeConnector.credentialId} IS NOT NULL) - AND ( - ${knowledgeConnector.status} IN ('pending', 'syncing') - OR ${hasDocumentsInState(inArray(document.processingStatus, ['pending', 'processing']))} - OR (${knowledgeConnector.accessMode} = 'members' AND ( - ${knowledgeConnector.memberSyncStatus} IN ('pending', 'running') - )) - ))`, - hasWaiting: sql`bool_or(NOT ${paused} AND ${knowledgeConnector.accessMode} = 'members' AND NOT ${hasActiveMembers})`, - hasPendingSync: sql`bool_or(NOT ${paused} AND ( - ${continuing} OR (${knowledgeConnector.accessMode} = 'members' AND ${hasMemberFirstListing}) - ))`, - hasUnstarted: sql`bool_or(NOT ${paused} AND ( - (${knowledgeConnector.accessMode} = 'admin' AND ${knowledgeConnector.lastSyncAt} IS NULL) - OR (${knowledgeConnector.accessMode} = 'members' AND ${hasMemberFirstListing}) - ))`, - }) - .from(knowledgeConnector) - .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) - .where( - and( - eq(knowledgeBase.organizationId, context.organizationId), - eq(knowledgeBase.isSearchIndex, true), - isNull(knowledgeBase.deletedAt), - inArray(knowledgeConnector.connectorType, providerTypes), - inArray(knowledgeConnector.accessMode, ['admin', 'members']), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .groupBy(knowledgeConnector.connectorType) - .limit(MAX_SEARCH_SOURCE_PROVIDER_TYPES), - db - .select({ - connectorType: organizationSearchIntegration.connectorType, - approved: organizationSearchIntegration.approved, - }) - .from(organizationSearchIntegration) - .where( - and( - eq(organizationSearchIntegration.organizationId, context.organizationId), - inArray(organizationSearchIntegration.connectorType, providerTypes) - ) - ) - .limit(MAX_SEARCH_SOURCE_PROVIDER_TYPES), - ]) - const healthByType = new Map(health.map((provider) => [provider.connectorType, provider])) - const approvals = new Map( - decisions.map((decision) => [decision.connectorType, decision.approved]) - ) - return { - providers: SEARCH_SOURCE_TYPES.flatMap(([connectorType, meta]) => { - const state = healthByType.get(connectorType) - if (!state && !approvals.has(connectorType)) return [] - const approved = approvals.get(connectorType) ?? Boolean(state?.sourceCount) - const status = organizationSearchProviderStatus( - state, - approved, - canConnectWithDefaults(meta) && availability.memberScoped, - Boolean( - (meta.permissionScopedListing && availability.memberScoped) || - (meta.mirrorsSourceAcls && - availability.sourceMirrored && - (!meta.requiresMemberIdentity || availability.memberScoped)) - ) - ) - return [ - { - connectorType, - approved, - sourceCount: state?.sourceCount ?? 0, - status, - issue: - status === 'needs_attention' - ? state?.hasAccountError - ? ('account_sync_incomplete' as const) - : state?.hasPermissionError - ? ('permission_sync_incomplete' as const) - : state?.hasDocumentError - ? ('document_indexing_failed' as const) - : ('sync_failed' as const) - : null, - isSyncing: status !== 'paused' && Boolean(state?.hasIndexing), - hasPendingSync: status !== 'paused' && Boolean(state?.hasPendingSync), - }, - ] - }), - } - }, -}) diff --git a/apps/sim/lib/knowledge/application/organization-search-stats.test.ts b/apps/sim/lib/knowledge/application/organization-search-stats.test.ts deleted file mode 100644 index 050cf7e7bf3..00000000000 --- a/apps/sim/lib/knowledge/application/organization-search-stats.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { member } from '@sim/db/schema' -import { queueTableRows, resetDbChainMock } from '@sim/testing' -import { - createPersonalApiKeyPrincipal, - createSessionPrincipal, -} from '@sim/testing/factories/principal.factory' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' -import { - knowledgeAvailabilityMock, - knowledgeAvailabilityMockFns, -} from '@sim/testing/mocks/knowledge-availability.mock' -import { - knowledgeContextsMock, - knowledgeContextsMockFns, -} from '@sim/testing/mocks/knowledge-contexts.mock' -import { - permissionGroupsResolveMock, - permissionGroupsResolveMockFns, -} from '@sim/testing/mocks/permission-groups-resolve.mock' -import { workspaceAuthzMock } from '@sim/testing/mocks/workspace-authz.mock' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const mocks = vi.hoisted(() => ({ - load: vi.fn(), -})) -vi.mock('@/lib/knowledge/application/contexts', () => knowledgeContextsMock) -vi.mock('@/lib/permission-groups/resolve.server', () => permissionGroupsResolveMock) -vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) -vi.mock('@/lib/knowledge/access/availability', () => knowledgeAvailabilityMock) -vi.mock('@/lib/knowledge/search/activity-stats', () => ({ - loadOrganizationSearchStats: mocks.load, -})) - -import { readOrganizationSearchStats } from '@/lib/knowledge/application/organization-search-stats' -import { SearchIndexDormantError } from '@/lib/sim-search/indexed/gate' - -const principal = createSessionPrincipal({ userId: 'admin', sessionId: 'session' }) -const input = { organizationId: 'organization', period: '7d', surface: 'mcp' } as const - -afterEach(resetEnvFlagsMock) - -beforeEach(() => { - resetDbChainMock() - /** The Stats tab reports indexed organization search, so these cases run with it on. */ - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) - knowledgeContextsMockFns.mockResolveKnowledgeOwnerContext.mockResolvedValue({ - organizationId: 'organization', - }) - permissionGroupsResolveMockFns.mockGetUserPermissionConfigForOrganization.mockResolvedValue(null) - knowledgeAvailabilityMockFns.mockRequireOrganizationSearchAvailable.mockResolvedValue(undefined) - mocks.load.mockResolvedValue({ totals: { invocations: 3 } }) -}) - -describe('organization Search stats authorization', () => { - it.each([ - { rows: [{ role: 'member' }], code: 'forbidden' }, - { rows: [], code: 'not_found' }, - ])('rejects unauthorized access with $code before aggregation', async ({ rows, code }) => { - queueTableRows(member, rows) - await expect(readOrganizationSearchStats.execute({ principal, input })).rejects.toMatchObject({ - code, - }) - expect(mocks.load).not.toHaveBeenCalled() - }) - it('rejects API keys before protected loading', async () => { - await expect( - readOrganizationSearchStats.execute({ - principal: createPersonalApiKeyPrincipal({ userId: 'admin', keyId: 'key' }), - input, - }) - ).rejects.toThrow() - expect(knowledgeContextsMockFns.mockResolveKnowledgeOwnerContext).not.toHaveBeenCalled() - expect(mocks.load).not.toHaveBeenCalled() - }) - it('refuses while indexed organization search is dormant, before aggregation', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) - queueTableRows(member, [{ role: 'admin' }]) - await expect(readOrganizationSearchStats.execute({ principal, input })).rejects.toBeInstanceOf( - SearchIndexDormantError - ) - expect(mocks.load).not.toHaveBeenCalled() - }) - - it('fails closed when Search is disabled', async () => { - queueTableRows(member, [{ role: 'admin' }]) - knowledgeAvailabilityMockFns.mockRequireOrganizationSearchAvailable.mockRejectedValueOnce( - new Error('Search is disabled') - ) - await expect(readOrganizationSearchStats.execute({ principal, input })).rejects.toThrow( - 'Search is disabled' - ) - expect(mocks.load).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/knowledge/application/organization-search-stats.ts b/apps/sim/lib/knowledge/application/organization-search-stats.ts deleted file mode 100644 index 6b79404d65b..00000000000 --- a/apps/sim/lib/knowledge/application/organization-search-stats.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { requireOrganizationSearchAvailable } from '@/lib/knowledge/access/availability' -import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' -import { resolveKnowledgeOwnerContext } from '@/lib/knowledge/application/contexts' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { - loadOrganizationSearchStats, - type SearchStatsInput, -} from '@/lib/knowledge/search/activity-stats' -import { assertIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' - -/** The indexed Stats tab's activity report; refused while indexed organization search is dormant. */ -export const readOrganizationSearchStats = defineAuthorizedKnowledgeUseCase({ - operation: knowledgeOperations.readOrganizationSearchStats, - resolveContext: ({ input }: { input: SearchStatsInput }) => - resolveKnowledgeOwnerContext({ organizationId: input.organizationId }), - async execute({ context, input }) { - assertIndexedOrgSearchEnabled() - if (!context.organizationId) - throw new OrchestrationError('validation', 'Organization is required') - await requireOrganizationSearchAvailable(context.organizationId) - return loadOrganizationSearchStats({ ...input, organizationId: context.organizationId }) - }, -}) diff --git a/apps/sim/lib/knowledge/application/personal-search-account.test.ts b/apps/sim/lib/knowledge/application/personal-search-account.test.ts deleted file mode 100644 index cfbcd2aa52d..00000000000 --- a/apps/sim/lib/knowledge/application/personal-search-account.test.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { user } from '@sim/db/schema' -import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' -import { - createPersonalApiKeyPrincipal, - createSessionPrincipal, -} from '@sim/testing/factories/principal.factory' -import { - integrationsAvailabilityMock, - integrationsAvailabilityMockFns, -} from '@sim/testing/mocks/integrations-availability.mock' -import { knowledgeAvailabilityMock } from '@sim/testing/mocks/knowledge-availability.mock' -import { - knowledgeSearchIntegrationPolicyMock, - knowledgeSearchIntegrationPolicyMockFns, -} from '@sim/testing/mocks/knowledge-search-integration-policy.mock' -import { - organizationAuthorizationMock, - organizationAuthorizationMockFns, -} from '@sim/testing/mocks/organization-authorization.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const hoisted = vi.hoisted(() => ({ - accounts: vi.fn(), -})) -vi.mock('@/lib/core/application/organization-authorization', () => organizationAuthorizationMock) -vi.mock('@/lib/knowledge/search/integration-policy', () => knowledgeSearchIntegrationPolicyMock) -vi.mock('@/lib/knowledge/access/availability', () => knowledgeAvailabilityMock) -vi.mock('@/lib/integrations/availability.server', () => integrationsAvailabilityMock) -vi.mock('@/lib/credentials/organization-managed', () => ({ - getOwnOrganizationManagedOAuthCredentials: hoisted.accounts, -})) - -import { - authorizePersonalSearchSetup, - authorizePersonalSearchSetupCredential, -} from '@/lib/knowledge/application/personal-search-account' - -const mocks = { - ...hoisted, - approval: knowledgeSearchIntegrationPolicyMockFns.mockRequireOrganizationSearchApproval, - deployed: integrationsAvailabilityMockFns.mockIsOAuthServiceDeploymentAvailable, -} - -const principal = createSessionPrincipal({ userId: 'member-1' }) -const input = { - organizationId: 'organization-1', - connectorType: 'jira', - credentialId: 'own-account', -} as const - -describe('personal Search setup authorization', () => { - beforeEach(() => { - vi.resetAllMocks() - resetDbChainMock() - queueTableRows(user, [{ emailVerified: true }]) - mocks.deployed.mockReturnValue(true) - mocks.accounts.mockResolvedValue([ - { - id: 'own-account', - providerId: 'jira', - displayName: 'Work account', - scopes: ['read:jira-work'], - }, - ]) - }) - - it('permits a verified member and scopes the account lookup to that person and provider', async () => { - await expect(authorizePersonalSearchSetupCredential(principal, input)).resolves.toMatchObject({ - id: 'own-account', - }) - expect(organizationAuthorizationMockFns.mockRequireOrganizationMembership).toHaveBeenCalledWith( - principal, - input.organizationId, - 'member', - 'knowledge.use' - ) - expect(mocks.approval).toHaveBeenCalledWith(input.organizationId, 'jira') - expect(mocks.accounts).toHaveBeenCalledWith({ - organizationId: input.organizationId, - userId: principal.userId, - providerId: 'jira', - credentialId: input.credentialId, - }) - }) - - it('rejects non-session callers before membership or protected account reads', async () => { - await expect( - authorizePersonalSearchSetup(createPersonalApiKeyPrincipal({ userId: 'member-1' }), input) - ).rejects.toThrow('Sign in') - expect( - organizationAuthorizationMockFns.mockRequireOrganizationMembership - ).not.toHaveBeenCalled() - expect(dbChainMockFns.select).not.toHaveBeenCalled() - }) - - it('rejects non-members before looking up credentials', async () => { - organizationAuthorizationMockFns.mockRequireOrganizationMembership.mockRejectedValue( - new Error('Membership ended') - ) - await expect(authorizePersonalSearchSetupCredential(principal, input)).rejects.toThrow( - 'Membership ended' - ) - expect(mocks.accounts).not.toHaveBeenCalled() - }) - - it('requires a verified email before approving or provisioning setup', async () => { - resetDbChainMock() - queueTableRows(user, [{ emailVerified: false }]) - await expect(authorizePersonalSearchSetup(principal, input)).rejects.toThrow( - 'Verify your email' - ) - expect(mocks.approval).not.toHaveBeenCalled() - }) - - it.each([ - { accounts: [] }, - { accounts: [{ id: 'another-account', providerId: 'jira' }] }, - { accounts: [{ id: 'own-account', providerId: 'confluence' }] }, - ])('rejects revoked, foreign or mismatched grants %#', async ({ accounts }) => { - mocks.accounts.mockResolvedValue(accounts) - await expect(authorizePersonalSearchSetupCredential(principal, input)).rejects.toThrow( - 'Connect your account again' - ) - }) -}) diff --git a/apps/sim/lib/knowledge/application/personal-search-account.ts b/apps/sim/lib/knowledge/application/personal-search-account.ts deleted file mode 100644 index 04a25443acb..00000000000 --- a/apps/sim/lib/knowledge/application/personal-search-account.ts +++ /dev/null @@ -1,68 +0,0 @@ -import type { Principal } from '@sim/auth/principal' -import { db } from '@sim/db' -import { user } from '@sim/db/schema' -import { eq } from 'drizzle-orm' -import { requireOrganizationMembership } from '@/lib/core/application/organization-authorization' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { getOwnOrganizationManagedOAuthCredentials } from '@/lib/credentials/organization-managed' -import { isOAuthServiceDeploymentAvailable } from '@/lib/integrations/availability.server' -import { requireKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' -import { requireOrganizationSearchApproval } from '@/lib/knowledge/search/integration-policy' - -export type PersonalSearchSetupConnector = 'jira' | 'confluence' - -/** Rechecks the caller's organization and approved integration before personal account discovery. */ -export async function authorizePersonalSearchSetup( - principal: Principal, - input: { organizationId: string; connectorType: PersonalSearchSetupConnector } -) { - if ( - principal.kind !== 'session' || - (input.connectorType !== 'jira' && input.connectorType !== 'confluence') - ) { - throw new OrchestrationError('forbidden', 'Sign in to connect this Search source') - } - await requireOrganizationMembership(principal, input.organizationId, 'member', 'knowledge.use') - const [viewer] = await db - .select({ emailVerified: user.emailVerified }) - .from(user) - .where(eq(user.id, principal.userId)) - .limit(1) - if (!viewer?.emailVerified) { - throw new OrchestrationError( - 'validation', - 'Verify your email address before connecting an account' - ) - } - await requireOrganizationSearchApproval(input.organizationId, input.connectorType) - await requireKnowledgeMemberAccessAvailable({ organizationId: input.organizationId }) - if (!isOAuthServiceDeploymentAvailable(input.connectorType)) { - throw new OrchestrationError('validation', 'This account connection is unavailable') - } - return principal.userId -} - -/** Personal setup can browse only a currently live grant owned by the signed-in member. */ -export async function authorizePersonalSearchSetupCredential( - principal: Principal, - input: { - organizationId: string - connectorType: PersonalSearchSetupConnector - credentialId: string - } -) { - const userId = await authorizePersonalSearchSetup(principal, input) - const accounts = await getOwnOrganizationManagedOAuthCredentials({ - organizationId: input.organizationId, - userId, - providerId: input.connectorType, - credentialId: input.credentialId, - }) - const account = accounts.find( - (entry) => entry.id === input.credentialId && entry.providerId === input.connectorType - ) - if (!account) { - throw new OrchestrationError('not_found', 'Connect your account again before choosing sources') - } - return account -} diff --git a/apps/sim/lib/knowledge/application/personal-search-integration-pages.ts b/apps/sim/lib/knowledge/application/personal-search-integration-pages.ts deleted file mode 100644 index 709ee5f81b2..00000000000 --- a/apps/sim/lib/knowledge/application/personal-search-integration-pages.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { Principal } from '@sim/auth/principal' -import { - type ListPersonalSearchIntegrationsInput, - listPersonalSearchIntegrations, -} from '@/lib/knowledge/application/personal-search-integrations' - -/** One page of the viewer's personal Search integrations. */ -export type PersonalSearchIntegrationsPage = Awaited< - ReturnType -> - -/** The most pages one walk of the personal inventory reads before it stops. */ -const MAX_PERSONAL_SEARCH_INTEGRATION_PAGES = 100 - -/** - * Every page of the viewer's personal Search integrations, in order: Live Search answers in one - * page, indexed search in one per batch of sources. A cursor that repeats, or a walk past - * {@link MAX_PERSONAL_SEARCH_INTEGRATION_PAGES}, is a defect and throws rather than answering from - * part of the inventory. - */ -export async function* personalSearchIntegrationPages({ - principal, - input, - signal, -}: { - principal: Principal - input: Omit - signal?: AbortSignal -}): AsyncGenerator { - const seen = new Set() - let cursor: string | undefined - for (let page = 0; page < MAX_PERSONAL_SEARCH_INTEGRATION_PAGES; page++) { - signal?.throwIfAborted() - const inventory = await listPersonalSearchIntegrations.execute({ - principal, - input: { ...input, ...(cursor ? { cursor } : {}) }, - }) - signal?.throwIfAborted() - yield inventory - if (inventory.nextCursor === null) return - if (seen.has(inventory.nextCursor)) - throw new Error('Personal Search integration pagination did not advance') - seen.add(inventory.nextCursor) - cursor = inventory.nextCursor - } - throw new Error( - `Personal Search integration pagination exceeded ${MAX_PERSONAL_SEARCH_INTEGRATION_PAGES} pages` - ) -} diff --git a/apps/sim/lib/knowledge/application/personal-search-integrations.test.ts b/apps/sim/lib/knowledge/application/personal-search-integrations.test.ts index aeaf4296001..92bc535c87b 100644 --- a/apps/sim/lib/knowledge/application/personal-search-integrations.test.ts +++ b/apps/sim/lib/knowledge/application/personal-search-integrations.test.ts @@ -1,5 +1,5 @@ import { user } from '@sim/db/schema' -import { queueTableRows, resetDbChainMock, resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { queueTableRows, resetDbChainMock, resetEnvFlagsMock } from '@sim/testing' import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' import { credentialGroupsAvailabilityMock, @@ -127,108 +127,6 @@ beforeEach(() => { ) mockGetConnectorAccessAvailability.mockReturnValue({ members: true }) }) -describe('personal Search inventory', () => { - beforeEach(() => setEnvFlags({ isLiveEnterpriseSearchEnabled: false })) - - it.each([ - [{}, 'connected', 'not_indexed'], - [{ isSyncing: true }, 'connected', 'indexing'], - [{ hasViewerDocuments: true }, 'connected', 'indexed'], - [{ hasSyncError: true }, 'connected', 'sync_failed'], - [ - { - viewerAccounts: [{ credentialId: 'mine', displayName: 'My mail', status: 'needs_reauth' }], - }, - 'reconnect_needed', - 'not_indexed', - ], - ])( - 'separates account state from index state %#', - async (changes, connectionStatus, indexingStatus) => { - m.sources.mockResolvedValue({ sources: [{ ...source, ...changes }], nextCursor: null }) - const result = await listPersonalSearchIntegrations.execute({ principal, input }) - expect(result.connections[0]).toMatchObject({ connectionStatus, indexingStatus }) - expect(personalSearchIntegrationPageSchema.safeParse(result).success).toBe(true) - expect(m.sources).toHaveBeenCalledWith({ principal, input }) - expect(m.configuredTypes).toHaveBeenCalledWith({ organizationId: 'org' }) - expect(JSON.stringify(result)).not.toMatch(/accessToken|authorizationUrl|other-person/) - } - ) - it('offers approved ready providers with no index, excluding app setup and unapproved providers', async () => { - m.sources.mockResolvedValue({ sources: [], nextCursor: null }) - m.configuredTypes.mockResolvedValue([]) - const result = await listPersonalSearchIntegrations.execute({ principal, input }) - expect(result.available.map((entry) => entry.target.connectorType)).toEqual(['gmail']) - expect(result.connections).toEqual([]) - }) - it('omits every connection action when the person has an unverified email', async () => { - resetDbChainMock() - queueTableRows(user, [{ emailVerified: false }]) - m.sources.mockResolvedValue({ sources: [], nextCursor: null }) - m.configuredTypes.mockResolvedValue([]) - expect((await listPersonalSearchIntegrations.execute({ principal, input })).available).toEqual( - [] - ) - }) - it('forwards the bounded cursor and provider filter without exposing other people’s accounts', async () => { - m.sources.mockResolvedValue({ - sources: [{ ...source, viewerAccounts: [], viewerMembership: 'not_enrolled' }], - nextCursor: 'next', - }) - const filtered = { ...input, connectorType: 'gmail', cursor: 'page' } - const result = await listPersonalSearchIntegrations.execute({ principal, input: filtered }) - expect(result.connections).toEqual([]) - expect(result.nextCursor).toBe('next') - expect(result.available).toEqual([{ name: 'gmail', description: '', target }]) - expect(m.sources).toHaveBeenCalledWith({ principal, input: filtered }) - }) - it('rechecks authorization before every read and returns nothing after membership is revoked', async () => { - organizationAuthorizationMockFns.mockAuthorizeOrganizationOperation.mockRejectedValue( - new Error('Membership revoked') - ) - await expect(listPersonalSearchIntegrations.execute({ principal, input })).rejects.toThrow( - 'Membership revoked' - ) - expect(m.sources).not.toHaveBeenCalled() - }) - it.each([ - { ...target, credentialId: 'another-person' }, - { ...target, connectorId: 'other-source' }, - { ...target, provider: 'notion' }, - ])('rejects a forged or stale reconnect target', async (forged) => { - m.sources.mockResolvedValue({ - sources: [ - { - ...source, - viewerAccounts: [ - { credentialId: 'mine', displayName: 'My mail', status: 'needs_reauth' }, - ], - }, - ], - nextCursor: null, - }) - await expect( - resolvePersonalSearchConnection.execute({ principal, input: { ...input, target: forged } }) - ).rejects.toThrow('no longer available') - }) - it('returns the precise owned reconnect target', async () => { - m.sources.mockResolvedValue({ - sources: [ - { - ...source, - viewerAccounts: [ - { credentialId: 'mine', displayName: 'My mail', status: 'needs_reauth' }, - ], - }, - ], - nextCursor: null, - }) - const selected = { ...target, credentialId: 'mine' } - await expect( - resolvePersonalSearchConnection.execute({ principal, input: { ...input, target: selected } }) - ).resolves.toEqual({ name: 'gmail', target: selected }) - }) -}) describe('live Search connection controls', () => { const liveTarget = { @@ -238,7 +136,6 @@ describe('live Search connection controls', () => { connectionMode: 'live', optionId: 'slack-option', } as const - beforeEach(() => setEnvFlags({ isLiveEnterpriseSearchEnabled: true })) it('offers Slack without a knowledge base or indexed connector and round-trips the response', async () => { const result = await listPersonalSearchIntegrations.execute({ principal, input }) @@ -298,7 +195,6 @@ describe('live Search connection controls', () => { ...liveTarget, credentialId: 'mine', }) - expect(result.connections[0].indexingStatus).toBeUndefined() expect(personalSearchIntegrationPageSchema.safeParse(result).success).toBe(true) }) @@ -306,7 +202,6 @@ describe('live Search connection controls', () => { { ...liveTarget, credentialId: 'another-person' }, { ...liveTarget, optionId: 'another-option' }, { ...liveTarget, provider: 'gmail' }, - { ...liveTarget, connectionMode: undefined, optionId: undefined }, ])('rejects a forged or stale live target: %j', async (target) => { await expect( resolvePersonalSearchConnection.execute({ principal, input: { ...input, target } }) @@ -367,14 +262,4 @@ describe('live Search connection controls', () => { completionId: 'attempt', }) }) - - it('rejects an indexed source target after switching to live search', async () => { - await expect( - listPersonalSearchIntegrations.execute({ - principal, - input: { ...input, connectorId: 'stale-source' }, - }) - ).rejects.toThrow('Refresh your live account connections') - expect(m.group).not.toHaveBeenCalled() - }) }) diff --git a/apps/sim/lib/knowledge/application/personal-search-integrations.ts b/apps/sim/lib/knowledge/application/personal-search-integrations.ts index 5121c564f0c..3a910921cb5 100644 --- a/apps/sim/lib/knowledge/application/personal-search-integrations.ts +++ b/apps/sim/lib/knowledge/application/personal-search-integrations.ts @@ -14,15 +14,11 @@ import { knowledgeOperations } from '@/lib/knowledge/application/operations' import type { SearchConnectionTarget } from '@/lib/knowledge/search/connection-target' import { listOrganizationSearchApprovals } from '@/lib/knowledge/search/integration-policy' import { SEARCH_CONNECTORS } from '@/lib/sim-search/connectors' -import { isIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' -import { listIndexedPersonalSearchIntegrations } from '@/lib/sim-search/indexed/integrations/personal-search-integrations' import { LIVE_SEARCH_SCOPE_FIELDS } from '@/lib/sim-search/live/policy-schema' export interface ListPersonalSearchIntegrationsInput { organizationId: string connectorType?: string - connectorId?: string - cursor?: string completionId?: string } @@ -39,10 +35,6 @@ export const listPersonalSearchIntegrations = defineAuthorizedKnowledgeUseCase({ .where(eq(user.id, userId)) .limit(1) if (!viewer) throw new OrchestrationError('forbidden', 'The current person is unavailable') - if (isIndexedOrgSearchEnabled()) - return listIndexedPersonalSearchIntegrations({ principal, input, context, userId, viewer }) - if (input.connectorId || input.cursor) - throw new OrchestrationError('validation', 'Refresh your live account connections') const scope = { kind: 'organization', organizationId: context.organizationId } as const if (!(await isScopedCredentialGroupsAvailable(scope))) return { completedCredentialId: null, connections: [], available: [], nextCursor: null } @@ -95,9 +87,6 @@ export const listPersonalSearchIntegrations = defineAuthorizedKnowledgeUseCase({ name: connector.meta.name, providerId: option.provider, connectorType: connector.type, - connectorId: undefined, - knowledgeBaseId: undefined, - indexingStatus: undefined, description: '', accounts: own, connectionStatus: !ready @@ -144,7 +133,6 @@ export const resolvePersonalSearchConnection = defineAuthorizedKnowledgeUseCase( input: { organizationId: input.organizationId, connectorType: input.target.connectorType, - connectorId: input.target.connectorId, }, }) const targets = [ @@ -159,7 +147,6 @@ export const resolvePersonalSearchConnection = defineAuthorizedKnowledgeUseCase( ({ target }) => target.provider === input.target.provider && target.connectorType === input.target.connectorType && - target.connectorId === input.target.connectorId && target.credentialId === input.target.credentialId && target.connectionMode === input.target.connectionMode && target.optionId === input.target.optionId diff --git a/apps/sim/lib/knowledge/application/personal-source-setup.test.ts b/apps/sim/lib/knowledge/application/personal-source-setup.test.ts deleted file mode 100644 index 506037b2245..00000000000 --- a/apps/sim/lib/knowledge/application/personal-source-setup.test.ts +++ /dev/null @@ -1,271 +0,0 @@ -import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' -import { - credentialGroupsCredentialsMock, - credentialGroupsCredentialsMockFns, -} from '@sim/testing/mocks/credential-groups-credentials.mock' -import { - credentialGroupsEnrollmentsMock, - credentialGroupsEnrollmentsMockFns, -} from '@sim/testing/mocks/credential-groups-enrollments.mock' -import { - credentialGroupsSelfEnrollmentMock, - credentialGroupsSelfEnrollmentMockFns, -} from '@sim/testing/mocks/credential-groups-self-enrollment.mock' -import { knowledgeContextsMock } from '@sim/testing/mocks/knowledge-contexts.mock' -import { - knowledgeMemberQueueMock, - knowledgeMemberQueueMockFns, -} from '@sim/testing/mocks/knowledge-member-queue.mock' -import { - organizationAuthorizationMock, - organizationAuthorizationMockFns, -} from '@sim/testing/mocks/organization-authorization.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const hoisted = vi.hoisted(() => ({ - authorize: vi.fn(), - ownAccount: vi.fn(), - listAccounts: vi.fn(), - completion: vi.fn(), - provision: vi.fn(), - oauth: vi.fn(), - selector: vi.fn(), - configure: vi.fn(), - billing: vi.fn(), -})) -vi.mock('@/lib/core/application/organization-authorization', () => organizationAuthorizationMock) -vi.mock('@/lib/knowledge/application/contexts', () => knowledgeContextsMock) -vi.mock('@/lib/knowledge/application/personal-search-account', () => ({ - authorizePersonalSearchSetup: hoisted.authorize, - authorizePersonalSearchSetupCredential: hoisted.ownAccount, -})) -vi.mock('@/lib/credentials/organization-managed', () => ({ - getOwnOrganizationManagedOAuthCredentials: hoisted.listAccounts, -})) -vi.mock('@/lib/credential-groups/search-connection-completion', () => ({ - readSearchConnectionCompletion: hoisted.completion, -})) -vi.mock('@/lib/knowledge/connectors/member-provisioning', () => ({ - provisionKnowledgeConnectorMembersBinding: hoisted.provision, -})) -vi.mock('@/lib/credential-groups/self-enrollment', () => credentialGroupsSelfEnrollmentMock) -vi.mock('@/lib/credential-groups/enrollments', () => credentialGroupsEnrollmentsMock) -vi.mock('@/lib/credential-groups/oauth', () => ({ startCredentialGroupOAuth: hoisted.oauth })) -vi.mock('@/lib/selectors/application/execute-selector', () => ({ - executeSelector: { execute: hoisted.selector }, -})) -vi.mock('@/lib/knowledge/application/sim-search', () => ({ - configureSimSearchConnector: { execute: hoisted.configure }, -})) -vi.mock('@/lib/credential-groups/credentials', () => credentialGroupsCredentialsMock) -vi.mock('@/lib/knowledge/connectors/member-queue', () => knowledgeMemberQueueMock) -vi.mock('@/lib/knowledge/application/billing', () => ({ - resolveKnowledgeBillingAttribution: hoisted.billing, -})) -vi.mock('@/connectors/registry', () => ({ - CONNECTOR_META_REGISTRY: { jira: { name: 'Jira' }, confluence: { name: 'Confluence' } }, -})) - -import { - listPersonalSourceSetupAccounts, - personalSourceSetup, -} from '@/lib/knowledge/application/personal-source-setup' -import type { SelectorRequest } from '@/lib/selectors/types' - -const mocks = { - ...hoisted, - enrollment: credentialGroupsSelfEnrollmentMockFns.mockCreateViewerCredentialGroupEnrollment, - oauthContext: credentialGroupsEnrollmentsMockFns.mockGetCredentialGroupOAuthContextForEnrollment, - binding: credentialGroupsCredentialsMockFns.mockLoadManagedCredentialGroupBinding, - group: credentialGroupsCredentialsMockFns.mockLoadScopedAccountsCredentialListContext, - dispatch: knowledgeMemberQueueMockFns.mockDispatchMemberSync, -} - -interface ValidationSelectorCall { - input: { request: SelectorRequest; signal: AbortSignal } -} - -const principal = createSessionPrincipal({ userId: 'member-1' }) -const owner = { organizationId: 'organization-1', connectorType: 'jira' } as const -const credential = { credentialId: 'own-account', domain: 'example.atlassian.net' } -const connect = { ...owner, ...credential, action: 'connect', keys: ['PROJECT'] } as const -const account = { - id: 'own-account', - displayName: 'Work account', - providerId: 'jira', - scopes: ['read:jira-work'], -} -const runConnect = (changes = {}) => - personalSourceSetup.execute({ principal, input: { ...connect, keys: ['PROJECT'], ...changes } }) - -describe('personal source setup', () => { - it.each(['confluence', 'jira'] as const)( - 'rejects mixed All and explicit %s keys before discovery or saving', - async (connectorType) => { - await expect(runConnect({ connectorType, keys: ['*', 'ENG'] })).rejects.toThrow( - 'Use "*" by itself for All, or remove it to select individual items.' - ) - expect(mocks.selector).not.toHaveBeenCalled() - expect(mocks.configure).not.toHaveBeenCalled() - } - ) - - it('does not let All bypass a failed site or credential check', async () => { - mocks.selector.mockRejectedValue(new Error('Site unavailable')) - await expect(runConnect({ keys: ['*'] })).rejects.toThrow('Site unavailable') - expect(mocks.configure).not.toHaveBeenCalled() - }) - - beforeEach(() => { - vi.resetAllMocks() - mocks.authorize.mockResolvedValue(principal.userId) - mocks.ownAccount.mockResolvedValue(account) - mocks.listAccounts.mockResolvedValue([account]) - mocks.completion.mockResolvedValue(account.id) - mocks.provision.mockResolvedValue({ - credentialGroupId: 'group-1', - credentialGroupOptionId: 'option-1', - }) - mocks.enrollment.mockResolvedValue({ - enrollment: { id: 'enrollment-1', email: 'member@example.com' }, - invitationLink: 'https://example.com/credential-groups/enroll/invitation-token', - }) - mocks.oauthContext.mockResolvedValue({ option: { id: 'option-1' } }) - mocks.oauth.mockResolvedValue('https://auth.atlassian.com/authorize') - mocks.selector.mockResolvedValue({ kind: 'list', items: [{ id: 'PROJECT', label: 'Project' }] }) - mocks.configure.mockResolvedValue({ knowledgeBaseId: 'kb-1', connectorId: 'source-1' }) - mocks.binding.mockResolvedValue({ - organizationId: owner.organizationId, - credentialGroupId: 'group-1', - credentialGroupOptionId: 'option-1', - }) - mocks.group.mockResolvedValue({ credentialGroupId: 'group-1' }) - mocks.billing.mockResolvedValue({ actorUserId: principal.userId }) - mocks.dispatch.mockResolvedValue({ queued: true }) - }) - - it('does not return a receipt for a foreign or no longer active credential', async () => { - mocks.completion.mockResolvedValue('another-account') - expect( - ( - await listPersonalSourceSetupAccounts.execute({ - principal, - input: { ...owner, completionId: 'completion-1' }, - }) - ).completedCredentialId - ).toBeNull() - }) - - it('does not start OAuth if enrollment was revoked', async () => { - mocks.enrollment.mockRejectedValue(new Error('An admin removed your access')) - await expect( - personalSourceSetup.execute({ - principal, - input: { ...owner, action: 'authorize', oauthCompletionId: 'completion-1' }, - }) - ).rejects.toThrow('admin removed') - expect(mocks.oauth).not.toHaveBeenCalled() - }) - - it.each([ - { kind: 'list', items: [], truncated: true, nextCursor: '50' }, - { kind: 'list', items: [] }, - ])('rejects unavailable manually entered keys before source creation %#', async (page) => { - mocks.selector.mockResolvedValueOnce(page).mockResolvedValue({ kind: 'detail', item: null }) - await expect(runConnect({ keys: ['MISSING'] })).rejects.toThrow('could not be found') - expect(mocks.configure).not.toHaveBeenCalled() - }) - - it('fails before mutation when direct validation also exceeds its deadline', async () => { - const listing = new AbortController() - const details = new AbortController() - vi.spyOn(AbortSignal, 'timeout') - .mockReturnValueOnce(listing.signal) - .mockReturnValueOnce(details.signal) - mocks.selector - .mockResolvedValueOnce({ kind: 'list', items: [] }) - .mockImplementationOnce(({ input }: ValidationSelectorCall) => { - details.abort(new DOMException('Validation timed out', 'TimeoutError')) - input.signal.throwIfAborted() - }) - await expect(runConnect()).rejects.toThrow('took too long') - expect(mocks.configure).not.toHaveBeenCalled() - }) - - it('rejects a detail response for a different key', async () => { - mocks.selector - .mockResolvedValueOnce({ kind: 'list', items: [] }) - .mockResolvedValueOnce({ kind: 'detail', item: { id: 'OTHER', label: 'Other project' } }) - await expect(runConnect()).rejects.toThrow('could not be found') - expect(mocks.configure).not.toHaveBeenCalled() - }) - - it('bounds direct validation concurrency and validates each unresolved key only once', async () => { - let release = () => {} - let allStarted = () => {} - const gate = new Promise((resolve) => { - release = resolve - }) - const started = new Promise((resolve) => { - allStarted = resolve - }) - let active = 0 - let maximumActive = 0 - mocks.selector.mockImplementation(async ({ input }: ValidationSelectorCall) => { - if (input.request.kind === 'list') return { kind: 'list', items: [] } - active++ - maximumActive = Math.max(maximumActive, active) - if (active === 5) allStarted() - await gate - active-- - return { kind: 'detail', item: { id: input.request.id, label: input.request.id } } - }) - const keys = Array.from({ length: 12 }, (_, index) => `PROJECT${index}`) - const connection = runConnect({ keys: [...keys, ...keys] }) - await started - try { - expect(mocks.selector).toHaveBeenCalledTimes(6) - expect(mocks.configure).not.toHaveBeenCalled() - } finally { - release() - } - await expect(connection).resolves.toMatchObject({ kind: 'connected' }) - expect(maximumActive).toBe(5) - expect(mocks.selector).toHaveBeenCalledTimes(keys.length + 1) - }) - - it('rejects stale or wrong-user credentials before provider calls', async () => { - mocks.ownAccount.mockRejectedValue(new Error('Account unavailable')) - await expect(runConnect()).rejects.toThrow('Account unavailable') - expect(mocks.selector).not.toHaveBeenCalled() - expect(mocks.configure).not.toHaveBeenCalled() - }) - - it('rejects an account whose enrollment group differs from the organization setup', async () => { - mocks.binding.mockResolvedValue({ - organizationId: owner.organizationId, - credentialGroupId: 'other-group', - credentialGroupOptionId: 'option-1', - }) - await expect(runConnect()).rejects.toThrow('Connect your account again') - expect(mocks.configure).not.toHaveBeenCalled() - }) - - it('rejects a current operation authorization denial before any setup effects', async () => { - organizationAuthorizationMockFns.mockAuthorizeOrganizationOperation.mockRejectedValue( - new Error('Membership ended') - ) - await expect(runConnect()).rejects.toThrow('Membership ended') - expect(mocks.authorize).not.toHaveBeenCalled() - expect(mocks.configure).not.toHaveBeenCalled() - }) - - it.each([ - 'example.atlassian.net/wiki/spaces', - 'user:password@example.atlassian.net', - 'example.atlassian.net?site=other', - ])('rejects an invalid site value before discovery: %s', async (domain) => { - await expect(runConnect({ domain })).rejects.toThrow('hostname') - expect(mocks.selector).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/knowledge/application/personal-source-setup.ts b/apps/sim/lib/knowledge/application/personal-source-setup.ts deleted file mode 100644 index e38870f5103..00000000000 --- a/apps/sim/lib/knowledge/application/personal-source-setup.ts +++ /dev/null @@ -1,305 +0,0 @@ -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { normalizeAtlassianSiteUrl } from '@/lib/atlassian/discovery' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { mapWithConcurrency } from '@/lib/core/utils/concurrency' -import { - loadManagedCredentialGroupBinding, - loadScopedAccountsCredentialListContext, -} from '@/lib/credential-groups/credentials' -import { getCredentialGroupOAuthContextForEnrollment } from '@/lib/credential-groups/enrollments' -import { startCredentialGroupOAuth } from '@/lib/credential-groups/oauth' -import { readSearchConnectionCompletion } from '@/lib/credential-groups/search-connection-completion' -import { createViewerCredentialGroupEnrollment } from '@/lib/credential-groups/self-enrollment' -import { getOwnOrganizationManagedOAuthCredentials } from '@/lib/credentials/organization-managed' -import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' -import { resolveKnowledgeBillingAttribution } from '@/lib/knowledge/application/billing' -import { resolveKnowledgeOrganizationContext } from '@/lib/knowledge/application/contexts' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { - authorizePersonalSearchSetup, - authorizePersonalSearchSetupCredential, - type PersonalSearchSetupConnector, -} from '@/lib/knowledge/application/personal-search-account' -import { configureSimSearchConnector } from '@/lib/knowledge/application/sim-search' -import { provisionKnowledgeConnectorMembersBinding } from '@/lib/knowledge/connectors/member-provisioning' -import { executeSelector } from '@/lib/selectors/application/execute-selector' -import { MAX_SELECTOR_PAGES } from '@/lib/selectors/limits' -import type { SelectorExecutionResult, SelectorRequest } from '@/lib/selectors/types' -import { MAX_PERSONAL_SOURCE_SETUP_KEYS } from '@/lib/sim-search/personal-source-setup' -import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' -import { getSourceSelectionError, isAllSourceItems } from '@/connectors/selection' - -const logger = createLogger('PersonalSourceSetup') -const VALIDATION_PHASE_TIMEOUT_MS = 30_000 -const DETAIL_VALIDATION_CONCURRENCY = 5 - -interface PersonalSourceSetupOwner { - organizationId: string - connectorType: PersonalSearchSetupConnector -} - -type PersonalSourceSetupInput = PersonalSourceSetupOwner & - ( - | { action: 'authorize'; oauthCompletionId: string } - | { action: 'options'; credentialId: string; domain: string; request: SelectorRequest } - | { action: 'connect'; credentialId: string; domain: string; keys: string[] } - ) - -type PersonalSourceSetupResult = - | { kind: 'authorization'; url: string } - | { kind: 'connected'; knowledgeBaseId: string; connectorId: string } - | SelectorExecutionResult - -function normalizeSetupDomain(value: string) { - let url: URL - try { - url = new URL(normalizeAtlassianSiteUrl(value)) - } catch { - throw new OrchestrationError('validation', 'Enter your Atlassian site hostname') - } - if ( - url.username || - url.password || - url.port || - url.pathname !== '/' || - url.search || - url.hash || - !url.hostname.includes('.') - ) { - throw new OrchestrationError( - 'validation', - 'Enter your Atlassian site hostname without a page path' - ) - } - return url.hostname -} - -/** Lists the viewer's live accounts independently of whether any source exists yet. */ -export const listPersonalSourceSetupAccounts = defineAuthorizedKnowledgeUseCase({ - operation: knowledgeOperations.listPersonalSourceSetupAccounts, - resolveContext: ({ input }: { input: PersonalSourceSetupOwner & { completionId?: string } }) => - resolveKnowledgeOrganizationContext(input), - async execute({ principal, input }) { - const userId = await authorizePersonalSearchSetup(principal, input) - const accounts = await getOwnOrganizationManagedOAuthCredentials({ - organizationId: input.organizationId, - userId, - providerId: input.connectorType, - }) - const completedCredentialId = input.completionId - ? await readSearchConnectionCompletion({ - organizationId: input.organizationId, - userId, - completionId: input.completionId, - }) - : null - return { - accounts: accounts.map((account) => ({ - id: account.id, - name: account.displayName, - provider: input.connectorType, - type: 'managed_oauth' as const, - scopes: account.scopes, - })), - completedCredentialId: accounts.some((account) => account.id === completedCredentialId) - ? completedCredentialId - : null, - } - }, -}) - -/** Connects the viewer first, then configures only an approved personal-account Search source. */ -export const personalSourceSetup = defineAuthorizedKnowledgeUseCase({ - operation: knowledgeOperations.personalSourceSetup, - resolveContext: ({ input }: { input: PersonalSourceSetupInput }) => - resolveKnowledgeOrganizationContext(input), - async execute({ principal, input, context, request }): Promise { - const userId = await authorizePersonalSearchSetup(principal, input) - if (input.action === 'authorize') { - const binding = await provisionKnowledgeConnectorMembersBinding({ - organizationId: input.organizationId, - connectorMeta: CONNECTOR_META_REGISTRY[input.connectorType]!, - userId, - }) - const { enrollment, invitationLink } = await createViewerCredentialGroupEnrollment({ - organizationId: input.organizationId, - userId, - credentialGroupId: binding.credentialGroupId, - }) - const token = new URL(invitationLink).pathname.split('/').at(-1) - if (!token) throw new Error('Account enrollment did not return an invitation token') - const oauth = await getCredentialGroupOAuthContextForEnrollment( - { - organizationId: input.organizationId, - credentialGroupId: binding.credentialGroupId, - enrollmentId: enrollment.id, - email: enrollment.email, - userId, - }, - binding.credentialGroupOptionId - ) - if (!oauth) - throw new OrchestrationError('forbidden', 'This account connection is no longer available') - return { - kind: 'authorization', - url: await startCredentialGroupOAuth(oauth, token, { - completionRedirect: true, - returnTo: 'search', - completionId: input.oauthCompletionId, - connectionIntent: { kind: 'create' }, - }), - } - } - - await authorizePersonalSearchSetupCredential(principal, input) - const domain = normalizeSetupDomain(input.domain) - const selectorInput = { - selectorKey: - input.connectorType === 'jira' - ? ('jira.projectKeys' as const) - : ('confluence.spaces' as const), - scope: { kind: 'organization' as const, organizationId: input.organizationId }, - context: { oauthCredential: input.credentialId, domain }, - personalSearchSetup: input.connectorType, - } - if (input.action === 'options') { - return executeSelector.execute({ - principal, - request, - input: { ...selectorInput, request: input.request }, - }) - } - if ( - !input.keys.length || - input.keys.length > MAX_PERSONAL_SOURCE_SETUP_KEYS || - input.keys.some((key) => !key.trim() || key.length > 255) - ) { - throw new OrchestrationError('validation', 'Select between 1 and 1,000 projects or spaces') - } - const keys = [...new Set(input.keys.map((key) => key.trim()))] - const selectionError = getSourceSelectionError(keys) - if (selectionError) throw new OrchestrationError('validation', selectionError) - const remaining = new Set(keys) - const cursors = new Set() - const timeout = AbortSignal.timeout(VALIDATION_PHASE_TIMEOUT_MS) - const signal = request?.signal ? AbortSignal.any([request.signal, timeout]) : timeout - let cursor: string | undefined - for (let page = 0; page < MAX_SELECTOR_PAGES; page++) { - let result: SelectorExecutionResult - try { - signal.throwIfAborted() - result = await executeSelector.execute({ - principal, - request, - input: { - ...selectorInput, - signal, - request: { kind: 'list', ...(cursor ? { cursor } : {}) }, - }, - }) - signal.throwIfAborted() - } catch (error) { - if (timeout.aborted && !request?.signal?.aborted) break - throw error - } - if (result.kind !== 'list') throw new Error('Source discovery returned an unexpected result') - if (isAllSourceItems(keys)) { - remaining.clear() - break - } - for (const option of result.items) remaining.delete(option.id) - if (remaining.size === 0) break - if (!result.nextCursor || result.truncated || cursors.has(result.nextCursor)) break - cursor = result.nextCursor - cursors.add(cursor) - } - request?.signal?.throwIfAborted() - if (remaining.size > 0) { - const detailTimeout = AbortSignal.timeout(VALIDATION_PHASE_TIMEOUT_MS) - const details = new AbortController() - const detailSignal = AbortSignal.any([ - detailTimeout, - details.signal, - ...(request?.signal ? [request.signal] : []), - ]) - try { - await mapWithConcurrency([...remaining], DETAIL_VALIDATION_CONCURRENCY, async (key) => { - detailSignal.throwIfAborted() - const result = await executeSelector.execute({ - principal, - request, - input: { - ...selectorInput, - signal: detailSignal, - request: { kind: 'detail', id: key }, - }, - }) - detailSignal.throwIfAborted() - if (result.kind !== 'detail' || result.item?.id !== key) { - throw new OrchestrationError( - 'validation', - 'Some selected projects or spaces could not be found with this account. Refresh the choices and try again.' - ) - } - }) - } catch (error) { - details.abort(error) - if (detailTimeout.aborted && !request?.signal?.aborted) { - throw new OrchestrationError( - 'validation', - 'Checking the selected projects or spaces took too long. Try fewer selections.' - ) - } - throw error - } - } - const [binding, group] = await Promise.all([ - loadManagedCredentialGroupBinding(input.credentialId), - loadScopedAccountsCredentialListContext({ - kind: 'organization', - organizationId: input.organizationId, - }), - ]) - if ( - !binding || - !group || - binding.credentialGroupId !== group.credentialGroupId || - binding.organizationId !== input.organizationId - ) { - throw new OrchestrationError( - 'not_found', - 'Connect your account again before choosing sources' - ) - } - await authorizePersonalSearchSetupCredential(principal, input) - request?.signal?.throwIfAborted() - const result = await configureSimSearchConnector.execute({ - principal, - request, - input: { - organizationId: input.organizationId, - connectorType: input.connectorType, - memberCredentialBinding: { - credentialGroupId: binding.credentialGroupId, - credentialGroupOptionId: binding.credentialGroupOptionId, - }, - sourceConfig: { - domain, - [input.connectorType === 'jira' ? 'projectKey' : 'spaceKey']: [...keys].join(','), - }, - }, - }) - try { - const { dispatchMemberSync } = await import('@/lib/knowledge/connectors/member-queue') - await dispatchMemberSync(result.connectorId, { - billingAttribution: await resolveKnowledgeBillingAttribution(principal, context), - }) - } catch (error) { - logger.warn('Initial personal source sync will retry on its schedule', { - error: getErrorMessage(error), - }) - } - return { kind: 'connected', ...result } - }, -}) diff --git a/apps/sim/lib/knowledge/application/search-diagnostics.ts b/apps/sim/lib/knowledge/application/search-diagnostics.ts index 7643ad14836..da404865994 100644 --- a/apps/sim/lib/knowledge/application/search-diagnostics.ts +++ b/apps/sim/lib/knowledge/application/search-diagnostics.ts @@ -1,5 +1,4 @@ import type { AuthorizingUseCase } from '@/lib/core/application' -import type { ResourceOwner } from '@/lib/core/resource-scope' import type { knowledgeOperations } from '@/lib/knowledge/application/operations' import type { SearchKnowledgeInput } from '@/lib/knowledge/application/search' import { @@ -44,29 +43,3 @@ export function instrumentSearchUseCase< ), } } - -/** - * Times the source overview, whose cost is access batching and live source proof rather than - * retrieval. It shares the search trace so one log line explains a slow Sim Search surface. - */ -export function instrumentSourceOverviewUseCase( - useCase: AuthorizingUseCase -): AuthorizingUseCase { - return { - ...useCase, - execute: (args) => - withSearchDiagnostics( - { - operation: 'read_search_source_overview', - principalKind: args.principal.kind, - /** Derived without `resourceScopeFromOwner`, which throws before authorization runs. */ - scopeKind: args.input.organizationId - ? 'organization' - : args.input.workspaceId - ? 'workspace' - : undefined, - }, - () => measureSearchStage('source_overview', () => useCase.execute(args)) - ), - } -} diff --git a/apps/sim/lib/knowledge/application/search-integrations.test.ts b/apps/sim/lib/knowledge/application/search-integrations.test.ts index 1f00c17269a..e353084631e 100644 --- a/apps/sim/lib/knowledge/application/search-integrations.test.ts +++ b/apps/sim/lib/knowledge/application/search-integrations.test.ts @@ -4,13 +4,7 @@ import { organization, organizationSearchIntegration, } from '@sim/db/schema' -import { - dbChainMockFns, - queueTableRows, - resetDbChainMock, - resetEnvFlagsMock, - setEnvFlags, -} from '@sim/testing' +import { dbChainMockFns, queueTableRows, resetDbChainMock, resetEnvFlagsMock } from '@sim/testing' import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' import { auditMock, auditMockFns } from '@sim/testing/mocks/audit.mock' import { @@ -153,7 +147,7 @@ describe('organization Search approval', () => { }) it('preserves existing sources while an explicit deactivation overrides them', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) + queueTableRows(organization, [{ metadata: {} }]) queueTableRows(member, [{ role: 'member' }]) queueTableRows(organizationSearchIntegration, [{ connectorType: 'gmail', approved: false }]) queueTableRows(knowledgeConnector, [ @@ -162,12 +156,12 @@ describe('organization Search approval', () => { ]) await expect( listSearchIntegrations.execute({ principal, input: { organizationId: 'organization' } }) - ).resolves.toEqual([ - { connectorType: 'gmail', approved: false }, - { connectorType: 'google_drive', approved: true }, - { connectorType: 'github', approved: false }, - { connectorType: 'jira', approved: false }, - ]) + ).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ connectorType: 'gmail', approved: false }), + expect.objectContaining({ connectorType: 'google_drive', approved: true }), + ]) + ) }) }) @@ -224,7 +218,7 @@ describe('organization Search controls through Mothership', () => { }) it('allows delegated members to read approval state without granting writes', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) + queueTableRows(organization, [{ metadata: {} }]) queueTableRows(member, [{ role: 'member' }]) queueTableRows(organizationSearchIntegration, [{ connectorType: 'gmail', approved: true }]) queueTableRows(knowledgeConnector, []) @@ -233,7 +227,7 @@ describe('organization Search controls through Mothership', () => { principal: delegatedPrincipal, input: { organizationId: 'organization' }, }) - ).resolves.toContainEqual({ connectorType: 'gmail', approved: true }) + ).resolves.toContainEqual(expect.objectContaining({ connectorType: 'gmail', approved: true })) expect(dbChainMockFns.insert).not.toHaveBeenCalled() }) }) @@ -254,7 +248,6 @@ describe('live organization search policies', () => { it.each([principal, delegatedPrincipal])( 'repairs an already approved member source through the same atomic admin action', async (actor) => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) queueTableRows(member, [{ role: 'admin' }]) mocks.memberSetup.mockResolvedValueOnce({ groupId: 'group', changed: true }) const result = await approveSearchIntegration.execute({ @@ -282,7 +275,6 @@ describe('live organization search policies', () => { it.each([principal, delegatedPrincipal])( 'requires integrations capability before provisioning sign-in for an authorized admin', async (actor) => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) queueTableRows(member, [{ role: 'admin' }]) permissionGroupsResolveMockFns.mockGetUserPermissionConfigForOrganization.mockResolvedValue({ hideIntegrationsTab: true, @@ -299,7 +291,6 @@ describe('live organization search policies', () => { ) it('does not approve a source when member sign-in setup fails', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) queueTableRows(member, [{ role: 'admin' }]) mocks.memberSetup.mockRejectedValueOnce(new Error('Provider configuration is unavailable')) await expect(approveSearchIntegration.execute({ principal, input })).rejects.toThrow( @@ -310,7 +301,6 @@ describe('live organization search policies', () => { }) it('explains missing OAuth app setup instead of approving a source with an unusable connection', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) queueTableRows(member, [{ role: 'admin' }]) mocks.memberSetup.mockRejectedValueOnce( new CredentialGroupProviderConfigurationError('Managed Jira authorization is not configured') @@ -326,13 +316,11 @@ describe('live organization search policies', () => { }) it('does not provision sign-in while removing a source', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) queueTableRows(member, [{ role: 'admin' }]) await approveSearchIntegration.execute({ principal, input: { ...input, approved: false } }) expect(mocks.memberSetup).not.toHaveBeenCalled() }) it('clears source settings when switching to member accounts', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) queueTableRows(member, [{ role: 'admin' }]) const result = await approveSearchIntegration.execute({ principal, @@ -357,7 +345,6 @@ describe('live organization search policies', () => { ])( 'rejects unavailable sources before writing without masking infrastructure errors', async ({ error, code }) => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) queueTableRows(member, [{ role: 'admin' }]) mocks.source.mockRejectedValueOnce(error) const attempt = approveSearchIntegration.execute({ @@ -377,21 +364,7 @@ describe('live organization search policies', () => { expect(dbChainMockFns.insert).not.toHaveBeenCalled() } ) - - it('rejects policy writes when the rollout flag is off', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) - queueTableRows(member, [{ role: 'owner' }]) - await expect( - approveSearchIntegration.execute({ - principal, - input: { ...input, policy: defaultLiveSearchPolicy() }, - }) - ).rejects.toMatchObject({ code: 'validation' }) - expect(dbChainMockFns.insert).not.toHaveBeenCalled() - expect(dbChainMockFns.update).not.toHaveBeenCalled() - }) it('saves a validated service source and its approval in one transaction', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) queueTableRows(member, [{ role: 'admin' }]) const result = await approveSearchIntegration.execute({ principal, @@ -424,7 +397,6 @@ describe('live organization search policies', () => { expect(result.changed).toBe(true) }) it('allows GitHub App mode without a single source ID', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) queueTableRows(member, [{ role: 'admin' }]) const result = await approveSearchIntegration.execute({ principal, @@ -440,7 +412,6 @@ describe('live organization search policies', () => { expect(dbChainMockFns.transaction).toHaveBeenCalledOnce() }) it('saves an unfinished service-account source without exposing member-mode search', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) queueTableRows(member, [{ role: 'admin' }]) const result = await approveSearchIntegration.execute({ principal, @@ -454,7 +425,6 @@ describe('live organization search policies', () => { expect(mocks.source).not.toHaveBeenCalled() }) it('rejects invalid scope data before any protected write', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) queueTableRows(member, [{ role: 'admin' }]) await expect( approveSearchIntegration.execute({ @@ -466,7 +436,6 @@ describe('live organization search policies', () => { expect(dbChainMockFns.insert).not.toHaveBeenCalled() }) it('prevents members from changing live search scopes', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) queueTableRows(member, [{ role: 'member' }]) await expect( approveSearchIntegration.execute({ @@ -477,7 +446,6 @@ describe('live organization search policies', () => { expect(dbChainMockFns.transaction).not.toHaveBeenCalled() }) it('reads saved policies alongside inherited approval without requiring an index', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) queueTableRows(member, [{ role: 'member' }]) queueTableRows(organizationSearchIntegration, [{ connectorType: 'gmail', approved: true }]) queueTableRows(organization, [ diff --git a/apps/sim/lib/knowledge/application/search-integrations.ts b/apps/sim/lib/knowledge/application/search-integrations.ts index 9456a2de42f..421eda03983 100644 --- a/apps/sim/lib/knowledge/application/search-integrations.ts +++ b/apps/sim/lib/knowledge/application/search-integrations.ts @@ -3,7 +3,6 @@ import { requirePrincipalSubjectUserId } from '@sim/auth/principal' import { db } from '@sim/db' import { organization, organizationSearchIntegration } from '@sim/db/schema' import { eq, sql } from 'drizzle-orm' -import { isLiveEnterpriseSearchEnabled } from '@/lib/core/config/env-flags' import { OrchestrationError } from '@/lib/core/orchestration/types' import { CredentialGroupProviderConfigurationError } from '@/lib/credential-groups/provider-adapter' import { isScopedCredentialGroupsAvailable } from '@/lib/credential-groups/scoped-availability' @@ -14,7 +13,6 @@ import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { listOrganizationSearchApprovals } from '@/lib/knowledge/search/integration-policy' import { refuseCapability } from '@/lib/permission-groups/capabilities' import { isOrganizationCapabilityWithheld } from '@/lib/permission-groups/capability-assertions' -import { SEARCH_SOURCE_TYPES } from '@/lib/sim-search/connectors' import { NativeSearchError } from '@/lib/sim-search/live/http' import { addOrganizationSearchMcpProvider, @@ -52,24 +50,19 @@ export const listSearchIntegrations = defineAuthorizedKnowledgeUseCase({ if (!context.organizationId) throw new OrchestrationError('validation', 'Organization is required') const approvals = await listOrganizationSearchApprovals(context.organizationId) - const policies = isLiveEnterpriseSearchEnabled - ? await loadLiveSearchPolicies({ organizationId: context.organizationId }) - : undefined + const policies = await loadLiveSearchPolicies({ organizationId: context.organizationId }) const scope = { kind: 'organization', organizationId: context.organizationId } as const - const zoomEnabled = - !isLiveEnterpriseSearchEnabled || (await isSearchProviderEnabled('zoom', scope)) - return (isLiveEnterpriseSearchEnabled ? LIVE_SEARCH_SOURCE_TYPES : SEARCH_SOURCE_TYPES).map( - ([connectorType]) => ({ - connectorType, - approved: approvals.get(connectorType) ?? false, - ...(policies - ? { - policy: livePolicyFor(policies, connectorType), - available: connectorType !== 'zoom' || zoomEnabled, - } - : {}), - }) - ) + const zoomEnabled = await isSearchProviderEnabled('zoom', scope) + return LIVE_SEARCH_SOURCE_TYPES.map(([connectorType]) => ({ + connectorType, + approved: approvals.get(connectorType) ?? false, + ...(policies + ? { + policy: livePolicyFor(policies, connectorType), + available: connectorType !== 'zoom' || zoomEnabled, + } + : {}), + })) }, }) @@ -81,14 +74,11 @@ export const approveSearchIntegration = defineAuthorizedKnowledgeUseCase({ async execute({ input, context, principal }) { if (!context.organizationId) throw new OrchestrationError('validation', 'Organization is required') - const source = ( - isLiveEnterpriseSearchEnabled ? LIVE_SEARCH_SOURCE_TYPES : SEARCH_SOURCE_TYPES - ).find(([type]) => type === input.connectorType) + const source = LIVE_SEARCH_SOURCE_TYPES.find(([type]) => type === input.connectorType) if (!source) { throw new OrchestrationError('validation', 'This integration is not supported by Sim Search') } if ( - isLiveEnterpriseSearchEnabled && input.approved && !(await isSearchProviderEnabled(input.connectorType, { kind: 'organization', @@ -99,16 +89,10 @@ export const approveSearchIntegration = defineAuthorizedKnowledgeUseCase({ 'forbidden', 'Zoom Search is not available for this organization' ) - if (input.policy && !isLiveEnterpriseSearchEnabled) - throw new OrchestrationError('validation', 'Live search settings are not enabled') - const memberProvider = - isLiveEnterpriseSearchEnabled && input.approved - ? liveSearchMemberAccountProvider(input.connectorType) - : null - const mcpProvider = - isLiveEnterpriseSearchEnabled && input.approved - ? liveSearchMcpConnector(input.connectorType) - : null + const memberProvider = input.approved + ? liveSearchMemberAccountProvider(input.connectorType) + : null + const mcpProvider = input.approved ? liveSearchMcpConnector(input.connectorType) : null if (memberProvider || mcpProvider) { /** permission-group-enforced: integrations.manage — adding sign-in is part of this explicit source action. */ if (await isOrganizationCapabilityWithheld(context.organizationId, 'integrations.manage')) diff --git a/apps/sim/lib/knowledge/application/search-source-overview.ts b/apps/sim/lib/knowledge/application/search-source-overview.ts deleted file mode 100644 index dbc4f92ec0a..00000000000 --- a/apps/sim/lib/knowledge/application/search-source-overview.ts +++ /dev/null @@ -1,220 +0,0 @@ -import { db } from '@sim/db' -import { document, embedding, knowledgeBase, knowledgeConnector } from '@sim/db/schema' -import { and, eq, exists, inArray, isNull, notInArray, or } from 'drizzle-orm' -import type { SearchSourceOverview } from '@/lib/api/contracts/knowledge/connectors' -import { type ResourceOwner, resourceScopeFromOwner } from '@/lib/core/resource-scope' -import { resourceScopeCondition } from '@/lib/core/resource-scope.server' -import { resolveKnowledgeAccessAvailability } from '@/lib/knowledge/access/availability' -import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' -import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' -import { resolveKnowledgeOwnerContext } from '@/lib/knowledge/application/contexts' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { instrumentSourceOverviewUseCase } from '@/lib/knowledge/application/search-diagnostics' -import { MAX_SEARCH_SOURCE_PROVIDER_TYPES } from '@/lib/knowledge/constants' -import { knowledgeReadAccessBatches } from '@/lib/knowledge/read-access' -import { annotateSearchDiagnostics, measureSearchStage } from '@/lib/knowledge/search/diagnostics' -import { searchIntegrationAccessCondition } from '@/lib/knowledge/search/integration-policy' -import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' - -function searchProviderTypes() { - const providerTypes = Object.keys(CONNECTOR_META_REGISTRY) - if (providerTypes.length > MAX_SEARCH_SOURCE_PROVIDER_TYPES) { - throw new Error('Search provider catalog exceeds the overview bound') - } - return providerTypes -} - -/** Live search-index sources of a known provider, over `knowledge_connector` joined to its base. */ -function configuredSearchSourceCondition(owner: ResourceOwner, providerTypes: string[]) { - return and( - resourceScopeCondition(knowledgeBase, resourceScopeFromOwner(owner)), - eq(knowledgeBase.isSearchIndex, true), - isNull(knowledgeBase.deletedAt), - inArray(knowledgeConnector.connectorType, providerTypes), - inArray(knowledgeConnector.accessMode, ['admin', 'members']), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) -} - -function configuredProvidersQuery() { - return db - .selectDistinct({ connectorType: knowledgeConnector.connectorType }) - .from(knowledgeConnector) - .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) -} - -/** - * Provider types with at least one configured search source in an already authorized owner. - * Reads no documents, so callers needing only setup state avoid the overview's access probes. - */ -export async function listConfiguredSearchProviderTypes(owner: ResourceOwner): Promise { - const rows = await configuredProvidersQuery() - .where(configuredSearchSourceCondition(owner, searchProviderTypes())) - .limit(MAX_SEARCH_SOURCE_PROVIDER_TYPES) - return rows.map(({ connectorType }) => connectorType) -} - -/** Provider-level existence probes keep setup cards independent of the loaded source pages. */ -export const readSearchSourceOverview = instrumentSourceOverviewUseCase( - defineAuthorizedKnowledgeUseCase({ - operation: knowledgeOperations.readSearchSourceOverview, - resolveContext: ({ input }: { input: ResourceOwner }) => resolveKnowledgeOwnerContext(input), - async execute({ principal, context }): Promise { - const availability = await measureSearchStage('source_overview.availability', () => - resolveKnowledgeAccessAvailability(context) - ) - const access = createKnowledgeAccessProvider(principal, context) - const providerTypes = searchProviderTypes() - const configured = configuredSearchSourceCondition(context, providerTypes) - const available = or( - availability.memberScoped ? eq(knowledgeConnector.accessMode, 'members') : undefined, - availability.sourceMirrored - ? and( - eq(knowledgeConnector.accessMode, 'admin'), - inArray( - knowledgeConnector.connectorType, - providerTypes.filter( - (type) => - availability.memberScoped || - !CONNECTOR_META_REGISTRY[type].requiresMemberIdentity - ) - ) - ) - : undefined - ) - const syncingEnabled = and( - available, - searchIntegrationAccessCondition(), - notInArray(knowledgeConnector.status, ['paused', 'disabled']), - or( - eq(knowledgeConnector.accessMode, 'admin'), - notInArray(knowledgeConnector.memberSyncStatus, ['disabled']) - ) - ) - const documentConditions = and( - eq(document.connectorId, knowledgeConnector.id), - eq(document.knowledgeBaseId, knowledgeConnector.knowledgeBaseId), - eq(document.enabled, true), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - const providers = await measureSearchStage('source_overview.providers', () => - configuredProvidersQuery().where(configured).limit(MAX_SEARCH_SOURCE_PROVIDER_TYPES) - ) - annotateSearchDiagnostics({ configuredProviderCount: providers.length }) - const indexingTypes = new Set() - let searchableProbes = 0 - let hasSearchableDocuments = false - const probesSources: boolean = availability.memberScoped || availability.sourceMirrored - /** One searchable document is the whole answer, so later batches skip the probe entirely. */ - const probesSearchable = (): boolean => probesSources && !hasSearchableDocuments - /** - * A provider type is only read back as membership of `indexingTypes`, so once every - * configured type is in the set no later batch can change the answer. - */ - const probesIndexing = (): boolean => - probesSources && providers.some(({ connectorType }) => !indexingTypes.has(connectorType)) - for await (const accessCondition of knowledgeReadAccessBatches(access, [ - configured, - available, - documentConditions, - ])) { - const readableDocument = and(documentConditions, accessCondition) - const probesSearchableNow = probesSearchable() - if (probesSearchableNow) searchableProbes += 1 - /** Annotated so the searchable probe's guard does not infer through its own result. */ - const [indexing, searchable]: [{ connectorType: string }[], { id: string }[]] = - await Promise.all([ - probesIndexing() - ? measureSearchStage('source_overview.indexing', () => - configuredProvidersQuery() - .where( - and( - configured, - syncingEnabled, - /** - * The probe narrows the configured set the provider list came from, so a - * type already found stays found; excluding it only drops repeated work. - * An empty set adds no predicate rather than a no-op one. - */ - indexingTypes.size > 0 - ? notInArray(knowledgeConnector.connectorType, [...indexingTypes]) - : undefined, - or( - inArray(knowledgeConnector.status, ['pending', 'syncing']), - and( - eq(knowledgeConnector.accessMode, 'members'), - inArray(knowledgeConnector.memberSyncStatus, ['pending', 'running']) - ), - exists( - db - .select({ id: document.id }) - .from(document) - .where( - and( - readableDocument, - inArray(document.processingStatus, ['pending', 'processing']) - ) - ) - ) - ) - ) - ) - .limit(MAX_SEARCH_SOURCE_PROVIDER_TYPES) - ) - : [], - probesSearchableNow - ? measureSearchStage('source_overview.searchable', () => - db - .select({ id: document.id }) - .from(document) - .innerJoin(knowledgeConnector, eq(knowledgeConnector.id, document.connectorId)) - .innerJoin( - knowledgeBase, - eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId) - ) - .where( - and( - configured, - available, - readableDocument, - eq(document.processingStatus, 'completed'), - exists( - db - .select({ id: embedding.id }) - .from(embedding) - .where( - and( - eq(embedding.documentId, document.id), - eq(embedding.enabled, true) - ) - ) - ) - ) - ) - .limit(1) - ) - : [], - ]) - for (const provider of indexing) indexingTypes.add(provider.connectorType) - hasSearchableDocuments ||= searchable.length > 0 - /** - * Both probes are saturated, so every remaining batch would be discovered and live-proved - * for no probe. `accessBatchCount` and `liveProofConnectorCount` stay what they document: - * the batches and proofs this read actually spent, not the batches the owner could produce. - */ - if (!probesSearchable() && !probesIndexing()) break - } - annotateSearchDiagnostics({ searchableProbeCount: searchableProbes }) - return { - providers: providers.map(({ connectorType }) => ({ - connectorType, - isSyncing: indexingTypes.has(connectorType), - })), - hasSearchableDocuments, - } - }, - }) -) diff --git a/apps/sim/lib/knowledge/application/search-source-progress.ts b/apps/sim/lib/knowledge/application/search-source-progress.ts deleted file mode 100644 index 89ad3efa680..00000000000 --- a/apps/sim/lib/knowledge/application/search-source-progress.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { requirePrincipalSubjectUserId } from '@sim/auth/principal' -import { db } from '@sim/db' -import { document, knowledgeBase, knowledgeConnector } from '@sim/db/schema' -import { and, eq, exists, inArray, isNull, type SQL, sql } from 'drizzle-orm' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { type ResourceOwner, resourceScopeFromOwner } from '@/lib/core/resource-scope' -import { resourceScopeCondition } from '@/lib/core/resource-scope.server' -import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' -import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' -import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' -import { resolveKnowledgeOwnerContext } from '@/lib/knowledge/application/contexts' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { hasViewerMemberSyncError } from '@/lib/knowledge/connectors/viewer-member-sync-error' -import { MAX_SEARCH_SOURCE_PROGRESS_ITEMS } from '@/lib/knowledge/constants' -import { failedDocumentCondition } from '@/lib/knowledge/documents/processing-status' -import { searchIntegrationAccessCondition } from '@/lib/knowledge/search/integration-policy' - -interface ReadSearchSourceProgressInput extends ResourceOwner { - connectorIds: string[] -} - -/** Progress probes stop at the first visible document, without recounting completed chunks. */ -export const readSearchSourceProgress = defineAuthorizedKnowledgeUseCase({ - operation: knowledgeOperations.readSearchSourceProgress, - resolveContext: ({ input }: { input: ReadSearchSourceProgressInput }) => - resolveKnowledgeOwnerContext(input), - async execute({ principal, input, context }) { - if ( - input.connectorIds.length === 0 || - input.connectorIds.length > MAX_SEARCH_SOURCE_PROGRESS_ITEMS - ) { - throw new OrchestrationError( - 'validation', - `Provide between 1 and ${MAX_SEARCH_SOURCE_PROGRESS_ITEMS} sources` - ) - } - const access = await createKnowledgeAccessProvider(principal, context).getForConnectors( - input.connectorIds - ) - const hasDocumentsInState = (condition: SQL) => - sql`${exists( - db - .select({ id: document.id }) - .from(document) - .where( - and( - eq(document.knowledgeBaseId, knowledgeConnector.knowledgeBaseId), - eq(document.connectorId, knowledgeConnector.id), - condition, - eq(document.enabled, true), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - knowledgeAccessCondition(access) - ) - ) - )}` - const rows = await db - .select({ - connectorId: knowledgeConnector.id, - status: knowledgeConnector.status, - accessMode: knowledgeConnector.accessMode, - memberSyncStatus: knowledgeConnector.memberSyncStatus, - hasRetainedSyncError: sql`${knowledgeConnector.lastSyncError} IS NOT NULL`, - hasViewerMemberSyncError: hasViewerMemberSyncError( - requirePrincipalSubjectUserId(principal) - ), - approved: sql`${searchIntegrationAccessCondition()}`, - isIndexing: hasDocumentsInState( - inArray(document.processingStatus, ['pending', 'processing']) - ), - hasIndexingError: hasDocumentsInState(failedDocumentCondition()), - }) - .from(knowledgeConnector) - .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) - .where( - and( - resourceScopeCondition(knowledgeBase, resourceScopeFromOwner(context)), - eq(knowledgeBase.isSearchIndex, true), - isNull(knowledgeBase.deletedAt), - inArray(knowledgeConnector.id, [...new Set(input.connectorIds)]), - inArray(knowledgeConnector.accessMode, ['admin', 'members']), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .limit(MAX_SEARCH_SOURCE_PROGRESS_ITEMS) - return { - sources: rows.map((row) => ({ - connectorId: row.connectorId, - isSyncing: - row.approved && - row.status !== 'paused' && - row.status !== 'disabled' && - !(row.accessMode === 'members' && row.memberSyncStatus === 'disabled') && - (row.status === 'pending' || - row.status === 'syncing' || - (row.accessMode === 'members' && - (row.memberSyncStatus === 'pending' || row.memberSyncStatus === 'running')) || - row.isIndexing), - hasSyncError: - row.status === 'error' || - row.hasRetainedSyncError === true || - row.hasViewerMemberSyncError === true || - (row.accessMode === 'members' && row.memberSyncStatus === 'error'), - hasIndexingError: row.hasIndexingError, - })), - } - }, -}) diff --git a/apps/sim/lib/knowledge/application/search-sources.test.ts b/apps/sim/lib/knowledge/application/search-sources.test.ts index e4c0154fc18..7792aeeac6e 100644 --- a/apps/sim/lib/knowledge/application/search-sources.test.ts +++ b/apps/sim/lib/knowledge/application/search-sources.test.ts @@ -1,14 +1,10 @@ -import { knowledgeBase, knowledgeConnector, member, user } from '@sim/db/schema' +import { knowledgeConnector, member } from '@sim/db/schema' import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' import { createPersonalApiKeyPrincipal, createSessionPrincipal, createWorkspaceApiKeyPrincipal, } from '@sim/testing/factories/principal.factory' -import { - knowledgeAccessScopeMock, - knowledgeAccessScopeMockFns, -} from '@sim/testing/mocks/knowledge-access-scope.mock' import { knowledgeAvailabilityMock, knowledgeAvailabilityMockFns, @@ -21,26 +17,10 @@ import { permissionGroupsResolveMock } from '@sim/testing/mocks/permission-group import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' -const hoisted = vi.hoisted(() => ({ - memberships: vi.fn(), - accounts: vi.fn(), - predicate: vi.fn(), -})) vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) vi.mock('@/lib/permission-groups/resolve.server', () => permissionGroupsResolveMock) vi.mock('@/lib/knowledge/application/contexts', () => knowledgeContextsMock) vi.mock('@/lib/knowledge/access/availability', () => knowledgeAvailabilityMock) -vi.mock('@/lib/knowledge/connectors/member-provisioning', () => ({ - resolveViewerConnectorMemberships: hoisted.memberships, -})) -vi.mock('@/lib/knowledge/connectors/viewer-source-accounts', () => ({ - resolveViewerSourceAccounts: hoisted.accounts, -})) -vi.mock('@/lib/knowledge/access/scope', () => knowledgeAccessScopeMock) -vi.mock('@/lib/knowledge/access/predicate', () => ({ - knowledgeAccessCondition: hoisted.predicate, - knowledgeMetadataCandidateAccessCondition: hoisted.predicate, -})) vi.mock('@/connectors/registry', () => { const registry = { google_drive: { id: 'google_drive', search: true, configFields: [{ id: 'folderId' }] }, @@ -60,24 +40,14 @@ vi.mock('@/connectors/registry', () => { } }) -import { searchSourceSummarySchema } from '@/lib/api/contracts/knowledge/connectors' -import { readSearchSourceOverview } from '@/lib/knowledge/application/search-source-overview' -import { readSearchSourceProgress } from '@/lib/knowledge/application/search-source-progress' import { listSearchSources } from '@/lib/knowledge/application/search-sources' -const mocks = { - ...hoisted, - access: knowledgeAccessScopeMockFns.mockCreateKnowledgeAccessProvider, -} - workspaceAuthzMockFns.mockPermissionSatisfies.mockImplementation( (actual: string | null) => actual !== null ) const principal = createSessionPrincipal({ userId: 'reader', sessionId: 'session' }) const input = { workspaceId: 'workspace' } -const access = { kind: 'user', userId: principal.userId, tokens: ['u:reader@example.test'] } -const ACL = { type: 'viewer-acl' } const LAST_SYNC = new Date('2026-09-05T12:00:00.000Z') function source(id: string, connectorType = 'google_drive', accessMode = 'admin') { @@ -103,9 +73,8 @@ function source(id: string, connectorType = 'google_drive', accessMode = 'admin' } } -function seed(rows: ReturnType[], emailVerified = true) { +function seed(rows: ReturnType[]) { queueTableRows(knowledgeConnector, rows) - queueTableRows(user, [{ emailVerified }]) } beforeEach(() => { @@ -120,15 +89,6 @@ beforeEach(() => { sourceMirrored: true, memberScoped: true, }) - mocks.memberships.mockResolvedValue(new Map()) - mocks.accounts.mockResolvedValue(new Map()) - mocks.access.mockReturnValue({ - get: async () => access, - getForConnectors: async () => access, - getForDocuments: async () => access, - liveSourceConnectorCondition: async () => null, - }) - mocks.predicate.mockReturnValue(ACL) }) describe('Search source summaries', () => { @@ -137,9 +97,6 @@ describe('Search source summaries', () => { async (role) => { workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue(role) seed([source('drive')]) - queueTableRows(knowledgeConnector, [ - { connectorId: 'drive', hasDocuments: true, failedCount: 0, isIndexing: false }, - ]) const result = await listSearchSources.execute({ principal, input }) expect(result.sources).toEqual([ { @@ -151,71 +108,21 @@ describe('Search source summaries', () => { isGitHubInstallation: false, availability: 'available', enabled: true, - isSyncing: false, - lastSyncAt: LAST_SYNC.toISOString(), - hasSyncError: false, - hasViewerDocuments: true, - viewerFailedDocumentCount: 0, - viewerEmailVerified: true, - viewerAccounts: [], - connectionRequired: false, - viewerMembership: null, }, ]) - expect(searchSourceSummarySchema.parse(result.sources[0])).toEqual(result.sources[0]) expect(JSON.stringify(result)).not.toMatch( /secret-fixture|admin@example|group-secret|option-secret|sourceConfig/ ) expect(knowledgeContextsMockFns.mockResolveKnowledgeOwnerContext).toHaveBeenCalledWith(input) - expect(mocks.access).toHaveBeenCalledWith(principal, { - workspaceId: 'workspace', - workspaceOrganizationId: null, - allowPersonalApiKeys: true, - }) - expect(mocks.predicate).toHaveBeenCalledWith(access) } ) - it('surfaces retained partial sync errors without returning the private error message', async () => { - seed([{ ...source('drive'), hasRetainedSyncError: true }]) - const result = await listSearchSources.execute({ principal, input }) - expect(result.sources[0].hasSyncError).toBe(true) - expect(result.sources[0]).not.toHaveProperty('lastSyncError') - }) - - it('restricts the source query to this workspace, the Search index, and live configured sources', async () => { - seed([]) - await expect(listSearchSources.execute({ principal, input })).resolves.toEqual({ - sources: [], - nextCursor: null, - }) - expect(dbChainMockFns.where).toHaveBeenCalledWith({ - type: 'and', - conditions: expect.arrayContaining([ - { - type: 'and', - conditions: [ - { type: 'eq', left: knowledgeBase.workspaceId, right: 'workspace' }, - { type: 'isNull', column: knowledgeBase.organizationId }, - ], - }, - { type: 'eq', left: knowledgeBase.isSearchIndex, right: true }, - { type: 'isNull', column: knowledgeBase.deletedAt }, - { type: 'isNull', column: knowledgeConnector.archivedAt }, - { type: 'isNull', column: knowledgeConnector.deletedAt }, - { type: 'inArray', column: knowledgeConnector.accessMode, values: ['admin', 'members'] }, - ]), - }) - expect(mocks.memberships).not.toHaveBeenCalled() - }) - it('rejects a former workspace member before querying source data', async () => { workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) await expect(listSearchSources.execute({ principal, input })).rejects.toMatchObject({ code: 'forbidden', }) expect(dbChainMockFns.select).not.toHaveBeenCalled() - expect(mocks.memberships).not.toHaveBeenCalled() }) it.each([ @@ -240,31 +147,20 @@ describe('Search source summaries', () => { describe('organization Search source summaries', () => { it.each(['member', 'admin'])( - 'returns only the current %s viewer ACL counts without a workspace membership', + 'returns configured sources to a current organization %s without workspace membership', async (role) => { knowledgeContextsMockFns.mockResolveKnowledgeOwnerContext.mockResolvedValue({ organizationId: 'org-1', }) queueTableRows(member, [{ role }]) seed([source('drive')]) - queueTableRows(knowledgeConnector, []) - queueTableRows(knowledgeConnector, [ - { connectorId: 'drive', hasDocuments: true, failedCount: 0, isIndexing: false }, - ]) const result = await listSearchSources.execute({ principal, input: { organizationId: 'org-1' }, }) expect(result.sources[0]).toMatchObject({ connectorId: 'drive', - hasViewerDocuments: true, - viewerEmailVerified: true, }) - expect(mocks.access).toHaveBeenCalledWith(principal, { organizationId: 'org-1' }) - expect(mocks.predicate).toHaveBeenCalledWith(access) - expect(mocks.memberships).toHaveBeenCalledWith( - expect.objectContaining({ organizationId: 'org-1', userId: 'reader' }) - ) expect(workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission).not.toHaveBeenCalled() expect(JSON.stringify(result)).not.toMatch( /secret-fixture|admin@example|group-secret|option-secret/ @@ -280,28 +176,6 @@ describe('organization Search source summaries', () => { await expect( listSearchSources.execute({ principal, input: { organizationId: 'org-1' } }) ).rejects.toThrow('Organization not found') - expect(mocks.memberships).not.toHaveBeenCalled() - expect(mocks.access).not.toHaveBeenCalled() - }) -}) - -describe('bounded Search progress', () => { - it('does not read progress for a former member', async () => { - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) - await expect( - readSearchSourceProgress.execute({ principal, input: { ...input, connectorIds: ['drive'] } }) - ).rejects.toMatchObject({ code: 'forbidden' }) - expect(mocks.access).not.toHaveBeenCalled() - }) - - it('rejects an oversized progress request before document work', async () => { - await expect( - readSearchSourceProgress.execute({ - principal, - input: { ...input, connectorIds: Array(101).fill('drive') }, - }) - ).rejects.toMatchObject({ code: 'validation' }) - expect(mocks.access).not.toHaveBeenCalled() }) }) @@ -327,7 +201,7 @@ describe('bounded Search source pagination', () => { input: { ...input, cursor: first.nextCursor!, - ...(change === 'filter' ? { mine: true } : {}), + ...(change === 'filter' ? { search: 'new' } : {}), ...(change === 'provider' ? { connectorType: 'gmail' } : {}), ...(change === 'excluded-provider' ? { excludeConnectorType: 'github' } : {}), }, @@ -337,14 +211,3 @@ describe('bounded Search source pagination', () => { } ) }) - -describe('Search source overview', () => { - it('rechecks current membership before reading the overview', async () => { - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) - await expect(readSearchSourceOverview.execute({ principal, input })).rejects.toMatchObject({ - code: 'forbidden', - }) - expect(mocks.access).not.toHaveBeenCalled() - expect(dbChainMockFns.select).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/knowledge/application/search-sources.ts b/apps/sim/lib/knowledge/application/search-sources.ts index 637ccc9145e..3a725f8c53f 100644 --- a/apps/sim/lib/knowledge/application/search-sources.ts +++ b/apps/sim/lib/knowledge/application/search-sources.ts @@ -1,8 +1,8 @@ import { requirePrincipalSubjectUserId } from '@sim/auth/principal' import { db } from '@sim/db' -import { document, embedding, knowledgeBase, knowledgeConnector, user } from '@sim/db/schema' +import { knowledgeBase, knowledgeConnector } from '@sim/db/schema' import { toRecord } from '@sim/utils/object' -import { and, desc, eq, exists, inArray, isNull, lt, ne, or, type SQL, sql } from 'drizzle-orm' +import { and, desc, eq, inArray, isNull, lt, ne, or, sql } from 'drizzle-orm' import { listSearchSourcesContract, searchSourceCursorSchema, @@ -12,19 +12,13 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { type ResourceOwner, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { resolveKnowledgeAccessAvailability } from '@/lib/knowledge/access/availability' -import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' -import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { resolveKnowledgeOwnerContext } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { resolveViewerConnectorMemberships } from '@/lib/knowledge/connectors/member-provisioning' -import { hasViewerMemberSyncError } from '@/lib/knowledge/connectors/viewer-member-sync-error' -import { resolveViewerSourceAccounts } from '@/lib/knowledge/connectors/viewer-source-accounts' import { SEARCH_SOURCE_CANDIDATE_PAGE_SIZE, SEARCH_SOURCE_PAGE_SIZE, } from '@/lib/knowledge/constants' -import { failedDocumentCondition } from '@/lib/knowledge/documents/processing-status' import { listOrganizationSearchApprovals } from '@/lib/knowledge/search/integration-policy' import { describeSearchSource } from '@/lib/sim-search/source-identity' import { getConnectorMeta } from '@/connectors/registry' @@ -35,10 +29,9 @@ export interface ListSearchSourcesInput extends ResourceOwner { connectorType?: string excludeConnectorType?: string search?: string - mine?: boolean } -/** Viewer-safe setup and indexing state; source credentials and other members never leave this use case. */ +/** Bounded live source configuration; credentials and account identities remain private. */ export const listSearchSources = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.listSearchSources, resolveContext: ({ input }: { input: ListSearchSourcesInput }) => @@ -56,7 +49,6 @@ export const listSearchSources = defineAuthorizedKnowledgeUseCase({ connectorType: connectorType ?? '', ...(excludeConnectorType ? { excludeConnectorType } : {}), connectorId: input.connectorId ?? '', - mine: input.mine === true, order: 'newest', }) const cursor = (() => { @@ -84,12 +76,6 @@ export const listSearchSources = defineAuthorizedKnowledgeUseCase({ accessMode: knowledgeConnector.accessMode, status: knowledgeConnector.status, memberSyncStatus: knowledgeConnector.memberSyncStatus, - lastSyncAt: knowledgeConnector.lastSyncAt, - hasRetainedSyncError: sql`${knowledgeConnector.lastSyncError} IS NOT NULL`, - hasViewerMemberSyncError: hasViewerMemberSyncError(userId), - lastMemberSyncAt: knowledgeConnector.lastMemberSyncAt, - credentialGroupId: knowledgeConnector.credentialGroupId, - credentialGroupOptionId: knowledgeConnector.credentialGroupOptionId, }) .from(knowledgeConnector) .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) @@ -122,38 +108,11 @@ export const listSearchSources = defineAuthorizedKnowledgeUseCase({ if (candidates.length === 0) return { sources: [], nextCursor: null } const scanned = candidates.slice(0, SEARCH_SOURCE_CANDIDATE_PAGE_SIZE) - const [availability, memberships, viewers, approvals, accounts] = await Promise.all([ + const [availability, approvals] = await Promise.all([ resolveKnowledgeAccessAvailability(context), - resolveViewerConnectorMemberships({ - userId: userId, - workspaceId: context.workspaceId, - organizationId: context.organizationId, - connectors: scanned, - }), - db - .select({ emailVerified: user.emailVerified }) - .from(user) - .where(eq(user.id, userId)) - .limit(1), context.organizationId ? listOrganizationSearchApprovals(context.organizationId) : null, - context.organizationId - ? resolveViewerSourceAccounts({ - organizationId: context.organizationId, - userId: userId, - connectors: scanned, - }) - : new Map(), ]) - /** Owned grants stay manageable even when the source can no longer authorize Search. */ const matches = scanned.filter((row) => { - const membership = memberships.get(row.id) - if ( - input.mine && - (context.organizationId - ? !accounts.has(row.id) - : membership !== 'connected' && membership !== 'needs_reauth') - ) - return false const meta = getConnectorMeta(row.connectorType) const label = meta ? `${meta.name ?? row.connectorType} ${describeSearchSource(meta, row.sourceConfig)}` @@ -174,67 +133,6 @@ export const listSearchSources = defineAuthorizedKnowledgeUseCase({ ).toString('base64url') : null if (rows.length === 0) return { sources: [], nextCursor } - const access = await createKnowledgeAccessProvider(principal, context).getForConnectors( - rows.map((row) => row.id) - ) - /** - * Per-source probes rather than one aggregate: the existence checks stop at the first - * visible document, and the failed and in-progress rows are few and indexed, so the cost no - * longer grows with every document the viewer can read. - */ - const readable = knowledgeAccessCondition(access) - const viewerDocument = (condition: SQL | undefined) => - and( - eq(document.connectorId, knowledgeConnector.id), - eq(document.enabled, true), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - condition, - readable - ) - const documentStates = await db - .select({ - connectorId: knowledgeConnector.id, - hasDocuments: sql`${exists( - db - .select({ id: document.id }) - .from(document) - .where( - viewerDocument( - and( - eq(document.processingStatus, 'completed'), - exists( - db - .select({ id: embedding.id }) - .from(embedding) - .where( - and(eq(embedding.documentId, document.id), eq(embedding.enabled, true)) - ) - ) - ) - ) - ) - )}`, - failedCount: sql`${db - .select({ count: sql`count(*)::int` }) - .from(document) - .where(viewerDocument(failedDocumentCondition()))}`, - isIndexing: sql`${exists( - db - .select({ id: document.id }) - .from(document) - .where(viewerDocument(inArray(document.processingStatus, ['pending', 'processing']))) - )}`, - }) - .from(knowledgeConnector) - .where( - inArray( - knowledgeConnector.id, - rows.map((row) => row.id) - ) - ) - const states = new Map(documentStates.map((state) => [state.connectorId, state])) return { nextCursor, @@ -252,7 +150,6 @@ export const listSearchSources = defineAuthorizedKnowledgeUseCase({ row.status !== 'paused' && row.status !== 'disabled' && (row.accessMode !== 'members' || row.memberSyncStatus !== 'disabled') - const state = states.get(row.id) const source = { knowledgeBaseId: row.knowledgeBaseId, connectorId: row.id, @@ -266,39 +163,8 @@ export const listSearchSources = defineAuthorizedKnowledgeUseCase({ availability: available ? ('available' as const) : ('unavailable' as const), enabled, ...(approvals ? { approved: approvals.get(row.connectorType) ?? true } : {}), - isSyncing: - available && - enabled && - approvals?.get(row.connectorType) !== false && - (row.status === 'pending' || - row.status === 'syncing' || - (row.accessMode === 'members' && - (row.memberSyncStatus === 'pending' || row.memberSyncStatus === 'running')) || - state?.isIndexing === true), - lastSyncAt: - (row.accessMode === 'members' ? row.lastMemberSyncAt : row.lastSyncAt)?.toISOString() ?? - null, - hasSyncError: - row.status === 'error' || - row.hasRetainedSyncError === true || - row.hasViewerMemberSyncError === true || - (row.accessMode === 'members' && row.memberSyncStatus === 'error'), - hasViewerDocuments: available && state?.hasDocuments === true, - viewerFailedDocumentCount: available ? (state?.failedCount ?? 0) : 0, - viewerEmailVerified: viewers[0]?.emailVerified === true, - viewerAccounts: accounts.get(row.id) ?? [], } as const - return [ - { - ...source, - ...(connectionRequired - ? { - connectionRequired: true as const, - viewerMembership: available ? (memberships.get(row.id) ?? null) : null, - } - : { connectionRequired: false as const, viewerMembership: null }), - }, - ] + return [source] }), } }, diff --git a/apps/sim/lib/knowledge/application/search.test.ts b/apps/sim/lib/knowledge/application/search.test.ts index 48f1b6e06ee..4134e49312d 100644 --- a/apps/sim/lib/knowledge/application/search.test.ts +++ b/apps/sim/lib/knowledge/application/search.test.ts @@ -17,7 +17,6 @@ import { billingUsageMonitorMock, billingUsageMonitorMockFns, } from '@sim/testing/mocks/billing-usage-monitor.mock' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' import { knowledgeAvailabilityMock, knowledgeAvailabilityMockFns, @@ -41,7 +40,7 @@ import { import { permissionGroupsResolveMock } from '@sim/testing/mocks/permission-groups-resolve.mock' import { getMockPlatformEvent, telemetryMock } from '@sim/testing/mocks/telemetry.mock' import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' import { OrchestrationError } from '@/lib/core/orchestration/types' const hoisted = vi.hoisted(() => ({ @@ -52,15 +51,10 @@ const hoisted = vi.hoisted(() => ({ getTagDefinitions: vi.fn(), importProvenance: vi.fn(), rerank: vi.fn(), - recordActivity: vi.fn(), })) vi.mock('@/lib/core/telemetry', () => telemetryMock) -vi.mock('@/lib/knowledge/search/activity', () => ({ - recordOrganizationSearchActivity: hoisted.recordActivity, -})) - vi.mock('@/lib/knowledge/reranker', () => ({ hasRerankerCredential: hoisted.hasRerankerCredential, rerank: hoisted.rerank, @@ -236,42 +230,6 @@ describe('knowledge search application use case', () => { } ) - describe.each(['workspace', 'organization'] as const)('%s ranking policy', (scope) => { - beforeEach(() => { - if (scope === 'organization') { - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) - mocks.getKnowledgeBase.mockResolvedValue({ - ...knowledgeBase, - workspaceId: null, - organizationId: 'org-canonical', - isSearchIndex: true, - }) - queueTableRows(member, [{ role: 'member' }]) - } - }) - afterEach(resetEnvFlagsMock) - - it('meters only successful organization calls under the acting person', async () => { - await searchKnowledge.execute({ - principal: createSessionPrincipal(), - input: { knowledgeBaseIds: ['knowledge-1'], query: 'answer', topK: 10, surface: 'mcp' }, - }) - if (scope === 'organization') { - expect(mocks.recordActivity).toHaveBeenCalledExactlyOnceWith({ - organizationId: 'org-canonical', - userId: 'user-1', - surface: 'mcp', - results: expect.any(Array), - }) - } else { - expect(mocks.recordActivity).not.toHaveBeenCalled() - } - }) - - const _principal = createSessionPrincipal() - const _input = { knowledgeBaseIds: ['knowledge-1'], query: 'answer', topK: 10 } - }) - it('gates organization search using the persisted owner even when the request omits it', async () => { mocks.getKnowledgeBase.mockResolvedValue({ ...knowledgeBase, @@ -288,7 +246,6 @@ describe('knowledge search application use case', () => { input: { knowledgeBaseIds: ['knowledge-1'], query: 'answer', topK: 5 }, }) ).rejects.toThrow('Search is not enabled for this organization') - expect(mocks.recordActivity).not.toHaveBeenCalled() expect( knowledgeAvailabilityMockFns.mockRequireOrganizationSearchAvailable ).toHaveBeenCalledExactlyOnceWith('org-canonical') @@ -297,10 +254,8 @@ describe('knowledge search application use case', () => { expect(mocks.executeSearch).not.toHaveBeenCalled() }) - describe('while indexed organization search is dormant', () => { + describe('Search-marked knowledge bases named explicitly', () => { const principal = createSessionPrincipal() - beforeEach(() => setEnvFlags({ isLiveEnterpriseSearchEnabled: true })) - afterEach(resetEnvFlagsMock) it.each([ ['an organization', { workspaceId: null, organizationId: 'org-canonical' }], @@ -318,7 +273,7 @@ describe('knowledge search application use case', () => { }) expect(result.results).toHaveLength(1) expect(mocks.executeSearch).toHaveBeenCalledWith( - expect.objectContaining({ knowledgeBaseIds: ['knowledge-1'], indexedRetrieval: false }) + expect.objectContaining({ knowledgeBaseIds: ['knowledge-1'] }) ) }) }) diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index 319e2622ee1..515b3f73826 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -37,7 +37,6 @@ import type { ActiveKnowledgeBaseReference } from '@/lib/knowledge/knowledge-bas import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' import { hasRerankerCredential, rerank } from '@/lib/knowledge/reranker' import type { RerankerStatus } from '@/lib/knowledge/reranker-models' -import { recordOrganizationSearchActivity } from '@/lib/knowledge/search/activity' import { SearchDeadlineError } from '@/lib/knowledge/search/budget' import type { SearchResult } from '@/lib/knowledge/search/candidates' import { resolveKnowledgeSearchDefaults } from '@/lib/knowledge/search/defaults' @@ -53,7 +52,6 @@ import { import { getDocumentTagDefinitionsByKnowledgeBaseIds } from '@/lib/knowledge/tags/service' import type { DocumentTagDefinition } from '@/lib/knowledge/tags/types' import type { StructuredFilter } from '@/lib/knowledge/types' -import { usesIndexedRetrieval } from '@/lib/sim-search/indexed/gate' import { estimateTokenCount } from '@/lib/tokenization/estimators' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { getRerankModelPricing } from '@/providers/models' @@ -473,7 +471,6 @@ export async function runKnowledgeSearch({ } : undefined, structuredFilters: structuredFilters.length > 0 ? structuredFilters : undefined, - indexedRetrieval: usesIndexedRetrieval(context.knowledgeBases), }) ) @@ -770,7 +767,7 @@ export async function runKnowledgeSearch({ } } -/** What follows a completed search on every surface: the organization's activity record and the platform event. */ +/** Records the platform event after an authorized knowledge search. */ export async function afterKnowledgeSearch({ principal, context, @@ -782,17 +779,6 @@ export async function afterKnowledgeSearch({ input: Pick result: SearchKnowledgeResult }): Promise { - const actorUserId = resolvePrincipalSubjectUserId(principal) - if (context.organizationId && actorUserId) { - await measureSearchStage('activity_recording', () => - recordOrganizationSearchActivity({ - organizationId: context.organizationId, - userId: actorUserId, - surface: input.surface ?? 'other', - results: result.results, - }) - ) - } PlatformEvents.knowledgeBaseSearched({ knowledgeBaseId: result.knowledgeBaseId, knowledgeBaseIds: result.knowledgeBaseIds, diff --git a/apps/sim/lib/knowledge/application/sim-search.test.ts b/apps/sim/lib/knowledge/application/sim-search.test.ts index b06c4ef7087..d9bca6e9378 100644 --- a/apps/sim/lib/knowledge/application/sim-search.test.ts +++ b/apps/sim/lib/knowledge/application/sim-search.test.ts @@ -1,4 +1,4 @@ -import { knowledgeBase, knowledgeConnector, member } from '@sim/db/schema' +import { member } from '@sim/db/schema' import { queueTableRows, resetDbChainMock } from '@sim/testing' import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' import { auditMock } from '@sim/testing/mocks/audit.mock' @@ -6,7 +6,6 @@ import { credentialGroupsServiceMock, credentialGroupsServiceMockFns, } from '@sim/testing/mocks/credential-groups-service.mock' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' import { knowledgeAvailabilityMock, knowledgeAvailabilityMockFns, @@ -23,10 +22,6 @@ import { knowledgeEmbeddingsMock, knowledgeEmbeddingsMockFns, } from '@sim/testing/mocks/knowledge-embeddings.mock' -import { - knowledgeSearchIntegrationPolicyMock, - knowledgeSearchIntegrationPolicyMockFns, -} from '@sim/testing/mocks/knowledge-search-integration-policy.mock' import { knowledgeServiceMock, knowledgeServiceMockFns, @@ -36,14 +31,8 @@ import { permissionGroupsResolveMockFns, } from '@sim/testing/mocks/permission-groups-resolve.mock' import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' -import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' -const hoisted = vi.hoisted(() => ({ - createConnector: vi.fn(), - createApprovedSource: vi.fn(), - deleteConnector: vi.fn(), - enroll: vi.fn(), -})) vi.mock('@/lib/credential-groups/service', () => credentialGroupsServiceMock) vi.mock('@sim/audit', () => auditMock) @@ -58,35 +47,12 @@ vi.mock('@/lib/knowledge/service', () => knowledgeServiceMock) vi.mock('@/lib/knowledge/embeddings', () => knowledgeEmbeddingsMock) vi.mock('@/lib/knowledge/application/knowledge-bases', () => knowledgeBaseUseCasesMock) -vi.mock('@/lib/knowledge/search/integration-policy', () => knowledgeSearchIntegrationPolicyMock) - -vi.mock('@/lib/knowledge/application/connectors', () => ({ - createApprovedSearchSource: { execute: hoisted.createApprovedSource }, - createKnowledgeConnector: { execute: hoisted.createConnector }, - deleteKnowledgeConnector: { execute: hoisted.deleteConnector }, -})) - -vi.mock('@/lib/knowledge/application/connector-access', () => ({ - startKnowledgeConnectorMemberEnrollment: { execute: hoisted.enroll }, -})) - vi.mock('@/lib/permission-groups/resolve.server', () => permissionGroupsResolveMock) vi.mock('@/lib/sim-search/connectors', () => ({ SIM_SEARCH_KNOWLEDGE_BASE_NAME: 'Sim Search', canConnectPersonally: (meta: { permissionScopedListing?: unknown }) => Boolean(meta.permissionScopedListing), - withSearchSourceDefaults: ( - meta: { searchDefaultSourceConfig?: Record }, - sourceConfig: Record = {} - ) => ({ ...(meta.searchDefaultSourceConfig ?? {}), ...sourceConfig }), - missingSetupFields: ( - meta: { configFields: Array<{ id: string; title: string; required?: boolean }> }, - sourceConfig: Record - ) => - meta.configFields.filter( - (field) => field.required && typeof sourceConfig[field.id] !== 'string' - ), })) vi.mock('@/connectors/registry', () => ({ @@ -124,21 +90,13 @@ vi.mock('@/connectors/registry', () => ({ })) import { OrchestrationError } from '@/lib/core/orchestration/types' -import { - configureSimSearchConnector, - connectSimSearchConnector, - prepareSearchSource, -} from '@/lib/knowledge/application/sim-search' +import { prepareSearchSource } from '@/lib/knowledge/application/sim-search' import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' -import { SearchIndexDormantError } from '@/lib/sim-search/indexed/gate' const mocks = { - ...hoisted, ensureAccounts: credentialGroupsServiceMockFns.mockEnsureWorkspaceAccountsGroup, createOrganizationKnowledgeBase: knowledgeServiceMockFns.mockCreateAuthorizedKnowledgeBase, createKnowledgeBase: knowledgeBaseUseCasesMockFns.mockCreateKnowledgeBaseExecute, - deleteKnowledgeBase: knowledgeBaseUseCasesMockFns.mockDeleteKnowledgeBaseOperationExecute, - requireApproval: knowledgeSearchIntegrationPolicyMockFns.mockRequireOrganizationSearchApproval, } knowledgeEmbeddingsMockFns.mockGetConfiguredKbEmbedding.mockResolvedValue({ @@ -164,81 +122,20 @@ const workspaceContext = { } const principal = createSessionPrincipal() -const existingConnector = { knowledgeBaseId: 'kb-search', connectorId: 'connector-drive' } - -/** The first lookup runs before the coalesced creation and the second inside it. */ -function queueConnectorLookups(...results: Array) { - for (const result of results) { - queueTableRows(knowledgeConnector, result ? [result] : []) - } -} - -describe('connectSimSearchConnector', () => { +describe('prepareSearchSource', () => { afterAll(resetDbChainMock) - afterEach(resetEnvFlagsMock) beforeEach(() => { resetDbChainMock() - /** A Search source crawls into the search index, which only indexed organization search reads. */ - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) knowledgeContextsMockFns.mockResolveKnowledgeOwnerContext.mockResolvedValue(workspaceContext) permissionGroupsResolveMockFns.mockGetUserPermissionConfig.mockResolvedValue( DEFAULT_PERMISSION_GROUP_CONFIG ) knowledgeAvailabilityMockFns.mockIsKnowledgeMemberAccessAvailable.mockResolvedValue(true) mocks.createKnowledgeBase.mockResolvedValue({ knowledgeBase: { id: 'kb-new' } }) - mocks.createConnector.mockResolvedValue({ connector: { id: 'connector-new' } }) - mocks.enroll.mockResolvedValue({ url: 'https://sim.test/enroll/token' }) mocks.ensureAccounts.mockResolvedValue({ id: 'accounts-group' }) }) - it('reuses a prepared account only when the source enrollment group and option match', async () => { - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue('read') - queueTableRows(knowledgeConnector, [ - { ...existingConnector, credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, - ]) - await expect( - configureSimSearchConnector.execute({ - principal, - input: { - workspaceId: 'workspace-1', - connectorType: 'google_drive', - memberCredentialBinding: { - credentialGroupId: 'group-1', - credentialGroupOptionId: 'option-1', - }, - }, - }) - ).resolves.toEqual(existingConnector) - expect(mocks.enroll).not.toHaveBeenCalled() - }) - - it.each([ - { credentialGroupId: 'other-group', credentialGroupOptionId: 'option-1' }, - { credentialGroupId: 'group-1', credentialGroupOptionId: 'other-option' }, - ])( - 'refuses an existing source with a mismatched prepared account binding %#', - async (sourceBinding) => { - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue('read') - queueTableRows(knowledgeConnector, [{ ...existingConnector, ...sourceBinding }]) - await expect( - configureSimSearchConnector.execute({ - principal, - input: { - workspaceId: 'workspace-1', - connectorType: 'google_drive', - memberCredentialBinding: { - credentialGroupId: 'group-1', - credentialGroupOptionId: 'option-1', - }, - }, - }) - ).rejects.toMatchObject({ code: 'conflict' }) - expect(mocks.enroll).not.toHaveBeenCalled() - expect(mocks.createConnector).not.toHaveBeenCalled() - } - ) - it('requires an administrator before preparing a managed source', async () => { workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue('read') await expect( @@ -260,153 +157,20 @@ describe('connectSimSearchConnector', () => { ).rejects.toMatchObject({ code: 'validation' }) expect(mocks.createKnowledgeBase).not.toHaveBeenCalled() }) - - it('refuses while indexed organization search is dormant, before creating anything', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue('admin') - queueConnectorLookups(null) - - await expect( - connectSimSearchConnector.execute({ - principal, - input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, - }) - ).rejects.toBeInstanceOf(SearchIndexDormantError) - expect(mocks.createKnowledgeBase).not.toHaveBeenCalled() - expect(mocks.createConnector).not.toHaveBeenCalled() - expect(mocks.enroll).not.toHaveBeenCalled() - }) - - it('refuses before creating anything when per-member access is unavailable', async () => { - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue('admin') - knowledgeAvailabilityMockFns.mockIsKnowledgeMemberAccessAvailable.mockResolvedValue(false) - queueConnectorLookups(null) - - await expect( - connectSimSearchConnector.execute({ - principal, - input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, - }) - ).rejects.toMatchObject({ code: 'validation' }) - expect(mocks.createKnowledgeBase).not.toHaveBeenCalled() - expect(mocks.createConnector).not.toHaveBeenCalled() - }) - - it('uses the source returned by transaction-level creation reuse without deleting another source', async () => { - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue('admin') - mocks.createKnowledgeBase.mockRejectedValueOnce(new Error('Duplicate knowledge base name')) - mocks.createConnector.mockResolvedValueOnce({ - connector: { id: existingConnector.connectorId }, - reused: true, - }) - queueTableRows(knowledgeBase, []) - queueTableRows(knowledgeBase, [{ id: existingConnector.knowledgeBaseId }]) - queueConnectorLookups(null, null) - const result = await connectSimSearchConnector.execute({ - principal, - input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, - }) - expect(mocks.deleteConnector).not.toHaveBeenCalled() - expect(mocks.createConnector).toHaveBeenCalledWith( - expect.objectContaining({ - input: expect.objectContaining({ reuseSearchSource: true }), - }) - ) - expect(result).toEqual({ ...existingConnector, url: 'https://sim.test/enroll/token' }) - }) - - it('requires an explicit source when legacy duplicate settings make selection ambiguous', async () => { - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue('read') - queueTableRows(knowledgeConnector, [ - { ...existingConnector, connectorId: 'one', sourceConfig: {} }, - { ...existingConnector, connectorId: 'two', sourceConfig: {} }, - ]) - await expect( - connectSimSearchConnector.execute({ - principal, - input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, - }) - ).rejects.toMatchObject({ code: 'conflict' }) - expect(mocks.enroll).not.toHaveBeenCalled() - }) - - it.each([ - { name: 'missing or outside the canonical index', rows: [], config: undefined }, - { - name: 'different settings', - rows: [{ ...existingConnector, sourceConfig: { spaceKey: 'OPS' } }], - config: { spaceKey: 'ENG' }, - }, - ])('rejects an explicitly selected source that is $name', async ({ rows, config }) => { - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue('read') - queueTableRows(knowledgeConnector, rows) - await expect( - connectSimSearchConnector.execute({ - principal, - input: { - workspaceId: 'workspace-1', - connectorType: 'confluence', - connectorId: existingConnector.connectorId, - sourceConfig: config, - }, - }) - ).rejects.toMatchObject({ code: 'not_found' }) - expect(mocks.enroll).not.toHaveBeenCalled() - expect(mocks.createConnector).not.toHaveBeenCalled() - }) }) describe('organization Search setup', () => { const owner = { organizationId: 'org-1' } - afterEach(resetEnvFlagsMock) beforeEach(() => { resetDbChainMock() - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) knowledgeContextsMockFns.mockResolveKnowledgeOwnerContext.mockResolvedValue(owner) knowledgeAvailabilityMockFns.mockIsKnowledgeMemberAccessAvailable.mockResolvedValue(true) mocks.ensureAccounts.mockResolvedValue({ id: 'org-accounts' }) mocks.createOrganizationKnowledgeBase.mockResolvedValue({ id: 'org-index' }) - mocks.enroll.mockResolvedValue({ url: 'https://fixture.test/enroll' }) }) function asRole(role: string) { for (let i = 0; i < 4; i++) queueTableRows(member, [{ role }]) } - it('lets an approved member create a personal source without impersonating an admin', async () => { - asRole('member') - queueTableRows(knowledgeBase, []) - mocks.createApprovedSource.mockResolvedValue({ connector: { id: 'approved-source' } }) - await expect( - connectSimSearchConnector.execute({ - principal, - input: { ...owner, connectorType: 'google_drive' }, - }) - ).resolves.toMatchObject({ knowledgeBaseId: 'org-index', connectorId: 'approved-source' }) - expect(mocks.requireApproval).toHaveBeenCalledWith('org-1', 'google_drive') - expect(mocks.createApprovedSource).toHaveBeenCalledWith( - expect.objectContaining({ - principal, - input: { - knowledgeBaseId: 'org-index', - assertedOrganizationId: 'org-1', - connectorType: 'google_drive', - sourceConfig: {}, - }, - }) - ) - expect(mocks.createConnector).not.toHaveBeenCalled() - }) - it('refuses unapproved members before provisioning anything', async () => { - asRole('member') - mocks.requireApproval.mockRejectedValueOnce(new Error('Approval required')) - await expect( - connectSimSearchConnector.execute({ - principal, - input: { ...owner, connectorType: 'google_drive' }, - }) - ).rejects.toThrow('Approval required') - expect(mocks.createOrganizationKnowledgeBase).not.toHaveBeenCalled() - expect(mocks.enroll).not.toHaveBeenCalled() - }) it('refuses organization source setup by a member before provisioning accounts or an index', async () => { asRole('member') await expect( @@ -418,17 +182,6 @@ describe('organization Search setup', () => { expect(mocks.ensureAccounts).not.toHaveBeenCalled() expect(mocks.createOrganizationKnowledgeBase).not.toHaveBeenCalled() }) - it('refuses a former organization member before looking up any configured source', async () => { - queueTableRows(member, []) - await expect( - connectSimSearchConnector.execute({ - principal, - input: { ...owner, connectorType: 'google_drive' }, - }) - ).rejects.toThrow('Organization not found') - expect(mocks.enroll).not.toHaveBeenCalled() - expect(mocks.createOrganizationKnowledgeBase).not.toHaveBeenCalled() - }) }) describe('Mothership Search setup authorization', () => { @@ -461,7 +214,6 @@ describe('Mothership Search setup authorization', () => { else await expect(action).resolves.toBeUndefined() expect(mocks.createOrganizationKnowledgeBase).not.toHaveBeenCalled() expect(mocks.ensureAccounts).not.toHaveBeenCalled() - expect(mocks.enroll).not.toHaveBeenCalled() } ) }) diff --git a/apps/sim/lib/knowledge/application/sim-search.ts b/apps/sim/lib/knowledge/application/sim-search.ts index 75c10b00e4c..859916de119 100644 --- a/apps/sim/lib/knowledge/application/sim-search.ts +++ b/apps/sim/lib/knowledge/application/sim-search.ts @@ -1,13 +1,9 @@ import { type Principal, resolvePrincipalSubjectUserId } from '@sim/auth/principal' import { db } from '@sim/db' -import { knowledgeBase, knowledgeConnector } from '@sim/db/schema' +import { knowledgeBase } from '@sim/db/schema' import { and, eq, isNull } from 'drizzle-orm' import { coalesceLocally } from '@/lib/concurrency/singleflight' import { requireOrganizationMembership } from '@/lib/core/application/organization-authorization' -import { - InsufficientWorkspacePermissionsError, - requireCurrentHumanRole, -} from '@/lib/core/application/workspace-authorization' import { OrchestrationError, type OrchestrationRequestContext, @@ -15,13 +11,10 @@ import { import { type ResourceOwner, type ResourceScope, - resourceScopeFields, resourceScopeFromOwner, resourceScopeKey, } from '@/lib/core/resource-scope' -import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { generateRequestId } from '@/lib/core/utils/request' -import type { CredentialGroupConnectionIntent } from '@/lib/credential-groups/oauth-intent' import { ensureWorkspaceAccountsGroup } from '@/lib/credential-groups/service' import { requireKnowledgeMemberAccessAvailable, @@ -29,116 +22,20 @@ import { requireSourceMirroredAccessAvailable, } from '@/lib/knowledge/access/availability' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' -import { startKnowledgeConnectorMemberEnrollment } from '@/lib/knowledge/application/connector-access' -import { - createApprovedSearchSource, - createKnowledgeConnector, -} from '@/lib/knowledge/application/connectors' -import { - type KnowledgeOrganizationContext, - type KnowledgeWorkspaceContext, - resolveKnowledgeOwnerContext, -} from '@/lib/knowledge/application/contexts' +import { resolveKnowledgeOwnerContext } from '@/lib/knowledge/application/contexts' import { createKnowledgeBase } from '@/lib/knowledge/application/knowledge-bases' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { DEFAULT_CHUNKING_CONFIG } from '@/lib/knowledge/constants' import { getConfiguredKbEmbedding } from '@/lib/knowledge/embeddings' -import { requireOrganizationSearchApproval } from '@/lib/knowledge/search/integration-policy' import { findSearchIndex } from '@/lib/knowledge/search/search-index' import { createAuthorizedKnowledgeBase } from '@/lib/knowledge/service' -import { - canConnectPersonally, - missingSetupFields, - SIM_SEARCH_KNOWLEDGE_BASE_NAME, - withSearchSourceDefaults, -} from '@/lib/sim-search/connectors' -import { SIM_SEARCH_SYNC_INTERVAL_MINUTES } from '@/lib/sim-search/constants' -import { assertIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' -import { searchSourceIdentity } from '@/lib/sim-search/source-identity' +import { canConnectPersonally, SIM_SEARCH_KNOWLEDGE_BASE_NAME } from '@/lib/sim-search/connectors' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' const SIM_SEARCH_KNOWLEDGE_BASE_DESCRIPTION = 'What each person can open in the sources they connected, searched as them.' -export interface ConnectSimSearchConnectorInput extends ResourceOwner { - /** `CONNECTOR_META_REGISTRY` key of the source to connect. */ - connectorType: string - /** An existing source selected by the person, scoped to the canonical workspace index. */ - connectorId?: string - /** Source settings identify a compatible configuration when creating or reusing a source. */ - sourceConfig?: Record - /** Correlates a direct provider authorization with the initiating Integrations tab. */ - connectionIntent?: CredentialGroupConnectionIntent - oauthCompletionId?: string - /** Internal setup assertion; a reused source must use the viewer's prepared enrollment option. */ - memberCredentialBinding?: { credentialGroupId: string; credentialGroupOptionId: string } -} - -export interface ConnectSimSearchConnectorResult { - knowledgeBaseId: string - connectorId: string - /** The invitation link or provider authorization URL for the caller's own account. */ - url: string -} - -async function findSimSearchConnector(input: ConnectSimSearchConnectorInput) { - const rows = await db - .select({ - knowledgeBaseId: knowledgeBase.id, - connectorId: knowledgeConnector.id, - sourceConfig: knowledgeConnector.sourceConfig, - credentialGroupId: knowledgeConnector.credentialGroupId, - credentialGroupOptionId: knowledgeConnector.credentialGroupOptionId, - }) - .from(knowledgeConnector) - .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) - .where( - and( - resourceScopeCondition(knowledgeBase, resourceScopeFromOwner(input)), - eq(knowledgeBase.isSearchIndex, true), - isNull(knowledgeBase.deletedAt), - eq(knowledgeConnector.connectorType, input.connectorType), - input.connectorId ? eq(knowledgeConnector.id, input.connectorId) : undefined, - eq(knowledgeConnector.accessMode, 'members'), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - const meta = CONNECTOR_META_REGISTRY[input.connectorType]! - const identity = searchSourceIdentity(meta, input.sourceConfig ?? {}) - const matches = rows.filter((row) => - input.connectorId && input.sourceConfig === undefined - ? true - : searchSourceIdentity(meta, row.sourceConfig) === identity - ) - if (input.connectorId && matches.length === 0) { - throw new OrchestrationError( - 'not_found', - 'This Search source is unavailable or its settings have changed' - ) - } - if (matches.length > 1) { - throw new OrchestrationError( - 'conflict', - 'Several Search sources use these settings. Choose the source you want to connect.' - ) - } - const match = matches[0] - if ( - match && - input.memberCredentialBinding && - (match.credentialGroupId !== input.memberCredentialBinding.credentialGroupId || - match.credentialGroupOptionId !== input.memberCredentialBinding.credentialGroupOptionId) - ) { - throw new OrchestrationError( - 'conflict', - 'This source uses a different account configuration. Ask an admin to review its settings.' - ) - } - return match ? { knowledgeBaseId: match.knowledgeBaseId, connectorId: match.connectorId } : null -} - -/** Resolves the current workspace search index without creating one. */ +/** Resolves the current Search source configuration without creating it. */ export const readSearchIndex = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.readSearchIndex, resolveContext: ({ input }: { input: ResourceOwner }) => resolveKnowledgeOwnerContext(input), @@ -151,12 +48,11 @@ export const readSearchIndex = defineAuthorizedKnowledgeUseCase({ }, }) -/** Admin setup adopts legacy indexes; reads and deletion always use the persisted index marker. */ +/** Search service sources share a marked knowledge base that stores their configuration. */ async function ensureSearchKnowledgeBase( scope: ResourceScope, principal: Principal, - request?: OrchestrationRequestContext, - approvedConnectorType?: string + request?: OrchestrationRequestContext ): Promise { return coalesceLocally(`sim-search:base:${resourceScopeKey(scope)}`, async () => { const existing = await findSearchIndex(scope) @@ -165,12 +61,10 @@ async function ensureSearchKnowledgeBase( if (scope.kind === 'organization') { const userId = resolvePrincipalSubjectUserId(principal) if (!userId) throw new OrchestrationError('forbidden', 'Sign in to configure sources') - if (approvedConnectorType) - await requireOrganizationSearchApproval(scope.organizationId, approvedConnectorType) await requireOrganizationMembership( principal, scope.organizationId, - approvedConnectorType ? 'member' : 'admin', + 'admin', 'knowledge.create' ) const embedding = await getConfiguredKbEmbedding() @@ -223,7 +117,7 @@ async function ensureSearchKnowledgeBase( }) } -/** Prepares the shared search index before the existing connector setup modal collects credentials. */ +/** Prepares shared Search source configuration before collecting connector credentials. */ export const prepareSearchSource = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.prepareSearchSource, resolveContext: ({ @@ -255,166 +149,3 @@ export const prepareSearchSource = defineAuthorizedKnowledgeUseCase({ } }, }) - -/** - * The first connect of a source turns it on for the whole workspace, which is - * an admin decision the same way a members-mode connector is. Refused with - * the way forward rather than the nested operations' generic role error, so a - * reader learns whom to ask and for what. - */ -async function requireSimSearchSetupAdmin( - principal: Principal, - context: KnowledgeWorkspaceContext | KnowledgeOrganizationContext, - sourceName: string -): Promise { - try { - if (context.organizationId) - await requireOrganizationMembership( - principal, - context.organizationId, - 'admin', - 'knowledge.use' - ) - else if (context.workspaceId) { - const userId = resolvePrincipalSubjectUserId(principal) - if (!userId) throw new OrchestrationError('forbidden', 'Sign in to configure sources') - await requireCurrentHumanRole(userId, context, 'admin') - } - } catch (error) { - if (!(error instanceof InsufficientWorkspacePermissionsError)) throw error - throw new OrchestrationError( - 'forbidden', - `${sourceName} is not connected in this workspace yet. Ask a workspace admin to connect ${sourceName} first; after that everyone connects their own account.` - ) - } -} - -/** - * Connects the caller's account, creating the owner's index and personal source - * when needed. Organization members may set up approved integrations; workspace - * setup requires an admin. OAuth completion queues indexing for the member. - * - * The database identifies one active search index per owner. Local - * singleflight also coalesces repeated setup clicks for each source; concurrent - * source creation is serialized by the connector insert transaction before enrollment. - * The source crawls into the owner's search index, so it is refused while indexed organization - * search is dormant. - */ -export const configureSimSearchConnector = defineAuthorizedKnowledgeUseCase({ - operation: knowledgeOperations.simSearchConnect, - resolveContext: ({ input }: { input: ConnectSimSearchConnectorInput }) => - resolveKnowledgeOwnerContext(input), - async execute({ principal, input, context, request }) { - assertIndexedOrgSearchEnabled() - const meta = CONNECTOR_META_REGISTRY[input.connectorType] - if (!meta || !canConnectPersonally(meta)) { - throw new OrchestrationError( - 'validation', - 'This source cannot be connected per person; a workspace admin sets it up from a knowledge base' - ) - } - const scope = resourceScopeFromOwner(context) - const owner = resourceScopeFields(scope) - const workspaceId = context.workspaceId - if (context.organizationId) { - await requireOrganizationSearchApproval(context.organizationId, input.connectorType) - } - /** - * Defaults are applied before any lookup so a second person connecting with - * an untouched form lands on the source the first connection created. - */ - const sourceConfig = - input.connectorId && input.sourceConfig === undefined - ? undefined - : withSearchSourceDefaults(meta, input.sourceConfig) - let target = await findSimSearchConnector({ ...input, sourceConfig, ...owner }) - if (!target) { - const userId = resolvePrincipalSubjectUserId(principal) - if (!userId) throw new OrchestrationError('forbidden', 'Sign in to connect your account') - const sourceConfig = withSearchSourceDefaults(meta, input.sourceConfig) - const missing = missingSetupFields(meta, sourceConfig) - if (missing.length > 0) { - throw new OrchestrationError( - 'validation', - `${meta.name} needs ${missing.map((field) => field.title).join(' and ')} to connect` - ) - } - /** - * Judged before anything is created: the connector creation below checks - * the same availability, but only after the knowledge base exists. - */ - await Promise.all([ - requireKnowledgeMemberAccessAvailable(owner), - context.organizationId - ? Promise.resolve() - : requireSimSearchSetupAdmin(principal, context, meta.name), - ]) - const knowledgeBaseId = await ensureSearchKnowledgeBase( - scope, - principal, - request, - context.organizationId ? input.connectorType : undefined - ) - target = await coalesceLocally( - `sim-search:connect:${resourceScopeKey(scope)}:${input.connectorType}:${searchSourceIdentity(meta, sourceConfig)}`, - async () => { - const existing = await findSimSearchConnector({ ...input, ...owner }) - if (existing) return existing - if (context.organizationId) { - const created = await createApprovedSearchSource.execute({ - principal, - input: { - knowledgeBaseId, - assertedOrganizationId: context.organizationId, - connectorType: input.connectorType, - sourceConfig, - }, - request, - }) - return { knowledgeBaseId, connectorId: created.connector.id } - } - const created = await createKnowledgeConnector.execute({ - principal, - input: { - knowledgeBaseId, - assertedWorkspaceId: workspaceId, - assertedOrganizationId: context.organizationId, - connectorType: input.connectorType, - sourceConfig, - syncIntervalMinutes: SIM_SEARCH_SYNC_INTERVAL_MINUTES, - accessMode: 'members', - reuseSearchSource: true, - source: 'ui', - }, - request, - }) - return { knowledgeBaseId, connectorId: created.connector.id } - } - ) - } - return target - }, -}) - -/** Creates or reuses the source, then authorizes the member when setup has not already done so. */ -export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ - operation: knowledgeOperations.simSearchConnect, - resolveContext: ({ input }: { input: ConnectSimSearchConnectorInput }) => - resolveKnowledgeOwnerContext(input), - async execute({ principal, input, context, request }): Promise { - const target = await configureSimSearchConnector.execute({ principal, input, request }) - const { url } = await startKnowledgeConnectorMemberEnrollment.execute({ - principal, - input: { - knowledgeBaseId: target.knowledgeBaseId, - connectorId: target.connectorId, - assertedWorkspaceId: context.workspaceId, - assertedOrganizationId: context.organizationId, - oauthCompletionId: input.oauthCompletionId, - connectionIntent: input.connectionIntent, - }, - request, - }) - return { ...target, url } - }, -}) diff --git a/apps/sim/lib/knowledge/connectors/indexing-policy.test.ts b/apps/sim/lib/knowledge/connectors/indexing-policy.test.ts deleted file mode 100644 index b30a175ce85..00000000000 --- a/apps/sim/lib/knowledge/connectors/indexing-policy.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { knowledgeBase } from '@sim/db/schema' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' -import { eq } from 'drizzle-orm' -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { - connectorIndexingCondition, - requiresConnectorIndexing, -} from '@/lib/knowledge/connectors/indexing-policy' - -describe('connector indexing policy', () => { - beforeEach(resetEnvFlagsMock) - - it('preserves ordinary knowledge-base indexing when Search uses live APIs', () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) - expect(requiresConnectorIndexing(false)).toBe(true) - expect(requiresConnectorIndexing(true)).toBe(false) - connectorIndexingCondition() - expect(vi.mocked(eq)).toHaveBeenCalledWith(knowledgeBase.isSearchIndex, false) - }) - - it('preserves indexed Search and existing schedules when live search is disabled', () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) - expect(requiresConnectorIndexing(true)).toBe(true) - expect(requiresConnectorIndexing(false)).toBe(true) - expect(connectorIndexingCondition()).toBeUndefined() - }) -}) diff --git a/apps/sim/lib/knowledge/connectors/indexing-policy.ts b/apps/sim/lib/knowledge/connectors/indexing-policy.ts index bbb61df298f..ac2ee44afcf 100644 --- a/apps/sim/lib/knowledge/connectors/indexing-policy.ts +++ b/apps/sim/lib/knowledge/connectors/indexing-policy.ts @@ -1,13 +1,12 @@ import { knowledgeBase } from '@sim/db/schema' import { eq } from 'drizzle-orm' -import { isIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' /** Federated Search keeps source configuration but does not crawl content into a knowledge base. */ export function requiresConnectorIndexing(isSearchIndex?: boolean | null): boolean { - return isIndexedOrgSearchEnabled() || isSearchIndex !== true + return isSearchIndex !== true } /** Keeps federated sources out of bounded indexing scheduler pages. */ export function connectorIndexingCondition() { - return isIndexedOrgSearchEnabled() ? undefined : eq(knowledgeBase.isSearchIndex, false) + return eq(knowledgeBase.isSearchIndex, false) } diff --git a/apps/sim/lib/knowledge/connectors/organization-account-indexing.test.ts b/apps/sim/lib/knowledge/connectors/organization-account-indexing.test.ts deleted file mode 100644 index 0882dc75a6f..00000000000 --- a/apps/sim/lib/knowledge/connectors/organization-account-indexing.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { credentialGroup, knowledgeBase, knowledgeConnector } from '@sim/db/schema' -import { - dbChainMockFns, - flattenMockConditions, - queueTableRows, - resetDbChainMock, -} from '@sim/testing' -import { - knowledgeMemberAccessMock, - knowledgeMemberAccessMockFns, -} from '@sim/testing/mocks/knowledge-member-access.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/knowledge/connectors/member-access', () => knowledgeMemberAccessMock) - -import { setOrganizationAccountIndexing } from '@/lib/knowledge/connectors/organization-account-indexing' - -const validateBinding = knowledgeMemberAccessMockFns.mockValidateKnowledgeConnectorMembersBinding - -const input = { - organizationId: 'org-1', - credentialGroupId: 'group-1', - optionId: 'gmail-option', - enabled: false, -} -const group = { - status: 'active', - options: [{ id: 'gmail-option', status: 'active', provider: 'gmail' }], -} -const source = { - id: 'source-1', - knowledgeBaseId: 'kb-1', - status: 'active', - memberSyncStatus: 'idle', - sourceConfig: {}, -} - -describe('organization provider indexing changes', () => { - beforeEach(() => { - resetDbChainMock() - validateBinding.mockReturnValue({ ok: true }) - queueTableRows(credentialGroup, [group]) - }) - - it('pauses all bound sources in one transaction, cancels queued work and retains documents', async () => { - queueTableRows(knowledgeConnector, [ - source, - { ...source, id: 'source-2', memberSyncStatus: 'pending' }, - ]) - await expect(setOrganizationAccountIndexing(input)).resolves.toMatchObject({ - enabled: false, - changed: true, - knowledgeBaseIds: ['kb-1'], - }) - expect(dbChainMockFns.set).toHaveBeenCalledWith( - expect.objectContaining({ - status: 'paused', - nextMemberSyncAt: null, - memberSyncLockToken: null, - memberSyncStatus: 'idle', - }) - ) - expect(dbChainMockFns.update).toHaveBeenCalledExactlyOnceWith(knowledgeConnector) - expect(dbChainMockFns.delete).not.toHaveBeenCalled() - const predicates = dbChainMockFns.where.mock.calls.flatMap(([condition]) => - flattenMockConditions(condition) - ) - for (const expected of [ - { type: 'eq', left: knowledgeBase.organizationId, right: 'org-1' }, - { type: 'eq', left: knowledgeBase.isSearchIndex, right: true }, - { type: 'eq', left: knowledgeConnector.credentialGroupId, right: 'group-1' }, - { type: 'eq', left: knowledgeConnector.credentialGroupOptionId, right: 'gmail-option' }, - { type: 'eq', left: knowledgeConnector.accessMode, right: 'members' }, - ]) - expect(predicates).toContainEqual(expected) - }) - - it('refuses the entire change when one source has an active run', async () => { - queueTableRows(knowledgeConnector, [ - source, - { ...source, id: 'source-2', memberSyncStatus: 'running' }, - ]) - await expect(setOrganizationAccountIndexing(input)).rejects.toMatchObject({ code: 'conflict' }) - expect(dbChainMockFns.update).not.toHaveBeenCalled() - }) - - it('refuses a stale or foreign option before touching any source', async () => { - await expect( - setOrganizationAccountIndexing({ ...input, optionId: 'foreign-option' }) - ).rejects.toMatchObject({ code: 'not_found' }) - expect(dbChainMockFns.update).not.toHaveBeenCalled() - }) - - it('requires setup instead of reporting indexing enabled without a source', async () => { - queueTableRows(knowledgeConnector, []) - await expect(setOrganizationAccountIndexing({ ...input, enabled: true })).rejects.toMatchObject( - { code: 'not_found' } - ) - expect(dbChainMockFns.update).not.toHaveBeenCalled() - }) - - it('rejects re-enabling a source whose scopes no longer meet the ingestion requirements', async () => { - queueTableRows(knowledgeConnector, [{ ...source, status: 'paused' }]) - validateBinding.mockReturnValue({ ok: false, message: 'Reconnect with the required scopes' }) - await expect(setOrganizationAccountIndexing({ ...input, enabled: true })).rejects.toThrow( - 'Reconnect with the required scopes' - ) - expect(dbChainMockFns.update).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/knowledge/connectors/organization-account-indexing.ts b/apps/sim/lib/knowledge/connectors/organization-account-indexing.ts deleted file mode 100644 index a1e34897ede..00000000000 --- a/apps/sim/lib/knowledge/connectors/organization-account-indexing.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { db } from '@sim/db' -import { credentialGroup, knowledgeBase, knowledgeConnector } from '@sim/db/schema' -import { isPlainRecord } from '@sim/utils/object' -import { and, asc, eq, inArray, isNull } from 'drizzle-orm' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { resourceScopeCondition } from '@/lib/core/resource-scope.server' -import { getCredentialGroupIndexingConnector } from '@/lib/credential-groups/indexing' -import { ORGANIZATION_ACCOUNT_INDEXING_SOURCE_LIMIT } from '@/lib/credential-groups/limits' -import { isCredentialGroupProvider } from '@/lib/credential-groups/providers' -import { validateKnowledgeConnectorMembersBinding } from '@/lib/knowledge/connectors/member-access' - -export interface SetOrganizationAccountIndexingInput { - organizationId: string - credentialGroupId: string - optionId: string - enabled: boolean -} - -/** Changes every Search source bound to this org option atomically, respecting running sync leases. */ -export async function setOrganizationAccountIndexing(input: SetOrganizationAccountIndexingInput) { - const scope = { kind: 'organization' as const, organizationId: input.organizationId } - return db.transaction(async (tx) => { - const [group] = await tx - .select({ options: credentialGroup.options, status: credentialGroup.status }) - .from(credentialGroup) - .where( - and( - eq(credentialGroup.id, input.credentialGroupId), - resourceScopeCondition(credentialGroup, scope) - ) - ) - .limit(1) - .for('update') - if (!group) - throw new OrchestrationError('not_found', 'Organization connected accounts were not found') - const option = group.options.find( - (candidate) => candidate.id === input.optionId && candidate.status === 'active' - ) - if (!option || !isCredentialGroupProvider(option.provider)) - throw new OrchestrationError('not_found', 'Connected account provider was not found') - const connector = getCredentialGroupIndexingConnector(option.provider) - if (!connector) - throw new OrchestrationError('validation', 'Indexing is not supported for this provider') - const sources = await tx - .select({ - id: knowledgeConnector.id, - knowledgeBaseId: knowledgeBase.id, - sourceConfig: knowledgeConnector.sourceConfig, - status: knowledgeConnector.status, - memberSyncStatus: knowledgeConnector.memberSyncStatus, - }) - .from(knowledgeConnector) - .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) - .where( - and( - resourceScopeCondition(knowledgeBase, scope), - eq(knowledgeBase.isSearchIndex, true), - isNull(knowledgeBase.deletedAt), - eq(knowledgeConnector.credentialGroupId, input.credentialGroupId), - eq(knowledgeConnector.credentialGroupOptionId, option.id), - eq(knowledgeConnector.connectorType, connector.type), - eq(knowledgeConnector.accessMode, 'members'), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .orderBy(asc(knowledgeConnector.id)) - .limit(ORGANIZATION_ACCOUNT_INDEXING_SOURCE_LIMIT + 1) - .for('update') - if (sources.length > ORGANIZATION_ACCOUNT_INDEXING_SOURCE_LIMIT) - throw new OrchestrationError('validation', 'Too many indexing sources for one provider') - if (!sources.length) - throw new OrchestrationError('not_found', 'Set up an indexing source for this provider first') - const changed = sources.filter((source) => - input.enabled - ? source.status === 'paused' || - source.status === 'disabled' || - source.memberSyncStatus === 'disabled' - : source.status !== 'paused' - ) - if ( - changed.some((source) => source.status === 'syncing' || source.memberSyncStatus === 'running') - ) - throw new OrchestrationError( - 'conflict', - 'Indexing is running. Wait for the current sync to finish, then try again.' - ) - if (input.enabled) { - for (const source of changed) { - if (!isPlainRecord(source.sourceConfig)) - throw new OrchestrationError('validation', 'Indexing source settings are invalid') - const validation = validateKnowledgeConnectorMembersBinding({ - connectorMeta: connector.meta, - group, - credentialGroupOptionId: option.id, - sourceConfig: source.sourceConfig, - }) - if (!validation.ok) throw new OrchestrationError('validation', validation.message) - } - } - if (changed.length) { - const now = new Date() - await tx - .update(knowledgeConnector) - .set({ - status: input.enabled ? 'active' : 'paused', - memberSyncStatus: 'idle', - memberSyncLockToken: null, - memberSyncLockLeaseAt: null, - syncLockToken: null, - syncLockLeaseAt: null, - nextMemberSyncAt: input.enabled ? now : null, - updatedAt: now, - ...(input.enabled - ? { consecutiveFailures: 0, lastSyncError: null, lastMemberSyncError: null } - : {}), - }) - .where( - and( - inArray( - knowledgeConnector.id, - changed.map((source) => source.id) - ), - eq(knowledgeConnector.credentialGroupId, input.credentialGroupId), - eq(knowledgeConnector.credentialGroupOptionId, option.id) - ) - ) - } - return { - enabled: input.enabled, - changed: changed.length > 0, - providerName: connector.meta.name, - knowledgeBaseIds: [...new Set(sources.map((source) => source.knowledgeBaseId))], - } - }) -} diff --git a/apps/sim/lib/knowledge/connectors/viewer-member-sync-error.ts b/apps/sim/lib/knowledge/connectors/viewer-member-sync-error.ts deleted file mode 100644 index 1d8e1919cde..00000000000 --- a/apps/sim/lib/knowledge/connectors/viewer-member-sync-error.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { db } from '@sim/db' -import { - credential, - credentialGroupEnrollment, - knowledgeConnector, - knowledgeConnectorMember, -} from '@sim/db/schema' -import { and, eq, exists, inArray, isNotNull, isNull, sql } from 'drizzle-orm' - -/** Retains the viewer's account-level failure even after a run claims no due members. */ -export function hasViewerMemberSyncError(userId: string) { - return sql`${exists( - db - .select({ id: knowledgeConnectorMember.id }) - .from(knowledgeConnectorMember) - .innerJoin(credential, eq(credential.id, knowledgeConnectorMember.credentialId)) - .innerJoin( - credentialGroupEnrollment, - eq(credentialGroupEnrollment.id, credential.credentialGroupEnrollmentId) - ) - .where( - and( - eq(knowledgeConnector.accessMode, 'members'), - eq(knowledgeConnectorMember.connectorId, knowledgeConnector.id), - eq(credentialGroupEnrollment.userId, userId), - inArray(knowledgeConnectorMember.status, ['active', 'suspended']), - inArray(credential.managedOauthStatus, ['active', 'needs_reauth']), - isNull(credential.revokedAt), - isNotNull(knowledgeConnectorMember.lastError) - ) - ) - )}` -} diff --git a/apps/sim/lib/knowledge/connectors/viewer-source-accounts.test.ts b/apps/sim/lib/knowledge/connectors/viewer-source-accounts.test.ts deleted file mode 100644 index d2337681577..00000000000 --- a/apps/sim/lib/knowledge/connectors/viewer-source-accounts.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { credential, credentialGroup, credentialGroupEnrollment } from '@sim/db/schema' -import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' -import { eq, inArray, isNull } from 'drizzle-orm' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/connectors/registry', () => ({ - getConnectorMeta: (id: string) => ({ - requiresMemberIdentity: id === 'slack', - auth: { mode: 'oauth', provider: id }, - }), -})) - -import { resolveViewerSourceAccounts } from '@/lib/knowledge/connectors/viewer-source-accounts' -import { SEARCH_SOURCE_CANDIDATE_PAGE_SIZE } from '@/lib/knowledge/constants' - -const source = { - id: 'gmail-source', - connectorType: 'gmail', - accessMode: 'members', - credentialGroupId: 'group-1', - credentialGroupOptionId: 'gmail-option', -} -const input = { organizationId: 'org-1', userId: 'viewer', connectors: [source] } -const account = { - credentialId: 'mine', - displayName: 'My Gmail', - groupId: 'group-1', - optionId: 'gmail-option', - providerId: 'gmail', - status: 'active', -} - -describe('personal source account projection', () => { - beforeEach(() => { - resetDbChainMock() - }) - - it('binds the current contributor and organization on both credentials and groups', async () => { - queueTableRows(credential, [account]) - const result = await resolveViewerSourceAccounts(input) - expect(eq).toHaveBeenCalledWith(credentialGroupEnrollment.userId, 'viewer') - expect(eq).toHaveBeenCalledWith(credential.organizationId, 'org-1') - expect(eq).toHaveBeenCalledWith(credentialGroup.organizationId, 'org-1') - expect(isNull).toHaveBeenCalledWith(credential.workspaceId) - expect(isNull).toHaveBeenCalledWith(credentialGroup.workspaceId) - expect(isNull).toHaveBeenCalledWith(credential.revokedAt) - expect(inArray).toHaveBeenCalledWith(credential.managedOauthStatus, ['active', 'needs_reauth']) - expect(result.get(source.id)).toEqual([ - { credentialId: 'mine', displayName: 'My Gmail', status: 'active' }, - ]) - expect(dbChainMockFns.select).toHaveBeenCalledWith({ - credentialId: credential.id, - displayName: credential.displayName, - status: credential.managedOauthStatus, - groupId: credentialGroup.id, - optionId: credential.credentialGroupOptionId, - providerId: credential.providerId, - }) - }) - - it('does not attach an account from another source option or group', async () => { - queueTableRows(credential, [ - { ...account, groupId: 'other' }, - { ...account, optionId: 'other' }, - ]) - expect(await resolveViewerSourceAccounts(input)).toEqual(new Map()) - }) - - it('fails instead of silently truncating too many accounts', async () => { - queueTableRows( - credential, - Array.from({ length: SEARCH_SOURCE_CANDIDATE_PAGE_SIZE + 1 }, () => account) - ) - await expect(resolveViewerSourceAccounts(input)).rejects.toThrow('Too many personal accounts') - }) -}) diff --git a/apps/sim/lib/knowledge/connectors/viewer-source-accounts.ts b/apps/sim/lib/knowledge/connectors/viewer-source-accounts.ts deleted file mode 100644 index 924dd0f386b..00000000000 --- a/apps/sim/lib/knowledge/connectors/viewer-source-accounts.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { credential, credentialGroup } from '@sim/db/schema' -import { and, eq, or } from 'drizzle-orm' -import { listViewerOrganizationAccounts } from '@/lib/credential-groups/viewer-accounts' -import { getConnectorMeta } from '@/connectors/registry' - -interface ViewerSourceAccount { - credentialId: string - displayName: string - status: 'active' | 'needs_reauth' -} - -interface SourceAccountBinding { - id: string - connectorType: string - accessMode: string - credentialGroupId: string | null - credentialGroupOptionId: string | null -} - -/** - * Own account controls remain available when provider setup, enrollment, or sync is disabled. - * Called inside the authorized source read; selects no token material or other contributors. - */ -export async function resolveViewerSourceAccounts(input: { - organizationId: string - userId: string - connectors: ReadonlyArray -}): Promise> { - const bindings = input.connectors.map((source) => { - const meta = getConnectorMeta(source.connectorType) - const providerId = - source.accessMode === 'admin' && meta?.requiresMemberIdentity && meta.auth.mode === 'oauth' - ? meta.auth.provider - : null - return { source, providerId } - }) - const matches = bindings.flatMap(({ source, providerId }) => { - if ( - source.accessMode === 'members' && - source.credentialGroupId && - source.credentialGroupOptionId - ) - return [ - and( - eq(credentialGroup.id, source.credentialGroupId), - eq(credential.credentialGroupOptionId, source.credentialGroupOptionId) - ), - ] - return providerId ? [eq(credential.providerId, providerId)] : [] - }) - const result = new Map() - if (!matches.length) return result - const accounts = await listViewerOrganizationAccounts({ - organizationId: input.organizationId, - userId: input.userId, - matching: or(...matches)!, - }) - for (const { source, providerId } of bindings) { - const own = accounts.filter((account) => - source.accessMode === 'members' - ? account.groupId === source.credentialGroupId && - account.optionId === source.credentialGroupOptionId - : providerId !== null && account.providerId === providerId - ) - if (own.length) - result.set( - source.id, - own.map(({ credentialId, displayName, status }) => { - if (status !== 'active' && status !== 'needs_reauth') - throw new Error('Invalid personal account status') - return { credentialId, displayName, status } - }) - ) - } - return result -} diff --git a/apps/sim/lib/knowledge/constants.ts b/apps/sim/lib/knowledge/constants.ts index f6de98cf0a0..a292948b174 100644 --- a/apps/sim/lib/knowledge/constants.ts +++ b/apps/sim/lib/knowledge/constants.ts @@ -30,7 +30,6 @@ export const MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE = 200 export const MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_SEARCH_LENGTH = 200 /** Maximum source IDs in one viewer-authorized progress request. */ -export const MAX_SEARCH_SOURCE_PROGRESS_ITEMS = 100 /** Bound viewer-specific source resolution and document counts to a single page. */ export const SEARCH_SOURCE_PAGE_SIZE = 25 export const SEARCH_SOURCE_CANDIDATE_PAGE_SIZE = 100 diff --git a/apps/sim/lib/knowledge/documents/ocr-recovery.md b/apps/sim/lib/knowledge/documents/ocr-recovery.md index 91723bbf352..caaca290016 100644 --- a/apps/sim/lib/knowledge/documents/ocr-recovery.md +++ b/apps/sim/lib/knowledge/documents/ocr-recovery.md @@ -1,6 +1,6 @@ # OCR capacity and indexing recovery -Regular knowledge bases and legacy indexed Sim Search (`SIM_SEARCH_LIVE=false`) use the connector content pass, document processor, embeddings, and processing continuations described here. Live Search, the default, calls provider APIs and does not run this OCR/indexing recovery path. Authorization and source visibility remain specific to each access mode. +Regular knowledge bases use the connector content pass, document processor, embeddings, and processing continuations described here. Enterprise Search calls provider APIs and does not run this OCR/indexing recovery path. Authorization and source visibility remain specific to each access mode. ## Operating budgets diff --git a/apps/sim/lib/knowledge/mcp/route-handler.test.ts b/apps/sim/lib/knowledge/mcp/route-handler.test.ts index 074656fc040..41daf2f7ba2 100644 --- a/apps/sim/lib/knowledge/mcp/route-handler.test.ts +++ b/apps/sim/lib/knowledge/mcp/route-handler.test.ts @@ -63,7 +63,6 @@ vi.mock('@/connectors/registry', () => ({ CONNECTOR_META_REGISTRY: {} })) vi.mock('@/lib/sim-search/connectors', () => ({ SIM_SEARCH_KNOWLEDGE_BASE_NAME: 'Sim Search', canConnectPersonally: vi.fn(), - missingSetupFields: vi.fn(), })) import { V2ApiKeyUnauthenticatedError } from '@/lib/api/server/routes/v2-api-key-auth' @@ -160,7 +159,7 @@ describe('organization MCP request admission', () => { expect(mocks.index).not.toHaveBeenCalled() }) - it('admits a current personal-key member and binds the canonical organization index', async () => { + it('admits a current personal-key member for the canonical organization', async () => { const result = await post() expect(result.status).toBe(200) expect(result.headers.get('Cache-Control')).toBe('private, no-store') @@ -169,7 +168,7 @@ describe('organization MCP request admission', () => { knowledgeAvailabilityMockFns.mockRequireOrganizationSearchAvailable ).toHaveBeenCalledExactlyOnceWith('org-1') expect(mocks.createServer).toHaveBeenCalledWith( - expect.objectContaining({ auth, organizationId: 'org-1', searchIndexId: 'index-1' }) + expect.objectContaining({ auth, organizationId: 'org-1' }) ) expect(mocks.close).toHaveBeenCalledOnce() expect(v2RouteMocks.authenticate).toHaveBeenCalledWith( diff --git a/apps/sim/lib/knowledge/mcp/route-handler.ts b/apps/sim/lib/knowledge/mcp/route-handler.ts index 006b69ad43e..980672d468b 100644 --- a/apps/sim/lib/knowledge/mcp/route-handler.ts +++ b/apps/sim/lib/knowledge/mcp/route-handler.ts @@ -42,7 +42,7 @@ export function createKnowledgeMcpHandlers() { maxBodyBytes: 64 * 1024, }) if (!parsed.success) return parsed.response - const index = await readSearchIndex.execute({ + await readSearchIndex.execute({ principal: admission.auth.principal, input: parsed.data.params, request, @@ -51,7 +51,6 @@ export function createKnowledgeMcpHandlers() { request, auth: admission.auth, ...parsed.data.params, - searchIndexId: index.knowledgeBaseId, }) return await serveStatelessMcp(server, request, parsed.data.body) } catch (error) { diff --git a/apps/sim/lib/knowledge/mcp/server.protocol.test.ts b/apps/sim/lib/knowledge/mcp/server.protocol.test.ts index a0ea9244d10..594af0f2070 100644 --- a/apps/sim/lib/knowledge/mcp/server.protocol.test.ts +++ b/apps/sim/lib/knowledge/mcp/server.protocol.test.ts @@ -1,33 +1,19 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js' import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' import { createPersonalApiKeyPrincipal } from '@sim/testing/factories/principal.factory' -import { - knowledgeSearchUseCaseMock, - knowledgeSearchUseCaseMockFns, -} from '@sim/testing/mocks/knowledge-search-use-case.mock' import { urlsMockFns } from '@sim/testing/mocks/urls.mock' import { NextRequest } from 'next/server' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { describe, expect, it, vi } from 'vitest' const hoisted = vi.hoisted(() => ({ liveSearch: vi.fn(), liveRead: vi.fn(), - indexedRead: vi.fn(), })) vi.mock('@/lib/core/utils/after-response', () => ({ afterResponse: vi.fn() })) vi.mock('@/lib/knowledge/mcp/activity', () => ({ recordOrganizationSearchMcpActivity: vi.fn() })) vi.mock('@/lib/api/server/routes/v2-json-route', () => ({ v2RateLimits: { publicApi: { enforce: vi.fn().mockResolvedValue(null) } }, })) -vi.mock('@/lib/knowledge/application/search', () => knowledgeSearchUseCaseMock) -vi.mock('@/lib/sim-search/indexed/documents/read-indexed-document', () => ({ - readIndexedKnowledgeDocument: { execute: hoisted.indexedRead }, -})) -vi.mock('@/lib/sim-search/indexed', async () => ({ - registerIndexedKnowledgeMcpTools: (await import('@/lib/sim-search/indexed/mcp/register-tools')) - .registerIndexedKnowledgeMcpTools, -})) vi.mock('@/lib/sim-search/live/application', () => ({ searchLiveKnowledge: { execute: hoisted.liveSearch }, readLiveDocument: { execute: hoisted.liveRead }, @@ -35,99 +21,68 @@ vi.mock('@/lib/sim-search/live/application', () => ({ import { createKnowledgeMcpServer } from '@/lib/knowledge/mcp/server' -const mocks = { - ...hoisted, - indexedSearch: knowledgeSearchUseCaseMockFns.mockSearchKnowledgeExecute, -} +const mocks = hoisted urlsMockFns.mockGetBaseUrl.mockReturnValue('http://localhost') describe('Search MCP protocol', () => { - beforeEach(() => { - resetEnvFlagsMock() - }) - - it.each([true, false])( - 'lists and calls the selected backend through the SDK (live: %s)', - async (live) => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: live }) - const documentId = live ? `live:${'a'.repeat(500)}` : 'doc-1' - const row = { - documentId, - knowledgeBaseId: live ? '' : 'index-1', - documentName: 'Release notes', - sourceUrl: 'https://example.com/notes', - connectorType: 'google_drive', - content: 'Evidence', - chunkIndex: 0, - similarity: 0.5, - } - mocks.liveSearch.mockResolvedValue({ - results: [row], - live: { backend: 'live', accounts: [], guidance: '' }, - }) - mocks.indexedSearch.mockResolvedValue({ results: [row] }) - mocks.liveRead.mockResolvedValue({ - ...row, - knowledgeBaseName: 'Drive', - chunks: [{ chunkIndex: 0, content: 'Read evidence' }], - hasMore: false, - next: null, - }) - mocks.indexedRead.mockResolvedValue({ - ...row, - title: row.documentName, - chunks: [{ chunkIndex: 0, content: 'Read evidence' }], - }) - - const server = createKnowledgeMcpServer({ - organizationId: 'org-1', - searchIndexId: live ? null : 'index-1', - request: new NextRequest('http://localhost/api/mcp/search/organizations/org-1'), - auth: { - principal: createPersonalApiKeyPrincipal(), - keyType: 'personal', - keyExpiresAt: null, - rateLimitSubjectIds: ['user-1'], - rateLimitSubscription: null, - }, - }) - const client = new Client({ name: 'Search test', version: '1.0.0' }) - const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair() - try { - await server.connect(serverTransport) - await client.connect(clientTransport) - const { tools } = await client.listTools() - expect(tools.map((tool) => tool.name)).toEqual(['search', 'read_document', 'chat']) - const searchSchema = tools.find((tool) => tool.name === 'search')?.inputSchema - const readSchema = tools.find((tool) => tool.name === 'read_document')?.inputSchema - if (live) { - expect(searchSchema?.properties).toHaveProperty('nativeQueries') - expect(readSchema?.properties).toHaveProperty('startOffset') - expect(readSchema?.properties).not.toHaveProperty('url') - } else { - expect(searchSchema?.properties).not.toHaveProperty('nativeQueries') - expect(readSchema?.properties).toHaveProperty('offset') - expect(readSchema?.properties).toHaveProperty('url') - } - const search = await client.callTool({ name: 'search', arguments: { query: 'release' } }) - expect(search.isError).not.toBe(true) - expect(search.content).toEqual([ - { type: 'text', text: expect.stringContaining(documentId) }, - ]) - const read = await client.callTool({ name: 'read_document', arguments: { documentId } }) - expect(read.isError).not.toBe(true) - expect(read.content).toEqual([ - { type: 'text', text: expect.stringContaining('Read evidence') }, - ]) - expect(live ? mocks.liveSearch : mocks.indexedSearch).toHaveBeenCalledOnce() - expect(live ? mocks.liveRead : mocks.indexedRead).toHaveBeenCalledOnce() - expect(live ? mocks.indexedSearch : mocks.liveSearch).not.toHaveBeenCalled() - expect(live ? mocks.indexedRead : mocks.liveRead).not.toHaveBeenCalled() - } finally { - await client.close() - await server.close() - } + it('lists and calls live Search through the SDK', async () => { + const documentId = `live:${'a'.repeat(500)}` + const row = { + documentId, + knowledgeBaseId: '', + documentName: 'Release notes', + sourceUrl: 'https://example.com/notes', + connectorType: 'google_drive', + content: 'Evidence', + chunkIndex: 0, + similarity: 0.5, + } + mocks.liveSearch.mockResolvedValue({ + results: [row], + live: { backend: 'live', accounts: [], guidance: '' }, + }) + mocks.liveRead.mockResolvedValue({ + ...row, + knowledgeBaseName: 'Drive', + chunks: [{ chunkIndex: 0, content: 'Read evidence' }], + hasMore: false, + next: null, + }) + const server = createKnowledgeMcpServer({ + organizationId: 'org-1', + request: new NextRequest('http://localhost/api/mcp/search/organizations/org-1'), + auth: { + principal: createPersonalApiKeyPrincipal(), + keyType: 'personal', + keyExpiresAt: null, + rateLimitSubjectIds: ['user-1'], + rateLimitSubscription: null, + }, + }) + const client = new Client({ name: 'Search test', version: '1.0.0' }) + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair() + try { + await server.connect(serverTransport) + await client.connect(clientTransport) + const { tools } = await client.listTools() + expect(tools.map((tool) => tool.name)).toEqual(['search', 'read_document', 'chat']) + const searchSchema = tools.find((tool) => tool.name === 'search')?.inputSchema + const readSchema = tools.find((tool) => tool.name === 'read_document')?.inputSchema + expect(searchSchema?.properties).toHaveProperty('nativeQueries') + expect(readSchema?.properties).toHaveProperty('startOffset') + expect(readSchema?.properties).not.toHaveProperty('url') + const search = await client.callTool({ name: 'search', arguments: { query: 'release' } }) + expect(search.isError).not.toBe(true) + expect(search.content).toEqual([{ type: 'text', text: expect.stringContaining(documentId) }]) + const read = await client.callTool({ name: 'read_document', arguments: { documentId } }) + expect(read.isError).not.toBe(true) + expect(read.content).toEqual([ + { type: 'text', text: expect.stringContaining('Read evidence') }, + ]) + } finally { + await client.close() + await server.close() } - ) + }) }) diff --git a/apps/sim/lib/knowledge/mcp/server.test.ts b/apps/sim/lib/knowledge/mcp/server.test.ts index aca1577fd0d..4617bf078af 100644 --- a/apps/sim/lib/knowledge/mcp/server.test.ts +++ b/apps/sim/lib/knowledge/mcp/server.test.ts @@ -1,10 +1,5 @@ import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' import { createPersonalApiKeyPrincipal } from '@sim/testing/factories/principal.factory' -import { - knowledgeSearchUseCaseMock, - knowledgeSearchUseCaseMockFns, -} from '@sim/testing/mocks/knowledge-search-use-case.mock' import { getMockLogger } from '@sim/testing/mocks/logger.mock' import { urlsMockFns } from '@sim/testing/mocks/urls.mock' import { NextRequest } from 'next/server' @@ -19,7 +14,6 @@ const hoisted = vi.hoisted(() => ({ string, { description: string; inputSchema: { parse: (input: unknown) => unknown } } >(), - read: vi.fn(), liveSearch: vi.fn(), liveRead: vi.fn(), chat: vi.fn(), @@ -49,14 +43,6 @@ vi.mock('@modelcontextprotocol/sdk/server/mcp.js', () => ({ vi.mock('@/lib/api/server/routes/v2-json-route', () => ({ v2RateLimits: { publicApi: { enforce: hoisted.rateLimit } }, })) -vi.mock('@/lib/knowledge/application/search', () => knowledgeSearchUseCaseMock) -vi.mock('@/lib/sim-search/indexed/documents/read-indexed-document', () => ({ - readIndexedKnowledgeDocument: { execute: hoisted.read }, -})) -vi.mock('@/lib/sim-search/indexed', async () => ({ - registerIndexedKnowledgeMcpTools: (await import('@/lib/sim-search/indexed/mcp/register-tools')) - .registerIndexedKnowledgeMcpTools, -})) vi.mock('@/lib/sim-search/live/application', () => ({ searchLiveKnowledge: { execute: hoisted.liveSearch }, readLiveDocument: { execute: hoisted.liveRead }, @@ -69,10 +55,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { createKnowledgeMcpServer } from '@/lib/knowledge/mcp/server' import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' -const mocks = { - ...hoisted, - search: knowledgeSearchUseCaseMockFns.mockSearchKnowledgeExecute, -} +const mocks = hoisted urlsMockFns.mockGetBaseUrl.mockReturnValue('https://sim.example') @@ -85,8 +68,8 @@ const auth = { rateLimitSubscription: null, } const request = new NextRequest('http://localhost/api/mcp/search/organizations/org-1') -function create(searchIndexId: string | null = 'index-1') { - createKnowledgeMcpServer({ organizationId: 'org-1', searchIndexId, request, auth }) +function create() { + createKnowledgeMcpServer({ organizationId: 'org-1', request, auth }) } function call(tool: string, input: Record, signal = new AbortController().signal) { const run = mocks.tools.get(tool) @@ -94,28 +77,11 @@ function call(tool: string, input: Record, signal = new AbortCo return run(input, { signal }) } -function _payload(result: CallToolResult): unknown { - const first = result.content[0] - if (first.type !== 'text') throw new Error('Expected a text result') - return JSON.parse(first.text) -} - beforeEach(() => { - resetEnvFlagsMock() - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) mocks.tools.clear() mocks.configs.clear() mocks.rateLimit.mockReset().mockResolvedValue(null) - mocks.search.mockResolvedValue({ results: [] }) - mocks.read.mockResolvedValue({ - knowledgeBaseId: 'index-1', - documentId: 'doc-1', - title: 'A source', - sourceUrl: 'https://example.com/source', - processingStatus: 'completed', - chunks: [{ id: 'chunk-1', chunkIndex: 0, content: 'Indexed text' }], - pagination: { total: 1, offset: 0, limit: 20, hasMore: false }, - }) + mocks.liveSearch.mockResolvedValue({ results: [] }) mocks.chat.mockResolvedValue({ content: 'An answer', citations: [] }) }) @@ -139,7 +105,6 @@ describe('live organization Search MCP', () => { guidance: 'Use nativeQueries to continue.', } beforeEach(() => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) mocks.liveSearch.mockResolvedValue({ results: [document], live: coverage, @@ -163,15 +128,13 @@ describe('live organization Search MCP', () => { }) it.each(['search', 'read_document'])( - 'does not fall back to indexed data after a live %s denial', + 'does not return live %s content after an authorization denial', async (tool) => { create() const backend = tool === 'search' ? mocks.liveSearch : mocks.liveRead backend.mockRejectedValueOnce(new OrchestrationError('forbidden', 'Access denied')) const result = await call(tool, tool === 'search' ? { query: 'release' } : { documentId }) expect(result).toEqual({ isError: true, content: [{ type: 'text', text: 'Access denied' }] }) - expect(mocks.search).not.toHaveBeenCalled() - expect(mocks.read).not.toHaveBeenCalled() } ) @@ -198,40 +161,10 @@ describe('live organization Search MCP', () => { ) }) -describe('search', () => { - const tool = 'search' - it('searches only the canonical index using the actual personal key principal', async () => { - create() - expect((await call(tool, { query: 'find it', topK: 10 })).isError).toBeUndefined() - expect(mocks.search).toHaveBeenCalledWith({ - principal, - request, - input: expect.objectContaining({ - organizationId: 'org-1', - knowledgeBaseIds: ['index-1'], - query: 'find it', - surface: 'mcp', - }), - }) - }) -}) - describe('organization Search MCP tools', () => { - it('delegates URL resolution without fetching the URL in the adapter', async () => { - create() - await call('read_document', { url: 'https://example.com/source', limit: 20 }) - expect(mocks.read).toHaveBeenCalledWith({ - principal, - request, - input: expect.objectContaining({ - organizationId: 'org-1', - target: { kind: 'url', url: 'https://example.com/source' }, - }), - }) - }) it('does not return content denied by the canonical document ACL operation', async () => { create() - mocks.read.mockRejectedValueOnce(new OrchestrationError('not_found', 'Document not found')) + mocks.liveRead.mockRejectedValueOnce(new OrchestrationError('not_found', 'Document not found')) const result = await call('read_document', { documentId: 'foreign-document', }) @@ -243,16 +176,16 @@ describe('organization Search MCP tools', () => { it('does not expose cached success after a later membership or policy denial', async () => { create() await call('search', { query: 'first', topK: 10 }) - mocks.search.mockRejectedValueOnce( + mocks.liveSearch.mockRejectedValueOnce( new OrchestrationError('forbidden', 'Knowledge access is disabled') ) expect((await call('search', { query: 'second', topK: 10 })).isError).toBe(true) - expect(mocks.search).toHaveBeenCalledTimes(2) + expect(mocks.liveSearch).toHaveBeenCalledTimes(2) }) it('does not return partial metadata when the chunk read is denied', async () => { create() - mocks.read.mockRejectedValueOnce(new OrchestrationError('not_found', 'Document not found')) + mocks.liveRead.mockRejectedValueOnce(new OrchestrationError('not_found', 'Document not found')) expect(await call('read_document', { documentId: 'doc-1' })).toEqual({ isError: true, content: [{ type: 'text', text: 'Document not found' }], @@ -295,7 +228,9 @@ describe('MCP tool completion records', () => { it('records an authorization failure without including the query or error message', async () => { create() - mocks.search.mockRejectedValueOnce(new OrchestrationError('forbidden', 'Private denial reason')) + mocks.liveSearch.mockRejectedValueOnce( + new OrchestrationError('forbidden', 'Private denial reason') + ) await call('search', { query: 'private query' }) expect(getMockLogger('KnowledgeMcp').info).toHaveBeenCalledExactlyOnceWith( 'Knowledge MCP tool completed', diff --git a/apps/sim/lib/knowledge/mcp/server.ts b/apps/sim/lib/knowledge/mcp/server.ts index e3a9c600186..996e352885a 100644 --- a/apps/sim/lib/knowledge/mcp/server.ts +++ b/apps/sim/lib/knowledge/mcp/server.ts @@ -25,8 +25,6 @@ import { } from '@/lib/knowledge/mcp/tool-runner' import { liveCitationId } from '@/lib/knowledge/search/citation' import { toolError } from '@/lib/mcp/tool-result' -import { registerIndexedKnowledgeMcpTools } from '@/lib/sim-search/indexed' -import { isIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' import { readLiveDocument, searchLiveKnowledge } from '@/lib/sim-search/live/application' import { v2CaughtOrchestrationError } from '@/app/api/v2/lib/response' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' @@ -36,12 +34,11 @@ interface KnowledgeMcpContext { organizationId: string request: NextRequest auth: V2ApiKeyAuthContext - searchIndexId: string | null } /** A request owns its server; no credential or principal survives into another HTTP request. */ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServer { - const { request, auth, searchIndexId, organizationId } = context + const { request, auth, organizationId } = context const principal = auth.principal const server = new McpServer({ name: 'Sim Search', version: '1.0.0' }) @@ -107,103 +104,92 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe } } - if (isIndexedOrgSearchEnabled()) { - registerIndexedKnowledgeMcpTools({ - server, - principal, - request, - organizationId, - searchIndexId, - execute, - }) - } else { - server.registerTool( - 'search', - { - title: 'Search', - description: - 'Search this organization’s sources through their live APIs, within your access and the admin’s source settings. Use source and date filters to narrow results, or nativeQueries for provider queries and pagination. Inspect live.accounts for provider status and continuation cursors, and live.guidance for query syntax. Results are candidates, not proof of complete coverage. Use read_document with the exact returned documentId for context and cite citationUrl when available.', - inputSchema: liveSearchMcpSchema, - annotations: KNOWLEDGE_MCP_READ_ONLY, - }, - async (input: unknown, extra: { signal: AbortSignal }) => - execute('search', knowledgeOperations.search, extra.signal, async (registry, signal) => { - const { query, topK, nativeQueries, ...filters } = liveSearchMcpSchema.parse(input) - const result = await searchLiveKnowledge.execute({ + server.registerTool( + 'search', + { + title: 'Search', + description: + 'Search this organization’s sources through their live APIs, within your access and the admin’s source settings. Use source and date filters to narrow results, or nativeQueries for provider queries and pagination. Inspect live.accounts for provider status and continuation cursors, and live.guidance for query syntax. Results are candidates, not proof of complete coverage. Use read_document with the exact returned documentId for context and cite citationUrl when available.', + inputSchema: liveSearchMcpSchema, + annotations: KNOWLEDGE_MCP_READ_ONLY, + }, + async (input: unknown, extra: { signal: AbortSignal }) => + execute('search', knowledgeOperations.search, extra.signal, async (registry, signal) => { + const { query, topK, nativeQueries, ...filters } = liveSearchMcpSchema.parse(input) + const result = await searchLiveKnowledge.execute({ + principal, + input: { + organizationId, + query, + topK, + nativeQueries, + filters, + resultSecretRegistry: registry, + signal, + }, + request, + }) + return projectResult( + { + results: result.results.map((row) => ({ + documentId: row.documentId, + title: row.documentName, + sourceUrl: row.sourceUrl, + citationId: liveCitationId(row.documentId), + citationUrl: row.sourceUrl, + sourceModifiedAt: row.sourceModifiedAt ?? null, + sourceDate: row.sourceDate, + sourceContainerName: row.sourceContainerName, + sourceContainerUrl: row.sourceContainerUrl, + connectorType: row.connectorType, + content: row.content, + chunkIndex: row.chunkIndex, + })), + retrieval: result.retrieval, + live: result.live, + }, + registry + ) + }) + ) + server.registerTool( + 'read_document', + { + title: 'Read document', + description: + 'Read a live document using the exact documentId returned by search. Access and the admin’s source settings are checked again on every read. When hasMore is true, pass next.startChunkIndex and next.startOffset with the same documentId to continue. Cite citationUrl when available.', + inputSchema: readLiveDocumentMcpSchema, + annotations: KNOWLEDGE_MCP_READ_ONLY, + }, + async (raw: unknown, extra: { signal: AbortSignal }) => + execute( + 'read_document', + knowledgeOperations.readDocument, + extra.signal, + async (registry, signal) => { + const input = readLiveDocumentMcpSchema.parse(raw) + const result = await readLiveDocument.execute({ principal, - input: { - organizationId, - query, - topK, - nativeQueries, - filters, - resultSecretRegistry: registry, - signal, - }, + input: { ...input, organizationId, resultSecretRegistry: registry, signal }, request, }) + const { + knowledgeBaseId: _knowledgeBaseId, + knowledgeBaseName: _name, + ...document + } = result return projectResult( { - results: result.results.map((row) => ({ - documentId: row.documentId, - title: row.documentName, - sourceUrl: row.sourceUrl, - citationId: liveCitationId(row.documentId), - citationUrl: row.sourceUrl, - sourceModifiedAt: row.sourceModifiedAt ?? null, - sourceDate: row.sourceDate, - sourceContainerName: row.sourceContainerName, - sourceContainerUrl: row.sourceContainerUrl, - connectorType: row.connectorType, - content: row.content, - chunkIndex: row.chunkIndex, - })), - retrieval: result.retrieval, - live: result.live, + ...document, + title: result.documentName, + citationId: liveCitationId(result.documentId), + citationUrl: result.sourceUrl, }, registry ) - }) - ) - server.registerTool( - 'read_document', - { - title: 'Read document', - description: - 'Read a live document using the exact documentId returned by search. Access and the admin’s source settings are checked again on every read. When hasMore is true, pass next.startChunkIndex and next.startOffset with the same documentId to continue. Cite citationUrl when available.', - inputSchema: readLiveDocumentMcpSchema, - annotations: KNOWLEDGE_MCP_READ_ONLY, - }, - async (raw: unknown, extra: { signal: AbortSignal }) => - execute( - 'read_document', - knowledgeOperations.readDocument, - extra.signal, - async (registry, signal) => { - const input = readLiveDocumentMcpSchema.parse(raw) - const result = await readLiveDocument.execute({ - principal, - input: { ...input, organizationId, resultSecretRegistry: registry, signal }, - request, - }) - const { - knowledgeBaseId: _knowledgeBaseId, - knowledgeBaseName: _name, - ...document - } = result - return projectResult( - { - ...document, - title: result.documentName, - citationId: liveCitationId(result.documentId), - citationUrl: result.sourceUrl, - }, - registry - ) - } - ) - ) - } + } + ) + ) server.registerTool( 'chat', diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index 7a3170a6934..af72cbeac37 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -9,7 +9,7 @@ import { knowledgeConnectorMember, } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { getErrorMessage, getPostgresErrorCode, toError } from '@sim/utils/errors' +import { getPostgresErrorCode, toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { and, eq, isNull, sql } from 'drizzle-orm' import { encryptApiKey } from '@/lib/api-key/crypto' @@ -67,7 +67,6 @@ import { type KnowledgeOperationContext, type KnowledgeOrchestrationResult, } from '@/lib/knowledge/orchestration/shared' -import { dropSourceVectorIndex } from '@/lib/knowledge/search/source-vector-indexes' import { createTagDefinition } from '@/lib/knowledge/tags/service' import { captureServerEvent } from '@/lib/posthog/server' import { searchSourceIdentity } from '@/lib/sim-search/source-identity' @@ -1414,14 +1413,6 @@ export async function performDeleteKnowledgeConnector( }) } - /** The source is gone, so its vector index is too; ranking falls back to the exact path. */ - await dropSourceVectorIndex(connectorId).catch((error: unknown) => { - logger.warn('Could not drop the source vector index', { - connectorId, - error: getErrorMessage(error), - }) - }) - return { success: true, documentsDeleted: deleteDocuments ? docCount : 0, diff --git a/apps/sim/lib/knowledge/projection/enqueue.ts b/apps/sim/lib/knowledge/projection/enqueue.ts index 93b8887c845..69592a29e2a 100644 --- a/apps/sim/lib/knowledge/projection/enqueue.ts +++ b/apps/sim/lib/knowledge/projection/enqueue.ts @@ -7,7 +7,6 @@ import { import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isTriggerAvailable } from '@/lib/core/config/trigger-availability' -import { isIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' const logger = createLogger('KnowledgeProjectionEnqueue') @@ -69,20 +68,19 @@ export interface KnowledgeProjectionSweepResult { * The knowledge projector's only trigger: one pass per window while marks need one, so marks are * settled within about a minute of their write, a document a pass gave up is retried by the next, * and an idle deployment starts no pass at all. The sweep first releases, on the pooled database, - * the marks no pass is owed, so the always-on marking of writes never starts one. While indexed - * organization search is off that is every mark without content to project, search-index ones - * included, since nothing reads their mirrored source and ACL. + * the marks no pass is owed, so the always-on marking of writes never starts one. + * Search marks and permission-only marks are released without copying their rows. Only deferred + * ordinary-KB content admits a repair pass. */ export async function enqueueKnowledgeProjectionSweep(): Promise { - const scope = { searchIndexes: isIndexedOrgSearchEnabled() } let release = { drained: false, empty: false } try { - release = await releaseSettledMarks(db.$client, Date.now() + MARK_RELEASE_BUDGET_MS, scope) + release = await releaseSettledMarks(db.$client, Date.now() + MARK_RELEASE_BUDGET_MS) } catch (error) { /** A release that failed leaves its marks for the next sweep; whether a pass is owed still stands. */ logger.warn('Releasing settled projection marks failed', { error: getErrorMessage(error) }) } - if (release.empty || !(await hasKnowledgeProjectionWork(db.$client, release, scope))) { + if (release.empty || !(await hasKnowledgeProjectionWork(db.$client))) { return { triggered: false, backend: null, jobId: null } } if (!isTriggerAvailable()) { diff --git a/apps/sim/lib/knowledge/projection/run.ts b/apps/sim/lib/knowledge/projection/run.ts index b0f2c6a5cad..2cc64fe56e7 100644 --- a/apps/sim/lib/knowledge/projection/run.ts +++ b/apps/sim/lib/knowledge/projection/run.ts @@ -9,21 +9,20 @@ import { withUtcTimestamps } from '@sim/db/timestamps' import { createLogger } from '@sim/logger' import postgres, { type Sql } from 'postgres' import { env, envNumber } from '@/lib/core/config/env' -import { isIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' const logger = createLogger('KnowledgeProjectionPass') /** * Most documents one pass projects at once, each worker on a connection of its own. Measured * locally on content-heavy load, projection throughput kept rising to eight workers without - * deadlocks, so eight is the default; `KB_CONFIG_PROJECTION_CONCURRENCY` raises or lowers it per + * deadlocks, so eight is the ceiling; `KB_CONFIG_PROJECTION_CONCURRENCY` can lower it per * deployment. A round opens only as many workers as there are marks, so a pass over a few * documents holds a few connections. */ -const PROJECTION_CONCURRENCY = envNumber(env.KB_CONFIG_PROJECTION_CONCURRENCY, 8, { - min: 1, - integer: true, -}) +const PROJECTION_CONCURRENCY = Math.min( + 8, + envNumber(env.KB_CONFIG_PROJECTION_CONCURRENCY, 8, { min: 1, integer: true }) +) /** How the projector's connections name themselves in `pg_stat_activity`. */ const PROJECTOR_APPLICATION_NAME = 'sim-knowledge-projector' @@ -45,17 +44,14 @@ export interface KnowledgeProjectionPassResult extends KnowledgeProjectionProgre * One pass of the knowledge projector: settles every marked document. Each round first releases, * for no longer than a release's own short budget, the marks no pass is owed (see * `releaseSettledMarks`), so a backlog of them shrinks every round without holding up the content - * behind it. What is left is projected: content a writer deferred, and, while indexed organization - * search is on, search-index documents, whose rows mirror their source and ACL. + * behind it. Only ordinary-KB vector content that older writers deferred is repaired; retired + * Search content and copied ACLs are never projected. * * Workers project documents in parallel, each on a connection of its own that holds its * per-document advisory locks; a pass this long should not hold the pool's connections. Workers * read the same oldest marks and split them at those locks. A round ends when every worker found * nothing more it could take; the pass goes on while rounds settle documents, and `remaining` * reports marks it left for the next sweep. - * - * The pass writes Tin keyword rows only while indexed organization search is enabled: only that - * search reads them. */ export async function runKnowledgeProjectionPass(options: { budgetMs: number @@ -75,7 +71,6 @@ export async function runKnowledgeProjectionPass(options: { ) ) const deadline = Date.now() + options.budgetMs - const scope = { searchIndexes: isIndexedOrgSearchEnabled() } const result: KnowledgeProjectionPassResult = { settled: 0, deferred: 0, @@ -88,8 +83,7 @@ export async function runKnowledgeProjectionPass(options: { while (Date.now() < deadline) { const release = await releaseSettledMarks( sessions[0], - Math.min(deadline, Date.now() + MARK_RELEASE_BUDGET_MS), - scope + Math.min(deadline, Date.now() + MARK_RELEASE_BUDGET_MS) ) result.released += release.released const workers = sessions.slice(0, await workersFor(sessions[0])) @@ -98,7 +92,6 @@ export async function runKnowledgeProjectionPass(options: { workers.map((session) => runKnowledgeProjection(session, { budgetMs: Math.max(0, deadline - Date.now()), - ...scope, }) ) ) diff --git a/apps/sim/lib/knowledge/search/activity-stats.ts b/apps/sim/lib/knowledge/search/activity-stats.ts deleted file mode 100644 index 29108baa858..00000000000 --- a/apps/sim/lib/knowledge/search/activity-stats.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { dbReplica } from '@sim/db' -import { organizationSearchInvocation, user } from '@sim/db/schema' -import { sql } from 'drizzle-orm' -import { - getSearchStatsWindow, - SEARCH_STATS_PEOPLE_LIMIT, - type SEARCH_STATS_PERIODS, - SEARCH_STATS_SOURCE_LIMIT, - type SEARCH_STATS_SURFACES, - type SearchStatsDateRange, -} from '@/lib/knowledge/search/stats' - -export interface SearchStatsInput extends SearchStatsDateRange { - organizationId: string - period: (typeof SEARCH_STATS_PERIODS)[number] - surface?: (typeof SEARCH_STATS_SURFACES)[number] -} - -type StatsRow = { - totals: { invocations: number; activePeople: number; results: number } - series: { timestamp: string; invocations: number }[] - surfaces: { surface: (typeof SEARCH_STATS_SURFACES)[number]; invocations: number }[] - sources: { sourceType: string; invocations: number }[] - people: { - userId: string | null - name: string | null - email: string | null - invocations: number - sourceTypes: string[] - }[] -} - -/** One statement snapshot; only bounded aggregates leave Postgres, never individual search records. */ -export async function loadOrganizationSearchStats(input: SearchStatsInput, now = new Date()) { - const { start, end, days } = getSearchStatsWindow(input.period, now, input) - const rows = await dbReplica.transaction( - async (tx) => { - await tx.execute(sql`SET LOCAL statement_timeout = '10s'`) - return tx.execute(sql` - WITH activity AS ( - SELECT id, user_id, surface, source_types, result_count, created_at - FROM ${organizationSearchInvocation} - WHERE organization_id = ${input.organizationId} - AND created_at >= ${start.toISOString()}::timestamptz - AND created_at < ${end.toISOString()}::timestamptz - ${input.surface ? sql`AND surface = ${input.surface}` : sql``} - ), people AS ( - SELECT user_id, count(*)::float8 AS invocations - FROM activity GROUP BY user_id - ORDER BY invocations DESC, user_id NULLS LAST - LIMIT ${SEARCH_STATS_PEOPLE_LIMIT} - ) - SELECT - (SELECT jsonb_build_object( - 'invocations', count(*)::float8, - 'activePeople', count(DISTINCT user_id)::float8, - 'results', coalesce(sum(result_count), 0)::float8 - ) FROM activity) AS totals, - (SELECT coalesce(jsonb_agg(row ORDER BY row.timestamp), '[]'::jsonb) FROM ( - SELECT to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD') || 'T00:00:00.000Z' AS timestamp, - count(*)::float8 AS invocations - FROM activity GROUP BY 1 - ) row) AS series, - (SELECT coalesce(jsonb_agg(row ORDER BY row.invocations DESC, row.surface), '[]'::jsonb) FROM ( - SELECT surface, count(*)::float8 AS invocations FROM activity GROUP BY surface - ) row) AS surfaces, - (SELECT coalesce(jsonb_agg(row ORDER BY row.invocations DESC, row."sourceType"), '[]'::jsonb) FROM ( - SELECT source_type AS "sourceType", count(*)::float8 AS invocations - FROM activity CROSS JOIN LATERAL unnest(source_types) AS source_type - GROUP BY source_type ORDER BY invocations DESC, source_type - LIMIT ${SEARCH_STATS_SOURCE_LIMIT} - ) row) AS sources, - (SELECT coalesce(jsonb_agg(row ORDER BY row.invocations DESC, row."userId" NULLS LAST), '[]'::jsonb) FROM ( - SELECT people.user_id AS "userId", ${user.name} AS name, ${user.email} AS email, - people.invocations, - ARRAY( - SELECT DISTINCT source_type FROM activity - CROSS JOIN LATERAL unnest(source_types) AS source_type - WHERE activity.user_id IS NOT DISTINCT FROM people.user_id - ORDER BY source_type LIMIT ${SEARCH_STATS_SOURCE_LIMIT} - ) AS "sourceTypes" - FROM people LEFT JOIN ${user} ON ${user.id} = people.user_id - ) row) AS people - `) - }, - { accessMode: 'read only' } - ) - const row = rows[0] - if (!row) throw new Error('Search activity aggregate returned no result') - const byDay = new Map(row.series.map((point) => [point.timestamp, point.invocations])) - return { - ...row, - start: start.toISOString(), - end: end.toISOString(), - series: Array.from({ length: days }, (_, index) => { - const timestamp = new Date(start.getTime() + index * 86_400_000).toISOString() - return { timestamp, invocations: byDay.get(timestamp) ?? 0 } - }), - } -} diff --git a/apps/sim/lib/knowledge/search/activity.test.ts b/apps/sim/lib/knowledge/search/activity.test.ts deleted file mode 100644 index f2bdb78ca85..00000000000 --- a/apps/sim/lib/knowledge/search/activity.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { dbChainMockFns } from '@sim/testing/mocks/database.mock' -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const mocks = vi.hoisted(() => ({ - values: vi.fn(), - insert: vi.fn(), - execute: vi.fn(), -})) - -import { recordOrganizationSearchActivity } from '@/lib/knowledge/search/activity' - -beforeEach(() => { - mocks.insert.mockReturnValue({ values: mocks.values }) - mocks.values.mockResolvedValue(undefined) - mocks.execute.mockResolvedValue(undefined) - dbChainMockFns.transaction.mockImplementation((callback) => - callback({ execute: mocks.execute, insert: mocks.insert }) - ) -}) - -describe('Search activity metering', () => { - it('counts documents once per invocation and stores no document identity or content', async () => { - await recordOrganizationSearchActivity({ - organizationId: 'org', - userId: 'actor', - surface: 'mcp', - results: [ - { documentId: 'private-document-1', connectorType: 'confluence' }, - { documentId: 'private-document-1', connectorType: 'confluence' }, - { documentId: 'private-document-2', connectorType: 'jira' }, - { documentId: 'private-document-3', connectorType: null }, - ], - }) - expect(mocks.values).toHaveBeenCalledWith({ - id: expect.any(String), - organizationId: 'org', - userId: 'actor', - surface: 'mcp', - sourceTypes: ['confluence', 'jira', 'uploads'], - resultCount: 3, - }) - expect(JSON.stringify(mocks.values.mock.calls)).not.toContain('private-document') - }) - it('does not attempt an unbounded insert if setting the deadline fails', async () => { - mocks.execute.mockRejectedValueOnce(new Error('Could not set statement timeout')) - await expect( - recordOrganizationSearchActivity({ - organizationId: 'org', - userId: 'actor', - surface: 'slack', - results: [], - }) - ).resolves.toBeUndefined() - expect(mocks.insert).not.toHaveBeenCalled() - }) - it('does not fail Search when activity storage is unavailable', async () => { - mocks.values.mockRejectedValueOnce(new Error('offline')) - await expect( - recordOrganizationSearchActivity({ - organizationId: 'org', - userId: 'actor', - surface: 'dashboard', - results: [], - }) - ).resolves.toBeUndefined() - }) -}) diff --git a/apps/sim/lib/knowledge/search/activity.ts b/apps/sim/lib/knowledge/search/activity.ts deleted file mode 100644 index f3a68393de7..00000000000 --- a/apps/sim/lib/knowledge/search/activity.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { db } from '@sim/db' -import { organizationSearchInvocation } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { generateId } from '@sim/utils/id' -import { sql } from 'drizzle-orm' -import type { SEARCH_STATS_SURFACES } from '@/lib/knowledge/search/stats' - -const logger = createLogger('OrganizationSearchActivity') - -interface SearchActivityInput { - organizationId: string - userId: string - surface: (typeof SEARCH_STATS_SURFACES)[number] - results: ReadonlyArray<{ documentId: string; connectorType: string | null }> -} - -/** Records completed, authorized searches without query text, document IDs, or content. */ -export async function recordOrganizationSearchActivity(input: SearchActivityInput): Promise { - const sourceTypes = [...new Set(input.results.map((result) => result.connectorType ?? 'uploads'))] - try { - await db.transaction(async (tx) => { - await tx.execute(sql`SET LOCAL statement_timeout = '2s'`) - await tx.insert(organizationSearchInvocation).values({ - id: generateId(), - organizationId: input.organizationId, - userId: input.userId, - surface: input.surface, - sourceTypes, - resultCount: new Set(input.results.map((result) => result.documentId)).size, - }) - }) - } catch (error) { - logger.warn('Failed to record organization Search activity', { - error: getErrorMessage(error), - }) - } -} diff --git a/apps/sim/lib/knowledge/search/author.ts b/apps/sim/lib/knowledge/search/author.ts deleted file mode 100644 index bd6bff80d5e..00000000000 --- a/apps/sim/lib/knowledge/search/author.ts +++ /dev/null @@ -1,34 +0,0 @@ -/** - * The tag names connectors give the person behind a document, in the order - * they are tried. Connectors were never asked to agree on a name, so the - * result's author is derived here rather than in each of them. - */ -const AUTHOR_TAG_NAMES = [ - 'From', - 'Author', - 'Sender', - 'Owner', - 'Organizer', - 'Creator', - 'Reporter', - 'Assignee', -] as const - -/** - * The person a search result shows beside its source: the first author-like - * tag the document carries, reduced to a display name when the connector - * stored an address form such as `Name `. - */ -export function sourceAuthor(metadata: Record): string | null { - for (const name of AUTHOR_TAG_NAMES) { - const value = metadata[name] - if (typeof value !== 'string') continue - const display = value - .replace(/<[^>]*>/g, '') - .trim() - .replace(/^"|"$/g, '') - .trim() - if (display) return display - } - return null -} diff --git a/apps/sim/lib/knowledge/search/connection-attempt.ts b/apps/sim/lib/knowledge/search/connection-attempt.ts index 3eec8ee04c5..a820a04afee 100644 --- a/apps/sim/lib/knowledge/search/connection-attempt.ts +++ b/apps/sim/lib/knowledge/search/connection-attempt.ts @@ -5,7 +5,6 @@ export const SEARCH_CONNECTION_ATTEMPT_EVENT = 'sim:search-connection-attempt' const attemptSchema = z.object({ completionId: z.string().uuid(), requestedAt: z.number(), - connectorId: z.string().optional(), credentialId: z.string().optional(), status: z.enum(['pending', 'connected', 'failed']), error: z.string().nullable(), diff --git a/apps/sim/lib/knowledge/search/connection-target.test.ts b/apps/sim/lib/knowledge/search/connection-target.test.ts index 42ccb94de8b..fbadb5ae518 100644 --- a/apps/sim/lib/knowledge/search/connection-target.test.ts +++ b/apps/sim/lib/knowledge/search/connection-target.test.ts @@ -5,13 +5,14 @@ const target = { type: 'link', provider: 'google-email', connectorType: 'gmail', - connectorId: 'source', + connectionMode: 'live', + optionId: 'option', } describe('shared Search connection tags', () => { it.each([ { ...target, value: 'https://evil.test' }, { ...target, organizationId: 'other' }, - { ...target, connectorId: undefined, credentialId: 'account' }, + { ...target, optionId: undefined, credentialId: 'account' }, ])('rejects model URLs, scope and incomplete reconnects', (forged) => { expect( parseSearchConnectionTargets(`${JSON.stringify(forged)}`) diff --git a/apps/sim/lib/knowledge/search/connection-target.ts b/apps/sim/lib/knowledge/search/connection-target.ts index 003a8234d98..040224f912a 100644 --- a/apps/sim/lib/knowledge/search/connection-target.ts +++ b/apps/sim/lib/knowledge/search/connection-target.ts @@ -7,28 +7,11 @@ export const searchConnectionTargetSchema = z type: z.literal('link'), provider: z.string().trim().min(1).max(100), connectorType: z.string().trim().min(1).max(100), - connectorId: z.string().min(1).max(200).optional(), credentialId: z.string().min(1).max(128).optional(), - connectionMode: z.literal('live').optional(), - optionId: z.string().min(1).max(128).optional(), + connectionMode: z.literal('live'), + optionId: z.string().min(1).max(128), }) .strict() - .refine( - (target) => - !target.credentialId || Boolean(target.connectorId) || target.connectionMode === 'live', - { - message: 'A reconnect requires a configured source', - } - ) - .refine( - (target) => - target.connectionMode === 'live' - ? Boolean(target.optionId) && !target.connectorId - : !target.optionId, - { - message: 'Live account connections require an account option instead of an indexed source', - } - ) export type SearchConnectionTarget = z.infer @@ -63,5 +46,11 @@ export function parseSearchConnectionTargets(text: string): SearchConnectionTarg /** Carries an untrusted selection to the existing authenticated Integrations page, without OAuth state. */ export function searchConnectionPath(organizationId: string, target: SearchConnectionTarget) { - return `${organizationRoutes(organizationId).integrations}?${new URLSearchParams({ connectorType: target.connectorType, ...(target.connectorId ? { connectorId: target.connectorId } : {}), ...(target.credentialId ? { credentialId: target.credentialId } : {}), ...(target.connectionMode ? { connectionMode: target.connectionMode, ...(target.optionId ? { optionId: target.optionId } : {}), provider: target.provider } : {}) })}` + return `${organizationRoutes(organizationId).integrations}?${new URLSearchParams({ + connectorType: target.connectorType, + ...(target.credentialId ? { credentialId: target.credentialId } : {}), + connectionMode: target.connectionMode, + optionId: target.optionId, + provider: target.provider, + })}` } diff --git a/apps/sim/lib/knowledge/search/diagnostics.ts b/apps/sim/lib/knowledge/search/diagnostics.ts index 95ab1a6505b..2468262b182 100644 --- a/apps/sim/lib/knowledge/search/diagnostics.ts +++ b/apps/sim/lib/knowledge/search/diagnostics.ts @@ -14,11 +14,9 @@ export type SearchStage = | 'tool_presentation' | 'workspace_application' | 'organization_application' - | 'scoped_application' | 'knowledge_application' | 'scope_resolution' | 'knowledge_context' - | 'index_resolution' | 'availability' | 'billing_attribution' | 'usage_admission' @@ -28,10 +26,7 @@ export type SearchStage = | 'access_scope' | 'defaults' | 'retrieval' - | 'access_plan' | 'live_source_grants' - | 'vector.source_exact' - | 'vector.source_walk' | 'permitted_documents' | 'result_provenance' | 'reranking' @@ -39,7 +34,6 @@ export type SearchStage = | 'overage_billing' | 'tag_definitions' | 'metadata_provenance' - | 'activity_recording' | RetrievalLeg | `${RetrievalLeg}.candidates` | `${RetrievalLeg}.hydration` @@ -48,20 +42,9 @@ export type SearchStage = | 'vector.settings' | 'vector.probe' | 'vector.page' - | 'vector.projection_filled' - | 'vector.source_indexes' - | 'keyword.projection_filled' | 'vector.exact_candidates' | 'vector.exact' | 'vector.candidate_search' - | 'keyword.tin' - | 'keyword.tin_readiness' - | 'keyword.tin_query' - | 'source_overview' - | 'source_overview.availability' - | 'source_overview.providers' - | 'source_overview.indexing' - | 'source_overview.searchable' | 'access_batch.connectors' | 'access_batch.live_proof' | 'live.policies' @@ -74,7 +57,7 @@ export type SearchStage = /** Fixed, content-free fields. Never pass queries, filters, document identities, SQL, or errors. */ export interface SearchDiagnosticMetadata { - operation?: 'search_workspace' | 'read_document' | 'read_search_source_overview' + operation?: 'search_workspace' | 'read_document' surface?: 'dashboard' | 'mcp' | 'copilot' | 'workflow' | 'api' | 'slack' | 'other' toolCallId?: string executionId?: string @@ -91,7 +74,7 @@ export interface SearchDiagnosticMetadata { searchMode?: 'hybrid' | 'vector' boostRecency?: boolean embeddingDimensions?: number - vectorRanking?: 'exact' | 'exact-candidates' | 'projection-walk' | 'per-source' + vectorRanking?: 'exact' | 'exact-candidates' | 'projection-walk' /** * Whether the bounded traversal filled its candidate limit. `underfilled` means visibility * removed enough neighbours that the rerank pool is smaller than requested, which lowers recall @@ -102,25 +85,6 @@ export interface SearchDiagnosticMetadata { vectorCandidateLimit?: number /** Visible documents the tractability probe enumerated, capped at its own document limit. */ vectorProbeDocumentCount?: number - /** - * Whether a user-scoped search resolved its permitted documents before retrieval: `bounded` - * ranks inside that set, `unbounded` means it exceeded the probe's limit and both legs search - * the index with the access predicate applied per candidate. - */ - permittedDocuments?: 'bounded' | 'unbounded' - /** Documents in a bounded permitted set. */ - permittedDocumentCount?: number - vectorSourcesSliced?: number - /** The sliced sources held more readable documents than one exact ranking may enumerate. */ - vectorSlicedSaturated?: boolean - vectorSourcesWalked?: number - /** - * Which index ranked an unbounded keyword leg: `tin` ranks by BM25 and checks access on the top - * of that ranking; `gin` ranks every match. Absent when the leg ranked inside a bounded set. - */ - keywordRanking?: 'tin' | 'gin' - /** Candidates Tin ranked before access was checked on the last keyword page. */ - keywordTinWindow?: number vectorCandidateCount?: number vectorCandidateDimensions?: number resultCount?: number @@ -139,10 +103,6 @@ export interface SearchDiagnosticMetadata { accessBatchCount?: number /** Connector identities sent for live proof, summed over every batch after the first. */ liveProofConnectorCount?: number - /** Provider types with a configured search source, before any access probe. */ - configuredProviderCount?: number - /** Searchable-document probes actually issued; one per batch until the answer is known. */ - searchableProbeCount?: number } interface StageTiming { diff --git a/apps/sim/lib/knowledge/search/keyword-ranking.ts b/apps/sim/lib/knowledge/search/keyword-ranking.ts index ed9edf76869..0859183b1ab 100644 --- a/apps/sim/lib/knowledge/search/keyword-ranking.ts +++ b/apps/sim/lib/knowledge/search/keyword-ranking.ts @@ -1,4 +1,4 @@ -import { document, type embedding, type embeddingKeywordSearch } from '@sim/db/schema' +import { document, type embedding } from '@sim/db/schema' import { and, type SQL, sql } from 'drizzle-orm' /** @@ -18,7 +18,7 @@ export function keywordCandidateRankingQuery(input: { /** What a matched document must satisfy to be readable. */ documentConditions: (SQL | undefined)[] /** The table whose text-search vector `rank` reads, joined to the matches on `id`. */ - rankTable: typeof embedding | typeof embeddingKeywordSearch + rankTable: typeof embedding rank: SQL limit: number offset: number diff --git a/apps/sim/lib/knowledge/search/prewarm.test.ts b/apps/sim/lib/knowledge/search/prewarm.test.ts index 07790f39e3f..55dd5339544 100644 --- a/apps/sim/lib/knowledge/search/prewarm.test.ts +++ b/apps/sim/lib/knowledge/search/prewarm.test.ts @@ -1,9 +1,4 @@ import { describe, expect, it, vi } from 'vitest' - -vi.mock('@sim/db/knowledge-projection', () => ({ - SOURCE_ACL_PROJECTIONS: ['embedding_search', 'embedding_keyword_tin'], -})) - import { prewarmSearchProjection } from '@/lib/knowledge/search/prewarm' interface Statement { @@ -60,7 +55,7 @@ describe('prewarmSearchProjection', () => { installed: true, relations: [ 'embedding_search', - 'embedding_keyword_tin', + 'embedding_search_cosine_hnsw_idx', 'embedding_search_512_cosine_hnsw_idx', ], }) @@ -74,7 +69,7 @@ describe('prewarmSearchProjection', () => { const warmed = await prewarmSearchProjection(fake, { budgetMs: 1000 }) expect(warmed.map((item) => item.relation)).toEqual([ 'embedding_search', - 'embedding_keyword_tin', + 'embedding_search_cosine_hnsw_idx', 'embedding_search_512_cosine_hnsw_idx', ]) const timeouts = fake.statements diff --git a/apps/sim/lib/knowledge/search/prewarm.ts b/apps/sim/lib/knowledge/search/prewarm.ts index 239907f1b13..5713d58d5bf 100644 --- a/apps/sim/lib/knowledge/search/prewarm.ts +++ b/apps/sim/lib/knowledge/search/prewarm.ts @@ -1,15 +1,13 @@ -import { SOURCE_ACL_PROJECTIONS } from '@sim/db/knowledge-projection' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' const logger = createLogger('SearchProjectionPrewarm') /** - * The access methods a ranking touches at random: the vector graphs, the Tin keyword index, and - * the GIN index the on-row permission test reads. The remaining b-trees serve hydration, which + * Vector ranking touches graph pages at random. The remaining b-trees serve hydration, which * reads a handful of rows by key and is fast cold. */ -const RANKING_ACCESS_METHODS = ['hnsw', 'tin', 'gin'] as const +const RANKING_ACCESS_METHODS = ['hnsw'] as const /** The one call the helper needs from a `postgres` connection or a reserved session. */ export interface PrewarmSession { @@ -143,7 +141,7 @@ async function rankingRelations(session: PrewarmSession): Promise { AND am.amname = ANY($2::text[]) ) ORDER BY c.relkind = 'r' DESC, pg_relation_size(c.oid)`, - [toArrayLiteral(SOURCE_ACL_PROJECTIONS), toArrayLiteral(RANKING_ACCESS_METHODS)] + [toArrayLiteral(['embedding_search']), toArrayLiteral(RANKING_ACCESS_METHODS)] ) return Array.from(rows, (row) => String(row.relation)) } diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index 0002fd9bb6b..0e55d49ad21 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -646,8 +646,6 @@ interface ExecuteKnowledgeSearchParams { queryVector?: KnowledgeQueryVector structuredFilters?: StructuredFilter[] filters?: WorkspaceSearchFilters - /** Runs the search-index retrieval legs; see `usesIndexedRetrieval`. */ - indexedRetrieval?: boolean } export interface RetrievalStatus { @@ -704,27 +702,9 @@ export async function retrieveKnowledgeSearch( if (hasQuery && !queryVector) { throw new Error('Query vector is required when searching with a query') } - /** - * The one seam between the two retrieval strategies. A signed-in reader's search over search - * indexes, while indexed organization search is on, binds the reader's resolved plan and ranks - * on the projection rows; the dormant module loads only then. Everything else decides - * readability on each candidate's document, under the caller's own tokens and any live source - * proof they hold. - */ - const legs: RetrievalLegs = - access.kind === 'user' && accessProvider && params.indexedRetrieval === true - ? await (await import('@/lib/sim-search/indexed/retrieval')).prepareIndexedRetrieval({ - knowledgeBaseIds, - access, - accessProvider, - filters: params.filters, - signal: params.signal, - ranked: hasQuery, - budget: budgets.vector, - }) - : documentRetrievalLegs( - await resolveDocumentReadAccess(knowledgeBaseIds, access, accessProvider, params.signal) - ) + const legs = documentRetrievalLegs( + await resolveDocumentReadAccess(knowledgeBaseIds, access, accessProvider, params.signal) + ) const common = { knowledgeBaseIds, access, diff --git a/apps/sim/lib/knowledge/search/source-vector-indexes.test.ts b/apps/sim/lib/knowledge/search/source-vector-indexes.test.ts deleted file mode 100644 index 31c4e68ca10..00000000000 --- a/apps/sim/lib/knowledge/search/source-vector-indexes.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' -import { beforeEach, describe, expect, it } from 'vitest' -import { dropSourceVectorIndex } from '@/lib/knowledge/search/source-vector-indexes' - -describe('source vector indexes', () => { - let statements: string[] - - beforeEach(() => { - resetDbChainMock() - statements = [] - dbChainMockFns.execute.mockImplementation(async (query: unknown) => { - statements.push(typeof query === 'string' ? query : JSON.stringify(query)) - return [] - }) - }) - - it('never spells an unexpected identifier into DDL', async () => { - await dropSourceVectorIndex("x'; DROP TABLE document; --") - expect(statements.some((text) => text.includes('DROP TABLE'))).toBe(false) - }) -}) diff --git a/apps/sim/lib/knowledge/search/source-vector-indexes.ts b/apps/sim/lib/knowledge/search/source-vector-indexes.ts deleted file mode 100644 index 4a12dea2038..00000000000 --- a/apps/sim/lib/knowledge/search/source-vector-indexes.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { db } from '@sim/db' -import { sql } from 'drizzle-orm' - -/** - * An index's name and predicate are spelled into DDL, which takes no parameters, so a connector id - * is only ever used after it matches the shape connectors carry. - */ -const CONNECTOR_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/ - -/** Derived, never stored, so a dropped source leaves nothing to reconcile. */ -function indexName(connectorId: string): string { - return `embedding_search_src_${connectorId.replaceAll('-', '')}_hnsw` -} - -/** - * Drops a deleted source's per-source vector index, where an earlier build left one. Nothing - * builds these any more; the dormant indexed search (`lib/sim-search/indexed/retrieval/`) still - * reads the ones that exist, and its brief cache of them only names sources a search can no - * longer reach once their connector is deleted. - */ -export async function dropSourceVectorIndex(connectorId: string): Promise { - if (!CONNECTOR_ID.test(connectorId)) return - await db.execute(sql.raw(`DROP INDEX CONCURRENTLY IF EXISTS "${indexName(connectorId)}"`)) -} diff --git a/apps/sim/lib/knowledge/search/stats.test.ts b/apps/sim/lib/knowledge/search/stats.test.ts deleted file mode 100644 index 14f2cd5146b..00000000000 --- a/apps/sim/lib/knowledge/search/stats.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { organizationSearchStatsQuerySchema } from '@/lib/api/contracts/knowledge/search-stats' -import { getSearchStatsRangeError, getSearchStatsWindow } from '@/lib/knowledge/search/stats' - -const now = new Date('2026-09-10T20:00:00.000Z') -beforeEach(() => { - vi.useFakeTimers() - vi.setSystemTime(now) -}) -afterEach(() => vi.useRealTimers()) - -describe('Search stats window and contract', () => { - it('includes the last custom day with an exclusive next-midnight bound', () => { - expect( - getSearchStatsWindow('custom', now, { startDate: '2026-08-31', endDate: '2026-09-02' }) - ).toEqual({ - start: new Date('2026-08-31T00:00:00.000Z'), - end: new Date('2026-09-03T00:00:00.000Z'), - days: 3, - }) - expect( - getSearchStatsWindow('custom', now, { startDate: '2026-09-10', endDate: '2026-09-10' }) - ).toEqual({ - start: new Date('2026-09-10T00:00:00.000Z'), - end: now, - days: 1, - }) - }) - it.each([ - { startDate: '2026-09-01' }, - { endDate: '2026-09-01' }, - { startDate: '2026-02-29', endDate: '2026-03-01' }, - { startDate: '2026-04-31', endDate: '2026-05-01' }, - { startDate: '2026-9-01', endDate: '2026-09-02' }, - { startDate: '2026-09-03', endDate: '2026-09-02' }, - { startDate: '2026-09-10', endDate: '2026-09-11' }, - { startDate: '2026-06-12', endDate: '2026-09-10' }, - ])('rejects invalid or unbounded custom ranges: %j', (range) => { - expect(getSearchStatsRangeError(range, now)).not.toBeNull() - expect( - organizationSearchStatsQuerySchema.safeParse({ - organizationId: 'org', - period: 'custom', - ...range, - }).success - ).toBe(false) - expect(() => getSearchStatsWindow('custom', now, range)).toThrow() - }) - it('bounds scans and rejects caller-supplied surfaces', () => { - expect( - organizationSearchStatsQuerySchema.safeParse({ organizationId: 'org', period: '365d' }) - .success - ).toBe(false) - expect( - organizationSearchStatsQuerySchema.safeParse({ organizationId: 'org', surface: 'forged' }) - .success - ).toBe(false) - expect(organizationSearchStatsQuerySchema.parse({ organizationId: 'org' })).toEqual({ - organizationId: 'org', - period: '30d', - }) - }) -}) diff --git a/apps/sim/lib/knowledge/search/stats.ts b/apps/sim/lib/knowledge/search/stats.ts deleted file mode 100644 index 35dbf4ded42..00000000000 --- a/apps/sim/lib/knowledge/search/stats.ts +++ /dev/null @@ -1,74 +0,0 @@ -export const SEARCH_STATS_PERIODS = ['today', '3d', '7d', '14d', '30d', '90d', 'custom'] as const -export const SEARCH_STATS_SURFACES = [ - 'dashboard', - 'copilot', - 'mcp', - 'slack', - 'api', - 'workflow', - 'other', -] as const -export const SEARCH_STATS_MAX_DAYS = 90 -const DAY_MS = 86_400_000 - -export const SEARCH_STATS_PEOPLE_LIMIT = 20 -export const SEARCH_STATS_SOURCE_LIMIT = 100 - -export const SEARCH_STATS_SURFACE_LABELS: Record<(typeof SEARCH_STATS_SURFACES)[number], string> = { - dashboard: 'Search', - copilot: 'Assistant', - mcp: 'MCP', - slack: 'Slack', - api: 'API', - workflow: 'Workflows', - other: 'Other', -} - -export interface SearchStatsDateRange { - startDate?: string - endDate?: string -} - -/** Checks the date-only UTC range shared by the picker, API, and aggregate query. */ -export function getSearchStatsRangeError( - range: SearchStatsDateRange, - now = new Date() -): string | null { - const { startDate, endDate } = range - if (!startDate || !endDate) return 'Select both a start date and an end date.' - for (const date of [startDate, endDate]) { - if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return 'Use dates in YYYY-MM-DD format.' - const parsed = new Date(`${date}T00:00:00.000Z`) - if (!Number.isFinite(parsed.getTime()) || parsed.toISOString().slice(0, 10) !== date) { - return 'Select valid calendar dates.' - } - } - if (endDate < startDate) return 'The end date must be on or after the start date.' - if (endDate > now.toISOString().slice(0, 10)) return 'Select dates on or before today (UTC).' - const days = (Date.parse(endDate) - Date.parse(startDate)) / DAY_MS + 1 - if (days > SEARCH_STATS_MAX_DAYS) - return `Select a range of ${SEARCH_STATS_MAX_DAYS} days or fewer.` - return null -} - -/** Daily UTC buckets, with the selected end date included and today capped at the current time. */ -export function getSearchStatsWindow( - period: (typeof SEARCH_STATS_PERIODS)[number], - now = new Date(), - range: SearchStatsDateRange = {} -) { - if (period === 'custom') { - const error = getSearchStatsRangeError(range, now) - if (error) throw new Error(error) - const start = new Date(`${range.startDate}T00:00:00.000Z`) - const endDay = new Date(`${range.endDate}T00:00:00.000Z`) - const end = new Date(Math.min(endDay.getTime() + DAY_MS, now.getTime())) - const days = (endDay.getTime() - start.getTime()) / DAY_MS + 1 - return { start, end, days } - } - const daysByPeriod = { today: 1, '3d': 3, '7d': 7, '14d': 14, '30d': 30, '90d': 90 } as const - const days = daysByPeriod[period] - const start = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) - start.setUTCDate(start.getUTCDate() - days + 1) - return { start, end: now, days } -} diff --git a/apps/sim/lib/mothership/application/load-search-integrations.test.ts b/apps/sim/lib/mothership/application/load-search-integrations.test.ts index e1d38de0db7..e2bf1887645 100644 --- a/apps/sim/lib/mothership/application/load-search-integrations.test.ts +++ b/apps/sim/lib/mothership/application/load-search-integrations.test.ts @@ -1,4 +1,4 @@ -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' +import { resetEnvFlagsMock } from '@sim/testing' import { mothershipOrganizationChatsMock, mothershipOrganizationChatsMockFns, @@ -42,14 +42,17 @@ const emptyPage: InventoryPage = { describe('loadCopilotSearchIntegrations', () => { beforeEach(() => { resetEnvFlagsMock() - /** The paged inventory below is the indexed arm; the live test opts back in. */ - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) + authorizeChat.mockResolvedValue(undefined) listIntegrations.mockResolvedValue(emptyPage) + liveAccounts.mockResolvedValue({ backend: 'live', accounts: [] }) }) it('authorizes the private chat and reads only for the authenticated person and organization', async () => { - expect(await loadCopilotSearchIntegrations(context)).toBe('{"connections":[],"available":[]}') + expect(JSON.parse(await loadCopilotSearchIntegrations(context))).toMatchObject({ + connections: [], + available: [], + }) const principal = authorizeChat.mock.calls[0][0].principal expect(principal).toMatchObject({ kind: 'organization_delegated', @@ -70,7 +73,6 @@ describe('loadCopilotSearchIntegrations', () => { }) it('includes exact live connection targets alongside current provider search accounts', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) const target = { type: 'link', provider: 'slack', @@ -97,45 +99,6 @@ describe('loadCopilotSearchIntegrations', () => { }) }) - it('loads every page and preserves account status and exact connection controls', async () => { - const available: InventoryPage['available'][number] = { - name: 'Gmail', - description: 'Personal mail', - target: { type: 'link', provider: 'google-email', connectorType: 'gmail' }, - } - const connection: InventoryPage['connections'][number] = { - name: 'Gmail', - providerId: 'google-email', - connectorType: 'gmail', - connectorId: 'source-1', - knowledgeBaseId: 'kb-1', - description: 'Personal mail', - accounts: [ - { - credentialId: 'account-1', - displayName: 'me@example.com', - status: 'reconnect_needed', - action: { ...available.target, connectorId: 'source-1', credentialId: 'account-1' }, - }, - ], - connectionStatus: 'reconnect_needed', - indexingStatus: 'indexed', - action: null, - } - listIntegrations - .mockResolvedValueOnce({ ...emptyPage, available: [available], nextCursor: 'page-2' }) - .mockResolvedValueOnce({ ...emptyPage, connections: [connection], available: [available] }) - - expect(JSON.parse(await loadCopilotSearchIntegrations(context))).toEqual({ - connections: [connection], - available: [available], - }) - expect(listIntegrations.mock.calls[1][0]).toEqual({ - principal: authorizeChat.mock.calls[0][0].principal, - input: { organizationId: 'org-1', cursor: 'page-2' }, - }) - }) - it('does not read inventory when private-chat authorization fails', async () => { authorizeChat.mockRejectedValueOnce(new Error('Chat belongs to another person')) await expect(loadCopilotSearchIntegrations(context)).rejects.toThrow( @@ -144,30 +107,6 @@ describe('loadCopilotSearchIntegrations', () => { expect(listIntegrations).not.toHaveBeenCalled() }) - it('fails the turn if a later page cannot be read', async () => { - listIntegrations - .mockResolvedValueOnce({ ...emptyPage, nextCursor: 'page-2' }) - .mockRejectedValueOnce(new Error('Inventory unavailable')) - await expect(loadCopilotSearchIntegrations(context)).rejects.toThrow('Inventory unavailable') - }) - - it('rejects a repeated cursor instead of looping or returning partial inventory', async () => { - listIntegrations.mockResolvedValue({ ...emptyPage, nextCursor: 'page-2' }) - await expect(loadCopilotSearchIntegrations(context)).rejects.toThrow( - 'pagination did not advance' - ) - expect(listIntegrations).toHaveBeenCalledTimes(2) - }) - - it('bounds page loading when the inventory never ends', async () => { - listIntegrations.mockImplementation(async () => ({ - ...emptyPage, - nextCursor: `page-${listIntegrations.mock.calls.length + 1}`, - })) - await expect(loadCopilotSearchIntegrations(context)).rejects.toThrow('exceeded 100 pages') - expect(listIntegrations).toHaveBeenCalledTimes(100) - }) - it('rejects oversized prompt content instead of silently truncating it', async () => { listIntegrations.mockResolvedValueOnce({ ...emptyPage, diff --git a/apps/sim/lib/mothership/application/load-search-integrations.ts b/apps/sim/lib/mothership/application/load-search-integrations.ts index 68c85837496..6a728fbef7e 100644 --- a/apps/sim/lib/mothership/application/load-search-integrations.ts +++ b/apps/sim/lib/mothership/application/load-search-integrations.ts @@ -5,8 +5,6 @@ import { createTrustedOrganizationCopilotPrincipal, } from '@/lib/mothership/auth/application-delegation' import { authorizeOrganizationChatDelegation } from '@/lib/mothership/chat/organization-chats' -import { loadIndexedSearchIntegrationInventory } from '@/lib/sim-search/indexed' -import { isIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' import { listLiveSearchAccounts } from '@/lib/sim-search/live/application' const MAX_INVENTORY_BYTES = 256 * 1024 @@ -33,14 +31,6 @@ export async function loadCopilotSearchIntegrations( ) await authorizeOrganizationChatDelegation.execute({ principal }) - if (isIndexedOrgSearchEnabled()) - return loadIndexedSearchIntegrationInventory({ - principal, - organizationId: context.organizationId, - signal: context.signal, - maxBytes: MAX_INVENTORY_BYTES, - }) - const [inventory, connections] = await Promise.all([ listLiveSearchAccounts.execute({ principal, diff --git a/apps/sim/lib/mothership/assistant/connected-account-tool.test.ts b/apps/sim/lib/mothership/assistant/connected-account-tool.test.ts index d3e7b8e58c3..360f6356c3d 100644 --- a/apps/sim/lib/mothership/assistant/connected-account-tool.test.ts +++ b/apps/sim/lib/mothership/assistant/connected-account-tool.test.ts @@ -7,9 +7,8 @@ import { getIssueV2Tool } from '@/tools/github/get_issue' import { searchIssuesV2Tool } from '@/tools/github/search_issues' describe('GitHub Assistant connected-account adapter', () => { - it('preserves Build and flag-off tool configuration without mutating the registry', () => { - expect(projectAssistantConnectedAccountTool(getIssueV2Tool, false)).toBe(getIssueV2Tool) - const adapted = projectAssistantConnectedAccountTool(getIssueV2Tool, true) + it('preserves Build tool configuration without mutating the registry', () => { + const adapted = projectAssistantConnectedAccountTool(getIssueV2Tool) expect(adapted).not.toBe(getIssueV2Tool) expect(getIssueV2Tool.params.apiKey.required).toBe(true) expect(getIssueV2Tool.oauth).toBeUndefined() @@ -23,7 +22,7 @@ describe('GitHub Assistant connected-account adapter', () => { expect(assistantConnectedAccountTokenParam(adapted)).toBe('apiKey') }) it('uses the same adapter for the existing issue/PR search tool with total_count', () => { - const adapted = projectAssistantConnectedAccountTool(searchIssuesV2Tool, true) + const adapted = projectAssistantConnectedAccountTool(searchIssuesV2Tool) expect(adapted.request).toBe(searchIssuesV2Tool.request) expect(adapted.transformResponse).toBe(searchIssuesV2Tool.transformResponse) expect(adapted.outputs).toBe(searchIssuesV2Tool.outputs) @@ -31,7 +30,7 @@ describe('GitHub Assistant connected-account adapter', () => { }) it('does not turn unrelated API-key tools or GitLab admin sources into personal credentials', () => { const tool = { ...getIssueV2Tool, id: 'gitlab_get_project' } - expect(projectAssistantConnectedAccountTool(tool, true)).toBe(tool) + expect(projectAssistantConnectedAccountTool(tool)).toBe(tool) expect(assistantConnectedAccountTokenParam(tool)).toBeUndefined() }) }) diff --git a/apps/sim/lib/mothership/assistant/connected-account-tool.ts b/apps/sim/lib/mothership/assistant/connected-account-tool.ts index 3569a8d70dd..910b40bf8ce 100644 --- a/apps/sim/lib/mothership/assistant/connected-account-tool.ts +++ b/apps/sim/lib/mothership/assistant/connected-account-tool.ts @@ -1,12 +1,8 @@ import type { ToolMetadata } from '@/tools/metadata' /** Adapt legacy GitHub API-token operations only at the Assistant boundary; Build schemas stay unchanged. */ -export function projectAssistantConnectedAccountTool( - tool: T, - liveSearch: boolean -): T { - if (!liveSearch || !/^github_[a-z0-9_]+$/.test(tool.id) || !tool.params.apiKey || tool.oauth) - return tool +export function projectAssistantConnectedAccountTool(tool: T): T { + if (!/^github_[a-z0-9_]+$/.test(tool.id) || !tool.params.apiKey || tool.oauth) return tool return { ...tool, oauth: { diff --git a/apps/sim/lib/mothership/assistant/tool-policy.ts b/apps/sim/lib/mothership/assistant/tool-policy.ts index 3ccb8b824e6..0c8bcf5251e 100644 --- a/apps/sim/lib/mothership/assistant/tool-policy.ts +++ b/apps/sim/lib/mothership/assistant/tool-policy.ts @@ -1,4 +1,3 @@ -import { isLiveEnterpriseSearchEnabled } from '@/lib/core/config/env-flags' import { projectAssistantConnectedAccountTool } from '@/lib/mothership/assistant/connected-account-tool' import type { ToolMetadata } from '@/tools/metadata' @@ -14,7 +13,7 @@ const CREDENTIAL_PARAMS = new Set(['credential', 'credentialId', 'oauthCredentia /** Assistant uses the regular integration registry, with authentication supplied by the caller's account. */ export function isAssistantIntegrationTool(tool: ToolMetadata | undefined): boolean { if (!tool) return false - tool = projectAssistantConnectedAccountTool(tool, isLiveEnterpriseSearchEnabled) + tool = projectAssistantConnectedAccountTool(tool) const tokenBinding = tool.personalToken const supportsToken = tokenBinding && tool.params[tokenBinding.tokenParam] && tool.params[tokenBinding.hostParam] @@ -35,7 +34,7 @@ export function isAssistantIntegrationTool(tool: ToolMetadata | undefined): bool } export function isAssistantIntegrationParameter(tool: ToolMetadata, name: string): boolean { - tool = projectAssistantConnectedAccountTool(tool, isLiveEnterpriseSearchEnabled) + tool = projectAssistantConnectedAccountTool(tool) if (CREDENTIAL_PARAMS.has(name)) return true if (name === tool.personalToken?.tokenParam || name === tool.personalToken?.hostParam) return false diff --git a/apps/sim/lib/mothership/chat/payload.ts b/apps/sim/lib/mothership/chat/payload.ts index ed649d21e75..7ca78e3d76f 100644 --- a/apps/sim/lib/mothership/chat/payload.ts +++ b/apps/sim/lib/mothership/chat/payload.ts @@ -7,7 +7,7 @@ import { LRUCache } from 'lru-cache' import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription' import { isPaid } from '@/lib/billing/plan-helpers' import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' -import { isHosted, isLiveEnterpriseSearchEnabled } from '@/lib/core/config/env-flags' +import { isHosted } from '@/lib/core/config/env-flags' import { isOAuthServiceDeploymentAvailable } from '@/lib/integrations/availability.server' import { type IntegrationGateConfig, @@ -218,9 +218,7 @@ export async function buildIntegrationToolSchemas( return structuredClone( schemas.filter((schema) => { const original = getToolMetadata(schema.name) - const metadata = original - ? projectAssistantConnectedAccountTool(original, isLiveEnterpriseSearchEnabled) - : undefined + const metadata = original ? projectAssistantConnectedAccountTool(original) : undefined return ( !metadata?.personalToken && metadata?.oauth?.required && @@ -250,7 +248,7 @@ async function buildIntegrationToolSchemasUncached({ const metadata = getToolMetadata(toolId) if (options.personalAccountsOnly && !isAssistantIntegrationTool(metadata)) continue const projectedTool = options.personalAccountsOnly - ? projectAssistantConnectedAccountTool(toolConfig, isLiveEnterpriseSearchEnabled) + ? projectAssistantConnectedAccountTool(toolConfig) : toolConfig const userSchema = createUserToolSchema(projectedTool, { surface: options.schemaSurface, diff --git a/apps/sim/lib/mothership/tools/server/knowledge/workspace-search.test.ts b/apps/sim/lib/mothership/tools/server/knowledge/workspace-search.test.ts index 39ff46cf777..1b78b765008 100644 --- a/apps/sim/lib/mothership/tools/server/knowledge/workspace-search.test.ts +++ b/apps/sim/lib/mothership/tools/server/knowledge/workspace-search.test.ts @@ -1,4 +1,4 @@ -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' +import { resetEnvFlagsMock } from '@sim/testing/mocks/env-flags.mock' import { getMockLogger } from '@sim/testing/mocks/logger.mock' import { mothershipOrganizationChatsMock, @@ -11,20 +11,14 @@ const hoisted = vi.hoisted(() => ({ read: vi.fn(), })) vi.mock('@/lib/mothership/chat/organization-chats', () => mothershipOrganizationChatsMock) -vi.mock('@/lib/sim-search/indexed', () => ({ - searchOrganizationKnowledge: { +vi.mock('@/lib/sim-search/live/application', () => ({ + searchLiveKnowledge: { get operation() { return knowledgeOperations.search }, execute: hoisted.search, }, - searchWorkspaceKnowledge: { - get operation() { - return knowledgeOperations.search - }, - execute: hoisted.search, - }, - readSearchDocument: { + readLiveDocument: { get operation() { return knowledgeOperations.readDocument }, @@ -32,9 +26,7 @@ vi.mock('@/lib/sim-search/indexed', () => ({ }, })) -import { EmbeddingConfigurationError } from '@/lib/embeddings/configuration-error' import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { annotateSearchDiagnostics } from '@/lib/knowledge/search/diagnostics' import { readDocumentServerTool, searchWorkspaceServerTool, @@ -61,9 +53,8 @@ const context = { describe('Assistant retrieval tools', () => { afterEach(resetEnvFlagsMock) beforeEach(() => { - /** These cover the indexed arm of Sim's search and read tools. */ - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) - mocks.search.mockResolvedValue({ + mocks.search.mockImplementation(async ({ input }: { input: { query: string } }) => ({ + query: input.query, retrieval: { status: 'complete', timedOutLegs: [] }, knowledgeBases: [{ id: 'index', name: 'Enterprise Search' }], results: [ @@ -79,7 +70,7 @@ describe('Assistant retrieval tools', () => { similarity: 1, }, ], - }) + })) mocks.read.mockResolvedValue({ knowledgeBaseId: 'index', documentId: 'doc', @@ -93,7 +84,6 @@ describe('Assistant retrieval tools', () => { it.each([{ startDate: '2026-09-01T00:00:00Z' }, { sortBy: 'newest' }, { sortBy: 'oldest' }])( 'returns actionable validation for empty Notion native queries with %j', async (bound) => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) const result = await searchWorkspaceServerTool.execute( { ...bound, @@ -113,7 +103,6 @@ describe('Assistant retrieval tools', () => { { provider: 'slack', kind: 'meeting' }, { provider: 'google_drive', kind: 'transcript' }, ])('rejects $provider searches with an unsupported $kind selector', async (selection) => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) const result = await searchWorkspaceServerTool.execute( { query: 'release', nativeQueries: [{ ...selection, query: 'release' }] }, { ...context, assistantSearch: undefined } @@ -124,89 +113,6 @@ describe('Assistant retrieval tools', () => { }) expect(result).not.toHaveProperty('data') }) - it('returns a safe permanent configuration failure instead of empty results or opaque error', async () => { - mocks.search.mockRejectedValue(new EmbeddingConfigurationError()) - const result = await searchWorkspaceServerTool.execute({ query: 'policy' }, context) - expect(result).toMatchObject({ - success: false, - retryable: false, - capability: 'semantic_retrieval', - reason: 'provider_not_configured', - recovery: expect.stringContaining('authorized original'), - message: expect.stringContaining('embedding provider is not configured'), - }) - expect(result).not.toHaveProperty('data') - expect(result).not.toHaveProperty('results') - }) - it('returns empty incomplete retrieval as a recoverable search outcome and logs coverage', async () => { - mocks.search.mockImplementation(async () => { - annotateSearchDiagnostics({ - retrievalStatus: 'partial', - timedOutLegs: ['vector', 'keyword'], - }) - return { - retrieval: { status: 'partial', timedOutLegs: ['vector', 'keyword'] }, - knowledgeBases: [{ id: 'index', name: 'Enterprise Search' }], - results: [], - } - }) - - const result = await searchWorkspaceServerTool.execute({ query: 'canaries' }, context) - - expect(result).toMatchObject({ - success: true, - message: expect.stringContaining('cannot establish absence or completeness'), - data: { - retrieval: { status: 'partial', timedOutLegs: ['vector', 'keyword'] }, - results: [], - }, - }) - expect(result).not.toHaveProperty('error') - expect(mocks.info).toHaveBeenCalledWith( - 'Knowledge search completed', - expect.objectContaining({ passageBytes: 0, originalPassageBytes: 0, outcome: 'partial' }) - ) - }) - it('pins organization and private chat while reusing the canonical search index and citations', async () => { - const orgContext = { - ...context, - workspaceId: undefined, - organizationId: 'org-1', - chatId: 'chat-1', - requestMode: 'assistant', - } - const result = await searchWorkspaceServerTool.execute( - { query: 'policy', organizationId: 'forged' }, - orgContext - ) - expect(result).toMatchObject({ - success: true, - data: { - results: [ - expect.objectContaining({ - citationUrl: expect.stringContaining('/o/org-1/knowledge/index/doc'), - }), - ], - }, - }) - expect(mocks.authorizeChat).toHaveBeenCalledWith({ - principal: expect.objectContaining({ - kind: 'organization_delegated', - subjectUserId: 'reader', - organizationId: 'org-1', - resourceScope: { chatId: 'chat-1' }, - }), - }) - expect(mocks.search).toHaveBeenCalledWith({ - principal: expect.objectContaining({ organizationId: 'org-1' }), - input: expect.objectContaining({ organizationId: 'org-1', filters: context.assistantSearch }), - }) - await readDocumentServerTool.execute({ documentId: 'doc' }, orgContext) - expect(mocks.read).toHaveBeenCalledWith({ - principal: expect.objectContaining({ organizationId: 'org-1' }), - input: expect.objectContaining({ assertedOrganizationId: 'org-1' }), - }) - }) it.each([ { searchSurface: undefined, expected: 'copilot' }, @@ -284,97 +190,6 @@ describe('Assistant retrieval tools', () => { ) expect(JSON.stringify(result)).not.toContain(secret) }) - - it.each([0, 20, 50])( - 'measures UTF-8 bytes for %i passages without logging their content', - async (count) => { - const content = 'Confidential passage é🔎'.repeat(100) - mocks.search.mockResolvedValueOnce({ - retrieval: { status: 'complete', timedOutLegs: [] }, - knowledgeBases: [{ id: 'index', name: 'Enterprise Search' }], - results: Array.from({ length: count }, (_, index) => ({ - knowledgeBaseId: 'index', - documentId: `doc-${index % 4}`, - documentName: 'Private title', - sourceUrl: null, - sourceModifiedAt: null, - metadata: {}, - content, - chunkIndex: index, - similarity: 1, - })), - }) - - const output = await searchWorkspaceServerTool.execute( - { query: 'Private query', ...(count === 50 ? { topK: 50 } : {}) }, - context - ) - - expect(output.success).toBe(true) - expect(mocks.info).toHaveBeenCalledWith( - 'Knowledge search completed', - expect.objectContaining({ - toolCallId: 'call', - toolResultBytes: Buffer.byteLength(JSON.stringify(output)), - passageBytes: count * Buffer.byteLength(content.slice(0, 1200)), - originalPassageBytes: count * Buffer.byteLength(content), - maxPassageBytes: count ? Buffer.byteLength(content.slice(0, 1200)) : 0, - uniqueDocumentCount: Math.min(count, 4), - }) - ) - const logged = JSON.stringify(mocks.info.mock.calls) - expect(logged).not.toContain('Confidential passage') - expect(logged).not.toContain('Private title') - expect(logged).not.toContain('Private query') - } - ) - - it('returns stable citation IDs with internal links for uploaded documents', async () => { - const result = await searchWorkspaceServerTool.execute({ query: 'orion' }, context) - expect(result).toMatchObject({ - success: true, - data: { - results: [ - expect.objectContaining({ - citationId: 'document:doc', - citationUrl: expect.stringContaining('/workspace/workspace/knowledge/index/doc'), - }), - ], - }, - }) - }) - it('projects the provider name for connected-source citations instead of the index name', async () => { - mocks.search.mockResolvedValueOnce({ - retrieval: { status: 'complete', timedOutLegs: [] }, - knowledgeBases: [{ id: 'index', name: 'Sim Search' }], - results: [ - { - knowledgeBaseId: 'index', - documentId: 'doc', - documentName: 'Launch checklist', - sourceUrl: 'https://mail.google.com/thread', - connectorType: 'gmail', - sourceModifiedAt: null, - metadata: {}, - content: 'body', - chunkIndex: 0, - similarity: 1, - }, - ], - }) - expect(await searchWorkspaceServerTool.execute({ query: 'launch' }, context)).toMatchObject({ - success: true, - data: { - results: [ - expect.objectContaining({ - documentName: 'Launch checklist', - siteName: 'Gmail', - knowledgeBaseName: 'Sim Search', - }), - ], - }, - }) - }) it('rejects untrusted contexts, incompatible sources and out-of-scope document reads', async () => { expect( await searchWorkspaceServerTool.execute( @@ -391,34 +206,4 @@ describe('Assistant retrieval tools', () => { expect(mocks.search).not.toHaveBeenCalled() expect(mocks.read).not.toHaveBeenCalled() }) - it('reads a selected document through the shared use case and caps oversized pages', async () => { - expect( - await readDocumentServerTool.execute({ documentId: 'doc', startChunkIndex: 20 }, context) - ).toMatchObject({ success: true }) - expect(mocks.read).toHaveBeenCalledWith( - expect.objectContaining({ - input: expect.objectContaining({ - assertedWorkspaceId: 'workspace', - filters: context.assistantSearch, - startChunkIndex: 20, - limit: 3, - }), - }) - ) - mocks.read.mockImplementationOnce(async ({ input }: { input: { limit: number } }) => ({ - knowledgeBaseId: 'index', - documentId: 'doc', - documentName: 'Title', - sourceUrl: 'https://source.test/doc', - chunks: Array.from({ length: input.limit }, (_, chunkIndex) => ({ - content: 'body', - chunkIndex, - })), - hasMore: true, - next: null, - })) - const capped = await readDocumentServerTool.execute({ documentId: 'doc', limit: 9 }, context) - expect(capped).toMatchObject({ success: true }) - expect((capped as { data: { chunks: unknown[] } }).data.chunks).toHaveLength(8) - }) }) diff --git a/apps/sim/lib/mothership/tools/server/knowledge/workspace-search.ts b/apps/sim/lib/mothership/tools/server/knowledge/workspace-search.ts index e50144f149a..eb748e5bc45 100644 --- a/apps/sim/lib/mothership/tools/server/knowledge/workspace-search.ts +++ b/apps/sim/lib/mothership/tools/server/knowledge/workspace-search.ts @@ -7,17 +7,10 @@ import { import { getValidationErrorMessage } from '@/lib/api/server/validation' import { getBaseUrl } from '@/lib/core/utils/urls' import { EmbeddingConfigurationError } from '@/lib/embeddings/configuration-error' -import { sourceAuthor } from '@/lib/knowledge/search/author' import { SearchDeadlineError } from '@/lib/knowledge/search/budget' import { createKnowledgeDocumentCitation, liveCitationId } from '@/lib/knowledge/search/citation' -import { - annotateSearchDiagnostics, - measureSearchStage, - recordSearchStageDuration, - withSearchDiagnostics, -} from '@/lib/knowledge/search/diagnostics' +import { withSearchDiagnostics } from '@/lib/knowledge/search/diagnostics' import { intersectWorkspaceSearchFilters } from '@/lib/knowledge/search/filters' -import { matchPassage } from '@/lib/knowledge/search/snippet' import { executeCopilotKnowledgeUseCase, executeCopilotOrganizationKnowledgeUseCase, @@ -26,12 +19,6 @@ import { } from '@/lib/mothership/application/execute-knowledge-use-case' import type { BaseServerTool, ServerToolContext } from '@/lib/mothership/tools/server/base-tool' import { connectorDisplayName } from '@/lib/sim-search/connectors' -import { - readSearchDocument, - searchOrganizationKnowledge, - searchWorkspaceKnowledge, -} from '@/lib/sim-search/indexed' -import { isIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' import { readLiveDocument, searchLiveKnowledge } from '@/lib/sim-search/live/application' import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' @@ -39,9 +26,7 @@ const logger = createLogger('WorkspaceSearchTool') const CITATION_INSTRUCTION = 'Cite the evidence you use as {"id":""}. Use only IDs returned by these tools.' + - (isIndexedOrgSearchEnabled() - ? '' - : ' When referring to a Slack conversation, link the returned sourceContainerName to its sourceContainerUrl when available.') + ' When referring to a Slack conversation, link the returned sourceContainerName to its sourceContainerUrl when available.' export const searchWorkspaceServerTool: BaseServerTool = { name: 'search_workspace', @@ -56,7 +41,6 @@ export const searchWorkspaceServerTool: BaseServerTool = { }, async () => { try { - const inputStarted = performance.now() const scope = requireCopilotKnowledgeScope(context) const { query, topK, nativeQueries, ...requestedFilters } = searchWorkspaceInputSchema.parse(raw) @@ -79,127 +63,47 @@ export const searchWorkspaceServerTool: BaseServerTool = { resultSecretRegistry: registry, signal: context?.abortSignal, } as const - if (!isIndexedOrgSearchEnabled()) { - const nativeProjection = projectResolvedSecretModelContent( - nativeQueries ?? [], - registry - ) - if (!nativeProjection.safe) - return { - success: false, - message: 'Native queries contain protected content. Rephrase them.', - } - const liveInput = { - ...input, - nativeQueries: searchWorkspaceInputSchema.shape.nativeQueries.parse( - nativeQueries ? nativeProjection.value : undefined - ), - } - const data = - scope.kind === 'organization' - ? await executeCopilotOrganizationKnowledgeUseCase(context, searchLiveKnowledge, { - ...liveInput, - organizationId: scope.organizationId, - }) - : await executeCopilotKnowledgeUseCase(context, searchLiveKnowledge, { - ...liveInput, - workspaceId: scope.workspaceId, - }) - return { - success: true, - message: `Found ${data.results.length} live results. Read a documentId when its passage does not answer the question or more context is needed. ${CITATION_INSTRUCTION}`, - data: { - ...data, - results: data.results.map((item) => ({ - ...item, - siteName: connectorDisplayName(item.connectorType ?? ''), - ...createKnowledgeDocumentCitation({ - scope, - knowledgeBaseId: '', - documentId: item.documentId, - sourceUrl: item.sourceUrl, - baseUrl: getBaseUrl(), - }), - citationId: liveCitationId(item.documentId), - })), - }, - } - } - if ( - nativeQueries || - !safeQuery.trim() || - requestedFilters.startDate || - requestedFilters.endDate || - requestedFilters.sortBy - ) + const nativeProjection = projectResolvedSecretModelContent(nativeQueries ?? [], registry) + if (!nativeProjection.safe) return { success: false, - message: - 'Native queries, date-only search, startDate/endDate and sorting require live search. Use modifiedAfter/modifiedBefore with a text query for indexed search.', + message: 'Native queries contain protected content. Rephrase them.', } - recordSearchStageDuration('tool_input', performance.now() - inputStarted) - const result = await measureSearchStage('tool_application', () => + const liveInput = { + ...input, + nativeQueries: searchWorkspaceInputSchema.shape.nativeQueries.parse( + nativeQueries ? nativeProjection.value : undefined + ), + } + const data = scope.kind === 'organization' - ? executeCopilotOrganizationKnowledgeUseCase(context, searchOrganizationKnowledge, { - ...input, + ? await executeCopilotOrganizationKnowledgeUseCase(context, searchLiveKnowledge, { + ...liveInput, organizationId: scope.organizationId, }) - : executeCopilotKnowledgeUseCase(context, searchWorkspaceKnowledge, { - ...input, + : await executeCopilotKnowledgeUseCase(context, searchLiveKnowledge, { + ...liveInput, workspaceId: scope.workspaceId, }) - ) - return await measureSearchStage('tool_presentation', () => { - const names = new Map(result.knowledgeBases.map((base) => [base.id, base.name])) - const output = { - success: true, - message: `${result.retrieval.status === 'partial' ? 'Search coverage is incomplete. Continue with a more specific query or source filter; these results cannot establish absence or completeness. ' : ''}Found ${result.results.length} passage previews. Read a document at its chunkIndex for more context. ${CITATION_INSTRUCTION}`, - data: { - query: safeQuery, - retrieval: result.retrieval, - results: result.results.map((item) => { - const content = projectResolvedSecretModelContent(item.content, registry) - if (!content.safe || typeof content.value !== 'string') - throw new Error('Knowledge result provenance is unavailable') - return { - documentId: item.documentId, - knowledgeBaseId: item.knowledgeBaseId, - knowledgeBaseName: names.get(item.knowledgeBaseId) ?? '', - siteName: item.connectorType - ? connectorDisplayName(item.connectorType) - : names.get(item.knowledgeBaseId), - documentName: item.documentName, - sourceUrl: item.sourceUrl, - connectorType: item.connectorType, - sourceModifiedAt: item.sourceModifiedAt?.toISOString() ?? null, - author: sourceAuthor(item.metadata), - ...matchPassage(content.value, safeQuery, 1200), - chunkIndex: item.chunkIndex, - similarity: item.similarity, - ...createKnowledgeDocumentCitation({ - scope, - knowledgeBaseId: item.knowledgeBaseId, - documentId: item.documentId, - sourceUrl: item.sourceUrl, - baseUrl: getBaseUrl(), - }), - } + return { + success: true, + message: `Found ${data.results.length} live results. Read a documentId when its passage does not answer the question or more context is needed. ${CITATION_INSTRUCTION}`, + data: { + ...data, + results: data.results.map((item) => ({ + ...item, + siteName: connectorDisplayName(item.connectorType ?? ''), + ...createKnowledgeDocumentCitation({ + scope, + knowledgeBaseId: '', + documentId: item.documentId, + sourceUrl: item.sourceUrl, + baseUrl: getBaseUrl(), }), - }, - } - const passageBytes = output.data.results.map((item) => Buffer.byteLength(item.content)) - annotateSearchDiagnostics({ - toolResultBytes: Buffer.byteLength(JSON.stringify(output)), - passageBytes: passageBytes.reduce((total, bytes) => total + bytes, 0), - originalPassageBytes: result.results.reduce( - (total, item) => total + Buffer.byteLength(item.content), - 0 - ), - maxPassageBytes: Math.max(0, ...passageBytes), - uniqueDocumentCount: new Set(output.data.results.map((item) => item.documentId)).size, - }) - return output - }) + citationId: liveCitationId(item.documentId), + })), + }, + } } catch (error) { logger.error('Workspace search failed', { error }) return { @@ -238,47 +142,8 @@ export const readDocumentServerTool: BaseServerTool = { const input = readDocumentInputSchema.parse(raw) const registry = context?.resolvedSecretTraceRegistry if (!registry) throw new Error('Knowledge result provenance is unavailable') - if (!isIndexedOrgSearchEnabled()) { - const liveInput = { - ...input, - filters: intersectWorkspaceSearchFilters( - { documentIds: [input.documentId] }, - context?.assistantSearch - ), - resultSecretRegistry: registry, - signal: context?.abortSignal, - } - const data = - scope.kind === 'organization' - ? await executeCopilotOrganizationKnowledgeUseCase(context, readLiveDocument, { - ...liveInput, - organizationId: scope.organizationId, - }) - : await executeCopilotKnowledgeUseCase(context, readLiveDocument, { - ...liveInput, - workspaceId: scope.workspaceId, - }) - return { - success: true, - message: CITATION_INSTRUCTION, - data: { - ...data, - ...createKnowledgeDocumentCitation({ - scope, - knowledgeBaseId: '', - documentId: data.documentId, - sourceUrl: data.sourceUrl, - baseUrl: getBaseUrl(), - }), - citationId: liveCitationId(data.documentId), - }, - } - } - const readInput = { + const liveInput = { ...input, - ...(scope.kind === 'organization' - ? { assertedOrganizationId: scope.organizationId } - : { assertedWorkspaceId: scope.workspaceId }), filters: intersectWorkspaceSearchFilters( { documentIds: [input.documentId] }, context?.assistantSearch @@ -286,33 +151,31 @@ export const readDocumentServerTool: BaseServerTool = { resultSecretRegistry: registry, signal: context?.abortSignal, } - const result = await measureSearchStage('document_read', () => + const data = scope.kind === 'organization' - ? executeCopilotOrganizationKnowledgeUseCase(context, readSearchDocument, readInput) - : executeCopilotKnowledgeUseCase(context, readSearchDocument, readInput) - ) - const output = { + ? await executeCopilotOrganizationKnowledgeUseCase(context, readLiveDocument, { + ...liveInput, + organizationId: scope.organizationId, + }) + : await executeCopilotKnowledgeUseCase(context, readLiveDocument, { + ...liveInput, + workspaceId: scope.workspaceId, + }) + return { success: true, message: CITATION_INSTRUCTION, data: { - ...result, + ...data, ...createKnowledgeDocumentCitation({ scope, - knowledgeBaseId: result.knowledgeBaseId, - documentId: result.documentId, - sourceUrl: result.sourceUrl, + knowledgeBaseId: '', + documentId: data.documentId, + sourceUrl: data.sourceUrl, baseUrl: getBaseUrl(), }), + citationId: liveCitationId(data.documentId), }, } - annotateSearchDiagnostics({ - toolResultBytes: Buffer.byteLength(JSON.stringify(output)), - passageBytes: result.chunks.reduce( - (total, chunk) => total + Buffer.byteLength(chunk.content), - 0 - ), - }) - return output } catch (error) { logger.error('Document read failed', { error }) return { diff --git a/apps/sim/lib/mothership/tools/server/search-sources.test.ts b/apps/sim/lib/mothership/tools/server/search-sources.test.ts index e16971bcf44..4af61f7fd45 100644 --- a/apps/sim/lib/mothership/tools/server/search-sources.test.ts +++ b/apps/sim/lib/mothership/tools/server/search-sources.test.ts @@ -53,9 +53,11 @@ beforeEach(() => { }) describe('Search source direct tool', () => { it('uses authenticated actor and org and forwards viewer-safe pagination', async () => { - expect(await tool.execute({ action: 'list', cursor: 'previous', mine: true }, context)).toEqual( - { action: 'list', sources: [], nextCursor: 'next' } - ) + expect(await tool.execute({ action: 'list', cursor: 'previous' }, context)).toEqual({ + action: 'list', + sources: [], + nextCursor: 'next', + }) expect(mocks.list).toHaveBeenCalledWith({ principal: expect.objectContaining({ kind: 'organization_delegated', @@ -64,7 +66,7 @@ describe('Search source direct tool', () => { audience: 'sim:knowledge', resourceScope: { chatId: 'actual-chat' }, }), - input: { organizationId: 'actual-org', cursor: 'previous', mine: true }, + input: { organizationId: 'actual-org', cursor: 'previous' }, }) expect(mocks.chat).toHaveBeenCalledBefore(mocks.list) }) diff --git a/apps/sim/lib/mothership/tools/server/settings-connected-accounts.ts b/apps/sim/lib/mothership/tools/server/settings-connected-accounts.ts index 24baafe6418..30bf4b31c4e 100644 --- a/apps/sim/lib/mothership/tools/server/settings-connected-accounts.ts +++ b/apps/sim/lib/mothership/tools/server/settings-connected-accounts.ts @@ -9,14 +9,12 @@ import { inviteOrganizationAccountPeopleBodySchema, listOrganizationAccountPeopleQuerySchema, startOrganizationAccountConnectionBodySchema, - updateOrganizationAccountIndexingBodySchema, updateOrganizationAccountWorkspaceAccessBodySchema, } from '@/lib/api/contracts/organization-accounts' import { getOrganizationAccountWorkspaceAccess, updateOrganizationAccountWorkspaceAccess, } from '@/lib/credential-groups/application/organization-access' -import { updateOrganizationAccountIndexing } from '@/lib/credential-groups/application/organization-account-indexing' import { addOrganizationAccountMcpProvider, inviteOrganizationAccountPeople, @@ -152,17 +150,6 @@ export const connectedAccountSettingsActions = { return { revision: result.revision, grants: result.grants } } ), - set_indexing: settingsOperation( - 'write', - updateOrganizationAccountIndexingBodySchema, - async (context, input) => { - const result = await updateOrganizationAccountIndexing.execute({ - principal: context.principal, - input: { ...input, organizationId: settingsOrganizationId(context) }, - }) - return { enabled: result.enabled, knowledgeBaseIds: result.knowledgeBaseIds } - } - ), add_mcp_provider: settingsOperation( 'write', z.union([ diff --git a/apps/sim/lib/selectors/application/execute-selector.test.ts b/apps/sim/lib/selectors/application/execute-selector.test.ts index 71563f35f56..f3611cd2469 100644 --- a/apps/sim/lib/selectors/application/execute-selector.test.ts +++ b/apps/sim/lib/selectors/application/execute-selector.test.ts @@ -21,12 +21,8 @@ const hoisted = vi.hoisted(() => ({ resolveReferences: vi.fn(), resolveScope: vi.fn(), sanitize: vi.fn(), - authorizePersonalSearch: vi.fn(), })) -vi.mock('@/lib/knowledge/application/personal-search-account', () => ({ - authorizePersonalSearchSetup: hoisted.authorizePersonalSearch, -})) vi.mock('@/lib/core/application/organization-authorization', () => organizationAuthorizationMock) vi.mock('@sim/audit', () => auditMock) @@ -169,7 +165,6 @@ describe('executeSelector', () => { 'admin', 'knowledge.use' ) - expect(mocks.authorizePersonalSearch).not.toHaveBeenCalled() expect(mocks.getAttachment).not.toHaveBeenCalled() } ) @@ -190,39 +185,6 @@ describe('executeSelector', () => { expect(mocks.executeAttachment).not.toHaveBeenCalled() }) - it('rejects a personal setup marker outside its approved provider selector and organization scope', async () => { - await expect(execute({ personalSearchSetup: 'jira' })).rejects.toBeInstanceOf( - SelectorContextUnavailableError - ) - await expect( - execute({ - scope: { kind: 'organization', organizationId: 'org-1' }, - selectorKey: 'jira.issues', - personalSearchSetup: 'jira', - }) - ).rejects.toBeInstanceOf(SelectorContextUnavailableError) - expect(mocks.authorizeCredential).not.toHaveBeenCalled() - expect(mocks.executeAttachment).not.toHaveBeenCalled() - }) - - it('requires the personal setup authorization before canonical discovery and provider calls', async () => { - mocks.authorizePersonalSearch.mockRejectedValueOnce(new Error('Integration unapproved')) - await expect( - execute({ - scope: { kind: 'organization', organizationId: 'org-1' }, - selectorKey: 'jira.projectKeys', - context: { oauthCredential: 'managed-1', domain: 'example.atlassian.net' }, - personalSearchSetup: 'jira', - }) - ).rejects.toThrow('Integration unapproved') - expect(mocks.authorizePersonalSearch).toHaveBeenCalledWith(principal, { - organizationId: 'org-1', - connectorType: 'jira', - }) - expect(mocks.resolveScope).not.toHaveBeenCalled() - expect(mocks.executeAttachment).not.toHaveBeenCalled() - }) - /** * The picker is a use of the integration, not a neutral list: it reaches the * provider's API with the caller's credential. The authorization funnel never diff --git a/apps/sim/lib/selectors/application/execute-selector.ts b/apps/sim/lib/selectors/application/execute-selector.ts index df881f217fb..a1b2c38fe8d 100644 --- a/apps/sim/lib/selectors/application/execute-selector.ts +++ b/apps/sim/lib/selectors/application/execute-selector.ts @@ -10,7 +10,6 @@ import type { OperationUseCase } from '@/lib/core/application/operation' import { requireOrganizationMembership } from '@/lib/core/application/organization-authorization' import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' -import { authorizePersonalSearchSetup } from '@/lib/knowledge/application/personal-search-account' import { type CredentialAuditRequest, recordCredentialAccess } from '@/lib/oauth/token-resolution' import { SELECTOR_DELEGATION_AUDIENCE, @@ -45,8 +44,6 @@ const logger = createLogger('ExecuteSelector') export interface ExecuteSelectorInput extends ExecuteSelectorRequest { signal?: AbortSignal auditRequest?: CredentialAuditRequest - /** Set only by the personal Search setup use case; excluded from the public selector contract. */ - personalSearchSetup?: 'jira' | 'confluence' } function validateAuthorizedInput( @@ -172,7 +169,6 @@ async function executeAuthorizedSelector(args: { scope: args.input.scope, workspaceId: args.context.workspaceId, organizationId, - personalSearchSetup: args.input.personalSearchSetup, policy: attachment.credential, protectedValues, references: resolved.references, @@ -203,9 +199,7 @@ async function executeAuthorizedSelector(args: { }) const credentialAccess = credential?.access - const credentialResourceId = credential?.personalSearchSetup - ? credential.suppliedId - : credentialAccess?.resolvedCredentialId + const credentialResourceId = credentialAccess?.resolvedCredentialId let credentialUseRecorded = false const recordCredentialUse = attachment.auditCredentialUse && credentialResourceId @@ -355,25 +349,15 @@ export const executeSelector: OperationUseCase< }, } if (args.input.scope.kind !== 'organization') { - if (args.input.personalSearchSetup) throw new SelectorContextUnavailableError() return executeWorkspaceSelector.execute(args) } if (args.principal.kind !== 'session') throw new SelectorContextUnavailableError() - if (args.input.personalSearchSetup) { - const selectorKey = - args.input.personalSearchSetup === 'jira' ? 'jira.projectKeys' : 'confluence.spaces' - if (args.input.selectorKey !== selectorKey) throw new SelectorContextUnavailableError() - await authorizePersonalSearchSetup(args.principal, { - organizationId: args.input.scope.organizationId, - connectorType: args.input.personalSearchSetup, - }) - } else - await requireOrganizationMembership( - args.principal, - args.input.scope.organizationId, - 'admin', - 'knowledge.use' - ) + await requireOrganizationMembership( + args.principal, + args.input.scope.organizationId, + 'admin', + 'knowledge.use' + ) const context = await resolveSelectorApplicationContext({ selectorKey: args.input.selectorKey as ServerSelectorKey, scope: args.input.scope, diff --git a/apps/sim/lib/selectors/server/credentials.test.ts b/apps/sim/lib/selectors/server/credentials.test.ts index 46ebb61fd5f..ef2d76f9514 100644 --- a/apps/sim/lib/selectors/server/credentials.test.ts +++ b/apps/sim/lib/selectors/server/credentials.test.ts @@ -1,10 +1,6 @@ import { credential } from '@sim/db/schema' import { queueTableRows, resetDbChainMock } from '@sim/testing' import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing/mocks/auth-oauth-utils.mock' -import { - credentialsManagedOauthMock, - credentialsManagedOauthMockFns, -} from '@sim/testing/mocks/credentials-managed-oauth.mock' import { oauthUtilsMock, oauthUtilsMockFns } from '@sim/testing/mocks/oauth-utils.mock' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -12,7 +8,6 @@ const mocksHoisted = vi.hoisted(() => ({ authorizeCredentialUse: vi.fn(), authorizeOrganizationCredentialUse: vi.fn(), resolveOrganizationCredentialTokenBundle: vi.fn(), - authorizePersonalSearch: vi.fn(), })) vi.mock('@/lib/auth/credential-access', () => ({ @@ -26,11 +21,6 @@ vi.mock('@/lib/credentials/application/organization-credentials', () => ({ vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) -vi.mock('@/lib/knowledge/application/personal-search-account', () => ({ - authorizePersonalSearchSetupCredential: mocksHoisted.authorizePersonalSearch, -})) -vi.mock('@/lib/credentials/managed-oauth', () => credentialsManagedOauthMock) - vi.mock('@/lib/oauth/utils', () => oauthUtilsMock) import { @@ -45,7 +35,6 @@ const mocks = { resolveCredentialTokenBundle: authOAuthUtilsMockFns.mockResolveCredentialTokenBundle, credentialProviderMatchesService: oauthUtilsMockFns.mockCredentialProviderMatchesService, getServiceConfig: oauthUtilsMockFns.mockGetServiceConfigByServiceId, - resolveManagedOAuthToken: credentialsManagedOauthMockFns.mockResolveManagedOAuthToken, } const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } @@ -124,83 +113,6 @@ describe('authorizeSelectorCredential', () => { }) }) - it('browses with only the member’s own prepared account and refreshes it without the admin credential path', async () => { - mocks.authorizePersonalSearch.mockResolvedValue({ id: 'managed-1', providerId: 'jira' }) - mocks.resolveManagedOAuthToken.mockResolvedValue({ accessToken: 'private-token' }) - const selected = await authorizeSelectorCredential({ - principal, - context: { oauthCredential: 'managed-1' }, - scope: { kind: 'organization', organizationId: 'org-1' }, - organizationId: 'org-1', - personalSearchSetup: 'jira', - policy: { kind: 'stored', field: 'oauthCredential', serviceIds: ['jira'] }, - protectedValues: createSelectorProtectedValues(), - references: new Map(), - }) - const protectedValues = createSelectorProtectedValues() - const recordCredentialUse = vi.fn() - await expect( - resolveSelectorOAuthAccessToken({ - credential: selected, - serviceId: 'jira', - scopes: ['read:jira-work'], - protectedValues, - recordCredentialUse, - }) - ).resolves.toBe('private-token') - expect(mocks.authorizePersonalSearch).toHaveBeenCalledTimes(2) - expect(mocks.resolveManagedOAuthToken).toHaveBeenCalledWith({ - organizationId: 'org-1', - credentialId: 'managed-1', - expectedProviderId: 'jira', - requiredScopes: ['read:jira-work'], - }) - expect(protectedValues.contains('private-token')).toBe(true) - expect(recordCredentialUse).toHaveBeenCalledWith('jira') - expect(mocks.authorizeOrganizationCredentialUse).not.toHaveBeenCalled() - expect(mocks.resolveOrganizationCredentialTokenBundle).not.toHaveBeenCalled() - }) - - it('rejects a revoked prepared credential before refreshing its token', async () => { - mocks.authorizePersonalSearch.mockRejectedValue(new Error('Credential revoked')) - await expect( - resolveSelectorOAuthAccessToken({ - credential: { - suppliedId: 'managed-1', - providerId: 'jira', - personalSearchSetup: { principal, organizationId: 'org-1', connectorType: 'jira' }, - }, - serviceId: 'jira', - protectedValues: createSelectorProtectedValues(), - }) - ).rejects.toThrow('Credential revoked') - expect(mocks.resolveManagedOAuthToken).not.toHaveBeenCalled() - }) - - it('refuses using a prepared Jira credential for another provider or impersonation', async () => { - const selected = { - suppliedId: 'managed-1', - providerId: 'jira', - personalSearchSetup: { principal, organizationId: 'org-1', connectorType: 'jira' as const }, - } - await expect( - resolveSelectorOAuthAccessToken({ - credential: selected, - serviceId: 'confluence', - protectedValues: createSelectorProtectedValues(), - }) - ).rejects.toBeInstanceOf(SelectorConnectionUnavailableError) - await expect( - resolveSelectorOAuthAccessToken({ - credential: selected, - serviceId: 'jira', - impersonateEmail: 'other@example.com', - protectedValues: createSelectorProtectedValues(), - }) - ).rejects.toBeInstanceOf(SelectorConnectionUnavailableError) - expect(mocks.resolveManagedOAuthToken).not.toHaveBeenCalled() - }) - it('rejects an organization connection for the wrong selector provider', async () => { mocks.authorizeOrganizationCredentialUse.mockResolvedValue({ credential: { id: 'managed-1', providerId: 'jira', type: 'managed_oauth' }, diff --git a/apps/sim/lib/selectors/server/credentials.ts b/apps/sim/lib/selectors/server/credentials.ts index aa4eb7ef2ed..1167e61306d 100644 --- a/apps/sim/lib/selectors/server/credentials.ts +++ b/apps/sim/lib/selectors/server/credentials.ts @@ -10,8 +10,6 @@ import { authorizeOrganizationCredentialUse, resolveOrganizationCredentialTokenBundle, } from '@/lib/credentials/application/organization-credentials' -import { resolveManagedOAuthToken } from '@/lib/credentials/managed-oauth' -import { authorizePersonalSearchSetupCredential } from '@/lib/knowledge/application/personal-search-account' import { resolveCredentialTokenBundle } from '@/lib/oauth/credential-service' import { credentialProviderMatchesService, getServiceConfigByServiceId } from '@/lib/oauth/utils' import { SelectorConnectionUnavailableError } from '@/lib/selectors/server/errors' @@ -107,7 +105,6 @@ export async function authorizeSelectorCredential(input: { scope: SelectorScope workspaceId?: string organizationId?: string - personalSearchSetup?: 'jira' | 'confluence' policy: SelectorCredentialPolicy protectedValues: SelectorProtectedValues references: ReadonlyMap @@ -115,32 +112,6 @@ export async function authorizeSelectorCredential(input: { const suppliedId = input.context[input.policy.field] if (!suppliedId) throw new SelectorConnectionUnavailableError() - if (input.personalSearchSetup) { - if ( - input.scope.kind !== 'organization' || - input.principal.kind !== 'session' || - input.organizationId !== input.scope.organizationId || - input.workspaceId || - !input.policy.serviceIds.includes(input.personalSearchSetup) - ) - throw new SelectorConnectionUnavailableError() - const row = await authorizePersonalSearchSetupCredential(input.principal, { - organizationId: input.scope.organizationId, - connectorType: input.personalSearchSetup, - credentialId: suppliedId, - }) - input.protectedValues.add(suppliedId, 'reference') - return { - suppliedId, - providerId: row.providerId, - personalSearchSetup: { - principal: input.principal, - organizationId: input.scope.organizationId, - connectorType: input.personalSearchSetup, - }, - } - } - if (input.scope.kind === 'organization') { if ( input.principal.kind !== 'session' || @@ -217,31 +188,6 @@ export async function resolveSelectorOAuthAccessToken(input: { input.credential.signal?.throwIfAborted() if (input.credential.fixedToken) return input.credential.fixedToken - if (input.credential.personalSearchSetup) { - const setup = input.credential.personalSearchSetup - if (input.impersonateEmail || input.serviceId !== setup.connectorType) { - throw new SelectorConnectionUnavailableError() - } - await authorizePersonalSearchSetupCredential(setup.principal, { - ...setup, - credentialId: input.credential.suppliedId, - }) - const result = await waitForSelectorCredentialResolution( - resolveManagedOAuthToken({ - organizationId: setup.organizationId, - credentialId: input.credential.suppliedId, - expectedProviderId: setup.connectorType, - requiredScopes: input.scopes ? [...input.scopes] : [], - }), - input.credential.signal - ) - input.credential.signal?.throwIfAborted() - if (!result?.accessToken) throw new SelectorConnectionUnavailableError() - input.protectedValues.add(result.accessToken) - input.recordCredentialUse?.(setup.connectorType) - return result.accessToken - } - if (input.credential.organization) { const result = await waitForSelectorCredentialResolution( resolveOrganizationCredentialTokenBundle({ diff --git a/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts b/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts index bbac1e777e9..a98da4a940e 100644 --- a/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts +++ b/apps/sim/lib/selectors/server/providers/credential-bundle.test.ts @@ -1,18 +1,7 @@ import { authOAuthUtilsMock, authOAuthUtilsMockFns } from '@sim/testing/mocks/auth-oauth-utils.mock' -import { - credentialsManagedOauthMock, - credentialsManagedOauthMockFns, -} from '@sim/testing/mocks/credentials-managed-oauth.mock' import { describe, expect, it, vi } from 'vitest' const mockResolveOrganizationToken = vi.hoisted(() => vi.fn()) -const mockOwnAccount = vi.hoisted(() => vi.fn()) - -vi.mock('@/lib/knowledge/application/personal-search-account', () => ({ - authorizePersonalSearchSetupCredential: mockOwnAccount, -})) -vi.mock('@/lib/credentials/managed-oauth', () => credentialsManagedOauthMock) - vi.mock('@/lib/credentials/application/organization-credentials', () => ({ resolveOrganizationCredentialTokenBundle: mockResolveOrganizationToken, })) @@ -24,45 +13,7 @@ import { resolveSelectorCredentialBundle } from '@/lib/selectors/server/provider const mockResolveCredentialAccessToken = authOAuthUtilsMockFns.mockResolveCredentialTokenBundle -const mockResolveManagedToken = credentialsManagedOauthMockFns.mockResolveManagedOAuthToken - describe('selector credential bundles', () => { - it('resolves a personal Atlassian grant through the owned managed-account path', async () => { - mockOwnAccount.mockResolvedValue({ id: 'managed-1', providerId: 'jira' }) - mockResolveManagedToken.mockResolvedValue({ accessToken: 'own-managed-token' }) - const principal = { kind: 'session', userId: 'member-1', sessionId: 'session-1' } as const - const protectedValues = createSelectorProtectedValues() - await expect( - resolveSelectorCredentialBundle({ - credential: { - suppliedId: 'managed-1', - providerId: 'jira', - personalSearchSetup: { principal, organizationId: 'org-1', connectorType: 'jira' }, - }, - providerId: 'jira', - scopes: ['read:jira-work'], - protectedValues, - }) - ).resolves.toEqual({ accessToken: 'own-managed-token' }) - expect(mockOwnAccount).toHaveBeenCalledWith( - principal, - expect.objectContaining({ - credentialId: 'managed-1', - organizationId: 'org-1', - connectorType: 'jira', - }) - ) - expect(mockResolveManagedToken).toHaveBeenCalledWith({ - organizationId: 'org-1', - credentialId: 'managed-1', - expectedProviderId: 'jira', - requiredScopes: ['read:jira-work'], - }) - expect(mockResolveOrganizationToken).not.toHaveBeenCalled() - expect(mockResolveCredentialAccessToken).not.toHaveBeenCalled() - expect(protectedValues.contains('own-managed-token')).toBe(true) - }) - it('protects short credential-bound cloud ids as exact identifiers', async () => { mockResolveCredentialAccessToken.mockResolvedValue({ accessToken: 'server-only-token', diff --git a/apps/sim/lib/selectors/server/providers/credential-bundle.ts b/apps/sim/lib/selectors/server/providers/credential-bundle.ts index b134cf89047..97669ff8a8f 100644 --- a/apps/sim/lib/selectors/server/providers/credential-bundle.ts +++ b/apps/sim/lib/selectors/server/providers/credential-bundle.ts @@ -3,10 +3,7 @@ import { resolveCredentialTokenBundle, type ServiceAccountTokenResult, } from '@/lib/oauth/credential-service' -import { - resolveSelectorOAuthAccessToken, - waitForSelectorCredentialResolution, -} from '@/lib/selectors/server/credentials' +import { waitForSelectorCredentialResolution } from '@/lib/selectors/server/credentials' import { SelectorConnectionUnavailableError } from '@/lib/selectors/server/errors' import type { AuthorizedSelectorCredential, @@ -29,16 +26,6 @@ export async function resolveSelectorCredentialBundle(input: { if (!credential) throw new SelectorConnectionUnavailableError() credential.signal?.throwIfAborted() - if (credential.personalSearchSetup) { - if (!input.providerId) throw new SelectorConnectionUnavailableError() - return { - accessToken: await resolveSelectorOAuthAccessToken({ - ...input, - credential, - serviceId: input.providerId, - }), - } - } if (credential.fixedToken) { if (input.providerId) { input.recordCredentialUse?.(credential.providerId ?? input.providerId) diff --git a/apps/sim/lib/selectors/server/types.ts b/apps/sim/lib/selectors/server/types.ts index 113ac2e2174..b361c675c26 100644 --- a/apps/sim/lib/selectors/server/types.ts +++ b/apps/sim/lib/selectors/server/types.ts @@ -78,11 +78,6 @@ export type SelectorCredentialPolicy = export interface AuthorizedSelectorCredential { suppliedId: string organization?: { principal: SessionPrincipal; organizationId: string } - personalSearchSetup?: { - principal: SessionPrincipal - organizationId: string - connectorType: 'jira' | 'confluence' - } access?: CredentialAccessResult fixedToken?: string /** Trusted provider id loaded during server-side credential binding. */ diff --git a/apps/sim/lib/selectors/types.ts b/apps/sim/lib/selectors/types.ts index 2584863072c..67e9256d5cc 100644 --- a/apps/sim/lib/selectors/types.ts +++ b/apps/sim/lib/selectors/types.ts @@ -109,13 +109,6 @@ export type SelectorScope = workspaceId: string } -/** Chooses a dedicated client transport without granting access through the generic selector API. */ -export interface SelectorSurface { - kind: 'personal-search-setup' - organizationId: string - connectorType: 'jira' | 'confluence' -} - export type SelectorRequest = | { kind: 'list' diff --git a/apps/sim/lib/sim-search/connectors.ts b/apps/sim/lib/sim-search/connectors.ts index b8e892ebf8c..0e7da56295a 100644 --- a/apps/sim/lib/sim-search/connectors.ts +++ b/apps/sim/lib/sim-search/connectors.ts @@ -13,7 +13,7 @@ import { import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' import type { ConnectorConfigField, ConnectorMeta } from '@/connectors/types' -/** The workspace knowledge base Sim Search indexes into, one per workspace, created on first connect. */ +/** The knowledge-base shell that holds live Search source configuration. */ export const SIM_SEARCH_KNOWLEDGE_BASE_NAME = 'Sim Search' /** @@ -130,42 +130,6 @@ export function canConnectWithDefaults(meta: ConnectorMeta): boolean { return canConnectPersonally(meta) && meta.id !== 'slack' && personalSetupFields(meta).length === 0 } -/** - * The settings a person may supply when a source is created from Sim Search: - * its setup fields plus anything the connector's Search defaults cover. - */ -export function personalSourceConfigFieldIds(meta: ConnectorMeta): Set { - return new Set([ - ...personalSetupFields(meta).map((field) => field.id), - ...Object.keys(meta.searchDefaultSourceConfig ?? {}), - ]) -} - -/** - * A Search source's settings, starting from the connector's Search defaults. - * A supplied value replaces its default; a blank one leaves the default in - * place, so an untouched form field never widens the source. - */ -export function withSearchSourceDefaults( - meta: Pick, - sourceConfig: Record = {} -): Record { - const merged: Record = { ...(meta.searchDefaultSourceConfig ?? {}) } - for (const [field, value] of Object.entries(sourceConfig)) { - if (typeof value === 'string' && value.trim() === '' && field in merged) continue - merged[field] = value - } - return merged -} - -/** The setup fields a source config leaves empty. */ -export function missingSetupFields( - meta: ConnectorMeta, - sourceConfig: Record -): ConnectorConfigField[] { - return personalSetupFields(meta).filter((field) => !sourceConfig[field.id]?.trim()) -} - /** The name a connector shows, from its registry entry. */ export function connectorDisplayName(connectorType: string): string { return CONNECTOR_META_REGISTRY[connectorType]?.name ?? connectorType diff --git a/apps/sim/lib/sim-search/indexed/README.md b/apps/sim/lib/sim-search/indexed/README.md deleted file mode 100644 index 07440bedb8e..00000000000 --- a/apps/sim/lib/sim-search/indexed/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Indexed organization search (dormant) - -The indexed backend for Sim Search: retrieval over `is_search_index` knowledge bases that organization and workspace connectors crawl into, ranked from the embedding projections. Live Search (`../live/`) replaced it. **This code is dormant**: it stays in the tree so it can be switched back on, but no request reaches it in a default deployment. - -Ordinary workspace knowledge bases, the Knowledge block, the embedding projections, the projector, and the document access predicate (`lib/knowledge/access/predicate.ts`) live outside this directory and behave the same whichever backend is selected. - -## The gate - -`isIndexedOrgSearchEnabled()` in `gate.ts` is the only switch. It is the inverse of `SIM_SEARCH_LIVE`, which defaults to `true`, so indexed search is on only where a deployment sets `SIM_SEARCH_LIVE=false`. Every use case in this directory calls `assertIndexedOrgSearchEnabled()` itself, so dormancy holds even for a caller that skipped the gate. - -While the gate is off: - -- Search, the MCP tools, and Sim's `search_workspace` and `read_document` tools serve Live Search, and personal Search integrations are read from live accounts. -- Indexed-only surfaces refuse with `SearchIndexDormantError` (a `409`): the Stats report and connecting a source that crawls into a search index. The indexed document page is not found. -- A knowledge search that names a search-index knowledge base (the Knowledge block, v1, v2, Sim's knowledge tool) still answers from the documents it already holds, decided on each document exactly as a workspace knowledge base is. -- Nothing crawls into search indexes: content syncs, member syncs, processing recovery, document dispatch, and queued document workers skip them (`lib/knowledge/connectors/indexing-policy.ts`). -- The projector owes search-index documents nothing: their marks are released with the rest, and it writes no Tin keyword rows. The GIN keyword projection follows `is_search_index` alone, so it keeps its search-index rows either way. - -## Layout - -- `gate.ts`: the switch, `SearchIndexDormantError`, and the search-index helpers. Anything may import it. -- `index.ts`: the use-case barrel. Its callers branch on the gate first. -- `search/`, `documents/`, `mcp/`, `integrations/`: indexed search, document reads, the indexed MCP tools, and the indexed arms of the personal Search integration inventory. -- `retrieval/`: the search-index retrieval legs behind one entry, `prepareIndexedRetrieval`, which `lib/knowledge/search/queries.ts` loads with a dynamic import only for a signed-in reader whose every base is a search index while the gate is on. Every other search, including every workspace knowledge base search, decides readability on the document and reads none of it. - -The dormant UI sits in `indexed/` folders next to the component that picks it from `features.liveEnterpriseSearch` (`useDeploymentShape()`), so each can be deleted in one step: `app/o/[organizationId]/integrations/indexed/`, `app/o/[organizationId]/settings/components/integrations/indexed/`, and `app/workspace/[workspaceId]/home/components/knowledge-search-results/indexed/`. - -## Re-enabling - -1. Set `SIM_SEARCH_LIVE=false` in both the app and the Trigger.dev environment, and deploy. The container entrypoint (`apps/sim/bootstrap.ts`) mirrors it to `NEXT_PUBLIC_SIM_SEARCH_LIVE` for the client; crawling, processing, and projection read it in whichever process runs them. -2. Confirm the keyword projection objects exist (`0019_tin_keyword_projection`, `0024_knowledge_projection_async`, `0025_scope_keyword_projections`). Both keyword projections, `embedding_keyword_search` and `embedding_keyword_tin`, hold only search-index rows, written by the chunk triggers and by the trigger on `knowledge_base.is_search_index`. Backfill both for every search-index knowledge base whose rows were removed while dormant, and build the Tin index. -3. If the `0027_retire_search_embeddings` cleanup ran for a base, deliberately restore its retired documents' eligibility before resyncing. Its deleted embeddings cannot be recovered by changing the backend flag alone. See `packages/db/script-migrations/search-embedding-retirement.md` for the cleanup lifecycle. -4. Resume and fully resync the connectors of search-index knowledge bases, so content that went stale while dormant is indexed again. - -Projection rows written before projections carried their document's source and ACL are decided on their document until they are rewritten. - -## Database objects it depends on - -`knowledge_base.is_search_index`, `document.acl`, `document.connector_id`, `knowledge_connector`, `embedding_search`, `embedding_keyword_search`, `embedding_keyword_tin`, `knowledge_projection_dirty`, and the Tin extension objects. All of them are owned by `packages/db`; no schema or migration belongs to this directory. diff --git a/apps/sim/lib/sim-search/indexed/documents/read-indexed-document.ts b/apps/sim/lib/sim-search/indexed/documents/read-indexed-document.ts deleted file mode 100644 index 30b2fa5e947..00000000000 --- a/apps/sim/lib/sim-search/indexed/documents/read-indexed-document.ts +++ /dev/null @@ -1,230 +0,0 @@ -import { db } from '@sim/db' -import { document, embedding } from '@sim/db/schema' -import { and, eq, isNull, lte, sql } from 'drizzle-orm' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' -import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' -import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' -import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' -import { - resolveActiveKnowledgeResourceContext, - resolveKnowledgeOrganizationContext, -} from '@/lib/knowledge/application/contexts' -import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import type { ChunkQueryResult } from '@/lib/knowledge/chunks/types' -import { knowledgeReadAccessBatches } from '@/lib/knowledge/read-access' -import { findSearchIndex } from '@/lib/knowledge/search/search-index' -import { isKnowledgeSourceUrl } from '@/lib/knowledge/search/source-url' -import { - createKnowledgeDocumentSourceValue, - importKnowledgePersistedResponseSecretProvenance, -} from '@/lib/knowledge/secret-provenance' -import { assertIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' -import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' - -type IndexedKnowledgeDocumentTarget = - | { kind: 'id'; documentId: string } - | { kind: 'url'; url: string } - -export interface ReadIndexedKnowledgeDocumentInput { - organizationId: string - target: IndexedKnowledgeDocumentTarget - limit: number - offset?: number - aroundChunkIndex?: number - resultSecretRegistry: ResolvedSecretTraceRegistry - signal?: AbortSignal -} - -export interface ReadIndexedKnowledgeDocumentResult { - knowledgeBaseId: string - documentId: string - title: string - sourceUrl: string | null - sourceModifiedAt: string | null - connectorType: string | null - processingStatus: string - chunks?: { id: string; chunkIndex: number; content: string }[] - pagination?: ChunkQueryResult['pagination'] -} - -function activeDocumentConditions(knowledgeBaseId: string, access?: KnowledgeAccessScope) { - return [ - eq(document.knowledgeBaseId, knowledgeBaseId), - eq(document.enabled, true), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - access ? knowledgeAccessCondition(access) : undefined, - ] -} - -function validateReadInput(input: ReadIndexedKnowledgeDocumentInput) { - if ( - !Number.isInteger(input.limit) || - input.limit < 1 || - input.limit > 50 || - (input.offset !== undefined && - (!Number.isInteger(input.offset) || input.offset < 0 || input.offset > 1_000_000)) || - (input.aroundChunkIndex !== undefined && - (!Number.isInteger(input.aroundChunkIndex) || - input.aroundChunkIndex < 0 || - input.aroundChunkIndex > 1_000_000)) - ) { - throw new OrchestrationError('validation', 'Invalid document page bounds') - } - if (input.offset !== undefined && input.aroundChunkIndex !== undefined) { - throw new OrchestrationError('validation', 'Use offset or aroundChunkIndex, not both') - } - if ( - input.target.kind === 'url' && - (input.target.url.length > 8192 || !isKnowledgeSourceUrl(input.target.url.trim())) - ) { - throw new OrchestrationError('validation', 'url must be an HTTP or HTTPS document URL') - } -} - -/** Resolves only indexed references; URL lookup never contacts the provider or fetches content. */ -export const readIndexedKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ - operation: knowledgeOperations.readDocument, - resolveContext: ({ input }: { input: ReadIndexedKnowledgeDocumentInput }) => { - assertIndexedOrgSearchEnabled() - input.signal?.throwIfAborted() - validateReadInput(input) - return resolveKnowledgeOrganizationContext({ organizationId: input.organizationId }) - }, - async execute({ - principal, - input, - context, - request, - }): Promise { - input.signal?.throwIfAborted() - const assertions = { - assertedOrganizationId: context.organizationId, - } - const index = await findSearchIndex({ - kind: 'organization', - organizationId: context.organizationId, - }) - if (!index) throw new OrchestrationError('not_found', 'Document not found') - const knowledgeBaseId = index.id - const knowledgeContext = await resolveActiveKnowledgeResourceContext( - { knowledgeBaseId, ...assertions }, - principal - ) - let documentId: string - if (input.target.kind === 'id') { - documentId = input.target.documentId - } else { - const conditions = [ - ...activeDocumentConditions(knowledgeBaseId), - eq(document.sourceUrl, input.target.url.trim()), - ] - const matches: { id: string }[] = [] - for await (const accessCondition of knowledgeReadAccessBatches( - knowledgeContext.access, - conditions, - input.signal - )) { - matches.push( - ...(await db - .select({ id: document.id }) - .from(document) - .where(and(...conditions, accessCondition)) - .limit(2 - matches.length)) - ) - if (matches.length > 1) break - } - if (!matches.length) throw new OrchestrationError('not_found', 'Document not found') - if (matches.length > 1) { - throw new OrchestrationError( - 'validation', - 'Multiple accessible documents use this URL. Use documentId from search.' - ) - } - documentId = matches[0].id - } - const access = await knowledgeContext.access.getForDocuments([documentId], input.signal) - input.signal?.throwIfAborted() - const { document: doc } = await readKnowledgeDocument.execute({ - principal, - input: { knowledgeBaseId, documentId, ...assertions, requireEnabledDocument: true }, - request, - }) - const value: ReadIndexedKnowledgeDocumentResult = { - knowledgeBaseId, - documentId: doc.id, - title: doc.filename, - sourceUrl: doc.sourceUrl, - sourceModifiedAt: doc.sourceModifiedAt?.toISOString() ?? null, - connectorType: doc.connectorType, - processingStatus: doc.processingStatus, - } - if ( - !(await importKnowledgePersistedResponseSecretProvenance({ - registry: input.resultSecretRegistry, - documents: [{ id: doc.id, source: createKnowledgeDocumentSourceValue(doc), value }], - })) - ) { - throw new Error('Knowledge document provenance is unavailable') - } - input.signal?.throwIfAborted() - if (doc.processingStatus !== 'completed') return value - - let offset = input.offset ?? 0 - if (input.aroundChunkIndex !== undefined) { - const [position] = await db - .select({ - matched: - sql`count(*) filter (where ${embedding.chunkIndex} = ${input.aroundChunkIndex})`.mapWith( - Number - ), - preceding: - sql`count(*) filter (where ${embedding.chunkIndex} < ${input.aroundChunkIndex})`.mapWith( - Number - ), - }) - .from(embedding) - .innerJoin(document, eq(document.id, embedding.documentId)) - .where( - and( - ...activeDocumentConditions(knowledgeBaseId, access), - eq(document.id, documentId), - eq(embedding.enabled, true), - lte(embedding.chunkIndex, input.aroundChunkIndex) - ) - ) - if (!position?.matched) throw new OrchestrationError('not_found', 'Document chunk not found') - offset = Math.max(0, position.preceding - Math.min(2, input.limit - 1)) - } - input.signal?.throwIfAborted() - const page = await listKnowledgeChunks.execute({ - principal, - input: { - knowledgeBaseId, - documentId, - ...assertions, - requireEnabledDocument: true, - enabled: 'true', - sortBy: 'chunkIndex', - sortOrder: 'asc', - limit: input.limit, - offset, - }, - request, - }) - const chunks = page.chunks.map(({ id, chunkIndex, content }) => ({ id, chunkIndex, content })) - if ( - !(await importKnowledgePersistedResponseSecretProvenance({ - registry: input.resultSecretRegistry, - chunks: chunks.map((chunk) => ({ ...chunk, documentId, value: chunk })), - })) - ) { - throw new Error('Knowledge chunk provenance is unavailable') - } - input.signal?.throwIfAborted() - return { ...value, chunks, pagination: page.pagination } - }, -}) diff --git a/apps/sim/lib/sim-search/indexed/documents/read-search-document.test.ts b/apps/sim/lib/sim-search/indexed/documents/read-search-document.test.ts deleted file mode 100644 index ce6a0052598..00000000000 --- a/apps/sim/lib/sim-search/indexed/documents/read-search-document.test.ts +++ /dev/null @@ -1,267 +0,0 @@ -import { member } from '@sim/db/schema' -import { queueTableRows, resetDbChainMock } from '@sim/testing' -import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' -import { - knowledgeContextsMock, - knowledgeContextsMockFns, -} from '@sim/testing/mocks/knowledge-contexts.mock' -import { permissionGroupsResolveMock } from '@sim/testing/mocks/permission-groups-resolve.mock' -import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@/lib/knowledge/search/search-index', () => ({ - findSearchIndex: async () => ({ id: 'index' }), -})) -const mocks = vi.hoisted(() => ({ - chunks: vi.fn(), - provenance: vi.fn(), - importProvenance: vi.fn(), -})) -vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) -vi.mock('@/lib/permission-groups/resolve.server', () => permissionGroupsResolveMock) -vi.mock('@/lib/knowledge/application/contexts', () => knowledgeContextsMock) -vi.mock('@/lib/knowledge/chunks/service', () => ({ queryChunks: mocks.chunks })) -vi.mock('@/lib/knowledge/secret-provenance', () => ({ - importKnowledgeSearchResultSecretProvenance: mocks.provenance, -})) -vi.mock('@/lib/execution/durable-secret-provenance', () => ({ - importDurableSecretProvenance: mocks.importProvenance, -})) - -import { readSearchDocument } from '@/lib/sim-search/indexed/documents/read-search-document' -import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' - -workspaceAuthzMockFns.mockPermissionSatisfies.mockImplementation( - (actual: string | null) => actual !== null -) - -const principal = createSessionPrincipal({ userId: 'reader', sessionId: 'session' }) -const access = { kind: 'user', workspaceId: 'workspace', tokens: ['u:reader'] } as const -const context = { - workspaceId: 'workspace', - workspaceOrganizationId: null, - allowPersonalApiKeys: true, - billedAccountUserId: 'payer', - knowledgeBaseId: 'index', - knowledgeBase: { isSearchIndex: true }, - documentId: 'doc', - document: { - enabled: true, - processingStatus: 'completed', - filename: 'Title', - sourceUrl: 'https://source.test/doc', - }, - access: { get: async () => access }, -} -const input = { - documentId: 'doc', - assertedWorkspaceId: 'workspace', - limit: 3, - filters: { source: 'slack', documentIds: ['doc'] }, - resultSecretRegistry: new ResolvedSecretTraceRegistry([], { - userId: 'reader', - workspaceId: 'workspace', - }), -} - -/** These use cases run only while indexed organization search is on. */ -beforeEach(() => setEnvFlags({ isLiveEnterpriseSearchEnabled: false })) -afterEach(resetEnvFlagsMock) - -describe('Assistant document read', () => { - beforeEach(() => { - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue('read') - knowledgeContextsMockFns.mockResolveCanonicalActiveKnowledgeDocumentContext.mockResolvedValue( - context - ) - mocks.chunks.mockResolvedValue({ - chunks: [{ id: 'chunk', chunkIndex: 0, content: 'body' }], - pagination: { total: 1, hasMore: false }, - }) - mocks.provenance.mockResolvedValue({ - imported: true, - documentMetadata: { - doc: { - filename: 'Title', - sourceUrl: 'https://source.test/doc', - provenance: { status: 'exact', entries: [] }, - }, - }, - }) - mocks.importProvenance.mockResolvedValue(true) - }) - it('uses canonical authorization and filters enabled chunks by the same scope', async () => { - await expect(readSearchDocument.execute({ principal, input })).resolves.toMatchObject({ - documentId: 'doc', - chunks: [{ content: 'body', chunkIndex: 0 }], - next: null, - }) - expect( - knowledgeContextsMockFns.mockResolveCanonicalActiveKnowledgeDocumentContext - ).toHaveBeenCalledWith({ ...input, knowledgeBaseId: 'index' }, principal) - expect(mocks.chunks).toHaveBeenCalledWith( - 'doc', - expect.objectContaining({ - documentFilters: input.filters, - enabled: 'true', - limit: 3, - }), - expect.any(String), - access - ) - }) - it('rejects ordinary KBs and disabled documents', async () => { - knowledgeContextsMockFns.mockResolveCanonicalActiveKnowledgeDocumentContext.mockResolvedValueOnce( - { ...context, knowledgeBase: { isSearchIndex: false } } - ) - await expect(readSearchDocument.execute({ principal, input })).rejects.toThrow( - 'Document not found' - ) - knowledgeContextsMockFns.mockResolveCanonicalActiveKnowledgeDocumentContext.mockResolvedValueOnce( - { - ...context, - document: { ...context.document, enabled: false }, - } - ) - await expect(readSearchDocument.execute({ principal, input })).rejects.toThrow( - 'Document not found' - ) - expect(mocks.chunks).not.toHaveBeenCalled() - }) - it('fails closed on absent filtered documents or unverified provenance', async () => { - mocks.chunks.mockResolvedValueOnce({ chunks: [], pagination: { total: 0, hasMore: false } }) - await expect(readSearchDocument.execute({ principal, input })).rejects.toThrow( - 'Document not found' - ) - mocks.provenance.mockResolvedValueOnce({ imported: false }) - await expect(readSearchDocument.execute({ principal, input })).rejects.toThrow('provenance') - }) - it('rechecks the current workspace role', async () => { - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) - await expect(readSearchDocument.execute({ principal, input })).rejects.toThrow( - 'Insufficient workspace' - ) - expect(mocks.chunks).not.toHaveBeenCalled() - }) -}) - -describe('organization Search document reads', () => { - beforeEach(() => { - resetDbChainMock() - knowledgeContextsMockFns.mockResolveCanonicalActiveKnowledgeDocumentContext.mockResolvedValue({ - ...context, - workspaceId: undefined, - organizationId: 'org-1', - knowledgeBase: { isSearchIndex: true, organizationId: 'org-1' }, - }) - mocks.chunks.mockResolvedValue({ - chunks: [{ id: 'chunk', chunkIndex: 0, content: 'body' }], - pagination: { total: 1, hasMore: false }, - }) - mocks.provenance.mockResolvedValue({ - imported: true, - documentMetadata: { - doc: { - filename: 'Title', - sourceUrl: 'https://source.test/doc', - provenance: { status: 'exact', entries: [] }, - }, - }, - }) - mocks.importProvenance.mockResolvedValue(true) - }) - const orgInput = { ...input, assertedWorkspaceId: undefined, assertedOrganizationId: 'org-1' } - it('uses the current org member and existing ACL-filtered document read pipeline', async () => { - queueTableRows(member, [{ role: 'member' }]) - await expect(readSearchDocument.execute({ principal, input: orgInput })).resolves.toMatchObject( - { documentId: 'doc', chunks: [{ content: 'body' }] } - ) - expect(mocks.chunks).toHaveBeenCalledWith( - 'doc', - expect.objectContaining({ enabled: 'true', documentFilters: orgInput.filters }), - expect.any(String), - access - ) - expect(workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission).not.toHaveBeenCalled() - }) - it('refuses a removed member before reading document content or importing provenance', async () => { - queueTableRows(member, []) - await expect(readSearchDocument.execute({ principal, input: orgInput })).rejects.toThrow( - 'Organization not found' - ) - expect(mocks.chunks).not.toHaveBeenCalled() - expect(mocks.provenance).not.toHaveBeenCalled() - }) - it('rejects ambiguous workspace and organization scope before canonical lookup', async () => { - await expect( - readSearchDocument.execute({ - principal, - input: { ...orgInput, assertedWorkspaceId: 'workspace' }, - }) - ).rejects.toThrow('exactly one') - expect( - knowledgeContextsMockFns.mockResolveCanonicalActiveKnowledgeDocumentContext - ).not.toHaveBeenCalled() - expect(mocks.chunks).not.toHaveBeenCalled() - }) -}) - -describe('precise bounded passage expansion', () => { - beforeEach(() => { - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue('read') - knowledgeContextsMockFns.mockResolveCanonicalActiveKnowledgeDocumentContext.mockResolvedValue( - context - ) - mocks.provenance.mockResolvedValue({ imported: true, documentMetadata: {} }) - }) - - it('projects a secret spanning the page boundary before slicing it', async () => { - const secret = 'private-token-that-crosses-the-boundary' - const registry = new ResolvedSecretTraceRegistry([ - { name: 'TOKEN', plaintext: secret, encryptedValue: 'ciphertext' }, - ]) - mocks.provenance.mockImplementationOnce(async () => { - registry.recordResolved('TOKEN', secret) - return { imported: true, documentMetadata: {} } - }) - mocks.chunks.mockResolvedValue({ - chunks: [{ id: 'secret-chunk', chunkIndex: 0, content: `${'x'.repeat(7990) + secret}tail` }], - pagination: { total: 1, hasMore: false }, - }) - const result = await readSearchDocument.execute({ - principal, - input: { ...input, resultSecretRegistry: registry }, - }) - expect(result.chunks[0].content).toContain('{{TOKEN}}') - expect(JSON.stringify(result)).not.toContain('private-token') - }) - - it('rejects a continuation when all remaining chunks have disappeared', async () => { - mocks.chunks.mockResolvedValue({ - chunks: [], - pagination: { total: 2, hasMore: false }, - }) - await expect( - readSearchDocument.execute({ principal, input: { ...input, startChunkIndex: 7 } }) - ).rejects.toThrow('no longer available') - expect(mocks.provenance).not.toHaveBeenCalled() - }) - - it('rejects positions without an anchor and stale within-chunk continuation', async () => { - await expect( - readSearchDocument.execute({ principal, input: { ...input, startOffset: 2 } }) - ).rejects.toThrow('startOffset requires startChunkIndex') - expect(mocks.chunks).not.toHaveBeenCalled() - mocks.chunks.mockResolvedValue({ - chunks: [{ id: 'c8', chunkIndex: 8, content: 'replacement' }], - pagination: { total: 1, hasMore: false }, - }) - await expect( - readSearchDocument.execute({ - principal, - input: { ...input, startChunkIndex: 7, startOffset: 100 }, - }) - ).rejects.toThrow('no longer available') - }) -}) diff --git a/apps/sim/lib/sim-search/indexed/documents/read-search-document.ts b/apps/sim/lib/sim-search/indexed/documents/read-search-document.ts deleted file mode 100644 index 309754a0482..00000000000 --- a/apps/sim/lib/sim-search/indexed/documents/read-search-document.ts +++ /dev/null @@ -1,173 +0,0 @@ -import type { Principal } from '@sim/auth/principal' -import type { ReadSearchDocumentResult } from '@/lib/api/contracts/knowledge/documents' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { generateRequestId } from '@/lib/core/utils/request' -import { importDurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' -import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' -import { KnowledgeDocumentNotReadyError } from '@/lib/knowledge/application/chunk-errors' -import { resolveCanonicalActiveKnowledgeDocumentContext } from '@/lib/knowledge/application/contexts' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { queryChunks } from '@/lib/knowledge/chunks/service' -import { measureSearchStage } from '@/lib/knowledge/search/diagnostics' -import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters' -import { findSearchIndex } from '@/lib/knowledge/search/search-index' -import { passageWindow } from '@/lib/knowledge/search/snippet' -import { importKnowledgeSearchResultSecretProvenance } from '@/lib/knowledge/secret-provenance' -import { assertIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' -import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' -import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' - -export interface ReadSearchDocumentInput { - documentId: string - assertedWorkspaceId?: string - assertedOrganizationId?: string - filters?: WorkspaceSearchFilters - limit: number - startChunkIndex?: number - startOffset?: number - resultSecretRegistry: ResolvedSecretTraceRegistry - signal?: AbortSignal -} - -/** Bounds model text to at most 24KB of UTF-8, with continuation even inside a large chunk. */ -const READ_PAGE_CHARACTERS = 8000 -const READ_PAGE_CHUNKS = 8 -const STALE_POSITION_MESSAGE = - 'The passage position is no longer available; search again or read from the chunk start' - -/** Reads enabled indexed passages with the same document scope and ACLs as search. */ -export const readSearchDocument = defineAuthorizedKnowledgeUseCase({ - operation: knowledgeOperations.readDocument, - resolveContext: async ({ - principal, - input, - }: { - principal: Principal - input: ReadSearchDocumentInput - }) => { - assertIndexedOrgSearchEnabled() - if (Boolean(input.assertedWorkspaceId) === Boolean(input.assertedOrganizationId)) - throw new OrchestrationError('validation', 'Document reads require exactly one search owner') - const index = await findSearchIndex( - input.assertedOrganizationId - ? { kind: 'organization', organizationId: input.assertedOrganizationId } - : { kind: 'workspace', workspaceId: input.assertedWorkspaceId! } - ) - if (!index) throw new OrchestrationError('not_found', 'Document not found') - return resolveCanonicalActiveKnowledgeDocumentContext( - { ...input, knowledgeBaseId: index.id }, - principal - ) - }, - async execute({ input, context }): Promise { - input.signal?.throwIfAborted() - if ( - !Number.isInteger(input.limit) || - input.limit < 1 || - input.limit > READ_PAGE_CHUNKS || - (input.startChunkIndex !== undefined && - (!Number.isSafeInteger(input.startChunkIndex) || - input.startChunkIndex < 0 || - input.startChunkIndex > 2147483647)) || - (input.startOffset !== undefined && - (!Number.isSafeInteger(input.startOffset) || - input.startOffset < 0 || - input.startOffset > 2147483647 || - input.startChunkIndex === undefined)) - ) { - throw new OrchestrationError( - 'validation', - 'Document reads require limit 1–8 and nonnegative chunk positions; startOffset requires startChunkIndex' - ) - } - if (context.document.processingStatus !== 'completed') { - throw new KnowledgeDocumentNotReadyError(context.document.processingStatus) - } - if (!context.knowledgeBase.isSearchIndex) - throw new OrchestrationError('not_found', 'Document not found') - if (!context.document.enabled) throw new OrchestrationError('not_found', 'Document not found') - const access = await measureSearchStage('access_scope', () => context.access.get()) - const page = await measureSearchStage('document_read.sql', () => - queryChunks( - context.documentId, - { - limit: input.limit, - startChunkIndex: input.startChunkIndex, - requireEnabledDocument: true, - enabled: 'true', - sortBy: 'chunkIndex', - sortOrder: 'asc', - documentFilters: input.filters, - }, - generateRequestId(), - access - ) - ) - if (page.pagination.total === 0) throw new OrchestrationError('not_found', 'Document not found') - if (input.startChunkIndex !== undefined && page.chunks.length === 0) { - throw new OrchestrationError('validation', STALE_POSITION_MESSAGE) - } - const provenance = await measureSearchStage('result_provenance', () => - importKnowledgeSearchResultSecretProvenance({ - registry: input.resultSecretRegistry, - results: page.chunks.map((chunk) => ({ ...chunk, documentId: context.documentId })), - }) - ) - if (!provenance.imported) throw new Error('Knowledge result provenance is unavailable') - const metadata = provenance.documentMetadata[context.documentId] - if ( - metadata && - !(await importDurableSecretProvenance(input.resultSecretRegistry, metadata.provenance, { - documentName: metadata.filename, - sourceUrl: metadata.sourceUrl, - })) - ) { - throw new Error('Knowledge document provenance is unavailable') - } - /** Project complete strings before slicing, so a window cannot expose part of a secret. */ - const projectedChunks = page.chunks.map((chunk) => { - const projected = projectResolvedSecretModelContent(chunk.content, input.resultSecretRegistry) - if (!projected.safe || typeof projected.value !== 'string') - throw new Error('Knowledge result provenance is unavailable') - return { ...chunk, content: projected.value } - }) - if ( - input.startOffset && - (projectedChunks[0]?.chunkIndex !== input.startChunkIndex || - input.startOffset >= projectedChunks[0].content.length) - ) { - throw new OrchestrationError('validation', STALE_POSITION_MESSAGE) - } - let remaining = READ_PAGE_CHARACTERS - const chunks: ReadSearchDocumentResult['chunks'] = [] - let next: ReadSearchDocumentResult['next'] = null - for (const chunk of projectedChunks) { - if (remaining < 2) { - next = { startChunkIndex: chunk.chunkIndex, startOffset: 0 } - break - } - const start = chunk.chunkIndex === input.startChunkIndex ? (input.startOffset ?? 0) : 0 - const excerpt = passageWindow(chunk.content, start, remaining) - chunks.push({ chunkIndex: chunk.chunkIndex, ...excerpt }) - remaining -= excerpt.content.length - if (excerpt.endOffset < chunk.content.length) { - next = { startChunkIndex: chunk.chunkIndex, startOffset: excerpt.endOffset } - break - } - } - const last = chunks.at(-1) - if (!next && page.pagination.hasMore && last) { - next = { startChunkIndex: last.chunkIndex + 1, startOffset: 0 } - } - input.signal?.throwIfAborted() - return { - documentId: context.documentId, - knowledgeBaseId: context.knowledgeBaseId, - documentName: metadata?.filename ?? null, - sourceUrl: metadata?.sourceUrl ?? null, - chunks, - hasMore: next !== null, - next, - } - }, -}) diff --git a/apps/sim/lib/sim-search/indexed/gate.ts b/apps/sim/lib/sim-search/indexed/gate.ts deleted file mode 100644 index 9b5771ad870..00000000000 --- a/apps/sim/lib/sim-search/indexed/gate.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { isLiveEnterpriseSearchEnabled } from '@/lib/core/config/env-flags' - -/** - * The single switch for indexed organization search: retrieval over `is_search_index` knowledge - * bases and the crawling that fills them. It is the inverse of the Live Search backend selector - * (`SIM_SEARCH_LIVE`, on by default), so indexed search is dormant unless a deployment sets - * `SIM_SEARCH_LIVE=false`. The selector is read once at startup, so the answer is constant for the - * life of the process. - */ -export function isIndexedOrgSearchEnabled(): boolean { - return !isLiveEnterpriseSearchEnabled -} - -/** An indexed-only surface was reached while indexed organization search is dormant. */ -export class SearchIndexDormantError extends Error { - constructor() { - super('This search index is inactive; use Sim Search.') - this.name = 'SearchIndexDormantError' - } -} - -/** - * Refuses entry to dormant indexed organization search. Every indexed use case and entry calls it - * itself, so dormancy holds even for a caller that forgot to ask the gate. - */ -export function assertIndexedOrgSearchEnabled(): void { - if (!isIndexedOrgSearchEnabled()) throw new SearchIndexDormantError() -} - -/** - * Whether a search runs the search-index retrieval legs: indexed organization search is on and - * every knowledge base it names is a search index. Every other search decides readability on - * each candidate's document. - */ -export function usesIndexedRetrieval( - knowledgeBases: ReadonlyArray<{ isSearchIndex?: boolean | null }> -): boolean { - return ( - isIndexedOrgSearchEnabled() && - knowledgeBases.every((knowledgeBase) => knowledgeBase.isSearchIndex === true) - ) -} diff --git a/apps/sim/lib/sim-search/indexed/index.ts b/apps/sim/lib/sim-search/indexed/index.ts deleted file mode 100644 index 65d2f4e08a9..00000000000 --- a/apps/sim/lib/sim-search/indexed/index.ts +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Dormant indexed organization search: the use cases that search and read `is_search_index` - * knowledge bases, and the indexed arms of the personal Search integration inventory. Callers - * check `isIndexedOrgSearchEnabled()` before reaching these, and each refuses on its own while the - * gate is off; see this directory's README. - */ - -export { readIndexedKnowledgeDocument } from '@/lib/sim-search/indexed/documents/read-indexed-document' -export { readSearchDocument } from '@/lib/sim-search/indexed/documents/read-search-document' -export { ownsIndexedPersonalSearchAccount } from '@/lib/sim-search/indexed/integrations/personal-account-ownership' -export { loadIndexedSearchIntegrationInventory } from '@/lib/sim-search/indexed/integrations/personal-inventory' -export { listIndexedPersonalSearchIntegrations } from '@/lib/sim-search/indexed/integrations/personal-search-integrations' -export { registerIndexedKnowledgeMcpTools } from '@/lib/sim-search/indexed/mcp/register-tools' -export { - searchOrganizationKnowledge, - searchScopedKnowledge, - searchWorkspaceKnowledge, -} from '@/lib/sim-search/indexed/search/scoped-search' diff --git a/apps/sim/lib/sim-search/indexed/integrations/personal-account-ownership.ts b/apps/sim/lib/sim-search/indexed/integrations/personal-account-ownership.ts deleted file mode 100644 index 561e7cbb039..00000000000 --- a/apps/sim/lib/sim-search/indexed/integrations/personal-account-ownership.ts +++ /dev/null @@ -1,29 +0,0 @@ -import type { Principal } from '@sim/auth/principal' -import { personalSearchIntegrationPages } from '@/lib/knowledge/application/personal-search-integration-pages' -import { assertIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' - -/** - * The indexed arm of organization personal-token ownership: whether the viewer's personal Search - * inventory lists `credentialId` as a connected account on this connector type. Indexed search - * answers ownership from its per-source inventory, where Live Search reads the live accounts. - */ -export async function ownsIndexedPersonalSearchAccount( - principal: Principal, - input: { organizationId: string; connectorType: string; credentialId: string } -): Promise { - assertIndexedOrgSearchEnabled() - for await (const page of personalSearchIntegrationPages({ - principal, - input: { organizationId: input.organizationId, connectorType: input.connectorType }, - })) { - if ( - page.connections.some((connection) => - connection.accounts.some( - (account) => account.credentialId === input.credentialId && account.status === 'connected' - ) - ) - ) - return true - } - return false -} diff --git a/apps/sim/lib/sim-search/indexed/integrations/personal-inventory.ts b/apps/sim/lib/sim-search/indexed/integrations/personal-inventory.ts deleted file mode 100644 index 6b7309b7055..00000000000 --- a/apps/sim/lib/sim-search/indexed/integrations/personal-inventory.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { Principal } from '@sim/auth/principal' -import { - type PersonalSearchIntegrationsPage, - personalSearchIntegrationPages, -} from '@/lib/knowledge/application/personal-search-integration-pages' -import { assertIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' - -/** - * The indexed arm of Sim's Search inventory for one chat turn: every page of the viewer's - * personal connections on the indexed sources, merged and serialized for the prompt. Indexed - * inventory pages by source, where Live Search returns a single page. - */ -export async function loadIndexedSearchIntegrationInventory({ - principal, - organizationId, - signal, - maxBytes, -}: { - principal: Principal - organizationId: string - signal?: AbortSignal - maxBytes: number -}): Promise { - assertIndexedOrgSearchEnabled() - const connections: Array = [] - const available = new Map() - let inventory = JSON.stringify({ connections, available: [] }) - for await (const page of personalSearchIntegrationPages({ - principal, - input: { organizationId }, - signal, - })) { - connections.push(...page.connections) - for (const entry of page.available) { - available.set(JSON.stringify(entry.target), entry) - } - inventory = JSON.stringify({ connections, available: [...available.values()] }) - if (Buffer.byteLength(inventory) > maxBytes) { - throw new Error('Search integration inventory exceeds the prompt size limit') - } - } - return inventory -} diff --git a/apps/sim/lib/sim-search/indexed/integrations/personal-search-integrations.ts b/apps/sim/lib/sim-search/indexed/integrations/personal-search-integrations.ts deleted file mode 100644 index 8762e38f8e9..00000000000 --- a/apps/sim/lib/sim-search/indexed/integrations/personal-search-integrations.ts +++ /dev/null @@ -1,164 +0,0 @@ -import type { Principal } from '@sim/auth/principal' -import { readSearchConnectionCompletion } from '@/lib/credential-groups/search-connection-completion' -import { - getIntegrationAvailability, - isOAuthServiceDeploymentAvailable, -} from '@/lib/integrations/availability.server' -import { resolveKnowledgeAccessAvailability } from '@/lib/knowledge/access/availability' -import type { KnowledgeOrganizationContext } from '@/lib/knowledge/application/contexts' -import type { ListPersonalSearchIntegrationsInput } from '@/lib/knowledge/application/personal-search-integrations' -import { listConfiguredSearchProviderTypes } from '@/lib/knowledge/application/search-source-overview' -import { listSearchSources } from '@/lib/knowledge/application/search-sources' -import type { SearchConnectionTarget } from '@/lib/knowledge/search/connection-target' -import { listOrganizationSearchApprovals } from '@/lib/knowledge/search/integration-policy' -import { getConnectorAccessAvailability, SEARCH_CONNECTORS } from '@/lib/sim-search/connectors' -import { assertIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' -import { findSharedSlackSearchInstallation } from '@/lib/slack-search/shared-app' - -/** - * The indexed arm of `listPersonalSearchIntegrations`: the viewer's accounts on the connectors - * that crawl the organization search index, with each source's indexing state. The caller has - * authorized the principal and loaded the viewer; it reaches this only while - * `isIndexedOrgSearchEnabled()` is on. - */ -export async function listIndexedPersonalSearchIntegrations({ - principal, - input, - context, - userId, - viewer, -}: { - principal: Principal - input: ListPersonalSearchIntegrationsInput - context: KnowledgeOrganizationContext - userId: string - viewer: { emailVerified: boolean } -}) { - assertIndexedOrgSearchEnabled() - const [page, configuredTypes, approvals, access, sharedSlack] = await Promise.all([ - listSearchSources.execute({ principal, input }), - listConfiguredSearchProviderTypes({ organizationId: context.organizationId }), - listOrganizationSearchApprovals(context.organizationId), - resolveKnowledgeAccessAvailability(context), - findSharedSlackSearchInstallation(context.organizationId), - ]) - const deployment = new Map( - getIntegrationAvailability().map((entry) => [entry.type.toLowerCase(), entry]) - ) - const oauth = new Map( - SEARCH_CONNECTORS.map((entry) => [ - entry.providerId, - isOAuthServiceDeploymentAvailable(entry.providerId), - ]) - ) - const configured = new Set(configuredTypes) - const eligible = (connectorType: string) => { - const connector = SEARCH_CONNECTORS.find((entry) => entry.type === connectorType) - return Boolean( - viewer.emailVerified && - connector && - approvals.get(connectorType) && - getConnectorAccessAvailability(connector.meta, deployment, { - memberAccessAvailable: access.memberScoped, - mirroredAccessAvailable: access.sourceMirrored, - oauthServiceAvailability: oauth, - isIntegrationAvailabilityReady: true, - }).members - ) - } - const projected = page.sources.flatMap((source) => { - const connector = SEARCH_CONNECTORS.find((entry) => entry.type === source.connectorType) - if (!connector) return [] - const target: SearchConnectionTarget = { - type: 'link', - provider: connector.providerId, - connectorType: source.connectorType, - connectorId: source.connectorId, - } - const canConnect = - eligible(source.connectorType) && - source.enabled && - source.availability === 'available' && - source.viewerEmailVerified && - source.connectionRequired && - source.viewerMembership !== null && - !['revoked', 'unverified_email'].includes(source.viewerMembership) - const accounts = source.viewerAccounts.map((account) => { - if (!account.status) throw new Error('Personal Search account status is missing') - return { - credentialId: account.credentialId, - displayName: account.displayName, - status: - account.status === 'active' ? ('connected' as const) : ('reconnect_needed' as const), - action: - canConnect && account.status === 'needs_reauth' - ? { ...target, credentialId: account.credentialId } - : null, - } - }) - return [ - { - name: connector.meta.name, - providerId: connector.providerId, - connectorType: connector.type, - connectorId: source.connectorId, - knowledgeBaseId: source.knowledgeBaseId, - description: source.sourceDescription, - accounts, - connectionStatus: accounts.some((account) => account.status === 'reconnect_needed') - ? ('reconnect_needed' as const) - : accounts.length - ? ('connected' as const) - : canConnect - ? ('not_connected' as const) - : ('unavailable' as const), - indexingStatus: - !source.enabled || source.availability !== 'available' || source.approved === false - ? ('paused' as const) - : source.isSyncing - ? ('indexing' as const) - : source.hasSyncError || source.viewerFailedDocumentCount > 0 - ? ('sync_failed' as const) - : source.hasViewerDocuments - ? ('indexed' as const) - : ('not_indexed' as const), - action: canConnect && !accounts.length ? target : null, - }, - ] - }) - const available: Array<{ name: string; description: string; target: SearchConnectionTarget }> = [ - ...projected.flatMap((entry) => - entry.action - ? [{ name: entry.name, description: entry.description, target: entry.action }] - : [] - ), - ...SEARCH_CONNECTORS.filter( - (connector) => - !input.connectorId && - (!input.connectorType || connector.type === input.connectorType) && - (connector.type !== 'slack' || sharedSlack !== null) && - (!configured.has(connector.type) || connector.setupFields.length > 0) && - eligible(connector.type) - ).map((connector) => ({ - name: connector.meta.name, - description: '', - target: { - type: 'link' as const, - provider: connector.providerId, - connectorType: connector.type, - }, - })), - ] - return { - completedCredentialId: input.completionId - ? await readSearchConnectionCompletion({ - organizationId: context.organizationId, - userId, - completionId: input.completionId, - }) - : null, - connections: projected.filter((entry) => entry.accounts.length > 0), - available, - nextCursor: page.nextCursor, - } -} diff --git a/apps/sim/lib/sim-search/indexed/mcp/register-tools.ts b/apps/sim/lib/sim-search/indexed/mcp/register-tools.ts deleted file mode 100644 index 174defef89b..00000000000 --- a/apps/sim/lib/sim-search/indexed/mcp/register-tools.ts +++ /dev/null @@ -1,148 +0,0 @@ -import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' -import type { Principal } from '@sim/auth/principal' -import type { NextRequest } from 'next/server' -import { readDocumentMcpSchema, searchMcpSchema } from '@/lib/api/contracts/knowledge/mcp' -import type { ResourceScope } from '@/lib/core/resource-scope' -import { getBaseUrl } from '@/lib/core/utils/urls' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { searchKnowledge } from '@/lib/knowledge/application/search' -import { - KNOWLEDGE_MCP_READ_ONLY, - type KnowledgeMcpToolRunner, - projectResult, -} from '@/lib/knowledge/mcp/tool-runner' -import { createKnowledgeDocumentCitation } from '@/lib/knowledge/search/citation' -import { toolError } from '@/lib/mcp/tool-result' -import { readIndexedKnowledgeDocument } from '@/lib/sim-search/indexed/documents/read-indexed-document' -import { assertIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' - -interface IndexedKnowledgeMcpToolsContext { - server: McpServer - principal: Principal - request: NextRequest - organizationId: string - /** The organization's search index, or null when no source is connected yet. */ - searchIndexId: string | null - execute: KnowledgeMcpToolRunner -} - -/** - * Registers the indexed `search` and `read_document` Search MCP tools: passages ranked from the - * organization's search index, and indexed documents read by id or source URL. Called only while - * indexed organization search is on, and refuses otherwise; both tools run through use cases that - * refuse a dormant search index on their own. - */ -export function registerIndexedKnowledgeMcpTools(context: IndexedKnowledgeMcpToolsContext): void { - assertIndexedOrgSearchEnabled() - const { server, principal, request, organizationId, searchIndexId, execute } = context - const scope: ResourceScope = { kind: 'organization', organizationId } - - server.registerTool( - 'search', - { - title: 'Search', - description: - 'Search accessible passages in this organization’s Search index. Use source (for example, jira), modifiedAfter (an ISO timestamp), or documentIds to narrow results. Results are candidates; score is similarity, not answer confidence. Use read_document for context and cite citationUrl.', - inputSchema: searchMcpSchema, - annotations: KNOWLEDGE_MCP_READ_ONLY, - }, - async (input: unknown, extra: { signal: AbortSignal }) => - execute('search', knowledgeOperations.search, extra.signal, async (registry, signal) => { - const { query, topK, ...filters } = searchMcpSchema.parse(input) - if (!searchIndexId) { - return projectResult( - { - results: [], - message: 'No Search index is configured. Ask an admin to connect a source.', - }, - registry - ) - } - const result = await searchKnowledge.execute({ - principal, - input: { - organizationId, - knowledgeBaseIds: [searchIndexId], - query, - topK, - filters, - resultSecretRegistry: registry, - surface: 'mcp', - signal, - }, - request, - }) - return projectResult( - { - results: result.results.map((row) => ({ - documentId: row.documentId, - title: row.documentName, - sourceUrl: row.sourceUrl, - ...createKnowledgeDocumentCitation({ - scope, - knowledgeBaseId: row.knowledgeBaseId, - documentId: row.documentId, - sourceUrl: row.sourceUrl, - baseUrl: getBaseUrl(), - }), - sourceModifiedAt: row.sourceModifiedAt?.toISOString() ?? null, - connectorType: row.connectorType, - content: row.content, - chunkIndex: row.chunkIndex, - score: row.similarity, - })), - }, - result.resultSecretRegistry ?? registry - ) - }) - ) - server.registerTool( - 'read_document', - { - title: 'Read document', - description: - 'Read an indexed document by documentId from search or its original URL. URLs must match an accessible indexed source; this tool does not browse the web. Set aroundChunkIndex to a search hit’s chunkIndex for nearby context, or use offset for sequential pages. When pagination.hasMore is true, continue with pagination.offset + pagination.limit. Cite citationUrl. Documents still indexing return metadata only.', - inputSchema: readDocumentMcpSchema, - annotations: KNOWLEDGE_MCP_READ_ONLY, - }, - async (raw: unknown, extra: { signal: AbortSignal }) => - execute( - 'read_document', - knowledgeOperations.readDocument, - extra.signal, - async (registry, signal) => { - const input = readDocumentMcpSchema.parse(raw) - if (!input.url && !input.documentId) return toolError('Document not found') - const result = await readIndexedKnowledgeDocument.execute({ - principal, - input: { - organizationId, - target: input.url - ? { kind: 'url', url: input.url } - : { kind: 'id', documentId: input.documentId! }, - limit: input.limit, - offset: input.offset, - aroundChunkIndex: input.aroundChunkIndex, - resultSecretRegistry: registry, - signal, - }, - request, - }) - const { knowledgeBaseId, ...document } = result - return projectResult( - { - ...document, - ...createKnowledgeDocumentCitation({ - scope, - knowledgeBaseId, - documentId: result.documentId, - sourceUrl: result.sourceUrl, - baseUrl: getBaseUrl(), - }), - }, - registry - ) - } - ) - ) -} diff --git a/apps/sim/lib/sim-search/indexed/retrieval/access-plan.ts b/apps/sim/lib/sim-search/indexed/retrieval/access-plan.ts deleted file mode 100644 index eba0bc5aa4f..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/access-plan.ts +++ /dev/null @@ -1,188 +0,0 @@ -import { db } from '@sim/db' -import { knowledgeConnector, knowledgeConnectorMember } from '@sim/db/schema' -import { and, eq, inArray, isNull, sql } from 'drizzle-orm' -import { SOURCE_ACL_MAX_AGE_MS } from '@/lib/knowledge/access/freshness' -import { textArrayLiteral } from '@/lib/knowledge/access/predicate' -import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' -import { searchIntegrationAccessCondition } from '@/lib/knowledge/search/integration-policy' - -/** - * The connectors a search may read from, resolved once per query: their ids grouped by the shape - * their documents' ACLs take, and separately those whose reader access is proven live per request. - */ -export interface KnowledgeConnectorEligibility { - /** Documents carry the workspace ACL. */ - workspace: readonly string[] - /** Documents carry mirrored source permissions verified as a whole. */ - admin: readonly string[] - /** Documents carry the subject tokens of the members who observe them. */ - members: readonly string[] - /** Of the above, those that additionally require this request's live source proof. */ - liveProofRequired: readonly string[] -} - -/** One of the caller's member identities and the connector it belongs to. */ -export interface KnowledgeMemberObserver { - id: string - connectorId: string -} - -/** - * The caller's active member identities on the connectors a search reads, by what makes their - * observations current: `confirmed` members drained their change feed inside the freshness window, - * so every observation they hold stands; `observed` members are trusted only where the observation - * itself is recent. - */ -export interface KnowledgeMemberObservers { - confirmed: readonly KnowledgeMemberObserver[] - observed: readonly KnowledgeMemberObserver[] -} - -/** What a search resolves once about its sources and the caller's standing in them. */ -export interface SearchAccessPlan { - connectors: KnowledgeConnectorEligibility - observers: KnowledgeMemberObservers - /** Connectors the caller is an active member of, whose documents they read broadly. */ - memberSources: readonly string[] - /** Each eligible connector's type, so a search may be confined to one kind of source. */ - connectorTypes: ReadonlyMap - /** Whether documents without a source — uploads — are in scope. */ - uploads: boolean -} - -/** - * The plan confined to one kind of source: the connectors of that type keep their eligibility and - * the rest lose it, so every predicate built from the plan — on the row and on the document — and - * every source the legs walk or rank are that kind alone. `upload` keeps only source-less documents. - */ -export function restrictSearchAccessPlan(plan: SearchAccessPlan, source: string): SearchAccessPlan { - const keep = (id: string) => source !== 'upload' && plan.connectorTypes.get(id) === source - const kept = (ids: readonly string[]) => ids.filter(keep) - return { - connectors: { - workspace: kept(plan.connectors.workspace), - admin: kept(plan.connectors.admin), - members: kept(plan.connectors.members), - liveProofRequired: kept(plan.connectors.liveProofRequired), - }, - observers: { - confirmed: plan.observers.confirmed.filter((observer) => keep(observer.connectorId)), - observed: plan.observers.observed.filter((observer) => keep(observer.connectorId)), - }, - memberSources: kept(plan.memberSources), - connectorTypes: plan.connectorTypes, - uploads: source === 'upload', - } -} - -/** - * The connectors a search may read from, grouped by access mode, with the ones whose reader access - * must be proven live marked. - * - * Deletion, archival, a pending access rewrite and the organization's integration approval are - * facts about a connector. Resolving them once per query — there are tens of connectors against - * hundreds of thousands of documents — leaves each candidate its own columns to check. - */ -async function resolveConnectorEligibility( - knowledgeBaseIds: readonly string[] -): Promise<{ eligibility: KnowledgeConnectorEligibility; types: Map }> { - const eligibility: { - workspace: string[] - admin: string[] - members: string[] - liveProofRequired: string[] - } = { workspace: [], admin: [], members: [], liveProofRequired: [] } - const types = new Map() - if (knowledgeBaseIds.length === 0) return { eligibility, types } - const rows = await db - .select({ - id: knowledgeConnector.id, - accessMode: knowledgeConnector.accessMode, - connectorType: knowledgeConnector.connectorType, - /** A GitHub connector is gated only where it names the immutable repository behind a grant. */ - githubRepository: sql`${knowledgeConnector.sourceConfig}::jsonb ? 'githubRepositoryId'`, - }) - .from(knowledgeConnector) - .where( - and( - inArray(knowledgeConnector.knowledgeBaseId, [...knowledgeBaseIds]), - isNull(knowledgeConnector.deletedAt), - isNull(knowledgeConnector.archivedAt), - eq(knowledgeConnector.accessRewritePending, false), - searchIntegrationAccessCondition() - ) - ) - for (const row of rows) { - if (row.accessMode === 'workspace') eligibility.workspace.push(row.id) - else if (row.accessMode === 'admin') eligibility.admin.push(row.id) - else if (row.accessMode === 'members') eligibility.members.push(row.id) - else continue - types.set(row.id, row.connectorType) - const live = - (row.connectorType === 'github' && row.githubRepository) || - (row.connectorType === 'confluence' && row.accessMode === 'admin') - if (live) eligibility.liveProofRequired.push(row.id) - } - return { eligibility, types } -} - -/** - * The caller's own member identities on these connectors, split by whether the member's change - * feed is itself current. - * - * A members-mode document is readable while one of the caller's active members observes it, - * freshly — and which members those are is a fact about the caller, not about any document. A - * member whose feed drained recently confirms every observation it holds, so its observations need - * no age check at all; the rest are checked against the age of the observation itself. Resolved - * once, the per-document check becomes one lookup on the observation key, with no join to the - * member behind it. - */ -async function resolveMemberObservers( - access: KnowledgeAccessScope, - connectorIds: readonly string[] -): Promise<{ observers: KnowledgeMemberObservers; memberSources: string[] }> { - if (access.kind !== 'user' || connectorIds.length === 0 || access.tokens.length === 0) { - return { observers: { confirmed: [], observed: [] }, memberSources: [] } - } - const rows = await db - .select({ - id: knowledgeConnectorMember.id, - connectorId: knowledgeConnectorMember.connectorId, - syncedThrough: knowledgeConnectorMember.memberSyncedThrough, - }) - .from(knowledgeConnectorMember) - .where( - and( - inArray(knowledgeConnectorMember.connectorId, [...connectorIds]), - eq(knowledgeConnectorMember.status, 'active'), - sql`${knowledgeConnectorMember.subjectToken} = ANY(${textArrayLiteral([...access.tokens])})` - ) - ) - const cutoff = Date.now() - SOURCE_ACL_MAX_AGE_MS - const confirmed: KnowledgeMemberObserver[] = [] - const observed: KnowledgeMemberObserver[] = [] - const memberSources = new Set() - for (const row of rows) { - const member = { id: row.id, connectorId: row.connectorId } - if (row.syncedThrough !== null && row.syncedThrough.getTime() > cutoff) confirmed.push(member) - else observed.push(member) - memberSources.add(row.connectorId) - } - return { observers: { confirmed, observed }, memberSources: [...memberSources] } -} - -/** - * Everything a search needs to know about its sources and the caller's standing in them, resolved - * once: which connectors it may read, the caller's member identities there, and the sources they - * are a member of. Each is a fact about a connector or a caller, so deriving them per candidate - * document is what made retrieval cost grow with the size of what someone may read. - */ -export async function resolveSearchAccessPlan( - knowledgeBaseIds: readonly string[], - access: KnowledgeAccessScope -): Promise { - const { eligibility: connectors, types: connectorTypes } = - await resolveConnectorEligibility(knowledgeBaseIds) - const { observers, memberSources } = await resolveMemberObservers(access, connectors.members) - return { connectors, observers, memberSources, connectorTypes, uploads: true } -} diff --git a/apps/sim/lib/sim-search/indexed/retrieval/index.ts b/apps/sim/lib/sim-search/indexed/retrieval/index.ts deleted file mode 100644 index 79691075482..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -/** - * The search-index retrieval legs shared retrieval (`lib/knowledge/search/queries.ts`) runs for a - * user-scoped search over search indexes while indexed organization search is on. Everything that - * decides readability on the projection row lives behind this barrel: the resolved access plan - * and the projection-row predicates built from it, reach and permitted sets, per-source vector - * walks, the projection-fill probe, live source proof, and keyword ranking over the GIN and Tin - * projections. Kept apart from the use-case barrel because the use cases depend on that retrieval - * layer, which depends on these. - */ -export { prepareIndexedRetrieval } from '@/lib/sim-search/indexed/retrieval/legs' diff --git a/apps/sim/lib/sim-search/indexed/retrieval/keyword.ts b/apps/sim/lib/sim-search/indexed/retrieval/keyword.ts deleted file mode 100644 index c0016e77518..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/keyword.ts +++ /dev/null @@ -1,325 +0,0 @@ -import { document, embedding, embeddingKeywordSearch, embeddingKeywordTin } from '@sim/db/schema' -import { and, eq, inArray, type SQL, sql } from 'drizzle-orm' -import { knowledgeAccessCondition, textArrayLiteral } from '@/lib/knowledge/access/predicate' -import { runSearchQuery } from '@/lib/knowledge/search/budget' -import { - candidateDocumentConditions, - excludeSearchSources, - FTS_CONFIG, - hydrateSearchCandidates, - type KeywordSearchParams, - type SearchReadCandidate, - type SearchReadCandidatePage, - type SearchResult, - selectAuthorizedSearchResults, -} from '@/lib/knowledge/search/candidates' -import { annotateSearchDiagnostics } from '@/lib/knowledge/search/diagnostics' -import { searchDateFilterCondition } from '@/lib/knowledge/search/filter-conditions' -import { keywordCandidateRankingQuery } from '@/lib/knowledge/search/keyword-ranking' -import { getStructuredTagFilters } from '@/lib/knowledge/search/tag-filters' -import { embeddingDistance } from '@/lib/knowledge/vector-columns' -import { - documentSatisfies, - type IndexedRetrievalContext, - PERMITTED_EXACT_DOCUMENT_LIMIT, -} from '@/lib/sim-search/indexed/retrieval/permitted' -import { - excludeSearchSourcesOnRow, - knowledgeCandidateAccessConditionForConnectors, - projectionCandidateAccessCondition, - projectionDecidedOnDocument, -} from '@/lib/sim-search/indexed/retrieval/projection-access' -import { isProjectionFilled } from '@/lib/sim-search/indexed/retrieval/projection-fill' -import { resolveTinKeywordQuery } from '@/lib/sim-search/indexed/retrieval/tin-keyword' - -/** - * Chunks Tin ranks before access is checked, widening while too few are readable to fill a page. - * A caller past the permitted-set limit reads a large share of the index, so the first window - * almost always fills; the widest bounds the work before the GIN ranking takes over. - */ -const TIN_KEYWORD_WINDOWS = [2000, 10_000, 50_000] as const - -/** Readable rows one wide window returns for a narrow reader: several pages' worth, ranked once. */ -const NARROW_KEYWORD_PAGE = 1000 - -/** - * The widest window a narrow reader ranks: wide enough that a few percent of it fills their page - * several times over, and less than half the cost of the widest window the broad readers reach. - * It is tried only after the first window came back short: ranking costs grow with the window, - * and a term that is common where the reader can read fills the page from the narrowest one. - */ -const NARROW_KEYWORD_WINDOWS = [TIN_KEYWORD_WINDOWS[0], 20_000] as const - -/** - * The keyword leg of a user-scoped search-index search. - * - * A bounded permitted set confines matching to the chunks the caller may read. A caller reaching - * past the permitted-set limit reads much of the index, so ranking every match before checking - * access is the leg's whole cost for a common term: where the Tin projection is complete, BM25 - * ranks inside the bases first and access is checked, on the row, only on the top of that - * ranking. Otherwise the GIN projection (`embedding_keyword_search`) ranks in three stages: - * match, authorize, rank. - * - * The visibility predicate carries correlated subqueries — one per connector, one per - * search-integration decision — so evaluating it across a base ahead of the query costs a table - * pass priced by how many documents the base holds rather than by how many the query matched. - * Matching first restricts that predicate to the documents the query actually matched. - * - * Two details keep that ordering from paying the saving back. Restricting the predicate with - * `document.id = ANY (...)` rather than a subquery keeps the narrowed lookup on a bitmap scan, - * which prefetches, where a plain `IN (SELECT ...)` plans as an index walk that does not. And - * the match stage carries identifiers only: ranking every match rather than every *visible* - * match would detoast one text-search vector per match, which on a mid-frequency term costs - * more than the pass it replaces. - */ -export async function executeIndexedKeywordSearch( - params: KeywordSearchParams, - context: IndexedRetrievalContext -): Promise { - const { knowledgeBaseIds, topK, query, queryVector, structuredFilters } = params - if (!query.trim()) return [] - const { access, accessPlan, permitted } = context - const tsQuery = sql`websearch_to_tsquery(${FTS_CONFIG}, ${query})` - const tagFilterConditions = structuredFilters?.length - ? getStructuredTagFilters(structuredFilters, embedding) - : [] - const candidateRank = sql`ts_rank_cd(${embeddingKeywordSearch.contentTsv}, ${tsQuery})` - /** - * A bounded set past the exact-ranking size is read on the row like an unbounded one: the - * bounded read materializes every chunk of the set before it matches a term, where a ranking - * decided on the row costs what the term matches. - */ - const largePermittedSet = - permitted?.kind === 'bounded' && permitted.documents.length >= PERMITTED_EXACT_DOCUMENT_LIMIT - const onRowReader = permitted?.kind === 'unbounded' || largePermittedSet - let tinQuery: Awaited> = null - if (onRowReader && tagFilterConditions.length === 0) { - try { - tinQuery = await resolveTinKeywordQuery(query, FTS_CONFIG, params.budget) - } catch (error) { - /** A leg whose deadline passed before it ranked anything is short, not failed. */ - if (!params.budget?.isTimeout(error)) throw error - return [] - } - } - if (onRowReader) annotateSearchDiagnostics({ keywordRanking: tinQuery ? 'tin' : 'gin' }) - /** A filled projection decides readability on the ranked row alone; none of its rows needs the document. */ - const tinFilled = tinQuery - ? await isProjectionFilled('embedding_keyword_tin', 'keyword.projection_filled', params.budget) - : false - /** The ranked CTE's mirrored columns, which the on-row predicates read. */ - const rankedTinRow = { - connectorId: sql`ranked_tin_chunks.connector_id`, - acl: sql`ranked_tin_chunks.acl`, - documentId: sql`ranked_tin_chunks.document_id`, - } - /** The projection predicate over the ranked CTE's mirrored columns, plus any excluded source. */ - const onRowKeywordVisibility = (excludedSources: readonly string[]) => - and( - projectionCandidateAccessCondition(rankedTinRow, access, accessPlan, { - filled: tinFilled, - }), - documentSatisfies( - sql`ranked_tin_chunks.document_id`, - searchDateFilterCondition(params.filters) - ), - excludeSearchSourcesOnRow(rankedTinRow, tinFilled, excludedSources) - ) - const documentConditions = (excludedSources: readonly string[]) => - and( - ...candidateDocumentConditions( - knowledgeBaseIds, - params.filters, - knowledgeCandidateAccessConditionForConnectors(access, accessPlan) - ), - excludeSearchSources(excludedSources) - ) - /** - * A page read on the row takes each candidate's source from the row, which is what decides - * whether its live source proof is asked for. A row decided on its document — not yet filled, - * or its document marked for the projector — takes it from the document: one primary-key read - * per such row of the page, after its limit, never per ranked row. - */ - const onRowPage = (ranked: SQL) => sql` - SELECT paged.id, paged."documentId", - CASE WHEN paged.decided_on_document - THEN (SELECT ${document.connectorId} FROM ${document} WHERE ${document.id} = paged."documentId") - ELSE paged."connectorId" - END AS "connectorId", - paged.keyword_rank - FROM (${ranked}) AS paged` - /** - * One page from the top of Tin's ranking. Readability is decided on the ranked row. The windows - * widen while the page is short, a narrow reader's to a wide one sooner and no further, and what - * the widest cannot fill is left short rather than handed to a ranking over every match. A large - * bounded set is the exception on its first page: its bounded read was exhaustive, so the widest - * window that still falls short hands that page to the GIN ranking, which covers every match. A - * later page stays with Tin: the two rankers order differently, so an offset advanced through - * one cannot resume the other. - */ - const selectTinPage = async ( - scopedQuery: SQL, - limit: number, - offset: number, - excludedSources: readonly string[] - ): Promise => { - const narrow = (permitted?.kind === 'unbounded' && !permitted.broad) || largePermittedSet - const windows: readonly number[] = narrow ? NARROW_KEYWORD_WINDOWS : TIN_KEYWORD_WINDOWS - /** - * A narrow reader's page is the readable remainder of a wide ranking, and that ranking is - * the cost: each page would rank the window again to find the next few readable rows, so one - * statement returns as many as several pages could ask for. - */ - const pageLimit = narrow ? Math.max(limit, NARROW_KEYWORD_PAGE) : limit - for (const window of windows) { - if (window < offset + limit) continue - const [page] = await runSearchQuery(params.budget, 'keyword.tin', (executor) => - executor.execute<{ ranked: number; candidates: SearchReadCandidate[] }>(sql` - WITH ranked_tin_chunks AS MATERIALIZED ( - SELECT ${embeddingKeywordTin.id} AS id, ${embeddingKeywordTin.documentId} AS document_id, - ${embeddingKeywordTin.enabled} AS enabled, ${embeddingKeywordTin.connectorId} AS connector_id, - ${embeddingKeywordTin.acl} AS acl, - tin.full_score(${embeddingKeywordTin}.ctid) AS keyword_rank - FROM ${embeddingKeywordTin} - WHERE ${embeddingKeywordTin.content} ==> (${scopedQuery}) - ORDER BY keyword_rank DESC - LIMIT ${window} - ), page AS ( - ${ - /** - * Readability decided on the ranked row: its source and ACL are mirrored there, so a - * window of mostly unreadable chunks costs an array test per row, not a document - * lookup. The full predicate follows at hydration. - */ - onRowPage( - sql` - SELECT ranked_tin_chunks.id, ranked_tin_chunks.document_id AS "documentId", - ranked_tin_chunks.connector_id AS "connectorId", ranked_tin_chunks.keyword_rank, - ${projectionDecidedOnDocument(rankedTinRow, tinFilled)} AS decided_on_document - FROM ranked_tin_chunks /* on-row visibility */ - WHERE ranked_tin_chunks.enabled AND ${onRowKeywordVisibility(excludedSources)} - ORDER BY ranked_tin_chunks.keyword_rank DESC, ranked_tin_chunks.id - LIMIT ${pageLimit} OFFSET ${offset}` - ) - } - ) - SELECT (SELECT count(*)::int FROM ranked_tin_chunks) AS ranked, - coalesce(( - SELECT json_agg(json_build_object( - 'id', page.id, 'documentId', page."documentId", 'connectorId', page."connectorId" - ) ORDER BY page.keyword_rank DESC, page.id) - FROM page - ), '[]'::json) AS candidates - `) - ) - annotateSearchDiagnostics({ keywordTinWindow: window }) - if ( - page.candidates.length >= limit || - page.ranked < window || - (!(largePermittedSet && offset === 0) && window === windows[windows.length - 1]) - ) { - return { candidates: page.candidates, nextOffset: offset + page.candidates.length } - } - } - return null - } - /** Parenthesized where used: `==>` binds tighter than `||`. */ - const tinScope = tinQuery - ? sql`'(' || ${sql.join( - knowledgeBaseIds.map((id) => sql`knowledge_tin_base_token(${id}) || '^0'`), - sql` || ' OR ' || ` - )} || ') AND (' || ${tinQuery} || ')'` - : undefined - /** - * Tin and GIN order candidates differently, so a search that once handed a page to GIN stays - * with GIN: an offset advanced through one ranking cannot resume the other. - */ - let handedToGin = false - /** Keep readable identities and rank scalars separate so sorts never carry full text-search vectors. */ - return selectAuthorizedSearchResults({ - leg: 'keyword', - access, - liveSourceAccess: context.liveSourceAccess, - signal: params.signal, - budget: params.budget, - topK, - selectPage: async (limit, offset, excludedSources) => { - /** - * A bounded permitted set confines matching to the chunks the caller may read, so a term - * common across the index is ranked only where it can surface. The visibility CTE below - * still re-applies the candidate predicate, so the restriction can only narrow. - */ - const permittedIds = - permitted?.kind === 'bounded' && !largePermittedSet - ? permitted.documents.map((entry) => entry.id) - : undefined - if (permittedIds?.length === 0) return { candidates: [], nextOffset: offset } - if (tinScope && !handedToGin) { - const tinPage = await selectTinPage(tinScope, limit, offset, excludedSources) - if (tinPage) return tinPage - handedToGin = true - annotateSearchDiagnostics({ keywordRanking: 'gin' }) - } - const baseScope = and( - inArray(embeddingKeywordSearch.knowledgeBaseId, knowledgeBaseIds), - eq(embeddingKeywordSearch.enabled, true) - ) - const chunkMatch = and( - sql`${embeddingKeywordSearch.contentTsv} @@ ${tsQuery}`, - tagFilterConditions.length - ? sql`EXISTS ( - SELECT 1 FROM ${embedding} WHERE ${embedding.id} = ${embeddingKeywordSearch.id} - AND ${and(...tagFilterConditions)} - )` - : undefined - ) - /** - * A bounded permitted set is read through its documents alone and matched row by row, at a - * cost linear in the permitted chunks. Offered the text or base indexes alongside, - * PostgreSQL may intersect the permitted chunks with every chunk in the base that holds the - * term or sits in the base; measured on an organization index that plan cost several - * times the direct read, and the direct read is never materially slower. The permitted - * documents were resolved inside these bases; the base check still applies to the rows - * read, so the read can never widen the scope. `OFFSET 0` keeps the read from being - * flattened back into an intersection; the alias lets the shared conditions bind to it. - */ - const matchedChunks = permittedIds - ? sql` - SELECT ${embeddingKeywordSearch.id} AS id, ${embeddingKeywordSearch.documentId} AS document_id - FROM ( - SELECT * FROM ${embeddingKeywordSearch} - WHERE ${embeddingKeywordSearch.documentId} = ANY(${textArrayLiteral(permittedIds)}) - OFFSET 0 - ) AS ${embeddingKeywordSearch} - WHERE ${and(baseScope, chunkMatch)}` - : sql` - SELECT ${embeddingKeywordSearch.id} AS id, ${embeddingKeywordSearch.documentId} AS document_id - FROM ${embeddingKeywordSearch} - WHERE ${and(baseScope, chunkMatch)}` - const candidates = await runSearchQuery(params.budget, 'keyword.sql', (executor) => - executor.execute( - keywordCandidateRankingQuery({ - matchedChunks, - documentConditions: [documentConditions(excludedSources)], - rankTable: embeddingKeywordSearch, - rank: candidateRank, - limit, - offset, - }) - ) - ) - return { candidates, nextOffset: offset + candidates.length } - }, - /** Every candidate already matched the query where it was ranked; matching it again here would detoast one text-search vector per result. */ - hydrate: (ids, authorized) => - hydrateSearchCandidates( - ids, - knowledgeAccessCondition(authorized), - embeddingDistance(queryVector.dimensions, queryVector.vector).as('distance'), - params.filters, - [inArray(embedding.knowledgeBaseId, knowledgeBaseIds), ...tagFilterConditions], - 'keyword', - params.budget - ), - }) -} diff --git a/apps/sim/lib/sim-search/indexed/retrieval/legs.test.ts b/apps/sim/lib/sim-search/indexed/retrieval/legs.test.ts deleted file mode 100644 index ed313c45f8c..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/legs.test.ts +++ /dev/null @@ -1,995 +0,0 @@ -import { - dbChainMockFns, - hasMockCondition, - queueTableRows, - resetDbChainMock, - resetEnvFlagsMock, - schemaMock, - setEnvFlags, -} from '@sim/testing' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockResolveTinKeywordQuery } = vi.hoisted(() => ({ - mockResolveTinKeywordQuery: vi.fn<() => Promise>(async () => null), -})) - -vi.mock('@/lib/sim-search/indexed/retrieval/tin-keyword', () => ({ - resolveTinKeywordQuery: mockResolveTinKeywordQuery, -})) - -import type { KnowledgeAccessProvider, UserAccessScope } from '@/lib/knowledge/access/types' -import { SearchBudget } from '@/lib/knowledge/search/budget' -import type { KeywordSearchParams, SearchParams } from '@/lib/knowledge/search/candidates' -import { retrieveKnowledgeSearch } from '@/lib/knowledge/search/queries' -import type { SearchAccessPlan } from '@/lib/sim-search/indexed/retrieval/access-plan' -import { executeIndexedKeywordSearch } from '@/lib/sim-search/indexed/retrieval/keyword' -import { selectIndexedTagResults } from '@/lib/sim-search/indexed/retrieval/legs' -import { - forgetSearchReach, - type IndexedRetrievalContext, - isSearchFiltered, - PERMITTED_EXACT_DOCUMENT_LIMIT, - type PermittedDocuments, - resolvePermittedDocuments, - resolveReach, -} from '@/lib/sim-search/indexed/retrieval/permitted' -import { forgetProjectionFilled } from '@/lib/sim-search/indexed/retrieval/projection-fill' -import { forgetIndexedVectorSources } from '@/lib/sim-search/indexed/retrieval/source-vector-indexes' -import { selectIndexedVectorResults } from '@/lib/sim-search/indexed/retrieval/vector' - -/** - * The projection-fill memo outlives a test; every case starts without one. Shared retrieval runs - * these legs only while indexed organization search is on. - */ -beforeEach(() => { - forgetProjectionFilled() - setEnvFlags({ isLiveEnterpriseSearchEnabled: false }) -}) -afterEach(resetEnvFlagsMock) - -/** A plan that admits every source and resolves no membership: what a test leaves unsaid. */ -const openPlan = (): SearchAccessPlan => ({ - connectors: { workspace: [], admin: [], members: [], liveProofRequired: [] }, - observers: { confirmed: [], observed: [] }, - memberSources: [], - connectorTypes: new Map(), - uploads: true, -}) - -type Resolved = Partial> & { - accessPlan?: Omit & { - connectors: Omit & { - liveProofRequired?: readonly string[] - } - } -} - -/** The caller's resolved state, split from the leg's own parameters. */ -function context( - { accessPlan, permitted, liveSourceAccess }: Resolved, - params: SearchParams -): IndexedRetrievalContext { - return { - access: params.access as UserAccessScope, - filtered: isSearchFiltered(params.filters), - accessPlan: accessPlan - ? { - ...accessPlan, - connectors: { liveProofRequired: [], ...accessPlan.connectors }, - } - : openPlan(), - permitted, - liveSourceAccess, - } -} - -const vectorSearch = ({ - accessPlan, - permitted, - liveSourceAccess, - ...params -}: SearchParams & Resolved) => - selectIndexedVectorResults(params, context({ accessPlan, permitted, liveSourceAccess }, params)) - -const tagSearch = ({ - accessPlan, - permitted, - liveSourceAccess, - ...params -}: SearchParams & Resolved) => - selectIndexedTagResults(params, context({ accessPlan, permitted, liveSourceAccess }, params)) - -const keywordSearch = ({ - accessPlan, - permitted, - liveSourceAccess, - ...params -}: KeywordSearchParams & Resolved) => - executeIndexedKeywordSearch(params, context({ accessPlan, permitted, liveSourceAccess }, params)) - -/** - * The global `drizzle-orm` mock renders `sql` fragments to a `?`-placeholder - * string via `toSQL()`, so we can assert the exact predicate each statement builds. - */ -function render(condition: unknown) { - return (condition as { toSQL: () => { sql: string; params: unknown[] } }).toSQL() -} - -/** The permitted-documents probe: the reach count and the saturation sentinel, never a slice. */ -function isProbeStatement(sql: string) { - return sql.includes('AS saturated') && !sql.includes('readable_chunks') -} - -/** A graph walk decided on the row it visits. */ -function isWalk(sql: string) { - return sql.includes('on-row visibility') && !sql.includes('ranked_tin_chunks') -} - -/** `+ 0` is what keeps the exact ranking off the ANN index, so it also identifies the statement. */ -function isExactRanking(sql: string) { - return sql.includes(') + 0 LIMIT') -} - -/** The page read: a slice of the pool's identities, from the projection and its documents. */ -function isPageStatement(sql: string) { - return ( - sql.includes('AS "connectorId"') && sql.includes('= ANY(') && !sql.includes('ranked_tin_chunks') - ) -} - -const statements = () => dbChainMockFns.execute.mock.calls.map(([query]) => render(query)) - -const bounded = ( - ...documents: Array<{ id: string; connectorId: string | null }> -): PermittedDocuments => ({ kind: 'bounded', documents }) - -describe('search-index legs rank identifiers before verification', () => { - const identity: UserAccessScope = { - kind: 'user', - userId: 'reader', - tokens: ['org', 's:github-repositories:-:42'], - } - const candidate = (id: string, connectorId: string) => ({ - id, - documentId: `doc-${id}`, - connectorId, - distance: 0.1, - }) - const provider: KnowledgeAccessProvider = { - get: async () => identity, - getForConnectors: async () => identity, - getForDocuments: async () => identity, - liveSourceConnectorCondition: async () => null, - } - const params: SearchParams = { - knowledgeBaseIds: ['org-index'], - topK: 1, - access: identity, - queryVector: { vector: '[0.1,0.2]', dimensions: 1536, model: 'text-embedding-3-small' }, - distanceThreshold: 0.8, - structuredFilters: [{ tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'release' }], - } - - const probePages: Array> = [] - const exactPages: Array> = [] - const candidatePages: Array> = [] - const rerankPages: Array>> = [] - const keywordPages: Array>> = [] - function queueRerank(rows: Array>) { - rerankPages.push(rows) - } - function queueCandidates(rows: Array<{ id: string }>, initialCount = rows.length) { - candidatePages.push(rows.map(({ id }) => ({ id, initial_count: initialCount }))) - } - - beforeEach(() => { - resetDbChainMock() - probePages.length = 0 - exactPages.length = 0 - candidatePages.length = 0 - rerankPages.length = 0 - keywordPages.length = 0 - dbChainMockFns.execute.mockImplementation(async (query) => { - const statement = render(query).sql - /** The fixtures model the page read, which only an unfilled projection makes. */ - if (statement.includes('AS unfilled')) return [{ unfilled: true }] - if (statement.includes('AS visible')) return candidatePages.shift() ?? [] - if (isPageStatement(statement)) return rerankPages.shift() ?? [] - if (statement.includes('WITH matched_keyword_chunks')) return keywordPages.shift() ?? [] - if (isExactRanking(statement)) return exactPages.shift() ?? [] - if (isProbeStatement(statement)) return probePages.shift() ?? [] - return [] - }) - }) - - it.each(['vector', 'tag-vector', 'tags', 'keyword'] as const)( - '%s ranks identifiers before verification and loads content under the full predicate', - async (mode) => { - const candidates = [candidate('selected', 'allowed-source')] - if (mode === 'vector' || mode === 'tag-vector') { - exactPages.push([{ id: 'selected' }]) - queueRerank(candidates) - } - if (mode === 'keyword') keywordPages.push(candidates) - if (mode === 'tags') queueTableRows(schemaMock.embedding, candidates) - queueTableRows(schemaMock.embedding, [{ id: 'selected', content: 'verified result' }]) - const permitted = bounded({ id: 'doc-selected', connectorId: 'allowed-source' }) - const rows = - mode === 'vector' - ? await vectorSearch({ ...params, structuredFilters: undefined, permitted }) - : mode === 'tag-vector' - ? await vectorSearch({ ...params, permitted }) - : mode === 'tags' - ? await tagSearch(params) - : await keywordSearch({ - ...params, - query: 'release', - queryVector: params.queryVector!, - }) - expect(rows).toEqual([{ id: 'selected', content: 'verified result' }]) - if (mode === 'keyword') { - const ranking = render(dbChainMockFns.execute.mock.calls[0][0]).sql - expect(ranking).toContain('matched_keyword_chunks AS MATERIALIZED') - expect(ranking).toContain('ORDER BY keyword_rank DESC, matched_keyword_chunks.id') - expect(ranking).not.toContain('<=>') - expect(ranking).not.toContain('"content"') - } else if (mode === 'tags') { - expect(Object.keys(dbChainMockFns.select.mock.calls[0][0]).sort()).toEqual( - ['id', 'documentId', 'connectorId'].sort() - ) - } else { - const ranking = statements().find((query) => isExactRanking(query.sql))! - /** The identities are one nested fragment; the mock renders it into the parameters. */ - expect(JSON.stringify(ranking)).toContain('connectorId') - expect(ranking.sql).not.toContain('"content"') - } - const rankingOrder = - mode === 'tags' - ? dbChainMockFns.select.mock.invocationCallOrder[0] - : dbChainMockFns.execute.mock.invocationCallOrder[0] - expect(rankingOrder).toBeLessThan(dbChainMockFns.select.mock.invocationCallOrder.at(-1)!) - const fullPredicate = dbChainMockFns.where.mock.calls.at(-1)![0] - expect(JSON.stringify(fullPredicate)).toContain('acl') - expect( - hasMockCondition( - fullPredicate, - (node) => - node.type === 'inArray' && - node.column === schemaMock.embedding.id && - Array.isArray(node.values) && - node.values.length === 1 && - node.values[0] === 'selected' - ) - ).toBe(true) - } - ) - - it('matches keyword chunks before the visibility predicate and ranks only what survives it', async () => { - keywordPages.push([candidate('selected', 'allowed-source')]) - queueTableRows(schemaMock.embedding, [{ id: 'selected', content: 'verified result' }]) - await keywordSearch({ ...params, query: 'release', queryVector: params.queryVector! }) - const ranking = render(dbChainMockFns.execute.mock.calls[0][0]).sql - const matched = ranking.indexOf('matched_keyword_chunks AS MATERIALIZED') - const visible = ranking.indexOf('visible_keyword_documents AS MATERIALIZED') - expect(matched).toBeGreaterThanOrEqual(0) - expect(visible).toBeGreaterThan(matched) - expect(ranking.slice(matched, visible)).not.toContain('keyword_rank') - expect(ranking.slice(visible)).toContain('FROM matched_keyword_chunks INNER JOIN') - /** The predicate fragments are parameterized, so the restriction is read off the query tree. */ - const fragments = JSON.stringify(dbChainMockFns.execute.mock.calls[0][0]) - expect(fragments).toContain('= ANY (ARRAY(SELECT document_id FROM matched_keyword_chunks))') - }) -}) - -describe('permitted-document planner', () => { - const reader: UserAccessScope = { - kind: 'user', - userId: 'reader', - tokens: ['u:reader@example.com'], - } - const provider: KnowledgeAccessProvider = { - get: async () => reader, - getForConnectors: async () => reader, - getForDocuments: async () => reader, - liveSourceConnectorCondition: async () => null, - } - const params: SearchParams = { - knowledgeBaseIds: ['org-index'], - topK: 1, - access: reader, - queryVector: { vector: '[0.1,0.2]', dimensions: 1536, model: 'text-embedding-3-small' }, - distanceThreshold: 1, - } - const hit = (id: string, connectorId: string | null) => ({ - id, - documentId: `doc-${id}`, - connectorId, - distance: 0.1, - }) - let probeRows: Array<{ id: string | null; connectorId: string | null; saturated: boolean }> - let exactRows: Array<{ id: string }> - let traversedRows: Array<{ id: string; distance?: number }> - let rerankRows: Array> - let indexedSourceRows: Array<{ name: string; connectorId: string }> - let sourceExactRows: Array<{ id: string; distance: number }> - - beforeEach(() => { - resetDbChainMock() - probeRows = [] - exactRows = [] - traversedRows = [] - rerankRows = [] - sourceExactRows = [] - indexedSourceRows = [] - forgetIndexedVectorSources() - forgetSearchReach() - dbChainMockFns.execute.mockImplementation(async (query) => { - const statement = render(query).sql - /** The fixtures model the page read, which only an unfilled projection makes. */ - if (statement.includes('AS unfilled')) return [{ unfilled: true }] - if (statement.includes('pg_index')) return indexedSourceRows - if (isWalk(statement)) return traversedRows - if (isPageStatement(statement)) return rerankRows - if (statement.includes('WITH readable_chunks')) return sourceExactRows - if (isExactRanking(statement)) return exactRows - if (isProbeStatement(statement)) return probeRows - return [] - }) - }) - - it('ranks a bounded permitted set exactly without walking the graph', async () => { - exactRows = [{ id: 'a' }] - rerankRows = [hit('a', null)] - queueTableRows(schemaMock.embedding, [hit('a', null)]) - const results = await vectorSearch({ - ...params, - permitted: bounded({ id: 'doc-a', connectorId: null }, { id: 'doc-b', connectorId: 'src' }), - }) - expect(results.map((row) => row.id)).toEqual(['a']) - const sqls = statements().map((query) => query.sql) - expect(sqls.some((sql) => sql.includes('hnsw.iterative_scan'))).toBe(false) - expect(sqls.some((sql) => sql.includes('AS visible'))).toBe(false) - expect(sqls.some(isProbeStatement)).toBe(false) - const exact = JSON.stringify(statements().find((query) => isExactRanking(query.sql))) - expect(exact).toContain('doc-a') - expect(exact).toContain('doc-b') - }) - - it('walks the whole graph once for a caller whose reach is broad', async () => { - const eligibility = { workspace: [], admin: ['other-src'], members: ['member-src'] } - indexedSourceRows = [{ name: 'idx', connectorId: 'member-src' }] - /** A full pool: the walk found as many readable neighbours as it was asked for. */ - traversedRows = Array.from({ length: 400 }, (_, i) => ({ id: `walked-${i}`, distance: 0.2 })) - rerankRows = [hit('walked-0', 'member-src')] - queueTableRows(schemaMock.embedding, rerankRows) - await vectorSearch({ - ...params, - permitted: { kind: 'unbounded', broad: true }, - accessPlan: { - connectors: eligibility, - observers: { confirmed: [{ id: 'm-1', connectorId: 'member-src' }], observed: [] }, - memberSources: ['member-src'], - connectorTypes: new Map(), - uploads: true, - }, - }) - /** One walk over every source, scoped to the bases alone — no source is singled out. */ - const walks = statements().filter((query) => isWalk(query.sql)) - expect(walks).toHaveLength(1) - expect(JSON.stringify(walks[0])).not.toContain('"right":"member-src"') - expect(statements().some((query) => query.sql.includes('WITH readable_chunks'))).toBe(false) - }) - - it('walks an indexed source a bounded caller is a member of instead of ranking it exactly', async () => { - const eligibility = { workspace: [], admin: ['small-src'], members: ['member-src'] } - indexedSourceRows = [{ name: 'idx', connectorId: 'member-src' }] - sourceExactRows = [{ id: 'small-hit', distance: 0.3, saturated: false }] - traversedRows = [{ id: 'walked-hit', distance: 0.2 }] - rerankRows = [hit('walked-hit', 'member-src'), hit('small-hit', 'small-src')] - queueTableRows(schemaMock.embedding, rerankRows) - await vectorSearch({ - ...params, - topK: 2, - permitted: bounded( - { id: 'doc-a', connectorId: 'member-src' }, - { id: 'doc-b', connectorId: 'small-src' } - ), - accessPlan: { - connectors: eligibility, - observers: { confirmed: [{ id: 'm-1', connectorId: 'member-src' }], observed: [] }, - memberSources: ['member-src'], - connectorTypes: new Map(), - uploads: true, - }, - }) - const walks = statements().filter((query) => isWalk(query.sql)) - expect(walks).toHaveLength(1) - expect(JSON.stringify(walks[0])).toContain('"right":"member-src"') - expect(statements().some((q) => isExactRanking(q.sql))).toBe(false) - }) - - it('walks the sliced sources when more documents are readable than one ranking may enumerate', async () => { - const eligibility = { workspace: [], admin: ['sliced-src'], members: [] } - /** The slice enumerates in no order, so a saturated one would rank an arbitrary subset. */ - sourceExactRows = [{ id: 'arbitrary-hit', distance: 0.4, saturated: true }] - traversedRows = [{ id: 'walked-hit', distance: 0.2 }] - rerankRows = [hit('walked-hit', 'sliced-src')] - queueTableRows(schemaMock.embedding, rerankRows) - await vectorSearch({ - ...params, - permitted: { kind: 'unbounded', broad: false }, - accessPlan: { - connectors: eligibility, - observers: { confirmed: [], observed: [] }, - memberSources: [], - connectorTypes: new Map(), - uploads: true, - }, - }) - const walks = statements().filter((query) => isWalk(query.sql)) - expect(walks).toHaveLength(1) - expect(JSON.stringify(walks[0])).toContain('sliced-src') - const reranked = JSON.stringify(statements().find((query) => isPageStatement(query.sql))) - expect(reranked).toContain('walked-hit') - expect(reranked).not.toContain('arbitrary-hit') - }) - - it('ranks uploaded documents even when every connector source is walked', async () => { - const eligibility = { workspace: [], admin: [], members: ['member-src'] } - indexedSourceRows = [{ name: 'idx', connectorId: 'member-src' }] - sourceExactRows = [{ id: 'upload-hit', distance: 0.05, saturated: false }] - traversedRows = [{ id: 'walked-hit', distance: 0.2 }] - rerankRows = [hit('upload-hit', null), hit('walked-hit', 'member-src')] - queueTableRows(schemaMock.embedding, rerankRows) - await vectorSearch({ - ...params, - topK: 2, - permitted: { kind: 'unbounded', broad: false }, - accessPlan: { - connectors: eligibility, - observers: { confirmed: [{ id: 'm-1', connectorId: 'member-src' }], observed: [] }, - memberSources: ['member-src'], - connectorTypes: new Map(), - uploads: true, - }, - }) - /** Uploads carry no connector, so their slice runs even with no sliced source beside them. */ - const exact = statements().filter((query) => query.sql.includes('WITH readable_chunks')) - expect(exact).toHaveLength(1) - expect(JSON.stringify(statements().find((q) => isPageStatement(q.sql)))).toContain('upload-hit') - }) - - it('confines keyword matching to the bounded permitted set', async () => { - await keywordSearch({ - ...params, - topK: 1, - query: 'release', - queryVector: params.queryVector!, - permitted: bounded({ id: 'doc-a', connectorId: null }), - }) - const keyword = statements().find((query) => query.sql.includes('WITH matched_keyword_chunks'))! - /** The mock renders the whole WHERE as one parameter, so the restriction shows up in it. */ - expect(JSON.stringify(keyword)).toContain('doc-a') - }) - - describe('Tin keyword ranking for an unbounded caller', () => { - const unbounded: PermittedDocuments = { kind: 'unbounded' } - const keyword = (overrides: Partial[0]> = {}) => - keywordSearch({ - ...params, - topK: 1, - query: 'release', - queryVector: params.queryVector!, - permitted: unbounded, - ...overrides, - }) - const tinStatements = () => - statements().filter((query) => query.sql.includes('ranked_tin_chunks')) - const ginStatements = () => - statements().filter((query) => query.sql.includes('WITH matched_keyword_chunks')) - let tinPages: Array<{ ranked: number; candidates: ReturnType[] }> - - beforeEach(() => { - mockResolveTinKeywordQuery.mockReset() - mockResolveTinKeywordQuery.mockResolvedValue('"releas"') - tinPages = [] - dbChainMockFns.execute.mockImplementation(async (query) => - render(query).sql.includes('ranked_tin_chunks') - ? [tinPages.shift() ?? { ranked: 0, candidates: [] }] - : [] - ) - }) - - it('ranks with Tin and checks access only on the top of that ranking', async () => { - tinPages = [{ ranked: 1500, candidates: [hit('a', null)] }] - queueTableRows(schemaMock.embedding, [{ ...hit('a', null), content: 'release notes' }]) - const results = await keyword() - expect(results.map((row) => row.id)).toEqual(['a']) - expect(mockResolveTinKeywordQuery).toHaveBeenCalledWith('release', 'english', undefined) - expect(ginStatements()).toHaveLength(0) - expect(JSON.stringify(tinStatements()[0])).toContain('2000') - /** `==>` binds tighter than `||`, so the concatenated query must be parenthesized. */ - expect(tinStatements()[0].sql).toContain('==> (?)') - }) - - it('decides a row the fill has not reached on its document while the fill runs', async () => { - tinPages.push({ - ranked: 1, - candidates: [{ id: 'a', documentId: 'doc-a', connectorId: 'src-a' }], - }) - const execute = dbChainMockFns.execute.getMockImplementation()! - dbChainMockFns.execute.mockImplementation(async (query) => - render(query).sql.includes('AS unfilled') ? [{ unfilled: true }] : execute(query) - ) - await keyword({ - accessPlan: { - connectors: { workspace: [], admin: ['src-a'], members: [] }, - observers: { confirmed: [], observed: [] }, - memberSources: [], - connectorTypes: new Map(), - uploads: true, - }, - }) - const statement = JSON.stringify(tinStatements()[0]) - /** A row the fill has not reached (`acl IS NULL`), or a marked document's row, is decided on its document. */ - expect(statement).toContain(' IS NULL OR ') - expect(statement).toContain('knowledgeProjectionDirty.documentId') - expect(statement).toContain('EXISTS (') - expect(statement).toContain('ranked_tin_chunks.document_id') - }) - - it('widens the window for a broad resolved scope whose first page came back short', async () => { - tinPages = [ - { ranked: 2000, candidates: [] }, - { ranked: 4000, candidates: [hit('b', 'src-a')] }, - ] - queueTableRows(schemaMock.embedding, [{ ...hit('b', 'src-a'), content: 'release notes' }]) - const results = await keyword({ - permitted: { kind: 'unbounded', broad: true }, - accessPlan: { - connectors: { workspace: [], admin: ['src-a'], members: [] }, - observers: { confirmed: [], observed: [] }, - memberSources: [], - connectorTypes: new Map(), - uploads: true, - }, - }) - expect(results.map((row) => row.id)).toEqual(['b']) - const windows = tinStatements().map((query) => JSON.stringify(query)) - expect(windows).toHaveLength(2) - expect(windows[0]).toContain('2000') - expect(windows[1]).toContain('10000') - }) - - it('hydrates an oversized keyword page in slices and stops at the results it needs', async () => { - const ranked = Array.from({ length: 1000 }, (_, i) => hit(`k-${i}`, 'src-a')) - tinPages = [{ ranked: 20_000, candidates: ranked }] - /** The first slice — as many candidates as results are wanted — fills the page of results. */ - queueTableRows( - schemaMock.embedding, - ranked.slice(0, 20).map((row) => ({ ...row, content: 'release notes' })) - ) - const results = await keyword({ - topK: 20, - permitted: { kind: 'unbounded', broad: false }, - accessPlan: { - connectors: { workspace: [], admin: ['src-a'], members: [] }, - observers: { confirmed: [], observed: [] }, - memberSources: [], - connectorTypes: new Map(), - uploads: true, - }, - }) - expect(results).toHaveLength(20) - expect(tinStatements()).toHaveLength(1) - /** One hydration, of one slice — never the whole page. */ - const hydrations = dbChainMockFns.where.mock.calls.filter(([condition]) => - hasMockCondition( - condition, - (node) => node.type === 'inArray' && node.column === schemaMock.embedding.id - ) - ) - expect(hydrations).toHaveLength(1) - expect( - hasMockCondition( - hydrations[0][0], - (node) => - node.type === 'inArray' && - node.column === schemaMock.embedding.id && - Array.isArray(node.values) && - node.values.length === 20 - ) - ).toBe(true) - }) - - describe('a bounded set past the exact-ranking size', () => { - const large = Array.from({ length: PERMITTED_EXACT_DOCUMENT_LIMIT }, (_, index) => ({ - id: `doc-${index}`, - connectorId: 'src-a', - })) - const accessPlan = { - connectors: { workspace: [], admin: ['src-a'], members: [], liveProofRequired: [] }, - observers: { confirmed: [], observed: [] }, - memberSources: [], - connectorTypes: new Map(), - uploads: true, - } - - it('ranks with Tin as a narrow reader, decided on the row', async () => { - tinPages = [{ ranked: 1500, candidates: [hit('a', 'src-a')] }] - queueTableRows(schemaMock.embedding, [{ ...hit('a', 'src-a'), content: 'release notes' }]) - const results = await keyword({ - permitted: { kind: 'bounded', documents: large }, - accessPlan, - }) - expect(results.map((row) => row.id)).toEqual(['a']) - expect(mockResolveTinKeywordQuery).toHaveBeenCalledTimes(1) - expect(tinStatements()).toHaveLength(1) - expect(JSON.stringify(tinStatements()[0])).toContain('2000') - expect(JSON.stringify(tinStatements()[0])).not.toContain('doc-4999') - expect(ginStatements()).toHaveLength(0) - }) - - it('leaves a later page short rather than resuming a different ranking at its offset', async () => { - /** The first page fills from Tin; hydration keeps half, so a second page is asked for. */ - const first = Array.from({ length: 40 }, (_, index) => hit(`t-${index}`, 'src-a')) - tinPages = [ - { ranked: 2000, candidates: first }, - { ranked: 2000, candidates: [] }, - { ranked: 20_000, candidates: [] }, - ] - queueTableRows( - schemaMock.embedding, - first.slice(0, 20).map((row) => ({ ...row, content: 'release notes' })) - ) - const results = await keyword({ - topK: 40, - permitted: { kind: 'bounded', documents: large }, - accessPlan, - }) - expect(results).toHaveLength(20) - expect(tinStatements()).toHaveLength(3) - expect(ginStatements()).toHaveLength(0) - }) - }) - - it('keeps GIN ranking when Tin is not ready or cannot express the query', async () => { - mockResolveTinKeywordQuery.mockResolvedValue(null) - await keyword() - expect(tinStatements()).toHaveLength(0) - expect(ginStatements()).toHaveLength(1) - }) - }) - - it('skips keyword SQL entirely when nothing is permitted', async () => { - expect( - await keywordSearch({ - ...params, - query: 'release', - queryVector: params.queryVector!, - permitted: bounded(), - }) - ).toEqual([]) - expect(dbChainMockFns.execute).not.toHaveBeenCalled() - }) - - it('reads a user scope through its reachable documents and reports saturation', async () => { - probeRows = [{ id: 'doc-a', connectorId: null, saturated: false }] - await resolvePermittedDocuments({ - knowledgeBaseIds: ['org-index'], - access: { ...reader, tokens: ['u:reachable-documents@example.com'] }, - accessPlan: openPlan(), - filtered: false, - }) - const user = statements().find((query) => isProbeStatement(query.sql))! - const userSql = user.sql - expect(userSql).toContain('WITH reach AS MATERIALIZED') - expect(userSql).toContain('reachable AS MATERIALIZED') - expect(userSql).toContain('FROM reachable AS') - expect(userSql).toContain('AS saturated') - /** - * Baseline tokens reach every tenant's org-wide, public, and uploaded documents, so both the - * count and the rows are confined to the requested bases, outside the fence around the index. - */ - const [reach, reachable] = userSql.split('reachable AS MATERIALIZED') - for (const cte of [reach, reachable.split('FROM reachable AS')[0]]) { - expect(cte).toMatch(/OFFSET 0\s*\) AS \?\s*WHERE \?/) - } - expect(JSON.stringify(user.params)).toContain('org-index') - }) - - it.each([ - [[{ id: null, connectorId: null, saturated: true }], 'unbounded'], - [[{ id: 'doc-a', connectorId: null, saturated: false }], 'bounded'], - ] as const)('resolves %j as %s', async (rows, kind) => { - probeRows = [...rows] - const permitted = await resolvePermittedDocuments({ - knowledgeBaseIds: ['org-index'], - access: { ...reader, tokens: [`u:resolves-${kind}@example.com`] }, - accessPlan: openPlan(), - filtered: false, - }) - expect(permitted.kind).toBe(kind) - if (permitted.kind === 'bounded') - expect(permitted.documents).toEqual([{ id: 'doc-a', connectorId: null }]) - }) - - describe('saturated reach', () => { - const scope = (name: string): UserAccessScope => ({ - ...reader, - tokens: [`u:${name}@example.com`], - }) - const resolve = (access: UserAccessScope, knowledgeBaseIds = ['org-index']) => - resolvePermittedDocuments({ - knowledgeBaseIds, - access, - accessPlan: openPlan(), - filtered: false, - }) - const probes = () => statements().filter((query) => isProbeStatement(query.sql)).length - - it('counts a saturated reach against the broad bound once, and remembers the answer', async () => { - /** The index holds a million documents; the bound is a quarter of them. */ - const counts = { index: 1_000_000, reached: 250_000 } - dbChainMockFns.execute.mockImplementation(async (query) => { - const statement = render(query).sql - if (isProbeStatement(statement)) return [{ id: null, connectorId: null, saturated: true }] - if (statement.includes(') reached')) return [{ n: counts.reached }] - if (statement.includes('EXPLAIN')) - return [{ 'QUERY PLAN': [{ Plan: { 'Plan Rows': counts.index } }] }] - return [] - }) - const reachCounts = () => statements().filter((query) => query.sql.includes(') reached')) - const broad = await resolve(scope('broad-reach')) - expect(broad).toEqual({ kind: 'unbounded', broad: true }) - expect(reachCounts()).toHaveLength(1) - expect(JSON.stringify(reachCounts()[0])).toContain('250000') - await resolve(scope('broad-reach')) - expect(reachCounts()).toHaveLength(1) - counts.reached = 120_000 - const narrow = await resolve(scope('narrow-reach')) - expect(narrow).toEqual({ kind: 'unbounded', broad: false }) - expect(reachCounts()).toHaveLength(2) - }) - - it('does not remember a saturated reach whose count ran out of time', async () => { - dbChainMockFns.execute.mockImplementation(async (query) => { - const statement = render(query).sql - if (isProbeStatement(statement)) return [{ id: null, connectorId: null, saturated: true }] - if (statement.includes('EXPLAIN')) - return [{ 'QUERY PLAN': [{ Plan: { 'Plan Rows': 1_000_000 } }] }] - if (statement.includes(') reached')) - throw Object.assign(new Error('canceling statement due to statement timeout'), { - code: '57014', - }) - return [] - }) - const reachCounts = () => statements().filter((query) => query.sql.includes(') reached')) - const budget = () => new SearchBudget('vector', performance.now() + 10_000) - expect( - await resolvePermittedDocuments({ - knowledgeBaseIds: ['org-index'], - access: scope('timed-saturated'), - budget: budget(), - accessPlan: openPlan(), - filtered: false, - }) - ).toEqual({ kind: 'unbounded', broad: true }) - expect(reachCounts()).toHaveLength(1) - /** The next search probes and counts again rather than trusting a reach that was never measured. */ - await resolvePermittedDocuments({ - knowledgeBaseIds: ['org-index'], - access: scope('timed-saturated'), - budget: budget(), - accessPlan: openPlan(), - filtered: false, - }) - expect(probes()).toBe(2) - expect(reachCounts()).toHaveLength(2) - }) - - it('does not read an unanalyzed index as a reach of nothing', async () => { - /** The planner knows no rows yet, so the bound is zero and the count looked at nothing. */ - dbChainMockFns.execute.mockImplementation(async (query) => { - const statement = render(query).sql - if (statement.includes('EXPLAIN')) return [{ 'QUERY PLAN': [{ Plan: { 'Plan Rows': 0 } }] }] - if (statement.includes(') reached')) return [{ n: 0 }] - return [] - }) - await expect( - resolveReach( - ['org-index'], - scope('unanalyzed'), - new SearchBudget('vector', performance.now() + 10_000), - { - connectors: { workspace: [], admin: [], members: [], liveProofRequired: [] }, - observers: { confirmed: [], observed: [] }, - memberSources: [], - connectorTypes: new Map(), - uploads: true, - } - ) - ).resolves.toEqual({ kind: 'unbounded', broad: true }) - }) - - it('is remembered per set of bases and tokens', async () => { - probeRows = [{ id: null, connectorId: null, saturated: true }] - await resolve(scope('per-key')) - probeRows = [{ id: 'doc-a', connectorId: null, saturated: false }] - expect((await resolve(scope('per-key'), ['other-index'])).kind).toBe('bounded') - expect((await resolve(scope('per-key-other'))).kind).toBe('bounded') - expect(probes()).toBe(3) - }) - }) - - it('reports an exhausted vector budget as unbounded instead of failing both legs', async () => { - const budget = new SearchBudget('vector', performance.now() - 1) - const permitted = await resolvePermittedDocuments({ - knowledgeBaseIds: ['org-index'], - access: reader, - budget, - accessPlan: openPlan(), - filtered: false, - }) - expect(permitted.kind).toBe('unbounded') - expect(budget.timedOut).toBe(true) - }) - - const liveSearch = { - knowledgeBaseIds: ['org-index'], - indexedRetrieval: true, - topK: 1, - searchMode: 'hybrid' as const, - query: 'release', - queryVector: params.queryVector!, - } - - it('never asks a source for live grants when the scope reads none', async () => { - const getForConnectors = vi.fn(async () => reader) - await retrieveKnowledgeSearch({ - ...liveSearch, - access: reader, - accessProvider: { ...provider, getForConnectors }, - }) - expect(getForConnectors).not.toHaveBeenCalled() - }) - - it('rebuilds the pool without a gated source the caller turns out not to hold', async () => { - queueTableRows(schemaMock.knowledgeConnector, [ - { - id: 'gated-src', - accessMode: 'admin', - connectorType: 'confluence', - githubRepository: false, - }, - ]) - /** - * The first pool is filled by the gated source alone; only a pool built without it — the - * exclusion carries the source id into the walk — reaches the accessible candidate. The - * projection is filled, so the walk carries each candidate's source and no page is read. - */ - dbChainMockFns.execute.mockImplementation(async (query) => { - const statement = render(query).sql - if (statement.includes('AS unfilled')) return [{ unfilled: false }] - const rebuilt = JSON.stringify(query).includes('/* excluded sources */') - if (isWalk(statement)) - return Array.from({ length: 400 }, (_, i) => - i === 0 - ? rebuilt - ? hit('b', 'other-src') - : hit('a', 'gated-src') - : hit(`w-${i}`, rebuilt ? 'other-src' : 'gated-src') - ) - return [] - }) - queueTableRows(schemaMock.embedding, []) - queueTableRows(schemaMock.embedding, [hit('b', 'other-src')]) - /** No grants come back, so the gated source is denied. */ - const getForConnectors = vi.fn(async () => reader) - const result = await retrieveKnowledgeSearch({ - ...liveSearch, - searchMode: 'vector', - access: reader, - accessProvider: { ...provider, getForConnectors }, - }) - expect(getForConnectors).toHaveBeenCalledOnce() - expect(result.rows.map((row) => row.id)).toEqual(['b']) - const walks = statements().filter((query) => isWalk(query.sql)) - expect(walks).toHaveLength(2) - expect(JSON.stringify(walks[0])).not.toContain('/* excluded sources */') - expect(JSON.stringify(walks[1])).toContain('/* excluded sources */') - expect(JSON.stringify(walks[1])).toContain('OR NOT (') - expect(statements().some((query) => isPageStatement(query.sql))).toBe(false) - }) -}) - -describe('filters on a resolved scope', () => { - const reader: UserAccessScope = { - kind: 'user', - userId: 'reader', - tokens: ['u:reader@example.com'], - } - const provider: KnowledgeAccessProvider = { - get: async () => reader, - getForConnectors: async () => reader, - getForDocuments: async () => reader, - liveSourceConnectorCondition: async () => null, - } - const params: SearchParams = { - knowledgeBaseIds: ['org-index'], - topK: 1, - access: reader, - queryVector: { vector: '[0.1,0.2]', dimensions: 1536, model: 'text-embedding-3-small' }, - distanceThreshold: 1, - } - const plan = (sources: string[] = ['src-a']) => ({ - connectors: { workspace: [], admin: sources, members: [], liveProofRequired: [] }, - observers: { confirmed: [], observed: [] }, - memberSources: [], - connectorTypes: new Map(sources.map((id) => [id, 'slack'])), - uploads: true, - }) - const hit = (id: string, connectorId: string | null) => ({ - id, - documentId: `doc-${id}`, - connectorId, - distance: 0.1, - }) - let probeRows: Array<{ id: string | null; connectorId: string | null; saturated: boolean }> - let traversedRows: Array<{ id: string; distance?: number }> - let rerankRows: Array> - let exactRows: Array<{ id: string }> - let indexedSourceRows: Array<{ name: string; connectorId: string }> - - beforeEach(() => { - resetDbChainMock() - forgetIndexedVectorSources() - forgetSearchReach() - probeRows = [] - traversedRows = [] - rerankRows = [] - exactRows = [] - indexedSourceRows = [] - dbChainMockFns.execute.mockImplementation(async (query) => { - const statement = render(query).sql - /** The fixtures model the page read, which only an unfilled projection makes. */ - if (statement.includes('AS unfilled')) return [{ unfilled: true }] - if (statement.includes('EXPLAIN')) - return [{ 'QUERY PLAN': [{ Plan: { 'Plan Rows': 1_000_000 } }] }] - if (statement.includes('pg_index')) return indexedSourceRows - if (isExactRanking(statement)) return exactRows - if (statement.includes(') reached')) return [{ n: 250_000 }] - if (isProbeStatement(statement)) return probeRows - if (isWalk(statement)) return traversedRows - if (isPageStatement(statement)) return rerankRows - if (statement.includes('ranked_tin_chunks')) return [{ ranked: 0, candidates: [] }] - return [] - }) - }) - - it('enumerates the documents a date filter admits even when the reach is remembered', async () => { - probeRows = [{ id: 'doc-recent', connectorId: 'src-a', saturated: false }] - const budget = () => new SearchBudget('vector', performance.now() + 10_000) - await resolveReach(['org-index'], reader, budget(), plan()) - const permitted = await resolvePermittedDocuments({ - knowledgeBaseIds: ['org-index'], - access: reader, - filters: { modifiedAfter: '2026-09-13T00:00:00.000Z' }, - budget: budget(), - accessPlan: plan(), - filtered: true, - }) - expect(permitted).toEqual({ - kind: 'bounded', - documents: [{ id: 'doc-recent', connectorId: 'src-a' }], - }) - const probes = statements().filter((query) => query.sql.includes('AS saturated')) - expect(probes).toHaveLength(1) - /** Filter first, over the date index: never the reach count that reports a broad reader saturated. */ - expect(probes[0].sql).not.toContain('WITH reach') - /** An index-driven probe earns its own budget: a window at the document limit fits inside it. */ - const deadlines = statements().filter((query) => query.sql.includes('statement_timeout')) - expect(deadlines.at(-1)?.params[0]).toBe('1500') - expect(JSON.stringify(probes[0])).toContain('"type":"gte"') - }) -}) diff --git a/apps/sim/lib/sim-search/indexed/retrieval/legs.ts b/apps/sim/lib/sim-search/indexed/retrieval/legs.ts deleted file mode 100644 index 2eb4a9aa2c6..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/legs.ts +++ /dev/null @@ -1,128 +0,0 @@ -import type { KnowledgeAccessProvider, UserAccessScope } from '@/lib/knowledge/access/types' -import type { SearchBudget } from '@/lib/knowledge/search/budget' -import { - liveSourceAccessForConnectors, - type RetrievalLegs, - type SearchParams, - type SearchResult, - VECTOR_PROBE_BUDGET_MS, - VECTOR_PROBE_DOCUMENT_LIMIT, -} from '@/lib/knowledge/search/candidates' -import { measureSearchStage } from '@/lib/knowledge/search/diagnostics' -import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters' -import { selectAuthorizedTagResults } from '@/lib/knowledge/search/tag-filters' -import { assertIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' -import { - resolveSearchAccessPlan, - restrictSearchAccessPlan, -} from '@/lib/sim-search/indexed/retrieval/access-plan' -import { executeIndexedKeywordSearch } from '@/lib/sim-search/indexed/retrieval/keyword' -import { - estimateFilteredDocuments, - type IndexedRetrievalContext, - isSearchFiltered, - type PermittedDocuments, - resolvePermittedDocuments, - resolveReach, -} from '@/lib/sim-search/indexed/retrieval/permitted' -import { knowledgeCandidateAccessConditionForConnectors } from '@/lib/sim-search/indexed/retrieval/projection-access' -import { selectIndexedVectorResults } from '@/lib/sim-search/indexed/retrieval/vector' - -/** - * The tag-only leg of a user-scoped search-index search: candidate identities under the caller's - * resolved plan, in id order, hydrated under the full read predicate once live source proof is - * known. - */ -export function selectIndexedTagResults( - params: SearchParams, - context: IndexedRetrievalContext -): Promise { - if (!params.structuredFilters || params.structuredFilters.length === 0) { - throw new Error('Tag filters are required for tag-only search') - } - return selectAuthorizedTagResults( - params, - knowledgeCandidateAccessConditionForConnectors(context.access, context.accessPlan), - context.liveSourceAccess - ) -} - -/** - * Resolves what a user-scoped search over search indexes needs before any leg ranks, and binds it - * to the search-index legs. Connector state is the same for every document a connector owns, so - * every leg reads it from one resolution instead of proving it per candidate. A source filter - * confines the plan rather than the rows: with only that kind of source eligible, every predicate - * the plan builds and every source the legs walk is that kind. - * - * A ranked search also resolves, once and on the vector leg's budget, what the caller may read. - * A filter that leaves few documents is enumerated and ranked exactly inside them, both legs: the - * row does not carry the document's date, and a keyword ranking of the whole base may hold few of - * a small source's matches. A filter that leaves many is ranked as the scope is — the source - * confined on the row, the date tested through the document — since a set that large holds most - * of the query's neighbours anyway. The planner's estimate decides which. Otherwise readability is - * decided on the projection row, so the caller's reach alone chooses between one walk over the - * whole graph and a search of each source. Explicit documents are already a bounded scope with - * their own exhaustive ordering. - */ -export async function prepareIndexedRetrieval(input: { - knowledgeBaseIds: string[] - access: UserAccessScope - accessProvider: KnowledgeAccessProvider - filters?: WorkspaceSearchFilters - signal?: AbortSignal - /** Whether the search ranks a query; a tag-only search needs no permitted set. */ - ranked: boolean - /** The vector leg's budget, which resolving the permitted set spends. */ - budget: SearchBudget -}): Promise { - assertIndexedOrgSearchEnabled() - const { knowledgeBaseIds, access, filters } = input - const resolvedPlan = await measureSearchStage('access_plan', () => - resolveSearchAccessPlan(knowledgeBaseIds, access) - ) - const accessPlan = filters?.source - ? restrictSearchAccessPlan(resolvedPlan, filters.source) - : resolvedPlan - const liveSourceAccess = liveSourceAccessForConnectors( - accessPlan.connectors.liveProofRequired, - input.accessProvider, - input.signal - ) - const filtered = isSearchFiltered(filters) - let permitted: PermittedDocuments | undefined - if (input.ranked && !filters?.documentIds?.length) { - /** Planning only, so a short cap of its own: running past it answers as the wide window it may be. */ - const estimateBudget = input.budget.capped(VECTOR_PROBE_BUDGET_MS) - const enumerateFiltered = - filters && filtered - ? await estimateFilteredDocuments(knowledgeBaseIds, filters, accessPlan, estimateBudget) - .then((estimate) => estimate <= VECTOR_PROBE_DOCUMENT_LIMIT) - .catch((error) => { - if (!estimateBudget.isTimeout(error)) throw error - return false - }) - : false - permitted = enumerateFiltered - ? await resolvePermittedDocuments({ - knowledgeBaseIds, - access, - filters, - budget: input.budget, - accessPlan, - filtered, - }) - : await resolveReach(knowledgeBaseIds, access, input.budget, accessPlan) - } - const context: IndexedRetrievalContext = { - access, - accessPlan, - filtered, - permitted, - liveSourceAccess, - } - return { - tags: (params) => selectIndexedTagResults(params, context), - vector: (params) => selectIndexedVectorResults(params, context), - keyword: (params) => executeIndexedKeywordSearch(params, context), - } -} diff --git a/apps/sim/lib/sim-search/indexed/retrieval/permitted.ts b/apps/sim/lib/sim-search/indexed/retrieval/permitted.ts deleted file mode 100644 index 70518416745..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/permitted.ts +++ /dev/null @@ -1,424 +0,0 @@ -import { document } from '@sim/db/schema' -import { sha256Hex } from '@sim/security/hash' -import { and, inArray, isNull, type SQL, sql } from 'drizzle-orm' -import type { AnyPgColumn } from 'drizzle-orm/pg-core' -import { LRUCache } from 'lru-cache' -import { textArrayLiteral } from '@/lib/knowledge/access/predicate' -import type { UserAccessScope } from '@/lib/knowledge/access/types' -import { runSearchQuery, type SearchBudget } from '@/lib/knowledge/search/budget' -import { - candidateDocumentConditions, - directVisibleDocumentsQuery, - type LiveSourceAccess, - type PermittedDocument, - type ProbeOutcome, - probeVisibleDocuments, - VECTOR_PROBE_BUDGET_MS, - VECTOR_PROBE_DOCUMENT_LIMIT, -} from '@/lib/knowledge/search/candidates' -import { annotateSearchDiagnostics } from '@/lib/knowledge/search/diagnostics' -import { searchDateFilterCondition } from '@/lib/knowledge/search/filter-conditions' -import type { WorkspaceSearchFilters } from '@/lib/knowledge/search/filters' -import type { SearchAccessPlan } from '@/lib/sim-search/indexed/retrieval/access-plan' -import { - knowledgeAclOverlapCondition, - knowledgeCandidateAccessConditionForConnectors, -} from '@/lib/sim-search/indexed/retrieval/projection-access' - -/** - * What a filter-first probe may spend: it reads the filtered documents off their own index and - * tests each one's access, bounded by the same document limit, and measures around 2 µs per - * document to enumerate plus the access test — a window at the limit fits with room. Its result - * is ranked exactly, at a cost that is predictable where a walk through a mostly-excluded - * neighbourhood is not. - */ -const FILTERED_PROBE_BUDGET_MS = 1500 - -/** - * Whether a filter narrows the documents in a way the projection row cannot see — a date window, - * or a source kind, which confines the plan — so the filtered set is worth estimating and, when - * small, enumerating directly. - */ -export function isSearchFiltered(filters: WorkspaceSearchFilters | undefined): boolean { - return Boolean(searchDateFilterCondition(filters) || filters?.source) -} - -/** A row whose document satisfies `condition`; nothing when there is nothing to ask the document. */ -export function documentSatisfies( - documentId: AnyPgColumn | SQL, - condition: SQL | undefined -): SQL | undefined { - if (condition === undefined) return undefined - return sql`EXISTS (SELECT 1 FROM ${document} WHERE ${and(sql`${document.id} = ${documentId}`, condition)})` -} - -/** - * Documents a bounded permitted set may hold before ranking it exactly costs more than walking - * the graph on the row. Exact ranking reads every chunk of the set, a few per document, where an - * on-row walk reads at most a capped number of tuples; at this size the two meet. A set past it - * is walked first and ranked exactly only if the walk cannot fill its pool, so its recall is never - * below the exact ranking's and its usual cost is the walk's. The same size turns the keyword leg - * from a read of the set's every chunk into a ranking decided on the row. - */ -export const PERMITTED_EXACT_DOCUMENT_LIMIT = 5_000 - -/** - * The documents a user-scoped search-index search may rank, resolved once before either leg runs. - * - * Organization search indexes grant most documents to a single mailbox, channel, or file owner, - * so a member typically reads a vanishing share of the index. Ranking the whole index and - * checking access afterwards then scans thousands of candidates to find none; ranking inside the - * permitted set finds every eligible chunk at a cost proportional to what the member can read. - * `unbounded` means the set exceeded the probe's limit, where post-filtered index search fills - * quickly because most candidates are readable. - */ -export type PermittedDocuments = - | { kind: 'bounded'; documents: readonly PermittedDocument[] } - | { kind: 'unbounded'; broad: boolean } - -/** - * The share of the index a caller must reach before the whole graph is walked for them. pgvector - * post-filters, so a walk returns a caller's own neighbours in proportion to their reach: above - * this share almost every neighbour the graph visits is theirs and one walk is the cheapest exact - * answer there is; below it the walk spends its budget on chunks they cannot read, and each - * readable source is searched on its own instead. - */ -const BROAD_REACH_SHARE = 0.25 - -/** - * How long a caller's saturated reach is remembered. Reach counts the documents a caller's tokens - * touch in the bases, which moves slowly, and an unbounded set only means the legs search the - * index with the full access predicate, so a stale answer costs speed, never access. - */ -const SATURATED_REACH_TTL_MS = 5 * 60 * 1000 - -/** - * A counted reach: whether it is broad enough to walk the whole graph for, or empty, in which - * case the caller reads nothing in these bases and no leg has anything to rank. - */ -interface CountedReach { - broad: boolean - empty: boolean -} - -/** - * Only breadth is remembered. Emptiness decides completeness, not strategy, so it is counted on - * every search: the count of a reach of nothing finds nothing and costs almost nothing. - */ -const saturatedReach = new LRUCache({ - max: 10_000, - ttl: SATURATED_REACH_TTL_MS, -}) - -/** How many documents the bases hold: the denominator of a reach share, and it moves slowly. */ -const indexDocumentCounts = new LRUCache({ - max: 1000, - ttl: SATURATED_REACH_TTL_MS, - /** - * The planner's estimate of the bases' documents, from the statistics it already keeps: a share - * threshold needs the order of magnitude, and counting every row to learn it costs more than the - * search it serves. The read that misses is the search's own, under its deadline. - */ - fetchMethod: async (key, _stale, { context: budget }) => { - const [row] = await runSearchQuery(budget, 'permitted_documents', (executor) => - executor.execute<{ 'QUERY PLAN': Array<{ Plan: { 'Plan Rows': number } }> }>(sql` - EXPLAIN (FORMAT JSON) SELECT 1 FROM ${document} - WHERE ${document.knowledgeBaseId} = ANY(${textArrayLiteral(key.split(','))}) - AND ${document.deletedAt} IS NULL`) - ) - /** An empty answer is not remembered; the bases may simply not have been analyzed yet. */ - return Number(row?.['QUERY PLAN']?.[0]?.Plan?.['Plan Rows'] ?? 0) || undefined - }, -}) - -/** - * The permitted-set probe's SQL, returning at most one row past the document limit. - * - * A user scope first materializes the documents its tokens reach in these bases, read through - * `doc_acl_gin_idx` alone, then applies the state and full access conditions to those rows in - * memory; the set is aliased as `document` so the shared conditions bind to it unchanged. Handed - * the combined predicate instead, PostgreSQL misjudges the token overlap as unselective and - * intersects it with base-wide indexes that read the whole search index. - * - * The index is global and every caller holds the baseline tokens every tenant's org-wide, public, - * and uploaded documents carry, so the reach must be counted inside these bases or those - * documents alone would saturate it. The base check is applied outside an `OFFSET 0` fence so it - * filters the index's rows instead of replacing the index with a base-wide scan. The reach is - * counted before any row is materialized, so a caller whose tokens reach past the limit pays only - * for the count, and a `saturated` sentinel row then reports the set as unbounded. Resolved scopes - * hold base-wide tokens, so they filter directly. - */ -function visibleDocumentsQuery( - knowledgeBaseIds: string[], - conditions: (SQL | undefined)[], - access: UserAccessScope, - shape: 'reach-first' | 'direct' = 'reach-first' -): SQL { - const limit = VECTOR_PROBE_DOCUMENT_LIMIT + 1 - /** - * `direct` applies the conditions as they are: a date filter is selective on its own and has - * its own index, so counting the reach first would only report a broad caller as saturated - * before the filter was consulted. - */ - if (shape === 'direct') return directVisibleDocumentsQuery(conditions) - /** Exactly `doc_acl_gin_idx`'s predicate, so both the count and the rows read that index alone. */ - const reached = sql`${document.deletedAt} IS NULL AND ${knowledgeAclOverlapCondition(access)}` - const underLimit = sql`(SELECT n FROM reach) < ${limit}` - const inBases = inArray(document.knowledgeBaseId, knowledgeBaseIds) - return sql` - WITH reach AS MATERIALIZED ( - SELECT count(*) AS n FROM ( - SELECT 1 FROM ( - SELECT ${document.knowledgeBaseId} FROM ${document} WHERE ${reached} OFFSET 0 - ) AS ${document} - WHERE ${inBases} - LIMIT ${limit} - ) AS reached - ), reachable AS MATERIALIZED ( - SELECT * FROM ( - SELECT * FROM ${document} WHERE ${underLimit} AND ${reached} OFFSET 0 - ) AS ${document} - WHERE ${inBases} - ) - ( - SELECT ${document.id} AS id, ${document.connectorId} AS "connectorId", false AS saturated - FROM reachable AS ${document} - WHERE ${underLimit} AND ${and(...conditions)} - LIMIT ${limit} - ) - UNION ALL - SELECT NULL, NULL, true WHERE (SELECT n FROM reach) >= ${limit} - ` -} - -/** - * The planner's estimate of the documents a filter leaves in the bases — a date filter from the - * statistics on its index, a source filter from its connectors' — so whether the filtered set is - * worth enumerating is decided from its order of magnitude, without reading a row. - */ -export async function estimateFilteredDocuments( - knowledgeBaseIds: string[], - filters: WorkspaceSearchFilters, - plan: SearchAccessPlan, - budget: SearchBudget | undefined -): Promise { - const [row] = await runSearchQuery(budget, 'permitted_documents', (executor) => - executor.execute<{ 'QUERY PLAN': Array<{ Plan: { 'Plan Rows': number } }> }>(sql` - EXPLAIN (FORMAT JSON) SELECT 1 FROM ${document} - WHERE ${and( - inArray(document.knowledgeBaseId, knowledgeBaseIds), - isNull(document.deletedAt), - searchDateFilterCondition(filters), - filters.source ? planSourceCondition(plan) : undefined - )}`) - ) - return Number(row?.['QUERY PLAN']?.[0]?.Plan?.['Plan Rows'] ?? 0) -} - -/** - * How far a caller reaches: broad when they reach at least {@link BROAD_REACH_SHARE} of the - * bases' documents, empty when they reach none. A reach of nothing is a bounded set of nothing: a - * caller who reads no document in these bases, such as a member with no source of their own yet, - * has nothing for any leg to rank, where an unbounded set would have each leg scan to its - * deadline for rows it cannot find. Breadth is counted once against the bound and remembered, so - * the first search after the window pays for it and the rest do not. A caller whose probe already - * saturated is known to reach past the probe's limit, so a bound inside that limit is met without - * counting. - * - * The count reads as many index entries as the caller reaches, so on a large index it can cost - * more than the leg it serves; it gets the probe's share of the deadline, never the whole leg's. - * A count that runs out of that share answers `null`: the leg keeps its time and its deadline - * intact, and the caller decides this search alone without remembering anything. - */ -async function countReach( - knowledgeBaseIds: string[], - access: UserAccessScope, - budget: SearchBudget | undefined, - plan: SearchAccessPlan, - saturated: boolean -): Promise { - const countBudget = budget?.capped(VECTOR_PROBE_BUDGET_MS) - try { - const total = - (await indexDocumentCounts.fetch([...knowledgeBaseIds].sort().join(','), { - context: countBudget, - })) ?? 0 - const bound = Math.ceil(total * BROAD_REACH_SHARE) - if (saturated && bound <= VECTOR_PROBE_DOCUMENT_LIMIT) return { broad: true, empty: false } - const [row] = await runSearchQuery(countBudget, 'permitted_documents', (executor) => - executor.execute<{ n: number }>(sql` - SELECT count(*) AS n FROM ( - SELECT 1 FROM ${document} - WHERE ${and( - isNull(document.deletedAt), - knowledgeAclOverlapCondition(access), - inArray(document.knowledgeBaseId, knowledgeBaseIds), - planSourceCondition(plan) - )} - LIMIT ${bound} - ) reached`) - ) - const reached = Number(row?.n ?? 0) - /** A count that looked and found nothing: only a bound of zero looks at nothing. */ - return { broad: reached >= bound, empty: bound > 0 && reached === 0 } - } catch (error) { - if (!budget || !countBudget?.isTimeout(error)) throw error - /** Only the count's share was spent; the leg's own deadline still governs. */ - budget.remaining() - return null - } -} - -/** - * Reach depends on the bases, the caller's tokens and, when the plan is confined to one kind of - * source, which sources those are; a date filter narrows the set, not the reach. - */ -function reachKey( - knowledgeBaseIds: readonly string[], - access: UserAccessScope, - plan: SearchAccessPlan -): string { - const sources = `:${sha256Hex([...planSources(plan)].sort().join('\n'))}:${plan.uploads}` - return `${[...knowledgeBaseIds].sort().join(',')}:${sha256Hex([...access.tokens].sort().join('\n'))}${sources}` -} - -/** Every connector the plan admits, whatever its access mode. */ -function planSources(plan: SearchAccessPlan): readonly string[] { - return [...plan.connectors.workspace, ...plan.connectors.admin, ...plan.connectors.members] -} - -/** The documents a plan's sources own, on the document row; every source when unconfined. */ -function planSourceCondition(plan: SearchAccessPlan): SQL { - const owned = planSources(plan) - const inSources = owned.length - ? sql`${document.connectorId} = ANY(${textArrayLiteral([...owned])})` - : sql`false` - return plan.uploads ? sql`(${document.connectorId} IS NULL OR ${inSources})` : inSources -} - -/** Forgets every remembered reach, after the bases' documents or a caller's tokens changed. */ -export function forgetSearchReach(): void { - saturatedReach.clear() - indexDocumentCounts.clear() -} - -/** A resolved scope's reach, remembered per bases and tokens, with no document enumerated. */ -export async function resolveReach( - knowledgeBaseIds: string[], - access: UserAccessScope, - budget: SearchBudget | undefined, - plan: SearchAccessPlan -): Promise { - const key = reachKey(knowledgeBaseIds, access, plan) - const remembered = saturatedReach.get(key) - if (remembered) return { kind: 'unbounded', broad: remembered.broad } - try { - const reach = await countReach(knowledgeBaseIds, access, budget, plan, false) - /** A count that ran out of time decides this search only; the next one counts again. */ - if (reach === null) return { kind: 'unbounded', broad: true } - if (reach.empty) return { kind: 'bounded', documents: [] } - saturatedReach.set(key, { broad: reach.broad }) - return { kind: 'unbounded', broad: reach.broad } - } catch (error) { - /** The leg's own deadline passed during the count: the leg is short, the search is not failed. */ - if (!budget?.isTimeout(error)) throw error - return { kind: 'unbounded', broad: true } - } -} - -/** - * Resolve the permitted set with the candidate predicate both legs apply, so restricting a leg - * to it never admits a document the leg would otherwise refuse. Tag filters stay chunk-level in - * each leg; the set is the document-level superset they narrow. - * - * It runs ahead of both legs on the vector leg's budget, so exhausting that budget here reports - * `unbounded` and marks the vector leg timed out rather than failing the keyword leg with it. - */ -export async function resolvePermittedDocuments(params: { - knowledgeBaseIds: string[] - access: UserAccessScope - filters?: WorkspaceSearchFilters - budget?: SearchBudget - accessPlan: SearchAccessPlan - /** Whether a date or source filter narrows the set, which then is enumerated directly. */ - filtered: boolean -}): Promise { - const key = reachKey(params.knowledgeBaseIds, params.access, params.accessPlan) - let probe: ProbeOutcome - let broad = true - /** - * A remembered reach says how much of the bases the caller reads, which a date filter does not - * change; the filtered set still has to be enumerated, so under one the probe always runs. A plan - * under a date or source filter enumerates the filtered set directly; reach cannot stand in for it. - */ - const remembered = params.filtered ? undefined : saturatedReach.get(key) - if (remembered) { - probe = { kind: 'saturated' } - broad = remembered.broad - } else { - try { - probe = await probeVisibleDocuments( - visibleDocumentsQuery( - params.knowledgeBaseIds, - candidateDocumentConditions( - params.knowledgeBaseIds, - params.filters, - knowledgeCandidateAccessConditionForConnectors(params.access, params.accessPlan) - ), - params.access, - params.filtered ? 'direct' : 'reach-first' - ), - params.budget, - 'permitted_documents', - params.filtered ? FILTERED_PROBE_BUDGET_MS : VECTOR_PROBE_BUDGET_MS - ) - } catch (error) { - if (!params.budget?.isTimeout(error)) throw error - probe = { kind: 'timed_out' } - } - if (probe.kind === 'saturated') { - try { - const reach = await countReach( - params.knowledgeBaseIds, - params.access, - params.budget, - params.accessPlan, - true - ) - /** A count that ran out of time decides this search only; the next one counts again. */ - if (reach?.empty) probe = { kind: 'documents', documents: [] } - else if (reach !== null) { - broad = reach.broad - saturatedReach.set(key, { broad }) - } - } catch (error) { - /** The leg's own deadline passed during the count: the leg is short, the search is not failed. */ - if (!params.budget?.isTimeout(error)) throw error - } - } - } - const permitted: PermittedDocuments = - probe.kind === 'documents' - ? { kind: 'bounded', documents: probe.documents } - : { kind: 'unbounded', broad } - annotateSearchDiagnostics({ - permittedDocuments: permitted.kind, - ...(probe.kind === 'documents' ? { permittedDocumentCount: probe.documents.length } : {}), - }) - return permitted -} - -/** - * What a user-scoped search-index search resolves about the caller once, before any leg ranks: - * the connectors it may read and the caller's standing in them, the permitted set or reach, and - * the proof the gated sources still need. - */ -export interface IndexedRetrievalContext { - access: UserAccessScope - accessPlan: SearchAccessPlan - /** Whether a date or source filter narrows the documents; see {@link isSearchFiltered}. */ - filtered: boolean - /** Absent for a tag-only search and for explicit documents, which are already a bounded scope. */ - permitted?: PermittedDocuments - liveSourceAccess?: LiveSourceAccess -} diff --git a/apps/sim/lib/sim-search/indexed/retrieval/projection-access.ts b/apps/sim/lib/sim-search/indexed/retrieval/projection-access.ts deleted file mode 100644 index 284b08e3f61..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/projection-access.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { - document, - knowledgeConnector, - knowledgeDocumentObservation, - knowledgeProjectionDirty, -} from '@sim/db/schema' -import { type SQL, sql } from 'drizzle-orm' -import type { AnyPgColumn } from 'drizzle-orm/pg-core' -import { - aclOverlap, - aclRequirementsSatisfied, - documentHasMirroredAcl, - documentHasWorkspaceAcl, - sourceAclFreshnessCutoff, - textArrayLiteral, -} from '@/lib/knowledge/access/predicate' -import type { UserAccessScope } from '@/lib/knowledge/access/types' -import type { - KnowledgeMemberObserver, - KnowledgeMemberObservers, - SearchAccessPlan, -} from '@/lib/sim-search/indexed/retrieval/access-plan' - -/** - * The candidate predicate with connector state resolved ahead of the query instead of per row. - * - * Deletion, archival, a pending access rewrite, the organization's integration approval and the - * access mode are facts about a connector, not a document, so checking them once per query leaves - * each candidate an id comparison plus its own columns. - * - * `liveSourceAccess` is the caller's live source proof, and defaults to admitting everything: - * candidate ranking defers that proof until after ranking, exactly as - * `knowledgeMetadataCandidateAccessCondition` does, and only a reader that already holds the - * grants — content hydration — passes it. The connectors it would gate are listed separately so - * that clause is applied to those alone. - * - * Either way it narrows exactly as the predicate it stands in for: the eligible ids are the - * connectors that predicate's `EXISTS` would admit, and every document-level clause is carried - * over unchanged. - */ -export function knowledgeCandidateAccessConditionForConnectors( - scope: UserAccessScope, - plan: SearchAccessPlan, - liveSourceAccess: SQL = sql`true` -): SQL { - const eligibility = plan.connectors - if (scope.tokens.length === 0) return sql`false` - const tokens = textArrayLiteral(scope.tokens) - const cutoff = sourceAclFreshnessCutoff() - const liveProof = new Set(eligibility.liveProofRequired) - const inConnectors = (ids: readonly string[]): SQL => - ids.length === 0 - ? sql`false` - : sql`${document.connectorId} = ANY(${textArrayLiteral([...ids])})` - const mirrored = (ids: readonly string[], current: SQL): SQL => { - const direct = ids.filter((id) => !liveProof.has(id)) - const gated = ids.filter((id) => liveProof.has(id)) - const currentAndMirrored = sql`${documentHasMirroredAcl()} AND ${current}` - return sql`( - (${inConnectors(direct)} AND ${currentAndMirrored}) - OR (${inConnectors(gated)} AND ${currentAndMirrored} AND EXISTS ( - SELECT 1 FROM ${knowledgeConnector} - WHERE ${knowledgeConnector.id} = ${document.connectorId} - AND ${liveSourceAccess} - )) - )` - } - const workspaceOwned = plan.uploads - ? sql`(${document.connectorId} IS NULL OR ${inConnectors(eligibility.workspace)})` - : inConnectors(eligibility.workspace) - return sql`( - ${aclOverlap(tokens)} - AND ${aclRequirementsSatisfied(tokens)} - AND ( - (${workspaceOwned} AND ${documentHasWorkspaceAcl()}) - OR ${mirrored(eligibility.admin, sql`${document.aclVerifiedAt} > ${cutoff}`)} - OR ${mirrored(eligibility.members, resolvedObservationCondition(plan.observers, cutoff))} - ) - )` -} - -/** - * Whether a projection row belongs to a document marked for the knowledge projector: its source, - * ACL, or chunks changed and its rows may not show it yet. A probe of the marks' primary key: the - * planner may instead hash the whole set once per statement, which is as cheap while the marks are - * few, and an `IN` would risk re-reading them per row once they outgrow the hash. - */ -export function projectionPending(documentId: AnyPgColumn | SQL): SQL { - return sql`(EXISTS (SELECT 1 FROM ${knowledgeProjectionDirty} WHERE ${knowledgeProjectionDirty.documentId} = ${documentId}))` -} - -/** - * Whether a projection row is decided on its document rather than on its own columns: its - * document is marked for the projector, or, while the source and ACL fill runs, the row has not - * been filled. - */ -export function projectionDecidedOnDocument( - projection: { acl: AnyPgColumn | SQL; documentId: AnyPgColumn | SQL }, - filled: boolean -): SQL { - const pending = projectionPending(projection.documentId) - return filled ? pending : sql`(${projection.acl} IS NULL OR ${pending})` -} - -/** - * The candidate predicate on a ranking projection's own row, for a scope whose connectors were - * resolved: `connectorId` and `acl` are mirrored there from the document, so a walk or a keyword - * window decides readability on the row it scores instead of joining `document` per candidate. - * - * It admits a superset of the document predicate, never a subset: a mirrored ACL names the members - * who observe a document, so overlap with the caller's tokens is the per-row test without the - * observation's freshness, and requirement clauses live on the document. Both are refused there, - * under the full predicate, before content is returned — this predicate only decides what is worth - * ranking. - * - * A row whose columns may be behind its document is decided on the document instead, under - * {@link knowledgeCandidateAccessConditionForConnectors} — the join per candidate that every row - * paid before the columns existed: a row the source and ACL fill has not reached (`acl IS NULL`), - * and every row of a document marked for the knowledge projector. A revoked grant still on such a - * row never admits it, and a new grant not yet on it never hides it from a statement that reaches - * the row. A source-scoped walk or slice reaches rows by the source on the row, though, so a - * document that moved to another source joins that source's ranking once the projector has - * rewritten its rows; until then it can be missing there, never shown where it is not readable. - * The projector and the fill run in the background, so search never waits on either. - */ -export function projectionCandidateAccessCondition( - projection: { - connectorId: AnyPgColumn | SQL - acl: AnyPgColumn | SQL - documentId: AnyPgColumn | SQL - }, - scope: UserAccessScope, - plan: SearchAccessPlan, - options: { - /** - * Whether every row of the projection carries its mirrored source and ACL. While the fill - * is under way, a row it has not reached is decided on its document; once it is complete only - * a marked document's rows are. - */ - filled?: boolean - } = {} -): SQL { - if (scope.tokens.length === 0) return sql`false` - const tokens = textArrayLiteral(scope.tokens) - const inSources = (ids: readonly string[]): SQL => - ids.length === 0 - ? sql`false` - : sql`${projection.connectorId} = ANY(${textArrayLiteral([...ids])})` - const mirrored = [ - ...plan.connectors.workspace, - ...plan.connectors.admin, - ...plan.connectors.members, - ] - const owned = plan.uploads - ? sql`(${projection.connectorId} IS NULL OR ${inSources(mirrored)})` - : inSources(mirrored) - const onRow = sql`(${projection.acl} && ${tokens} AND ${owned})` - /** - * A scalar subquery rather than `EXISTS`: the planner may turn an `EXISTS` into one hash of every - * readable document, a sequential scan of `document` for a statement that only needs a few rows - * decided. A scalar subquery is only ever a primary-key probe per row that needs it. - */ - const onDocument = sql`(SELECT ${document.id} FROM ${document} - WHERE ${document.id} = ${projection.documentId} - AND ${knowledgeCandidateAccessConditionForConnectors(scope, plan)} - LIMIT 1) IS NOT NULL` - return sql`((${projectionDecidedOnDocument(projection, options.filled ?? false)} AND ${onDocument}) - OR (${onRow} AND NOT ${projectionPending(projection.documentId)}))` -} - -/** - * The same membership, resolved ahead of the query: each candidate costs one lookup on the - * observation key instead of a join to the member behind it. Equivalent by construction — the ids - * are the members that join would have matched, and each one's freshness rule is carried over. - */ -function resolvedObservationCondition(observers: KnowledgeMemberObservers, cutoff: SQL): SQL { - if (observers.confirmed.length === 0 && observers.observed.length === 0) return sql`false` - /** - * An observation vouches for a document only from a member of the document's own connector: a - * document that changed hands keeps its old observations, which must not carry it. - */ - const byMember = (members: readonly KnowledgeMemberObserver[]): SQL => - sql`(${knowledgeDocumentObservation.memberId}, ${document.connectorId}) IN (${sql.join( - members.map((member) => sql`(${member.id}, ${member.connectorId})`), - sql`, ` - )})` - const current = - observers.confirmed.length === 0 - ? sql`${byMember(observers.observed)} AND ${knowledgeDocumentObservation.lastSeenAt} > ${cutoff}` - : observers.observed.length === 0 - ? byMember(observers.confirmed) - : sql`(${byMember(observers.confirmed)} - OR (${byMember(observers.observed)} AND ${knowledgeDocumentObservation.lastSeenAt} > ${cutoff}))` - return sql`EXISTS ( - SELECT 1 FROM ${knowledgeDocumentObservation} - WHERE ${knowledgeDocumentObservation.documentId} = ${document.id} - AND ${current} - )` -} - -/** - * The token half of the stored access predicate: the documents a caller's tokens reach before - * any source, freshness, or requirement check narrows them. It is a necessary condition of - * `knowledgeAccessCondition`, never a substitute for it. - * - * Paired with `deleted_at IS NULL` it matches `doc_acl_gin_idx` exactly, so a query can enumerate - * a member's reachable documents from that index alone. PostgreSQL cannot estimate array-overlap - * selectivity, so left to itself it intersects this highly selective bitmap with base-wide ones. - */ -export function knowledgeAclOverlapCondition(scope: UserAccessScope): SQL { - if (scope.tokens.length === 0) return sql`false` - return aclOverlap(textArrayLiteral(scope.tokens)) -} - -/** - * Keeps the rows of sources the caller turned out not to hold out of a ranking decided on the - * row. A row decided on its document — not yet filled, or its document marked for the projector, - * so its own source may be stale — asks the document instead. - */ -export function excludeSearchSourcesOnRow( - projection: { - connectorId: AnyPgColumn | SQL - acl: AnyPgColumn | SQL - documentId: AnyPgColumn | SQL - }, - filled: boolean, - excludedSources: readonly string[] -): SQL | undefined { - if (!excludedSources.length) return undefined - const excluded = textArrayLiteral([...excludedSources]) - const decided = projectionDecidedOnDocument(projection, filled) - return sql`((${decided} AND NOT EXISTS (SELECT 1 FROM ${document} WHERE ${document.id} = ${projection.documentId} AND ${document.connectorId} = ANY(${excluded}))) - OR (NOT ${decided} AND (${projection.connectorId} IS NULL OR NOT (${projection.connectorId} = ANY(${excluded}))))) /* excluded sources */` -} diff --git a/apps/sim/lib/sim-search/indexed/retrieval/projection-fill.test.ts b/apps/sim/lib/sim-search/indexed/retrieval/projection-fill.test.ts deleted file mode 100644 index 98f83e7d799..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/projection-fill.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' -import { SearchBudget } from '@/lib/knowledge/search/budget' -import { - forgetProjectionFilled, - isProjectionFilled, -} from '@/lib/sim-search/indexed/retrieval/projection-fill' - -const LEG_BUDGET_MS = 8000 - -describe('projection fill probe', () => { - beforeEach(() => forgetProjectionFilled()) - - it('remembers a failed probe as unfilled rather than probing again on every search', async () => { - const query = vi - .spyOn(SearchBudget.prototype, 'query') - .mockRejectedValue(new Error('connection reset')) - const budget = new SearchBudget('keyword', performance.now() + LEG_BUDGET_MS) - await expect( - isProjectionFilled('embedding_keyword_tin', 'keyword.projection_filled', budget) - ).resolves.toBe(false) - await expect( - isProjectionFilled('embedding_keyword_tin', 'keyword.projection_filled', budget) - ).resolves.toBe(false) - expect(query).toHaveBeenCalledOnce() - }) - - it('does not hold a later search past its own share while another search probes', async () => { - vi.useFakeTimers() - try { - let answerFirst: (rows: Array<{ unfilled: boolean }>) => void = () => {} - vi.spyOn(SearchBudget.prototype, 'query').mockImplementation( - () => - new Promise((resolve) => { - answerFirst = resolve as typeof answerFirst - }) as ReturnType - ) - const first = isProjectionFilled( - 'embedding_search', - 'vector.projection_filled', - new SearchBudget('vector', performance.now() + LEG_BUDGET_MS) - ) - let secondSettled = false - const second = isProjectionFilled( - 'embedding_search', - 'vector.projection_filled', - new SearchBudget('vector', performance.now() + 20) - ).finally(() => { - secondSettled = true - }) - await vi.advanceTimersByTimeAsync(20) - expect(secondSettled).toBe(true) - await expect(second).resolves.toBe(false) - answerFirst([{ unfilled: false }]) - await expect(first).resolves.toBe(true) - } finally { - vi.useRealTimers() - } - }) - - it('does not remember a probe its own search cancelled', async () => { - const query = vi - .spyOn(SearchBudget.prototype, 'query') - .mockRejectedValue(new DOMException('aborted', 'AbortError')) - const controller = new AbortController() - controller.abort() - const budget = new SearchBudget('vector', performance.now() + LEG_BUDGET_MS, controller.signal) - await expect( - isProjectionFilled('embedding_search', 'vector.projection_filled', budget) - ).resolves.toBe(false) - await expect( - isProjectionFilled('embedding_search', 'vector.projection_filled', budget) - ).resolves.toBe(false) - expect(query).toHaveBeenCalledTimes(2) - }) -}) diff --git a/apps/sim/lib/sim-search/indexed/retrieval/projection-fill.ts b/apps/sim/lib/sim-search/indexed/retrieval/projection-fill.ts deleted file mode 100644 index 04b8a1592ff..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/projection-fill.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { SOURCE_ACL_PROJECTIONS, type SourceAclProjection } from '@sim/db/knowledge-projection' -import { embeddingKeywordTin, embeddingSearch } from '@sim/db/schema' -import { sleep } from '@sim/utils/helpers' -import { sql } from 'drizzle-orm' -import { LRUCache } from 'lru-cache' -import { runSearchQuery, type SearchBudget } from '@/lib/knowledge/search/budget' -import type { SearchStage } from '@/lib/knowledge/search/diagnostics' - -/** How long a fully filled projection is taken on trust before its unfilled rows are looked for again. */ -const PROJECTION_FILLED_TTL_MS = 60_000 - -/** - * The most of a leg's budget the probe may spend. The probe is one index read that answers in - * milliseconds when the partial index serves it; a read slower than this is fighting a cold cache - * or a busy database, and waiting longer would spend the leg's ranking time on an optimization. - * An unanswered probe costs only the slower plan, so a small cap loses nothing. - */ -export const PROJECTION_FILLED_PROBE_BUDGET_MS = 250 - -/** - * How long an unanswered probe is remembered as unfilled. Short enough that a recovered database - * is asked again within seconds, long enough that searches arriving during an outage do not each - * spend their own probe budget rediscovering it. - */ -const PROJECTION_FILLED_UNKNOWN_TTL_MS = 5_000 - -/** - * Whether the ranking projection still holds rows the source and ACL fill has not reached. Read off the - * unfilled-rows index in milliseconds and remembered briefly: the answer only ever changes once. - * - * The read asks for the last unfilled row by id, not whether one exists: an `EXISTS` drops its - * order and limit, and while most rows are unfilled the planner expects a sequential scan to - * meet one at once, then walks the whole projection when the unfilled rows sit past the filled - * ones. Ordered by id and capped at one row, the read can only be the partial index, whose - * last entry is the row the fill reaches last. - */ -const projectionFilled = new LRUCache< - SourceAclProjection, - boolean, - { budget: SearchBudget | undefined; stage: SearchStage } ->({ - max: SOURCE_ACL_PROJECTIONS.length, - ttl: PROJECTION_FILLED_TTL_MS, - /** - * The read that misses the cache is the search's own, capped to a small share of its budget, - * and the searches that miss together share it. A read that fails or runs out of that share - * answers unfilled, the slower and safe form, and that answer is remembered briefly so the - * searches behind it do not each pay for the same failure. A read cut short by its own - * search's cancellation learned nothing about the projection and is not remembered. - */ - fetchMethod: async (projection, _stale, { context, options }) => { - const table = projection === 'embedding_search' ? embeddingSearch : embeddingKeywordTin - try { - const [row] = await runSearchQuery( - context.budget?.capped(PROJECTION_FILLED_PROBE_BUDGET_MS), - context.stage, - (executor) => - executor.execute<{ unfilled: boolean }>(sql` - SELECT ( - SELECT ${table.id} FROM ${table} WHERE ${table.acl} IS NULL - ORDER BY ${table.id} DESC LIMIT 1 - ) IS NOT NULL AS unfilled`) - ) - return !row?.unfilled - } catch { - if (context.budget?.signal?.aborted) return undefined - options.ttl = PROJECTION_FILLED_UNKNOWN_TTL_MS - return false - } - }, -}) - -/** - * Whether every row of the projection carries its mirrored source and ACL; unknown counts as not yet. - * - * Searches that miss the cache together share the first one's read, which is capped to that - * search's share. Each caller still waits no longer than its own share, or its own deadline if - * nearer, and reads an unanswered probe as unfilled: a caller that joined late, with less of its - * leg left, never waits on another search's timetable. A remembered answer is returned at once, - * so only a search that missed the memo starts a wait. - */ -export async function isProjectionFilled( - projection: SourceAclProjection, - stage: SearchStage, - budget: SearchBudget | undefined -): Promise { - const remembered = projectionFilled.get(projection) - if (remembered !== undefined) return remembered - const answer = projectionFilled.fetch(projection, { context: { budget, stage } }) - if (!budget) return (await answer) ?? false - const waitMs = Math.max( - 0, - Math.min(PROJECTION_FILLED_PROBE_BUDGET_MS, budget.deadline - performance.now()) - ) - const unanswered = sleep(waitMs).then(() => undefined) - return (await Promise.race([answer.catch(() => undefined), unanswered])) ?? false -} - -/** Forgets whether the projections were filled; the memo is per process and otherwise expires on its own. */ -export function forgetProjectionFilled(): void { - projectionFilled.clear() -} diff --git a/apps/sim/lib/sim-search/indexed/retrieval/source-vector-indexes.ts b/apps/sim/lib/sim-search/indexed/retrieval/source-vector-indexes.ts deleted file mode 100644 index 165757779fe..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/source-vector-indexes.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { sql } from 'drizzle-orm' -import { LRUCache } from 'lru-cache' -import { runSearchQuery, type SearchBudget } from '@/lib/knowledge/search/budget' - -/** - * The sources that have their own index, cached briefly: every unbounded ranking asks, and the - * answer changes only when a connector deletion drops one. - */ -const indexedSources = new LRUCache<'sources', ReadonlySet>({ max: 1, ttl: 60 * 1000 }) - -/** Forgets the cached answer. */ -export function forgetIndexedVectorSources(): void { - indexedSources.clear() -} - -/** The sources with a graph of their own; a search that misses the memo reads under its own deadline. */ -export async function indexedVectorSources(budget?: SearchBudget): Promise> { - const cached = indexedSources.get('sources') - if (cached) return cached - const rows = await runSearchQuery(budget, 'vector.source_indexes', (executor) => - executor.execute<{ connectorId: string | null }>(sql` - SELECT substring(pg_get_expr(i.indpred, i.indrelid) from '''([0-9a-f-]+)''') AS "connectorId" - FROM pg_index i - JOIN pg_class c ON c.oid = i.indexrelid - WHERE i.indrelid = 'embedding_search'::regclass - AND c.relname LIKE 'embedding_search_src_%' AND i.indisvalid AND i.indisready`) - ) - const sources = new Set( - rows.map((row) => row.connectorId).filter((id): id is string => id !== null) - ) - indexedSources.set('sources', sources) - return sources -} diff --git a/apps/sim/lib/sim-search/indexed/retrieval/tin-keyword-readiness.test.ts b/apps/sim/lib/sim-search/indexed/retrieval/tin-keyword-readiness.test.ts deleted file mode 100644 index d6b1fbece87..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/tin-keyword-readiness.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' -import { expect, it } from 'vitest' -import { resolveTinKeywordQuery } from '@/lib/sim-search/indexed/retrieval/tin-keyword' - -/** Its own file, so the process-wide readiness cache starts empty. */ -it('stays on the GIN projection while the Tin index is incomplete, and remembers that', async () => { - resetDbChainMock() - let indexValid = false - dbChainMockFns.execute.mockImplementation(async (query) => { - const text = JSON.stringify(query) - if (text.includes('indisvalid')) return [{ valid: indexValid }] - if (text.includes('websearch_to_tsquery')) return [{ rendered: "'releas'" }] - return [] - }) - expect(await resolveTinKeywordQuery('release', 'english', undefined)).toBeNull() - indexValid = true - expect(await resolveTinKeywordQuery('release', 'english', undefined)).toBeNull() - const readinessReads = dbChainMockFns.execute.mock.calls.filter(([query]) => - JSON.stringify(query).includes('indisvalid') - ) - expect(readinessReads).toHaveLength(1) -}) diff --git a/apps/sim/lib/sim-search/indexed/retrieval/tin-keyword.test.ts b/apps/sim/lib/sim-search/indexed/retrieval/tin-keyword.test.ts deleted file mode 100644 index 4d6558042c4..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/tin-keyword.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' -import { beforeEach, describe, expect, it } from 'vitest' -import { SearchBudget, SearchDeadlineError } from '@/lib/knowledge/search/budget' -import { resolveTinKeywordQuery } from '@/lib/sim-search/indexed/retrieval/tin-keyword' - -/** Readiness is cached per process; the incomplete-index case lives in its own file, where the cache starts empty. */ -describe('resolveTinKeywordQuery', () => { - let indexValid: boolean - let rendered: string - - beforeEach(() => { - resetDbChainMock() - indexValid = true - rendered = "'releas' & 'note'" - dbChainMockFns.execute.mockImplementation(async (query) => { - const text = JSON.stringify(query) - if (text.includes('indisvalid')) return indexValid ? [{ valid: true }] : [{ valid: false }] - if (text.includes('websearch_to_tsquery')) return [{ rendered }] - return [] - }) - }) - - it('translates the analyzed query', async () => { - expect(await resolveTinKeywordQuery('release notes', 'english', undefined)).toBe( - '("releas" AND "note")' - ) - }) - - it('reads under the keyword budget, so an expired deadline ends the leg instead of querying', async () => { - const expired = new SearchBudget('keyword', performance.now() - 1) - await expect( - resolveTinKeywordQuery('release notes', 'english', expired) - ).rejects.toBeInstanceOf(SearchDeadlineError) - expect(dbChainMockFns.execute).not.toHaveBeenCalled() - }) - - it('falls back to GIN instead of failing the search when the query cannot be analyzed', async () => { - dbChainMockFns.execute.mockRejectedValue(new Error('connection reset')) - expect(await resolveTinKeywordQuery('release notes', 'english', undefined)).toBeNull() - }) -}) diff --git a/apps/sim/lib/sim-search/indexed/retrieval/tin-keyword.ts b/apps/sim/lib/sim-search/indexed/retrieval/tin-keyword.ts deleted file mode 100644 index 37360aae927..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/tin-keyword.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { EMBEDDING_KEYWORD_TIN_INDEX } from '@sim/db/schema' -import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' -import { sql } from 'drizzle-orm' -import { LRUCache } from 'lru-cache' -import { runSearchQuery, type SearchBudget } from '@/lib/knowledge/search/budget' -import { tinQueryFromTsquery } from '@/lib/sim-search/indexed/retrieval/tin-query' - -const logger = createLogger('TinKeywordSearch') - -/** - * How long a readiness answer holds. The index only becomes valid once the projection is fully - * backfilled, and a newly valid or dropped index is noticed within this window. - */ -const READINESS_TTL_MS = 60 * 1000 - -/** - * Whether the Tin index exists and finished building, i.e. the projection is complete. The read - * spends the budget of the search that missed the cache. - */ -const indexReadiness = new LRUCache<'index', boolean, SearchBudget | undefined>({ - max: 1, - ttl: READINESS_TTL_MS, - fetchMethod: async (_key, _stale, { context }) => { - const [row] = await runSearchQuery(context, 'keyword.tin_readiness', (executor) => - executor.execute<{ valid: boolean }>(sql` - SELECT i.indisvalid AS valid FROM pg_index i - WHERE i.indexrelid = to_regclass(${EMBEDDING_KEYWORD_TIN_INDEX})`) - ) - return row?.valid === true - }, -}) - -/** - * The TINQL query that ranks `query` inside a search index's bases, or null when keyword search - * must keep the GIN projection: the database has no complete Tin index, or the query uses a shape - * TINQL cannot express. The text is analyzed by the same `websearch_to_tsquery` the GIN path - * uses, so both engines match the same stemmed terms. - * - * Every read runs under the keyword leg's `budget`, so deciding the engine cannot outlast the leg's - * deadline. A read that fails for another reason, including a shared cache read cut short by - * another search's deadline, keeps the GIN projection; this search's own expired deadline or - * cancellation propagates like any other keyword query's. - */ -export async function resolveTinKeywordQuery( - query: string, - ftsConfig: string, - budget: SearchBudget | undefined -): Promise { - try { - if (!(await indexReadiness.fetch('index', { context: budget }))) return null - const [{ rendered }] = await runSearchQuery(budget, 'keyword.tin_query', (executor) => - executor.execute<{ rendered: string }>( - sql`SELECT websearch_to_tsquery(${ftsConfig}::regconfig, ${query})::text AS rendered` - ) - ) - return tinQueryFromTsquery(rendered) - } catch (error) { - budget?.remaining() - logger.warn('Tin keyword readiness check failed; using the GIN projection', { - error: getErrorMessage(error), - }) - return null - } -} diff --git a/apps/sim/lib/sim-search/indexed/retrieval/tin-query.test.ts b/apps/sim/lib/sim-search/indexed/retrieval/tin-query.test.ts deleted file mode 100644 index 472a52d9665..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/tin-query.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { tinQueryFromTsquery } from '@/lib/sim-search/indexed/retrieval/tin-query' - -/** Inputs are `websearch_to_tsquery('english', …)::text` exactly as PostgreSQL renders them. */ -describe('tinQueryFromTsquery', () => { - it('keeps punctuation and reserved words literal', () => { - expect(tinQueryFromTsquery("'user@example.com' & 'https' & '/sim.ai/docs'")).toBe( - '("user@example.com" AND "https" AND "/sim.ai/docs")' - ) - expect(tinQueryFromTsquery("'near' & 'and'")).toBe('("near" AND "and")') - expect(tinQueryFromTsquery("'snake_case' & 'it''s'")).toBe('("snake\\_case" AND "it\'s")') - }) - - it.each([ - ['', 'a query of stopwords only'], - ["!'onlyneg'", 'a lone negation'], - ["!'a' & !'b'", 'a conjunction of negations'], - ["'a' | !'b'", 'a negated disjunct'], - ["'appl':*", 'a prefix match'], - ])('declines %j (%s)', (rendered) => { - expect(tinQueryFromTsquery(rendered)).toBeNull() - }) -}) diff --git a/apps/sim/lib/sim-search/indexed/retrieval/tin-query.ts b/apps/sim/lib/sim-search/indexed/retrieval/tin-query.ts deleted file mode 100644 index aa7b88d5063..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/tin-query.ts +++ /dev/null @@ -1,195 +0,0 @@ -/** - * Translates PostgreSQL's text rendering of a `tsquery` into TINQL, the query language of the Tin - * text index. The keyword leg already derives its `tsquery` with `websearch_to_tsquery`, so the - * lexemes arrive stemmed by the same configuration that stemmed the indexed text; translating - * them keeps Tin matching the same documents the GIN path matches, without re-implementing - * stemming. - * - * Supported shapes are the ones `websearch_to_tsquery` produces: `&`, `|`, `!` applied to a - * lexeme, phrases of lexemes joined by `<->` or ``, and parentheses. TINQL has no standalone - * negation, so a query that is only negated, or negates inside a disjunction, has no translation - * and returns `null`; the caller then keeps the GIN path. - */ - -type Node = - | { kind: 'term'; lexeme: string } - | { kind: 'phrase'; terms: string[]; gaps: number[] } - | { kind: 'not'; operand: Node } - | { kind: 'and'; operands: Node[] } - | { kind: 'or'; operands: Node[] } - -type Token = - | { kind: 'lexeme'; value: string } - | { kind: 'and' | 'or' | 'not' | 'open' | 'close' } - | { kind: 'follow'; distance: number } - -class UntranslatableQuery extends Error {} - -function tokenize(text: string): Token[] { - const tokens: Token[] = [] - let index = 0 - while (index < text.length) { - const char = text[index] - if (char === ' ') { - index++ - } else if (char === "'") { - let value = '' - index++ - for (;;) { - if (index >= text.length) throw new UntranslatableQuery('unterminated lexeme') - if (text[index] === "'" && text[index + 1] === "'") { - value += "'" - index += 2 - } else if (text[index] === "'") { - index++ - break - } else { - value += text[index++] - } - } - /** Weight and prefix suffixes (`:A`, `:*`) change matching and are never emitted by websearch. */ - if (text[index] === ':') throw new UntranslatableQuery('lexeme modifiers') - tokens.push({ kind: 'lexeme', value }) - } else if (char === '&') { - tokens.push({ kind: 'and' }) - index++ - } else if (char === '|') { - tokens.push({ kind: 'or' }) - index++ - } else if (char === '!') { - tokens.push({ kind: 'not' }) - index++ - } else if (char === '(') { - tokens.push({ kind: 'open' }) - index++ - } else if (char === ')') { - tokens.push({ kind: 'close' }) - index++ - } else if (char === '<') { - const end = text.indexOf('>', index) - if (end < 0) throw new UntranslatableQuery('unterminated distance') - const body = text.slice(index + 1, end) - const distance = body === '-' ? 1 : Number(body) - if (!Number.isInteger(distance) || distance < 1) { - throw new UntranslatableQuery('unsupported distance') - } - tokens.push({ kind: 'follow', distance }) - index = end + 1 - } else { - throw new UntranslatableQuery(`unexpected character ${char}`) - } - } - return tokens -} - -/** Recursive descent over tsquery precedence: `|` binds loosest, then `&`, then ``, then `!`. */ -function parse(tokens: Token[]): Node { - let position = 0 - const peek = () => tokens[position] - - function parseOr(): Node { - const operands = [parseAnd()] - while (peek()?.kind === 'or') { - position++ - operands.push(parseAnd()) - } - return operands.length === 1 ? operands[0] : { kind: 'or', operands } - } - - function parseAnd(): Node { - const operands = [parseFollow()] - while (peek()?.kind === 'and') { - position++ - operands.push(parseFollow()) - } - return operands.length === 1 ? operands[0] : { kind: 'and', operands } - } - - function parseFollow(): Node { - const first = parseUnary() - if (peek()?.kind !== 'follow') return first - if (first.kind !== 'term') throw new UntranslatableQuery('phrase over a compound operand') - const terms = [first.lexeme] - const gaps: number[] = [] - for (let token = peek(); token?.kind === 'follow'; token = peek()) { - position++ - const next = parseUnary() - if (next.kind !== 'term') throw new UntranslatableQuery('phrase over a compound operand') - gaps.push(token.distance) - terms.push(next.lexeme) - } - return { kind: 'phrase', terms, gaps } - } - - function parseUnary(): Node { - const token = tokens[position++] - if (!token) throw new UntranslatableQuery('unexpected end') - if (token.kind === 'not') return { kind: 'not', operand: parseUnary() } - if (token.kind === 'lexeme') return { kind: 'term', lexeme: token.value } - if (token.kind === 'open') { - const inner = parseOr() - if (tokens[position++]?.kind !== 'close') throw new UntranslatableQuery('unbalanced group') - return inner - } - throw new UntranslatableQuery(`unexpected ${token.kind}`) - } - - const root = parseOr() - if (position !== tokens.length) throw new UntranslatableQuery('trailing input') - return root -} - -/** A lexeme is always quoted, so reserved words and punctuation stay literal terms. */ -function quote(lexeme: string): string { - return `"${lexeme.replace(/[\\"_[\]]/g, (char) => `\\${char}`)}"` -} - -function render(node: Node): string { - switch (node.kind) { - case 'term': - return quote(node.lexeme) - case 'phrase': - /** - * Adjacent lexemes form a phrase. A wider gap marks stopwords the analyzer removed; the - * indexed stream omits them too, so the gap becomes ordered proximity within that distance. - */ - if (node.gaps.every((gap) => gap === 1)) { - return `"${node.terms.map((term) => quote(term).slice(1, -1)).join(' ')}"` - } - return `(${node.terms - .map((term, index) => - index === 0 ? quote(term) : `THEN/${node.gaps[index - 1]} ${quote(term)}` - ) - .join(' ')})` - case 'not': - throw new UntranslatableQuery('negation outside a conjunction') - case 'and': { - const positive = node.operands.filter((operand) => operand.kind !== 'not') - const negative = node.operands.filter( - (operand): operand is Extract => operand.kind === 'not' - ) - if (positive.length === 0) throw new UntranslatableQuery('conjunction of negations only') - return `(${[ - positive.map(render).join(' AND '), - ...negative.map((operand) => `NOT ${render(operand.operand)}`), - ].join(' AND ')})` - } - case 'or': - return `(${node.operands.map(render).join(' OR ')})` - } -} - -/** - * The TINQL equivalent of a rendered `tsquery`, or `null` when the query has no lexemes or uses a - * shape TINQL cannot express. - */ -export function tinQueryFromTsquery(rendered: string): string | null { - const text = rendered.trim() - if (!text) return null - try { - return render(parse(tokenize(text))) - } catch (error) { - if (error instanceof UntranslatableQuery) return null - throw error - } -} diff --git a/apps/sim/lib/sim-search/indexed/retrieval/vector.ts b/apps/sim/lib/sim-search/indexed/retrieval/vector.ts deleted file mode 100644 index 9ccb72b1fd9..00000000000 --- a/apps/sim/lib/sim-search/indexed/retrieval/vector.ts +++ /dev/null @@ -1,491 +0,0 @@ -import { document, embeddingSearch } from '@sim/db/schema' -import { and, eq, inArray, type SQL, sql } from 'drizzle-orm' -import type { AnyPgColumn } from 'drizzle-orm/pg-core' -import { mapWithConcurrency } from '@/lib/core/utils/concurrency' -import { knowledgeAccessCondition, textArrayLiteral } from '@/lib/knowledge/access/predicate' -import type { UserAccessScope } from '@/lib/knowledge/access/types' -import { runSearchQuery, type SearchBudget } from '@/lib/knowledge/search/budget' -import { - excludeSearchSources, - getVisibilityConditions, - type SearchParams, - type SearchReadCandidate, - type SearchResult, - SOURCE_RANKING_CONCURRENCY, - selectAuthorizedSearchResults, -} from '@/lib/knowledge/search/candidates' -import { annotateSearchDiagnostics } from '@/lib/knowledge/search/diagnostics' -import { searchDateFilterCondition } from '@/lib/knowledge/search/filter-conditions' -import { - annotateVectorPoolPlanned, - annotateVectorPoolSelected, - CANDIDATE_HNSW_MAX_SCAN_TUPLES, - gatheredVectorCandidatePool, - hydrateVectorCandidates, - prepareVectorLeg, - rankVectorCandidatesExactly, - readVectorCandidatePool, - selectExactVectorPage, - sliceVectorCandidatePool, - type VectorCandidatePool, - withVectorScanSettings, -} from '@/lib/knowledge/search/vector-leg' -import type { SearchAccessPlan } from '@/lib/sim-search/indexed/retrieval/access-plan' -import { - documentSatisfies, - type IndexedRetrievalContext, - PERMITTED_EXACT_DOCUMENT_LIMIT, -} from '@/lib/sim-search/indexed/retrieval/permitted' -import { - excludeSearchSourcesOnRow, - knowledgeCandidateAccessConditionForConnectors, - projectionCandidateAccessCondition, - projectionPending, -} from '@/lib/sim-search/indexed/retrieval/projection-access' -import { isProjectionFilled } from '@/lib/sim-search/indexed/retrieval/projection-fill' -import { indexedVectorSources } from '@/lib/sim-search/indexed/retrieval/source-vector-indexes' - -/** - * How far a walk that decides readability on the row may go before giving up: a cap, not a target, - * since the scan stops as soon as the limit is met. The default cap was sized for a walk that looked - * a document up per visited tuple; on the row a tuple costs a fraction of that, so a caller whose - * neighbourhood is mostly unreadable can be carried past it for tens of milliseconds rather than - * left with what the neighbourhood happened to hold. - */ -const ON_ROW_WALK_SCAN_TUPLES = 100_000 - -/** - * How far a walk may go when readability is on the row: the on-row cap, unless the walk still - * has to ask the document about tuples — a tag or date filter, or rows the source and ACL fill - * has not reached yet — in which case such a tuple costs what it did before the columns were mirrored, - * and the default cap keeps a walk through a mostly-excluded neighbourhood at a short answer - * rather than a missed deadline. - */ -function onRowWalkScanTuples( - documentCondition: SQL | undefined, - projectionFilled: boolean -): number { - return documentCondition === undefined && projectionFilled - ? ON_ROW_WALK_SCAN_TUPLES - : CANDIDATE_HNSW_MAX_SCAN_TUPLES -} - -/** - * A candidate's source: the row's, unless the row's document is marked for the projector, whose - * source may have moved since the row was written — then the document's, read for that row only. - */ -function projectionCandidateSource(projection: { - connectorId: AnyPgColumn | SQL - documentId: AnyPgColumn | SQL -}): SQL { - return sql`CASE WHEN ${projectionPending(projection.documentId)} - THEN (SELECT ${document.connectorId} FROM ${document} WHERE ${document.id} = ${projection.documentId}) - ELSE ${projection.connectorId} END` -} - -/** - * The same identities read off a projection row in raw SQL: the aliases are what - * `SearchReadCandidate` deserializes, so every walk reads them from one place. - */ -const PROJECTION_CANDIDATE_COLUMNS = sql`${embeddingSearch.id} AS id, ${embeddingSearch.documentId} AS "documentId", ${projectionCandidateSource(embeddingSearch)} AS "connectorId"` - -/** - * How many chunks the sliced sources contribute to exact ranking. A caller's slice of mirrored - * sources — their mail, their files, the spaces they belong to — sits below this, and ranking that - * many exactly, on the projection's half-precision vectors, measures in tens of milliseconds. - */ -const SOURCE_EXACT_CHUNK_LIMIT = 150_000 - -/** Sources whose own index a caller's ranking walks, and whether anything is left to rank exactly. */ -interface SourceVectorPlan { - walked: readonly string[] - sliced: readonly string[] -} - -/** - * How each readable source contributes its nearest chunks. - * - * Membership decides it, not a count: a member of a source reads essentially all of it, so its own - * index is walked and the graph's neighbours are chunks they can read. Every other source is - * sliced — mirrored permissions give a caller their own mail, their own files — and those slices - * are ranked exactly together, which is cheaper than a walk and exact by construction. A source - * the caller is a member of but which has no index of its own is sliced too. - */ -function planSourceVectorCandidates(input: { - plan: SearchAccessPlan - indexedSources: ReadonlySet -}): SourceVectorPlan { - const eligible = [ - ...new Set([ - ...input.plan.connectors.workspace, - ...input.plan.connectors.admin, - ...input.plan.connectors.members, - ]), - ] - const walked = input.plan.memberSources.filter((id) => input.indexedSources.has(id)) - const walking = new Set(walked) - return { walked, sliced: eligible.filter((id) => !walking.has(id)) } -} - -/** - * The nearest readable chunks, gathered per source and merged by distance. - * - * Walking one source at a time is what keeps recall: pgvector post-filters, so a walk over every - * source spends its scan budget on the sources this caller cannot read and returns few of their - * true neighbours. Inside one source they read, almost every neighbour qualifies. - * - * Nothing but the merged identities crosses the wire — each source's readable documents are - * resolved inside its own statement. - */ -async function selectSourceVectorCandidates(input: { - access: UserAccessScope - knowledgeBaseIds: string[] - plan: SearchAccessPlan - tagCondition: SQL | undefined - documentCondition: SQL | undefined - /** Sources the caller turned out not to hold, kept out of every source's ranking. */ - exclusion: SQL | undefined - /** Whether every projection row carries its mirrored columns, so a walk needs no document. */ - projectionFilled: boolean - candidateDistance: SQL - candidateLimit: number - budget?: SearchBudget -}): Promise { - const sources = planSourceVectorCandidates({ - plan: input.plan, - indexedSources: await indexedVectorSources(input.budget), - }) - annotateSearchDiagnostics({ - vectorRanking: 'per-source', - vectorSourcesWalked: sources.walked.length, - vectorSourcesSliced: sources.sliced.length, - }) - const base = and( - inArray(embeddingSearch.knowledgeBaseId, input.knowledgeBaseIds), - eq(embeddingSearch.enabled, true), - input.tagCondition, - input.exclusion - ) - type RankedChunks = Promise> - /** - * Walks one source's own index, or the sliced sources together when their slice saturated. - * Readability is decided on the row the walk visits — the source and ACL are mirrored there — - * so the graph is not stalled by a document lookup per candidate; the tag filter, which lives on - * the chunk, still joins. - */ - const onRow = projectionCandidateAccessCondition(embeddingSearch, input.access, input.plan, { - filled: input.projectionFilled, - }) - const walk = - (scope: SQL): (() => RankedChunks) => - () => - withVectorScanSettings( - (executor) => - executor.execute(sql` - SELECT ${PROJECTION_CANDIDATE_COLUMNS}, ${input.candidateDistance} AS distance - FROM ${embeddingSearch} /* on-row visibility */ - WHERE ${and( - base, - scope, - onRow, - documentSatisfies(embeddingSearch.documentId, input.documentCondition) - )} - ORDER BY ${input.candidateDistance} LIMIT ${input.candidateLimit}`), - input.budget, - 'vector.source_walk', - onRowWalkScanTuples(input.documentCondition, input.projectionFilled) - ) - const walks: Array<() => RankedChunks> = sources.walked.map((connectorId) => - walk(eq(embeddingSearch.connectorId, connectorId)) - ) - const slicedScope = sql`(${embeddingSearch.connectorId} IS NULL - OR ${embeddingSearch.connectorId} = ANY(${textArrayLiteral([...sources.sliced])}))` - /** - * One statement for the sliced sources and, with them, every uploaded document: uploads carry no - * connector, so a caller who is a member of all the indexed sources would otherwise rank none. - */ - const slice: Array<() => RankedChunks> = [ - async () => { - /** - * The sliced sources' readable chunks, decided on the row, ranked exactly: the ACL index - * enumerates them and `+ 0` keeps the planner off the graph. The chunks are counted one past - * the bound in the same statement, so a set too large to rank exactly is known before it is. - */ - const readableChunks = and( - base, - slicedScope, - onRow, - documentSatisfies(embeddingSearch.documentId, input.documentCondition) - ) - const rows = await runSearchQuery(input.budget, 'vector.source_exact', (executor) => - executor.execute(sql` - WITH readable_chunks AS MATERIALIZED ( - SELECT ${PROJECTION_CANDIDATE_COLUMNS}, ${input.candidateDistance} AS distance - FROM ${embeddingSearch} - WHERE ${readableChunks} - LIMIT ${SOURCE_EXACT_CHUNK_LIMIT + 1} - ) - SELECT id, "documentId", "connectorId", distance + 0 AS distance, - (SELECT count(*) FROM readable_chunks) > ${SOURCE_EXACT_CHUNK_LIMIT} AS saturated - FROM readable_chunks - ORDER BY distance LIMIT ${input.candidateLimit}`) - ) - /** - * The slice enumerates readable chunks in no particular order, so a set past its bound - * would rank an arbitrary subset and could miss the nearest chunks entirely. Walk those - * sources instead: approximate, but drawn from the whole of them. - */ - if (!rows.some((row) => row.saturated)) return rows - annotateSearchDiagnostics({ vectorSlicedSaturated: true }) - return walk(slicedScope)() - }, - ] - /** A source whose search runs out of budget marks the leg partial; the others' results stand. */ - const scored = await mapWithConcurrency( - [...walks, ...slice], - SOURCE_RANKING_CONCURRENCY, - async (run) => { - try { - return await run() - } catch (error) { - if (!input.budget?.isTimeout(error)) throw error - return [] - } - } - ) - const ranked: Array = scored.flat() - return ranked - .sort((a, b) => Number(a.distance) - Number(b.distance)) - .slice(0, input.candidateLimit) -} - -/** - * The vector leg of a user-scoped search-index search: a bounded candidate pool decided on the - * projection row through the caller's resolved plan, then hydrated under the full read predicate. - * - * A bounded permitted set is ranked exactly; a reader who is a member of indexed sources has each - * walked on its own; a broad reader walks the whole graph once. Live source authorization and - * content hydration still run after candidate ranking. - */ -export async function selectIndexedVectorResults( - params: SearchParams, - context: IndexedRetrievalContext -): Promise { - const setup = prepareVectorLeg(params) - const { access, accessPlan: plan, permitted } = context - /** Only live-verified readers may defer source authorization until after candidate ranking. */ - const candidateAccess = knowledgeCandidateAccessConditionForConnectors(access, plan) - /** - * What an on-row walk still has to ask the document: the tags, which live on chunks, and the - * date filter, which the row does not carry. A bounded set never walks, so this only runs when - * the filtered documents were too many to enumerate. - */ - const dateCondition = searchDateFilterCondition(params.filters) - const documentCondition = - setup.documentTagCondition || dateCondition - ? and(setup.documentTagCondition, dateCondition) - : undefined - /** `filled`: whether the pool's rows carry their source, so a page needs no read of its own. */ - let candidatePool: (VectorCandidatePool & { filled: boolean }) | undefined - return selectAuthorizedSearchResults({ - leg: 'vector', - access: params.access, - liveSourceAccess: context.liveSourceAccess, - signal: params.signal, - budget: params.budget, - topK: params.topK, - compareResults: (a, b) => a.distance - b.distance, - selectPage: async (limit, offset, excludedSources) => { - if (params.filters?.documentIds?.length) { - return selectExactVectorPage( - setup, - params.budget, - [ - ...getVisibilityConditions(params.filters, candidateAccess), - excludeSearchSources(excludedSources), - ], - limit, - offset - ) - } - if (permitted?.kind === 'bounded' && permitted.documents.length === 0) - return { candidates: [], nextOffset: offset } - candidatePool = await readVectorCandidatePool( - candidatePool, - excludedSources, - offset, - limit, - async ({ excludedKey, candidateLimit, previous: previousPool }) => { - /** Two remembered facts, read together when neither is remembered. */ - const [filled, plannedIndexedSources] = await Promise.all([ - isProjectionFilled('embedding_search', 'vector.projection_filled', params.budget), - plan.memberSources.length ? indexedVectorSources(params.budget) : undefined, - ]) - /** - * A source the caller turned out not to hold is left out where the pool is built: the - * pool is the page's order now, so a denied source's chunks would otherwise keep their - * slots. The row's mirrored source decides it, unless the row is decided on its document. - */ - const excludedOnRow = excludeSearchSourcesOnRow(embeddingSearch, filled, excludedSources) - annotateVectorPoolPlanned(setup, candidateLimit) - /** - * Exact ranking reads what the permitted set costs rather than re-deriving permission - * across the whole index, and honours `statement_timeout`, which a traversal cannot. - * `read` are chunks a pool already holds, ranked past. - */ - const rankPermittedExactly = (documentIds: string[], read?: readonly string[]) => - rankVectorCandidatesExactly({ - setup, - knowledgeBaseIds: params.knowledgeBaseIds, - documentIds, - columns: PROJECTION_CANDIDATE_COLUMNS, - conditions: [ - read?.length - ? sql`NOT (${embeddingSearch.id} = ANY(${textArrayLiteral([...read])}))` - : undefined, - excludedOnRow, - ], - candidateLimit, - budget: params.budget, - }) - let selected: SearchReadCandidate[] - /** Set where a pool's end is known better than by its length. */ - let exhausted: boolean | undefined - const walkGraph = () => - withVectorScanSettings( - (executor) => - executor.execute(sql` - SELECT ${PROJECTION_CANDIDATE_COLUMNS} - FROM ${embeddingSearch} /* on-row visibility */ - WHERE ${and( - inArray(embeddingSearch.knowledgeBaseId, params.knowledgeBaseIds), - eq(embeddingSearch.enabled, true), - excludedOnRow, - projectionCandidateAccessCondition(embeddingSearch, access, plan, { filled }), - documentSatisfies(embeddingSearch.documentId, documentCondition) - )} - ORDER BY ${setup.distance} LIMIT ${candidateLimit} - `), - params.budget, - 'vector.candidate_search', - onRowWalkScanTuples(documentCondition, filled) - ) - /** - * A source the caller is a member of that has its own index is walked on its own, which - * beats ranking it exactly once it is large enough to have earned that index. - */ - const walksASource = plan.memberSources.some( - (id) => plannedIndexedSources?.has(id) ?? false - ) - if ( - permitted?.kind === 'bounded' && - filled && - permitted.documents.length >= PERMITTED_EXACT_DOCUMENT_LIMIT - ) { - /** - * A set this large costs more to rank exactly than to walk: exact ranking reads every - * chunk of every document in it, while the walk decides readability on the rows it - * visits and stops at its tuple cap. The walk answers whenever the set is a fair share - * of the graph; where it is not, the walk underfills and the exact ranking that was - * always complete takes over, so nothing is lost but the walk's bounded cost. - * - * The walk decides readability on the projection row, which is broader than the - * document predicate hydration applies, so a pool it filled can still run short of - * readable rows. That shortfall is what refills a pool: the refill is the exact ranking, - * complete over the set, ranked past the rows already read and placed behind them, so - * the pages keep their offsets and every refill is a full window of fresh rows. - */ - const permittedIds = permitted.documents.map((entry) => entry.id) - const previous = previousPool?.ids - if (previous) { - const exact = await rankPermittedExactly( - permittedIds, - previous.map((candidate) => candidate.id) - ) - selected = [...previous, ...exact] - exhausted = exact.length < candidateLimit - } else { - selected = await walkGraph() - if (selected.length < candidateLimit) - selected = await rankPermittedExactly(permittedIds) - } - } else if (permitted?.kind === 'bounded' && (!walksASource || context.filtered)) { - /** - * A bounded permitted set is ranked exactly without walking the graph first: the walk - * post-filters, so when the caller reads a small share of the index it spends its whole - * uninterruptible tuple budget and still returns almost none of their neighbours. A - * member's indexed source is otherwise walked instead, but not under a filter: the walk - * cannot see the date, and a filtered set is small by construction. - */ - selected = await rankPermittedExactly(permitted.documents.map((entry) => entry.id)) - } else if (!(permitted?.kind === 'unbounded' && permitted.broad)) { - /** - * Readability follows sources, so each readable source is searched in its own index and - * the results merged. A member reads a source whole or barely at all: walking one source - * spends its budget among chunks they can read, where a walk over every source spends it - * on the sources they cannot. A caller whose reach is broad skips this: for them the - * whole graph's neighbours are mostly theirs already, and one walk is the cheaper answer. - */ - selected = await selectSourceVectorCandidates({ - access, - knowledgeBaseIds: params.knowledgeBaseIds, - plan, - exclusion: excludedOnRow, - projectionFilled: filled, - tagCondition: setup.candidateTagCondition, - documentCondition, - candidateDistance: setup.distance, - candidateLimit, - budget: params.budget, - }) - } else { - /** - * A broad reader's nearest chunks are mostly theirs, so one walk over the whole graph, - * deciding readability on the row, is the cheapest exact answer there is. - */ - selected = await walkGraph() - } - const pool = { - ...gatheredVectorCandidatePool(excludedKey, selected, candidateLimit, exhausted), - filled, - } - annotateVectorPoolSelected(selected.length, candidateLimit) - return pool - } - ) - /** - * The walk carries each candidate's document and source, so a page is a slice of the pool. - * Rescoring the pool against the original vectors here read one out-of-line vector per - * candidate from storage no cache holds, seconds on a query nobody had run before; the page - * is scored at hydration instead. - */ - if (candidatePool.filled) return sliceVectorCandidatePool(candidatePool, offset, limit) - /** - * While the source and ACL fill runs, a row it has not reached carries no source, so the - * page's identities are read off the documents; a slice whose documents all went away since - * the walk is passed over, not mistaken for the pool's end. - */ - for (let start = offset; start < candidatePool.ids.length; start += limit) { - const slice = candidatePool.ids.slice(start, start + limit) - const ranked = new Map(slice.map((candidate, index) => [candidate.id, index])) - const identities = await runSearchQuery(params.budget, 'vector.page', (executor) => - executor.execute(sql` - SELECT ${embeddingSearch.id} AS id, ${document.id} AS "documentId", - ${document.connectorId} AS "connectorId" - FROM ${embeddingSearch} - INNER JOIN ${document} ON ${document.id} = ${embeddingSearch.documentId} - WHERE ${embeddingSearch.id} = ANY(${textArrayLiteral(slice.map((candidate) => candidate.id))}) - `) - ) - if (!identities.length) continue - const page = [...identities].sort( - (a, b) => (ranked.get(a.id) ?? 0) - (ranked.get(b.id) ?? 0) - ) - return { candidates: page, nextOffset: start + slice.length } - } - return { candidates: [], nextOffset: candidatePool.ids.length } - }, - hydrate: (ids, authorized) => - hydrateVectorCandidates(ids, knowledgeAccessCondition(authorized), setup, params), - }) -} diff --git a/apps/sim/lib/sim-search/indexed/search/scoped-search.activity.test.ts b/apps/sim/lib/sim-search/indexed/search/scoped-search.activity.test.ts deleted file mode 100644 index 612f5617545..00000000000 --- a/apps/sim/lib/sim-search/indexed/search/scoped-search.activity.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { member } from '@sim/db/schema' -import { queueTableRows, resetDbChainMock } from '@sim/testing' -import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' -import { - knowledgeAvailabilityMock, - knowledgeAvailabilityMockFns, -} from '@sim/testing/mocks/knowledge-availability.mock' -import { - knowledgeContextsMock, - knowledgeContextsMockFns, -} from '@sim/testing/mocks/knowledge-contexts.mock' -import { - knowledgeSearchUseCaseMock, - knowledgeSearchUseCaseMockFns, -} from '@sim/testing/mocks/knowledge-search-use-case.mock' -import { - permissionGroupsResolveMock, - permissionGroupsResolveMockFns, -} from '@sim/testing/mocks/permission-groups-resolve.mock' -import { workspaceAuthzMock } from '@sim/testing/mocks/workspace-authz.mock' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -const hoisted = vi.hoisted(() => ({ - findIndex: vi.fn(), - activity: vi.fn(), -})) -vi.mock('@/lib/knowledge/application/contexts', () => knowledgeContextsMock) -vi.mock('@/lib/permission-groups/resolve.server', () => permissionGroupsResolveMock) -vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) -vi.mock('@/lib/knowledge/search/search-index', () => ({ - findSearchIndex: hoisted.findIndex, -})) -vi.mock('@/lib/knowledge/access/availability', () => knowledgeAvailabilityMock) -vi.mock('@/lib/knowledge/search/activity', () => ({ - recordOrganizationSearchActivity: hoisted.activity, -})) -vi.mock('@/lib/knowledge/application/search', () => knowledgeSearchUseCaseMock) - -import { - searchOrganizationKnowledge, - searchScopedKnowledge, -} from '@/lib/sim-search/indexed/search/scoped-search' - -const mocks = { - ...hoisted, - afterSearch: knowledgeSearchUseCaseMockFns.mockAfterKnowledgeSearch, - search: knowledgeSearchUseCaseMockFns.mockRunKnowledgeSearch, -} -mocks.afterSearch.mockImplementation(async () => undefined) - -knowledgeContextsMockFns.mockResolveKnowledgeOwnerContext.mockImplementation((...args: unknown[]) => - knowledgeContextsMockFns.mockResolveKnowledgeOrganizationContext(...args) -) -knowledgeContextsMockFns.mockResolveKnowledgeWorkspaceContext.mockImplementation( - (...args: unknown[]) => knowledgeContextsMockFns.mockResolveKnowledgeOrganizationContext(...args) -) - -const principal = createSessionPrincipal({ userId: 'reader', sessionId: 'session' }) -const input = { organizationId: 'org', query: 'policy', topK: 20, surface: 'slack' } as const - -/** These use cases run only while indexed organization search is on. */ -beforeEach(() => setEnvFlags({ isLiveEnterpriseSearchEnabled: false })) -afterEach(resetEnvFlagsMock) - -beforeEach(() => { - resetDbChainMock() - knowledgeContextsMockFns.mockResolveKnowledgeOrganizationContext.mockResolvedValue({ - organizationId: 'org', - }) - permissionGroupsResolveMockFns.mockGetUserPermissionConfigForOrganization.mockResolvedValue(null) - mocks.findIndex.mockResolvedValue(null) - knowledgeAvailabilityMockFns.mockRequireOrganizationSearchAvailable.mockResolvedValue(undefined) - mocks.activity.mockResolvedValue(undefined) - mocks.search.mockResolvedValue({ - results: [], - knowledgeBases: [{ id: 'index' }], - knowledgeBaseId: 'index', - }) -}) - -describe.each([ - { name: 'organization Assistant', operation: searchOrganizationKnowledge }, - { name: 'scoped Search', operation: searchScopedKnowledge }, -])('$name activity before an index exists', ({ operation }) => { - it('records an authorized empty invocation for the acting member', async () => { - queueTableRows(member, [{ role: 'member' }]) - expect(await operation.execute({ principal, input })).toEqual({ - results: [], - retrieval: { status: 'complete', timedOutLegs: [] }, - query: 'policy', - knowledgeBases: [], - }) - expect( - knowledgeAvailabilityMockFns.mockRequireOrganizationSearchAvailable - ).toHaveBeenCalledExactlyOnceWith('org') - expect(mocks.activity).toHaveBeenCalledExactlyOnceWith({ - organizationId: 'org', - userId: 'reader', - surface: 'slack', - results: [], - }) - expect(mocks.search).not.toHaveBeenCalled() - }) - - it('does not meter an unavailable Search request', async () => { - queueTableRows(member, [{ role: 'member' }]) - knowledgeAvailabilityMockFns.mockRequireOrganizationSearchAvailable.mockRejectedValueOnce( - new Error('Search is disabled') - ) - await expect(operation.execute({ principal, input })).rejects.toThrow('Search is disabled') - expect(mocks.activity).not.toHaveBeenCalled() - expect(mocks.search).not.toHaveBeenCalled() - }) - - it('does not discover the index or meter a nonmember request', async () => { - queueTableRows(member, []) - await expect(operation.execute({ principal, input })).rejects.toMatchObject({ - code: 'not_found', - }) - expect(mocks.findIndex).not.toHaveBeenCalled() - expect(mocks.activity).not.toHaveBeenCalled() - }) - - it('does not meter a request that was already cancelled', async () => { - queueTableRows(member, [{ role: 'member' }]) - const controller = new AbortController() - controller.abort(new Error('Search cancelled')) - await expect( - operation.execute({ principal, input: { ...input, signal: controller.signal } }) - ).rejects.toThrow('Search cancelled') - expect(mocks.activity).not.toHaveBeenCalled() - expect(mocks.search).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/sim-search/indexed/search/scoped-search.test.ts b/apps/sim/lib/sim-search/indexed/search/scoped-search.test.ts deleted file mode 100644 index 70891390be9..00000000000 --- a/apps/sim/lib/sim-search/indexed/search/scoped-search.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { - dbChainMockFns, - hasMockCondition, - queueTableRows, - resetDbChainMock, - schemaMock, -} from '@sim/testing' -import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' -import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing/mocks/env-flags.mock' -import { - knowledgeContextsMock, - knowledgeContextsMockFns, -} from '@sim/testing/mocks/knowledge-contexts.mock' -import { - knowledgeSearchUseCaseMock, - knowledgeSearchUseCaseMockFns, -} from '@sim/testing/mocks/knowledge-search-use-case.mock' -import { workspaceAuthzMock, workspaceAuthzMockFns } from '@sim/testing/mocks/workspace-authz.mock' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' - -vi.mock('@sim/platform-authz/workspace', () => workspaceAuthzMock) -vi.mock('@/lib/knowledge/application/contexts', () => knowledgeContextsMock) -vi.mock('@/lib/knowledge/application/search', () => knowledgeSearchUseCaseMock) - -import { SearchIndexDormantError } from '@/lib/sim-search/indexed/gate' -import { searchWorkspaceKnowledge } from '@/lib/sim-search/indexed/search/scoped-search' - -const mocks = { - afterSearch: knowledgeSearchUseCaseMockFns.mockAfterKnowledgeSearch, - search: knowledgeSearchUseCaseMockFns.mockRunKnowledgeSearch, -} -mocks.afterSearch.mockImplementation(async () => undefined) - -workspaceAuthzMockFns.mockPermissionSatisfies.mockImplementation( - (actual: string | null) => actual !== null -) - -const principal = createSessionPrincipal({ userId: 'reader', sessionId: 'session' }) -const input = { workspaceId: 'workspace', query: 'orion', topK: 20, filters: { source: 'slack' } } - -/** These use cases run only while indexed organization search is on. */ -beforeEach(() => setEnvFlags({ isLiveEnterpriseSearchEnabled: false })) -afterEach(resetEnvFlagsMock) - -describe('canonical workspace search', () => { - beforeEach(() => { - resetDbChainMock() - knowledgeContextsMockFns.mockResolveKnowledgeWorkspaceContext.mockResolvedValue({ - workspaceId: 'workspace', - workspaceOrganizationId: null, - allowPersonalApiKeys: true, - billedAccountUserId: 'payer', - }) - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue('read') - mocks.search.mockResolvedValue({ - results: [], - knowledgeBases: [{ id: 'index', name: 'Enterprise Search' }], - }) - }) - it('authorizes the person before selecting the canonical active index and passes the same principal and filters', async () => { - queueTableRows(schemaMock.knowledgeBase, [{ id: 'index' }]) - await searchWorkspaceKnowledge.execute({ principal, input }) - /** The search runs under the context this use case resolved; the index is its one base. */ - expect(mocks.search).toHaveBeenCalledWith( - expect.objectContaining({ - principal, - input: { ...input, knowledgeBaseIds: ['index'] }, - context: expect.objectContaining({ - workspaceId: 'workspace', - knowledgeBases: [expect.objectContaining({ id: 'index' })], - }), - }) - ) - expect( - hasMockCondition( - dbChainMockFns.where.mock.calls[0][0], - (node) => - node.type === 'eq' && - node.left === schemaMock.knowledgeBase.isSearchIndex && - node.right === true - ) - ).toBe(true) - expect(dbChainMockFns.limit).toHaveBeenCalledWith(1) - }) - it('refuses on its own while indexed organization search is dormant', async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) - await expect(searchWorkspaceKnowledge.execute({ principal, input })).rejects.toBeInstanceOf( - SearchIndexDormantError - ) - expect(knowledgeContextsMockFns.mockResolveKnowledgeWorkspaceContext).not.toHaveBeenCalled() - expect(dbChainMockFns.where).not.toHaveBeenCalled() - }) - it('refuses a nonmember before querying the protected index', async () => { - workspaceAuthzMockFns.mockResolveEffectiveWorkspacePermission.mockResolvedValue(null) - await expect(searchWorkspaceKnowledge.execute({ principal, input })).rejects.toThrow( - 'Insufficient workspace' - ) - expect(dbChainMockFns.where).not.toHaveBeenCalled() - }) -}) diff --git a/apps/sim/lib/sim-search/indexed/search/scoped-search.ts b/apps/sim/lib/sim-search/indexed/search/scoped-search.ts deleted file mode 100644 index 63c038f4c75..00000000000 --- a/apps/sim/lib/sim-search/indexed/search/scoped-search.ts +++ /dev/null @@ -1,184 +0,0 @@ -import type { Principal } from '@sim/auth/principal' -import { resolvePrincipalSubjectUserId } from '@sim/auth/principal' -import { OrchestrationError } from '@/lib/core/orchestration/types' -import { type ResourceOwner, resourceScopeFromOwner } from '@/lib/core/resource-scope' -import { requireOrganizationSearchAvailable } from '@/lib/knowledge/access/availability' -import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' -import { - type KnowledgeResourceContext, - resolveKnowledgeOrganizationContext, - resolveKnowledgeOwnerContext, - resolveKnowledgeWorkspaceContext, -} from '@/lib/knowledge/application/contexts' -import { knowledgeOperations } from '@/lib/knowledge/application/operations' -import { - afterKnowledgeSearch, - buildKnowledgeSearchContext, - runKnowledgeSearch, - type SearchKnowledgeInput, - type SearchKnowledgeResult, - validateKnowledgeSearchInput, -} from '@/lib/knowledge/application/search' -import { instrumentSearchUseCase } from '@/lib/knowledge/application/search-diagnostics' -import type { ActiveKnowledgeBaseReference } from '@/lib/knowledge/knowledge-base-reference' -import { recordOrganizationSearchActivity } from '@/lib/knowledge/search/activity' -import { measureSearchStage } from '@/lib/knowledge/search/diagnostics' -import { findSearchIndex } from '@/lib/knowledge/search/search-index' -import { assertIndexedOrgSearchEnabled } from '@/lib/sim-search/indexed/gate' - -export type SearchWorkspaceKnowledgeInput = Omit< - SearchKnowledgeInput, - 'knowledgeBaseIds' | 'workspaceId' -> & { - workspaceId: string -} - -export type SearchOrganizationKnowledgeInput = Omit< - SearchWorkspaceKnowledgeInput, - 'workspaceId' -> & { organizationId: string } - -export type SearchScopedKnowledgeInput = Omit< - SearchKnowledgeInput, - 'knowledgeBaseIds' | 'workspaceId' | 'organizationId' -> & - ResourceOwner - -/** What an owner without an index answers: nothing, completely. */ -interface SearchWithoutIndex { - results: [] - query: string - knowledgeBases: [] - retrieval: { status: 'complete'; timedOutLegs: [] } -} - -type ScopedSearchResult = SearchKnowledgeResult | SearchWithoutIndex - -/** Whether an index was searched, which is what the follow-up to a search is for. */ -function searchedAnIndex(result: ScopedSearchResult): result is SearchKnowledgeResult { - return 'knowledgeBaseId' in result -} - -/** - * An owner without an index answers empty. An organization still has to be allowed to search, - * and its empty search is recorded like any other, so the activity view shows the attempt. - */ -async function searchWithoutIndex( - principal: Principal, - context: KnowledgeResourceContext, - input: Pick -): Promise { - if (context.organizationId) { - await requireOrganizationSearchAvailable(context.organizationId) - input.signal?.throwIfAborted() - const userId = resolvePrincipalSubjectUserId(principal) - if (userId) - await recordOrganizationSearchActivity({ - organizationId: context.organizationId, - userId, - surface: input.surface ?? 'other', - results: [], - }) - } - return { - results: [], - query: input.query ?? '', - knowledgeBases: [], - retrieval: { status: 'complete', timedOutLegs: [] }, - } -} - -type ScopedSearchInput = Omit - -/** - * A search surface that resolves an owner, finds the owner's index and searches it. The owner is - * resolved and authorized once, here; the search runs under that context, and nothing about the - * index is read twice. Surfaces differ only in how they name the owner and find the index. - */ -function defineScopedSearchUseCase< - I extends Pick, ->(surface: { - resolveContext: (input: I) => Promise - findIndex: (context: KnowledgeResourceContext) => Promise - searchInput: (input: I, context: KnowledgeResourceContext) => ScopedSearchInput -}) { - return defineAuthorizedKnowledgeUseCase({ - operation: knowledgeOperations.search, - resolveContext: ({ input }: { input: I }) => { - assertIndexedOrgSearchEnabled() - return measureSearchStage('scope_resolution', () => surface.resolveContext(input)) - }, - async execute({ principal, input, context }): Promise { - input.signal?.throwIfAborted() - if ( - !input.query?.trim() || - input.filters?.startDate || - input.filters?.endDate || - input.filters?.sortBy - ) - throw new OrchestrationError( - 'validation', - 'Date-only search, startDate/endDate and sorting require live search. Use a text query and modification filters for indexed search.' - ) - const index = await measureSearchStage('index_resolution', () => surface.findIndex(context)) - if (!index) return searchWithoutIndex(principal, context, input) - const searchInput: SearchKnowledgeInput = { - ...surface.searchInput(input, context), - knowledgeBaseIds: [index.id], - } - /** An owner the request asserts is the one that was resolved, or the request names none. */ - if ( - (searchInput.organizationId && searchInput.organizationId !== context.organizationId) || - (searchInput.workspaceId && searchInput.workspaceId !== context.workspaceId) - ) { - throw new OrchestrationError('not_found', 'Knowledge base not found') - } - validateKnowledgeSearchInput(searchInput) - return runKnowledgeSearch({ - principal, - input: searchInput, - context: buildKnowledgeSearchContext(principal, context, [index], searchInput), - }) - }, - afterSuccess: ({ principal, context, input, result }) => - searchedAnIndex(result) - ? afterKnowledgeSearch({ principal, context, input, result }) - : undefined, - }) -} - -/** Search and Assistant share the workspace's canonical Enterprise Search index. */ -export const searchWorkspaceKnowledge = instrumentSearchUseCase( - 'workspace_application', - defineScopedSearchUseCase({ - resolveContext: (input) => resolveKnowledgeWorkspaceContext(input), - findIndex: (context) => - findSearchIndex({ kind: 'workspace', workspaceId: context.workspaceId! }), - searchInput: (input, context) => ({ ...input, workspaceId: context.workspaceId }), - }) -) - -/** Organization Search and Assistant resolve the same index and provider ACLs. */ -export const searchOrganizationKnowledge = instrumentSearchUseCase( - 'organization_application', - defineScopedSearchUseCase({ - resolveContext: (input) => resolveKnowledgeOrganizationContext(input), - findIndex: (context) => - findSearchIndex({ kind: 'organization', organizationId: context.organizationId! }), - searchInput: (input) => input, - }) -) - -/** The routed owner selects the index; current membership and provider ACLs select its documents. */ -export const searchScopedKnowledge = instrumentSearchUseCase( - 'scoped_application', - defineScopedSearchUseCase({ - resolveContext: (input) => resolveKnowledgeOwnerContext(input), - findIndex: (context) => findSearchIndex(resourceScopeFromOwner(context)), - searchInput: (input) => ({ - ...input, - workspaceId: input.workspaceId ?? undefined, - organizationId: input.organizationId ?? undefined, - }), - }) -) diff --git a/apps/sim/lib/sim-search/live/README.md b/apps/sim/lib/sim-search/live/README.md index ccde37b1e19..4e90300c998 100644 --- a/apps/sim/lib/sim-search/live/README.md +++ b/apps/sim/lib/sim-search/live/README.md @@ -1,6 +1,6 @@ # Federated Search access and connector behavior -This describes the live enterprise-search path. Credential Groups and ordinary knowledge-base indexing retain their existing behavior. Live Search is enabled by default; only an explicit `SIM_SEARCH_LIVE=false` selects the legacy indexed backend, which is kept dormant in `../indexed/` (see its README). Live Search sources do not create content-indexing jobs, and queued content or persisted-directory jobs stop before crawling, embedding, or building ACL snapshots. Ordinary KB jobs remain enabled. Administrators can still maintain GitLab CSV grants, and request-time source permission checks remain required. +This describes the live enterprise-search path. Credential Groups and ordinary knowledge-base indexing retain their existing behavior. Enterprise Search uses this live backend. The legacy indexed backend and its runtime toggle have been removed. Live Search sources do not create content-indexing jobs, and queued content or persisted-directory jobs stop before crawling, embedding, or building ACL snapshots. Ordinary KB jobs remain enabled. Administrators can still maintain GitLab CSV grants, and request-time source permission checks remain required. ## Admin and member surfaces diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 5e61b8cd208..08304f4748b 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -18,7 +18,6 @@ import { } from '@/lib/api/contracts/mothership-assistant-tools' import { canonicalJson, fingerprint, instantScopePart } from '@/lib/api/cursor-binding' import { env } from '@/lib/core/config/env' -import { isLiveEnterpriseSearchEnabled } from '@/lib/core/config/env-flags' import { OrchestrationError } from '@/lib/core/orchestration/types' import { type ResourceOwner, @@ -160,10 +159,6 @@ interface LiveSearchOptions { } export type LiveSearchInput = ResourceOwner & LiveSearchOptions -function requireLiveSearch() { - if (!isLiveEnterpriseSearchEnabled) - throw new OrchestrationError('not_found', 'Live search is not enabled') -} function safeContent(content: string, registry?: ResolvedSecretTraceRegistry): string { if (!registry) return content const projected = projectResolvedSecretModelContent(content, registry) @@ -345,7 +340,6 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.search, resolveContext: ({ input }: { input: LiveSearchInput }) => resolveKnowledgeOwnerContext(input), async execute({ principal, input }): Promise { - requireLiveSearch() const userId = requirePrincipalSubjectUserId(principal) if (input.organizationId) await requireOrganizationSearchAvailable(input.organizationId) input.signal?.throwIfAborted() @@ -740,7 +734,6 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.readDocument, resolveContext: ({ input }: { input: LiveReadInput }) => resolveKnowledgeOwnerContext(input), async execute({ principal, input }) { - requireLiveSearch() const userId = requirePrincipalSubjectUserId(principal) if (input.organizationId) await requireOrganizationSearchAvailable(input.organizationId) if (!Number.isInteger(input.limit) || input.limit < 1 || input.limit > 8) @@ -845,7 +838,6 @@ export const listLiveSearchAccounts = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.listPersonalSearchIntegrations, resolveContext: ({ input }: { input: ResourceOwner }) => resolveKnowledgeOwnerContext(input), async execute({ principal, input }) { - requireLiveSearch() if (input.organizationId) await requireOrganizationSearchAvailable(input.organizationId) const accounts = await listLiveAccounts(input, requirePrincipalSubjectUserId(principal)) return { diff --git a/apps/sim/lib/sim-search/personal-source-setup.ts b/apps/sim/lib/sim-search/personal-source-setup.ts deleted file mode 100644 index 7155fd26ed9..00000000000 --- a/apps/sim/lib/sim-search/personal-source-setup.ts +++ /dev/null @@ -1,2 +0,0 @@ -/** Personal Atlassian setup bounds explicitly selected project or space keys. */ -export const MAX_PERSONAL_SOURCE_SETUP_KEYS = 1000 diff --git a/apps/sim/lib/slack-search/connections.test.ts b/apps/sim/lib/slack-search/connections.test.ts index 3b2ff89c94b..c07699c071d 100644 --- a/apps/sim/lib/slack-search/connections.test.ts +++ b/apps/sim/lib/slack-search/connections.test.ts @@ -16,7 +16,8 @@ const target = { type: 'link', provider: 'google-email', connectorType: 'gmail', - connectorId: 'source', + connectionMode: 'live', + optionId: 'gmail-option', } as const const input = { targets: [target], diff --git a/apps/sim/scripts/fixtures/desktop-source-connect.tsx b/apps/sim/scripts/fixtures/desktop-source-connect.tsx index 0e4dfe86046..521fe50ade9 100644 --- a/apps/sim/scripts/fixtures/desktop-source-connect.tsx +++ b/apps/sim/scripts/fixtures/desktop-source-connect.tsx @@ -1,13 +1,13 @@ import { StrictMode, useEffect, useRef, useState } from 'react' import { ToastProvider } from '@sim/emcn' -import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { QueryClient, QueryClientProvider, useMutation } from '@tanstack/react-query' import { createRoot } from 'react-dom/client' import { isCredentialGroupOAuthFailure } from '@/lib/credential-groups/oauth-completion' import { startDesktopSourceBrowser } from '@/lib/desktop/source-browser' +import { connectDesktopSource } from '@/lib/desktop/source-connect' import { CredentialGroupCompletionHandoff } from '@/app/credential-groups/complete/completion-handoff' import { SlackCompletion } from '@/app/credential-groups/slack-complete/slack-completion' import { SourceCompletion } from '@/app/desktop/connect/source-completion' -import { useMemberEnrollment } from '@/app/o/[organizationId]/integrations/indexed/use-member-enrollment' import { useConnectOrganizationAccount, useOrganizationAccounts, @@ -17,16 +17,16 @@ import { useSlackSearchInstallations, useStartSlackSearchOAuth } from '@/hooks/q import { useGitHubInstallationSetup } from '@/hooks/use-github-installation-setup' import { useSearchIntegrationConnection } from '@/hooks/use-search-integration-connection' -const NO_CONNECTIONS = new Set() -const MEMBERSHIP_KEYS: readonly (readonly string[])[] = [] - function SourceConnectFixture() { const accountConnection = useConnectOrganizationAccount() const reconnect = useReconnectPersonalOrganizationAccount() const accounts = useOrganizationAccounts('fixture-organization') - const enrollment = useMemberEnrollment({ - membershipQueryKeys: MEMBERSHIP_KEYS, - connectedConnectorIds: NO_CONNECTIONS, + const enrollment = useMutation({ + mutationFn: () => + connectDesktopSource({ + kind: 'member-enrollment', + params: { id: '00000000-0000-4000-8000-000000000001', connectorId: 'fixture-connector' }, + }), }) const [githubCredential, setGithubCredential] = useState('') const github = useGitHubInstallationSetup({ @@ -120,16 +120,11 @@ function SourceConnectFixture() { {String(github.pending)} {githubCredential} {github.error} - {String(enrollment.isPending)} - {enrollment.error} + {enrollment.error?.message} {connection.status} {inventory.data?.installations.length ?? 0} {connection.error &&

{connection.error.message}

} diff --git a/apps/sim/tools/index.test.ts b/apps/sim/tools/index.test.ts index b5ce9ee0134..c00162deb52 100644 --- a/apps/sim/tools/index.test.ts +++ b/apps/sim/tools/index.test.ts @@ -7163,7 +7163,6 @@ describe('organization scratch internal entrance', () => { describe('Live Search Assistant GitHub OAuth binding', () => { beforeEach(async () => { - setEnvFlags({ isLiveEnterpriseSearchEnabled: true }) const metadata = await import('@/tools/metadata') const actual = await vi.importActual('@/tools/metadata') vi.mocked(metadata.getToolMetadata).mockImplementation(actual.getToolMetadata) @@ -7199,8 +7198,6 @@ describe('Live Search Assistant GitHub OAuth binding', () => { }) it('executes the existing issue/PR counting tool using the selected personal OAuth account', async () => { const { getToolMetadata } = await import('@/tools/metadata') - const { isLiveEnterpriseSearchEnabled } = await import('@/lib/core/config/env-flags') - expect(isLiveEnterpriseSearchEnabled).toBe(true) expect(getToolMetadata('github_search_issues_v2')).toMatchObject({ id: 'github_search_issues_v2', params: { apiKey: { required: true } }, diff --git a/apps/sim/tools/index.ts b/apps/sim/tools/index.ts index b5710fe348b..d7fa5f42bce 100644 --- a/apps/sim/tools/index.ts +++ b/apps/sim/tools/index.ts @@ -15,7 +15,7 @@ import { type BillingAttributionSnapshot, serializeBillingAttributionHeader, } from '@/lib/billing/core/billing-attribution' -import { isHosted, isLiveEnterpriseSearchEnabled } from '@/lib/core/config/env-flags' +import { isHosted } from '@/lib/core/config/env-flags' import { findDatabaseQueryError } from '@/lib/core/errors/database-query-error' import { createTimeoutAbortController, @@ -1829,7 +1829,7 @@ async function executeToolImplementation( } if (operationContext?.requestMode === 'assistant' && tool) { - tool = projectAssistantConnectedAccountTool(tool, isLiveEnterpriseSearchEnabled) + tool = projectAssistantConnectedAccountTool(tool) } // Ensure context is preserved if it exists diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index 142be6253c8..126c1efe6b1 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -134,8 +134,6 @@ WORKDIR /app # Runtime flags override these image defaults; dev images opt into Plan. ARG MSHIP_PLAN_MODE_DEFAULT=false ENV MSHIP_PLAN_MODE_DEFAULT=$MSHIP_PLAN_MODE_DEFAULT -ARG SIM_SEARCH_LIVE_DEFAULT=true -ENV SIM_SEARCH_LIVE_DEFAULT=$SIM_SEARCH_LIVE_DEFAULT # Node.js 24, Python, ffmpeg, etc. are already installed in base stage ENV NODE_ENV=production diff --git a/packages/db/knowledge-projection.test.ts b/packages/db/knowledge-projection.test.ts deleted file mode 100644 index 22e9d4b86f6..00000000000 --- a/packages/db/knowledge-projection.test.ts +++ /dev/null @@ -1,245 +0,0 @@ -import { runKnowledgeProjection } from '@sim/db/knowledge-projection' -import type { Sql } from 'postgres' -import { describe, expect, it, vi } from 'vitest' - -interface Mark { - generation: number - content: boolean -} - -interface FakeDatabase { - marks: Map - /** Documents whose advisory lock another pass holds. */ - lockedElsewhere: Set - tin: boolean - /** Chunk count per document, paged by chunk index. */ - chunks: Map - /** Called before each page statement; may throw to fail the page or change marks. */ - beforePage?: (page: { documentId: string; projection: string; after: number }) => void - /** Called before a settle. */ - beforeSettle?: (documentId: string) => void -} - -/** The page statements a pass ran, in order. */ -interface Trace { - pages: Array<{ documentId: string; projection: string; after: number; mode: 'content' | 'acl' }> - locks: string[] - unlocks: string[] - claims: string[][] -} - -function postgresError(code: string, message: string): Error { - return Object.assign(new Error(message), { code }) -} - -/** - * A stand-in for a postgres.js session that answers the projector's statements from `state`, by - * the statement's text: enough of the database to drive the pass's control flow. - */ -function fakeSql(state: FakeDatabase): { sql: Sql; trace: Trace } { - const trace: Trace = { pages: [], locks: [], unlocks: [], claims: [] } - const answer = async (text: string, values: unknown[]): Promise => { - if (text.includes('to_regprocedure')) return [{ installed: state.tin }] - if (text.includes('ORDER BY marked_at')) { - const skipped = new Set(values[0] as string[]) - const claimed = [...state.marks.keys()].filter((id) => !skipped.has(id)).slice(0, 50) - trace.claims.push(claimed) - return claimed.map((document_id) => ({ document_id })) - } - if (text.includes('pg_try_advisory_lock')) { - const documentId = String(values[0]) - trace.locks.push(documentId) - return [{ acquired: !state.lockedElsewhere.has(documentId) }] - } - if (text.includes('pg_advisory_unlock')) { - trace.unlocks.push(String(values[0])) - return [] - } - if (text.includes('SELECT m.generation')) { - const mark = state.marks.get(String(values[0])) - return mark - ? [{ generation: String(mark.generation), content: mark.content, knowledge_base_id: 'kb' }] - : [] - } - if (text.includes('WITH removed AS')) { - const documentId = String(values[0]) - state.beforeSettle?.(documentId) - const mark = state.marks.get(documentId) - if (mark && mark.generation === values[1]) { - state.marks.delete(documentId) - return [{ removed: 1, marked: true }] - } - return [{ removed: 0, marked: Boolean(mark) }] - } - if (text.includes('SELECT EXISTS (SELECT 1 FROM knowledge_projection_dirty')) { - return [{ marked: state.marks.has(String(values[0])) }] - } - return [] - } - const tagged = (strings: TemplateStringsArray, ...values: unknown[]) => - answer(strings.join('?'), values) - const unsafe = async (text: string, values: unknown[] = []) => { - if (!text.includes('WITH page AS')) return answer(text, values) - const [documentId, after, pageSize] = values as [string, number, number] - const projection = - /INSERT INTO (\w+)|UPDATE (\w+) s SET/.exec(text)?.slice(1).find(Boolean) ?? 'unknown' - const mode = text.includes('INSERT INTO') ? 'content' : 'acl' - state.beforePage?.({ documentId, projection, after }) - trace.pages.push({ documentId, projection, after, mode }) - const total = state.chunks.get(documentId) ?? 1 - const first = after + 1 - const scanned = Math.max(0, Math.min(pageSize, total - first)) - return [ - { - scanned, - written: scanned, - last_chunk: scanned === 0 ? null : first + scanned - 1, - }, - ] - } - const session = Object.assign(tagged, { - unsafe, - begin: async (work: (tx: unknown) => Promise) => - work(Object.assign(tagged, { unsafe })), - }) - return { sql: session as unknown as Sql, trace } -} - -function database(overrides: Partial = {}): FakeDatabase { - return { - marks: new Map(), - lockedElsewhere: new Set(), - tin: true, - chunks: new Map(), - ...overrides, - } -} - -describe('runKnowledgeProjection', () => { - it('projects every marked document under its lock and removes each mark it settled', async () => { - const state = database({ - marks: new Map([ - ['doc-a', { generation: 1, content: true }], - ['doc-b', { generation: 3, content: false }], - ]), - }) - const { sql, trace } = fakeSql(state) - const progress = await runKnowledgeProjection(sql, { searchIndexes: true }) - expect(progress).toMatchObject({ settled: 2, deferred: 0, remaining: false }) - expect(state.marks.size).toBe(0) - expect(trace.locks).toEqual(['doc-a', 'doc-b']) - expect(trace.unlocks).toEqual(['doc-a', 'doc-b']) - /** A content mark rewrites every projection; a source and ACL mark only the mirrored ones. */ - expect(trace.pages.filter((page) => page.documentId === 'doc-a')).toEqual([ - { documentId: 'doc-a', projection: 'embedding_search', after: -1, mode: 'content' }, - { documentId: 'doc-a', projection: 'embedding_keyword_search', after: -1, mode: 'content' }, - { documentId: 'doc-a', projection: 'embedding_keyword_tin', after: -1, mode: 'content' }, - ]) - expect(trace.pages.filter((page) => page.documentId === 'doc-b')).toEqual([ - { documentId: 'doc-b', projection: 'embedding_search', after: -1, mode: 'acl' }, - { documentId: 'doc-b', projection: 'embedding_keyword_tin', after: -1, mode: 'acl' }, - ]) - }) - - it('keeps a mark whose generation moved while the pass ran', async () => { - const state = database({ marks: new Map([['doc', { generation: 1, content: false }]]) }) - state.beforeSettle = () => state.marks.set('doc', { generation: 2, content: false }) - const { sql, trace } = fakeSql(state) - const progress = await runKnowledgeProjection(sql, { searchIndexes: true }) - expect(progress).toMatchObject({ settled: 0, deferred: 1, remaining: true }) - expect(state.marks.get('doc')?.generation).toBe(2) - /** A document given up is not claimed again by the same pass. */ - expect(trace.claims).toEqual([['doc'], []]) - }) - - it('leaves a document another pass holds to that pass', async () => { - const state = database({ - marks: new Map([ - ['held', { generation: 1, content: false }], - ['free', { generation: 1, content: false }], - ]), - lockedElsewhere: new Set(['held']), - }) - const { sql, trace } = fakeSql(state) - const progress = await runKnowledgeProjection(sql, { searchIndexes: true }) - expect(progress).toMatchObject({ settled: 1, deferred: 1, remaining: true }) - expect(state.marks.has('held')).toBe(true) - expect(trace.pages.some((page) => page.documentId === 'held')).toBe(false) - expect(trace.unlocks).toEqual(['free']) - }) - - it.each([ - ['a lock timeout', postgresError('55P03', 'canceling statement due to lock timeout')], - ['a statement timeout', postgresError('57014', 'canceling statement due to statement timeout')], - ['a deadlock', postgresError('40P01', 'deadlock detected')], - ['a chunk deleted under the page', postgresError('23503', 'insert violates foreign key')], - ])('gives a document up on %s and carries on with the next', async (_, failure) => { - const state = database({ - marks: new Map([ - ['slow', { generation: 1, content: false }], - ['fine', { generation: 1, content: false }], - ]), - }) - state.beforePage = ({ documentId }) => { - if (documentId === 'slow') throw failure - } - const { sql, trace } = fakeSql(state) - const progress = await runKnowledgeProjection(sql, { searchIndexes: true }) - expect(progress).toMatchObject({ settled: 1, deferred: 1, remaining: true }) - expect(state.marks.has('slow')).toBe(true) - expect(trace.unlocks).toEqual(['slow', 'fine']) - }) - - it('counts a document deleted during its pass as gone rather than left marked', async () => { - const state = database({ marks: new Map([['doc', { generation: 1, content: false }]]) }) - state.beforePage = () => { - state.marks.delete('doc') - throw postgresError('23503', 'insert violates foreign key') - } - const { sql } = fakeSql(state) - await expect(runKnowledgeProjection(sql, { searchIndexes: true })).resolves.toMatchObject({ - settled: 0, - deferred: 0, - remaining: false, - }) - }) - - it('fails the pass on any other error, releasing the document it held', async () => { - const state = database({ marks: new Map([['doc', { generation: 1, content: false }]]) }) - state.beforePage = () => { - throw postgresError('42P01', 'relation does not exist') - } - const { sql, trace } = fakeSql(state) - await expect(runKnowledgeProjection(sql, { searchIndexes: true })).rejects.toThrow( - 'relation does not exist' - ) - expect(trace.unlocks).toEqual(['doc']) - expect(state.marks.has('doc')).toBe(true) - }) - - it('stops between pages of a long document at its deadline and keeps that mark', async () => { - vi.useFakeTimers() - try { - const state = database({ - marks: new Map([['long', { generation: 1, content: false }]]), - chunks: new Map([['long', 10]]), - tin: false, - }) - state.beforePage = () => { - vi.advanceTimersByTime(40) - } - const { sql, trace } = fakeSql(state) - const progress = await runKnowledgeProjection(sql, { - budgetMs: 100, - pageSize: 2, - searchIndexes: true, - }) - expect(trace.pages).toHaveLength(3) - expect(progress).toMatchObject({ settled: 0, deferred: 1, remaining: true }) - expect(state.marks.has('long')).toBe(true) - expect(trace.unlocks).toEqual(['long']) - } finally { - vi.useRealTimers() - } - }) -}) diff --git a/packages/db/knowledge-projection.ts b/packages/db/knowledge-projection.ts index fa55824873f..75511f56fe8 100644 --- a/packages/db/knowledge-projection.ts +++ b/packages/db/knowledge-projection.ts @@ -14,9 +14,8 @@ const logger = createLogger('KnowledgeProjection') const KNOWLEDGE_PROJECTION_MODE_SETTING = 'sim.projection_mode' /** - * Selected by every projector transaction. The projector writes each projection row's source and - * ACL itself, so the projection tables' own source and ACL triggers, which would re-read the - * document for every row, are skipped. + * Selected by every repair transaction so still-installed legacy source/ACL triggers do not copy + * permissions onto repaired vectors. Ordinary KB reads authorize against the parent document. */ const SKIP_SYNCHRONOUS_PROJECTION = `set_config('${KNOWLEDGE_PROJECTION_MODE_SETTING}', 'async', true)` @@ -53,6 +52,9 @@ const SETTLE_LOCK_TIMEOUT_MS = 2_000 /** Marks claimed per round; each is then projected under its own advisory lock. */ const CLAIM_BATCH_SIZE = 50 +/** Caps the IDs retained and resent to each claim query when documents cannot be repaired. */ +const MAX_DEFERRED_DOCUMENTS = 1_000 + /** The embedding models trained for prefix retrieval, whose 512 projection is a prefix. */ const SHORTENED_EMBEDDING_MODELS = `('text-embedding-3-small', 'text-embedding-3-large')` @@ -92,23 +94,10 @@ function searchVectorShortened(model: string, prefix: string): string { return `${model} IN ${SHORTENED_EMBEDDING_MODELS} AND ${prefix}.embedding_384 IS NULL` } -/** The projections the projector keeps; the Tin projection exists only where `tin` is installed. */ -const KNOWLEDGE_PROJECTIONS = [ - 'embedding_search', - 'embedding_keyword_search', - 'embedding_keyword_tin', -] as const -export type KnowledgeProjection = (typeof KNOWLEDGE_PROJECTIONS)[number] +export type KnowledgeProjection = 'embedding_search' -/** The projections that mirror their document's source and ACL, and so can be unfilled. */ +/** Historical trigger installers retain these immutable names until their contract migration. */ export const SOURCE_ACL_PROJECTIONS = ['embedding_search', 'embedding_keyword_tin'] as const -export type SourceAclProjection = (typeof SOURCE_ACL_PROJECTIONS)[number] -const mirrorsSourceAcl = (projection: KnowledgeProjection): projection is SourceAclProjection => - (SOURCE_ACL_PROJECTIONS as readonly string[]).includes(projection) - -/** The keyword projections, which hold only the rows of search-index knowledge bases. */ -const holdsSearchIndexesOnly = (projection: KnowledgeProjection) => - projection === 'embedding_keyword_search' || projection === 'embedding_keyword_tin' /** * The chunks one page covers: the next {@link PROJECTION_ROW_BATCH_SIZE} of the document in @@ -127,96 +116,31 @@ const PAGE_RESULT = `SELECT (SELECT count(*)::int FROM page) AS scanned, (SELECT max(chunk_index) FROM page) AS last_chunk` /** - * Rewrites a page's projection rows from their chunks and the document, for a mark whose chunk - * write skipped the synchronous triggers (only releases that deferred projection wrote those), - * writing only the rows that differ, so a document whose rows are already current costs reads and no index writes. The - * source and ACL are the document's as this statement reads it; a change that commits after it - * marks the document again, so the projector's settle leaves the mark for the next pass. + * Repairs only ordinary-KB vectors left by older deferred writers. The parent KB is checked and + * share-locked on every page so a concurrent Search-marker change cannot admit retired content. + * Binary columns remain compatibility writes until their database width constraint is replaced. */ -function contentPageStatement(projection: KnowledgeProjection): string { - if (projection === 'embedding_search') { - const vectors = SEARCH_VECTOR_COLUMNS.join(', ') - const compared = [ - 'knowledge_base_id', - 'document_id', - 'enabled', - 'connector_id', - 'acl', - ...SEARCH_VECTOR_COLUMNS, - ] - return `WITH ${PAGE}, source AS MATERIALIZED ( - SELECT e.id, e.knowledge_base_id, e.document_id, e.enabled, e.embedding, e.embedding_384, - e.embedding_768, e.embedding_1024, e.embedding_3072, - ${searchVectorShortened('k.embedding_model', 'e')} AS shortened, d.connector_id, d.acl - FROM page p JOIN embedding e ON e.id = p.id - JOIN knowledge_base k ON k.id = e.knowledge_base_id - JOIN document d ON d.id = e.document_id - ), written AS ( - INSERT INTO embedding_search AS s - (id, knowledge_base_id, document_id, enabled, ${SEARCH_BINARY_COLUMNS.join(', ')}, ${vectors}, - connector_id, acl) - SELECT id, knowledge_base_id, document_id, enabled, ${searchBinaryProjections('source')}, - ${searchVectorProjections('source', 'shortened')}, connector_id, acl - FROM source - ON CONFLICT (id) DO UPDATE SET - ${[...compared, ...SEARCH_BINARY_COLUMNS].map((column) => `${column} = EXCLUDED.${column}`).join(', ')} - WHERE (${compared.map((column) => `s.${column}`).join(', ')}) - IS DISTINCT FROM (${compared.map((column) => `EXCLUDED.${column}`).join(', ')}) - RETURNING s.id - ) ${PAGE_RESULT}` - } - if (projection === 'embedding_keyword_search') { - const compared = ['knowledge_base_id', 'document_id', 'enabled', 'content_tsv'] - return `WITH ${PAGE}, source AS MATERIALIZED ( - SELECT e.id, ${compared.map((column) => `e.${column}`).join(', ')}, k.is_search_index - FROM page p JOIN embedding e ON e.id = p.id - JOIN knowledge_base k ON k.id = e.knowledge_base_id - ), removed AS ( - DELETE FROM embedding_keyword_search s USING source - WHERE s.id = source.id AND NOT source.is_search_index - ), written AS ( - INSERT INTO embedding_keyword_search AS s (${['id', ...compared].join(', ')}) - SELECT id, ${compared.join(', ')} FROM source WHERE is_search_index - ON CONFLICT (id) DO UPDATE SET - ${compared.map((column) => `${column} = EXCLUDED.${column}`).join(', ')} - WHERE (${compared.map((column) => `s.${column}`).join(', ')}) - IS DISTINCT FROM (${compared.map((column) => `EXCLUDED.${column}`).join(', ')}) - RETURNING s.id - ) ${PAGE_RESULT}` - } - const compared = ['knowledge_base_id', 'document_id', 'enabled', 'content', 'connector_id', 'acl'] +function contentPageStatement(): string { + const vectors = SEARCH_VECTOR_COLUMNS.join(', ') + const compared = ['knowledge_base_id', 'document_id', 'enabled', ...SEARCH_VECTOR_COLUMNS] return `WITH ${PAGE}, source AS MATERIALIZED ( - SELECT e.id, e.knowledge_base_id, e.document_id, e.enabled, - knowledge_tin_base_token(e.knowledge_base_id) || ' ' || knowledge_tin_stream(e.content_tsv) AS content, - d.connector_id, d.acl, k.is_search_index + SELECT e.id, e.knowledge_base_id, e.document_id, e.enabled, e.embedding, e.embedding_384, + e.embedding_768, e.embedding_1024, e.embedding_3072, + ${searchVectorShortened('k.embedding_model', 'e')} AS shortened FROM page p JOIN embedding e ON e.id = p.id JOIN knowledge_base k ON k.id = e.knowledge_base_id - JOIN document d ON d.id = e.document_id - ), removed AS ( - DELETE FROM embedding_keyword_tin t USING source - WHERE t.id = source.id AND NOT source.is_search_index + WHERE NOT k.is_search_index + FOR SHARE OF k ), written AS ( - INSERT INTO embedding_keyword_tin AS t (${['id', ...compared].join(', ')}) - SELECT id, ${compared.join(', ')} FROM source WHERE is_search_index + INSERT INTO embedding_search AS s + (id, knowledge_base_id, document_id, enabled, ${SEARCH_BINARY_COLUMNS.join(', ')}, ${vectors}) + SELECT id, knowledge_base_id, document_id, enabled, ${searchBinaryProjections('source')}, + ${searchVectorProjections('source', 'shortened')} + FROM source ON CONFLICT (id) DO UPDATE SET - ${compared.map((column) => `${column} = EXCLUDED.${column}`).join(', ')} - WHERE (${compared.map((column) => `t.${column}`).join(', ')}) + ${[...compared, ...SEARCH_BINARY_COLUMNS].map((column) => `${column} = EXCLUDED.${column}`).join(', ')} + WHERE (${compared.map((column) => `s.${column}`).join(', ')}) IS DISTINCT FROM (${compared.map((column) => `EXCLUDED.${column}`).join(', ')}) - RETURNING t.id - ) ${PAGE_RESULT}` -} - -/** - * Copies the document's source and ACL onto a page's existing projection rows that differ, - * including rows written before projections carried a source and ACL. Chunks are untouched, so their vectors - * are never read; a row a chunk change has not projected yet is left to that change's own mark. - */ -function sourceAclPageStatement(projection: KnowledgeProjection): string { - return `WITH ${PAGE}, written AS ( - UPDATE ${projection} s SET connector_id = d.connector_id, acl = d.acl - FROM page p, document d - WHERE s.id = p.id AND d.id = $1 - AND (s.connector_id IS DISTINCT FROM d.connector_id OR s.acl IS DISTINCT FROM d.acl) RETURNING s.id ) ${PAGE_RESULT}` } @@ -245,7 +169,7 @@ async function enterProjectorTransaction(tx: TransactionSql, lockTimeoutMs: numb interface ProjectionMark { generation: number content: boolean - knowledgeBaseId: string + isSearchIndex: boolean } export interface KnowledgeProjectionOptions { @@ -256,14 +180,6 @@ export interface KnowledgeProjectionOptions { */ budgetMs?: number pageSize?: number - /** - * Whether search-index rows are read, that is whether indexed organization search is on (see - * {@link MarkScope}). Only then is the Tin keyword projection written, since it holds only - * search-index rows; the other projections are written either way, the GIN keyword projection - * for search-index bases alone whatever this says, and Tin is still skipped where it is not - * installed. - */ - searchIndexes: boolean /** Called after each page commits, for tests that interleave writes with a run. */ onPage?: (page: { documentId: string @@ -283,13 +199,6 @@ export interface KnowledgeProjectionProgress { remaining: boolean } -/** Whether the Tin keyword projection is maintained here: its functions exist only where installed. */ -async function tinInstalled(sql: Sql): Promise { - const [row] = await sql>` - SELECT to_regprocedure('knowledge_tin_stream(tsvector)') IS NOT NULL AS installed` - return Boolean(row?.installed) -} - /** * The oldest marks, past the ones this run has already passed over. A read without row locks: a * pass owns a document through its advisory lock, and a row lock on the mark would either block @@ -311,16 +220,17 @@ async function claimMarks(sql: Sql, skipped: readonly string[]): Promise { const [row] = await sql< - Array<{ generation: string; content: boolean; knowledge_base_id: string }> + Array<{ generation: string; content: boolean; is_search_index: boolean }> >` - SELECT m.generation, m.content, d.knowledge_base_id + SELECT m.generation, m.content, k.is_search_index FROM knowledge_projection_dirty m JOIN document d ON d.id = m.document_id + JOIN knowledge_base k ON k.id = d.knowledge_base_id WHERE m.document_id = ${documentId}` return row ? { generation: Number(row.generation), content: row.content, - knowledgeBaseId: row.knowledge_base_id, + isSearchIndex: row.is_search_index, } : null } @@ -370,15 +280,11 @@ async function stillMarked(sql: Sql, documentId: string): Promise { async function projectDocumentRows( sql: Sql, documentId: string, - projection: KnowledgeProjection, - mark: ProjectionMark, options: KnowledgeProjectionOptions, deadline: number ): Promise<{ pages: number; written: number; finished: boolean }> { const pageSize = options.pageSize ?? PROJECTION_ROW_BATCH_SIZE - const statement = mark.content - ? contentPageStatement(projection) - : sourceAclPageStatement(projection) + const statement = contentPageStatement() let after = -1 let pages = 0 let written = 0 @@ -386,9 +292,6 @@ async function projectDocumentRows( if (Date.now() >= deadline) return { pages, written, finished: false } const page = await sql.begin(async (tx) => { await enterProjectorTransaction(tx, PROJECTION_PAGE_LOCK_TIMEOUT_MS) - /** Shares the base's membership lock, as the embedding triggers do, so a flip of its marker waits. */ - if (holdsSearchIndexesOnly(projection)) - await tx`SELECT pg_advisory_xact_lock_shared(knowledge_tin_membership_key(${mark.knowledgeBaseId}))` const [row] = await tx.unsafe< Array<{ scanned: number; written: number; last_chunk: number | null }> >(statement, [documentId, after, pageSize]) @@ -396,7 +299,7 @@ async function projectDocumentRows( }) pages += 1 written += page.written - await options.onPage?.({ documentId, projection, written: page.written }) + await options.onPage?.({ documentId, projection: 'embedding_search', written: page.written }) if (page.last_chunk === null || page.scanned < pageSize) break after = page.last_chunk } @@ -412,7 +315,6 @@ type DocumentOutcome = 'settled' | 'deferred' | 'gone' async function projectMarkedDocument( sql: Sql, documentId: string, - projections: readonly KnowledgeProjection[], options: KnowledgeProjectionOptions, totals: { pages: number; written: number }, deadline: number @@ -423,9 +325,8 @@ async function projectMarkedDocument( try { const mark = await readMark(sql, documentId) if (!mark) return 'gone' - for (const projection of projections) { - if (!mark.content && !mirrorsSourceAcl(projection)) continue - const done = await projectDocumentRows(sql, documentId, projection, mark, options, deadline) + if (mark.content && !mark.isSearchIndex) { + const done = await projectDocumentRows(sql, documentId, options, deadline) totals.pages += done.pages totals.written += done.written if (!done.finished) return 'deferred' @@ -445,7 +346,7 @@ async function projectMarkedDocument( } /** - * Converges the search projections of every marked document, oldest mark first, until none is + * Repairs ordinary-KB vectors and releases obsolete marks, oldest mark first, until none is * left or the budget runs out. Runs on a connection of its own: each document is projected under * a session advisory lock, so concurrent runs never project the same document at once and a run * that dies releases its locks with its connection. A document keeps its mark, left to a later @@ -459,10 +360,6 @@ export async function runKnowledgeProjection( ): Promise { const deadline = options.budgetMs === undefined ? Number.POSITIVE_INFINITY : Date.now() + options.budgetMs - const projections = - options.searchIndexes && (await tinInstalled(sql)) - ? KNOWLEDGE_PROJECTIONS - : KNOWLEDGE_PROJECTIONS.filter((projection) => projection !== 'embedding_keyword_tin') const totals = { pages: 0, written: 0 } const skipped: string[] = [] let settled = 0 @@ -473,16 +370,12 @@ export async function runKnowledgeProjection( } for (const documentId of claimed) { if (Date.now() >= deadline) break - const outcome = await projectMarkedDocument( - sql, - documentId, - projections, - options, - totals, - deadline - ) + const outcome = await projectMarkedDocument(sql, documentId, options, totals, deadline) if (outcome === 'settled') settled += 1 else if (outcome === 'deferred') skipped.push(documentId) + if (skipped.length >= MAX_DEFERRED_DOCUMENTS) { + return { settled, deferred: skipped.length, ...totals, remaining: true } + } } } return { settled, deferred: skipped.length, ...totals, remaining: true } @@ -498,42 +391,15 @@ export const MARK_RELEASE_BUDGET_MS = 10_000 /** Marks one release statement removes; a release repeats it while statements come back full. */ const RELEASE_BATCH_SIZE = 1_000 -/** - * Which marks a pass is owed besides content: search-index documents, whose rows mirror their - * source and ACL, while indexed organization search reads them. With it off, a mark with no content - * to project is owed nothing, whatever its knowledge base. - */ -export interface MarkScope { - searchIndexes: boolean -} - -/** - * Whether any mark needs a projector pass: one carrying content to project, or, when `scope` owes - * search-index documents a pass, one on a search-index document. Every other mark is released by - * {@link releaseSettledMarks} without a pass. - * - * Asked right after a release. One that drained its marks left only the marks a pass is owed - * (and the few a writer held), so the join that tells a search-index mark apart walks a handful of - * rows. One cut short left a backlog the join would walk in full, so only the content marks are - * asked about: a search-index mark behind that backlog waits for the sweep whose release drains it. - */ -export async function hasKnowledgeProjectionWork( - sql: Sql | TransactionSql, - release: { drained: boolean }, - scope: MarkScope -): Promise { - const [row] = - release.drained && scope.searchIndexes - ? await sql>` - SELECT EXISTS (SELECT 1 FROM knowledge_projection_dirty WHERE content) - OR EXISTS ( - SELECT 1 FROM knowledge_projection_dirty d - JOIN document doc ON doc.id = d.document_id - JOIN knowledge_base k ON k.id = doc.knowledge_base_id - WHERE k.is_search_index - ) AS pending` - : await sql>` - SELECT EXISTS (SELECT 1 FROM knowledge_projection_dirty WHERE content) AS pending` +/** Only deferred ordinary-KB content requires projection work after indexed Search retirement. */ +export async function hasKnowledgeProjectionWork(sql: Sql | TransactionSql): Promise { + const [row] = await sql>` + SELECT EXISTS ( + SELECT 1 FROM knowledge_projection_dirty m + JOIN document d ON d.id = m.document_id + JOIN knowledge_base k ON k.id = d.knowledge_base_id + WHERE m.content AND NOT k.is_search_index + ) AS pending` return Boolean(row?.pending) } @@ -547,22 +413,11 @@ export interface SettledMarkRelease { } /** - * Transitional: removes the marks that carry no content to project and that `scope` owes no pass, - * until a follow-up migration scopes the mark triggers to search-index knowledge bases. Their - * synchronous writers already wrote every projection row a workspace search reads, and those - * searches decide nothing on a projection row's source, ACL, or mark, so a pass over them would - * only re-read rows it then leaves as they are. While indexed organization search is on, the marks - * of search-index documents, whose rows it reads by source and ACL, stay for a pass. - * - * An empty mark table costs one probe, and reports `empty` so its caller asks nothing further. Marks a writer holds are skipped rather than waited on, - * and a mark whose writer skipped the synchronous triggers carries content and stays for a pass. - * Stops once a statement comes back short or `deadline` passes. + * Releases obsolete ACL-only and Search marks in bounded transactions. Ordinary-KB content marks + * survive for vector repair. Locked marks and bases are skipped; locks prevent a content upgrade + * or a Search-marker change from making the deleted mark necessary before this commit. */ -export async function releaseSettledMarks( - sql: Sql, - deadline: number, - scope: MarkScope -): Promise { +export async function releaseSettledMarks(sql: Sql, deadline: number): Promise { const [marked] = await sql>` SELECT EXISTS (SELECT 1 FROM knowledge_projection_dirty) AS any` if (!marked?.any) return { released: 0, drained: true, empty: true } @@ -570,23 +425,18 @@ export async function releaseSettledMarks( while (Date.now() < deadline) { const count = await sql.begin(async (tx) => { await enterProjectorTransaction(tx, SETTLE_LOCK_TIMEOUT_MS) - const settled = scope.searchIndexes - ? tx` - SELECT d.document_id FROM knowledge_projection_dirty d - JOIN document doc ON doc.id = d.document_id - JOIN knowledge_base k ON k.id = doc.knowledge_base_id - WHERE NOT d.content AND NOT k.is_search_index - LIMIT ${RELEASE_BATCH_SIZE} - FOR UPDATE OF d SKIP LOCKED` - : tx` - SELECT d.document_id FROM knowledge_projection_dirty d - WHERE NOT d.content - LIMIT ${RELEASE_BATCH_SIZE} - FOR UPDATE SKIP LOCKED` + const settled = tx` + SELECT m.document_id FROM knowledge_projection_dirty m + JOIN document d ON d.id = m.document_id + JOIN knowledge_base k ON k.id = d.knowledge_base_id + WHERE NOT m.content OR k.is_search_index + LIMIT ${RELEASE_BATCH_SIZE} + FOR UPDATE OF m SKIP LOCKED + FOR SHARE OF k SKIP LOCKED` const [row] = await tx>` WITH released AS ( DELETE FROM knowledge_projection_dirty m - WHERE m.document_id IN (${settled}) AND NOT m.content + WHERE m.document_id IN (${settled}) RETURNING 1 ) SELECT count(*)::int AS released FROM released` diff --git a/packages/db/schema.ts b/packages/db/schema.ts index ac6a68ea216..0a47dffac6e 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -3749,6 +3749,7 @@ export const embedding = pgTable( ) /** Keyword ranking reads text-search vectors independently of chunk content and semantic vectors. */ +// contract-pending(after the indexed-search retirement release and all legacy projection writers have drained): drop embedding_keyword_search — regular KB keyword queries read embedding.content_tsv. export const embeddingKeywordSearch = pgTable( 'embedding_keyword_search', { @@ -3779,6 +3780,7 @@ export const EMBEDDING_KEYWORD_TIN_INDEX = 'embedding_keyword_tin_content_idx' * the index, and the embedding and knowledge base triggers that own these rows, and only where * `tin` exists; elsewhere the table stays empty and keyword search keeps the GIN projection. */ +// contract-pending(after the indexed-search retirement release and all legacy projection writers have drained): drop embedding_keyword_tin — only retired indexed Search ranks this projection. export const embeddingKeywordTin = pgTable( 'embedding_keyword_tin', { @@ -3825,10 +3827,12 @@ export const embeddingSearch = pgTable( * source spends its scan budget on chunks the graph reached but the member cannot read. * NULL for uploads. */ + // contract-pending(after the indexed-search retirement release and source/ACL projection writers have drained): drop connector_id — regular KB retrieval checks the parent document. connectorId: text('connector_id'), /** The document's ACL, mirrored by trigger, so a walk can test readability on the row it visits. */ + // contract-pending(after the indexed-search retirement release and source/ACL projection writers have drained): drop acl — regular KB retrieval retains document-level access checks. acl: text('acl').array(), - /** contract-pending(after the projection sync trigger stops writing them): drop the binary columns; their ANN indexes were dropped in 0372, and nothing reads them. */ + // contract-pending(after vector writers stop computing binary projections and embedding_search_width_check is replaced): drop binary and all binary_* columns — their ANN indexes were dropped in 0372 and no reader uses them. binary: bit('binary', { dimensions: 1536 }), binary384: bit('binary_384', { dimensions: 384 }), binary768: bit('binary_768', { dimensions: 768 }), @@ -3885,6 +3889,7 @@ export const embeddingSearch = pgTable( * holds the document row, but clearing it would otherwise take that row again, and readers probe * this small table instead of joining `document` per ranked row. */ +// contract-pending(after deferred vector content is repaired and projection mark writers/workers are retired): drop knowledge_projection_dirty — legacy ACL copies need no repair, but unfinished vector repairs must survive retirement. export const knowledgeProjectionDirty = pgTable( 'knowledge_projection_dirty', { @@ -5222,6 +5227,7 @@ export const usageLogSourceEnum = pgEnum('usage_log_source', [ ]) /** Content-free organization Search activity, independent of billable model usage. */ +// contract-pending(after the indexed-search retirement release and old activity writers have drained): drop organization_search_invocation — live Search does not record indexed result activity. export const organizationSearchInvocation = pgTable( 'organization_search_invocation', { diff --git a/packages/db/script-migrations/indexed-search-retirement.md b/packages/db/script-migrations/indexed-search-retirement.md new file mode 100644 index 00000000000..998e8e4f680 --- /dev/null +++ b/packages/db/script-migrations/indexed-search-retirement.md @@ -0,0 +1,152 @@ +# Indexed Search database retirement + +This inventory separates retired enterprise Search indexing from the database objects that still +serve ordinary knowledge bases and live Search. Removing an application reader does not remove an +installed PostgreSQL trigger or reclaim its table. This release removes legacy application paths; +physical removal must follow a fully deployed release boundary. + +No step in this document authorizes an unbounded data deletion, automatic HNSW rebuild, or production +execution. The existing Search retirement command remains operator-run maintenance. Its progress and +target snapshot must survive until a replacement cleanup has adopted them or retirement is verified. + +## Deferred physical removal + +| Object | Retired responsibility | Contract prerequisite | +| --- | --- | --- | +| `embedding_keyword_search` and its foreign key, primary key, `embedding_keyword_search_kb_idx`, `embedding_keyword_search_document_idx`, `embedding_keyword_search_content_idx` | Indexed Search's GIN keyword candidate projection | Remove indexed readers and all keyword projection writers, including older workers and database triggers. Ordinary KB keyword ranking uses `embedding.content_tsv`. | +| `embedding_keyword_tin` and its foreign key, primary key, `embedding_keyword_tin_document_idx`, `embedding_keyword_tin_content_idx`, `embedding_keyword_tin_acl_gin_idx`, `embedding_keyword_tin_acl_unfilled_idx` | Indexed Search's optional Tin/BM25 projection and permission copies | Remove indexed readers, Tin/ACL projection writers, and maintenance references. Some installations never installed the optional Tin index. | +| `embedding_search.connector_id`, `embedding_search.acl` | Denormalized per-vector source/permission filtering for indexed Search | Deploy removal of indexed readers, then retire source/ACL triggers and old projector/detachment writers. Regular KB retrieval must keep checking the parent document. | +| `embedding_search_source_idx`, `embedding_search_acl_gin_idx`, `embedding_search_acl_unfilled_idx` | Source filtering, copied ACL overlap, and ACL backfill probes | Remove indexed readers and ACL fill paths; drop indexes concurrently in a later contract migration. These indexes were created by script migrations and are not declared in `schema.ts`. | +| `embedding_search_src_*` partial HNSW indexes | Per-connector ANN graphs used only by indexed Search | Inventory actual index definitions, confirm each belongs to `embedding_search` and filters a retired source, then drop concurrently. Preserve the shared width-specific HNSW indexes. | +| `organization_search_invocation`, its two foreign keys, bounds checks and `organization_search_invocation_org_created_idx` / `organization_search_invocation_user_idx` | Indexed result activity counters and old organization statistics | Deploy removal of the indexed statistics API and activity writer, and drain old application versions. | +| `embedding_search.binary`, `binary_384`, `binary_768`, `binary_1024`, `binary_3072` | Obsolete binary-quantized candidate representation | Stop `sync_embedding_search()` and the recovery projector from calculating/writing these columns, and replace the binary-based `embedding_search_width_check`. The five binary ANN indexes were already removed in migration `0372`. | +| `knowledge_projection_dirty`, its document FK/primary key, and `knowledge_projection_dirty_marked_at_idx` | Tracks copied source/ACL changes and older deferred vector writes | First complete or adopt every pending vector content repair, stop mark writers, and drain old workers. A `content = true` mark can represent a missing ordinary-KB vector; discarding it can lose retrieval coverage. | +| `search_embedding_cleanup_progress`, `search_embedding_cleanup_targets` | Durable retirement scope, cursor, and maintenance checkpoints | Complete and verify retirement, or explicitly adopt this state into a successor. These script-owned tables are not application schema objects and must not be dropped merely because deployment no longer runs cleanup. | + +The binary-width constraint currently requires exactly one populated binary column. Stopping binary +writes without changing that constraint would reject new ordinary-KB vectors. A replacement vector +table can omit the binary columns and use an appropriate vector-width invariant from the outset. +Changing the existing table requires a separately reviewed compatible transition; no table-wide +rewrite or validation scan belongs in this application-removal release. + +## Installed triggers and functions + +These live in the database after their TypeScript installer has finished. Deleting an installer or +an application import alone leaves its database work active. + +| Retire after dependent old code drains | Installed by | +| --- | --- | +| `embedding_keyword_search_sync` on `embedding`; `sync_embedding_keyword_search()` | `0016`, subsequently scoped/guarded by `0024` and `0025` | +| `knowledge_base_keyword_search_sync` on `knowledge_base`; `sync_knowledge_base_keyword_search()` | `0025` | +| `embedding_keyword_tin_sync` on `embedding`; `sync_embedding_keyword_tin()` | `0019`, subsequently guarded/scoped by `0024` and `0025` | +| `knowledge_base_keyword_tin_sync` on `knowledge_base`; `sync_knowledge_base_keyword_tin()` | `0019`, updated by `0025` | +| `knowledge_tin_stream(tsvector)`, `knowledge_tin_base_token(text)`, `knowledge_tin_membership_key(text)` | `0019`; the membership helper is also installed by `0025` and used by both keyword writers and the projector | +| `projection_source_acl_sync` on `document`; `sync_projection_source_acl()` | `0021`/`0022`, updated by `0023`, `0024`, and `0025` | +| `embedding_search_source_acl_set` on `embedding_search`; `embedding_keyword_tin_source_acl_set` on `embedding_keyword_tin`; `set_projection_source_acl()` | `0021`/`0022`, guarded by `0024` | +| `embedding_projection_mark_insert`, `embedding_projection_mark_update` on `embedding`; `mark_inserted_embedding_projection()`, `mark_updated_embedding_projection()`, `mark_knowledge_projection(text[], boolean)` | `0024`; remove only after vector-repair adoption/drain | + +Very old installations may retain the earlier `embedding_search_connector_sync` document trigger, +`embedding_search_connector_set` projection trigger, `sync_embedding_search_connector()`, and +`set_embedding_search_connector()`. The current ACL installer removes those names; an idempotent +contract should account for installations that skipped it without using `CASCADE`. + +Keep `embedding_search_sync` on `embedding` and `sync_embedding_search()` until their ordinary-KB +replacement is active. Their vector projection is shared. Their binary computations can be retired +under the constraint transition above. The `sim.projection_mode` transaction setting is also used by +existing repair workers to skip synchronous triggers, so removing that guard before those workers +drain needs separate review. + +Dropping triggers requires relation locks. Use short lock timeouts and bounded retries in the +contract; a waiting DDL statement must not queue production writers indefinitely. Remove triggers +before their functions, then dependent indexes/columns/tables. Do not use broad `DROP ... CASCADE`. +The optional `tin` extension is not automatically droppable: confirm no other schema or application +uses it before scheduling extension removal. + +## Application and bootstrap dependencies + +The app-removal release must remove or narrow these responsibilities: + +- Indexed Search retrieval, projection-fill checks, and keyword/Tin capability probes. +- Source/ACL copying and keyword projection in `packages/db/knowledge-projection.ts`; retain bounded + repair of ordinary-KB vector content left by older asynchronous writers. +- The `knowledge-projection` background task and its enqueue/sweep path only after vector repair is + complete or adopted. While retained, it must not recreate retired keyword data. +- `prewarmSearchProjection`: keep shared vector warming if useful, but stop warming retired keyword + heaps and ACL GIN indexes. `pg_prewarm` itself is not specific to enterprise Search. +- Connector detachment's direct updates of copied source/ACL fields. These writes are currently paged + to prevent document updates from causing unbounded trigger fan-out. Keep that protection until the + database fan-out trigger has been retired, even if the new application no longer reads the copies. +- ACL-change page sizing in member observations and source permission persistence. Canonical + document ACLs still matter; only their projection-row accounting can simplify after fan-out stops. +- Operator maintenance commands that refer to retired tables. In particular, `0028` currently lists + both keyword tables for vacuum and discovers every HNSW index on the shared vector table. Physical + contraction must first replace or retire this command so a retry does not target removed objects. + +Historical SQL migrations remain intact. Script migrations run **after all SQL migrations**; old +pending scripts must not reinstall retired objects after a contract or fail because their target +columns have disappeared. `db:push` has its own reconciliation command list and needs the same review. + +| Script migration | Treatment | +| --- | --- | +| `0015_backfill_embedding_search` | Already superseded by `0016`; historical binary projection installer, not a reason to recreate binary indexes. | +| `0016_backfill_search_vectors` and `0017_index_search_documents` | Mixed shared/retired responsibilities. They install/backfill ordinary-KB vectors and build required vector indexes as well as legacy keyword objects. Do not unregister them wholesale without a replacement shared-vector bootstrap and upgrade path. | +| `0019_tin_keyword_projection` | Entirely retired keyword projection, but its installed functions/triggers remain until contract. Supersede in a reviewed bootstrap transition. | +| `0021_embedding_search_connector`, `0022_projection_source_acl_backfill`, `0023_projection_acl_skip_unfilled` | Entirely retired projection-copy installation/indexing/backfill. `0022` already supersedes `0021`. Removing application use does not justify replaying their table-wide work. | +| `0024_knowledge_projection_async` | Mixed legacy ACL and old vector-repair machinery. Supersede only with a vector-safe replacement and pending-mark handling. | +| `0025_scope_keyword_projections` | Scopes legacy keyword writers and adjusts the document ACL trigger. It also installs the membership helper used by the old projector. Remove with the dependent machinery. | +| `0027`–`0029` Search retirement | Already absent from automatic deployment. Preserve their checkpoints until successor cleanup adoption; do not reintroduce automatic deletion, vacuum, or HNSW rebuilding. | + +The current registry and `scripts/push.ts` are intentionally not changed by the inventory alone. +Simply omitting legacy installers on fresh installs is unsafe while mixed-version application paths, +repair workers, or later installers still expect their functions. A successor must define the full +final bootstrap, explicitly supersede the old script receipts, and work for empty, partially +migrated, and already-running databases without scanning all existing data during deployment. + +## Shared objects that remain + +- `embedding`, its full-precision vectors, content TSV/GIN index, tags, and provenance sidecar. + Regular KB exact reranking, keyword retrieval, document filtering, and secret provenance use them. +- `embedding_search` identities, enabled state, half-precision vectors, six HNSW indexes, KB index, + document lookup index, and embedding FK. Ordinary KB ANN retrieval uses this table. A replacement + must preserve current inserts, updates, deletes, and document-level authorization before cutover. +- `document.acl`, `acl_requirements`, `acl_verified_at`, the ACL GIN index/shape check, `connector_id`, + source URL/modified/seen timestamps, and source indexes. Ordinary KB reads still enforce the + document access predicate; workspace connectors can use member/admin permissions, and ordinary + KB filters use source modification times. Source-seen time drives connector absence reconciliation. +- `knowledge_connector_member`, `knowledge_document_observation`, external directory/group tables, + permission snapshot/grant tables, member sync logs, and their indexes. They support ordinary KB + permission-scoped connectors; deleting them would remove authorization evidence. +- Connector access mode, credential groups, member/admin sync state, permission/listing checkpoints, + partitions, retry fields, and detachment billing reservations. These are shared connector machinery. +- `knowledge_base.is_search_index`, organization ownership, and Search KB uniqueness constraints. + Live Search still loads configured sources through Search-marked KBs. A marker is also the durable + retirement target boundary; it is not proof that the parent KB row itself can be deleted. +- `organization_search_integration`, live Search activity/OAuth structures, provider credentials, and live + source configuration. They belong to the live product as well as the retired implementation. +- `workspace_file_search_*`, Slack Search, and documentation embeddings. These are separate search + products and are outside this retirement. + +The old `doc_processing_recovery_idx` is independently marked as superseded by the per-source +recovery index in `schema.ts`. It is a separate contract candidate after its replacement release is +verified; it is not evidence that ordinary KB processing recovery can be removed. + +## Release order and completion evidence + +1. Deploy removal of indexed readers/admission paths and narrow remaining shared workers. Keep the + compatible schema and trigger protection while older web tasks, queued jobs, and rollback images + may still use them. Retain the `contract-pending` markers in `schema.ts`. +2. Verify the release is fully deployed, old workers have drained, and rollback cannot reactivate + indexed Search. Adopt or complete pending vector repairs before dropping their queue. Verify + ordinary KB retrieval and permission changes against real PostgreSQL boundaries. +3. Retire legacy triggers/functions and installers with a shared-vector-safe bootstrap. Preserve + historical migration replay and inspect unknown dependencies rather than cascading through them. + This stops future keyword/ACL copying without rewriting existing vector rows. +4. Build any replacement projection through separately controlled, resumable maintenance. Copy only + ordinary-KB rows, capture concurrent changes, validate retrieval/authorization, then switch readers + while retaining a tested rollback path. This inventory does not start that copy or rebuild. +5. Contract retired tables/columns/indexes only after those readers and writers are gone. Reference + the deployed removal release in migration safety acknowledgments and remove the corresponding + `contract-pending` markers. Keep cleanup state until content retirement is independently verified. + +Code deletion, stopped writes, copied data, a successful read cutover, and physical reclamation are +different completion conditions. Report them separately; none implies that all the others happened. diff --git a/packages/db/script-migrations/search-embedding-retirement.md b/packages/db/script-migrations/search-embedding-retirement.md index 86d48536e5b..7b8864c4499 100644 --- a/packages/db/script-migrations/search-embedding-retirement.md +++ b/packages/db/script-migrations/search-embedding-retirement.md @@ -15,8 +15,8 @@ KBs' live source/credential configuration, document metadata, and backing files ## Before running The app and workers must already use live Search, and older indexing jobs must be drained. -`SIM_SEARCH_LIVE=true` (the default) makes `isIndexedOrgSearchEnabled()` false. **`SIM_SEARCH_LIVE=false` -enables indexed Search again.** The cleanup does not inspect this flag. Live source setup may still +Enterprise Search no longer has an indexed backend or an environment toggle to re-enable it. +Older releases could re-enable indexed Search, so their workers must be drained before retirement. Live source setup may still create a Search KB for configuration; it does not index content. Document uploads, dispatch and queued processing also honor the indexed-search gate. diff --git a/packages/testing/src/mocks/deployment-shape.mock.ts b/packages/testing/src/mocks/deployment-shape.mock.ts index 443f6ba8082..0b61e4aaf26 100644 --- a/packages/testing/src/mocks/deployment-shape.mock.ts +++ b/packages/testing/src/mocks/deployment-shape.mock.ts @@ -8,7 +8,6 @@ export interface MockDeploymentShape { azureConfigured: boolean cohereConfigured: boolean features: { - liveEnterpriseSearch?: boolean accessControl: boolean auditLogs: boolean customBlocks: boolean @@ -43,7 +42,6 @@ export function createMockDeploymentShape( cohereConfigured: false, ...rest, features: { - liveEnterpriseSearch: true, accessControl: false, auditLogs: false, customBlocks: false, diff --git a/packages/testing/src/mocks/env-flags.mock.ts b/packages/testing/src/mocks/env-flags.mock.ts index 98666e661ec..cba6ac6fc4c 100644 --- a/packages/testing/src/mocks/env-flags.mock.ts +++ b/packages/testing/src/mocks/env-flags.mock.ts @@ -32,7 +32,6 @@ interface EnvFlagsMockState { isUsageMonitoringEnabled: boolean isAccessControlEnabled: boolean isOrganizationsEnabled: boolean - isLiveEnterpriseSearchEnabled: boolean isInboxEnabled: boolean isSandboxDeploymentEntitled: boolean isSandboxesEnabled: boolean @@ -87,7 +86,6 @@ const defaultEnvFlagsState: EnvFlagsMockState = { isAccessControlEnabled: false, isOrganizationsEnabled: false, /** Live Search is the default Sim Search backend; indexed search is dormant. */ - isLiveEnterpriseSearchEnabled: true, // True with billing off and no flags set — these carry a legacy default of // `true` so upgrades do not remove a feature. See // ENTERPRISE_FEATURE_LEGACY_DEFAULTS. diff --git a/packages/testing/src/mocks/indexed-org-search.mock.ts b/packages/testing/src/mocks/indexed-org-search.mock.ts deleted file mode 100644 index d13b8db4a63..00000000000 --- a/packages/testing/src/mocks/indexed-org-search.mock.ts +++ /dev/null @@ -1,22 +0,0 @@ -/** - * The `vi.mock` factory a real-infrastructure suite of indexed organization search passes for - * `@/lib/core/config/env-flags`: the real module, with Live Search off so the dormant indexed - * backend is the one selected. Unit suites use the shared env-flags mock's `setEnvFlags` instead. - * - * Loaded inside the factory, which runs before the test file's own imports are initialized. - * - * @example - * ```ts - * vi.mock('@/lib/core/config/env-flags', async (importOriginal) => - * (await import('@sim/testing/mocks/indexed-org-search.mock')).indexedOrgSearchEnvFlags(importOriginal) - * ) - * ``` - */ -export async function indexedOrgSearchEnvFlags( - importOriginal: () => Promise -): Promise> { - return { - ...(await importOriginal>()), - isLiveEnterpriseSearchEnabled: false, - } -} diff --git a/packages/testing/src/mocks/kb-connectors-queries.mock.ts b/packages/testing/src/mocks/kb-connectors-queries.mock.ts index 107e821d42d..f5e3ef8fe05 100644 --- a/packages/testing/src/mocks/kb-connectors-queries.mock.ts +++ b/packages/testing/src/mocks/kb-connectors-queries.mock.ts @@ -21,13 +21,6 @@ const connectorKeys = { details: (knowledgeBaseId?: string) => [...connectorKeys.all(knowledgeBaseId), 'detail'] as const, detail: (knowledgeBaseId?: string, connectorId?: string) => [...connectorKeys.details(knowledgeBaseId), connectorId ?? ''] as const, - progress: (knowledgeBaseId?: string, connectorId?: string, scope?: MockResourceScope) => - [ - ...connectorKeys.progresses(knowledgeBaseId, connectorId), - scope ? resourceScopeKey(scope) : '', - ] as const, - progresses: (knowledgeBaseId?: string, connectorId?: string) => - [...connectorKeys.detail(knowledgeBaseId, connectorId), 'progress'] as const, } const searchIndexKeys = { @@ -55,7 +48,7 @@ const mutationHook = () => vi.fn((..._args: unknown[]): unknown => createMutatio * - `mockIsConnectorSyncingOrPending` is the real predicate (`status` `pending`/`syncing`, or * `memberSyncStatus` `pending`/`running`). * - Query hooks (`useConnectorList`, `useConnectorDetail`, `useSearchIndex`, - * `useSearchSourceOverview`, `useOrganizationSearchOverview`, `useSearchSources`, + * `useSearchSources`, * `useConnectorDocuments`) return a fresh {@link createQueryResultMock} (`data: undefined`, * `isPending: true`). * - Every mutation hook returns a fresh {@link createMutationResultMock} (`idle`, no-op @@ -82,8 +75,6 @@ export const kbConnectorsQueriesMockFns = { mockUseCreateConnector: mutationHook(), mockUseUpdateConnector: mutationHook(), mockUseSearchIndex: queryHook(), - mockUseSearchSourceOverview: queryHook(), - mockUseOrganizationSearchOverview: queryHook(), mockUseSearchSources: queryHook(), mockUseStartConnectorMemberEnrollment: mutationHook(), mockUseUpdateConnectorAccess: mutationHook(), @@ -92,7 +83,6 @@ export const kbConnectorsQueriesMockFns = { mockUseConnectorDocuments: queryHook(), mockUseExcludeConnectorDocument: mutationHook(), mockUseRestoreConnectorDocument: mutationHook(), - mockUseConnectSimSearchConnector: mutationHook(), mockUsePrepareSearchSource: mutationHook(), } @@ -121,8 +111,6 @@ export const kbConnectorsQueriesMock = { useCreateConnector: kbConnectorsQueriesMockFns.mockUseCreateConnector, useUpdateConnector: kbConnectorsQueriesMockFns.mockUseUpdateConnector, useSearchIndex: kbConnectorsQueriesMockFns.mockUseSearchIndex, - useSearchSourceOverview: kbConnectorsQueriesMockFns.mockUseSearchSourceOverview, - useOrganizationSearchOverview: kbConnectorsQueriesMockFns.mockUseOrganizationSearchOverview, useSearchSources: kbConnectorsQueriesMockFns.mockUseSearchSources, useStartConnectorMemberEnrollment: kbConnectorsQueriesMockFns.mockUseStartConnectorMemberEnrollment, @@ -132,6 +120,5 @@ export const kbConnectorsQueriesMock = { useConnectorDocuments: kbConnectorsQueriesMockFns.mockUseConnectorDocuments, useExcludeConnectorDocument: kbConnectorsQueriesMockFns.mockUseExcludeConnectorDocument, useRestoreConnectorDocument: kbConnectorsQueriesMockFns.mockUseRestoreConnectorDocument, - useConnectSimSearchConnector: kbConnectorsQueriesMockFns.mockUseConnectSimSearchConnector, usePrepareSearchSource: kbConnectorsQueriesMockFns.mockUsePrepareSearchSource, } diff --git a/scripts/test-patterns-baseline.json b/scripts/test-patterns-baseline.json index 0702a5bd1e4..dc44ea44c49 100644 --- a/scripts/test-patterns-baseline.json +++ b/scripts/test-patterns-baseline.json @@ -59,7 +59,6 @@ "local-factory\tapps/sim/executor/utils/output-filter.test.ts\t@/blocks", "local-factory\tapps/sim/hooks/use-table-undo.test.ts\t@/lib/table/constants", "local-factory\tapps/sim/lib/credentials/application/resolve-organization-personal-token.test.ts\t@/lib/sim-search/connectors", - "local-factory\tapps/sim/lib/knowledge/application/organization-search-overview.test.ts\t@/lib/sim-search/connectors", "local-factory\tapps/sim/lib/knowledge/application/search-integrations.test.ts\t@/lib/sim-search/connectors", "local-factory\tapps/sim/lib/knowledge/application/sim-search.test.ts\t@/lib/sim-search/connectors", "local-factory\tapps/sim/lib/knowledge/mcp/route-handler.test.ts\t@/lib/sim-search/connectors", From edfc88e112646b9fd9a0fa4cb24a5c62ad521f83 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 1 Oct 2026 11:44:06 -0700 Subject: [PATCH 2/4] fix(search): preserve live onboarding and Slack scope policies --- .github/workflows/ci.yml | 1 - .../components/get-started/get-started.tsx | 66 +++++++++++++++++-- .../integrations/page.test.tsx | 25 ------- .../integrations/search-source-setup.tsx | 4 +- apps/sim/hooks/queries/search-integrations.ts | 3 +- .../mothership-management-tools.test.ts | 2 +- .../slack-managed-user-scopes.ts | 5 +- .../slack-managed-users.test.ts | 38 +++++++++-- .../credential-groups/slack-managed-users.ts | 15 ++++- .../credential-groups/slack-provider.test.ts | 16 +---- .../lib/credential-groups/slack-provider.ts | 6 +- .../knowledge/application/operations.test.ts | 15 ----- apps/sim/lib/knowledge/constants.ts | 1 - .../server/knowledge/workspace-search.ts | 2 +- apps/sim/lib/organizations/surface.test.ts | 21 ++++++ apps/sim/lib/organizations/surface.ts | 4 ++ apps/sim/lib/sim-search/live/scopes.ts | 2 +- 17 files changed, 144 insertions(+), 82 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a30f3f9eec..547db6f3459 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -222,7 +222,6 @@ jobs: platforms: linux/amd64 tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:${{ github.sha }}-dev build-args: | - SIM_SEARCH_LIVE_DEFAULT=true MSHIP_PLAN_MODE_DEFAULT=true max-cache-size-mb: ${{ matrix.cache_mb }} diff --git a/apps/sim/app/o/[organizationId]/home/components/get-started/get-started.tsx b/apps/sim/app/o/[organizationId]/home/components/get-started/get-started.tsx index ac6e75ecd0b..92772134d27 100644 --- a/apps/sim/app/o/[organizationId]/home/components/get-started/get-started.tsx +++ b/apps/sim/app/o/[organizationId]/home/components/get-started/get-started.tsx @@ -7,9 +7,14 @@ import Link from 'next/link' import { HomeSection } from '@/components/home/home-section' import { OAUTH_SEARCH_READ_SCOPE, oauthScopeSatisfies } from '@/lib/auth/oauth-provider' import { organizationRoutes } from '@/lib/navigation/paths' +import { + liveSearchProviderForCredential, + supportsLiveSearchMode, +} from '@/lib/sim-search/live/provider-catalog' import { useOrganizationContext } from '@/app/o/[organizationId]/providers/organization-provider' import { useAuthorizedApps } from '@/hooks/queries/oauth-provider' import { useOrganizationAccounts } from '@/hooks/queries/organization-accounts' +import { useSearchIntegrations } from '@/hooks/queries/search-integrations' type StepId = 'connect-integration' | 'connect-sim-search' @@ -69,9 +74,15 @@ function StepMark({ complete }: { complete: boolean }) { * and reads as done from the organization's real state: a connected account and an OAuth app authorized to use Search. */ export function GetStarted() { - const { organization, viewer } = useOrganizationContext() + const { organization, viewer, connectedAccountsAvailable } = useOrganizationContext() const routes = organizationRoutes(organization.id) - const { data: accounts } = useOrganizationAccounts(organization.id) + const canConnectIntegrations = viewer.canConnectSearchIntegrations && connectedAccountsAvailable + const { data: accounts } = useOrganizationAccounts( + canConnectIntegrations ? organization.id : undefined + ) + const { data: integrations } = useSearchIntegrations(organization.id, { + enabled: canConnectIntegrations, + }) const { data: authorizedApps, fetchNextPage, @@ -83,6 +94,46 @@ export function GetStarted() { authorizedApps?.pages.some((page) => page.apps.some((app) => oauthScopeSatisfies(app.scopes, OAUTH_SEARCH_READ_SCOPE)) ) ?? false + const approvedProviders = new Set( + integrations + ?.filter((integration) => integration.approved && integration.available !== false) + .map((integration) => integration.connectorType) + ) + const readyOptions = new Set( + accounts?.credentialGroup?.options + .filter((option) => { + const provider = liveSearchProviderForCredential(option.provider) + return ( + option.status === 'active' && + option.configurationStatus === 'ready' && + provider && + supportsLiveSearchMode(provider, 'member') && + approvedProviders.has(provider) + ) + }) + .map((option) => option.id) + ) + const readyMcpServers = new Set( + accounts?.credentialGroup?.mcpServers + .filter((server) => { + const provider = liveSearchProviderForCredential(`mcp:${server.managedConnectorId}`) + return ( + server.enabled && + provider && + approvedProviders.has(provider) && + accounts.availableMcpConnectors.some((id) => id === server.managedConnectorId) + ) + }) + .map((server) => server.id) + ) + const hasSearchConnection = + accounts?.credentialGroup?.status === 'active' && + (accounts.viewerAccounts?.some( + (account) => account.status === 'active' && readyOptions.has(account.optionId) + ) || + accounts.viewerMcpAccounts?.some( + (account) => account.status === 'active' && readyMcpServers.has(account.mcpServerId) + )) const hrefs: Record = { 'connect-integration': viewer.isAdmin @@ -91,13 +142,12 @@ export function GetStarted() { 'connect-sim-search': routes.settingsSection('search-mcp'), } const completed: Record = { - 'connect-integration': Boolean( - accounts?.viewerAccounts?.some((account) => account.status === 'active') || - accounts?.viewerMcpAccounts?.some((account) => account.status === 'active') - ), + 'connect-integration': Boolean(hasSearchConnection), 'connect-sim-search': hasSearchAuthorization, } - const steps = STEPS.filter((step) => step.id !== 'connect-sim-search' || viewer.canUseSearchMcp) + const steps = STEPS.filter((step) => + step.id === 'connect-sim-search' ? viewer.canUseSearchMcp : canConnectIntegrations + ) const [expanded, setExpanded] = useState(true) /** @@ -132,6 +182,8 @@ export function GetStarted() { setExpanded((prev) => !prev) } + if (steps.length === 0) return null + return ( { }) describe('integrations page Slack context', () => { - it('preserves a requested connection across login and validates it in the existing organization page', async () => { - const selected = { - ...props, - searchParams: Promise.resolve({ - connectorType: 'gmail', - connectorId: 'source', - credentialId: 'account', - }), - } - const page = await OrganizationIntegrationsPage(selected) - expect(page.props.connectionRequest).toMatchObject({ - userId: 'viewer', - target: { - type: 'link', - connectorType: 'gmail', - connectorId: 'source', - credentialId: 'account', - }, - }) - authMockFns.mockGetSession.mockResolvedValue(null) - await expect(OrganizationIntegrationsPage(selected)).rejects.toThrow('NEXT_REDIRECT') - expect(mockRedirect).toHaveBeenCalledWith( - `/login?callbackUrl=${encodeURIComponent('/o/organization-a/integrations?connectorType=gmail&connectorId=source&credentialId=account')}` - ) - }) it('rejects unknown providers and reconnects without a source', async () => { for (const query of [ { connectorType: 'invented' }, diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.tsx b/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.tsx index d20238282f5..ddbb853ebe9 100644 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.tsx +++ b/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.tsx @@ -133,7 +133,7 @@ export function SearchSourceSetup({ } const failedQuery = index.isError ? index : null const initialMode = (type: string) => - type === 'github' ? ('members' as const) : ('admin' as const) + type === 'github' || type === 'slack' ? ('members' as const) : ('admin' as const) const selectedAccessMode = selectedType ? initialMode(selectedType) : undefined const selectedAvailability = selectedMeta @@ -359,7 +359,7 @@ export function SearchSourceSetup({ isIntegrationAvailabilityReady, } ) - const available = type === 'github' ? members : central + const available = initialMode(type) === 'members' ? members : central return ( (await requestJson(listSearchIntegrationsContract, { query: { organizationId }, signal })) .data, diff --git a/apps/sim/lib/api/contracts/mothership-management-tools.test.ts b/apps/sim/lib/api/contracts/mothership-management-tools.test.ts index 07ee0bc30bf..e64a8040214 100644 --- a/apps/sim/lib/api/contracts/mothership-management-tools.test.ts +++ b/apps/sim/lib/api/contracts/mothership-management-tools.test.ts @@ -26,7 +26,7 @@ const examples = { { action: 'update', scope: 'account', section: 'profile', changes: { timezone: 'UTC' } }, ], search_sources: [ - { action: 'list', connectorType: 'google_drive', mine: true }, + { action: 'list', connectorType: 'google_drive' }, { action: 'get', connectorId: 'source-1' }, { action: 'providers' }, { action: 'setup', connectorType: 'google_drive', accessMode: 'admin' }, diff --git a/apps/sim/lib/credential-groups/slack-managed-user-scopes.ts b/apps/sim/lib/credential-groups/slack-managed-user-scopes.ts index 4be542eefdf..1193daefb55 100644 --- a/apps/sim/lib/credential-groups/slack-managed-user-scopes.ts +++ b/apps/sim/lib/credential-groups/slack-managed-user-scopes.ts @@ -1,3 +1,5 @@ +import { SLACK_RTS_USER_SCOPES } from '@/lib/sim-search/live/scopes' + /** * User-token policy requested and verified by Credential Group Slack OAuth. * This is independent of the custom bot manifest and its configuration UI. @@ -48,12 +50,13 @@ export const SLACK_CHANNEL_READ_SCOPES = [ export const SLACK_DM_READ_SCOPES = ['im:history', 'im:read', 'mpim:history', 'mpim:read'] as const -/** The shared organization app grants member access for channel and DM indexing. */ +/** Explicit Search setup grants channel, DM, and live retrieval permissions together. */ export const SLACK_SEARCH_USER_SCOPES = [ ...SLACK_CHANNEL_READ_SCOPES, ...SLACK_DM_READ_SCOPES, 'users:read', 'users:read.email', + ...SLACK_RTS_USER_SCOPES, ] as const /** Existing workflow options retain their scope policy; every user grant must attest identity. */ diff --git a/apps/sim/lib/credential-groups/slack-managed-users.test.ts b/apps/sim/lib/credential-groups/slack-managed-users.test.ts index 4a3e89d887f..ce9e8301da7 100644 --- a/apps/sim/lib/credential-groups/slack-managed-users.test.ts +++ b/apps/sim/lib/credential-groups/slack-managed-users.test.ts @@ -199,6 +199,7 @@ describe('Slack managed-user authorization', () => { existingScopes: undefined, requestedScopes: SLACK_MANAGED_USER_SCOPES, scopes: SLACK_SEARCH_USER_SCOPES, + upgradesSearchPolicy: false, }, { name: 'legacy workflow pool without explicit scopes', @@ -206,6 +207,7 @@ describe('Slack managed-user authorization', () => { existingScopes: undefined, requestedScopes: SLACK_SEARCH_USER_SCOPES, scopes: SLACK_MANAGED_USER_SCOPES, + upgradesSearchPolicy: false, }, { name: 'existing Search pool', @@ -213,10 +215,30 @@ describe('Slack managed-user authorization', () => { existingScopes: SLACK_SEARCH_USER_SCOPES, requestedScopes: SLACK_MANAGED_USER_SCOPES, scopes: SLACK_SEARCH_USER_SCOPES, + upgradesSearchPolicy: false, + }, + { + name: 'legacy Search pool', + existing: true, + existingScopes: [ + 'channels:history', + 'channels:read', + 'groups:history', + 'groups:read', + 'im:history', + 'im:read', + 'mpim:history', + 'mpim:read', + 'users:read', + 'users:read.email', + ], + requestedScopes: SLACK_MANAGED_USER_SCOPES, + scopes: SLACK_SEARCH_USER_SCOPES, + upgradesSearchPolicy: true, }, ])( - 'verifies an organization $name without replacing its scope policy or disconnecting members', - async ({ existing, existingScopes, requestedScopes, scopes }) => { + 'verifies an organization $name with its explicit Search or workflow scope policy', + async ({ existing, existingScopes, requestedScopes, scopes, upgradesSearchPolicy }) => { const updatedAt = new Date('2026-08-12T00:00:00Z') const group = { id: 'group-1', @@ -232,7 +254,9 @@ describe('Slack managed-user authorization', () => { required: true, authorizationAppId: 'slack:A123:T123', requiredScopes: existingScopes, - scopeVersion: credentialGroupScopePolicyVersion([...scopes]), + scopeVersion: credentialGroupScopePolicyVersion([ + ...(existingScopes ?? SLACK_MANAGED_USER_SCOPES), + ]), }, ] : [], @@ -264,7 +288,6 @@ describe('Slack managed-user authorization', () => { const attempt = await consumeSlackManagedUsersAttempt(created.state) expect(attempt?.requiredScopes).toEqual([...scopes]) if (!attempt) throw new Error('Expected an organization authorization attempt') - if (!existing) expect(attempt.requiredScopes).toHaveLength(10) queueTableRows(schemaMock.slackApp, [app]) queueTableRows(schemaMock.credentialGroup, [group]) @@ -301,9 +324,10 @@ describe('Slack managed-user authorization', () => { options: [expect.objectContaining({ requiredScopes: [...scopes] })], }) ) - expect(dbChainMockFns.set).not.toHaveBeenCalledWith( - expect.objectContaining({ managedOauthStatus: 'needs_reauth' }) - ) + if (!upgradesSearchPolicy) + expect(dbChainMockFns.set).not.toHaveBeenCalledWith( + expect.objectContaining({ managedOauthStatus: 'needs_reauth' }) + ) } ) diff --git a/apps/sim/lib/credential-groups/slack-managed-users.ts b/apps/sim/lib/credential-groups/slack-managed-users.ts index ce7982f217f..e7a12675a1e 100644 --- a/apps/sim/lib/credential-groups/slack-managed-users.ts +++ b/apps/sim/lib/credential-groups/slack-managed-users.ts @@ -24,6 +24,8 @@ import { } from '@/lib/credential-groups/provider-configuration' import { resolveSlackManagedUserScopes, + SLACK_CHANNEL_READ_SCOPES, + SLACK_DM_READ_SCOPES, SLACK_MANAGED_USER_CONFIGURATION_CALLBACK_PATH, SLACK_SEARCH_USER_SCOPES, } from '@/lib/credential-groups/slack-managed-user-scopes' @@ -532,8 +534,19 @@ export async function createSlackManagedUsersAttempt(params: { clientId = app.clientId clientSecret = app.clientSecret appRevision = app.revision + const retiredSearchScopes = new Set([ + ...SLACK_CHANNEL_READ_SCOPES, + ...SLACK_DM_READ_SCOPES, + 'users:read', + 'users:read.email', + ]) + const upgradesSearchPolicy = + existingOption?.requiredScopes?.length === retiredSearchScopes.size && + existingOption.requiredScopes.every((scope) => retiredSearchScopes.has(scope)) requiredScopes = resolveSlackManagedUserScopes( - existingOption ? existingOption.requiredScopes : SLACK_SEARCH_USER_SCOPES + !existingOption || upgradesSearchPolicy + ? SLACK_SEARCH_USER_SCOPES + : existingOption.requiredScopes ) } else { if (!params.slackBotCredentialId) diff --git a/apps/sim/lib/credential-groups/slack-provider.test.ts b/apps/sim/lib/credential-groups/slack-provider.test.ts index e8d0a6b0b23..5acbe0394cf 100644 --- a/apps/sim/lib/credential-groups/slack-provider.test.ts +++ b/apps/sim/lib/credential-groups/slack-provider.test.ts @@ -39,7 +39,7 @@ describe('Slack member scope policy', () => { clientSecret: 'secret', appId: 'A1', teamId: 'T1', - scopes: [...SLACK_MANAGED_USER_SCOPES], + scopes: [...new Set([...SLACK_MANAGED_USER_SCOPES, ...SLACK_SEARCH_USER_SCOPES])], }) mocks.exchange.mockResolvedValue({ appId: 'A1', @@ -79,7 +79,7 @@ describe('Slack member scope policy', () => { } it.each([{ scopes: SLACK_SEARCH_USER_SCOPES }, { scopes: SLACK_MANAGED_USER_SCOPES }])( - 'requests RTS consent while retaining the stored option policy', + 'requests exactly the configured permissions without broadening workflow consent', async ({ scopes }) => { const current = context(scopes) const policy = await adapter.getPolicy(current.option, { @@ -90,17 +90,7 @@ describe('Slack member scope policy', () => { const url = new URL( await authorization.buildAuthorizationUrl({ state: 'state', nonce: 'nonce' }) ) - expect(url.searchParams.get('user_scope')?.split(',')).toEqual( - expect.arrayContaining([ - ...scopes, - 'search:read.public', - 'search:read.private', - 'search:read.im', - 'search:read.mpim', - 'search:read.files', - 'files:read', - ]) - ) + expect(url.searchParams.get('user_scope')?.split(',')).toEqual([...scopes]) expect(policy.requiredScopes).toEqual([...scopes]) } ) diff --git a/apps/sim/lib/credential-groups/slack-provider.ts b/apps/sim/lib/credential-groups/slack-provider.ts index 8acc6001224..f94d443e428 100644 --- a/apps/sim/lib/credential-groups/slack-provider.ts +++ b/apps/sim/lib/credential-groups/slack-provider.ts @@ -28,7 +28,6 @@ import { verifySlackUserIdentity, } from '@/lib/credential-groups/slack-managed-users' import type { DbOrTx } from '@/lib/db/types' -import { SLACK_RTS_USER_SCOPES } from '@/lib/sim-search/live/scopes' const PROVIDER = 'slack' as const @@ -180,10 +179,7 @@ export const slackCredentialGroupProviderAdapter: CredentialGroupProviderAdapter buildAuthorizationUrl: ({ state }) => { const authorizationUrl = new URL('https://slack.com/oauth/v2/authorize') authorizationUrl.searchParams.set('client_id', currentPolicy.clientId) - authorizationUrl.searchParams.set( - 'user_scope', - [...new Set([...policy.requiredScopes, ...SLACK_RTS_USER_SCOPES])].join(',') - ) + authorizationUrl.searchParams.set('user_scope', policy.requiredScopes.join(',')) authorizationUrl.searchParams.set('redirect_uri', redirectUri) authorizationUrl.searchParams.set('state', state) authorizationUrl.searchParams.set('team', currentPolicy.teamId) diff --git a/apps/sim/lib/knowledge/application/operations.test.ts b/apps/sim/lib/knowledge/application/operations.test.ts index 0f91cbc3c8d..1f700e47137 100644 --- a/apps/sim/lib/knowledge/application/operations.test.ts +++ b/apps/sim/lib/knowledge/application/operations.test.ts @@ -42,26 +42,11 @@ describe('knowledge operation registry', () => { knowledgeOperations.search, knowledgeOperations.readSearchIndex, knowledgeOperations.enrollConnectorMember, - knowledgeOperations.simSearchConnect, - knowledgeOperations.listPersonalSourceSetupAccounts, - knowledgeOperations.personalSourceSetup, ]) { expect(operation.organizationOperation.minimumRole).toBe('member') } }) - it('limits personal source setup to the signed-in member without delegating credential discovery', () => { - for (const operation of [ - knowledgeOperations.listPersonalSourceSetupAccounts, - knowledgeOperations.personalSourceSetup, - ]) { - expect(operation.principalKinds).toEqual(['session']) - expect(operation.organizationOperation.principalKinds).not.toContain('organization_delegated') - expect(operation.workspaceApiKey).toBe('deny') - expect(operation.capability).toBe('knowledge.use') - } - }) - it('keeps human-delegated tag, connector, and composed document operations off workspace keys', () => { const operations = [ knowledgeOperations.updateDocument, diff --git a/apps/sim/lib/knowledge/constants.ts b/apps/sim/lib/knowledge/constants.ts index a292948b174..3f9093870d8 100644 --- a/apps/sim/lib/knowledge/constants.ts +++ b/apps/sim/lib/knowledge/constants.ts @@ -29,7 +29,6 @@ export const DEFAULT_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE = 100 export const MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE = 200 export const MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_SEARCH_LENGTH = 200 -/** Maximum source IDs in one viewer-authorized progress request. */ /** Bound viewer-specific source resolution and document counts to a single page. */ export const SEARCH_SOURCE_PAGE_SIZE = 25 export const SEARCH_SOURCE_CANDIDATE_PAGE_SIZE = 100 diff --git a/apps/sim/lib/mothership/tools/server/knowledge/workspace-search.ts b/apps/sim/lib/mothership/tools/server/knowledge/workspace-search.ts index eb748e5bc45..ce9957cfec1 100644 --- a/apps/sim/lib/mothership/tools/server/knowledge/workspace-search.ts +++ b/apps/sim/lib/mothership/tools/server/knowledge/workspace-search.ts @@ -87,7 +87,7 @@ export const searchWorkspaceServerTool: BaseServerTool = { }) return { success: true, - message: `Found ${data.results.length} live results. Read a documentId when its passage does not answer the question or more context is needed. ${CITATION_INSTRUCTION}`, + message: `${data.retrieval.status === 'partial' ? 'Search coverage is incomplete. Continue with a more specific query or source filter; these results cannot establish absence or completeness. ' : ''}Found ${data.results.length} live results. Read a documentId when its passage does not answer the question or more context is needed. ${CITATION_INSTRUCTION}`, data: { ...data, results: data.results.map((item) => ({ diff --git a/apps/sim/lib/organizations/surface.test.ts b/apps/sim/lib/organizations/surface.test.ts index 469a4b660a0..474b6ba9991 100644 --- a/apps/sim/lib/organizations/surface.test.ts +++ b/apps/sim/lib/organizations/surface.test.ts @@ -49,6 +49,27 @@ describe('getOrganizationSurfaceContext', () => { setEnvFlags({ isInvitationsDisabled: false, isHosted: true, isBillingEnabled: true }) }) + it.each([ + { integrationsDenied: false, knowledgeDenied: false, allowed: true }, + { integrationsDenied: true, knowledgeDenied: false, allowed: false }, + { integrationsDenied: false, knowledgeDenied: true, allowed: false }, + ])( + 'respects Search connection permissions: %o', + async ({ integrationsDenied, knowledgeDenied, allowed }) => { + queueTableRows(member, [{ role: 'member' }]) + queueTableRows(organization, [{ id: 'org-1', name: 'Acme', slug: 'acme', logo: null }]) + queueTableRows(member, [{ memberCount: 1 }]) + mockPermissionConfig.mockResolvedValue({ + ...DEFAULT_PERMISSION_GROUP_CONFIG, + hideIntegrationsTab: integrationsDenied, + hideKnowledgeBaseTab: knowledgeDenied, + }) + await expect(getOrganizationSurfaceContext('org-1', 'viewer')).resolves.toMatchObject({ + viewer: { canConnectSearchIntegrations: allowed }, + }) + } + ) + it.each([ { role: 'owner', billing: true, denied: false, expected: true }, { role: 'admin', billing: true, denied: true, expected: false }, diff --git a/apps/sim/lib/organizations/surface.ts b/apps/sim/lib/organizations/surface.ts index 39237785860..91eb4a34a23 100644 --- a/apps/sim/lib/organizations/surface.ts +++ b/apps/sim/lib/organizations/surface.ts @@ -36,6 +36,7 @@ interface OrganizationSurfaceViewer { role: OrganizationRole isAdmin: boolean canInviteMembers: boolean + canConnectSearchIntegrations: boolean canUsePersonalApiKeys: boolean canUseSearchMcp: boolean } @@ -126,6 +127,9 @@ async function resolveOrganizationSurfaceContext( isAdmin: access.isAdmin, canInviteMembers: access.isAdmin && !isInvitationsDisabled && !capabilityDeniedBy('invitations.send', config), + canConnectSearchIntegrations: + !capabilityDeniedBy('integrations.manage', config) && + !capabilityDeniedBy('knowledge.use', config), canUsePersonalApiKeys: !capabilityDeniedBy('personal_api_key.use', config) && !capabilityDeniedBy('api_keys.manage', config), diff --git a/apps/sim/lib/sim-search/live/scopes.ts b/apps/sim/lib/sim-search/live/scopes.ts index a1d88b949b8..4de22a12977 100644 --- a/apps/sim/lib/sim-search/live/scopes.ts +++ b/apps/sim/lib/sim-search/live/scopes.ts @@ -1,4 +1,4 @@ -/** User-token RTS permissions. Existing grant policies remain valid when switching back to indexed search. */ +/** User-token permissions required by Slack's live retrieval API. */ export const SLACK_RTS_USER_SCOPES = [ 'search:read.files', 'files:read', From b87206079d63fdf20b78fd708b6b0f487f904b0c Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 1 Oct 2026 12:17:55 -0700 Subject: [PATCH 3/4] fix(search): close retired document writes and refresh Search consent --- .../integrations/search-source-setup.tsx | 4 +- .../lib/copy/copy-resources.test.ts | 15 +- .../lib/copy/copy-resources.ts | 59 ++++- apps/sim/lib/credential-groups/service.ts | 78 +++++-- .../sim/lib/credentials/managed-oauth.test.ts | 5 +- .../search-mcp-setup.integration.ts | 177 +++++++++++++-- .../search-source-progress.integration.ts | 96 +++++++++ .../lib/knowledge/application/connectors.ts | 3 + .../lib/knowledge/application/documents.ts | 10 + .../application/search-integrations.ts | 15 +- .../__integration__/fork-sync.integration.ts | 203 ++++++++++++++++++ .../indexed-search-retirement.md | 152 ------------- 12 files changed, 617 insertions(+), 200 deletions(-) delete mode 100644 packages/db/script-migrations/indexed-search-retirement.md diff --git a/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.tsx b/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.tsx index ddbb853ebe9..017f383ef90 100644 --- a/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.tsx +++ b/apps/sim/app/o/[organizationId]/settings/components/integrations/search-source-setup.tsx @@ -133,7 +133,9 @@ export function SearchSourceSetup({ } const failedQuery = index.isError ? index : null const initialMode = (type: string) => - type === 'github' || type === 'slack' ? ('members' as const) : ('admin' as const) + type === 'github' || type === 'slack' || setup['source-access'] === 'members' + ? ('members' as const) + : ('admin' as const) const selectedAccessMode = selectedType ? initialMode(selectedType) : undefined const selectedAvailability = selectedMeta diff --git a/apps/sim/ee/workspace-forking/lib/copy/copy-resources.test.ts b/apps/sim/ee/workspace-forking/lib/copy/copy-resources.test.ts index 37b3cf2df7b..248dccc9cd9 100644 --- a/apps/sim/ee/workspace-forking/lib/copy/copy-resources.test.ts +++ b/apps/sim/ee/workspace-forking/lib/copy/copy-resources.test.ts @@ -3,6 +3,7 @@ import { sha256Hex } from '@sim/security/hash' import { dbChainMockFns, flattenMockConditions, + queueTableRows, resetDbChainMock, schemaMock, storageServiceMock, @@ -105,6 +106,12 @@ function mappedDocumentPlan(): ForkContentPlan { describe('copyForkResourceContent', () => { beforeEach(() => { resetDbChainMock() + for (let attempt = 0; attempt < 4; attempt++) + queueTableRows(schemaMock.knowledgeBase, [ + { id: 'src-kb', isSearchIndex: false }, + { id: 'child-kb', isSearchIndex: false }, + { id: 'existing-target-kb', isSearchIndex: false }, + ]) dbChainMockFns.returning.mockResolvedValue([{ id: 'activated-document' }]) dbChainMockFns.for.mockResolvedValue([{ workspaceId: 'child-ws' }]) storageServiceMockFns.mockHeadObject.mockResolvedValue(null) @@ -1138,11 +1145,15 @@ describe('planForkMappedKbDocumentCopies', () => { let selectCalls = 0 const tx = { select: () => { - const rows = selectCalls++ === 0 ? docs : existingTargets return { - from: () => ({ + from: (table: unknown) => ({ where: (condition: unknown) => { + if (table === schemaMock.knowledgeBase) { + const rows = Promise.resolve([{ id: 'target-kb' }]) + return Object.assign(rows, { for: () => rows }) + } wheres.push(condition) + const rows = selectCalls++ === 0 ? docs : existingTargets return Promise.resolve(rows) }, }), diff --git a/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts b/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts index c5cd463b170..57e0772d7d4 100644 --- a/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts +++ b/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts @@ -48,6 +48,10 @@ import { hashDurableSecretProvenanceValue, } from '@/lib/execution/durable-secret-provenance' import { WORKSPACE_ACCESS_TOKEN } from '@/lib/knowledge/access/types' +import { + connectorIndexingCondition, + requiresConnectorIndexing, +} from '@/lib/knowledge/connectors/indexing-policy' import { createKnowledgeDocumentSourceValue, type KnowledgeDocumentSourceValue, @@ -758,6 +762,7 @@ export async function copyForkResourceContainers( and( inArray(knowledgeBase.id, selection.knowledgeBases), eq(knowledgeBase.workspaceId, sourceWorkspaceId), + connectorIndexingCondition(), isNull(knowledgeBase.deletedAt) ) ) @@ -951,17 +956,43 @@ export async function planForkMappedKbDocumentCopies(params: { .where( and( inArray(document.id, candidateIds), + exists( + tx + .select({ id: knowledgeBase.id }) + .from(knowledgeBase) + .where( + and( + eq(knowledgeBase.id, document.knowledgeBaseId), + connectorIndexingCondition(), + isNull(knowledgeBase.deletedAt) + ) + ) + ), isNull(document.connectorId), isNull(document.deletedAt), isNull(document.archivedAt) ) ) - const planned = docs.flatMap((doc) => { + const candidates = docs.flatMap((doc) => { const targetKbId = resolver('knowledge-base', doc.knowledgeBaseId) if (targetKbId == null) return [] return [{ doc, targetKbId, childDocId: deriveCopyIdentity('document', targetKbId, doc.id) }] }) + if (candidates.length === 0) return { documents, docIdMap, mappingEntries } + const targets = await tx + .select({ id: knowledgeBase.id }) + .from(knowledgeBase) + .where( + and( + inArray(knowledgeBase.id, [...new Set(candidates.map(({ targetKbId }) => targetKbId))]), + connectorIndexingCondition(), + isNull(knowledgeBase.deletedAt) + ) + ) + .for('share') + const targetIds = new Set(targets.map(({ id }) => id)) + const planned = candidates.filter(({ targetKbId }) => targetIds.has(targetKbId)) const existingTargets = planned.length === 0 ? [] @@ -1690,7 +1721,10 @@ async function finalizeKbDocument(params: { } = params return db.transaction(async (tx) => { const [lockedKnowledgeBase] = await tx - .select({ workspaceId: knowledgeBase.workspaceId }) + .select({ + workspaceId: knowledgeBase.workspaceId, + isSearchIndex: knowledgeBase.isSearchIndex, + }) .from(knowledgeBase) .where(eq(knowledgeBase.id, childKnowledgeBaseId)) .for('update') @@ -1698,6 +1732,9 @@ async function finalizeKbDocument(params: { if (!lockedKnowledgeBase) { throw new Error(`Copied document knowledge base ${childKnowledgeBaseId} is missing`) } + if (!requiresConnectorIndexing(lockedKnowledgeBase.isSearchIndex)) { + throw new Error('Retired Search documents cannot be activated by a workspace copy') + } if (lockedKnowledgeBase.workspaceId !== billingContext.workspaceId) { throw new Error( `Copied document knowledge base ${childKnowledgeBaseId} moved from workspace ${billingContext.workspaceId}; refusing stale storage charge` @@ -1809,6 +1846,24 @@ async function copyKbDocument(params: { billingContext, } = params assertForkCopyActive(params.control) + const bases = await db + .select({ id: knowledgeBase.id, isSearchIndex: knowledgeBase.isSearchIndex }) + .from(knowledgeBase) + .where( + and( + inArray(knowledgeBase.id, [source.knowledgeBaseId, childKnowledgeBaseId]), + isNull(knowledgeBase.deletedAt) + ) + ) + const basesById = new Map(bases.map((base) => [base.id, base])) + if ( + [source.knowledgeBaseId, childKnowledgeBaseId].some((id) => { + const base = basesById.get(id) + return !base || !requiresConnectorIndexing(base.isSearchIndex) + }) + ) { + throw new Error('Workspace copies require active ordinary knowledge bases') + } const sourceSecretContext = await loadKnowledgeDocumentDurableSecretProvenance(source.id) const sourceSnapshotHash = hashDurableSecretProvenanceValue( createKnowledgeDocumentSourceValue(source) diff --git a/apps/sim/lib/credential-groups/service.ts b/apps/sim/lib/credential-groups/service.ts index e92b6684f6e..72b09db19f1 100644 --- a/apps/sim/lib/credential-groups/service.ts +++ b/apps/sim/lib/credential-groups/service.ts @@ -25,7 +25,7 @@ import { credentialGroupScopePolicyVersion } from '@/lib/credential-groups/provi import { decryptCredentialGroupProviderConfiguration } from '@/lib/credential-groups/provider-configuration' import { getCredentialGroupProviderAdapter } from '@/lib/credential-groups/provider-registry' import { - type CredentialGroupStandardOAuthProvider, + type CredentialGroupProvider, isCredentialGroupProvider, } from '@/lib/credential-groups/providers' import { credentialGroupScope } from '@/lib/credential-groups/scope' @@ -87,7 +87,13 @@ function scopesEqual(left: string[], right: string[]): boolean { async function buildOption( scope: ResourceScope, - option: CredentialGroupOptionInput, + option: { + provider: CredentialGroupProvider + label: string + required: boolean + slackBotCredentialId?: string + requiredScopes?: string[] + }, credentialGroupId?: string, executor: DbOrTx = db ): Promise { @@ -395,11 +401,11 @@ export async function ensureWorkspaceAccountsGroup( } } -/** Adds a provider explicitly selected by an organization administrator, preserving all grants. */ +/** Adds a provider or extends its required consent during an explicit administrator action. */ export async function addOrganizationAccountProvider( organizationId: string, userId: string, - option: { provider: CredentialGroupStandardOAuthProvider; label: string }, + option: { provider: CredentialGroupProvider; label: string; requiredScopes?: string[] }, executor: DbTransaction ): Promise<{ groupId: string; changed: boolean }> { const scope = { kind: 'organization', organizationId } as const @@ -418,11 +424,37 @@ export async function addOrganizationAccountProvider( `Connected accounts contains duplicate ${option.label} settings` ) if (matching[0]) { - if (matching[0].status !== 'active') + const current = matching[0] + if (current.status !== 'active') throw new OrchestrationError( 'validation', `Enable ${option.label} in Connected accounts first` ) + if (option.requiredScopes) { + const previousScopes = + current.provider === 'slack' + ? resolveSlackManagedUserScopes(current.requiredScopes) + : current.requiredScopes + const requiredScopes = [...new Set([...previousScopes, ...option.requiredScopes])] + const scopeVersion = credentialGroupScopePolicyVersion(requiredScopes) + if (!scopesEqual(requiredScopes, previousScopes) || scopeVersion !== current.scopeVersion) { + const [updated] = await executor + .update(credentialGroup) + .set({ + options: existing.options.map((entry) => + entry.id === current.id ? { ...entry, requiredScopes, scopeVersion } : entry + ), + updatedAt: new Date(), + }) + .where( + and(eq(credentialGroup.id, group.id), resourceScopeCondition(credentialGroup, scope)) + ) + .returning({ id: credentialGroup.id }) + if (!updated) throw new Error('Connected accounts policy update returned no row') + await invalidateOptionGrants(executor, group.id, [current.id]) + return { groupId: group.id, changed: true } + } + } return { groupId: group.id, changed: group.created } } if ( @@ -449,6 +481,27 @@ export async function addOrganizationAccountProvider( return { groupId: group.id, changed: true } } +async function invalidateOptionGrants( + executor: DbTransaction, + groupId: string, + optionIds: string[] +) { + const enrollmentIds = executor + .select({ id: credentialGroupEnrollment.id }) + .from(credentialGroupEnrollment) + .where(eq(credentialGroupEnrollment.credentialGroupId, groupId)) + await executor + .update(credential) + .set({ managedOauthStatus: 'needs_reauth', updatedAt: new Date() }) + .where( + and( + eq(credential.type, 'managed_oauth'), + inArray(credential.credentialGroupEnrollmentId, enrollmentIds), + inArray(credential.credentialGroupOptionId, optionIds) + ) + ) +} + /** * Refuses to remove account options while a knowledge * connector syncs per member through one of them: the connector would be left @@ -570,20 +623,7 @@ export async function updateCredentialGroup( if (!updated) throw new Error('Credential group update returned no row') if (invalidatedOptionIds.length > 0) { - const enrollmentIds = tx - .select({ id: credentialGroupEnrollment.id }) - .from(credentialGroupEnrollment) - .where(eq(credentialGroupEnrollment.credentialGroupId, groupId)) - await tx - .update(credential) - .set({ managedOauthStatus: 'needs_reauth', updatedAt: new Date() }) - .where( - and( - eq(credential.type, 'managed_oauth'), - inArray(credential.credentialGroupEnrollmentId, enrollmentIds), - inArray(credential.credentialGroupOptionId, invalidatedOptionIds) - ) - ) + await invalidateOptionGrants(tx, groupId, invalidatedOptionIds) } return toCredentialGroup(updated, await listLinkedMcpServers(updated.id, tx)) }) diff --git a/apps/sim/lib/credentials/managed-oauth.test.ts b/apps/sim/lib/credentials/managed-oauth.test.ts index 67935bb0fb2..74b8427d31d 100644 --- a/apps/sim/lib/credentials/managed-oauth.test.ts +++ b/apps/sim/lib/credentials/managed-oauth.test.ts @@ -410,7 +410,10 @@ describe('managed OAuth token resolution', () => { }) it('allows Search reads when Slack retains a broader grant', async () => { - seedSlackSearchCredential('option-1', 'active', SLACK_MANAGED_USER_SCOPES) + seedSlackSearchCredential('option-1', 'active', [ + ...SLACK_MANAGED_USER_SCOPES, + ...SLACK_SEARCH_USER_SCOPES, + ]) await expect( resolveManagedOAuthToken({ credentialId: 'credential-1', diff --git a/apps/sim/lib/knowledge/__integration__/search-mcp-setup.integration.ts b/apps/sim/lib/knowledge/__integration__/search-mcp-setup.integration.ts index f1a99f53971..41edfd940a4 100644 --- a/apps/sim/lib/knowledge/__integration__/search-mcp-setup.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-mcp-setup.integration.ts @@ -1,6 +1,8 @@ import { db, runOutsideTransactionContext } from '@sim/db' import { + credential, credentialGroup, + credentialGroupEnrollment, mcpServers, member, organization, @@ -9,6 +11,7 @@ import { user, } from '@sim/db/schema' import * as dns from '@sim/security/dns' +import { sha256Hex } from '@sim/security/hash' import { createSessionPrincipal } from '@sim/testing/factories/principal.factory' import { createDeferred } from '@sim/testing/helpers/deferred' import { getPostgresErrorCode } from '@sim/utils/errors' @@ -18,6 +21,14 @@ import { eq, inArray, sql } from 'drizzle-orm' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' import { listSearchIntegrationsContract } from '@/lib/api/contracts/knowledge/search-integrations' import { env } from '@/lib/core/config/env' +import { encryptSecret } from '@/lib/core/security/encryption' +import { credentialGroupScopePolicyVersion } from '@/lib/credential-groups/provider-adapter' +import { + emptyCredentialGroupProviderConfiguration, + encryptCredentialGroupProviderConfiguration, +} from '@/lib/credential-groups/provider-configuration' +import { getCredentialGroup } from '@/lib/credential-groups/service' +import { SLACK_MANAGED_USER_SCOPES } from '@/lib/credential-groups/slack-managed-user-scopes' import { createOrganizationAccountsGroup } from '@/lib/credential-groups/workspace-accounts' import { acquireAdvisoryXactLock, tryAcquireAdvisoryXactLock } from '@/lib/db/advisory-locks' import { @@ -25,6 +36,7 @@ import { listSearchIntegrations, } from '@/lib/knowledge/application/search-integrations' import { defaultLiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' +import { SLACK_RTS_USER_SCOPES } from '@/lib/sim-search/live/scopes' /** * Real authorization, transactions, constraints and persistence; only DNS is a fixture. @@ -98,7 +110,7 @@ describe('atomic organization live Search MCP setup', () => { }) async function snapshot() { - const [groups, servers, approvals, policies, organizations] = await Promise.all([ + const [groups, servers, approvals, policies, organizations, credentials] = await Promise.all([ db.select().from(credentialGroup).where(eq(credentialGroup.organizationId, ids.organization)), db.select().from(mcpServers).where(eq(mcpServers.organizationId, ids.organization)), db @@ -110,10 +122,131 @@ describe('atomic organization live Search MCP setup', () => { .select({ metadata: organization.metadata }) .from(organization) .where(eq(organization.id, ids.organization)), + db.select().from(credential).where(eq(credential.organizationId, ids.organization)), ]) - return { groups, servers, approvals, policies, metadata: toRecord(organizations[0]?.metadata) } + return { + groups, + servers, + approvals, + policies, + credentials, + metadata: toRecord(organizations[0]?.metadata), + } } + async function seedWorkflowSlack(scopes: readonly string[] = SLACK_MANAGED_USER_SCOPES) { + const option = { + id: generateId(), + provider: 'slack', + label: 'Slack', + authorizationAppId: 'slack:fixture-app:fixture-team', + requiredScopes: [...scopes], + scopeVersion: credentialGroupScopePolicyVersion([...scopes]), + required: false, + status: 'active' as const, + } + const other = { ...option, id: generateId(), provider: 'gmail', label: 'Gmail' } + const group = await db.transaction((tx) => + createOrganizationAccountsGroup(tx, ids.organization, ids.owner, [option, other]) + ) + await db + .update(credentialGroup) + .set({ + encryptedProviderConfiguration: await encryptCredentialGroupProviderConfiguration({ + ...emptyCredentialGroupProviderConfiguration(), + slack: { + clientId: 'fixture-client', + clientSecret: 'fixture-secret', + appId: 'fixture-app', + teamId: 'fixture-team', + scopes: [...scopes], + verifiedAt: new Date().toISOString(), + }, + }), + }) + .where(eq(credentialGroup.id, group.id)) + const enrollmentId = generateId() + await db.insert(credentialGroupEnrollment).values({ + id: enrollmentId, + credentialGroupId: group.id, + userId: ids.owner, + email: `${ids.owner}@fixture.test`, + status: 'completed', + invitationTokenHash: sha256Hex(generateId()), + invitationExpiresAt: new Date(Date.now() + 60_000), + invitedAt: new Date(), + }) + const encrypted = (await encryptSecret('{"access_token":"fixture-token"}')).encrypted + await db.insert(credential).values( + [option, other].map((entry) => ({ + id: generateId(), + organizationId: ids.organization, + type: 'managed_oauth' as const, + providerId: entry.provider, + displayName: entry.label, + createdBy: ids.owner, + authorizationAppId: entry.authorizationAppId, + providerSubjectId: ids.owner, + credentialGroupEnrollmentId: enrollmentId, + credentialGroupOptionId: entry.id, + managedOauthStatus: 'active' as const, + managedOauthScopeVersion: entry.scopeVersion, + grantedScopes: [...scopes], + encryptedOauthTokenSet: encrypted, + grantedAt: new Date(), + })) + ) + return { groupId: group.id, optionId: option.id, otherOptionId: other.id } + } + + it.each([ + { name: 'workflow policy', scopes: SLACK_MANAGED_USER_SCOPES }, + { name: 'custom policy', scopes: ['chat:write', 'users:read', 'users:read.email'] }, + ])( + 'upgrades an existing Slack $name only through explicit Search approval', + async ({ scopes }) => { + const seeded = await seedWorkflowSlack(scopes) + const before = await snapshot() + await approve('slack') + const state = await snapshot() + const upgraded = state.groups[0].options.find((option) => option.id === seeded.optionId)! + expect(upgraded.requiredScopes).toEqual( + expect.arrayContaining([...scopes, ...SLACK_RTS_USER_SCOPES]) + ) + expect(upgraded.scopeVersion).not.toBe(before.groups[0].options[0].scopeVersion) + expect(state.groups[0].options.find((option) => option.id === seeded.otherOptionId)).toEqual( + before.groups[0].options[1] + ) + expect(state.policies).toEqual(before.policies) + expect(state.groups[0].encryptedProviderConfiguration).toBe( + before.groups[0].encryptedProviderConfiguration + ) + expect( + state.credentials.find((entry) => entry.credentialGroupOptionId === seeded.optionId) + ?.managedOauthStatus + ).toBe('needs_reauth') + expect( + state.credentials.find((entry) => entry.credentialGroupOptionId === seeded.otherOptionId) + ).toEqual( + before.credentials.find((entry) => entry.credentialGroupOptionId === seeded.otherOptionId) + ) + expect( + await getCredentialGroup( + { kind: 'organization', organizationId: ids.organization }, + seeded.groupId + ) + ).toMatchObject({ + options: expect.arrayContaining([ + expect.objectContaining({ id: seeded.optionId, configurationStatus: 'needs_update' }), + ]), + }) + await expect(approve('slack')).resolves.toMatchObject({ memberAccounts: { changed: false } }) + const repeated = await snapshot() + expect(repeated.groups).toEqual(state.groups) + expect(repeated.credentials).toEqual(state.credentials) + } + ) + it('keeps disabled Zoom approvals visible and removable without permitting reapproval', async () => { const connectorType = 'zoom' await db.insert(organizationSearchIntegration).values({ @@ -363,25 +496,29 @@ describe('atomic organization live Search MCP setup', () => { } ) - it('rolls back sign-in resources and policy metadata when the final approval write fails', async () => { - const constraint = `search_setup_${generateId().replace(/-/g, '')}` - await db.execute( - sql`ALTER TABLE organization_search_integration ADD CONSTRAINT ${sql.identifier(constraint)} CHECK (organization_id <> ${sql.raw(`'${ids.organization}'`)}) NOT VALID` - ) - const before = await snapshot() - try { - let failure: unknown - try { - await approve('fireflies') - } catch (error) { - failure = error - } - expect(getPostgresErrorCode(failure)).toBe('23514') - expect(await snapshot()).toEqual(before) - } finally { + it.each(['fireflies', 'slack'])( + 'rolls back %s sign-in policy and credentials when the final approval write fails', + async (provider) => { + if (provider === 'slack') await seedWorkflowSlack() + const constraint = `search_setup_${generateId().replace(/-/g, '')}` await db.execute( - sql`ALTER TABLE organization_search_integration DROP CONSTRAINT ${sql.identifier(constraint)}` + sql`ALTER TABLE organization_search_integration ADD CONSTRAINT ${sql.identifier(constraint)} CHECK (organization_id <> ${sql.raw(`'${ids.organization}'`)}) NOT VALID` ) + const before = await snapshot() + try { + let failure: unknown + try { + await approve(provider) + } catch (error) { + failure = error + } + expect(getPostgresErrorCode(failure)).toBe('23514') + expect(await snapshot()).toEqual(before) + } finally { + await db.execute( + sql`ALTER TABLE organization_search_integration DROP CONSTRAINT ${sql.identifier(constraint)}` + ) + } } - }) + ) }) diff --git a/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts b/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts index 34ca5cf96c8..e4580b084e3 100644 --- a/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts @@ -22,8 +22,10 @@ import { import { deleteKnowledgeConnector, listKnowledgeConnectorDocuments, + updateKnowledgeConnectorDocuments, } from '@/lib/knowledge/application/connectors' import { + bulkUpdateKnowledgeDocuments, listKnowledgeDocuments, readKnowledgeDocument, updateKnowledgeDocument, @@ -93,6 +95,100 @@ describe('viewer-isolated knowledge-base recovery lists', () => { }) }) +describe('retired Search document admission', () => { + const fixture = createKnowledgeAclFixtureIds() + const viewer = { + kind: 'session' as const, + userId: fixture.aliceId, + sessionId: 'fixture-retirement', + } + const documentId = generateId() + + beforeAll(async () => { + await seedKnowledgeAclFixture(fixture, { connectorType: 'google_drive' }) + await db.insert(document).values({ + id: documentId, + knowledgeBaseId: fixture.knowledgeBaseId, + connectorId: fixture.connectorId, + externalId: documentId, + filename: 'Retirement fixture', + fileUrl: '', + fileSize: 0, + mimeType: 'text/plain', + processingStatus: 'completed', + acl: [`u:${fixture.aliceId}@fixture.test`], + aclVerifiedAt: new Date(), + }) + }) + afterAll(async () => { + await db.delete(workspace).where(eq(workspace.id, fixture.workspaceId)) + await db.delete(organization).where(eq(organization.id, fixture.organizationId)) + await db.delete(user).where(inArray(user.id, [fixture.aliceId, fixture.bobId])) + }) + + it.each([ + 'connector restore', + 'document enable', + 'document updates enable', + 'selected documents enable', + 'all documents enable', + ] as const)('refuses %s for retired Search while preserving ordinary KBs', async (operation) => { + const change = () => { + const input = { knowledgeBaseId: fixture.knowledgeBaseId, documentId } + if (operation === 'connector restore') + return updateKnowledgeConnectorDocuments.execute({ + principal: viewer, + input: { + ...input, + connectorId: fixture.connectorId, + operation: 'restore', + documentIds: [documentId], + }, + }) + if (operation === 'document enable') + return updateKnowledgeDocument.execute({ + principal: viewer, + input: { ...input, enabled: true }, + }) + if (operation === 'document updates enable') + return updateKnowledgeDocument.execute({ + principal: viewer, + input: { ...input, updates: { enabled: true } }, + }) + return bulkUpdateKnowledgeDocuments.execute({ + principal: viewer, + input: { + knowledgeBaseId: fixture.knowledgeBaseId, + operation: 'enable', + ...(operation === 'all documents enable' + ? { selectAll: true } + : { documentIds: [documentId] }), + }, + }) + } + await db + .update(document) + .set({ enabled: false, userExcluded: operation === 'connector restore' }) + .where(eq(document.id, documentId)) + await db + .update(knowledgeBase) + .set({ isSearchIndex: true }) + .where(eq(knowledgeBase.id, fixture.knowledgeBaseId)) + const [before] = await db.select().from(document).where(eq(document.id, documentId)) + + await expect(change()).rejects.toMatchObject({ code: 'validation' }) + expect(await db.select().from(document).where(eq(document.id, documentId))).toEqual([before]) + + await db + .update(knowledgeBase) + .set({ isSearchIndex: false }) + .where(eq(knowledgeBase.id, fixture.knowledgeBaseId)) + await change() + const [restored] = await db.select().from(document).where(eq(document.id, documentId)) + expect(restored).toMatchObject({ enabled: true, userExcluded: false, acl: before.acl }) + }) +}) + describe('connector document filename search and document sets', () => { const fixture = createKnowledgeAclFixtureIds() const viewer = { kind: 'session' as const, userId: fixture.aliceId, sessionId: 'fixture-search' } diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index fe3be8719f5..45994904047 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -1289,6 +1289,9 @@ export const updateKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCas } const documentIds = [...new Set(input.documentIds)] const restoring = input.operation === 'restore' + if (restoring && !requiresConnectorIndexing(context.knowledgeBase.isSearchIndex)) { + throw new OrchestrationError('validation', 'This search index is inactive; use Sim Search.') + } const updated = await db .update(document) .set({ userExcluded: !restoring, enabled: restoring }) diff --git a/apps/sim/lib/knowledge/application/documents.ts b/apps/sim/lib/knowledge/application/documents.ts index c2de94c0d5b..6a29c50085f 100644 --- a/apps/sim/lib/knowledge/application/documents.ts +++ b/apps/sim/lib/knowledge/application/documents.ts @@ -27,6 +27,7 @@ import { resolveCanonicalActiveKnowledgeDocumentContext, } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { requiresConnectorIndexing } from '@/lib/knowledge/connectors/indexing-policy' import { ALL_TAG_SLOTS, type AllTagSlot, @@ -837,6 +838,9 @@ export const updateKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ const updates: KnowledgeDocumentUpdates = input.updates ? { ...input.updates } : { filename: input.filename, enabled: input.enabled } + if (updates.enabled && !requiresConnectorIndexing(context.knowledgeBase.isSearchIndex)) { + throw new OrchestrationError('validation', 'This search index is inactive; use Sim Search.') + } if (input.tagValues !== undefined) { Object.assign( updates, @@ -889,6 +893,12 @@ export const bulkUpdateKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ input: BulkKnowledgeDocumentsInput }) => resolveActiveKnowledgeResourceContext(input, principal), async execute({ input, context }) { + if ( + input.operation === 'enable' && + !requiresConnectorIndexing(context.knowledgeBase.isSearchIndex) + ) { + throw new OrchestrationError('validation', 'This search index is inactive; use Sim Search.') + } const result = input.selectAll ? await bulkDocumentOperationByFilter( context.knowledgeBaseId, diff --git a/apps/sim/lib/knowledge/application/search-integrations.ts b/apps/sim/lib/knowledge/application/search-integrations.ts index 421eda03983..2e5b97ca5db 100644 --- a/apps/sim/lib/knowledge/application/search-integrations.ts +++ b/apps/sim/lib/knowledge/application/search-integrations.ts @@ -7,6 +7,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { CredentialGroupProviderConfigurationError } from '@/lib/credential-groups/provider-adapter' import { isScopedCredentialGroupsAvailable } from '@/lib/credential-groups/scoped-availability' import { addOrganizationAccountProvider } from '@/lib/credential-groups/service' +import { SLACK_SEARCH_USER_SCOPES } from '@/lib/credential-groups/slack-managed-user-scopes' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { resolveKnowledgeOwnerContext } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' @@ -90,7 +91,9 @@ export const approveSearchIntegration = defineAuthorizedKnowledgeUseCase({ 'Zoom Search is not available for this organization' ) const memberProvider = input.approved - ? liveSearchMemberAccountProvider(input.connectorType) + ? input.connectorType === 'slack' + ? 'slack' + : liveSearchMemberAccountProvider(input.connectorType) : null const mcpProvider = input.approved ? liveSearchMcpConnector(input.connectorType) : null if (memberProvider || mcpProvider) { @@ -169,7 +172,13 @@ export const approveSearchIntegration = defineAuthorizedKnowledgeUseCase({ memberAccounts = await addOrganizationAccountProvider( context.organizationId!, requirePrincipalSubjectUserId(principal), - { provider: memberProvider, label: source[1].name }, + { + provider: memberProvider, + label: source[1].name, + ...(memberProvider === 'slack' + ? { requiredScopes: [...SLACK_SEARCH_USER_SCOPES] } + : {}), + }, tx ).catch((error: unknown) => { if (error instanceof CredentialGroupProviderConfigurationError) @@ -217,7 +226,7 @@ export const approveSearchIntegration = defineAuthorizedKnowledgeUseCase({ action: AuditAction.CREDENTIAL_GROUP_UPDATED, resourceType: AuditResourceType.CREDENTIAL_GROUP, resourceId: result.memberAccounts.groupId, - description: `Added ${result.connectorType} member sign-in for Sim Search`, + description: `Configured ${result.connectorType} member sign-in for Sim Search`, metadata: { connectorType: result.connectorType }, }, ] diff --git a/apps/sim/lib/workspaces/__integration__/fork-sync.integration.ts b/apps/sim/lib/workspaces/__integration__/fork-sync.integration.ts index 619a4068957..bfd53dbe599 100644 --- a/apps/sim/lib/workspaces/__integration__/fork-sync.integration.ts +++ b/apps/sim/lib/workspaces/__integration__/fork-sync.integration.ts @@ -2,7 +2,10 @@ import { AuditAction } from '@sim/audit' import { db } from '@sim/db' import { auditLog, + document, + embedding, folder, + knowledgeBase, outboxEvent, permissions, user, @@ -37,7 +40,13 @@ import { } from '@/ee/workspace-forking/application/create-and-sync' import { assertForkSourceVersions } from '@/ee/workspace-forking/application/revision' import { setForkSyncDefault } from '@/ee/workspace-forking/application/sync-default' +import { + copyForkResourceContainers, + copyForkResourceContent, + planForkMappedKbDocumentCopies, +} from '@/ee/workspace-forking/lib/copy/copy-resources' import { loadSourceDeployedStates } from '@/ee/workspace-forking/lib/copy/deploy-bridge' +import type { ForkCopyProgress } from '@/ee/workspace-forking/lib/copy/progress' import type { WorkflowState } from '@/stores/workflows/workflow/types' const userId = generateId() @@ -917,4 +926,198 @@ describe('authorized fork and sync against PostgreSQL', () => { await setDefault(sourceWorkspaceId, false) } }) + async function seedKnowledgeCopy() { + const childWorkspaceId = generateId() + const sourceWorkspaceId = generateId() + createdWorkspaceIds.push(sourceWorkspaceId, childWorkspaceId) + await db.insert(workspace).values( + [sourceWorkspaceId, childWorkspaceId].map((id) => ({ + id, + name: 'Knowledge copy fixture', + ownerId: userId, + billedAccountUserId: userId, + })) + ) + const sourceId = generateId() + const childId = generateId() + await db.insert(knowledgeBase).values([ + { id: sourceId, workspaceId: sourceWorkspaceId, userId, name: `Source ${sourceId}` }, + { id: childId, workspaceId: childWorkspaceId, userId, name: 'Target fixture' }, + ]) + const [source] = await db + .insert(document) + .values({ + id: generateId(), + knowledgeBaseId: sourceId, + filename: 'Copy fixture', + fileUrl: '', + fileSize: 0, + mimeType: 'text/plain', + processingStatus: 'completed', + }) + .returning() + return { sourceWorkspaceId, childWorkspaceId, sourceId, childId, source } + } + + it('copies ordinary knowledge containers but excludes retired Search containers', async () => { + const fixture = await seedKnowledgeCopy() + const retiredId = generateId() + await db.insert(knowledgeBase).values({ + id: retiredId, + workspaceId: fixture.sourceWorkspaceId, + userId, + name: 'Retired Search fixture', + isSearchIndex: true, + }) + const copied = await db.transaction((tx) => + copyForkResourceContainers({ + tx, + sourceWorkspaceId: fixture.sourceWorkspaceId, + childWorkspaceId: fixture.childWorkspaceId, + userId, + now: new Date(), + selection: { + customTools: [], + skills: [], + mcpServers: [], + workflowMcpServers: [], + tables: [], + knowledgeBases: [fixture.sourceId, retiredId], + }, + workflowIdMap: new Map(), + documentMappingContext: { + edgeChildWorkspaceId: fixture.childWorkspaceId, + sourceIsParent: true, + }, + }) + ) + expect(copied.contentPlan.knowledgeBases.map((entry) => entry.sourceId)).toEqual([ + fixture.sourceId, + ]) + expect( + await db + .select() + .from(knowledgeBase) + .where( + and( + eq(knowledgeBase.workspaceId, fixture.childWorkspaceId), + eq(knowledgeBase.isSearchIndex, true) + ) + ) + ).toEqual([]) + }) + + it.each(['source', 'target'] as const)( + 'does not plan document copies for a retired Search %s', + async (retiredSide) => { + const fixture = await seedKnowledgeCopy() + const retiredId = retiredSide === 'source' ? fixture.sourceId : fixture.childId + await db + .update(knowledgeBase) + .set({ isSearchIndex: true }) + .where(eq(knowledgeBase.id, retiredId)) + const plan = () => + db.transaction((tx) => + planForkMappedKbDocumentCopies({ + tx, + resolver: (kind, id) => + kind === 'knowledge-base' && id === fixture.sourceId ? fixture.childId : null, + referencedDocumentIds: [fixture.source.id], + alreadyCopiedSourceDocIds: new Set(), + now: new Date(), + }) + ) + const refused = await plan() + expect(refused.documents).toEqual([]) + expect(refused.mappingEntries).toEqual([]) + expect( + await db.select().from(document).where(eq(document.knowledgeBaseId, fixture.childId)) + ).toEqual([]) + + await db + .update(knowledgeBase) + .set({ isSearchIndex: false }) + .where(eq(knowledgeBase.id, retiredId)) + const allowed = await plan() + expect(allowed.documents).toHaveLength(1) + const [placeholder] = await db + .select() + .from(document) + .where(eq(document.knowledgeBaseId, fixture.childId)) + expect(placeholder.archivedAt).not.toBeNull() + expect(allowed.docIdMap.get(fixture.source.id)).toBe(placeholder.id) + } + ) + + it.each(['source', 'target', 'target during copy', 'ordinary'] as const)( + 'checks retired Search admission for queued content with %s', + async (retiredSide) => { + const fixture = await seedKnowledgeCopy() + const childDocId = generateId() + await db.insert(document).values({ + ...fixture.source, + id: childDocId, + knowledgeBaseId: fixture.childId, + archivedAt: new Date(), + }) + if (retiredSide === 'source' || retiredSide === 'target') { + await db + .update(knowledgeBase) + .set({ isSearchIndex: true }) + .where( + eq(knowledgeBase.id, retiredSide === 'source' ? fixture.sourceId : fixture.childId) + ) + } + const progress: ForkCopyProgress = { completed: [], tables: {}, embeddings: {} } + const result = await copyForkResourceContent({ + contentPlan: { + sourceWorkspaceId: fixture.sourceWorkspaceId, + childWorkspaceId: fixture.childWorkspaceId, + userId, + tables: [], + knowledgeBases: [], + skills: [], + documents: [ + { + sourceDocId: fixture.source.id, + childDocId, + childKnowledgeBaseId: fixture.childId, + storageKey: null, + fileUrl: '', + fileSize: 0, + filename: fixture.source.filename, + mimeType: fixture.source.mimeType, + }, + ], + }, + control: { + progress, + checkpoint: async () => { + if (retiredSide === 'target during copy') + await db + .update(knowledgeBase) + .set({ isSearchIndex: true }) + .where(eq(knowledgeBase.id, fixture.childId)) + }, + }, + }) + const [copied] = await db.select().from(document).where(eq(document.id, childDocId)) + if (retiredSide === 'ordinary') { + expect(result).toMatchObject({ copied: 1, failed: 0 }) + expect(copied.archivedAt).toBeNull() + } else { + expect(result).toMatchObject({ copied: 0, failed: 1 }) + expect(copied.archivedAt).not.toBeNull() + expect( + await db.select().from(embedding).where(eq(embedding.documentId, childDocId)) + ).toEqual([]) + if (retiredSide !== 'target during copy') expect(progress.embeddings).toEqual({}) + } + const [targetWorkspace] = await db + .select() + .from(workspace) + .where(eq(workspace.id, fixture.childWorkspaceId)) + expect(targetWorkspace.storageUsedBytes).toBe(0) + } + ) }) diff --git a/packages/db/script-migrations/indexed-search-retirement.md b/packages/db/script-migrations/indexed-search-retirement.md deleted file mode 100644 index 998e8e4f680..00000000000 --- a/packages/db/script-migrations/indexed-search-retirement.md +++ /dev/null @@ -1,152 +0,0 @@ -# Indexed Search database retirement - -This inventory separates retired enterprise Search indexing from the database objects that still -serve ordinary knowledge bases and live Search. Removing an application reader does not remove an -installed PostgreSQL trigger or reclaim its table. This release removes legacy application paths; -physical removal must follow a fully deployed release boundary. - -No step in this document authorizes an unbounded data deletion, automatic HNSW rebuild, or production -execution. The existing Search retirement command remains operator-run maintenance. Its progress and -target snapshot must survive until a replacement cleanup has adopted them or retirement is verified. - -## Deferred physical removal - -| Object | Retired responsibility | Contract prerequisite | -| --- | --- | --- | -| `embedding_keyword_search` and its foreign key, primary key, `embedding_keyword_search_kb_idx`, `embedding_keyword_search_document_idx`, `embedding_keyword_search_content_idx` | Indexed Search's GIN keyword candidate projection | Remove indexed readers and all keyword projection writers, including older workers and database triggers. Ordinary KB keyword ranking uses `embedding.content_tsv`. | -| `embedding_keyword_tin` and its foreign key, primary key, `embedding_keyword_tin_document_idx`, `embedding_keyword_tin_content_idx`, `embedding_keyword_tin_acl_gin_idx`, `embedding_keyword_tin_acl_unfilled_idx` | Indexed Search's optional Tin/BM25 projection and permission copies | Remove indexed readers, Tin/ACL projection writers, and maintenance references. Some installations never installed the optional Tin index. | -| `embedding_search.connector_id`, `embedding_search.acl` | Denormalized per-vector source/permission filtering for indexed Search | Deploy removal of indexed readers, then retire source/ACL triggers and old projector/detachment writers. Regular KB retrieval must keep checking the parent document. | -| `embedding_search_source_idx`, `embedding_search_acl_gin_idx`, `embedding_search_acl_unfilled_idx` | Source filtering, copied ACL overlap, and ACL backfill probes | Remove indexed readers and ACL fill paths; drop indexes concurrently in a later contract migration. These indexes were created by script migrations and are not declared in `schema.ts`. | -| `embedding_search_src_*` partial HNSW indexes | Per-connector ANN graphs used only by indexed Search | Inventory actual index definitions, confirm each belongs to `embedding_search` and filters a retired source, then drop concurrently. Preserve the shared width-specific HNSW indexes. | -| `organization_search_invocation`, its two foreign keys, bounds checks and `organization_search_invocation_org_created_idx` / `organization_search_invocation_user_idx` | Indexed result activity counters and old organization statistics | Deploy removal of the indexed statistics API and activity writer, and drain old application versions. | -| `embedding_search.binary`, `binary_384`, `binary_768`, `binary_1024`, `binary_3072` | Obsolete binary-quantized candidate representation | Stop `sync_embedding_search()` and the recovery projector from calculating/writing these columns, and replace the binary-based `embedding_search_width_check`. The five binary ANN indexes were already removed in migration `0372`. | -| `knowledge_projection_dirty`, its document FK/primary key, and `knowledge_projection_dirty_marked_at_idx` | Tracks copied source/ACL changes and older deferred vector writes | First complete or adopt every pending vector content repair, stop mark writers, and drain old workers. A `content = true` mark can represent a missing ordinary-KB vector; discarding it can lose retrieval coverage. | -| `search_embedding_cleanup_progress`, `search_embedding_cleanup_targets` | Durable retirement scope, cursor, and maintenance checkpoints | Complete and verify retirement, or explicitly adopt this state into a successor. These script-owned tables are not application schema objects and must not be dropped merely because deployment no longer runs cleanup. | - -The binary-width constraint currently requires exactly one populated binary column. Stopping binary -writes without changing that constraint would reject new ordinary-KB vectors. A replacement vector -table can omit the binary columns and use an appropriate vector-width invariant from the outset. -Changing the existing table requires a separately reviewed compatible transition; no table-wide -rewrite or validation scan belongs in this application-removal release. - -## Installed triggers and functions - -These live in the database after their TypeScript installer has finished. Deleting an installer or -an application import alone leaves its database work active. - -| Retire after dependent old code drains | Installed by | -| --- | --- | -| `embedding_keyword_search_sync` on `embedding`; `sync_embedding_keyword_search()` | `0016`, subsequently scoped/guarded by `0024` and `0025` | -| `knowledge_base_keyword_search_sync` on `knowledge_base`; `sync_knowledge_base_keyword_search()` | `0025` | -| `embedding_keyword_tin_sync` on `embedding`; `sync_embedding_keyword_tin()` | `0019`, subsequently guarded/scoped by `0024` and `0025` | -| `knowledge_base_keyword_tin_sync` on `knowledge_base`; `sync_knowledge_base_keyword_tin()` | `0019`, updated by `0025` | -| `knowledge_tin_stream(tsvector)`, `knowledge_tin_base_token(text)`, `knowledge_tin_membership_key(text)` | `0019`; the membership helper is also installed by `0025` and used by both keyword writers and the projector | -| `projection_source_acl_sync` on `document`; `sync_projection_source_acl()` | `0021`/`0022`, updated by `0023`, `0024`, and `0025` | -| `embedding_search_source_acl_set` on `embedding_search`; `embedding_keyword_tin_source_acl_set` on `embedding_keyword_tin`; `set_projection_source_acl()` | `0021`/`0022`, guarded by `0024` | -| `embedding_projection_mark_insert`, `embedding_projection_mark_update` on `embedding`; `mark_inserted_embedding_projection()`, `mark_updated_embedding_projection()`, `mark_knowledge_projection(text[], boolean)` | `0024`; remove only after vector-repair adoption/drain | - -Very old installations may retain the earlier `embedding_search_connector_sync` document trigger, -`embedding_search_connector_set` projection trigger, `sync_embedding_search_connector()`, and -`set_embedding_search_connector()`. The current ACL installer removes those names; an idempotent -contract should account for installations that skipped it without using `CASCADE`. - -Keep `embedding_search_sync` on `embedding` and `sync_embedding_search()` until their ordinary-KB -replacement is active. Their vector projection is shared. Their binary computations can be retired -under the constraint transition above. The `sim.projection_mode` transaction setting is also used by -existing repair workers to skip synchronous triggers, so removing that guard before those workers -drain needs separate review. - -Dropping triggers requires relation locks. Use short lock timeouts and bounded retries in the -contract; a waiting DDL statement must not queue production writers indefinitely. Remove triggers -before their functions, then dependent indexes/columns/tables. Do not use broad `DROP ... CASCADE`. -The optional `tin` extension is not automatically droppable: confirm no other schema or application -uses it before scheduling extension removal. - -## Application and bootstrap dependencies - -The app-removal release must remove or narrow these responsibilities: - -- Indexed Search retrieval, projection-fill checks, and keyword/Tin capability probes. -- Source/ACL copying and keyword projection in `packages/db/knowledge-projection.ts`; retain bounded - repair of ordinary-KB vector content left by older asynchronous writers. -- The `knowledge-projection` background task and its enqueue/sweep path only after vector repair is - complete or adopted. While retained, it must not recreate retired keyword data. -- `prewarmSearchProjection`: keep shared vector warming if useful, but stop warming retired keyword - heaps and ACL GIN indexes. `pg_prewarm` itself is not specific to enterprise Search. -- Connector detachment's direct updates of copied source/ACL fields. These writes are currently paged - to prevent document updates from causing unbounded trigger fan-out. Keep that protection until the - database fan-out trigger has been retired, even if the new application no longer reads the copies. -- ACL-change page sizing in member observations and source permission persistence. Canonical - document ACLs still matter; only their projection-row accounting can simplify after fan-out stops. -- Operator maintenance commands that refer to retired tables. In particular, `0028` currently lists - both keyword tables for vacuum and discovers every HNSW index on the shared vector table. Physical - contraction must first replace or retire this command so a retry does not target removed objects. - -Historical SQL migrations remain intact. Script migrations run **after all SQL migrations**; old -pending scripts must not reinstall retired objects after a contract or fail because their target -columns have disappeared. `db:push` has its own reconciliation command list and needs the same review. - -| Script migration | Treatment | -| --- | --- | -| `0015_backfill_embedding_search` | Already superseded by `0016`; historical binary projection installer, not a reason to recreate binary indexes. | -| `0016_backfill_search_vectors` and `0017_index_search_documents` | Mixed shared/retired responsibilities. They install/backfill ordinary-KB vectors and build required vector indexes as well as legacy keyword objects. Do not unregister them wholesale without a replacement shared-vector bootstrap and upgrade path. | -| `0019_tin_keyword_projection` | Entirely retired keyword projection, but its installed functions/triggers remain until contract. Supersede in a reviewed bootstrap transition. | -| `0021_embedding_search_connector`, `0022_projection_source_acl_backfill`, `0023_projection_acl_skip_unfilled` | Entirely retired projection-copy installation/indexing/backfill. `0022` already supersedes `0021`. Removing application use does not justify replaying their table-wide work. | -| `0024_knowledge_projection_async` | Mixed legacy ACL and old vector-repair machinery. Supersede only with a vector-safe replacement and pending-mark handling. | -| `0025_scope_keyword_projections` | Scopes legacy keyword writers and adjusts the document ACL trigger. It also installs the membership helper used by the old projector. Remove with the dependent machinery. | -| `0027`–`0029` Search retirement | Already absent from automatic deployment. Preserve their checkpoints until successor cleanup adoption; do not reintroduce automatic deletion, vacuum, or HNSW rebuilding. | - -The current registry and `scripts/push.ts` are intentionally not changed by the inventory alone. -Simply omitting legacy installers on fresh installs is unsafe while mixed-version application paths, -repair workers, or later installers still expect their functions. A successor must define the full -final bootstrap, explicitly supersede the old script receipts, and work for empty, partially -migrated, and already-running databases without scanning all existing data during deployment. - -## Shared objects that remain - -- `embedding`, its full-precision vectors, content TSV/GIN index, tags, and provenance sidecar. - Regular KB exact reranking, keyword retrieval, document filtering, and secret provenance use them. -- `embedding_search` identities, enabled state, half-precision vectors, six HNSW indexes, KB index, - document lookup index, and embedding FK. Ordinary KB ANN retrieval uses this table. A replacement - must preserve current inserts, updates, deletes, and document-level authorization before cutover. -- `document.acl`, `acl_requirements`, `acl_verified_at`, the ACL GIN index/shape check, `connector_id`, - source URL/modified/seen timestamps, and source indexes. Ordinary KB reads still enforce the - document access predicate; workspace connectors can use member/admin permissions, and ordinary - KB filters use source modification times. Source-seen time drives connector absence reconciliation. -- `knowledge_connector_member`, `knowledge_document_observation`, external directory/group tables, - permission snapshot/grant tables, member sync logs, and their indexes. They support ordinary KB - permission-scoped connectors; deleting them would remove authorization evidence. -- Connector access mode, credential groups, member/admin sync state, permission/listing checkpoints, - partitions, retry fields, and detachment billing reservations. These are shared connector machinery. -- `knowledge_base.is_search_index`, organization ownership, and Search KB uniqueness constraints. - Live Search still loads configured sources through Search-marked KBs. A marker is also the durable - retirement target boundary; it is not proof that the parent KB row itself can be deleted. -- `organization_search_integration`, live Search activity/OAuth structures, provider credentials, and live - source configuration. They belong to the live product as well as the retired implementation. -- `workspace_file_search_*`, Slack Search, and documentation embeddings. These are separate search - products and are outside this retirement. - -The old `doc_processing_recovery_idx` is independently marked as superseded by the per-source -recovery index in `schema.ts`. It is a separate contract candidate after its replacement release is -verified; it is not evidence that ordinary KB processing recovery can be removed. - -## Release order and completion evidence - -1. Deploy removal of indexed readers/admission paths and narrow remaining shared workers. Keep the - compatible schema and trigger protection while older web tasks, queued jobs, and rollback images - may still use them. Retain the `contract-pending` markers in `schema.ts`. -2. Verify the release is fully deployed, old workers have drained, and rollback cannot reactivate - indexed Search. Adopt or complete pending vector repairs before dropping their queue. Verify - ordinary KB retrieval and permission changes against real PostgreSQL boundaries. -3. Retire legacy triggers/functions and installers with a shared-vector-safe bootstrap. Preserve - historical migration replay and inspect unknown dependencies rather than cascading through them. - This stops future keyword/ACL copying without rewriting existing vector rows. -4. Build any replacement projection through separately controlled, resumable maintenance. Copy only - ordinary-KB rows, capture concurrent changes, validate retrieval/authorization, then switch readers - while retaining a tested rollback path. This inventory does not start that copy or rebuild. -5. Contract retired tables/columns/indexes only after those readers and writers are gone. Reference - the deployed removal release in migration safety acknowledgments and remove the corresponding - `contract-pending` markers. Keep cleanup state until content retirement is independently verified. - -Code deletion, stopped writes, copied data, a successful read cutover, and physical reclamation are -different completion conditions. Report them separately; none implies that all the others happened. From 1bed203ba88a7e1a97e0d0f5b3610643035edfc6 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 1 Oct 2026 12:30:41 -0700 Subject: [PATCH 4/4] fix(search): preserve ordinary knowledge base classification --- .../search-source-setup.integration.ts | 38 +++++++++++++++---- .../lib/knowledge/application/sim-search.ts | 16 -------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/apps/sim/lib/knowledge/__integration__/search-source-setup.integration.ts b/apps/sim/lib/knowledge/__integration__/search-source-setup.integration.ts index ef488c24b48..eee347e4058 100644 --- a/apps/sim/lib/knowledge/__integration__/search-source-setup.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-source-setup.integration.ts @@ -236,7 +236,7 @@ describe('Search source identity and concurrent creation', () => { ).toHaveLength(0) }) - it('adopts a legacy index only during admin setup and persists its canonical identity', async () => { + it('preserves ordinary KB name collisions and creates one explicit Search configuration', async () => { await db .update(knowledgeBase) .set({ name: 'Sim Search' }) @@ -258,18 +258,40 @@ describe('Search source identity and concurrent creation', () => { principal: { kind: 'session', userId: other.aliceId, sessionId: 'fixture-admin' }, input, }) - const results = await Promise.all([prepare(), prepare()]) - expect(results.map((result) => result.knowledgeBaseId)).toEqual([ - other.knowledgeBaseId, - other.knowledgeBaseId, - ]) + await expect(prepare()).rejects.toMatchObject({ code: 'conflict' }) + expect( + await db + .select({ name: knowledgeBase.name, isSearchIndex: knowledgeBase.isSearchIndex }) + .from(knowledgeBase) + .where(eq(knowledgeBase.id, other.knowledgeBaseId)) + ).toEqual([{ name: 'Sim Search', isSearchIndex: false }]) await db .update(knowledgeBase) - .set({ name: 'Renamed adopted index' }) + .set({ name: 'Ordinary knowledge' }) .where(eq(knowledgeBase.id, other.knowledgeBaseId)) + const results = await Promise.all([prepare(), prepare()]) + const searchId = results[0].knowledgeBaseId + expect(searchId).not.toBe(other.knowledgeBaseId) + expect(results[1].knowledgeBaseId).toBe(searchId) + expect( + await db + .select({ id: knowledgeBase.id }) + .from(knowledgeBase) + .where( + and( + eq(knowledgeBase.workspaceId, other.workspaceId), + eq(knowledgeBase.isSearchIndex, true) + ) + ) + ).toEqual([{ id: searchId }]) + await db + .update(knowledgeBase) + .set({ name: 'Renamed Search configuration' }) + .where(eq(knowledgeBase.id, searchId)) await expect(readSearchIndex.execute({ principal: reader, input })).resolves.toMatchObject({ - knowledgeBaseId: other.knowledgeBaseId, + knowledgeBaseId: searchId, }) + await expect(prepare()).resolves.toMatchObject({ knowledgeBaseId: searchId }) }) it('serializes matching creates across independent database transactions and keeps one grant', async () => { diff --git a/apps/sim/lib/knowledge/application/sim-search.ts b/apps/sim/lib/knowledge/application/sim-search.ts index 859916de119..90e3f839efe 100644 --- a/apps/sim/lib/knowledge/application/sim-search.ts +++ b/apps/sim/lib/knowledge/application/sim-search.ts @@ -1,7 +1,4 @@ import { type Principal, resolvePrincipalSubjectUserId } from '@sim/auth/principal' -import { db } from '@sim/db' -import { knowledgeBase } from '@sim/db/schema' -import { and, eq, isNull } from 'drizzle-orm' import { coalesceLocally } from '@/lib/concurrency/singleflight' import { requireOrganizationMembership } from '@/lib/core/application/organization-authorization' import { @@ -84,19 +81,6 @@ async function ensureSearchKnowledgeBase( return created.id } const workspaceId = scope.workspaceId - const [legacy] = await db - .update(knowledgeBase) - .set({ isSearchIndex: true, updatedAt: new Date() }) - .where( - and( - eq(knowledgeBase.workspaceId, workspaceId), - eq(knowledgeBase.name, SIM_SEARCH_KNOWLEDGE_BASE_NAME), - eq(knowledgeBase.isSearchIndex, false), - isNull(knowledgeBase.deletedAt) - ) - ) - .returning({ id: knowledgeBase.id }) - if (legacy) return legacy.id const created = await createKnowledgeBase.execute({ principal, input: {