From e3533c16d8804076040c6ed06b6ca4048bc530b4 Mon Sep 17 00:00:00 2001 From: No Name <36251971+zrm625@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:48:51 -0400 Subject: [PATCH] Repair empty provider catalogs --- .../src/db/data-migrations.test.ts | 3 + .../host-cloudflare/src/db/data-migrations.ts | 2 + apps/host-selfhost/src/db/data-migrations.ts | 6 +- apps/local/src/db/data-migrations.ts | 6 +- e2e/selfhost/toolkits-ui.test.ts | 89 ++++++++++++++ .../plugins/openapi/src/sdk/plugin.test.ts | 59 ++++++++++ packages/plugins/openapi/src/sdk/plugin.ts | 109 +++++++++++++++++- .../src/catalog-repair.test.ts | 63 ++++++++++ .../src/catalog-repair.ts | 81 +++++++++++++ .../provider-service-split/src/index.ts | 1 + packages/plugins/toolkits/src/page.tsx | 63 ++++++++-- 11 files changed, 470 insertions(+), 12 deletions(-) create mode 100644 packages/plugins/provider-service-split/src/catalog-repair.test.ts create mode 100644 packages/plugins/provider-service-split/src/catalog-repair.ts diff --git a/apps/host-cloudflare/src/db/data-migrations.test.ts b/apps/host-cloudflare/src/db/data-migrations.test.ts index 02dc9be27..78100ffcd 100644 --- a/apps/host-cloudflare/src/db/data-migrations.test.ts +++ b/apps/host-cloudflare/src/db/data-migrations.test.ts @@ -240,6 +240,7 @@ describe("runCloudflareDataMigrations", () => { expect(yield* Effect.promise(() => runCloudflareDataMigrations(d1, bucket))).toEqual([ "2026-06-20-google-openapi-ownership", "2026-07-08-provider-service-split", + "2026-07-10-provider-service-empty-catalog-repair", "2026-07-09-openapi-ndjson-output-arrays", ]); expect(yield* Effect.promise(() => runCloudflareDataMigrations(d1, bucket))).toEqual([]); @@ -284,6 +285,7 @@ describe("runCloudflareDataMigrations", () => { expect(yield* Effect.promise(() => runCloudflareDataMigrations(d1, bucket))).toEqual([ "2026-06-20-google-openapi-ownership", "2026-07-08-provider-service-split", + "2026-07-10-provider-service-empty-catalog-repair", "2026-07-09-openapi-ndjson-output-arrays", ]); expect(yield* Effect.promise(() => runCloudflareDataMigrations(d1, bucket))).toEqual([]); @@ -335,6 +337,7 @@ describe("runCloudflareDataMigrations", () => { expect(yield* Effect.promise(() => runCloudflareDataMigrations(d1, bucket))).toEqual([ "2026-06-20-google-openapi-ownership", "2026-07-08-provider-service-split", + "2026-07-10-provider-service-empty-catalog-repair", "2026-07-09-openapi-ndjson-output-arrays", ]); expect(yield* Effect.promise(() => runCloudflareDataMigrations(d1, bucket))).toEqual([]); diff --git a/apps/host-cloudflare/src/db/data-migrations.ts b/apps/host-cloudflare/src/db/data-migrations.ts index 85f48b371..d3896aaa2 100644 --- a/apps/host-cloudflare/src/db/data-migrations.ts +++ b/apps/host-cloudflare/src/db/data-migrations.ts @@ -11,6 +11,7 @@ import { openApiNdjsonOutputDataMigration } from "@executor-js/plugin-openapi"; import { googleOpenApiOwnershipDataMigration } from "@executor-js/plugin-openapi/providers/google"; import { + providerServiceCatalogRepairDataMigration, providerServiceSplitDataMigration, runSqliteProviderServiceSplitMigration, type BlobRow, @@ -243,6 +244,7 @@ const cloudflareDataMigrations = (bucket: R2Bucket | undefined): readonly Sqlite }).pipe(Effect.asVoid); }), }, + providerServiceCatalogRepairDataMigration, // Stale-mark connections whose operations return NDJSON so their tool rows // rebuild with array-wrapped output schemas (mirrors cloud's drizzle 0010). openApiNdjsonOutputDataMigration, diff --git a/apps/host-selfhost/src/db/data-migrations.ts b/apps/host-selfhost/src/db/data-migrations.ts index 295f77d5b..6ac1d3763 100644 --- a/apps/host-selfhost/src/db/data-migrations.ts +++ b/apps/host-selfhost/src/db/data-migrations.ts @@ -15,7 +15,10 @@ import { import { graphqlIntrospectionBlobDataMigration } from "@executor-js/plugin-graphql"; import { googleOpenApiOwnershipDataMigration } from "@executor-js/plugin-openapi/providers/google"; -import { providerServiceSplitDataMigration } from "@executor-js/plugin-provider-service-split"; +import { + providerServiceCatalogRepairDataMigration, + providerServiceSplitDataMigration, +} from "@executor-js/plugin-provider-service-split"; import { authConfigTransforms } from "./auth-config-migration"; export const selfHostDataMigrations: readonly SqliteDataMigration[] = [ @@ -34,6 +37,7 @@ export const selfHostDataMigrations: readonly SqliteDataMigration[] = [ graphqlIntrospectionBlobDataMigration, googleOpenApiOwnershipDataMigration, providerServiceSplitDataMigration, + providerServiceCatalogRepairDataMigration, // Stale-mark connections whose operations return NDJSON so their tool rows // rebuild with array-wrapped output schemas (mirrors cloud's drizzle 0010). openApiNdjsonOutputDataMigration, diff --git a/apps/local/src/db/data-migrations.ts b/apps/local/src/db/data-migrations.ts index 522a7321d..deda4283c 100644 --- a/apps/local/src/db/data-migrations.ts +++ b/apps/local/src/db/data-migrations.ts @@ -20,7 +20,10 @@ import { import { graphqlIntrospectionBlobDataMigration } from "@executor-js/plugin-graphql"; import { googleOpenApiOwnershipDataMigration } from "@executor-js/plugin-openapi/providers/google"; -import { providerServiceSplitDataMigration } from "@executor-js/plugin-provider-service-split"; +import { + providerServiceCatalogRepairDataMigration, + providerServiceSplitDataMigration, +} from "@executor-js/plugin-provider-service-split"; import { authConfigTransforms } from "./auth-config-migration"; import { LOCAL_V1_V2_LEDGER_NAME } from "./v1-v2-migration"; @@ -45,6 +48,7 @@ export const localDataMigrations: readonly SqliteDataMigration[] = [ graphqlIntrospectionBlobDataMigration, googleOpenApiOwnershipDataMigration, providerServiceSplitDataMigration, + providerServiceCatalogRepairDataMigration, // GC dead DCR oauth_client rows (old always-register duplicates) and backfill // the surviving DCR rows' origin_issuer from token_url (issue #1120, Part C). oauthClientGcSqliteMigration, diff --git a/e2e/selfhost/toolkits-ui.test.ts b/e2e/selfhost/toolkits-ui.test.ts index aac0e96dd..b4c21725e 100644 --- a/e2e/selfhost/toolkits-ui.test.ts +++ b/e2e/selfhost/toolkits-ui.test.ts @@ -27,6 +27,27 @@ const hiddenPersonalSpec = (baseUrl: string): string => }, }); +const connectedGoogleSpec = (): string => + JSON.stringify({ + openapi: "3.0.3", + info: { title: "Gmail", version: "1.0.0" }, + paths: { + "/messages": { + get: { + operationId: "gmail.users.messages.list", + responses: { "200": { description: "Messages" } }, + }, + }, + }, + }); + +const connectedWithoutToolsSpec = (): string => + JSON.stringify({ + openapi: "3.0.3", + info: { title: "Gmail", version: "1.0.0" }, + paths: {}, + }); + scenario( "Toolkits · self-host UI creates a toolkit and configures tools", { timeout: 180_000 }, @@ -43,6 +64,8 @@ scenario( const slug = name; const hiddenPersonalIntegration = `${prefix}-personal-api`; const hiddenPersonalConnection = "mine"; + const connectedWithoutToolsIntegration = `${prefix}-gmail`; + const connectedWithoutToolsConnection = "localcoregoogle"; const seededToolkits = [ { owner: "org" as const, name: `${prefix}-workspace-a` }, { owner: "org" as const, name: `${prefix}-workspace-b` }, @@ -63,9 +86,21 @@ scenario( }, }) .pipe(Effect.ignore); + yield* client.connections + .remove({ + params: { + owner: "org", + integration: IntegrationSlug.make(connectedWithoutToolsIntegration), + name: ConnectionName.make(connectedWithoutToolsConnection), + }, + }) + .pipe(Effect.ignore); yield* client.openapi .removeSpec({ params: { slug: hiddenPersonalIntegration } }) .pipe(Effect.ignore); + yield* client.openapi + .removeSpec({ params: { slug: connectedWithoutToolsIntegration } }) + .pipe(Effect.ignore); const listed = yield* client.toolkits.list(); yield* Effect.forEach( listed.toolkits.filter((row) => row.slug.startsWith(prefix)), @@ -103,6 +138,42 @@ scenario( value: "unused-token", }, }); + yield* client.openapi.addSpec({ + payload: { + spec: { kind: "blob", value: connectedGoogleSpec() }, + slug: IntegrationSlug.make(connectedWithoutToolsIntegration), + authenticationTemplate: [ + { + slug: "apiKey", + type: "apiKey", + headers: { "x-api-key": [{ type: "variable", name: "token" }] }, + }, + ], + }, + }); + yield* client.connections.create({ + payload: { + owner: "org", + name: ConnectionName.make(connectedWithoutToolsConnection), + integration: IntegrationSlug.make(connectedWithoutToolsIntegration), + template: AuthTemplateSlug.make("apiKey"), + value: "unused-token", + }, + }); + yield* client.openapi.updateSpec({ + params: { slug: connectedWithoutToolsIntegration }, + payload: { spec: { kind: "blob", value: connectedWithoutToolsSpec() } }, + }); + const connectedAccounts = yield* client.connections.list({ + query: { + owner: "org", + integration: IntegrationSlug.make(connectedWithoutToolsIntegration), + }, + }); + expect( + connectedAccounts.map((connection) => connection.name), + "the Google account remains connected while its refreshed tool catalog is empty", + ).toContain(connectedWithoutToolsConnection); yield* browser.session(identity, async ({ page, step }) => { await step("Open the Toolkits plugin page", async () => { @@ -213,6 +284,24 @@ scenario( .waitFor(); }); + await step( + "A connected Google account remains addable before its tools are available", + async () => { + const dialog = page.getByRole("dialog", { name: "Manage connections" }); + await dialog + .getByLabel("Search connections and tools") + .fill(connectedWithoutToolsConnection); + await dialog.getByText(connectedWithoutToolsConnection, { exact: true }).waitFor(); + await dialog.getByText("Gmail · 0 tools", { exact: true }).waitFor(); + const add = dialog.getByRole("button", { name: /^Add connection / }); + await add.click(); + const remove = dialog.getByRole("button", { name: /^Remove connection / }); + await remove.waitFor(); + await remove.click(); + await add.waitFor(); + }, + ); + await step("Add a connection to the toolkit", async () => { const dialog = page.getByRole("dialog", { name: "Manage connections" }); await dialog.waitFor(); diff --git a/packages/plugins/openapi/src/sdk/plugin.test.ts b/packages/plugins/openapi/src/sdk/plugin.test.ts index 1617e8b55..2a1d8797c 100644 --- a/packages/plugins/openapi/src/sdk/plugin.test.ts +++ b/packages/plugins/openapi/src/sdk/plugin.test.ts @@ -1301,6 +1301,65 @@ paths: ), ); + it.effect("repairs a migrated provider connection whose operation catalog is missing", () => + Effect.scoped( + Effect.gen(function* () { + const specServer = yield* serveMutableOpenApiSpecTestServer({ initialApi: TestApi }); + const config = makeTestConfig({ plugins: testPlugins() }); + const executor = yield* createExecutor(config); + const integration = IntegrationSlug.make("google_gmail"); + + yield* executor.openapi.addSpec({ + spec: { kind: "url", url: specServer.specUrl }, + slug: integration, + family: "google", + baseUrl: specServer.baseUrl, + authenticationTemplate: [apiKeyTemplate], + }); + yield* executor.connections.create({ + owner: "org", + name: ConnectionName.make("main"), + integration, + template: AuthTemplateSlug.make("apiKey"), + value: "secret-key-123", + }); + expect((yield* executor.tools.list({ integration })).length).toBeGreaterThan(0); + + // Reproduce the bad service-split state: the integration + source URL + // and connection survived, but operation storage and tool rows did not, + // while the follow-up migration has marked the connection stale. + yield* Effect.promise(() => + config.db.deleteMany("plugin_storage", { + where: (b) => b.and(b("plugin_id", "=", "openapi"), b("collection", "=", "operation")), + }), + ); + yield* Effect.promise(() => + config.db.deleteMany("tool", { + where: (b) => b("integration", "=", String(integration)), + }), + ); + yield* Effect.promise(() => + config.db.updateMany("connection", { + where: (b) => b("integration", "=", String(integration)), + set: { tools_synced_at: null }, + }), + ); + + const repaired = yield* executor.tools.list({ integration }); + expect( + repaired.map((tool) => String(tool.name)), + "the next catalog read refetches the provider spec and restores tools", + ).toContain("items.listItems"); + const operations = yield* Effect.promise(() => + config.db.findMany("plugin_storage", { + where: (b) => b.and(b("plugin_id", "=", "openapi"), b("collection", "=", "operation")), + }), + ); + expect(operations.length, "the repaired operation catalog is persisted").toBeGreaterThan(0); + }), + ), + ); + it.effect("updateSpec accepts new inline content for blob-sourced integrations", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/plugins/openapi/src/sdk/plugin.ts b/packages/plugins/openapi/src/sdk/plugin.ts index da4cd2f4f..48d6976c9 100644 --- a/packages/plugins/openapi/src/sdk/plugin.ts +++ b/packages/plugins/openapi/src/sdk/plugin.ts @@ -7,6 +7,7 @@ import { IntegrationDetectionResult, IntegrationNotFoundError, IntegrationSlug, + StorageError, ToolResult, definePlugin, HealthCheckSpec, @@ -629,6 +630,81 @@ export const openApiPlugin = definePlugin< return { specText: config.spec.value }; }); + const migratedProviderFamilies = new Set(["google", "microsoft"]); + + const repairMissingMigratedProviderCatalog = (input: { + readonly ctx: PluginCtx; + readonly integration: Integration; + readonly current: OpenApiIntegrationConfig; + readonly storage: OpenapiStore; + readonly httpClientLayer: Layer.Layer; + }) => + Effect.gen(function* () { + const specUrl = input.current.specUrl; + if ( + specUrl === undefined || + input.current.family === undefined || + !migratedProviderFamilies.has(input.current.family) + ) { + return false; + } + + const slug = String(input.integration.slug); + const operations = yield* input.storage.listOperations(slug); + if (operations.length > 0) return false; + + const resolved = yield* resolveSpecForInput( + { + spec: { kind: "url", url: specUrl }, + specFormat: input.current.specFormat, + headers: input.current.headers, + queryParams: input.current.queryParams, + baseUrl: input.current.baseUrl, + }, + input.httpClientLayer, + ); + const compiled = resolved.keepPathItem + ? undefined + : yield* compileOpenApiSpec(resolved.specText); + const specHash = yield* sha256Hex(resolved.specText); + yield* input.storage.putSpec(specHash, resolved.specText); + if (compiled) { + yield* input.storage.putDefs(specHash, JSON.stringify(compiled.hoistedDefs)); + } + + const nextConfig: OpenApiIntegrationConfig = { + ...input.current, + specHash, + specUrl: resolved.specUrl ?? specUrl, + }; + yield* input.ctx.transaction( + Effect.gen(function* () { + yield* input.ctx.core.integrations.update(input.integration.slug, { + config: nextConfig satisfies OpenApiIntegrationConfig as IntegrationConfig, + }); + if (compiled) { + yield* input.storage.putOperations( + slug, + openApiStoredOperationsFromCompiled(slug, compiled), + ); + } else { + yield* compileAndPersistOpenApiSpecStreaming({ + specText: resolved.specText, + integration: slug, + storage: input.storage, + specHash, + keepPathItem: resolved.keepPathItem, + }); + } + }), + ); + return true; + }).pipe( + Effect.withSpan("openapi.plugin.repair_missing_migrated_provider_catalog", { + attributes: { "openapi.integration.slug": String(input.integration.slug) }, + }), + ); + return { id: "openapi" as const, packageName: "@executor-js/plugin-openapi", @@ -1172,8 +1248,37 @@ export const openApiPlugin = definePlugin< // operation bindings invokeTool needs are persisted at addSpec time; this // hook only shapes the per-connection ToolDefs from the spec blob the // catalog config points at. - resolveTools: ({ integration, config, storage }) => - resolveOpenApiBackedTools({ integration, config, storage }), + resolveTools: (input) => + Effect.gen(function* () { + const stored = yield* resolveOpenApiBackedTools(input); + if (stored.tools.length > 0 || input.ctx === undefined) return stored; + + const current = decodeOpenApiIntegrationConfig(input.config); + if ( + current?.family === undefined || + !migratedProviderFamilies.has(current.family) || + current.specUrl === undefined + ) { + return stored; + } + + const repaired = yield* repairMissingMigratedProviderCatalog({ + ctx: input.ctx, + integration: input.integration, + current, + storage: input.storage, + httpClientLayer: input.httpClientLayer, + }).pipe( + Effect.mapError( + (cause) => + new StorageError({ + message: `Failed to repair missing ${current.family} catalog for ${input.integration.slug}`, + cause, + }), + ), + ); + return repaired ? yield* resolveOpenApiBackedTools(input) : stored; + }), invokeTool: ({ ctx: invokeCtx, toolRow, credential, args }) => { const httpClientLayer = options?.httpClientLayer ?? invokeCtx.httpClientLayer; diff --git a/packages/plugins/provider-service-split/src/catalog-repair.test.ts b/packages/plugins/provider-service-split/src/catalog-repair.test.ts new file mode 100644 index 000000000..908940c87 --- /dev/null +++ b/packages/plugins/provider-service-split/src/catalog-repair.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import type { SqliteDataMigrationClient } from "@executor-js/sdk/core"; + +import { runSqliteProviderServiceCatalogRepair } from "./catalog-repair"; + +const makeFakeClient = ( + affectedRows: Record[], + options?: { readonly missingTable?: string }, +) => { + const log: (string | { readonly sql: string; readonly args: readonly unknown[] })[] = []; + const client: SqliteDataMigrationClient = { + execute: (stmt) => { + log.push(stmt); + const sql = typeof stmt === "string" ? stmt : stmt.sql; + if (sql.includes("sqlite_master")) { + const table = typeof stmt === "string" ? "" : String(stmt.args[0]); + return Promise.resolve({ + rows: table === options?.missingTable ? [] : [{ name: table }], + }); + } + if (sql.includes("SELECT c.row_id")) return Promise.resolve({ rows: affectedRows }); + return Promise.resolve({ rows: [] }); + }, + }; + return { client, log }; +}; + +describe("runSqliteProviderServiceCatalogRepair", () => { + it.effect("stale-marks only the empty migrated provider connections selected by the query", () => + Effect.gen(function* () { + const { client, log } = makeFakeClient([ + { row_id: "gmail-main" }, + { row_id: "calendar-work" }, + ]); + expect(yield* runSqliteProviderServiceCatalogRepair(client)).toBe(2); + + const selection = log.find( + (stmt) => typeof stmt !== "string" && stmt.sql.includes("SELECT c.row_id"), + ); + expect(typeof selection === "string" ? selection : selection?.sql).toContain("NOT EXISTS"); + expect(typeof selection === "string" ? selection : selection?.sql).toContain( + "'google', 'microsoft'", + ); + const updates = log.filter( + (stmt) => typeof stmt !== "string" && stmt.sql.includes("tools_synced_at = NULL"), + ); + expect(updates.map((stmt) => (typeof stmt === "string" ? [] : stmt.args))).toEqual([ + ["gmail-main"], + ["calendar-work"], + ]); + }), + ); + + it.effect("skips fresh databases that do not have the split-era tables", () => + Effect.gen(function* () { + for (const missingTable of ["connection", "integration", "tool"]) { + const { client } = makeFakeClient([{ row_id: "gmail-main" }], { missingTable }); + expect(yield* runSqliteProviderServiceCatalogRepair(client)).toBe(0); + } + }), + ); +}); diff --git a/packages/plugins/provider-service-split/src/catalog-repair.ts b/packages/plugins/provider-service-split/src/catalog-repair.ts new file mode 100644 index 000000000..7e89164fd --- /dev/null +++ b/packages/plugins/provider-service-split/src/catalog-repair.ts @@ -0,0 +1,81 @@ +// --------------------------------------------------------------------------- +// Follow-up repair for provider-service splits whose connection rows were +// cloned as "synced" even though no per-service tool rows were persisted. +// +// The OpenAPI resolver now repairs a missing migrated Google/Microsoft catalog +// from its stored spec URL. Stale-mark only affected connections so their next +// normal tools read enters that resolver. Connections that already have tools, +// non-provider OpenAPI integrations, and integrations owned by other plugins +// are left untouched. +// --------------------------------------------------------------------------- + +import { Effect } from "effect"; +import { DataMigrationError, type SqliteDataMigrationClient } from "@executor-js/sdk/core"; + +const MIGRATION_NAME = "2026-07-10-provider-service-empty-catalog-repair"; + +const execute = ( + client: SqliteDataMigrationClient, + stmt: string | { readonly sql: string; readonly args: readonly unknown[] }, +) => + Effect.tryPromise({ + try: () => client.execute(stmt), + catch: (cause) => new DataMigrationError({ migration: MIGRATION_NAME, cause }), + }); + +const tableExists = (client: SqliteDataMigrationClient, table: string) => + Effect.map( + execute(client, { + sql: "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", + args: [table], + }), + (result) => result.rows.length > 0, + ); + +export const runSqliteProviderServiceCatalogRepair = ( + client: SqliteDataMigrationClient, +): Effect.Effect => + Effect.gen(function* () { + for (const table of ["connection", "integration", "tool"]) { + if (!(yield* tableExists(client, table))) return 0; + } + + const affected = yield* execute(client, { + sql: `SELECT c.row_id AS row_id + FROM connection c + INNER JOIN integration i + ON i.tenant = c.tenant AND i.slug = c.integration + WHERE i.plugin_id = 'openapi' + AND i.config IS NOT NULL + AND json_valid(i.config) + AND json_extract(i.config, '$.family') IN ('google', 'microsoft') + AND json_extract(i.config, '$.specUrl') IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM tool t + WHERE t.tenant = c.tenant + AND t.owner = c.owner + AND t.subject = c.subject + AND t.integration = c.integration + AND t.connection = c.name + )`, + args: [], + }); + + const rowIds = affected.rows.flatMap((row) => + typeof row.row_id === "string" ? [row.row_id] : [], + ); + for (const rowId of rowIds) { + yield* execute(client, { + sql: "UPDATE connection SET tools_synced_at = NULL WHERE row_id = ?", + args: [rowId], + }); + } + return rowIds.length; + }); + +export const providerServiceCatalogRepairDataMigration = { + name: MIGRATION_NAME, + run: (client: SqliteDataMigrationClient) => + runSqliteProviderServiceCatalogRepair(client).pipe(Effect.asVoid), +}; diff --git a/packages/plugins/provider-service-split/src/index.ts b/packages/plugins/provider-service-split/src/index.ts index 26d727a3a..56105782e 100644 --- a/packages/plugins/provider-service-split/src/index.ts +++ b/packages/plugins/provider-service-split/src/index.ts @@ -1,2 +1,3 @@ export * from "./planner"; export * from "./sqlite"; +export * from "./catalog-repair"; diff --git a/packages/plugins/toolkits/src/page.tsx b/packages/plugins/toolkits/src/page.tsx index 9e78eaddb..a419af78d 100644 --- a/packages/plugins/toolkits/src/page.tsx +++ b/packages/plugins/toolkits/src/page.tsx @@ -12,13 +12,18 @@ import { } from "@executor-js/sdk/client"; import { matchPattern, + type Connection, type EffectivePolicy, type Integration, type Owner, type ToolAddress, type ToolPolicyAction, } from "@executor-js/sdk/shared"; -import { integrationsOptimisticAtom, toolsAllAtom } from "@executor-js/react/api/atoms"; +import { + connectionsAllAtom, + integrationsOptimisticAtom, + toolsAllAtom, +} from "@executor-js/react/api/atoms"; import { ReactivityKey } from "@executor-js/react/api/reactivity-keys"; import { useOrganizationSlug } from "@executor-js/react/api/organization-context"; import { @@ -159,6 +164,9 @@ const toolMatchId = (tool: ToolRow): string => const toolCanAppearInToolkit = (toolkit: ToolkitResponse, tool: ToolRow): boolean => toolkit.owner === "user" || tool.static === true || tool.owner !== "user"; +const connectionCanAppearInToolkit = (toolkit: ToolkitResponse, connection: Connection): boolean => + toolkit.owner === "user" || connection.owner !== "user"; + const toolkitUrlFor = (orgSlug: string | undefined, slug: string): string => { const path = orgSlug ? `/${orgSlug}/mcp/toolkits/${slug}` : `/mcp/toolkits/${slug}`; if (typeof window === "undefined") return path; @@ -261,8 +269,24 @@ const reducePolicyPatterns = (patterns: readonly string[]): readonly string[] => ); }; -const buildConnectionGroups = (tools: readonly ToolRow[]): readonly ToolkitConnectionGroup[] => { +const buildConnectionGroups = ( + tools: readonly ToolRow[], + connections: readonly Connection[] = [], +): readonly ToolkitConnectionGroup[] => { const groups = new Map(); + for (const connection of connections) { + const integration = String(connection.integration); + const name = String(connection.name); + const key = `${connection.owner}:${integration}:${name}`; + groups.set(key, { + id: key, + owner: connection.owner, + integration, + connection: name, + patterns: [`${integration}.${connection.owner}.${name}.*`], + tools: [], + }); + } for (const tool of tools) { const owner = toolOwner(tool); const key = `${owner}:${tool.integration}:${toolConnectionName(tool)}`; @@ -285,7 +309,10 @@ const buildConnectionGroups = (tools: readonly ToolRow[]): readonly ToolkitConne const sortedTools = [...group.tools].sort(compareTools); return { ...group, - patterns: reducePolicyPatterns(sortedTools.map(connectionPatternForTool)), + patterns: reducePolicyPatterns([ + ...group.patterns, + ...sortedTools.map(connectionPatternForTool), + ]), tools: sortedTools, }; }) @@ -1036,6 +1063,7 @@ function ToolkitWorkspace(props: { showOwnerLabels: boolean; policies: readonly ToolkitPolicyResponse[]; connections: readonly ToolkitConnectionResponse[]; + availableConnections: readonly Connection[]; tools: readonly ToolRow[]; integrations: readonly Integration[]; integrationPlugins: readonly IntegrationPlugin[]; @@ -1053,11 +1081,21 @@ function ToolkitWorkspace(props: { () => props.tools.filter((tool) => toolCanAppearInToolkit(props.toolkit, tool)), [props.toolkit, props.tools], ); - const connectionGroups = useMemo(() => buildConnectionGroups(visibleTools), [visibleTools]); + const visibleConnections = useMemo( + () => + props.availableConnections.filter((connection) => + connectionCanAppearInToolkit(props.toolkit, connection), + ), + [props.availableConnections, props.toolkit], + ); + const connectionGroups = useMemo( + () => buildConnectionGroups(visibleTools, visibleConnections), + [visibleConnections, visibleTools], + ); const hiddenPersonalConnectionCount = useMemo(() => { if (props.toolkit.owner !== "org") return 0; - return buildConnectionGroups(props.tools.filter((tool) => toolOwner(tool) === "user")).length; - }, [props.toolkit.owner, props.tools]); + return props.availableConnections.filter((connection) => connection.owner === "user").length; + }, [props.availableConnections, props.toolkit.owner]); const configuredConnections = useMemo( () => configuredConnectionViews( @@ -1290,6 +1328,7 @@ function ToolkitDetailSkeleton() { function ToolkitDetailView(props: { toolkit: ToolkitResponse; showOwnerLabels: boolean; + availableConnections: readonly Connection[]; tools: readonly ToolRow[]; integrations: readonly Integration[]; integrationPlugins: readonly IntegrationPlugin[]; @@ -1361,6 +1400,7 @@ function ToolkitDetailView(props: { showOwnerLabels={props.showOwnerLabels} policies={policyRows} connections={connectionRows} + availableConnections={props.availableConnections} tools={props.tools} integrations={props.integrations} integrationPlugins={props.integrationPlugins} @@ -1382,6 +1422,7 @@ export function ToolkitsPage(props: PluginPageProps) { const integrationPlugins = useIntegrationPlugins(); const toolkits = useAtomValue(toolkitsAtom); const tools = useAtomValue(toolsAllAtom); + const availableConnections = useAtomValue(connectionsAllAtom); const integrations = useAtomValue(integrationsOptimisticAtom); const doCreateToolkit = useAtomSet(createToolkit, { mode: "promiseExit" }); const doRemoveToolkit = useAtomSet(removeToolkit, { mode: "promiseExit" }); @@ -1392,6 +1433,9 @@ export function ToolkitsPage(props: PluginPageProps) { selectedToolkitSlug === null ? null : toolkitByRouteSlug(toolkitRows, selectedToolkitSlug); const toolRows = AsyncResult.isSuccess(tools) ? (tools.value as readonly ToolRow[]) : []; + const availableConnectionRows = AsyncResult.isSuccess(availableConnections) + ? (availableConnections.value as readonly Connection[]) + : []; const integrationRows = AsyncResult.isSuccess(integrations) ? (integrations.value as readonly Integration[]) : []; @@ -1399,6 +1443,8 @@ export function ToolkitsPage(props: PluginPageProps) { const toolkitsFailed = AsyncResult.isFailure(toolkits); const toolsReady = AsyncResult.isSuccess(tools); const toolsFailed = AsyncResult.isFailure(tools); + const connectionsReady = AsyncResult.isSuccess(availableConnections); + const connectionsFailed = AsyncResult.isFailure(availableConnections); const navigateToIndex = () => navigate({ to: "/{-$orgSlug}/toolkits", @@ -1447,14 +1493,15 @@ export function ToolkitsPage(props: PluginPageProps) { ) ) : selectedToolkit ? ( - toolsFailed ? ( + toolsFailed || connectionsFailed ? (
Failed to load toolkit tools
- ) : !toolsReady ? ( + ) : !toolsReady || !connectionsReady ? ( ) : (