diff --git a/apps/sim/connectors/google-workspace/api-errors.ts b/apps/sim/connectors/google-workspace/api-errors.ts
index c1b67f95bd6..8ca0b403d16 100644
--- a/apps/sim/connectors/google-workspace/api-errors.ts
+++ b/apps/sim/connectors/google-workspace/api-errors.ts
@@ -54,6 +54,13 @@ const RATE_LIMIT_REASONS = new Set([
'RATE_LIMIT_EXCEEDED',
])
+/** Whether a Google error reason reports an exhausted rate or usage quota rather than a denial. */
+export function isGoogleQuotaReason(reason: string): boolean {
+ return (
+ RATE_LIMIT_REASONS.has(reason) || reason === 'dailyLimitExceeded' || reason === 'quotaExceeded'
+ )
+}
+
export function safeGoogleErrorReasons(reasons: readonly string[]): string[] {
return [...new Set(reasons.filter((reason) => SAFE_REASONS.has(reason)))]
}
@@ -119,13 +126,7 @@ export class GoogleApiError extends ConnectorSourceError {
const safeReasons = safeGoogleErrorReasons(reasons)
const suffix = safeReasons.length ? ` (${safeReasons.join(', ')})` : ''
const category =
- status === 429 ||
- safeReasons.some(
- (reason) =>
- RATE_LIMIT_REASONS.has(reason) ||
- reason === 'dailyLimitExceeded' ||
- reason === 'quotaExceeded'
- )
+ status === 429 || safeReasons.some(isGoogleQuotaReason)
? 'rate_limit'
: status >= 500
? 'provider_unavailable'
diff --git a/apps/sim/connectors/utils.test.ts b/apps/sim/connectors/utils.test.ts
index b8640d29a75..4325ac4fde4 100644
--- a/apps/sim/connectors/utils.test.ts
+++ b/apps/sim/connectors/utils.test.ts
@@ -72,6 +72,7 @@ import {
isIndexableConnectorFile,
isSkippableMicrosoftGraphFolderError,
isSkippedDocument,
+ looksLikeHtml,
MICROSOFT_GRAPH_MAX_CURSOR_ENCODED_BYTES,
MICROSOFT_GRAPH_MAX_ITEM_ID_BYTES,
MICROSOFT_GRAPH_MAX_PENDING_FOLDERS,
@@ -1754,3 +1755,14 @@ describe('BoundedLines', () => {
})
})
})
+
+describe('looksLikeHtml', () => {
+ it('recognizes elements with valid tag syntax', () => {
+ for (const markup of ['', '
', '
', '
', '', ''])
+ expect(looksLikeHtml(`text ${markup} text`)).toBe(true)
+ })
+ it('does not mistake an address whose name starts like a tag for markup', () => {
+ for (const text of ['Invite ', 'From ', ''])
+ expect(looksLikeHtml(text)).toBe(false)
+ })
+})
diff --git a/apps/sim/connectors/utils.ts b/apps/sim/connectors/utils.ts
index 88aee7614f7..7688e920b1c 100644
--- a/apps/sim/connectors/utils.ts
+++ b/apps/sim/connectors/utils.ts
@@ -339,11 +339,11 @@ function decodeCharacterReference(raw: string, code: number): string {
* A false positive therefore does not merely pass text through untouched — it
* deletes the bracketed span and flattens the document's line structure. Plain
* text routinely contains angle brackets that are not markup: an email address
- * (`Reply from John `), a markdown autolink
+ * (`Reply from John `, or ``, whose name is a tag's), a markdown autolink
* (``), or a placeholder (``).
*/
const HTML_TAG_PATTERN =
- /<\/?(?:p|div|br|hr|ul|ol|li|h[1-6]|table|thead|tbody|tr|td|th|span|strong|em|b|i|u|a|code|pre|blockquote|img|figure)\b[^>]*>/i
+ /<\/?(?:p|div|br|hr|ul|ol|li|h[1-6]|table|thead|tbody|tr|td|th|span|strong|em|b|i|u|a|code|pre|blockquote|img|figure)(?=[\s/>])[^>]*>/i
/**
* Reports whether a value carries real HTML markup and is therefore worth routing
@@ -362,15 +362,25 @@ export function looksLikeHtml(value: string): boolean {
* punctuation as numeric references, which previously reached the index verbatim.
*/
export function htmlToPlainText(html: string): string {
- const text = html
- .replace(/<[^>]*>/g, ' ')
- .replace(HTML_ENTITY_PATTERN, (raw: string, hex?: string, decimal?: string, named?: string) => {
+ return decodeHtmlEntities(html.replace(/<[^>]*>/g, ' '))
+ .replace(/\s+/g, ' ')
+ .trim()
+}
+
+/**
+ * Decodes HTML character references without touching markup or whitespace. Use for text a
+ * provider HTML-escapes but does not mark up, such as Gmail message snippets.
+ */
+export function decodeHtmlEntities(text: string): string {
+ return text.replace(
+ HTML_ENTITY_PATTERN,
+ (raw: string, hex?: string, decimal?: string, named?: string) => {
if (named !== undefined) return NAMED_ENTITIES[named] ?? raw
if (hex !== undefined) return decodeCharacterReference(raw, Number.parseInt(hex, 16))
if (decimal !== undefined) return decodeCharacterReference(raw, Number.parseInt(decimal, 10))
return raw
- })
- return text.replace(/\s+/g, ' ').trim()
+ }
+ )
}
/**
diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts
index 9c638d35fc1..0bf028fa4ba 100644
--- a/apps/sim/lib/core/security/input-validation.server.ts
+++ b/apps/sim/lib/core/security/input-validation.server.ts
@@ -356,6 +356,19 @@ export interface SecureFetchOptions {
proxyUrl?: string
/** Hide credential-derived URL details from validation logs. */
logUrlValidationDetails?: boolean
+ /**
+ * Ask for a gzip or brotli body. The body is decoded before it is returned, and
+ * `maxResponseBytes` bounds the decoded bytes, so a compression bomb still stops at the cap.
+ */
+ acceptCompressed?: boolean
+ /**
+ * Reuses keep-alive connections to the same pinned address across requests. A connection is
+ * only ever reused for the IP it was opened to, so every request keeps its DNS pinning. The
+ * owner must call {@link PinnedConnectionPool.destroy} once its requests have finished. Bun
+ * keeps sockets in its own per-address pool, so there reuse spans pools and `destroy` releases
+ * only the agents.
+ */
+ connectionPool?: PinnedConnectionPool
/**
* Where this request's URL came from. Carried on the options so the same
* policy is re-applied to every redirect hop rather than re-derived — a hop
@@ -447,6 +460,43 @@ function resolveRedirectUrl(baseUrl: string, location: string): string {
}
}
+/** Keep-alive agents keyed by protocol, host, port, and the pinned address they connect to. */
+export interface PinnedConnectionPool {
+ /** Undefined once destroyed, so a late request falls back to a single-use pinned agent. */
+ agent(isHttps: boolean, host: string, port: number, resolvedIP: string): http.Agent | undefined
+ destroy(): void
+}
+
+/**
+ * Creates a request-scoped pool of pinned keep-alive agents. Reusing a connection skips the TCP
+ * and TLS handshakes that otherwise dominate short provider API calls.
+ */
+export function createPinnedConnectionPool(): PinnedConnectionPool {
+ const agents = new Map()
+ let destroyed = false
+ return {
+ agent(isHttps, host, port, resolvedIP) {
+ if (destroyed) return undefined
+ const key = JSON.stringify([isHttps, host, port, resolvedIP])
+ let agent = agents.get(key)
+ if (!agent) {
+ const options: http.AgentOptions = {
+ keepAlive: true,
+ lookup: createPinnedLookup(resolvedIP),
+ }
+ agent = isHttps ? new https.Agent(options) : new http.Agent(options)
+ agents.set(key, agent)
+ }
+ return agent
+ },
+ destroy() {
+ destroyed = true
+ for (const agent of agents.values()) agent.destroy()
+ agents.clear()
+ },
+ }
+}
+
/**
* Creates a DNS lookup function that always returns a pre-resolved IP address.
* Use this to prevent DNS rebinding (TOCTOU) attacks when connecting to
@@ -1088,6 +1138,11 @@ export async function secureFetchWithPinnedIP(
const port = parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort
let agent: http.Agent | undefined
+ /**
+ * Bun ignores a `lookup` set on an Agent and honors one on the request, while Node honors
+ * both. A pinned direct connection sets it in both places so pinning holds in either runtime.
+ */
+ let pinnedLookup: LookupFunction | undefined
if (outboundDispatcher) {
agent = undefined
} else if (options.proxyUrl) {
@@ -1096,12 +1151,17 @@ export async function secureFetchWithPinnedIP(
// targets tunnel via CONNECT, http targets use absolute-URI forwarding.
agent = isHttps ? new HttpsProxyAgent(options.proxyUrl) : new HttpProxyAgent(options.proxyUrl)
} else {
- const lookup = createPinnedLookup(resolvedIP)
- const agentOptions: http.AgentOptions = { lookup }
- agent = isHttps ? new https.Agent(agentOptions) : new http.Agent(agentOptions)
+ pinnedLookup = createPinnedLookup(resolvedIP)
+ agent =
+ options.connectionPool?.agent(isHttps, parsed.hostname, port, resolvedIP) ??
+ (isHttps
+ ? new https.Agent({ lookup: pinnedLookup })
+ : new http.Agent({ lookup: pinnedLookup }))
}
const { 'accept-encoding': _, ...sanitizedHeaders } = options.headers ?? {}
+ /** Raw deflate streams are ambiguous to decode, so only gzip and brotli are requested. */
+ if (options.acceptCompressed) sanitizedHeaders['accept-encoding'] = 'gzip, br'
if (!Object.keys(sanitizedHeaders).some((name) => name.toLowerCase() === 'user-agent')) {
sanitizedHeaders['user-agent'] = DEFAULT_USER_AGENT
}
@@ -1123,6 +1183,7 @@ export async function secureFetchWithPinnedIP(
method: options.method || 'GET',
headers: sanitizedHeaders,
agent,
+ ...(pinnedLookup ? { lookup: pinnedLookup } : {}),
timeout: options.timeout || 300000,
}
@@ -1265,7 +1326,13 @@ export async function secureFetchWithPinnedIP(
statusCode === 204 ||
statusCode === 205 ||
statusCode === 304
- const contentLength = headersRecord['content-length']
+ /**
+ * An encoded body's Content-Length is its wire size, not the decoded size the cap bounds,
+ * so only an identity body is rejected up front; the decoded stream is capped as it reads.
+ */
+ const contentLength = headersRecord['content-encoding']
+ ? undefined
+ : headersRecord['content-length']
if (contentLength && !isBodylessResponse) {
const parsedLength = Number.parseInt(contentLength, 10)
if (Number.isFinite(parsedLength) && parsedLength > maxResponseBytes) {
diff --git a/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts b/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts
new file mode 100644
index 00000000000..adfdd76d20b
--- /dev/null
+++ b/apps/sim/lib/core/security/secure-fetch-connection-pool.server.test.ts
@@ -0,0 +1,192 @@
+/**
+ * @vitest-environment node
+ */
+import http from 'node:http'
+import type { AddressInfo } from 'node:net'
+import { gzipSync } from 'node:zlib'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@sim/security/dns', () => ({
+ resolveHostAddresses: vi.fn(),
+ preferIpv4: (addresses: string[]) => addresses[0],
+}))
+
+vi.mock('@/lib/core/config/env-flags', () => ({
+ isHosted: false,
+ getEgressAllowedHosts: () => undefined,
+ getEgressAllowedIpRanges: () => undefined,
+ isLegacyPrivateDatabaseAccessAllowed: () => false,
+ getProxyUrl: () => undefined,
+}))
+
+import {
+ createPinnedConnectionPool,
+ secureFetchWithPinnedIP,
+} from '@/lib/core/security/input-validation.server'
+
+const servers: http.Server[] = []
+
+afterEach(() => {
+ for (const server of servers.splice(0)) {
+ server.closeAllConnections()
+ server.close()
+ }
+})
+
+/** Starts a loopback server that counts the TCP connections it accepts. */
+async function startServer(handler: http.RequestListener) {
+ const server = http.createServer(handler)
+ servers.push(server)
+ let connections = 0
+ server.on('connection', () => connections++)
+ await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve))
+ return {
+ origin: `http://127.0.0.1:${(server.address() as AddressInfo).port}`,
+ connections: () => connections,
+ }
+}
+
+describe('secureFetchWithPinnedIP connection reuse', () => {
+ it('reuses one pinned connection across requests that share a pool', async () => {
+ const server = await startServer((_req, res) => res.end('ok'))
+ const pool = createPinnedConnectionPool()
+ try {
+ for (let index = 0; index < 3; index++) {
+ const response = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', {
+ profile: 'configuredEndpoint',
+ connectionPool: pool,
+ })
+ await expect(response.text()).resolves.toBe('ok')
+ }
+ expect(server.connections()).toBe(1)
+ } finally {
+ pool.destroy()
+ }
+ })
+
+ it('opens a fresh connection per request without a pool', async () => {
+ const server = await startServer((_req, res) => res.end('ok'))
+ for (let index = 0; index < 2; index++) {
+ const response = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', {
+ profile: 'configuredEndpoint',
+ })
+ await response.text()
+ }
+ expect(server.connections()).toBe(2)
+ })
+
+ it('never shares an agent between different pinned addresses', () => {
+ const pool = createPinnedConnectionPool()
+ try {
+ const first = pool.agent(true, 'api.example.com', 443, '203.0.113.1')
+ expect(pool.agent(true, 'api.example.com', 443, '203.0.113.1')).toBe(first)
+ expect(pool.agent(true, 'api.example.com', 443, '203.0.113.2')).not.toBe(first)
+ expect(pool.agent(true, 'other.example.com', 443, '203.0.113.1')).not.toBe(first)
+ } finally {
+ pool.destroy()
+ }
+ })
+
+ it('falls back to a single-use agent for a request after the pool is destroyed', async () => {
+ const server = await startServer((_req, res) => res.end('ok'))
+ const pool = createPinnedConnectionPool()
+ pool.destroy()
+ expect(pool.agent(false, '127.0.0.1', 80, '127.0.0.1')).toBeUndefined()
+ const response = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', {
+ profile: 'configuredEndpoint',
+ connectionPool: pool,
+ })
+ await expect(response.text()).resolves.toBe('ok')
+ })
+})
+
+describe('secureFetchWithPinnedIP compressed responses', () => {
+ it('does not reject an encoded body by its wire size when the decoded body fits', async () => {
+ const payload = Buffer.from(JSON.stringify({ ok: true }))
+ const encoded = gzipSync(payload)
+ const server = await startServer((_req, res) => {
+ res.writeHead(200, {
+ 'Content-Encoding': 'gzip',
+ 'Content-Length': String(encoded.length),
+ })
+ res.end(encoded)
+ })
+ const response = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', {
+ profile: 'configuredEndpoint',
+ acceptCompressed: true,
+ maxResponseBytes: payload.length,
+ })
+ expect(encoded.length).toBeGreaterThan(payload.length)
+ await expect(response.json()).resolves.toEqual({ ok: true })
+ })
+ it('caps the decoded size of a compressed body it asked for', async () => {
+ const bomb = gzipSync(Buffer.alloc(64 * 1024, 0x41))
+ const server = await startServer((_req, res) => {
+ res.writeHead(200, { 'Content-Encoding': 'gzip' })
+ res.end(bomb)
+ })
+ const response = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', {
+ profile: 'configuredEndpoint',
+ acceptCompressed: true,
+ maxResponseBytes: 1024,
+ })
+ expect(bomb.length).toBeLessThan(1024)
+ await expect(response.text()).rejects.toThrow(/response body/i)
+ })
+ it('asks for compression only when requested and returns the decoded body', async () => {
+ const encodings: (string | undefined)[] = []
+ const server = await startServer((req, res) => {
+ encodings.push(req.headers['accept-encoding'])
+ if (req.headers['accept-encoding']?.includes('gzip')) {
+ res.writeHead(200, { 'Content-Encoding': 'gzip' })
+ res.end(gzipSync(Buffer.from('{"ok":true}')))
+ } else {
+ res.end('{"ok":true}')
+ }
+ })
+ const compressed = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', {
+ profile: 'configuredEndpoint',
+ acceptCompressed: true,
+ })
+ await expect(compressed.json()).resolves.toEqual({ ok: true })
+ const plain = await secureFetchWithPinnedIP(server.origin, '127.0.0.1', {
+ profile: 'configuredEndpoint',
+ })
+ await expect(plain.json()).resolves.toEqual({ ok: true })
+ expect(encodings).toEqual(['gzip, br', undefined])
+ })
+})
+
+describe('secureFetchWithPinnedIP address pinning', () => {
+ it.each([
+ ['a fresh agent', undefined],
+ ['a pooled agent', createPinnedConnectionPool()],
+ ])('pins the address on the request itself with %s', async (_label, pool) => {
+ const server = await startServer((_req, res) => res.end('ok'))
+ const request = vi.spyOn(http, 'request')
+ try {
+ const response = await secureFetchWithPinnedIP(
+ server.origin.replace('127.0.0.1', 'localhost'),
+ '127.0.0.1',
+ {
+ profile: 'configuredEndpoint',
+ connectionPool: pool,
+ }
+ )
+ await response.text()
+ const options = request.mock.calls[0]?.[0] as http.RequestOptions
+ const lookup = options.lookup as unknown as (
+ hostname: string,
+ options: object,
+ callback: (error: Error | null, address: string, family: number) => void
+ ) => void
+ const resolved = await new Promise((resolve) =>
+ lookup('localhost', {}, (_error, address) => resolve(address))
+ )
+ expect(resolved).toBe('127.0.0.1')
+ } finally {
+ request.mockRestore()
+ pool?.destroy()
+ }
+ })
+})
diff --git a/apps/sim/lib/knowledge/application/chat-citations.test.ts b/apps/sim/lib/knowledge/application/chat-citations.test.ts
new file mode 100644
index 00000000000..0fe6c9a8647
--- /dev/null
+++ b/apps/sim/lib/knowledge/application/chat-citations.test.ts
@@ -0,0 +1,30 @@
+/** @vitest-environment node */
+import { describe, expect, it } from 'vitest'
+import { stripInteractiveCards } from '@/lib/knowledge/application/chat-citations'
+
+describe('stripInteractiveCards', () => {
+ it('removes a JSON card, including closing markers inside its strings', () => {
+ expect(
+ stripInteractiveCards('Answer {"1":{"title":"Say x"}} end')
+ ).toBe('Answer end')
+ })
+ it('removes thinking blocks', () => {
+ expect(stripInteractiveCards('planAnswer')).toBe('Answer')
+ })
+ it('keeps tag-shaped prose', () => {
+ const prose = 'Wrap choices in your list tags, or use to ask.'
+ expect(stripInteractiveCards(prose)).toBe(prose)
+ })
+ it('keeps braces in prose that are not a JSON payload', () => {
+ const prose =
+ 'See {the Q4 report}, and {bold} styling applies.'
+ expect(stripInteractiveCards(prose)).toBe(prose)
+ })
+ it('keeps a closed card whose payload is not valid JSON, and the text after it', () => {
+ const text = 'Before {"a": oops} after.'
+ expect(stripInteractiveCards(text)).toBe(text)
+ })
+ it('drops a card left open before its JSON payload', () => {
+ expect(stripInteractiveCards('Connect here {"type":"link"')).toBe('Connect here ')
+ })
+})
diff --git a/apps/sim/lib/knowledge/application/chat-citations.ts b/apps/sim/lib/knowledge/application/chat-citations.ts
index 3e72158c715..1129c6dc892 100644
--- a/apps/sim/lib/knowledge/application/chat-citations.ts
+++ b/apps/sim/lib/knowledge/application/chat-citations.ts
@@ -8,6 +8,41 @@ export interface SearchChatCitation {
url: string
}
+/** Cards the Chat UI renders from a JSON payload; an MCP text answer drops them. */
+const CARD_TAGS = 'options|question|usage_upgrade|credential|workspace_resource'
+/** A JSON string owns its escaped quotes and any tag-shaped text inside it. */
+const JSON_STRING = '"(?:\\\\.|[^"\\\\\\r\\n])*"'
+/** A closed card candidate; its body is removed only when it parses as a JSON payload. */
+const CLOSED_CARD = new RegExp(
+ `<(${CARD_TAGS})>(\\s*[[{](?:${JSON_STRING}|[^"<])*?[\\]}]\\s*)`,
+ 'g'
+)
+/** A card the model left open, followed by the start of a JSON object with a quoted key. */
+const UNCLOSED_CARD = new RegExp(
+ `<(${CARD_TAGS})>(?![\\s\\S]*)\\s*(?:\\{\\s*"|\\[\\s*[{"])[\\s\\S]*$`
+)
+const THINKING = /[\s\S]*?<\/thinking>/g
+
+function isJsonPayload(body: string): boolean {
+ try {
+ const value: unknown = JSON.parse(body)
+ return typeof value === 'object' && value !== null
+ } catch {
+ return false
+ }
+}
+
+/**
+ * Removes interactive Chat cards so a text-only surface receives only the answer prose. A tag
+ * pair whose body is not a JSON payload is prose that happens to look like a card, and stays.
+ */
+export function stripInteractiveCards(content: string): string {
+ return content
+ .replace(CLOSED_CARD, (card, _tag: string, body: string) => (isJsonPayload(body) ? '' : card))
+ .replace(THINKING, '')
+ .replace(UNCLOSED_CARD, '')
+}
+
/** Resolves Assistant source tags only against successful, bounded retrieval evidence. */
export function resolveSearchChatCitations(content: string, toolCalls: ToolCallSummary[]) {
const evidence = new Map()
diff --git a/apps/sim/lib/knowledge/application/chat.test.ts b/apps/sim/lib/knowledge/application/chat.test.ts
index b1e6fc00650..40936b631f6 100644
--- a/apps/sim/lib/knowledge/application/chat.test.ts
+++ b/apps/sim/lib/knowledge/application/chat.test.ts
@@ -140,7 +140,7 @@ describe('organization Search Assistant chat', () => {
userId: 'member-1',
organizationId: 'org-1',
chatId: 'private-chat',
- goRoute: '/api/mothership/execute',
+ goRoute: '/api/mothership',
interactive: false,
autoExecuteTools: true,
secretActorUserId: null,
@@ -303,6 +303,24 @@ describe('organization Search Assistant chat', () => {
expect(dbChainMockFns.update).toHaveBeenCalledOnce()
})
+ it('drops interactive Chat tags from the MCP answer but keeps them in the transcript', async () => {
+ const tags =
+ '{"1":"Open it"}{"prompt":"Which kit?"}'
+ mocks.lifecycle.mockResolvedValue(createResult({ content: `Violet suitcase.${tags}` }))
+ const result = await execute()
+ expect(result.content).toBe('Violet suitcase.')
+ const [, messages] = mocks.persist.mock.calls[0]
+ expect(JSON.stringify(messages)).toContain('')
+ expect(JSON.stringify(messages)).toContain('')
+ })
+
+ it('refuses an answer that is only interactive Chat tags', async () => {
+ mocks.lifecycle.mockResolvedValue(
+ createResult({ content: '{"1":"Open it"}' })
+ )
+ await expect(execute()).rejects.toThrow('The assistant returned no answer')
+ })
+
it('fails closed when retrieval makes provenance incomplete', async () => {
const registry = new ResolvedSecretTraceRegistry()
mocks.lifecycle.mockImplementation(async () => {
diff --git a/apps/sim/lib/knowledge/application/chat.ts b/apps/sim/lib/knowledge/application/chat.ts
index cd49037d280..9d4d85fbdf5 100644
--- a/apps/sim/lib/knowledge/application/chat.ts
+++ b/apps/sim/lib/knowledge/application/chat.ts
@@ -19,6 +19,7 @@ import { requireOrganizationSearchAvailable } from '@/lib/knowledge/access/avail
import {
resolveSearchChatCitations,
type SearchChatCitation,
+ stripInteractiveCards,
} from '@/lib/knowledge/application/chat-citations'
import { organizationSearchChatOperation } from '@/lib/knowledge/application/chat-operations'
import { loadCopilotSearchIntegrations } from '@/lib/mothership/application/load-search-integrations'
@@ -172,7 +173,7 @@ export const organizationSearchChat: OperationUseCase<
organizationId,
chatId,
simRequestId: messageId,
- goRoute: '/api/mothership/execute',
+ goRoute: '/api/mothership',
interactive: false,
autoExecuteTools: true,
secretActorUserId: null,
@@ -201,7 +202,13 @@ export const organizationSearchChat: OperationUseCase<
buildPersistedAssistantMessage(result, messageId, 'assistant'),
registry
)
- const answer = resolveSearchChatCitations(assistantMessage.content, result.toolCalls)
+ /** MCP clients receive text; interactive Chat cards stay only in the saved transcript. */
+ const answer = resolveSearchChatCitations(
+ stripInteractiveCards(assistantMessage.content),
+ result.toolCalls
+ )
+ if (!answer.content.trim())
+ throw new OrchestrationError('internal', 'The assistant returned no answer. Try again.')
if (!isResolvedSecretModelContentUnchanged(answer, registry)) {
throw new OrchestrationError('internal', 'Search answer could not be returned safely')
}
diff --git a/apps/sim/lib/knowledge/search/diagnostics.ts b/apps/sim/lib/knowledge/search/diagnostics.ts
index fe0086eed4c..95ab1a6505b 100644
--- a/apps/sim/lib/knowledge/search/diagnostics.ts
+++ b/apps/sim/lib/knowledge/search/diagnostics.ts
@@ -64,6 +64,13 @@ export type SearchStage =
| 'source_overview.searchable'
| 'access_batch.connectors'
| 'access_batch.live_proof'
+ | 'live.policies'
+ | 'live.accounts'
+ | 'live.resolve'
+ | 'live.session'
+ | 'live.search'
+ | 'live.verify'
+ | 'live.read'
/** Fixed, content-free fields. Never pass queries, filters, document identities, SQL, or errors. */
export interface SearchDiagnosticMetadata {
diff --git a/apps/sim/lib/sim-search/live/README.md b/apps/sim/lib/sim-search/live/README.md
index 86bf763a9fb..f58cfaa420f 100644
--- a/apps/sim/lib/sim-search/live/README.md
+++ b/apps/sim/lib/sim-search/live/README.md
@@ -48,7 +48,7 @@ Selected labels are alternatives. Source date/query settings are authoritative r
### Google Calendar
-Member mode searches calendars accessible through the member's account. Service mode verifies the account's primary-calendar identity, Directory customer, and source user selection before delegating to that same Workspace user. Selected calendar IDs constrain retrieval and source verification; `primary` means that member's primary calendar. The admin picker browses the delegated administrator's calendar list and stores `primary` as a per-member alias. Service search delegates with only `calendar.events.readonly` (plus `admin.directory.user.readonly` for the Directory check), the same scope as the indexed crawl; all-day events take the calendar's time zone from the events list response. The picker additionally needs `calendar.readonly` ([CalendarList authorization](https://developers.google.com/workspace/calendar/api/v3/reference/calendarList/list)).
+Member mode searches calendars accessible through the member's account. A meeting visible in several calendars is returned once, preferring the member's primary calendar, and date-bounded agendas are ordered by scheduled start across calendars. Service mode verifies the account's primary-calendar identity, Directory customer, and source user selection before delegating to that same Workspace user. Selected calendar IDs constrain retrieval and source verification; `primary` means that member's primary calendar. The admin picker browses the delegated administrator's calendar list and stores `primary` as a per-member alias. Service search delegates with only `calendar.events.readonly` (plus `admin.directory.user.readonly` for the Directory check), the same scope as the indexed crawl; all-day events take the calendar's time zone from the events list response. The picker additionally needs `calendar.readonly` ([CalendarList authorization](https://developers.google.com/workspace/calendar/api/v3/reference/calendarList/list)).
Each event must exist under the source's delegated token, be in an allowed calendar, not be cancelled, and overlap the source's configured rolling time window. The existing default is 30 days before and after the request. A stable UTC-day envelope around that window is intersected with the user's date bounds in the provider query, ensuring recurring events expand and query bounds stay stable between pages. The exact rolling source window is still checked for each result. Nonoverlapping date ranges return no results; continuations spanning a UTC-day change may require a fresh search. A source search query is checked with Calendar's event search and exact event-ID matching. All-day events use the calendar's timezone. Attendee details follow the source's include-attendees setting. The member's API access still determines which event details they can see.
@@ -58,7 +58,7 @@ Member mode only. Search uses the connected user's Slack real-time search grant
### GitHub
-Member mode searches issues, code, and repositories permitted by the connected token. Explicit repository/organization/user qualifiers narrow the user's query. Default discovery is bounded to up to 100 affiliated repositories, and provider pagination/search caps still apply.
+Member mode searches issues, code, and repositories permitted by the connected token. Explicit repository/organization/user qualifiers narrow the user's query. Default discovery is bounded to up to 100 affiliated repositories, sent as `repo:` qualifiers in at most four batches per search kind. Code batches stay under code search's 1,000-byte query limit, issue batches are larger, and a long query that cannot fit every repository reports the searched subset. Date bounds become one `updated:start..end` range, since GitHub ORs repeated qualifiers. Provider pagination/search caps still apply. REST code search returns no file dates and accepts no date qualifier, so date-filtered searches cover issues and pull requests only.
In service mode, an administrator connects a GitHub App installation and selects repositories one by one in Sources. Each source pins the provider-verified repository ID and may narrow code files by directory and extension. Search queries the member's own GitHub connection with `repo:` qualifiers drawn only from active sources. For each candidate, Sim checks that the current App installation still covers that repository, mints a repository-scoped read token, and compares repository and owner IDs returned under both the App and member tokens. It then checks the per-repository code filters. Reads use the member token and repeat these checks. A personal repository outside the selected sources is never searched, even if the member can access it. GitHub REST code search covers the default branch; live Sources therefore do not offer a branch setting.
diff --git a/apps/sim/lib/sim-search/live/account-session.ts b/apps/sim/lib/sim-search/live/account-session.ts
new file mode 100644
index 00000000000..60df3975c8d
--- /dev/null
+++ b/apps/sim/lib/sim-search/live/account-session.ts
@@ -0,0 +1,128 @@
+import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge'
+import type { ResourceOwner } from '@/lib/core/resource-scope'
+import type { PinnedConnectionPool } from '@/lib/core/security/input-validation.server'
+import type { ResolvedLiveAccount } from '@/lib/sim-search/live/accounts'
+import { createCodaMcpClient, readCodaMcp, searchCodaMcp } from '@/lib/sim-search/live/coda-mcp'
+import { createAdminGitLabSession } from '@/lib/sim-search/live/gitlab-admin'
+import { createNativeClient } from '@/lib/sim-search/live/http'
+import { createPolicyVerifier } from '@/lib/sim-search/live/policy'
+import type { LiveSearchPolicy } from '@/lib/sim-search/live/policy-schema'
+import { livePolicyFor, loadLiveSearchPolicies } from '@/lib/sim-search/live/policy-store'
+import { LIVE_SEARCH_PROVIDER_CATALOG } from '@/lib/sim-search/live/provider-catalog'
+import { readNativeProvider, searchNativeProvider } from '@/lib/sim-search/live/providers'
+import { searchWithinPolicy } from '@/lib/sim-search/live/scoped-search'
+import { createLiveServiceSession } from '@/lib/sim-search/live/service-session'
+import type { NativeDocument, NativePage, NativeSearchInput } from '@/lib/sim-search/live/types'
+
+type Reference = Pick<
+ NativeDocument,
+ 'id' | 'container' | 'kind' | 'revision' | 'threadId' | 'accessMetadata'
+>
+
+/** One member account's provider clients and the source boundary that bounds its results. */
+export interface LiveAccountSession {
+ /** The effective policy: the service source's settings in service mode, else the member's. */
+ policy: LiveSearchPolicy
+ /** Service verification covered a bounded subset of the source's configured users. */
+ servicePartial: boolean
+ search(input: NativeSearchInput): Promise
+ /** True only when the document is inside the source boundary and the member may read it. */
+ verify(document: Reference): Promise
+ read(reference: Reference, filters?: WorkspaceSearchFilters): Promise
+ /** Checks a fetched document against the source settings saved now, not at session start. */
+ verifyCurrent(document: NativeDocument): Promise
+}
+
+interface OpenLiveAccountSessionInput {
+ owner: ResourceOwner
+ userId: string
+ resolved: ResolvedLiveAccount
+ policies: Record
+ signal: AbortSignal
+ pool?: PinnedConnectionPool
+}
+
+/**
+ * Opens the clients a search or read of one account needs: the member's provider client (or
+ * Coda MCP client), an administrator GitLab session, and the source verifier. Search and read
+ * share this so both apply exactly the same boundary.
+ */
+export async function openLiveAccountSession(
+ input: OpenLiveAccountSessionInput
+): Promise {
+ const { owner, userId, resolved, signal } = input
+ const { account } = resolved
+ const provider = account.provider
+ const origin =
+ 'origin' in resolved ? resolved.origin : LIVE_SEARCH_PROVIDER_CATALOG[provider].origin
+ const client =
+ account.type === 'managed_mcp'
+ ? null
+ : createNativeClient({ origin, accessToken: resolved.accessToken, signal, pool: input.pool })
+ const admin =
+ 'adminSource' in resolved && client
+ ? await createAdminGitLabSession({
+ owner,
+ userId,
+ source: resolved.adminSource,
+ token: resolved.accessToken,
+ client,
+ signal,
+ })
+ : undefined
+ const mcp = client ? undefined : await createCodaMcpClient(owner, userId, account.id, signal)
+
+ /** A service source replaces the member policy and verifies with its own credential. */
+ const sourceBoundary = async (memberPolicy: LiveSearchPolicy, fresh = false) => {
+ const service = await createLiveServiceSession({
+ owner,
+ userId,
+ provider,
+ policy: memberPolicy,
+ member: client,
+ mcp,
+ signal,
+ pool: input.pool,
+ })
+ if (service) return service
+ const verifyPolicy = createPolicyVerifier(provider, memberPolicy, client, origin, mcp, {
+ fresh,
+ })
+ return {
+ policy: memberPolicy,
+ partial: false,
+ scopeSearch: undefined,
+ verify: (document: Reference) => verifyPolicy(document, document.accessMetadata),
+ }
+ }
+ const boundary = await sourceBoundary(livePolicyFor(input.policies, provider))
+
+ return {
+ policy: boundary.policy,
+ servicePartial: boundary.partial,
+ async search(search) {
+ const scoped = boundary.scopeSearch ? boundary.scopeSearch(search) : search
+ if (scoped === null) return { documents: [] }
+ if (admin) return admin.search(scoped)
+ return searchWithinPolicy(provider, client, scoped, (request) =>
+ client ? searchNativeProvider(provider, client, request) : searchCodaMcp(mcp!, request)
+ )
+ },
+ async verify(document) {
+ if (!(await boundary.verify(document))) return false
+ return !admin || admin.verify(document, document.accessMetadata)
+ },
+ read(reference, filters) {
+ if (admin) return admin.read(reference)
+ if (client) return readNativeProvider(provider, client, reference, boundary.policy, filters)
+ return readCodaMcp(mcp!, reference.id)
+ },
+ async verifyCurrent(document) {
+ const current = await sourceBoundary(
+ livePolicyFor(await loadLiveSearchPolicies(owner), provider),
+ true
+ )
+ return current.verify(document)
+ },
+ }
+}
diff --git a/apps/sim/lib/sim-search/live/accounts.ts b/apps/sim/lib/sim-search/live/accounts.ts
index a7da24b1901..047480f9f2e 100644
--- a/apps/sim/lib/sim-search/live/accounts.ts
+++ b/apps/sim/lib/sim-search/live/accounts.ts
@@ -4,6 +4,7 @@ import { and, eq, inArray, isNull, ne } from 'drizzle-orm'
import { liveSearchProviderSchema } from '@/lib/api/contracts/mothership-assistant-tools'
import {
type ResourceOwner,
+ type ResourceScope,
resourceScopeFields,
resourceScopeFromOwner,
} from '@/lib/core/resource-scope'
@@ -27,74 +28,84 @@ import {
} from '@/lib/sim-search/live/provider-catalog'
import type { LiveAccount } from '@/lib/sim-search/live/types'
+async function listOAuthRows(scope: ResourceScope, userId: string) {
+ if (scope.kind === 'workspace') return getPersonalOAuthCredentials(scope.workspaceId, userId)
+ const [managed, personal] = await Promise.all([
+ getOwnOrganizationManagedOAuthCredentials({ organizationId: scope.organizationId, userId }),
+ db
+ .select({
+ id: credential.id,
+ providerId: credential.providerId,
+ displayName: credential.displayName,
+ })
+ .from(credential)
+ .innerJoin(account, eq(account.id, credential.accountId))
+ .where(
+ and(
+ resourceScopeCondition(credential, scope),
+ eq(credential.type, 'oauth'),
+ eq(account.userId, userId),
+ eq(account.providerId, credential.providerId),
+ ne(credential.providerId, 'slack')
+ )
+ ),
+ ])
+ return [
+ ...managed.map((row) => ({ ...row, type: 'managed_oauth' as const })),
+ ...personal.flatMap((row) =>
+ row.providerId ? [{ ...row, providerId: row.providerId, type: 'oauth' as const }] : []
+ ),
+ ]
+}
+
+async function listCodaTokenRows(scope: ResourceScope, userId: string) {
+ const rows = await db
+ .select({
+ id: credential.id,
+ providerId: credential.providerId,
+ displayName: credential.displayName,
+ })
+ .from(credential)
+ .where(
+ and(
+ resourceScopeCondition(credential, scope),
+ eq(credential.type, 'service_account'),
+ eq(credential.providerId, 'coda-service-account'),
+ eq(credential.createdBy, userId),
+ isNull(credential.revokedAt)
+ )
+ )
+ return rows.map((row) => ({
+ ...row,
+ providerId: 'coda-service-account',
+ type: 'service_account' as const,
+ }))
+}
+
/** Personal provider accounts and ACL-gated administrator-managed GitLab sources. */
export async function listLiveAccounts(
owner: ResourceOwner,
userId: string
): Promise {
const scope = resourceScopeFromOwner(owner)
- const oauth =
- scope.kind === 'workspace'
- ? await getPersonalOAuthCredentials(scope.workspaceId, userId)
- : [
- ...(
- await getOwnOrganizationManagedOAuthCredentials({
- organizationId: scope.organizationId,
- userId,
- })
- ).map((row) => ({ ...row, type: 'managed_oauth' as const })),
- ...(
- await db
- .select({
- id: credential.id,
- providerId: credential.providerId,
- displayName: credential.displayName,
- })
- .from(credential)
- .innerJoin(account, eq(account.id, credential.accountId))
- .where(
- and(
- resourceScopeCondition(credential, scope),
- eq(credential.type, 'oauth'),
- eq(account.userId, userId),
- eq(account.providerId, credential.providerId),
- ne(credential.providerId, 'slack')
- )
- )
- ).flatMap((row) =>
- row.providerId ? [{ ...row, providerId: row.providerId, type: 'oauth' as const }] : []
- ),
- ]
- const workspaceContext =
+ /** A workspace's organization is known only after its context loads; an organization's is not. */
+ const loadApprovals = (organizationId?: string | null) =>
+ organizationId ? listOrganizationSearchApprovals(organizationId) : null
+ const [oauth, coda, workspaceContext, organizationApprovals] = await Promise.all([
+ listOAuthRows(scope, userId),
+ listCodaTokenRows(scope, userId),
scope.kind === 'workspace'
- ? await resolveKnowledgeWorkspaceContext({ workspaceId: scope.workspaceId })
- : undefined
- const organizationId =
- scope.kind === 'organization' ? scope.organizationId : workspaceContext?.workspaceOrganizationId
- const approvals = organizationId ? await listOrganizationSearchApprovals(organizationId) : null
+ ? resolveKnowledgeWorkspaceContext({ workspaceId: scope.workspaceId })
+ : undefined,
+ scope.kind === 'organization' ? loadApprovals(scope.organizationId) : null,
+ ])
+ const approvals =
+ organizationApprovals ?? (await loadApprovals(workspaceContext?.workspaceOrganizationId))
const denied = new Set(
approvals
? liveSearchProviderSchema.options.filter((provider) => approvals.get(provider) !== true)
: []
)
- const coda = (
- await db
- .select({
- id: credential.id,
- providerId: credential.providerId,
- displayName: credential.displayName,
- })
- .from(credential)
- .where(
- and(
- resourceScopeCondition(credential, scope),
- eq(credential.type, 'service_account'),
- eq(credential.providerId, 'coda-service-account'),
- eq(credential.createdBy, userId),
- isNull(credential.revokedAt)
- )
- )
- ).map((row) => ({ ...row, providerId: 'coda-service-account', type: 'service_account' as const }))
const candidates = [...oauth, ...coda].flatMap((row) => {
const provider = liveSearchProviderForCredential(row.providerId)
return provider && supportsLiveSearchMode(provider, 'member') && !denied.has(provider)
@@ -110,11 +121,11 @@ export async function listLiveAccounts(
]
: []
})
- const visible = workspaceContext
- ? await filterWorkspaceAccountCredentials(workspaceContext, candidates)
- : candidates
- const mcp = denied.has('coda') ? [] : await listCodaMcpSearchAccounts(owner, userId)
- const admin = denied.has('gitlab') ? [] : await listAdminGitLabAccounts(owner)
+ const [visible, mcp, admin] = await Promise.all([
+ workspaceContext ? filterWorkspaceAccountCredentials(workspaceContext, candidates) : candidates,
+ denied.has('coda') ? [] : listCodaMcpSearchAccounts(owner, userId),
+ denied.has('gitlab') ? [] : listAdminGitLabAccounts(owner),
+ ])
if (visible.length === 0) return [...mcp, ...admin]
// Metadata is fetched in one batch; a fresh binding check still precedes token resolution.
const rows = await db
@@ -152,10 +163,25 @@ export async function listLiveAccounts(
]
}
+/** Re-lists the member's accounts so a reference from an earlier request is checked afresh. */
export async function resolveLiveAccount(owner: ResourceOwner, userId: string, accountId: string) {
const current = (await listLiveAccounts(owner, userId)).find((row) => row.id === accountId)
if (!current)
throw new NativeSearchError('reconnect', 'This search connection is no longer available.')
+ return resolveListedLiveAccount(owner, userId, current)
+}
+
+export type ResolvedLiveAccount = Awaited>
+
+/**
+ * Resolves credentials for an account {@link listLiveAccounts} returned earlier in the same
+ * request, which already performed the binding and revocation checks.
+ */
+export async function resolveListedLiveAccount(
+ owner: ResourceOwner,
+ userId: string,
+ current: LiveAccount
+) {
if (current.type === 'admin_source') return resolveAdminGitLabAccount(owner, current)
if (current.type === 'managed_mcp') return { account: current, accessToken: '', mcp: true }
const scope = resourceScopeFromOwner(owner)
diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts
index d5c1da01caf..0f733d3889c 100644
--- a/apps/sim/lib/sim-search/live/application.test.ts
+++ b/apps/sim/lib/sim-search/live/application.test.ts
@@ -16,6 +16,10 @@ const mocks = vi.hoisted(() => ({
adminVerify: vi.fn(),
json: vi.fn(),
service: vi.fn(),
+ destroyPool: vi.fn(),
+}))
+vi.mock('@/lib/core/security/input-validation.server', () => ({
+ createPinnedConnectionPool: () => ({ agent: vi.fn(), destroy: mocks.destroyPool }),
}))
vi.mock('@/lib/sim-search/live/service-session', () => ({
createLiveServiceSession: mocks.service,
@@ -49,6 +53,10 @@ vi.mock('@/lib/knowledge/application/contexts', () => ({
vi.mock('@/lib/sim-search/live/accounts', () => ({
listLiveAccounts: mocks.accounts,
resolveLiveAccount: mocks.resolveAccount,
+ resolveListedLiveAccount: async (owner: unknown, userId: string, listed: { id: string }) => ({
+ ...(await mocks.resolveAccount(owner, userId, listed.id)),
+ account: listed,
+ }),
}))
vi.mock('@/lib/sim-search/live/providers', () => ({
NATIVE_SEARCH_GUIDANCE: 'Live coverage',
@@ -233,6 +241,275 @@ describe('authorized live retrieval', () => {
expect.objectContaining({ filters })
)
})
+ it('reports more matches as a cursor, not as degraded coverage', async () => {
+ mocks.search.mockResolvedValue({ documents: [document], nextCursor: 'next', hasMore: true })
+ const result = await searchLiveKnowledge.execute({ principal, input: { ...input, topK: 1 } })
+ expect(result.retrieval.status).toBe('complete')
+ expect(result.live?.accounts[0]).toMatchObject({ status: 'ok', nextCursor: 'next' })
+ })
+ it('keeps more matches partial when a date order must cover them', async () => {
+ mocks.search.mockResolvedValue({ documents: [document], nextCursor: 'next' })
+ const result = await searchLiveKnowledge.execute({
+ principal,
+ input: { ...input, filters: { sortBy: 'newest' } },
+ })
+ expect(result.retrieval.status).toBe('partial')
+ })
+ it('reports candidates that could not be verified and keeps the verified ones', async () => {
+ mocks.search.mockResolvedValue({
+ documents: [document, { ...document, id: 'other', url: 'https://docs.google.com/other' }],
+ })
+ mocks.service.mockResolvedValue({
+ policy: defaultLiveSearchPolicy(),
+ partial: false,
+ verify: async ({ id }: { id: string }) => {
+ if (id === 'other') throw new NativeSearchError('unavailable', 'Budget')
+ return true
+ },
+ })
+ const result = await searchLiveKnowledge.execute({ principal, input })
+ expect(result.results.map((row) => decodeLiveReference(row.documentId).id)).toEqual(['doc'])
+ expect(result.live?.accounts[0]).toMatchObject({
+ status: 'partial',
+ message: expect.stringContaining('could not be verified'),
+ })
+ })
+ it('keeps verified results and names a rate limit hit during verification', async () => {
+ mocks.search.mockResolvedValue({
+ documents: [document, { ...document, id: 'other', url: 'https://docs.google.com/other' }],
+ })
+ mocks.service.mockResolvedValue({
+ policy: defaultLiveSearchPolicy(),
+ partial: false,
+ verify: async ({ id }: { id: string }) => {
+ if (id === 'other') throw new NativeSearchError('rate_limited', 'Later', 30)
+ return true
+ },
+ })
+ const result = await searchLiveKnowledge.execute({ principal, input })
+ expect(result.results).toHaveLength(1)
+ expect(result.live?.accounts[0]).toMatchObject({
+ status: 'partial',
+ message: expect.stringContaining('rate-limited verification'),
+ })
+ })
+ it('fails the account when a grant is revoked during verification', async () => {
+ mocks.service.mockResolvedValue({
+ policy: defaultLiveSearchPolicy(),
+ partial: false,
+ verify: async () => {
+ throw new NativeSearchError('reconnect', 'Revoked')
+ },
+ })
+ const result = await searchLiveKnowledge.execute({ principal, input })
+ expect(result.results).toEqual([])
+ expect(result.live?.accounts[0]).toMatchObject({ status: 'reconnect', message: 'Revoked' })
+ })
+ it('verifies GitLab candidates with the evidence from their own search response', async () => {
+ const gitlab = {
+ ...account,
+ id: 'gitlab-source:source',
+ provider: 'gitlab',
+ type: 'admin_source',
+ }
+ const evidence = { confidential: false, authorId: 7, assigneeIds: [] }
+ mocks.accounts.mockResolvedValue([gitlab])
+ mocks.resolveAccount.mockResolvedValue({
+ account: gitlab,
+ accessToken: 'admin-secret',
+ origin: 'https://gitlab.company.com',
+ adminSource: { id: 'source', config: { project: '42' } },
+ })
+ mocks.adminSearch.mockResolvedValue({
+ documents: [
+ { ...document, id: '5', container: '42', kind: 'issues', accessMetadata: evidence },
+ ],
+ })
+ await searchLiveKnowledge.execute({ principal, input })
+ expect(mocks.adminVerify).toHaveBeenCalledWith(expect.objectContaining({ id: '5' }), evidence)
+ })
+ it('merges equal ranks in a stable account order', async () => {
+ const gmail = { ...account, id: 'mail', provider: 'gmail', displayName: 'Mail' }
+ mocks.accounts.mockResolvedValue([gmail, account])
+ mocks.search.mockImplementation(async (provider: string) => ({
+ documents: [{ ...document, id: provider, url: `https://docs.google.com/${provider}` }],
+ }))
+ const result = await searchLiveKnowledge.execute({ principal, input })
+ mocks.accounts.mockResolvedValue([account, gmail])
+ const reversed = await searchLiveKnowledge.execute({ principal, input })
+ const order = (data: typeof result) => data.results.map((row) => row.connectorType)
+ expect(order(reversed)).toEqual(order(result))
+ })
+ it('reports partial coverage when more matches exist that no cursor reaches', async () => {
+ mocks.search.mockResolvedValue({ documents: [document], hasMore: true })
+ const result = await searchLiveKnowledge.execute({ principal, input })
+ expect(result.results).toHaveLength(1)
+ expect(result.live?.accounts[0]).toMatchObject({ status: 'partial' })
+ })
+ it('keeps the read page budget of indexed reads at any chunk limit', async () => {
+ mocks.read.mockResolvedValue({ ...document, content: 'x'.repeat(30_000) })
+ const search = await searchLiveKnowledge.execute({ principal, input })
+ const read = (limit: number) =>
+ readLiveDocument.execute({
+ principal,
+ input: {
+ workspaceId: 'workspace',
+ documentId: search.results[0]!.documentId,
+ limit,
+ resultSecretRegistry: new ResolvedSecretTraceRegistry([]),
+ },
+ })
+ expect((await read(3)).chunks[0]?.content).toHaveLength(8000)
+ expect((await read(8)).chunks[0]?.content).toHaveLength(8000)
+ })
+ it('centers the preview on the query match', async () => {
+ mocks.search.mockResolvedValue({
+ documents: [{ ...document, content: `${'filler '.repeat(500)}launch checklist` }],
+ })
+ const result = await searchLiveKnowledge.execute({ principal, input })
+ expect(result.results[0]?.content).toContain('launch checklist')
+ })
+ it('keeps the first verified copy of an item a provider returned twice', async () => {
+ mocks.search.mockResolvedValue({
+ documents: [
+ {
+ ...document,
+ id: 'primary-copy',
+ url: 'https://calendar.google.com/a',
+ dedupeKey: 'meeting',
+ },
+ {
+ ...document,
+ id: 'team-copy',
+ url: 'https://calendar.google.com/b',
+ dedupeKey: 'meeting',
+ },
+ ],
+ })
+ mocks.service.mockResolvedValue({
+ policy: defaultLiveSearchPolicy(),
+ partial: false,
+ verify: async ({ id }: { id: string }) => id === 'team-copy',
+ })
+ const result = await searchLiveKnowledge.execute({ principal, input })
+ expect(result.results.map((row) => decodeLiveReference(row.documentId).id)).toEqual([
+ 'team-copy',
+ ])
+ })
+ it('reports undated exclusions only for documents the member may read', async () => {
+ mocks.search.mockResolvedValue({
+ documents: [{ ...document, id: 'hidden', modifiedAt: undefined }],
+ })
+ mocks.service.mockResolvedValue({
+ policy: defaultLiveSearchPolicy(),
+ partial: false,
+ verify: async () => false,
+ })
+ const result = await searchLiveKnowledge.execute({
+ principal,
+ input: { ...input, filters: { modifiedAfter: '2026-01-01T00:00:00Z' } },
+ })
+ expect(result.live?.accounts[0]).toMatchObject({ status: 'ok' })
+ expect(result.live?.accounts[0]?.message).toBeUndefined()
+ })
+ it('drops the cursor of an account whose results were cut from the merge', async () => {
+ const gmail = { ...account, id: 'mail', provider: 'gmail', displayName: 'Mail' }
+ mocks.accounts.mockResolvedValue([account, gmail])
+ mocks.search.mockImplementation(async (provider: string) => ({
+ documents: [1, 2].map((rank) => ({
+ ...document,
+ id: `${provider}-${rank}`,
+ url: `https://docs.google.com/${provider}-${rank}`,
+ })),
+ nextCursor: 'next',
+ }))
+ const result = await searchLiveKnowledge.execute({ principal, input: { ...input, topK: 2 } })
+ expect(result.results).toHaveLength(2)
+ for (const status of result.live?.accounts ?? []) {
+ expect(status.nextCursor).toBeUndefined()
+ expect(status.message).toContain('Search this account alone')
+ }
+ })
+ it('drops the cursor only for the account whose results were cut from the merge', async () => {
+ const gmail = { ...account, id: 'mail', provider: 'gmail', displayName: 'Mail' }
+ mocks.accounts.mockResolvedValue([account, gmail])
+ mocks.search.mockImplementation(async (provider: string) => ({
+ documents: (provider === 'gmail' ? [1, 2] : [1]).map((rank) => ({
+ ...document,
+ id: `${provider}-${rank}`,
+ url: `https://docs.google.com/${provider}-${rank}`,
+ })),
+ nextCursor: 'next',
+ }))
+ const result = await searchLiveKnowledge.execute({ principal, input: { ...input, topK: 2 } })
+ const statusOf = (id: string) => result.live?.accounts.find((row) => row.accountId === id)
+ expect(result.results.map((row) => decodeLiveReference(row.documentId).id)).not.toContain(
+ 'gmail-2'
+ )
+ expect(statusOf('mail')?.nextCursor).toBeUndefined()
+ expect(statusOf('account')).toMatchObject({ nextCursor: 'next' })
+ })
+ it('reports a dated search partial when the provider has more it cannot continue', async () => {
+ mocks.search.mockResolvedValue({ documents: [document], hasMore: true })
+ const result = await searchLiveKnowledge.execute({
+ principal,
+ input: {
+ ...input,
+ filters: { startDate: '2026-08-01T00:00:00Z', endDate: '2026-10-01T00:00:00Z' },
+ },
+ })
+ expect(result.results).toHaveLength(1)
+ expect(result.live?.accounts[0]).toMatchObject({
+ status: 'partial',
+ message: expect.stringContaining('could return'),
+ })
+ })
+ it('applies date filters before spending provider verification', async () => {
+ const verify = vi.fn(async () => true)
+ mocks.service.mockResolvedValue({ policy: defaultLiveSearchPolicy(), verify, partial: false })
+ const outside = {
+ ...document,
+ id: 'outside',
+ url: 'https://docs.google.com/outside',
+ modifiedAt: '2026-01-01T00:00:00Z',
+ }
+ mocks.search.mockResolvedValue({ documents: [document, outside] })
+ const result = await searchLiveKnowledge.execute({
+ principal,
+ input: {
+ ...input,
+ filters: { startDate: '2026-08-01T00:00:00Z', endDate: '2026-10-01T00:00:00Z' },
+ },
+ })
+ expect(result.results.map((row) => decodeLiveReference(row.documentId).id)).toEqual(['doc'])
+ expect(verify).toHaveBeenCalledExactlyOnceWith(document)
+ })
+ it('releases the connection pool once, even when the search is cancelled mid-flight', async () => {
+ await searchLiveKnowledge.execute({ principal, input })
+ expect(mocks.destroyPool).toHaveBeenCalledOnce()
+ mocks.destroyPool.mockClear()
+ const controller = new AbortController()
+ mocks.search.mockImplementation(async () => {
+ controller.abort()
+ throw new NativeSearchError('unavailable', 'Cancelled')
+ })
+ await expect(
+ searchLiveKnowledge.execute({ principal, input: { ...input, signal: controller.signal } })
+ ).rejects.toThrow()
+ expect(mocks.destroyPool).toHaveBeenCalledOnce()
+ })
+ it('reports partial coverage when more matches exist but none could be returned', async () => {
+ mocks.search.mockResolvedValue({ documents: [], hasMore: true })
+ const result = await searchLiveKnowledge.execute({ principal, input })
+ expect(result.retrieval.status).toBe('partial')
+ expect(result.live?.accounts[0]?.message).toContain('could return')
+ })
+ it('reports partial coverage when a continuable page returned nothing readable', async () => {
+ mocks.search.mockResolvedValue({ documents: [], nextCursor: 'next' })
+ const result = await searchLiveKnowledge.execute({ principal, input })
+ expect(result.retrieval.status).toBe('partial')
+ expect(result.live?.accounts[0]).toMatchObject({ status: 'partial', nextCursor: 'next' })
+ })
it('rejects invalid dates before resolving provider credentials', async () => {
await expect(
searchLiveKnowledge.execute({
diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts
index 6a3759084f1..1841c2185dd 100644
--- a/apps/sim/lib/sim-search/live/application.ts
+++ b/apps/sim/lib/sim-search/live/application.ts
@@ -1,4 +1,5 @@
import { requirePrincipalSubjectUserId } from '@sim/auth/principal'
+import { compareStrings } from '@sim/utils/string'
import { z } from 'zod'
import type { WorkspaceSearchFilters } from '@/lib/api/contracts/knowledge'
import type {
@@ -19,37 +20,37 @@ import {
resourceScopeFromOwner,
resourceScopeKey,
} from '@/lib/core/resource-scope'
+import { createPinnedConnectionPool } from '@/lib/core/security/input-validation.server'
+import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
import { requireOrganizationSearchAvailable } from '@/lib/knowledge/access/availability'
import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case'
import { resolveKnowledgeOwnerContext } from '@/lib/knowledge/application/contexts'
import { knowledgeOperations } from '@/lib/knowledge/application/operations'
import { isKnowledgeSourceUrl } from '@/lib/knowledge/search/citation'
-import { listLiveAccounts, resolveLiveAccount } from '@/lib/sim-search/live/accounts'
-import { createCodaMcpClient, readCodaMcp, searchCodaMcp } from '@/lib/sim-search/live/coda-mcp'
+import { measureSearchStage } from '@/lib/knowledge/search/diagnostics'
+import { matchPassage } from '@/lib/knowledge/search/snippet'
import {
+ type LiveAccountSession,
+ openLiveAccountSession,
+} from '@/lib/sim-search/live/account-session'
+import {
+ listLiveAccounts,
+ resolveListedLiveAccount,
+ resolveLiveAccount,
+} from '@/lib/sim-search/live/accounts'
+import {
+ dateSortDirection,
hasDateBounds,
matchesSourceDates,
sourceDate,
sourceDateType,
} from '@/lib/sim-search/live/dates'
-import { createAdminGitLabSession } from '@/lib/sim-search/live/gitlab-admin'
-import { createNativeClient, NativeSearchError } from '@/lib/sim-search/live/http'
-import { createPolicyVerifier } from '@/lib/sim-search/live/policy'
-import { livePolicyFor, loadLiveSearchPolicies } from '@/lib/sim-search/live/policy-store'
-import { LIVE_SEARCH_PROVIDER_CATALOG } from '@/lib/sim-search/live/provider-catalog'
-import {
- NATIVE_SEARCH_GUIDANCE,
- readNativeProvider,
- searchNativeProvider,
-} from '@/lib/sim-search/live/providers'
-import { searchWithinPolicy } from '@/lib/sim-search/live/scoped-search'
-import { createLiveServiceSession } from '@/lib/sim-search/live/service-session'
-import type {
- LiveAccount,
- NativeDocument,
- NativePage,
- NativeSearchInput,
-} from '@/lib/sim-search/live/types'
+import { NativeSearchError } from '@/lib/sim-search/live/http'
+import { joinMessages } from '@/lib/sim-search/live/pages'
+import { loadLiveSearchPolicies } from '@/lib/sim-search/live/policy-store'
+import { LIVE_SEARCH_PROVIDER_IDS } from '@/lib/sim-search/live/provider-catalog'
+import { NATIVE_SEARCH_GUIDANCE } from '@/lib/sim-search/live/providers'
+import type { LiveAccount, NativeDocument } from '@/lib/sim-search/live/types'
import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection'
import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
@@ -137,27 +138,42 @@ export function matchesLiveFilters(
}
return true
}
-function resultFor(
+/** A provider result with the reference that binds it to this member, scope, and account. */
+interface LiveCandidate {
+ document: NativeDocument
+ documentId: string
+}
+
+function candidateFor(
document: NativeDocument,
account: LiveAccount,
owner: ResourceOwner,
- userId: string,
+ userId: string
+): LiveCandidate {
+ return {
+ document,
+ documentId: encodeLiveReference({
+ v: 1,
+ user: userId,
+ scope: resourceScopeKey(resourceScopeFromOwner(owner)),
+ account: account.id,
+ provider: account.provider,
+ id: document.id,
+ ...(document.container ? { container: document.container } : {}),
+ ...(document.kind ? { kind: document.kind } : {}),
+ ...(document.revision ? { revision: document.revision } : {}),
+ ...(document.threadId ? { threadId: document.threadId } : {}),
+ }),
+ }
+}
+
+function resultFor(
+ { document, documentId }: LiveCandidate,
+ account: LiveAccount,
rank: number,
+ previewQuery: string,
registry?: ResolvedSecretTraceRegistry
): WorkspaceKnowledgeSearchResult {
- const documentId = encodeLiveReference({
- v: 1,
- user: userId,
- scope: resourceScopeKey(resourceScopeFromOwner(owner)),
- account: account.id,
- provider: account.provider,
- id: document.id,
- ...(document.container ? { container: document.container } : {}),
- ...(document.kind ? { kind: document.kind } : {}),
- ...(document.revision ? { revision: document.revision } : {}),
- ...(document.threadId ? { threadId: document.threadId } : {}),
- })
- // The shared UI wire shape retains its index fields; live results never use them as identifiers.
return {
documentId,
knowledgeBaseId: '',
@@ -178,12 +194,83 @@ function resultFor(
sourceDate: sourceDate(document, account.provider) ?? null,
sourceDateType: sourceDateType(account.provider, document),
author: document.author ? safeContent(document.author, registry) : null,
- content: safeContent(document.content, registry).slice(0, 1800),
+ content: matchPassage(safeContent(document.content, registry), previewQuery, PREVIEW_CHARACTERS)
+ .content,
chunkIndex: 0,
similarity: 1 / (60 + rank),
}
}
+/** Accounts searched at once; bounds token refresh and provider fan-out in large organizations. */
+const ACCOUNT_CONCURRENCY = 4
+/** Candidates verified at once; each verification is one or more provider requests. */
+const VERIFY_CONCURRENCY = 5
+const MAX_ACCOUNTS = 20
+/** Previews center on the query's longest matching term, like indexed passages. */
+const PREVIEW_CHARACTERS = 1800
+/** Characters one read returns, the same page budget indexed document reads use. */
+const READ_WINDOW_CHARACTERS = 8000
+
+/**
+ * Verifies candidates in rank order. A revoked grant fails the whole account; any other
+ * verification failure only omits that candidate, and is reported as incomplete coverage.
+ */
+async function verifyCandidates(session: LiveAccountSession, candidates: LiveCandidate[]) {
+ const outcomes = await mapWithConcurrency(
+ candidates,
+ VERIFY_CONCURRENCY,
+ async ({ document }) => {
+ try {
+ return (await session.verify(document)) ? ('permitted' as const) : ('denied' as const)
+ } catch (error) {
+ if (error instanceof NativeSearchError && error.status === 'reconnect') throw error
+ return error instanceof NativeSearchError && error.status === 'rate_limited'
+ ? ('rate_limited' as const)
+ : ('unverified' as const)
+ }
+ }
+ )
+ return {
+ permitted: candidates.filter((_, index) => outcomes[index] === 'permitted'),
+ unverified: outcomes.some((outcome) => outcome === 'unverified' || outcome === 'rate_limited'),
+ rateLimited: outcomes.includes('rate_limited'),
+ }
+}
+
+/** Keeps the first of candidates a provider marked as the same item, such as a shared meeting. */
+function firstOfEachDocument(candidates: LiveCandidate[]): LiveCandidate[] {
+ const seen = new Set()
+ return candidates.filter(({ document }) => {
+ if (!document.dedupeKey) return true
+ if (seen.has(document.dedupeKey)) return false
+ seen.add(document.dedupeKey)
+ return true
+ })
+}
+
+/** Whether a result lacks the date metadata the requested date filters need. */
+function lacksFilterDate(
+ document: NativeDocument,
+ provider: string,
+ filters?: WorkspaceSearchFilters
+): boolean {
+ return (
+ (Boolean(filters?.startDate || filters?.endDate) && !sourceDate(document, provider)) ||
+ (Boolean(filters?.modifiedAfter || filters?.modifiedBefore) &&
+ !Number.isFinite(Date.parse(document.modifiedAt ?? '')))
+ )
+}
+
+/** Stable account order so equal-rank results from different accounts always merge the same way. */
+function compareAccounts(left: LiveAccount, right: LiveAccount): number {
+ return (
+ LIVE_SEARCH_PROVIDER_IDS.indexOf(left.provider) -
+ LIVE_SEARCH_PROVIDER_IDS.indexOf(right.provider) ||
+ compareStrings(left.displayName, right.displayName) ||
+ compareStrings(left.id, right.id)
+ )
+}
+
export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({
operation: knowledgeOperations.search,
resolveContext: ({ input }: { input: LiveSearchInput }) => resolveKnowledgeOwnerContext(input),
@@ -202,221 +289,218 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({
throw new OrchestrationError('validation', 'Invalid live search query or result limit')
if (input.filters)
input = { ...input, filters: workspaceSearchFiltersSchema.parse(input.filters) }
+ const filters = input.filters
if (
- input.filters?.startDate &&
- input.filters.endDate &&
- Date.parse(input.filters.startDate) >= Date.parse(input.filters.endDate)
+ filters?.startDate &&
+ filters.endDate &&
+ Date.parse(filters.startDate) >= Date.parse(filters.endDate)
)
throw new OrchestrationError('validation', 'endDate must be after startDate')
const queries = input.nativeQueries
? nativeSearchQueriesSchema.parse(input.nativeQueries)
: undefined
- if (queries?.some((query) => !query.query) && !hasDateBounds(input.filters))
+ if (queries?.some((query) => !query.query) && !hasDateBounds(filters))
throw new OrchestrationError('validation', 'Empty native queries require a date bound')
const searchSignal = input.signal
? AbortSignal.any([input.signal, AbortSignal.timeout(20_000)])
: AbortSignal.timeout(20_000)
- const policies = await loadLiveSearchPolicies(input)
- const allAccounts = await listLiveAccounts(input, userId)
- const eligible = allAccounts.filter(
- (account) =>
- (!input.filters?.source || input.filters.source === account.provider) &&
- (!queries ||
- queries.some(
- (query) =>
- query.provider === account.provider &&
- (!query.accountId || query.accountId === account.id)
- ))
- )
- const selected = eligible.slice(0, 20)
- const accounts: LiveSearchAccountStatus[] = []
- const results: WorkspaceKnowledgeSearchResult[] = []
- // Four accounts at a time bounds token refresh and API fanout across large organizations.
- for (let offset = 0; offset < selected.length; offset += 4) {
- const batch = await Promise.all(
- selected.slice(offset, offset + 4).map(async (account) => {
- const status = {
- accountId: account.id,
- provider: account.provider,
- displayName: account.displayName,
- }
- const signal = AbortSignal.any([searchSignal, AbortSignal.timeout(12_000)])
- try {
- signal.throwIfAborted()
- const resolved = await resolveLiveAccount(input, userId, account.id)
- const client =
- resolved.account.type === 'managed_mcp'
- ? null
- : createNativeClient({
- origin:
- 'origin' in resolved
- ? resolved.origin
- : LIVE_SEARCH_PROVIDER_CATALOG[account.provider].origin,
- accessToken: resolved.accessToken,
- signal,
- })
- const native = queries?.find(
- (query) =>
- query.provider === account.provider &&
- (!query.accountId || query.accountId === account.id)
- )
- let policy = livePolicyFor(policies, account.provider)
- const admin =
- 'adminSource' in resolved && client
- ? await createAdminGitLabSession({
- owner: input,
- userId,
- source: resolved.adminSource,
- token: resolved.accessToken,
- client,
- signal,
- })
- : undefined
- const mcp = client
- ? undefined
- : await createCodaMcpClient(input, userId, account.id, signal)
- const service = await createLiveServiceSession({
- owner: input,
- userId,
- provider: account.provider,
- policy,
- member: client,
- mcp,
- signal,
- })
- if (service) policy = service.policy
- const verify =
- service?.verify ??
- createPolicyVerifier(
- account.provider,
- policy,
- client,
- 'origin' in resolved
- ? resolved.origin
- : LIVE_SEARCH_PROVIDER_CATALOG[account.provider].origin,
- mcp
- )
- const searchInput: NativeSearchInput = {
- filters: input.filters,
- policy,
- query: input.query,
- native,
- limit: input.topK,
- scopes: resolved.account.scopes,
- }
- const scopedInput = service?.scopeSearch
- ? service.scopeSearch(searchInput)
- : searchInput
- const page: NativePage =
- scopedInput === null
- ? { documents: [] }
- : admin
- ? await admin.search(scopedInput)
- : await searchWithinPolicy(account.provider, client, scopedInput, (scoped) =>
- client
- ? searchNativeProvider(account.provider, client, scoped)
- : searchCodaMcp(mcp!, scoped)
- )
- const permitted: NativeDocument[] = []
- let unverified = false
- for (const document of page.documents) {
- try {
- if ((await verify(document)) && (!admin || (await admin.verify(document))))
- permitted.push(document)
- } catch (error) {
- if (error instanceof NativeSearchError && error.status === 'reconnect') throw error
- unverified = true
- }
- }
- const rows = permitted
- .filter((document) => document.id)
- .map((document, index) => ({
- document,
- result: resultFor(
- document,
- account,
- input,
- userId,
- index + 1,
- input.resultSecretRegistry
- ),
- }))
- const matching = rows
- .filter(({ document, result }) =>
- matchesLiveFilters(document, result.documentId, account.provider, input.filters)
- )
- .map(({ result }) => result)
- return {
- status: {
- ...status,
- status:
- (input.filters?.sortBy &&
- input.filters.sortBy !== 'relevance' &&
- rows.some(
- ({ document }) =>
- !Number.isFinite(Date.parse(sourceDate(document, account.provider) ?? ''))
- )) ||
- unverified ||
- service?.partial ||
- page.partial ||
- page.nextCursor ||
- matching.length < rows.length
- ? ('partial' as const)
- : ('ok' as const),
- message:
- [
- page.message,
- service?.partial
- ? 'Service account verification covered a bounded subset of the configured users. Narrow the source user list for complete coverage; external Drive users can only search files also visible to the source administrator.'
- : undefined,
- (input.filters?.startDate || input.filters?.endDate) &&
- rows.some(
- ({ document }) =>
- !Number.isFinite(Date.parse(sourceDate(document, account.provider) ?? ''))
- )
- ? 'Some results lacked date metadata and were excluded; date coverage is incomplete.'
- : undefined,
- input.filters?.sortBy && input.filters.sortBy !== 'relevance'
- ? 'Date order covers retrieved results; follow continuation before claiming an overall earliest or latest match.'
- : undefined,
- ]
- .filter(Boolean)
- .join(' ') || undefined,
- nextCursor: page.nextCursor,
- },
- results: matching,
- }
- } catch (error) {
- input.signal?.throwIfAborted()
- const failure =
- error instanceof NativeSearchError
- ? error
- : signal.aborted
- ? new NativeSearchError(
- 'timeout',
- 'Provider search timed out. Narrow the query or try again.'
- )
- : new NativeSearchError(
- 'unavailable',
- 'This account could not be searched. Check the connection and try again.'
- )
- return {
- status: {
- ...status,
- status: failure.status,
- message: failure.message,
- retryAfterSeconds: failure.retryAfterSeconds,
- },
- results: [],
- }
- }
- })
+ const nativeFor = (account: LiveAccount) =>
+ queries?.find(
+ (query) =>
+ query.provider === account.provider &&
+ (!query.accountId || query.accountId === account.id)
)
- for (const item of batch) {
- accounts.push(item.status)
- results.push(...item.results)
+ const [policies, allAccounts] = await Promise.all([
+ measureSearchStage('live.policies', () => loadLiveSearchPolicies(input)),
+ measureSearchStage('live.accounts', () => listLiveAccounts(input, userId)),
+ ])
+ const eligible = allAccounts
+ .filter(
+ (account) =>
+ (!filters?.source || filters.source === account.provider) &&
+ (!queries || nativeFor(account))
+ )
+ .sort(compareAccounts)
+ const selected = eligible.slice(0, MAX_ACCOUNTS)
+ const direction = dateSortDirection(filters)
+ const dateSorted = Boolean(direction)
+ const pool = createPinnedConnectionPool()
+ const searchAccount = async (
+ account: LiveAccount
+ ): Promise<{ status: LiveSearchAccountStatus; results: WorkspaceKnowledgeSearchResult[] }> => {
+ const status = {
+ accountId: account.id,
+ provider: account.provider,
+ displayName: account.displayName,
}
+ /** Cancels requests still in flight once the account settles, including after a failure. */
+ const settled = new AbortController()
+ const signal = AbortSignal.any([searchSignal, AbortSignal.timeout(12_000), settled.signal])
+ try {
+ signal.throwIfAborted()
+ const resolved = await measureSearchStage('live.resolve', () =>
+ resolveListedLiveAccount(input, userId, account)
+ )
+ const session = await measureSearchStage('live.session', () =>
+ openLiveAccountSession({ owner: input, userId, resolved, policies, signal, pool })
+ )
+ const native = nativeFor(account)
+ const page = await measureSearchStage('live.search', () =>
+ session.search({
+ filters,
+ policy: session.policy,
+ query: input.query,
+ native,
+ limit: input.topK,
+ scopes: resolved.account.scopes,
+ })
+ )
+ const candidates = page.documents
+ .filter((document) => document.id)
+ .map((document) => candidateFor(document, account, input, userId))
+ /**
+ * Local filters run first so provider verification is spent only on eligible results.
+ * Undated results are verified too, so their exclusion is reported only when readable.
+ */
+ const { permitted, unverified, rateLimited } = await measureSearchStage('live.verify', () =>
+ verifyCandidates(
+ session,
+ candidates.filter(
+ ({ document, documentId }) =>
+ matchesLiveFilters(document, documentId, account.provider, filters) ||
+ lacksFilterDate(document, account.provider, filters)
+ )
+ )
+ )
+ const matching = firstOfEachDocument(
+ permitted.filter(({ document, documentId }) =>
+ matchesLiveFilters(document, documentId, account.provider, filters)
+ )
+ )
+ const undatedExcluded = permitted.some(({ document }) =>
+ lacksFilterDate(document, account.provider, filters)
+ )
+ const undatedUnsorted =
+ dateSorted && matching.some(({ document }) => !sourceDate(document, account.provider))
+ const moreUnsorted = dateSorted && Boolean(page.nextCursor || page.hasMore)
+ /** More matches exist that no cursor can reach, so coverage is short. */
+ const moreUnreachable = Boolean(page.hasMore && !page.nextCursor)
+ /** A continuable page with nothing readable proves nothing about the pages after it. */
+ const emptyContinuable = Boolean(page.nextCursor && !matching.length)
+ const degraded =
+ unverified ||
+ session.servicePartial ||
+ page.partial ||
+ undatedExcluded ||
+ undatedUnsorted ||
+ moreUnsorted ||
+ moreUnreachable ||
+ emptyContinuable
+ return {
+ status: {
+ ...status,
+ status: degraded ? 'partial' : 'ok',
+ message: joinMessages([
+ page.message,
+ session.servicePartial
+ ? 'Service account verification covered a bounded subset of the configured users. Narrow the source user list for complete coverage; external Drive users can only search files also visible to the source administrator.'
+ : undefined,
+ unverified
+ ? rateLimited
+ ? 'The provider rate-limited verification, so some results were omitted. Try again later.'
+ : 'Some results could not be verified against the source settings and were omitted.'
+ : undefined,
+ undatedExcluded
+ ? 'Some results lacked date metadata and were excluded; date coverage is incomplete.'
+ : undefined,
+ dateSorted
+ ? 'Date order covers retrieved results; follow continuation before claiming an overall earliest or latest match.'
+ : undefined,
+ moreUnreachable
+ ? 'More matches exist than this search could return. Narrow the query or target one source.'
+ : undefined,
+ emptyContinuable
+ ? 'No readable matches on this page. Continue with nextCursor for more.'
+ : undefined,
+ ]),
+ nextCursor: page.nextCursor,
+ },
+ results: matching.map((candidate, index) =>
+ resultFor(
+ candidate,
+ account,
+ index + 1,
+ native?.query || input.query,
+ input.resultSecretRegistry
+ )
+ ),
+ }
+ } catch (error) {
+ input.signal?.throwIfAborted()
+ const failure =
+ error instanceof NativeSearchError
+ ? error
+ : signal.aborted
+ ? new NativeSearchError(
+ 'timeout',
+ 'Provider search timed out. Narrow the query or try again.'
+ )
+ : new NativeSearchError(
+ 'unavailable',
+ 'This account could not be searched. Check the connection and try again.'
+ )
+ return {
+ status: {
+ ...status,
+ status: failure.status,
+ message: failure.message,
+ retryAfterSeconds: failure.retryAfterSeconds,
+ },
+ results: [],
+ }
+ } finally {
+ settled.abort()
+ }
+ }
+ let searched: Awaited>[]
+ try {
+ searched = await mapWithConcurrency(selected, ACCOUNT_CONCURRENCY, searchAccount)
+ } finally {
+ pool.destroy()
}
+ const seen = new Set()
+ const ranked = searched
+ .flatMap(({ results }, account) => results.map((result) => ({ account, result })))
+ .sort(({ result: a }, { result: b }) => {
+ if (!dateSorted) return b.similarity - a.similarity
+ const left = Date.parse(a.sourceDate ?? '')
+ const right = Date.parse(b.sourceDate ?? '')
+ if (!Number.isFinite(left)) return Number.isFinite(right) ? 1 : b.similarity - a.similarity
+ if (!Number.isFinite(right)) return -1
+ return (direction === 'asc' ? left - right : right - left) || b.similarity - a.similarity
+ })
+ .filter(({ result }) => {
+ const key = result.sourceUrl || result.documentId
+ if (seen.has(key)) return false
+ seen.add(key)
+ return true
+ })
+ /**
+ * An account's cursor continues after its own page, so it would skip that account's results
+ * cut from this merge. Those accounts drop the cursor and point to a targeted search instead.
+ */
+ const truncated = new Set(ranked.slice(input.topK).map(({ account }) => account))
+ const accounts: LiveSearchAccountStatus[] = searched.map(({ status }, index) => {
+ if (!truncated.has(index)) return status
+ const { nextCursor: _, ...rest } = status
+ return {
+ ...rest,
+ message: joinMessages([
+ status.message,
+ 'More matches ranked below the returned results. Search this account alone to see them.',
+ ]),
+ }
+ })
for (const query of queries ?? []) {
if (
!selected.some(
@@ -433,34 +517,12 @@ export const searchLiveKnowledge = defineAuthorizedKnowledgeUseCase({
message: 'No connection with this provider is configured and approved in this scope.',
})
}
- const seen = new Set()
- const ranked = results
- .sort((a, b) => {
- if (!input.filters?.sortBy || input.filters.sortBy === 'relevance')
- return b.similarity - a.similarity
- const left = Date.parse(a.sourceDate ?? '')
- const right = Date.parse(b.sourceDate ?? '')
- if (!Number.isFinite(left)) return Number.isFinite(right) ? 1 : b.similarity - a.similarity
- if (!Number.isFinite(right)) return -1
- return (
- (input.filters.sortBy === 'oldest' ? left - right : right - left) ||
- b.similarity - a.similarity
- )
- })
- .filter((item) => {
- const key = item.sourceUrl || item.documentId
- if (seen.has(key)) return false
- seen.add(key)
- return true
- })
return {
query: input.query,
- results: ranked.slice(0, input.topK),
+ results: ranked.slice(0, input.topK).map(({ result }) => result),
retrieval: {
status:
- accounts.some((account) => account.status !== 'ok') ||
- eligible.length > selected.length ||
- ranked.length > input.topK
+ accounts.some((account) => account.status !== 'ok') || eligible.length > selected.length
? 'partial'
: 'complete',
timedOutLegs: [],
@@ -500,95 +562,40 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({
(input.filters?.documentIds && !input.filters.documentIds.includes(input.documentId))
)
throw new OrchestrationError('not_found', 'Document is outside the selected search filters')
- const resolved = await resolveLiveAccount(input, userId, reference.account)
+ const [resolved, policies] = await Promise.all([
+ resolveLiveAccount(input, userId, reference.account),
+ loadLiveSearchPolicies(input),
+ ])
if (resolved.account.provider !== reference.provider)
throw new OrchestrationError('not_found', 'Document account changed')
const signal = input.signal
? AbortSignal.any([input.signal, AbortSignal.timeout(15_000)])
: AbortSignal.timeout(15_000)
- const client =
- resolved.account.type === 'managed_mcp'
- ? null
- : createNativeClient({
- origin:
- 'origin' in resolved
- ? resolved.origin
- : LIVE_SEARCH_PROVIDER_CATALOG[reference.provider].origin,
- accessToken: resolved.accessToken,
- signal,
- })
- let policy = livePolicyFor(await loadLiveSearchPolicies(input), reference.provider)
- const admin =
- 'adminSource' in resolved && client
- ? await createAdminGitLabSession({
- owner: input,
- userId,
- source: resolved.adminSource,
- token: resolved.accessToken,
- client,
- signal,
- })
- : undefined
- const mcp = client
- ? undefined
- : await createCodaMcpClient(input, userId, reference.account, signal)
- const service = await createLiveServiceSession({
- owner: input,
- userId,
- provider: reference.provider,
- policy,
- member: client,
- mcp,
- signal,
- })
- if (service) policy = service.policy
- const verify =
- service?.verify ??
- createPolicyVerifier(
- reference.provider,
- policy,
- client,
- 'origin' in resolved
- ? resolved.origin
- : LIVE_SEARCH_PROVIDER_CATALOG[reference.provider].origin,
- mcp
- )
- if (!(await verify(reference)) || (admin && !(await admin.verify(reference))))
- throw new OrchestrationError(
- 'not_found',
- 'Document is outside your organization’s search scope'
- )
- const document = admin
- ? await admin.read(reference)
- : client
- ? await readNativeProvider(reference.provider, client, reference, policy, input.filters)
- : await readCodaMcp(mcp!, reference.id)
- const currentPolicy = livePolicyFor(await loadLiveSearchPolicies(input), reference.provider)
- const currentService = await createLiveServiceSession({
- owner: input,
- userId,
- provider: reference.provider,
- policy: currentPolicy,
- member: client,
- mcp,
- signal,
- })
- const verifyCurrent =
- currentService?.verify ??
- createPolicyVerifier(
- reference.provider,
- currentPolicy,
- client,
- 'origin' in resolved
- ? resolved.origin
- : LIVE_SEARCH_PROVIDER_CATALOG[reference.provider].origin,
- mcp
- )
- if (!(await verifyCurrent(document)))
- throw new OrchestrationError(
- 'not_found',
- 'Document is outside your organization’s search scope'
- )
+ const pool = createPinnedConnectionPool()
+ let document: NativeDocument
+ try {
+ const session = await openLiveAccountSession({
+ owner: input,
+ userId,
+ resolved,
+ policies,
+ signal,
+ pool,
+ })
+ if (!(await session.verify(reference)))
+ throw new OrchestrationError(
+ 'not_found',
+ 'Document is outside your organization’s search scope'
+ )
+ document = await measureSearchStage('live.read', () => session.read(reference, input.filters))
+ if (!(await session.verifyCurrent(document)))
+ throw new OrchestrationError(
+ 'not_found',
+ 'Document is outside your organization’s search scope'
+ )
+ } finally {
+ pool.destroy()
+ }
if (!matchesLiveFilters(document, input.documentId, reference.provider, input.filters))
throw new OrchestrationError('not_found', 'Document is outside the selected search filters')
const content = safeContent(document.content, input.resultSecretRegistry)
@@ -603,7 +610,7 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({
'validation',
'This document changed. Read again from the beginning'
)
- const end = Math.min(content.length, start + 8000)
+ const end = Math.min(content.length, start + READ_WINDOW_CHARACTERS)
return {
documentId: input.documentId,
knowledgeBaseId: '',
diff --git a/apps/sim/lib/sim-search/live/atlassian.ts b/apps/sim/lib/sim-search/live/atlassian.ts
index d3a108df0fc..c5a610c6b45 100644
--- a/apps/sim/lib/sim-search/live/atlassian.ts
+++ b/apps/sim/lib/sim-search/live/atlassian.ts
@@ -1,4 +1,4 @@
-import { nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates'
+import { dateSortDirection, nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates'
import {
array,
NativeSearchError,
@@ -7,6 +7,8 @@ import {
string,
textContent,
} from '@/lib/sim-search/live/http'
+import { interleaveByRank } from '@/lib/sim-search/live/pages'
+import { providerText } from '@/lib/sim-search/live/text'
import type {
NativeClient,
NativeDocument,
@@ -73,15 +75,16 @@ function page(row: Record, cloudId: string, site: string): Nati
title: string(content.title) || string(row.title),
url: `${site}/wiki${string(links.webui) || `/pages/${segment(string(content.id))}`}`,
content:
- string(object(object(content.body).view).value).replace(/<[^>]*>/g, ' ') ||
- string(row.excerpt).replace(/<[^>]*>/g, ' ') ||
+ providerText(string(object(object(content.body).view).value), 'html') ||
+ providerText(string(row.excerpt).replace(/@@@(?:end)?hl@@@/g, ''), 'html') ||
string(content.title),
modifiedAt: string(version.when) || string(row.lastModified),
author: string(object(version.by).displayName),
}
}
+/** The sites a grant can reach, requested once per client and shared with its verifier. */
async function sites(client: NativeClient) {
- return array(await client.json('/oauth/token/accessible-resources'))
+ return array(await client.json('/oauth/token/accessible-resources', { memo: true }))
}
export async function searchAtlassian(
@@ -102,56 +105,52 @@ export async function searchAtlassian(
'reconnect',
'No accessible Atlassian site matches this account or site ID.'
)
- const documents: NativeDocument[] = []
- let partial = !input.native?.project && allSites.length > selected.length
- let nextCursor: string | undefined
- for (const site of selected) {
- const cloudId = string(site.id)
- const origin = string(site.url).replace(/\/$/, '')
- const policyScope =
- input.policy?.mode === 'selected'
- ? `(${input.policy.included.map((id) => `${provider === 'jira' ? 'project' : 'space'} = ${JSON.stringify(id)}`).join(' OR ')})`
- : undefined
- const dates = nativeDateBounds(input)
- const field = provider === 'jira' ? 'updated' : 'lastmodified'
- const scope =
- [
- policyScope,
- dates.start
- ? `${field} >= "${new Date(Date.parse(dates.start) - 86400000).toISOString().slice(0, 10)}"`
- : '',
- dates.end
- ? `${field} <= "${new Date(Date.parse(dates.end) + 86400000).toISOString().slice(0, 10)}"`
- : '',
- ]
- .filter(Boolean)
- .join(' AND ') || undefined
- const order =
- input.filters?.sortBy && input.filters.sortBy !== 'relevance'
- ? `${field} ${input.filters.sortBy === 'oldest' ? 'ASC' : 'DESC'}`
- : undefined
- const text = nativeText(input)
- if (provider === 'jira') {
- const data = object(
- await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, {
- body: {
- jql: scopeAtlassianQuery(
- input.native?.query || (text ? `text ~ ${escapeSearchPhrase(text)}` : ''),
- scope,
- order
- ),
- maxResults: input.limit,
- fields: ['summary', 'description', 'updated', 'creator', 'status'],
- ...(input.native?.cursor && selected.length === 1
- ? { nextPageToken: input.native.cursor }
- : {}),
- },
- })
- )
- documents.push(...array(data.issues).map((row) => issue(row, cloudId, origin)))
- partial ||= Boolean(data.nextPageToken)
- if (selected.length === 1) nextCursor = string(data.nextPageToken) || undefined
- } else {
+ const single = selected.length === 1
+ const policyScope =
+ input.policy?.mode === 'selected'
+ ? `(${input.policy.included.map((id) => `${provider === 'jira' ? 'project' : 'space'} = ${JSON.stringify(id)}`).join(' OR ')})`
+ : undefined
+ const dates = nativeDateBounds(input)
+ const field = provider === 'jira' ? 'updated' : 'lastmodified'
+ const scope =
+ [
+ policyScope,
+ dates.start
+ ? `${field} >= "${new Date(Date.parse(dates.start) - 86400000).toISOString().slice(0, 10)}"`
+ : '',
+ dates.end
+ ? `${field} <= "${new Date(Date.parse(dates.end) + 86400000).toISOString().slice(0, 10)}"`
+ : '',
+ ]
+ .filter(Boolean)
+ .join(' AND ') || undefined
+ const direction = dateSortDirection(input.filters)
+ const order = direction ? `${field} ${direction.toUpperCase()}` : undefined
+ const text = nativeText(input)
+ const pages = await Promise.all(
+ selected.map(async (site) => {
+ const cloudId = string(site.id)
+ const origin = string(site.url).replace(/\/$/, '')
+ if (provider === 'jira') {
+ const data = object(
+ await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, {
+ body: {
+ jql: scopeAtlassianQuery(
+ input.native?.query || (text ? `text ~ ${escapeSearchPhrase(text)}` : ''),
+ scope,
+ order
+ ),
+ maxResults: input.limit,
+ fields: ['summary', 'description', 'updated', 'creator', 'status'],
+ ...(input.native?.cursor && single ? { nextPageToken: input.native.cursor } : {}),
+ },
+ })
+ )
+ return {
+ documents: array(data.issues).map((row) => issue(row, cloudId, origin)),
+ next: string(data.nextPageToken) || undefined,
+ }
+ }
const data = object(
await client.json(`/ex/confluence/${segment(cloudId)}/wiki/rest/api/search`, {
query: {
@@ -163,24 +162,25 @@ export async function searchAtlassian(
),
limit: String(input.limit),
expand: 'content.version',
- ...(input.native?.cursor && selected.length === 1
- ? { cursor: input.native.cursor }
- : {}),
+ ...(input.native?.cursor && single ? { cursor: input.native.cursor } : {}),
},
})
)
- documents.push(...array(data.results).map((row) => page(row, cloudId, origin)))
const next = string(object(data._links).next)
- partial ||= Boolean(next)
- if (next && selected.length === 1)
- nextCursor =
- new URL(next, 'https://api.atlassian.com').searchParams.get('cursor') ?? undefined
- }
- }
+ return {
+ documents: array(data.results).map((row) => page(row, cloudId, origin)),
+ next: next
+ ? (new URL(next, 'https://api.atlassian.com').searchParams.get('cursor') ?? undefined)
+ : undefined,
+ }
+ })
+ )
+ const documents = interleaveByRank(pages.map((result) => result.documents))
return {
documents,
- partial,
- nextCursor,
+ partial: !input.native?.project && allSites.length > selected.length,
+ hasMore: pages.some((result) => Boolean(result.next)),
+ nextCursor: single ? pages[0]?.next : undefined,
message:
'Searches up to four accessible Atlassian sites. For a specific site and pagination, set project to its cloud ID.',
}
diff --git a/apps/sim/lib/sim-search/live/coda-mcp.test.ts b/apps/sim/lib/sim-search/live/coda-mcp.test.ts
index 42bd317954a..742f675ce45 100644
--- a/apps/sim/lib/sim-search/live/coda-mcp.test.ts
+++ b/apps/sim/lib/sim-search/live/coda-mcp.test.ts
@@ -85,7 +85,7 @@ describe('Coda MCP content search', () => {
})
expect(page).toMatchObject({
nextCursor: 'next',
- partial: true,
+ partial: false,
documents: [{ id: 'coda://docs/doc/pages/page', kind: 'mcp', content: 'Release next week' }],
})
})
@@ -126,6 +126,19 @@ describe('Coda MCP content search', () => {
expect(page).toMatchObject({ partial: true, documents: [{ id: 'coda://docs/doc' }] })
expect(page.message).toContain('Date filters apply to returned timestamps in Sim')
})
+ it('reports more results without claiming degraded coverage', async () => {
+ const call = vi.fn().mockResolvedValue({
+ results: [{ uri: 'coda://docs/doc', title: 'Launch' }],
+ hasMore: true,
+ })
+ expect(await searchCodaMcp({ call }, input)).toMatchObject({ hasMore: true, partial: false })
+ expect(
+ await searchCodaMcp(
+ { call },
+ { ...input, filters: { startDate: '2026-09-01', endDate: '2026-09-30' } }
+ )
+ ).toMatchObject({ hasMore: true, partial: true })
+ })
it('rejects page-scoped document filters before calling the provider', async () => {
const call = vi.fn()
await expect(
diff --git a/apps/sim/lib/sim-search/live/coda-mcp.ts b/apps/sim/lib/sim-search/live/coda-mcp.ts
index bf33e26ed82..37520c4617c 100644
--- a/apps/sim/lib/sim-search/live/coda-mcp.ts
+++ b/apps/sim/lib/sim-search/live/coda-mcp.ts
@@ -190,11 +190,8 @@ export async function searchCodaMcp(
return {
documents,
nextCursor,
- partial:
- documents.length < rows.length ||
- Boolean(nextCursor) ||
- result.hasMore === true ||
- hasDateBounds(input.filters),
+ hasMore: result.hasMore === true,
+ partial: documents.length < rows.length || hasDateBounds(input.filters),
message:
(query
? 'Coda content search includes pages and table rows. Narrow the query or target a document for more results.'
diff --git a/apps/sim/lib/sim-search/live/dates.test.ts b/apps/sim/lib/sim-search/live/dates.test.ts
index 6a5acd34dbe..5599343dc2e 100644
--- a/apps/sim/lib/sim-search/live/dates.test.ts
+++ b/apps/sim/lib/sim-search/live/dates.test.ts
@@ -243,7 +243,7 @@ describe('generic live search dates', () => {
})
expect(api.json).toHaveBeenCalledTimes(1)
expect(api.json.mock.calls[0]?.[1].query).toMatchObject({
- q: expect.stringContaining('updated:>=2026-09-22T07:00:00.000Z'),
+ q: 'repo:team/repo is:pr updated:2026-09-22T07:00:00.000Z..2026-09-23T07:00:00.000Z',
sort: 'updated',
order: 'asc',
})
diff --git a/apps/sim/lib/sim-search/live/dates.ts b/apps/sim/lib/sim-search/live/dates.ts
index 992c07e9883..4e2cae914c1 100644
--- a/apps/sim/lib/sim-search/live/dates.ts
+++ b/apps/sim/lib/sim-search/live/dates.ts
@@ -24,6 +24,12 @@ export function hasDateBounds(filters?: WorkspaceSearchFilters): boolean {
)
}
+/** The provider-date order results must follow, or undefined when ranked by relevance. */
+export function dateSortDirection(filters?: WorkspaceSearchFilters): 'asc' | 'desc' | undefined {
+ if (!filters?.sortBy || filters.sortBy === 'relevance') return undefined
+ return filters.sortBy === 'oldest' ? 'asc' : 'desc'
+}
+
/** Native bounds may be widened for provider precision; returned metadata is checked exactly. */
export function nativeDateBounds(input: NativeSearchInput): { start?: string; end?: string } {
const filters = input.filters
diff --git a/apps/sim/lib/sim-search/live/github-service.ts b/apps/sim/lib/sim-search/live/github-service.ts
index 9b88313c352..7d2f2645e7d 100644
--- a/apps/sim/lib/sim-search/live/github-service.ts
+++ b/apps/sim/lib/sim-search/live/github-service.ts
@@ -1,4 +1,5 @@
import { decryptSecret } from '@/lib/core/security/encryption'
+import type { PinnedConnectionPool } from '@/lib/core/security/input-validation.server'
import {
assertGitHubInstallationRepositoryActive,
parseGitHubInstallationBinding,
@@ -47,7 +48,8 @@ function codeAllowed(source: LiveGitHubSource, document: Pick()
for (const source of sources) {
@@ -107,6 +109,7 @@ export function createGitHubServiceVerifier(
origin: 'https://api.github.com',
accessToken,
signal,
+ pool,
})
const [appRepository, memberRepository] = await Promise.all([
app.json(path).then(object),
diff --git a/apps/sim/lib/sim-search/live/github.ts b/apps/sim/lib/sim-search/live/github.ts
index d57e04c381c..e1968ec42fd 100644
--- a/apps/sim/lib/sim-search/live/github.ts
+++ b/apps/sim/lib/sim-search/live/github.ts
@@ -1,6 +1,11 @@
-import { hasDateBounds, nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates'
+import {
+ dateSortDirection,
+ hasDateBounds,
+ nativeDateBounds,
+ nativeText,
+} from '@/lib/sim-search/live/dates'
import { array, NativeSearchError, object, segment, string } from '@/lib/sim-search/live/http'
-import { collectNativePages } from '@/lib/sim-search/live/pages'
+import { collectNativePages, joinMessages } from '@/lib/sim-search/live/pages'
import type {
NativeClient,
NativeDocument,
@@ -41,6 +46,84 @@ function githubDocument(row: Record, kind: string): NativeDocum
}
}
+/**
+ * Code search rejects a `q` over 1,000 UTF-8 bytes with qualifiers counted, although the
+ * documentation cites only 256 characters of text. Batches keep a small margin under it.
+ */
+const GITHUB_CODE_QUERY_BYTES = 980
+/**
+ * Issue and repository search limit only free text to 256 characters and accept roughly 4,000
+ * bytes of repository qualifiers; batches leave room for type and date qualifiers appended later.
+ */
+const GITHUB_ISSUE_QUERY_BYTES = 3800
+/** Batches per kind; each code batch spends one of GitHub's ten code searches per minute. */
+const GITHUB_MAX_REPOSITORY_BATCHES = 4
+
+type GitHubKind = 'issues' | 'code' | 'repositories'
+const CODE_EXCLUDED_BY_DATES = 'Code has no dates and is excluded from date-filtered searches.'
+
+/** Code has no file dates, so a date-bounded search covers issues and pull requests only. */
+function githubKinds(input: NativeSearchInput): GitHubKind[] {
+ const kind = input.native?.kind
+ if (kind === 'issues' || kind === 'code' || kind === 'repositories') return [kind]
+ if (kind)
+ throw new NativeSearchError(
+ 'unavailable',
+ 'GitHub search supports issues, code, or repositories.'
+ )
+ return hasDateBounds(input.filters) ? ['issues'] : ['issues', 'code']
+}
+
+/**
+ * Splits repositories into qualifier batches that keep each query within `maxBytes`. A
+ * repository whose qualifier cannot fit beside the query at all is left out.
+ */
+function repositoryBatches(query: string, names: readonly string[], maxBytes: number): string[][] {
+ const base = Buffer.byteLength(query)
+ const batches: string[][] = []
+ let batch: string[] = []
+ let bytes = base
+ for (const name of names) {
+ const term = Buffer.byteLength(` repo:${name}`)
+ if (base + term > maxBytes) continue
+ if (bytes + term > maxBytes) {
+ if (batches.push(batch) === GITHUB_MAX_REPOSITORY_BATCHES) return batches
+ batch = []
+ bytes = base
+ }
+ batch.push(name)
+ bytes += term
+ }
+ if (batch.length) batches.push(batch)
+ return batches
+}
+
+/** `key:value` or `key:"quoted value"`, excluding URLs such as `https://…`. */
+const GITHUB_QUALIFIER = /^-?[a-z][\w-]*:(?!\/\/)\S/i
+
+/** Search terms, quoted phrases, and `key:"quoted value"` qualifiers as GitHub tokenizes them. */
+const githubTokens = (query: string) => query.match(/-?[\w-]+:"[^"]*"|-?"[^"]*"|\S+/g) ?? []
+
+/**
+ * Groups free text so appended qualifiers apply to all of it. GitHub treats a qualifier inside
+ * parentheses as search text, so qualifiers stay outside the group. A query that already uses
+ * parentheses or boolean operators is structured by its author and is left as written.
+ */
+function groupGitHubText(query: string): string {
+ if (!query || /[()]/.test(query) || /(?:^|\s)(?:AND|OR|NOT)(?=\s|$)/.test(query)) return query
+ const tokens = githubTokens(query)
+ const qualifiers = tokens.filter((token) => GITHUB_QUALIFIER.test(token))
+ const text = tokens.filter((token) => !GITHUB_QUALIFIER.test(token)).join(' ')
+ return [text ? `(${text})` : '', ...qualifiers].filter(Boolean).join(' ')
+}
+
+/** GitHub rejects more than 256 characters of search text; qualifiers do not count toward it. */
+const GITHUB_TEXT_CHARACTERS = 256
+const githubTextLength = (query: string) =>
+ githubTokens(query)
+ .filter((token) => !GITHUB_QUALIFIER.test(token))
+ .join(' ').length
+
export async function searchGitHub(
client: NativeClient,
input: NativeSearchInput
@@ -64,50 +147,78 @@ export async function searchGitHub(
documents: [],
message: 'No repositories are accessible through this GitHub connection.',
}
- const batches: string[][] = []
- for (let offset = 0; offset < names.length; offset += 25)
- batches.push(names.slice(offset, offset + 25))
- const pages: Promise[] = []
- for (const names of batches) {
- pages.push(
- searchGitHub(client, {
- ...input,
- native: {
- provider: 'github',
- ...input.native,
- query: `${query} ${names.map((name) => `repo:${name}`).join(' ')}`,
- },
- })
+ const kinds = githubKinds(input)
+ const planned = kinds.map((kind) => ({
+ kind,
+ repositories: repositoryBatches(
+ query,
+ names,
+ kind === 'code' ? GITHUB_CODE_QUERY_BYTES : GITHUB_ISSUE_QUERY_BYTES
+ ),
+ }))
+ const batches = planned.filter(({ repositories }) => repositories.length)
+ const skipped = planned.filter(({ repositories }) => !repositories.length)
+ if (!batches.length)
+ throw new NativeSearchError(
+ 'unavailable',
+ 'This query is too long to scope to your GitHub repositories. Shorten it or add a repo: qualifier.'
)
- }
+ const searched = Math.min(
+ ...batches.map(({ repositories }) =>
+ repositories.reduce((count, batch) => count + batch.length, 0)
+ )
+ )
+ /** Batches merge within their kind first, so a kind with more batches cannot crowd out another. */
const result = await collectNativePages(
- pages,
+ batches.map(({ kind, repositories }) =>
+ collectNativePages(
+ repositories.map((batch) =>
+ searchGitHub(client, {
+ ...input,
+ native: {
+ provider: 'github',
+ ...input.native,
+ kind,
+ query: [query, ...batch.map((name) => `repo:${name}`)].filter(Boolean).join(' '),
+ },
+ })
+ ),
+ ''
+ )
+ ),
'Searched repositories you own, collaborate on, or access through organization membership. Use a repo: qualifier to narrow results.'
)
+ const capped = repositories.length === 100 || searched < names.length
return {
...result,
- partial: result.partial || repositories.length === 100,
- message:
- repositories.length === 100
- ? `${result.message} Only the 100 most recently pushed repositories were searched; target a repository for broader coverage.`
- : result.message,
+ partial: result.partial || capped || skipped.length > 0,
+ message: joinMessages([
+ result.message,
+ ...skipped.map(
+ ({ kind }) =>
+ `GitHub ${kind} search was skipped because the query is too long to scope to repositories.`
+ ),
+ !input.native?.kind && !kinds.includes('code') ? CODE_EXCLUDED_BY_DATES : undefined,
+ capped
+ ? `Only the ${searched} most recently pushed repositories were searched; target a repository for broader coverage.`
+ : undefined,
+ ]),
}
}
- if (!input.native?.kind)
+ if (!input.native?.kind) {
+ const kinds = githubKinds(input)
return collectNativePages(
- ['issues', 'code'].map((kind) =>
+ kinds.map((kind) =>
searchGitHub(client, {
...input,
- native: {
- provider: 'github',
- ...input.native,
- query,
- kind: kind === 'code' ? 'code' : 'issues',
- },
+ native: { provider: 'github', ...input.native, query, kind },
})
),
- 'Searched GitHub issues, pull requests, and code.'
+ kinds.includes('code')
+ ? 'Searched GitHub issues, pull requests, and code.'
+ : `Searched GitHub issues and pull requests. ${CODE_EXCLUDED_BY_DATES}`
)
+ }
if (
input.native.kind === 'issues' &&
!/(?:^|\s)(?:is|type):(?:issue|pr|pull-request)(?:\s|$)/i.test(query)
@@ -121,12 +232,7 @@ export async function searchGitHub(
),
'Searched issues and pull requests separately.'
)
- const kind = input.native?.kind ?? 'issues'
- if (!['issues', 'code', 'repositories'].includes(kind))
- throw new NativeSearchError(
- 'unavailable',
- 'GitHub search supports issues, code, or repositories.'
- )
+ const [kind] = githubKinds(input)
if (kind === 'code' && hasDateBounds(input.filters))
throw new NativeSearchError(
'unavailable',
@@ -134,16 +240,22 @@ export async function searchGitHub(
)
const dates = nativeDateBounds(input)
const text = nativeText(input)
+ /** GitHub ORs repeated qualifiers, so both bounds must share one `updated:` range. */
+ const updated =
+ dates.start && dates.end
+ ? `updated:${dates.start}..${dates.end}`
+ : dates.start
+ ? `updated:>=${dates.start}`
+ : dates.end
+ ? `updated:<=${dates.end}`
+ : ''
const datedQuery =
- kind === 'issues'
- ? [
- text ? `(${text})` : '',
- dates.start ? `updated:>=${dates.start}` : '',
- dates.end ? `updated:<=${dates.end}` : '',
- ]
- .filter(Boolean)
- .join(' ')
- : text
+ kind === 'issues' ? [groupGitHubText(text), updated].filter(Boolean).join(' ') : text
+ if (githubTextLength(text) > GITHUB_TEXT_CHARACTERS)
+ throw new NativeSearchError(
+ 'unavailable',
+ 'GitHub search text is limited to 256 characters. Shorten the query.'
+ )
const page = input.native?.cursor ?? '1'
if (!/^\d{1,3}$/.test(page) || Number(page) < 1)
throw new NativeSearchError('unavailable', 'Invalid GitHub page.')
@@ -154,8 +266,8 @@ export async function searchGitHub(
q: hasDateBounds(input.filters) ? datedQuery : text,
per_page: String(input.limit),
page,
- ...(kind === 'issues' && input.filters?.sortBy && input.filters.sortBy !== 'relevance'
- ? { sort: 'updated', order: input.filters.sortBy === 'oldest' ? 'asc' : 'desc' }
+ ...(kind === 'issues' && dateSortDirection(input.filters)
+ ? { sort: 'updated', order: dateSortDirection(input.filters) }
: {}),
},
})
@@ -175,7 +287,8 @@ export async function searchGitHub(
return {
documents: array(data.items).map((row) => githubDocument(row, kind)),
nextCursor,
- partial: data.incomplete_results === true || total > 1000,
+ hasMore: total > 1000,
+ partial: data.incomplete_results === true,
message:
kind === 'code'
? 'GitHub REST code search covers the default branch and files below 384 KB; code queries have a separate rate limit. Read results for file contents.'
diff --git a/apps/sim/lib/sim-search/live/gitlab.ts b/apps/sim/lib/sim-search/live/gitlab.ts
index ce430e41d5a..67523fda03f 100644
--- a/apps/sim/lib/sim-search/live/gitlab.ts
+++ b/apps/sim/lib/sim-search/live/gitlab.ts
@@ -1,4 +1,9 @@
-import { hasDateBounds, nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates'
+import {
+ dateSortDirection,
+ hasDateBounds,
+ nativeDateBounds,
+ nativeText,
+} from '@/lib/sim-search/live/dates'
import { array, NativeSearchError, object, segment, string } from '@/lib/sim-search/live/http'
import { collectNativePages } from '@/lib/sim-search/live/pages'
import type {
@@ -13,7 +18,10 @@ function document(row: Record, kind: string): NativeDocument {
accessMetadata: {
confidential: row.confidential,
authorId: object(row.author).id,
- assigneeIds: array(row.assignees).map((person) => person.id),
+ /** Older merge request payloads carry only the single, deprecated `assignee`. */
+ assigneeIds: Array.isArray(row.assignees)
+ ? array(row.assignees).map((person) => person.id)
+ : [object(row.assignee).id].filter((id) => id !== undefined),
state: row.state,
labels: row.labels,
milestone: row.milestone,
@@ -109,8 +117,7 @@ export async function searchGitLab(
const listing = Boolean(
input.native?.project &&
['issues', 'merge_requests'].includes(kind) &&
- (hasDateBounds(input.filters) ||
- (input.filters?.sortBy && input.filters.sortBy !== 'relevance'))
+ (hasDateBounds(input.filters) || Boolean(dateSortDirection(input.filters)))
)
let response: unknown
try {
@@ -127,7 +134,7 @@ export async function searchGitLab(
...(listing && dates.start ? { updated_after: dates.start } : {}),
...(listing && dates.end ? { updated_before: dates.end } : {}),
...(listing
- ? { order_by: 'updated_at', sort: input.filters?.sortBy === 'oldest' ? 'asc' : 'desc' }
+ ? { order_by: 'updated_at', sort: dateSortDirection(input.filters) ?? 'desc' }
: {}),
per_page: String(input.limit),
page,
diff --git a/apps/sim/lib/sim-search/live/google-service.test.ts b/apps/sim/lib/sim-search/live/google-service.test.ts
index 49f2c14a9a9..46ed322e59c 100644
--- a/apps/sim/lib/sim-search/live/google-service.test.ts
+++ b/apps/sim/lib/sim-search/live/google-service.test.ts
@@ -3,7 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { ConnectorAccessToken } from '@/lib/knowledge/connectors/access-token'
import { searchCalendar } from '@/lib/sim-search/live/google'
import { createGoogleServiceVerifier } from '@/lib/sim-search/live/google-service'
-import { NativeSearchError } from '@/lib/sim-search/live/http'
+import { NativeSearchError, withJsonMemo } from '@/lib/sim-search/live/http'
import { liveSourcePolicy } from '@/lib/sim-search/live/source-policy'
import type { NativeClient, NativeSearchInput } from '@/lib/sim-search/live/types'
@@ -41,7 +41,7 @@ beforeEach(() => {
vi.setSystemTime(new Date('2026-09-22T12:00:00Z'))
member.json.mockResolvedValue({ emailAddress: 'reader@example.com' })
mocks.directory.mockImplementation(async (_token, email: string) => person(email))
- mocks.createClient.mockReturnValue(delegated)
+ mocks.createClient.mockImplementation(() => withJsonMemo(delegated))
mint.mockResolvedValue('delegated-token')
})
afterEach(() => vi.useRealTimers())
@@ -428,4 +428,12 @@ describe('Google service search scope and continuation', () => {
native,
})
})
+ it('caps Drive pages to what source verification can check within its request budget', async () => {
+ member.json.mockResolvedValue({ user: { emailAddress: 'reader@example.com' } })
+ const search = { query: 'launch', limit: 50, scopes: [] }
+ const scoped = await create('google_drive', { folderId: 'folder' })
+ expect(scoped.scopeSearch(search)).toMatchObject({ limit: 20 })
+ expect(scoped.scopeSearch({ ...search, limit: 10 })).toMatchObject({ limit: 10 })
+ expect((await create('google_drive')).scopeSearch(search)).toMatchObject({ limit: 29 })
+ })
})
diff --git a/apps/sim/lib/sim-search/live/google-service.ts b/apps/sim/lib/sim-search/live/google-service.ts
index 6c3b09fe90d..67f398d2ace 100644
--- a/apps/sim/lib/sim-search/live/google-service.ts
+++ b/apps/sim/lib/sim-search/live/google-service.ts
@@ -1,4 +1,5 @@
import { normalizeEmail } from '@sim/utils/string'
+import type { PinnedConnectionPool } from '@/lib/core/security/input-validation.server'
import { zonedWallClockToUtc } from '@/lib/core/utils/timezone'
import type { ConnectorAccessToken } from '@/lib/knowledge/connectors/access-token'
import {
@@ -20,6 +21,8 @@ import {
type GoogleProvider = 'google_drive' | 'gmail' | 'google_calendar'
type Reference = Pick
const SOURCE_REQUEST_BUDGET = 30
+/** Distinct ancestor folders a restricted Drive page may read while verifying its files. */
+const DRIVE_FOLDER_RESERVE = 9
const DAY_MS = 86400000
interface GoogleServiceSession {
@@ -43,8 +46,9 @@ export async function createGoogleServiceVerifier(input: {
config: Record
policy: LiveSearchPolicy
signal: AbortSignal
+ pool?: PinnedConnectionPool
}): Promise {
- const { provider, token, member, config, policy, signal } = input
+ const { provider, token, member, config, policy, signal, pool } = input
if (!token.getDelegatedAccessToken)
throw new NativeSearchError(
'unavailable',
@@ -99,7 +103,12 @@ export async function createGoogleServiceVerifier(input: {
)
continue
const accessToken = await token.getDelegatedAccessToken(person.email, signal)
- const client = createNativeClient({ origin: 'https://www.googleapis.com', accessToken, signal })
+ const client = createNativeClient({
+ origin: 'https://www.googleapis.com',
+ accessToken,
+ signal,
+ pool,
+ })
clients.push({
email: person.email,
client,
@@ -110,7 +119,12 @@ export async function createGoogleServiceVerifier(input: {
if (provider === 'google_drive' && !selected.length && !clients.length) {
signal.throwIfAborted()
const accessToken = await token.getDelegatedAccessToken(administrator.email, signal)
- const client = createNativeClient({ origin: 'https://www.googleapis.com', accessToken, signal })
+ const client = createNativeClient({
+ origin: 'https://www.googleapis.com',
+ accessToken,
+ signal,
+ pool,
+ })
clients.push({
email: administrator.email,
client,
@@ -261,6 +275,15 @@ export async function createGoogleServiceVerifier(input: {
partial,
scopeSearch(search) {
if (!clients.length) return null
+ if (provider === 'google_drive') {
+ /** Each file costs one read; a restricted source also reads folders shared across files. */
+ const folderReserve =
+ policy.mode === 'selected' || policy.excluded.length ? DRIVE_FOLDER_RESERVE : 0
+ return {
+ ...search,
+ limit: Math.min(search.limit, SOURCE_REQUEST_BUDGET - 1 - folderReserve),
+ }
+ }
if (provider === 'gmail') {
const requestsPerMessage = string(config.query).trim() ? 2 : 1
return {
diff --git a/apps/sim/lib/sim-search/live/google.ts b/apps/sim/lib/sim-search/live/google.ts
index e2eb88bc5e1..6c97f02c56c 100644
--- a/apps/sim/lib/sim-search/live/google.ts
+++ b/apps/sim/lib/sim-search/live/google.ts
@@ -1,7 +1,15 @@
+import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
import { zonedWallClockToUtc } from '@/lib/core/utils/timezone'
-import { hasDateBounds, nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates'
+import {
+ dateSortDirection,
+ hasDateBounds,
+ nativeDateBounds,
+ nativeText,
+} from '@/lib/sim-search/live/dates'
import { array, NativeSearchError, object, segment, string } from '@/lib/sim-search/live/http'
+import { interleaveByRank } from '@/lib/sim-search/live/pages'
import { permitsResources } from '@/lib/sim-search/live/policy'
+import { providerText } from '@/lib/sim-search/live/text'
import type {
NativeClient,
NativeDocument,
@@ -46,8 +54,10 @@ export async function searchDrive(
...(dates.start ? [`modifiedTime >= '${dates.start}'`] : []),
...(dates.end ? [`modifiedTime <= '${dates.end}'`] : []),
].join(' and '),
- ...(input.filters?.sortBy && input.filters.sortBy !== 'relevance'
- ? { orderBy: `modifiedTime${input.filters.sortBy === 'oldest' ? '' : ' desc'}` }
+ ...(dateSortDirection(input.filters)
+ ? {
+ orderBy: `modifiedTime${dateSortDirection(input.filters) === 'asc' ? '' : ' desc'}`,
+ }
: {}),
fields: `nextPageToken,incompleteSearch,files(${DRIVE_FIELDS})`,
pageSize: String(input.limit),
@@ -117,6 +127,11 @@ export async function readDrive(client: NativeClient, id: string): Promise): NativeDocument {
const payload = object(row.payload)
const headers = array(payload.headers)
@@ -124,10 +139,13 @@ function gmailDocument(row: Record): NativeDocument {
string(headers.find((h) => string(h.name).toLowerCase() === name)?.value)
const timestamp = Number(row.internalDate)
return {
+ ...(Array.isArray(row.labelIds)
+ ? { accessMetadata: { id: row.id, labelIds: row.labelIds } }
+ : {}),
id: string(row.id),
title: header('subject') || '(No subject)',
url: `https://mail.google.com/mail/u/0/#all/${segment(string(row.id))}`,
- content: string(row.snippet),
+ content: providerText(string(row.snippet), 'escaped'),
author: header('from'),
...(Number.isFinite(timestamp) && timestamp > 0
? { modifiedAt: new Date(timestamp).toISOString() }
@@ -157,24 +175,22 @@ export async function searchGmail(
},
})
)
- const documents: NativeDocument[] = []
- // Bound fanout while retaining provider rank. One metadata call per matching message.
- const messages = array(data.messages)
- for (let offset = 0; offset < messages.length; offset += 5) {
- documents.push(
- ...(await Promise.all(
- messages.slice(offset, offset + 5).map(async (message) =>
- gmailDocument(
- object(
- await client.json(`/gmail/v1/users/me/messages/${segment(string(message.id))}`, {
- query: { format: 'metadata' },
- })
- )
- )
+ const documents = await mapWithConcurrency(
+ array(data.messages),
+ GMAIL_METADATA_CONCURRENCY,
+ async (message) =>
+ gmailDocument(
+ object(
+ await client.json(`/gmail/v1/users/me/messages/${segment(string(message.id))}`, {
+ query: {
+ format: 'metadata',
+ metadataHeaders: ['Subject', 'From'],
+ fields: GMAIL_METADATA_FIELDS,
+ },
+ })
)
- ))
- )
- }
+ )
+ )
return {
documents,
nextCursor: string(data.nextPageToken) || undefined,
@@ -188,13 +204,11 @@ function mailText(value: unknown): string {
const children = array(part.parts).map(mailText).filter(Boolean)
const encoded = string(object(part.body).data)
if (string(part.mimeType) === 'text/plain' && encoded)
- return Buffer.from(encoded, 'base64url').toString('utf8')
+ return providerText(Buffer.from(encoded, 'base64url').toString('utf8'))
if (children.length) return children.join('\n')
// HTML-only messages remain text, never rendered as markup.
if (string(part.mimeType) === 'text/html' && encoded)
- return Buffer.from(encoded, 'base64url')
- .toString('utf8')
- .replace(/<[^>]*>/g, ' ')
+ return providerText(Buffer.from(encoded, 'base64url').toString('utf8'), 'html')
return ''
}
@@ -226,7 +240,7 @@ function eventDocument(
url: string(row.htmlLink),
content: [
string(row.summary),
- string(row.description),
+ providerText(string(row.description), 'auto'),
string(row.location),
`Start: ${string(object(row.start).dateTime) || string(object(row.start).date)}`,
`End: ${string(object(row.end).dateTime) || string(object(row.end).date)}`,
@@ -242,6 +256,27 @@ function eventDocument(
}
}
+/**
+ * A shared meeting appears once in every attendee calendar the member can see. Its iCalendar
+ * UID and start instant identify the occurrence across calendars, which may report the start in
+ * their own time zones; a modified recurring instance stays distinct from its series.
+ */
+function calendarOccurrenceKey(row: Record): string {
+ const start = object(row.originalStartTime ?? row.start)
+ const instant = Date.parse(string(start.dateTime))
+ return JSON.stringify([
+ string(row.iCalUID) || string(row.id),
+ row.recurringEventId ? 'instance' : 'event',
+ Number.isFinite(instant) ? instant : string(start.date),
+ ])
+}
+
+/** Events without a resolvable start sort after every scheduled one. */
+function eventStartTime(document: NativeDocument): number {
+ const time = Date.parse(document.eventStartAt ?? '')
+ return Number.isFinite(time) ? time : Number.MAX_SAFE_INTEGER
+}
+
export async function searchCalendar(
client: NativeClient,
input: NativeSearchInput
@@ -262,76 +297,68 @@ export async function searchCalendar(
])
)
.slice(0, 20)
- const documents: NativeDocument[] = []
- let partial = Boolean(calendars.nextPageToken)
- let nextCursor: string | undefined
- let failedCalendars = 0
- for (let offset = 0; offset < rows.length; offset += 4) {
- const pages = await Promise.allSettled(
- rows.slice(offset, offset + 4).map(async (row) => {
- const calendarId = string(row.id)
- const data = object(
- await client.json(`/calendar/v3/calendars/${segment(calendarId)}/events`, {
- query: {
- ...(nativeText(input) ? { q: nativeText(input) } : {}),
- ...(input.filters?.startDate
- ? {
- timeMin: new Date(
- Math.floor(Date.parse(input.filters.startDate) / 1000) * 1000 - 1000
- ).toISOString(),
- }
- : {}),
- ...(input.filters?.endDate
- ? {
- timeMax: new Date(
- Math.ceil(Date.parse(input.filters.endDate) / 1000) * 1000
- ).toISOString(),
- }
- : {}),
- ...(input.filters?.modifiedAfter ? { updatedMin: input.filters.modifiedAfter } : {}),
- ...(hasDateBounds(input.filters) ||
- (input.filters?.sortBy && input.filters.sortBy !== 'relevance')
- ? { singleEvents: 'true', orderBy: 'startTime' }
- : {}),
- maxResults: String(input.limit),
- showDeleted: 'false',
- ...(input.native?.cursor && input.native.project
- ? { pageToken: input.native.cursor }
- : {}),
- },
- })
- )
- return { data, calendarId, timeZone: string(data.timeZone) || string(row.timeZone) }
- })
- )
- for (const result of pages) {
- if (result.status === 'rejected') {
- if (input.native?.project) throw result.reason
- partial = true
- failedCalendars++
- continue
- }
- const { data, calendarId, timeZone } = result.value
- documents.push(
- ...array(data.items)
- .filter((event) => event.status !== 'cancelled')
- .map((event) =>
- eventDocument(event, calendarId, input.policy?.includeAttendees, timeZone)
- )
+ .sort((left, right) => Number(right.primary === true) - Number(left.primary === true))
+ const chronological = hasDateBounds(input.filters) || Boolean(dateSortDirection(input.filters))
+ const pages = await mapWithConcurrency(rows, 4, async (row) => {
+ const calendarId = string(row.id)
+ try {
+ const data = object(
+ await client.json(`/calendar/v3/calendars/${segment(calendarId)}/events`, {
+ query: {
+ ...(nativeText(input) ? { q: nativeText(input) } : {}),
+ ...(input.filters?.startDate
+ ? {
+ timeMin: new Date(
+ Math.floor(Date.parse(input.filters.startDate) / 1000) * 1000 - 1000
+ ).toISOString(),
+ }
+ : {}),
+ ...(input.filters?.endDate
+ ? {
+ timeMax: new Date(
+ Math.ceil(Date.parse(input.filters.endDate) / 1000) * 1000
+ ).toISOString(),
+ }
+ : {}),
+ ...(input.filters?.modifiedAfter ? { updatedMin: input.filters.modifiedAfter } : {}),
+ ...(chronological ? { singleEvents: 'true', orderBy: 'startTime' } : {}),
+ maxResults: String(input.limit),
+ showDeleted: 'false',
+ ...(input.native?.cursor && input.native.project
+ ? { pageToken: input.native.cursor }
+ : {}),
+ },
+ })
)
- partial ||= Boolean(data.nextPageToken)
- if (input.native?.project) nextCursor = string(data.nextPageToken) || undefined
+ return { data, calendarId, timeZone: string(data.timeZone) || string(row.timeZone) }
+ } catch (error) {
+ if (input.native?.project) throw error
+ return null
}
- }
- if (rows.length > 0 && failedCalendars === rows.length)
+ })
+ const searched = pages.filter((page) => page !== null)
+ if (rows.length > 0 && searched.length === 0)
throw new NativeSearchError(
'unavailable',
'None of the calendars could be searched. Check calendar permissions.'
)
+ const perCalendar = searched.map(({ data, calendarId, timeZone }) =>
+ array(data.items)
+ .filter((event) => event.status !== 'cancelled')
+ .map((event) => ({
+ ...eventDocument(event, calendarId, input.policy?.includeAttendees, timeZone),
+ dedupeKey: calendarOccurrenceKey(event),
+ }))
+ )
+ const documents = chronological
+ ? perCalendar.flat().sort((left, right) => eventStartTime(left) - eventStartTime(right))
+ : interleaveByRank(perCalendar)
+ const single = input.native?.project ? searched[0] : undefined
return {
documents,
- partial,
- nextCursor,
+ partial: Boolean(calendars.nextPageToken) || searched.length < rows.length,
+ hasMore: searched.some(({ data }) => Boolean(data.nextPageToken)),
+ nextCursor: single ? string(single.data.nextPageToken) || undefined : undefined,
message:
'Calendar supports text and date-only searches across up to 20 calendars. startDate/endDate filter scheduled starts; recurring events expand within the window. sourceDate is scheduled start and sourceModifiedAt is last edit. Use project = calendar ID to paginate; reverse ordering is limited to the fetched page.',
}
diff --git a/apps/sim/lib/sim-search/live/http.test.ts b/apps/sim/lib/sim-search/live/http.test.ts
index 61ba4c935bf..0f07ecccb07 100644
--- a/apps/sim/lib/sim-search/live/http.test.ts
+++ b/apps/sim/lib/sim-search/live/http.test.ts
@@ -6,7 +6,7 @@ vi.mock('@/lib/core/security/input-validation.server', () => ({
secureFetchWithValidation: mocks.fetch,
}))
-import { createNativeClient } from '@/lib/sim-search/live/http'
+import { createNativeClient, NativeSearchError, withJsonMemo } from '@/lib/sim-search/live/http'
describe('native search network boundary', () => {
beforeEach(() => {
@@ -95,6 +95,64 @@ describe('native search network boundary', () => {
message: expect.stringContaining('read permissions for Contents, Issues, and Pull requests'),
})
})
+ it('reports a Google quota 403 as a rate limit rather than a reconnect', async () => {
+ mocks.fetch.mockResolvedValue(
+ new Response(
+ JSON.stringify({
+ error: { code: 403, errors: [{ reason: 'userRateLimitExceeded' }], message: 'private' },
+ }),
+ { status: 403 }
+ )
+ )
+ const client = createNativeClient({
+ origin: 'https://www.googleapis.com',
+ accessToken: 'private',
+ signal: new AbortController().signal,
+ })
+ await expect(client.json('/drive/v3/files')).rejects.toMatchObject({
+ status: 'rate_limited',
+ message: 'Provider rate limit reached. Try again later.',
+ })
+ })
+ it('still asks for a reconnect when Google denies access', async () => {
+ mocks.fetch.mockResolvedValue(
+ new Response(JSON.stringify({ error: { errors: [{ reason: 'insufficientPermissions' }] } }), {
+ status: 403,
+ })
+ )
+ const client = createNativeClient({
+ origin: 'https://www.googleapis.com',
+ accessToken: 'private',
+ signal: new AbortController().signal,
+ })
+ await expect(client.json('/drive/v3/files')).rejects.toMatchObject({ status: 'reconnect' })
+ })
+ it('requests compressed bodies and passes the caller connection pool through', async () => {
+ const pool = { agent: vi.fn(), destroy: vi.fn() }
+ const client = createNativeClient({
+ origin: 'https://www.googleapis.com',
+ accessToken: 'private',
+ signal: new AbortController().signal,
+ pool,
+ })
+ await client.json('/drive/v3/files')
+ expect(mocks.fetch).toHaveBeenCalledWith(
+ expect.any(String),
+ expect.objectContaining({ acceptCompressed: true, connectionPool: pool })
+ )
+ })
+ it('reuses a memoized response, including a failure, only when asked to', async () => {
+ const json = vi
+ .fn()
+ .mockRejectedValueOnce(new NativeSearchError('rate_limited', 'Later'))
+ .mockResolvedValue({ ok: true })
+ const client = withJsonMemo({ json, text: vi.fn() })
+ await expect(client.json('/labels', { memo: true })).rejects.toThrow('Later')
+ await expect(client.json('/labels', { memo: true })).rejects.toThrow('Later')
+ expect(json).toHaveBeenCalledTimes(1)
+ await expect(client.json('/labels')).resolves.toEqual({ ok: true })
+ expect(json).toHaveBeenCalledTimes(2)
+ })
it('does not issue any request after cancellation', async () => {
const controller = new AbortController()
controller.abort()
@@ -106,4 +164,42 @@ describe('native search network boundary', () => {
await expect(client.json('/api/assistant.search.context')).rejects.toThrow()
expect(mocks.fetch).not.toHaveBeenCalled()
})
+ it('memoizes a repeated GET by path and query, never a request with a body', async () => {
+ mocks.fetch.mockImplementation(async () => new Response('{"ok":true}', { status: 200 }))
+ const client = createNativeClient({
+ origin: 'https://slack.com',
+ accessToken: 'private',
+ signal: new AbortController().signal,
+ })
+ await client.json('/api/team.info', { query: { team: 'a' }, memo: true })
+ await client.json('/api/team.info', { query: { team: 'a' }, memo: true })
+ expect(mocks.fetch).toHaveBeenCalledTimes(1)
+ await client.json('/api/team.info', { query: { team: 'b' }, memo: true })
+ expect(mocks.fetch).toHaveBeenCalledTimes(2)
+ await client.json('/api/search', { body: { query: 'launch' }, memo: true })
+ await client.json('/api/search', { body: { query: 'launch' }, memo: true })
+ expect(mocks.fetch).toHaveBeenCalledTimes(4)
+ })
+ it('reports a Google RESOURCE_EXHAUSTED 403 as a rate limit', async () => {
+ mocks.fetch.mockResolvedValue(
+ new Response(JSON.stringify({ error: { status: 'RESOURCE_EXHAUSTED' } }), { status: 403 })
+ )
+ const client = createNativeClient({
+ origin: 'https://www.googleapis.com',
+ accessToken: 'private',
+ signal: new AbortController().signal,
+ })
+ await expect(client.json('/drive/v3/files')).rejects.toMatchObject({ status: 'rate_limited' })
+ })
+ it('reads a quota-shaped 403 body only from Google origins', async () => {
+ mocks.fetch.mockResolvedValue(
+ new Response(JSON.stringify({ error: { status: 'RESOURCE_EXHAUSTED' } }), { status: 403 })
+ )
+ const client = createNativeClient({
+ origin: 'https://slack.com',
+ accessToken: 'private',
+ signal: new AbortController().signal,
+ })
+ await expect(client.json('/api/search.messages')).rejects.toMatchObject({ status: 'reconnect' })
+ })
})
diff --git a/apps/sim/lib/sim-search/live/http.ts b/apps/sim/lib/sim-search/live/http.ts
index c4282b043a1..b284f283ba7 100644
--- a/apps/sim/lib/sim-search/live/http.ts
+++ b/apps/sim/lib/sim-search/live/http.ts
@@ -1,5 +1,9 @@
-import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server'
+import {
+ type PinnedConnectionPool,
+ secureFetchWithValidation,
+} from '@/lib/core/security/input-validation.server'
import type { NativeClient } from '@/lib/sim-search/live/types'
+import { isGoogleQuotaReason } from '@/connectors/google-workspace/api-errors'
export class NativeSearchError extends Error {
constructor(
@@ -16,6 +20,8 @@ export function createNativeClient(input: {
origin: string
accessToken: string
signal: AbortSignal
+ /** Reuses connections across this client's requests; the caller owns its lifetime. */
+ pool?: PinnedConnectionPool
}): NativeClient {
let requests = 0
async function request(
@@ -57,10 +63,18 @@ export function createNativeClient(input: {
timeout: 10_000,
maxResponseBytes: 4 * 1024 * 1024,
maxRedirects: 0,
+ acceptCompressed: true,
+ connectionPool: input.pool,
})
if (!response.ok) {
+ /** Reading the small error body also returns a pooled connection for reuse. */
+ let quotaExceeded = false
+ if (response.status === 403 && url.hostname.endsWith('.googleapis.com'))
+ quotaExceeded = isGoogleQuotaError(await response.json().catch(() => null))
+ else await response.text().catch(() => '')
if (
response.status === 429 ||
+ quotaExceeded ||
(response.status === 403 &&
(response.headers.get('retry-after') !== null ||
response.headers.get('x-ratelimit-remaining') === '0'))
@@ -90,16 +104,53 @@ export function createNativeClient(input: {
}
return response
}
- return {
+ return withJsonMemo({
async json(path, options) {
return (await request(path, options)).json()
},
async text(path, query) {
return (await request(path, { query })).text()
},
+ })
+}
+
+/**
+ * Answers a repeated `memo` GET from the client's earlier response, including a failure, so a
+ * rate-limited endpoint is not requested again. A client lives for one search or read, so a
+ * memoized response never outlives the request that authorized it.
+ */
+export function withJsonMemo(client: NativeClient): NativeClient {
+ const memo = new Map>()
+ return {
+ ...client,
+ json(...request) {
+ const [path, options] = request
+ if (!options?.memo || options.body) return client.json(...request)
+ const key = JSON.stringify([path, options.query, options.googleService])
+ let pending = memo.get(key)
+ if (!pending) {
+ pending = client.json(...request)
+ memo.set(key, pending)
+ }
+ return pending
+ },
}
}
+/**
+ * Google reports quota exhaustion as a 403 whose body names the reason, without the rate-limit
+ * headers other providers send. Only the reason codes are read; the body is never surfaced.
+ */
+function isGoogleQuotaError(body: unknown): boolean {
+ const error = object(object(body).error)
+ return (
+ error.status === 'RESOURCE_EXHAUSTED' ||
+ [...array(error.errors), ...array(error.details)].some((entry) =>
+ isGoogleQuotaReason(string(entry.reason))
+ )
+ )
+}
+
/** Only extract typed fields from untrusted provider JSON. */
export function object(value: unknown): Record {
return value !== null && typeof value === 'object' && !Array.isArray(value)
diff --git a/apps/sim/lib/sim-search/live/pages.test.ts b/apps/sim/lib/sim-search/live/pages.test.ts
new file mode 100644
index 00000000000..b7a23e03038
--- /dev/null
+++ b/apps/sim/lib/sim-search/live/pages.test.ts
@@ -0,0 +1,39 @@
+/** @vitest-environment node */
+import { describe, expect, it } from 'vitest'
+import { NativeSearchError } from '@/lib/sim-search/live/http'
+import { collectNativePages, interleaveByRank, joinMessages } from '@/lib/sim-search/live/pages'
+
+const doc = (id: string) => ({ id, title: id, url: '', content: '' })
+
+describe('native page merging', () => {
+ it('interleaves ranked lists of different lengths', () => {
+ expect(interleaveByRank([[1, 2, 3], [4], [5, 6]])).toEqual([1, 4, 5, 2, 6, 3])
+ })
+ it('joins each message once', () => {
+ expect(joinMessages(['a.', undefined, 'b.', 'a.'])).toBe('a. b.')
+ expect(joinMessages([undefined])).toBeUndefined()
+ })
+ it('reports collection cursors as more results rather than degraded coverage', async () => {
+ const page = await collectNativePages(
+ [
+ Promise.resolve({ documents: [doc('a')], nextCursor: '2', message: 'Searched.' }),
+ Promise.resolve({ documents: [doc('b')], message: 'Searched.' }),
+ ],
+ 'Guidance.'
+ )
+ expect(page).toMatchObject({ partial: false, hasMore: true, message: 'Guidance.' })
+ expect(page.documents.map((item) => item.id)).toEqual(['a', 'b'])
+ })
+ it('reports a failed collection once, keeping successful evidence', async () => {
+ const failure = new NativeSearchError('unavailable', 'Code search failed.')
+ const page = await collectNativePages(
+ [
+ Promise.resolve({ documents: [doc('a')] }),
+ Promise.reject(failure),
+ Promise.reject(failure),
+ ],
+ 'Guidance.'
+ )
+ expect(page).toMatchObject({ partial: true, message: 'Guidance. Code search failed.' })
+ })
+})
diff --git a/apps/sim/lib/sim-search/live/pages.ts b/apps/sim/lib/sim-search/live/pages.ts
index 67aa635603a..a2f30959c16 100644
--- a/apps/sim/lib/sim-search/live/pages.ts
+++ b/apps/sim/lib/sim-search/live/pages.ts
@@ -1,6 +1,27 @@
import { NativeSearchError } from '@/lib/sim-search/live/http'
import type { NativePage } from '@/lib/sim-search/live/types'
+/**
+ * Merges ranked lists by position: every list's first item, then every second item, and so on.
+ * A populous list therefore cannot consume all of the previews ahead of the others.
+ */
+export function interleaveByRank(lists: readonly (readonly T[])[]): T[] {
+ const merged: T[] = []
+ const longest = Math.max(0, ...lists.map((list) => list.length))
+ for (let rank = 0; rank < longest; rank++) {
+ for (const list of lists) if (rank < list.length) merged.push(list[rank]!)
+ }
+ return merged
+}
+
+/** Joins status sentences once each, in first-seen order. */
+export function joinMessages(messages: readonly (string | undefined)[]): string | undefined {
+ return (
+ [...new Set(messages.filter((message): message is string => Boolean(message)))].join(' ') ||
+ undefined
+ )
+}
+
/** Independent collections retain their successful evidence if another collection fails. */
export async function collectNativePages(
requests: Promise[],
@@ -12,20 +33,16 @@ export async function collectNativePages(
result.status === 'rejected' ? [result.reason] : []
)
if (!pages.length && failures.length) throw failures[0]
- const documents = []
- // Interleave ranks so a populous collection cannot consume all cross-collection previews.
- for (let rank = 0; rank < Math.max(0, ...pages.map((page) => page.documents.length)); rank++) {
- for (const page of pages) if (page.documents[rank]) documents.push(page.documents[rank])
- }
return {
- documents,
- partial: failures.length > 0 || pages.some((page) => page.partial || page.nextCursor),
- message: [
+ documents: interleaveByRank(pages.map((page) => page.documents)),
+ partial: failures.length > 0 || pages.some((page) => page.partial),
+ hasMore: pages.some((page) => page.hasMore || page.nextCursor),
+ message: joinMessages([
guidance,
- ...pages.filter((page) => page.partial && page.message).map((page) => page.message),
+ ...pages.filter((page) => page.partial || page.hasMore).map((page) => page.message),
...failures.map((error) =>
error instanceof NativeSearchError ? error.message : 'One collection could not be searched.'
),
- ].join(' '),
+ ]),
}
}
diff --git a/apps/sim/lib/sim-search/live/policy.test.ts b/apps/sim/lib/sim-search/live/policy.test.ts
index 7aa5b44f554..41eee09e123 100644
--- a/apps/sim/lib/sim-search/live/policy.test.ts
+++ b/apps/sim/lib/sim-search/live/policy.test.ts
@@ -1,5 +1,6 @@
/** @vitest-environment node */
import { describe, expect, it, vi } from 'vitest'
+import { withJsonMemo } from '@/lib/sim-search/live/http'
import { createPolicyVerifier } from '@/lib/sim-search/live/policy'
import {
defaultLiveSearchPolicy,
@@ -243,4 +244,19 @@ describe('organization search scope enforcement', () => {
expect(await verify({ id: 'https://coda.io:444/d/doc' })).toBe(false)
expect(await verify({ id: 'https://evil.test/d/doc' })).toBe(false)
})
+ it('shares metadata within a client unless the verifier must read it fresh', async () => {
+ const api = client({
+ '/drive/v3/files/doc': { id: 'doc', parents: ['root'] },
+ '/drive/v3/files/root': { id: 'root', parents: [] },
+ })
+ const session = withJsonMemo(api)
+ const verify = createPolicyVerifier('google_drive', selected(['root']), session, '')
+ await verify({ id: 'doc' })
+ await verify({ id: 'doc' })
+ expect(api.json).toHaveBeenCalledTimes(2)
+ await createPolicyVerifier('google_drive', selected(['root']), session, '', undefined, {
+ fresh: true,
+ })({ id: 'doc' })
+ expect(api.json).toHaveBeenCalledTimes(4)
+ })
})
diff --git a/apps/sim/lib/sim-search/live/policy.ts b/apps/sim/lib/sim-search/live/policy.ts
index a8a5ce2a143..43ec5a3c339 100644
--- a/apps/sim/lib/sim-search/live/policy.ts
+++ b/apps/sim/lib/sim-search/live/policy.ts
@@ -30,7 +30,8 @@ export function permitsPath(policy: LiveSearchPolicy, path: string) {
}
/**
- * One request-local metadata cache; neither tokens nor permissions survive the request.
+ * Metadata reads are memoized on the request-scoped client, so neither tokens nor permissions
+ * survive the request.
* Optional document metadata must come from a current read using this verifier's client.
*/
export function createPolicyVerifier(
@@ -38,20 +39,15 @@ export function createPolicyVerifier(
policy: LiveSearchPolicy,
client: NativeClient | null,
origin: string,
- mcp?: CodaMcpClient
+ mcp?: CodaMcpClient,
+ /** A fresh verifier reads provider metadata again instead of reusing this client's responses. */
+ options: { fresh?: boolean } = {}
): PolicyVerifier {
if (!requiresScopedRetrieval(provider, policy)) return async () => true
- const calls = new Map>()
const json = (path: string, query?: Record) => {
if (!client)
throw new NativeSearchError('unavailable', 'This connection cannot verify the search scope.')
- const key = JSON.stringify([path, query])
- let result = calls.get(key)
- if (!result) {
- result = client.json(path, query ? { query } : undefined)
- calls.set(key, result)
- }
- return result
+ return client.json(path, { query, memo: !options.fresh })
}
const restricted = policy.mode === 'selected' || policy.excluded.length > 0
const siteAllowed = (value: string) => {
diff --git a/apps/sim/lib/sim-search/live/providers.test.ts b/apps/sim/lib/sim-search/live/providers.test.ts
index 4a19ae95e83..73aa4b46600 100644
--- a/apps/sim/lib/sim-search/live/providers.test.ts
+++ b/apps/sim/lib/sim-search/live/providers.test.ts
@@ -5,6 +5,7 @@ import { searchCoda } from '@/lib/sim-search/live/coda'
import { searchGitHub } from '@/lib/sim-search/live/github'
import { readGitLab, searchGitLab } from '@/lib/sim-search/live/gitlab'
import { readDrive, searchCalendar, searchDrive, searchGmail } from '@/lib/sim-search/live/google'
+import { withJsonMemo } from '@/lib/sim-search/live/http'
import { readSlack, searchSlack } from '@/lib/sim-search/live/slack'
import type { NativeClient } from '@/lib/sim-search/live/types'
@@ -105,13 +106,261 @@ describe('native search endpoints', () => {
const api = client()
api.json
.mockResolvedValueOnce({ items: [{ id: 'primary' }, { id: 'team@example.com' }] })
- .mockResolvedValue({
- items: [{ id: 'e', summary: 'Launch', htmlLink: 'https://calendar.google.com/event' }],
- })
+ .mockResolvedValueOnce({ items: [{ id: 'mine', summary: 'Launch' }] })
+ .mockResolvedValueOnce({ items: [{ id: 'team', summary: 'Launch review' }] })
const result = await searchCalendar(api, input)
expect(result.documents.map((item) => item.container)).toEqual(['primary', 'team@example.com'])
expect(api.json.mock.calls[2][0]).toContain('team%40example.com')
})
+ it('marks a meeting shared across calendars as one item, primary calendar first', async () => {
+ const api = client()
+ const shared = (calendar: string, dateTime: string) => ({
+ id: 'meeting',
+ iCalUID: 'meeting@google.com',
+ summary: 'Pilot planning',
+ htmlLink: `https://calendar.google.com/${calendar}`,
+ start: { dateTime },
+ })
+ api.json
+ .mockResolvedValueOnce({
+ items: [{ id: 'teammate@example.com' }, { id: 'member@example.com', primary: true }],
+ })
+ .mockResolvedValueOnce({ items: [shared('me', '2026-09-24T09:00:00-07:00')] })
+ .mockResolvedValueOnce({ items: [shared('emir', '2026-09-24T12:00:00-04:00')] })
+ const result = await searchCalendar(api, input)
+ expect(result.documents.map((item) => item.container)).toEqual([
+ 'member@example.com',
+ 'teammate@example.com',
+ ])
+ expect(new Set(result.documents.map((item) => item.dedupeKey)).size).toBe(1)
+ expect(api.json.mock.calls[1][0]).toContain('member%40example.com')
+ })
+ it('keeps recurring occurrences that share one iCalendar UID', async () => {
+ const api = client()
+ const occurrence = (day: string) => ({
+ id: `standup_${day}`,
+ iCalUID: 'standup@google.com',
+ summary: 'Standup',
+ start: { dateTime: `2026-09-${day}T09:00:00Z` },
+ })
+ api.json
+ .mockResolvedValueOnce({ items: [{ id: 'primary', primary: true }] })
+ .mockResolvedValueOnce({ items: [occurrence('24'), occurrence('25')] })
+ const result = await searchCalendar(api, {
+ ...input,
+ query: '',
+ filters: { startDate: '2026-09-24T00:00:00Z', endDate: '2026-09-26T00:00:00Z' },
+ })
+ expect(result.documents.map((item) => item.id)).toEqual(['standup_24', 'standup_25'])
+ expect(result.documents[0]?.dedupeKey).not.toBe(result.documents[1]?.dedupeKey)
+ })
+ it('orders a dated agenda by start time across calendars', async () => {
+ const api = client()
+ const event = (id: string, hour: string) => ({
+ id,
+ summary: id,
+ start: { dateTime: `2026-09-24T${hour}:00:00Z` },
+ })
+ api.json
+ .mockResolvedValueOnce({ items: [{ id: 'primary', primary: true }, { id: 'team' }] })
+ .mockResolvedValueOnce({ items: [event('late', '17'), event('later', '18')] })
+ .mockResolvedValueOnce({ items: [event('early', '08')], nextPageToken: 'more' })
+ const result = await searchCalendar(api, {
+ ...input,
+ query: '',
+ filters: { startDate: '2026-09-24T00:00:00Z', endDate: '2026-09-25T00:00:00Z' },
+ })
+ expect(result.documents.map((item) => item.id)).toEqual(['early', 'late', 'later'])
+ expect(result).toMatchObject({ hasMore: true, partial: false })
+ expect(api.json.mock.calls[1][1]?.query).toMatchObject({
+ singleEvents: 'true',
+ orderBy: 'startTime',
+ })
+ })
+ it('turns HTML event descriptions into readable text', async () => {
+ const api = client()
+ api.json
+ .mockResolvedValueOnce({ items: [{ id: 'primary', primary: true }] })
+ .mockResolvedValueOnce({
+ items: [
+ {
+ id: 'e',
+ summary: 'Demo',
+ description:
+ 'Who:
host@example.com
Where: https://meet.google.com/abc',
+ },
+ ],
+ })
+ const [event] = (await searchCalendar(api, input)).documents
+ expect(event.content).toContain('Who:\nhost@example.com\nWhere: https://meet.google.com/abc')
+ expect(event.content).not.toContain(' {
+ const api = client()
+ api.json.mockResolvedValueOnce({ messages: [{ id: 'm' }] }).mockResolvedValueOnce({
+ id: 'm',
+ labelIds: ['INBOX'],
+ snippet: 'Let's ship it \u034f \u034f \u200c\u200b\ufeff \u034f',
+ internalDate: '1700000000000',
+ payload: { headers: [{ name: 'Subject', value: 'Launch' }] },
+ })
+ const [message] = (await searchGmail(api, input)).documents
+ expect(message.content).toBe("Let's ship it")
+ expect(message.accessMetadata).toEqual({ id: 'm', labelIds: ['INBOX'] })
+ expect(api.json.mock.calls[1][1]?.query).toMatchObject({
+ format: 'metadata',
+ metadataHeaders: ['Subject', 'From'],
+ fields: 'id,labelIds,snippet,internalDate,payload/headers',
+ })
+ })
+ it('batches code search by bytes and searches issues in fewer, larger batches', async () => {
+ const api = client()
+ const repositories = Array.from({ length: 100 }, (_, index) => ({
+ full_name: `simstudioai/repository-with-a-long-name-${index}`,
+ }))
+ api.json.mockImplementation(async (path) =>
+ path === '/user/repos' ? repositories : { items: [], total_count: 0 }
+ )
+ const result = await searchGitHub(api, { ...input, query: 'déploiement 検索' })
+ const queries = (path: string) =>
+ api.json.mock.calls
+ .filter(([called]) => called === path)
+ .map(([, options]) => Buffer.byteLength(String(options?.query?.q)))
+ expect(Math.max(...queries('/search/code'))).toBeLessThanOrEqual(1000)
+ expect(queries('/search/code').length).toBeLessThanOrEqual(4)
+ expect(queries('/search/issues').length).toBeLessThan(queries('/search/code').length * 2)
+ expect(result.partial).toBe(true)
+ expect(result.message).toMatch(/Only the \d+ most recently pushed repositories were searched/)
+ })
+ it('interleaves GitHub kinds before batches so one kind cannot crowd out another', async () => {
+ const api = client()
+ const repositories = Array.from({ length: 100 }, (_, index) => ({
+ full_name: `simstudioai/repository-with-a-long-name-${index}`,
+ }))
+ api.json.mockImplementation(async (path) => {
+ if (path === '/user/repos') return repositories
+ return path === '/search/code'
+ ? { items: [{ path: 'src/a.ts', repository: { full_name: 'org/repo' } }], total_count: 1 }
+ : { items: [{ number: 1, title: 'Issue' }], total_count: 1 }
+ })
+ const result = await searchGitHub(api, input)
+ const count = (path: string) => api.json.mock.calls.filter(([called]) => called === path).length
+ expect(count('/search/code')).toBeGreaterThan(count('/search/issues') / 2)
+ expect(result.documents.slice(0, 2).map(({ kind }) => kind)).toEqual(['issues', 'code'])
+ })
+ it('batches GitHub repository search like issue search, not by the code query limit', async () => {
+ const api = client()
+ const repositories = Array.from({ length: 100 }, (_, index) => ({
+ full_name: `simstudioai/repository-with-a-long-name-${index}`,
+ }))
+ api.json.mockImplementation(async (path) =>
+ path === '/user/repos' ? repositories : { items: [], total_count: 0 }
+ )
+ await searchGitHub(api, {
+ ...input,
+ native: { provider: 'github', query: 'launch', kind: 'repositories' },
+ })
+ const searches = api.json.mock.calls.filter(([path]) => path === '/search/repositories')
+ expect(searches.length).toBeGreaterThan(0)
+ expect(searches.length).toBeLessThanOrEqual(2)
+ })
+ it('skips code search when the query leaves no room for a repository qualifier', async () => {
+ const api = client()
+ api.json.mockImplementation(async (path) =>
+ path === '/user/repos' ? [{ full_name: 'org/repo' }] : { items: [], total_count: 0 }
+ )
+ const result = await searchGitHub(api, { ...input, query: `launch label:"${'x'.repeat(960)}"` })
+ const searches = api.json.mock.calls.filter(([path]) => path.startsWith('/search/'))
+ expect(searches.map(([path]) => path)).toEqual(['/search/issues', '/search/issues'])
+ expect(result).toMatchObject({
+ partial: true,
+ message: expect.stringContaining('code search was skipped'),
+ })
+ })
+ it('leaves a boolean GitHub query as written when dates are appended', async () => {
+ const api = client()
+ api.json.mockResolvedValue({ items: [], total_count: 0 })
+ await searchGitHub(api, {
+ ...input,
+ native: {
+ provider: 'github',
+ query: 'repo:org/repo is:issue label:bug OR label:feature',
+ kind: 'issues',
+ },
+ filters: { startDate: '2026-09-20T00:00:00Z' },
+ })
+ expect(api.json.mock.calls[0][1]?.query?.q).toBe(
+ 'repo:org/repo is:issue label:bug OR label:feature updated:>=2026-09-20T00:00:00.000Z'
+ )
+ })
+ it("explains GitHub's search text limit instead of sending an oversized query", async () => {
+ const api = client()
+ await expect(
+ searchGitHub(api, {
+ ...input,
+ native: { provider: 'github', query: `repo:org/repo ${'word '.repeat(60)}`, kind: 'code' },
+ })
+ ).rejects.toThrow('limited to 256 characters')
+ expect(api.json).not.toHaveBeenCalled()
+ })
+ it('keeps GitHub qualifiers outside the grouped text of a dated search', async () => {
+ const api = client()
+ api.json.mockImplementation(async (path) =>
+ path === '/user/repos' ? [{ full_name: 'org/repo' }] : { items: [], total_count: 0 }
+ )
+ await searchGitHub(api, {
+ ...input,
+ filters: { startDate: '2026-09-20T00:00:00Z', endDate: '2026-09-24T00:00:00Z' },
+ })
+ const searches = api.json.mock.calls.filter(([path]) => path.startsWith('/search/'))
+ expect(searches.map(([path]) => path)).toEqual(['/search/issues', '/search/issues'])
+ expect(searches.map(([, options]) => options?.query?.q)).toEqual([
+ '(launch) repo:org/repo is:issue updated:2026-09-20T00:00:00.000Z..2026-09-24T00:00:00.000Z',
+ '(launch) repo:org/repo is:pull-request updated:2026-09-20T00:00:00.000Z..2026-09-24T00:00:00.000Z',
+ ])
+ })
+ it('lists dated GitHub issues without grouping an empty query', async () => {
+ const api = client()
+ api.json.mockResolvedValue({ items: [], total_count: 0 })
+ await searchGitHub(api, {
+ ...input,
+ query: '',
+ native: { provider: 'github', query: 'repo:org/repo is:issue', kind: 'issues' },
+ filters: { startDate: '2026-09-20T00:00:00Z' },
+ })
+ expect(api.json.mock.calls[0][1]?.query?.q).toBe(
+ 'repo:org/repo is:issue updated:>=2026-09-20T00:00:00.000Z'
+ )
+ })
+ it('searches Atlassian sites in parallel and reads accessible sites once per client', async () => {
+ const api = client()
+ api.json.mockImplementation(async (path) => {
+ if (path === '/oauth/token/accessible-resources')
+ return [
+ { id: 'one', url: 'https://one.atlassian.net' },
+ { id: 'two', url: 'https://two.atlassian.net' },
+ ]
+ const site = path.includes('/one/') ? 'one' : 'two'
+ return {
+ results: [
+ {
+ content: { id: `${site}-1`, title: 'A' },
+ excerpt: '@@@hl@@@Launch@@@endhl@@@ & plan',
+ },
+ { content: { id: `${site}-2`, title: 'B' } },
+ ],
+ }
+ })
+ const session = withJsonMemo(api)
+ const result = await searchAtlassian(session, 'confluence', input)
+ await searchAtlassian(session, 'confluence', input)
+ expect(result.documents.map((item) => item.id)).toEqual(['one-1', 'two-1', 'one-2', 'two-2'])
+ expect(result.documents[0].content).toBe('Launch & plan')
+ expect(
+ api.json.mock.calls.filter(([path]) => path === '/oauth/token/accessible-resources')
+ ).toHaveLength(1)
+ })
it('calls Slack RTS with only channel types granted by the user token', async () => {
const api = client()
api.json.mockResolvedValue({
@@ -278,6 +527,27 @@ describe('native search endpoints', () => {
).toHaveLength(3)
expect(api.json.mock.calls.some(([path]) => path === '/api/v4/search')).toBe(false)
})
+ it('reads GitLab merge request assignees from the deprecated single assignee', async () => {
+ const api = client()
+ const search = (row: Record) => {
+ api.json.mockResolvedValueOnce([
+ {
+ iid: 7,
+ project_id: 4,
+ web_url: 'https://gitlab.com/org/repo/-/merge_requests/7',
+ ...row,
+ },
+ ])
+ return searchGitLab(api, {
+ ...input,
+ native: { provider: 'gitlab', query: 'launch', kind: 'merge_requests', project: '4' },
+ })
+ }
+ expect(await search({ assignee: { id: 3 } })).toMatchObject({
+ documents: [{ accessMetadata: { assigneeIds: [3] } }],
+ })
+ expect(await search({})).toMatchObject({ documents: [{ accessMetadata: { assigneeIds: [] } }] })
+ })
it('binds GitLab code references and links to the searched revision', async () => {
const api = client()
api.json
@@ -365,7 +635,7 @@ describe('native search endpoints', () => {
...input,
native: { provider: 'github', query: 'repo:org/repo auth', kind: 'code' },
})
- expect(result).toMatchObject({ partial: true, nextCursor: '2' })
+ expect(result).toMatchObject({ partial: true, hasMore: true, nextCursor: '2' })
expect(api.json.mock.calls[0][0]).toBe('/search/code')
})
})
diff --git a/apps/sim/lib/sim-search/live/service-session.test.ts b/apps/sim/lib/sim-search/live/service-session.test.ts
index f08bf2118e3..8a131c5ae85 100644
--- a/apps/sim/lib/sim-search/live/service-session.test.ts
+++ b/apps/sim/lib/sim-search/live/service-session.test.ts
@@ -137,12 +137,27 @@ describe('service credential isolation', () => {
})
expect(session).toBeTruthy()
expect(mocks.githubSources).toHaveBeenCalledWith(base.owner)
- expect(mocks.github).toHaveBeenCalledWith([{ id: 'repository-source' }], api, base.signal)
+ expect(mocks.github).toHaveBeenCalledWith(
+ [{ id: 'repository-source' }],
+ api,
+ base.signal,
+ undefined
+ )
expect(mocks.source).not.toHaveBeenCalled()
await expect(
createLiveServiceSession({ ...base, member: null, provider: 'github', policy: configured })
).rejects.toThrow('personal GitHub account')
})
+ it('passes the caller connection pool to the GitHub verifier', async () => {
+ const pool = { agent: vi.fn(), destroy: vi.fn() }
+ await createLiveServiceSession({
+ ...base,
+ provider: 'github',
+ policy: { ...policy, sourceId: undefined },
+ pool,
+ })
+ expect(mocks.github).toHaveBeenCalledWith(expect.anything(), api, base.signal, pool)
+ })
})
describe('Confluence source namespace', () => {
diff --git a/apps/sim/lib/sim-search/live/service-session.ts b/apps/sim/lib/sim-search/live/service-session.ts
index a901dd92a28..9006ff86227 100644
--- a/apps/sim/lib/sim-search/live/service-session.ts
+++ b/apps/sim/lib/sim-search/live/service-session.ts
@@ -1,5 +1,6 @@
import type { LiveSearchProvider } from '@/lib/api/contracts/mothership-assistant-tools'
import type { ResourceOwner } from '@/lib/core/resource-scope'
+import type { PinnedConnectionPool } from '@/lib/core/security/input-validation.server'
import {
resolveConnectorAccessToken,
resolveConnectorTokenUserId,
@@ -41,12 +42,19 @@ export async function createLiveServiceSession(input: {
member: NativeClient | null
mcp?: CodaMcpClient
signal: AbortSignal
+ /** Reuses connections for the source credential's requests; owned by the caller. */
+ pool?: PinnedConnectionPool
}): Promise {
- const { provider, policy, member, signal } = input
+ const { provider, policy, member, signal, pool } = input
if (policy.accessMode !== 'service_account' || provider === 'gitlab') return undefined
if (provider === 'github') {
if (!member) throw new NativeSearchError('reconnect', 'Connect your personal GitHub account.')
- return createGitHubServiceVerifier(await loadLiveGitHubSources(input.owner), member, signal)
+ return createGitHubServiceVerifier(
+ await loadLiveGitHubSources(input.owner),
+ member,
+ signal,
+ pool
+ )
}
if (!policy.sourceId)
throw new NativeSearchError('unavailable', 'Ask an admin to select a service account source.')
@@ -86,6 +94,7 @@ export async function createLiveServiceSession(input: {
config: source.config,
policy: sourcePolicy,
signal,
+ pool,
})),
}
}
@@ -93,6 +102,7 @@ export async function createLiveServiceSession(input: {
origin: LIVE_SEARCH_PROVIDER_CATALOG[provider].origin,
accessToken: token.accessToken,
signal,
+ pool,
})
if (provider === 'coda')
return {
diff --git a/apps/sim/lib/sim-search/live/slack.ts b/apps/sim/lib/sim-search/live/slack.ts
index b222ec42c7f..9897f72653b 100644
--- a/apps/sim/lib/sim-search/live/slack.ts
+++ b/apps/sim/lib/sim-search/live/slack.ts
@@ -1,4 +1,4 @@
-import { nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates'
+import { dateSortDirection, nativeDateBounds, nativeText } from '@/lib/sim-search/live/dates'
import { array, NativeSearchError, object, string } from '@/lib/sim-search/live/http'
import {
slackConversationName,
@@ -78,11 +78,8 @@ export async function searchSlack(
limit: Math.min(input.limit, 20),
...(dates.start ? { after: Math.floor(Date.parse(dates.start) / 1000) - 1 } : {}),
...(dates.end ? { before: Math.ceil(Date.parse(dates.end) / 1000) } : {}),
- ...(input.filters?.sortBy && input.filters.sortBy !== 'relevance'
- ? {
- sort: 'timestamp',
- sort_dir: input.filters.sortBy === 'oldest' ? 'asc' : 'desc',
- }
+ ...(dateSortDirection(input.filters)
+ ? { sort: 'timestamp', sort_dir: dateSortDirection(input.filters) }
: {}),
...(input.native?.cursor ? { cursor: input.native.cursor } : {}),
...(input.native?.termClauses ? { term_clauses: input.native.termClauses } : {}),
diff --git a/apps/sim/lib/sim-search/live/text.test.ts b/apps/sim/lib/sim-search/live/text.test.ts
new file mode 100644
index 00000000000..5318f47e551
--- /dev/null
+++ b/apps/sim/lib/sim-search/live/text.test.ts
@@ -0,0 +1,69 @@
+/** @vitest-environment node */
+import { describe, expect, it } from 'vitest'
+import { providerText } from '@/lib/sim-search/live/text'
+
+describe('providerText', () => {
+ it('decodes an escaped snippet and drops preheader padding', () => {
+ const padding = String.fromCodePoint(0x034f, 0x20, 0x200c, 0x200b, 0x200d, 0x200e, 0xfeff, 0xad)
+ expect(providerText(`Receipt from Exa 'Labs' ${padding.repeat(3)}`, 'escaped')).toBe(
+ "Receipt from Exa 'Labs'"
+ )
+ })
+ it('keeps a zero-width joiner that composes an emoji', () => {
+ const family = String.fromCodePoint(0x1f468, 0x200d, 0x1f469, 0x200d, 0x1f467)
+ expect(providerText(`Team ${family}`)).toBe(`Team ${family}`)
+ })
+ it('keeps line structure when converting markup to text', () => {
+ expect(
+ providerText(
+ 'Who: a@b.co
Notes & links ',
+ 'html'
+ )
+ ).toBe('Who:\n\na@b.co\nNotes & links')
+ })
+ it('keeps an address whose name starts like a tag in auto mode', () => {
+ expect(providerText('Organizer ', 'auto')).toBe('Organizer ')
+ })
+ it('leaves plain text with angle brackets untouched in auto mode', () => {
+ expect(providerText('Reply from John ', 'auto')).toBe(
+ 'Reply from John '
+ )
+ })
+ it('keeps the spacing of plain text documents', () => {
+ const table = ' Name Qty\n Widget 2\n\n> quoted reply\n indented code'
+ expect(providerText(`${table} \n\n\n`)).toBe(table)
+ })
+ it('removes invisible characters inside a word without splitting it', () => {
+ const softHyphen = String.fromCodePoint(0xad)
+ expect(providerText(`hyphen${softHyphen}ation`)).toBe('hyphenation')
+ })
+ it('does not decode entities in plain text', () => {
+ expect(providerText('Use <b> for bold')).toBe('Use <b> for bold')
+ })
+ it('keeps each table row on one line', () => {
+ expect(providerText('', 'html')).toContain(
+ 'Widget 2'
+ )
+ })
+ it('separates a definition term from its definition', () => {
+ expect(providerText('- Term
- Def
', 'html')).toMatch(/Term\s+Def/)
+ })
+ it('does not throw on deeply nested markup', () => {
+ expect(() => providerText(`${''.repeat(20000)}x`, 'html')).not.toThrow()
+ })
+ it.each(['plain', 'escaped'] as const)(
+ 'cleans a long run of spaces in linear time (%s)',
+ (format) => {
+ const started = performance.now()
+ expect(providerText(`${' '.repeat(1_000_000)}x`, format)).toContain('x')
+ expect(performance.now() - started).toBeLessThan(1000)
+ }
+ )
+ it('keeps a zero-width non-joiner that shapes a Persian word', () => {
+ const word = String.fromCodePoint(0x0645, 0x06cc, 0x200c, 0x062e, 0x0648, 0x0627, 0x0645)
+ expect(providerText(word)).toBe(word)
+ })
+ it('removes a stray byte-order mark inside a word without adding a space', () => {
+ expect(providerText(`doc${String.fromCodePoint(0xfeff)}ument`)).toBe('document')
+ })
+})
diff --git a/apps/sim/lib/sim-search/live/text.ts b/apps/sim/lib/sim-search/live/text.ts
new file mode 100644
index 00000000000..049da163f93
--- /dev/null
+++ b/apps/sim/lib/sim-search/live/text.ts
@@ -0,0 +1,94 @@
+import { convert, type HtmlToTextOptions } from 'html-to-text'
+import { decodeHtmlEntities, htmlToPlainText, looksLikeHtml } from '@/connectors/utils'
+
+/**
+ * How a provider encodes a text field: `plain` is used as-is, `escaped` is HTML-escaped text
+ * without markup (Gmail snippets), `html` is markup, and `auto` treats the value as markup only
+ * when it carries real HTML tags (Calendar descriptions may be either).
+ */
+type ProviderTextFormat = 'plain' | 'escaped' | 'html' | 'auto'
+
+/**
+ * Invisible format characters that marketing mail pads preheaders with. A zero-width joiner
+ * followed by a pictograph is kept because it composes a single emoji. The combining grapheme
+ * joiner is its own alternative because it cannot share a character class with base characters.
+ */
+const INVISIBLE_CHARACTER =
+ '[\\u00AD\\u061C\\u115F\\u1160\\u17B4\\u17B5\\u180E\\u200B\\u200C\\u200E\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u206F\\u3164\\uFEFF\\uFFA0]|\\u034F|\\u200D(?!\\p{Extended_Pictographic})'
+/**
+ * A run of invisible characters with the spaces between them, as preheader padding is built.
+ * The lookbehind starts a match only at the beginning of a space run, which keeps long runs of
+ * spaces linear instead of rescanning them from every position.
+ */
+const INVISIBLE_RUN = new RegExp(
+ `(? ({
+ selector,
+ options: { uppercase: false },
+ })),
+ ],
+}
+
+/** Markup the converter cannot walk keeps its text through the flat tag-stripping path. */
+function markupText(html: string): string {
+ try {
+ return convert(html, HTML_TO_TEXT)
+ } catch {
+ return htmlToPlainText(html)
+ }
+}
+
+/**
+ * Normalizes provider text into readable plain text for previews and document reads. Markup is
+ * never rendered: tags are dropped and entities decoded into literal characters.
+ */
+export function providerText(value: string, format: ProviderTextFormat = 'plain'): string {
+ let text = value
+ if (format === 'html' || (format === 'auto' && looksLikeHtml(text))) {
+ text = markupText(text)
+ } else if (format === 'escaped') {
+ text = decodeHtmlEntities(text)
+ }
+ /** Padding collapses to one space; a stray mark inside a word is dropped unless it shapes it. */
+ text = text.replace(INVISIBLE_RUN, (run) =>
+ SPACING.test(run) ? ' ' : MEANINGFUL_MARK.test(run) ? run : ''
+ )
+ if (format === 'escaped') return text.replace(/\s+/g, ' ').trim()
+ return text
+ .replace(TRAILING_SPACES, '')
+ .replace(/\n{3,}/g, '\n\n')
+ .replace(/^\n+|\n+$/g, '')
+}
diff --git a/apps/sim/lib/sim-search/live/types.ts b/apps/sim/lib/sim-search/live/types.ts
index f0b38cfd7e2..f146546bf6b 100644
--- a/apps/sim/lib/sim-search/live/types.ts
+++ b/apps/sim/lib/sim-search/live/types.ts
@@ -21,8 +21,16 @@ export interface LiveAccount {
}
export interface NativeDocument {
- /** Server-only GitLab permission evidence from the same response as the content. */
+ /**
+ * Server-only permission evidence from the same response as the content. Only a verifier
+ * bound to the client that produced it may consume it; it is never projected to callers.
+ */
accessMetadata?: Record
+ /**
+ * Identifies one item a provider returns from several collections of the same account, such
+ * as a meeting on each attendee's calendar. Only the first verified copy is kept.
+ */
+ dedupeKey?: string
id: string
container?: string
containerName?: string
@@ -40,7 +48,11 @@ export interface NativeDocument {
export interface NativePage {
documents: NativeDocument[]
+ /** Continues this exact query and account; implies more results exist. */
nextCursor?: string
+ /** More matches exist beyond this page but cannot be continued through it. */
+ hasMore?: boolean
+ /** Coverage is degraded: a collection failed, a cap applied, or evidence was dropped. */
partial?: boolean
message?: string
}
@@ -52,6 +64,8 @@ export interface NativeClient {
query?: Record
body?: unknown
googleService?: 'sheets'
+ /** Reuse this client's earlier response to the same GET instead of requesting it again. */
+ memo?: boolean
}
): Promise
text(path: string, query?: Record): Promise
|