From a1e7e8e75b217c7dab527f39088185a38e6207b3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 28 Sep 2026 13:53:11 -0700 Subject: [PATCH 1/2] fix(search): preserve Gmail reads when replies disappear --- apps/sim/lib/sim-search/live/google.ts | 20 +++-- .../scripts/test-search-google-content-e2e.ts | 82 +++++++++++++++++++ 2 files changed, 97 insertions(+), 5 deletions(-) diff --git a/apps/sim/lib/sim-search/live/google.ts b/apps/sim/lib/sim-search/live/google.ts index 401caf2042c..d9f1b65588f 100644 --- a/apps/sim/lib/sim-search/live/google.ts +++ b/apps/sim/lib/sim-search/live/google.ts @@ -444,11 +444,21 @@ export async function readGmail( warnings.add('messages outside the source search scope were omitted') continue } - const row = object( - await client.json(`/gmail/v1/users/me/messages/${segment(candidateId)}`, { - query: { format: 'full' }, - }) - ) + let row: Record + try { + row = object( + await client.json(`/gmail/v1/users/me/messages/${segment(candidateId)}`, { + query: { format: 'full' }, + }) + ) + } catch (error) { + options.signal.throwIfAborted() + if (error instanceof NativeSearchError && error.httpStatus === 404) { + warnings.add('some messages are no longer available') + continue + } + throw error + } if (row.id !== candidateId || row.threadId !== threadId) throw new NativeSearchError( 'unavailable', diff --git a/apps/sim/scripts/test-search-google-content-e2e.ts b/apps/sim/scripts/test-search-google-content-e2e.ts index ad968f3f7d2..3ba09f9282f 100644 --- a/apps/sim/scripts/test-search-google-content-e2e.ts +++ b/apps/sim/scripts/test-search-google-content-e2e.ts @@ -79,6 +79,7 @@ const mailbox: GmailMessage[] = Array.from({ length: 12 }, (_, index) => ({ body: { data: Buffer.from(`Synthetic evidence ${index}.`).toString('base64url') }, }, })) +const messageFailures = new Map() const mediaStarted = createDeferred() const mediaClosed = createDeferred() const server = http.createServer((request, response) => { @@ -92,6 +93,15 @@ const server = http.createServer((request, response) => { const row = mailbox.find((message) => message.id === id) const isThread = url.pathname.startsWith('/gmail/v1/users/me/threads/') const isAttachment = url.pathname.includes('/attachments/') + const failure = + !isThread && url.searchParams.get('format') === 'full' ? messageFailures.get(id) : undefined + if (failure) { + requests.push({ id, resource: 'message', status: failure }) + response + .writeHead(failure, { 'Content-Type': 'application/json' }) + .end(JSON.stringify({ error: { code: failure } })) + return + } const threadMessages = isThread ? mailbox.filter((message) => message.threadId === id) : [] const metadata = (message: (typeof mailbox)[number]) => ({ ...message, @@ -284,6 +294,78 @@ try { server.listen(0, '127.0.0.1', () => started.resolve()) await started.promise origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}` + for (const { name, failedId, status, expectedStatus } of [ + { name: 'healthy conversation', failedId: '', status: 200 }, + { name: 'missing sibling', failedId: 'reply', status: 404 }, + { name: 'missing anchor', failedId: 'anchor', status: 404, expectedStatus: 'unavailable' }, + { name: 'expired credentials', failedId: 'reply', status: 401, expectedStatus: 'reconnect' }, + { name: 'denied credentials', failedId: 'reply', status: 403, expectedStatus: 'reconnect' }, + { name: 'rate limit', failedId: 'reply', status: 429, expectedStatus: 'rate_limited' }, + { name: 'server failure', failedId: 'reply', status: 500, expectedStatus: 'unavailable' }, + ]) { + await check( + `Gmail conversation handles ${name} without hiding unrelated failures`, + async () => { + const before = mailbox.length + mailbox.push( + ...['anchor', 'retained', 'reply'].map((id, index) => ({ + id, + threadId: 'failure-conversation', + labelIds: ['INBOX'], + internalDate: String(Date.UTC(2026, 0, 1, 12, index)), + payload: { + mimeType: 'text/plain', + body: { data: Buffer.from(`Synthetic ${id} evidence.`).toString('base64url') }, + }, + })) + ) + if (failedId) messageFailures.set(failedId, status) + try { + const signal = AbortSignal.timeout(5000) + const api = createNativeClient({ origin, accessToken: TOKEN, signal }) + const policy = { + ...defaultLiveSearchPolicy('gmail'), + mode: 'selected' as const, + included: ['INBOX'], + } + const verify = createPolicyVerifier('gmail', policy, api, origin) + assert.ok(await verify({ id: 'anchor' })) + const reading = readGmail(api, 'anchor', { + policy, + signal, + verify: (reference) => verify(reference, reference.accessMetadata), + }) + if (expectedStatus) { + await assert.rejects( + reading, + (error: unknown) => + error instanceof NativeSearchError && error.status === expectedStatus + ) + return + } + const document = await reading + assert.equal(document.id, 'anchor') + assert.ok(document.content.includes('Synthetic anchor evidence.')) + assert.ok(document.content.includes('Synthetic retained evidence.')) + assert.equal(document.content.includes('Synthetic reply evidence.'), status === 200) + assert.equal(document.content.includes('Coverage incomplete:'), status === 404) + assert.deepEqual( + document.accessDependencies, + status === 404 ? [{ id: 'retained' }] : [{ id: 'retained' }, { id: 'reply' }] + ) + const current = createPolicyVerifier('gmail', policy, api, origin, undefined, { + fresh: true, + }) + assert.ok(await current({ id: document.id })) + for (const dependency of document.accessDependencies ?? []) + assert.ok(await current(dependency)) + } finally { + mailbox.splice(before) + messageFailures.clear() + } + } + ) + } await check( 'Gmail conversation and fresh selected-label checks fit one native request budget', async () => { From 09ec7ca24aed8c853b7d33bb5f57b28184158085 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 28 Sep 2026 14:03:20 -0700 Subject: [PATCH 2/2] fix(search): handle replies missing during Gmail verification --- apps/sim/lib/sim-search/live/google.ts | 18 +-- .../scripts/test-search-google-content-e2e.ts | 134 +++++++++--------- 2 files changed, 77 insertions(+), 75 deletions(-) diff --git a/apps/sim/lib/sim-search/live/google.ts b/apps/sim/lib/sim-search/live/google.ts index d9f1b65588f..408ee7828bc 100644 --- a/apps/sim/lib/sim-search/live/google.ts +++ b/apps/sim/lib/sim-search/live/google.ts @@ -435,17 +435,17 @@ export async function readGmail( for (const candidate of latest) { options.signal.throwIfAborted() const candidateId = string(candidate.id) - if ( - !(await options.verify({ - id: candidateId, - accessMetadata: { id: candidateId, labelIds: candidate.labelIds }, - })) - ) { - warnings.add('messages outside the source search scope were omitted') - continue - } let row: Record try { + if ( + !(await options.verify({ + id: candidateId, + accessMetadata: { id: candidateId, labelIds: candidate.labelIds }, + })) + ) { + warnings.add('messages outside the source search scope were omitted') + continue + } row = object( await client.json(`/gmail/v1/users/me/messages/${segment(candidateId)}`, { query: { format: 'full' }, diff --git a/apps/sim/scripts/test-search-google-content-e2e.ts b/apps/sim/scripts/test-search-google-content-e2e.ts index 3ba09f9282f..1bae2589c73 100644 --- a/apps/sim/scripts/test-search-google-content-e2e.ts +++ b/apps/sim/scripts/test-search-google-content-e2e.ts @@ -79,7 +79,7 @@ const mailbox: GmailMessage[] = Array.from({ length: 12 }, (_, index) => ({ body: { data: Buffer.from(`Synthetic evidence ${index}.`).toString('base64url') }, }, })) -const messageFailures = new Map() +const messageFailures = new Map() const mediaStarted = createDeferred() const mediaClosed = createDeferred() const server = http.createServer((request, response) => { @@ -93,13 +93,12 @@ const server = http.createServer((request, response) => { const row = mailbox.find((message) => message.id === id) const isThread = url.pathname.startsWith('/gmail/v1/users/me/threads/') const isAttachment = url.pathname.includes('/attachments/') - const failure = - !isThread && url.searchParams.get('format') === 'full' ? messageFailures.get(id) : undefined - if (failure) { - requests.push({ id, resource: 'message', status: failure }) + const failure = !isThread ? messageFailures.get(id) : undefined + if (failure && url.searchParams.get('format') === failure.format) { + requests.push({ id, resource: 'message', status: failure.status }) response - .writeHead(failure, { 'Content-Type': 'application/json' }) - .end(JSON.stringify({ error: { code: failure } })) + .writeHead(failure.status, { 'Content-Type': 'application/json' }) + .end(JSON.stringify({ error: { code: failure.status } })) return } const threadMessages = isThread ? mailbox.filter((message) => message.threadId === id) : [] @@ -303,68 +302,71 @@ try { { name: 'rate limit', failedId: 'reply', status: 429, expectedStatus: 'rate_limited' }, { name: 'server failure', failedId: 'reply', status: 500, expectedStatus: 'unavailable' }, ]) { - await check( - `Gmail conversation handles ${name} without hiding unrelated failures`, - async () => { - const before = mailbox.length - mailbox.push( - ...['anchor', 'retained', 'reply'].map((id, index) => ({ - id, - threadId: 'failure-conversation', - labelIds: ['INBOX'], - internalDate: String(Date.UTC(2026, 0, 1, 12, index)), - payload: { - mimeType: 'text/plain', - body: { data: Buffer.from(`Synthetic ${id} evidence.`).toString('base64url') }, - }, - })) - ) - if (failedId) messageFailures.set(failedId, status) - try { - const signal = AbortSignal.timeout(5000) - const api = createNativeClient({ origin, accessToken: TOKEN, signal }) - const policy = { - ...defaultLiveSearchPolicy('gmail'), - mode: 'selected' as const, - included: ['INBOX'], - } - const verify = createPolicyVerifier('gmail', policy, api, origin) - assert.ok(await verify({ id: 'anchor' })) - const reading = readGmail(api, 'anchor', { - policy, - signal, - verify: (reference) => verify(reference, reference.accessMetadata), - }) - if (expectedStatus) { - await assert.rejects( - reading, - (error: unknown) => - error instanceof NativeSearchError && error.status === expectedStatus + for (const format of failedId === 'reply' ? ['metadata', 'full'] : ['full']) { + await check( + `Gmail conversation handles ${name} during ${format} reads without hiding unrelated failures`, + async () => { + const before = mailbox.length + mailbox.push( + ...['anchor', 'retained', 'reply'].map((id, index) => ({ + id, + threadId: 'failure-conversation', + labelIds: ['INBOX'], + internalDate: String(Date.UTC(2026, 0, 1, 12, index)), + payload: { + mimeType: 'text/plain', + body: { data: Buffer.from(`Synthetic ${id} evidence.`).toString('base64url') }, + }, + })) + ) + if (failedId) messageFailures.set(failedId, { status, format }) + try { + const signal = AbortSignal.timeout(5000) + const api = createNativeClient({ origin, accessToken: TOKEN, signal }) + const policy = { + ...defaultLiveSearchPolicy('gmail'), + mode: 'selected' as const, + included: ['INBOX'], + } + const verify = createPolicyVerifier('gmail', policy, api, origin) + assert.ok(await verify({ id: 'anchor' })) + const reading = readGmail(api, 'anchor', { + policy, + signal, + verify: (reference) => + verify(reference, format === 'full' ? reference.accessMetadata : undefined), + }) + if (expectedStatus) { + await assert.rejects( + reading, + (error: unknown) => + error instanceof NativeSearchError && error.status === expectedStatus + ) + return + } + const document = await reading + assert.equal(document.id, 'anchor') + assert.ok(document.content.includes('Synthetic anchor evidence.')) + assert.ok(document.content.includes('Synthetic retained evidence.')) + assert.equal(document.content.includes('Synthetic reply evidence.'), status === 200) + assert.equal(document.content.includes('Coverage incomplete:'), status === 404) + assert.deepEqual( + document.accessDependencies, + status === 404 ? [{ id: 'retained' }] : [{ id: 'retained' }, { id: 'reply' }] ) - return + const current = createPolicyVerifier('gmail', policy, api, origin, undefined, { + fresh: true, + }) + assert.ok(await current({ id: document.id })) + for (const dependency of document.accessDependencies ?? []) + assert.ok(await current(dependency)) + } finally { + mailbox.splice(before) + messageFailures.clear() } - const document = await reading - assert.equal(document.id, 'anchor') - assert.ok(document.content.includes('Synthetic anchor evidence.')) - assert.ok(document.content.includes('Synthetic retained evidence.')) - assert.equal(document.content.includes('Synthetic reply evidence.'), status === 200) - assert.equal(document.content.includes('Coverage incomplete:'), status === 404) - assert.deepEqual( - document.accessDependencies, - status === 404 ? [{ id: 'retained' }] : [{ id: 'retained' }, { id: 'reply' }] - ) - const current = createPolicyVerifier('gmail', policy, api, origin, undefined, { - fresh: true, - }) - assert.ok(await current({ id: document.id })) - for (const dependency of document.accessDependencies ?? []) - assert.ok(await current(dependency)) - } finally { - mailbox.splice(before) - messageFailures.clear() } - } - ) + ) + } } await check( 'Gmail conversation and fresh selected-label checks fit one native request budget',