fix(knowledge): keep a Google Workspace user's visible documents until a missing service outlasts propagation

Google applies service and organizational-unit changes within 24 hours, so a missing
mailbox or notACalendarUser can be transient for a user whose documents are already
indexed, and a mid-listing answer does not prove the whole account lacks the service.

- The scheduler skips a service-not-enabled user only on their first provider page, and
  only when readers see none of their documents or the condition was first observed at
  least 24 hours ago. Otherwise it is a retained failure, as before, whose first
  observation is kept in the partition failure; after 24 hours a mid-listing user
  restarts from their first page.
- Permission passes skip on the first page, since a retained failure refreshes nothing.
- Gmail Directory enumeration keeps scheduling a user without a mailbox while readers
  still see their mail; the crawl reports the missing mailbox to the scheduler instead of
  completing the user.
- Visibility is read through doc_acl_gin_idx for the user's token behind an OFFSET 0
  fence, bounded by that user's grants.

Author: waleedlatif1

apps/sim/connectors/google-workspace/company-crawl.test.ts

@@ -2,9 +2,11 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
 import { GoogleApiError, readGoogleApiError } from '@/connectors/google-workspace/api-errors'
 import {
+  GoogleWorkspaceMailboxNotSetup,
   getGoogleWorkspaceDocument,
   InvalidGoogleWorkspaceCursor,
   listGoogleWorkspaceDocuments,
+  serviceNotEnabledFailure,
   validateGoogleWorkspaceConfig,
 } from '@/connectors/google-workspace/company-crawl'
 import type { ConnectorConfig, ExternalDocument } from '@/connectors/types'
@@ -483,18 +485,18 @@ describe('Google Workspace per-user central crawl', () => {
     }
   )
 
-  it('skips an explicitly unprovisioned Gmail mailbox without a token or a listing failure', async () => {
+  it('leaves an explicitly unprovisioned Gmail mailbox to the scheduler before requesting a token', async () => {
     directory([USER('alice', undefined, { isMailboxSetup: false }), USER('bob')])
     const ctx: Record<string, unknown> = context()
-    const first = await list(ctx)
-    expect(first).toMatchObject({ documents: [], hasMore: true })
-    expect(first.listingFailures).toBeUndefined()
-    expect(first.reconciliationSafe).toBeUndefined()
+    const error = await list(ctx).catch((caught: unknown) => caught)
+    expect(error).toBeInstanceOf(GoogleWorkspaceMailboxNotSetup)
+    expect(serviceNotEnabledFailure(error)).toEqual({
+      operation: 'directory.users.get',
+      reasons: ['mailboxNotSetup'],
+    })
     expect(ctx.reconciliationUnsafe).toBeUndefined()
     expect(ctx.getDelegatedAccessToken).not.toHaveBeenCalled()
-    const second = await list(context(), first.nextCursor)
-    expect(second.documents[0].acl).toEqual(['u:bob@corp.com'])
-    expect(second.listingFailures).toBeUndefined()
+    expect(listUserDocuments).not.toHaveBeenCalled()
   })
 
   it('does not use Gmail mailbox eligibility for Calendar', async () => {

apps/sim/connectors/google-workspace/company-crawl.ts

@@ -161,30 +161,43 @@ function ownerDocument(document: ExternalDocument, access: DelegatedUser): Exter
   return { ...document, acl: [`u:${access.user.email}`] }
 }
 
+/** A Directory user without a Gmail mailbox; the user scheduler decides whether to skip them. */
+export class GoogleWorkspaceMailboxNotSetup extends Error {
+  constructor() {
+    super('Google Workspace user has no Gmail mailbox')
+    this.name = 'GoogleWorkspaceMailboxNotSetup'
+  }
+}
+
 /**
- * Calendar answers an account without the Calendar service with exactly this reason. That is a
- * standing property of the account, not a listing failure, so the user scheduler skips it.
+ * Evidence that a user lacks the service itself: no Gmail mailbox, or a Calendar list 403 whose
+ * only reason is `notACalendarUser`. Admin changes to a user's services can take up to a day to
+ * settle, so the user scheduler, which can see indexed documents, decides when this is a skip.
  */
-export function isGoogleWorkspaceServiceNotEnabled(error: unknown): boolean {
-  return (
-    error instanceof GoogleApiError &&
+export function serviceNotEnabledFailure(
+  error: unknown
+): Omit<ExternalListingFailures['samples'][number], 'scope'> | null {
+  if (error instanceof GoogleWorkspaceMailboxNotSetup)
+    return { operation: 'directory.users.get', reasons: ['mailboxNotSetup'] }
+  return error instanceof GoogleApiError &&
     error.reasonsComplete &&
     error.status === 403 &&
     error.diagnostic?.operation === 'calendar.events.list' &&
     error.diagnostic.reasons.length === 1 &&
     error.diagnostic.reasons[0] === 'notACalendarUser'
-  )
+    ? { operation: 'calendar.events.list', status: 403, reasons: ['notACalendarUser'] }
+    : null
 }
 
-/** Isolates narrow user-list failures; a missing Calendar service propagates for the scheduler to skip. */
+/** Isolates narrow user-list failures; a missing Calendar service propagates to the scheduler. */
 function userListingFailure(
   error: unknown,
   provider: GoogleWorkspaceProvider
 ): Omit<ExternalListingFailures['samples'][number], 'scope'> | null {
   if (!(error instanceof GoogleApiError) || !error.diagnostic || !error.reasonsComplete) return null
   const reasons = error.diagnostic.reasons
   const isolated =
-    !isGoogleWorkspaceServiceNotEnabled(error) &&
+    !serviceNotEnabledFailure(error) &&
     (provider === 'gmail'
       ? error.diagnostic.operation === 'gmail.threads.list' &&
         error.status === 400 &&
@@ -333,8 +346,8 @@ export async function listGoogleWorkspaceDocuments(
     })
     return emptyPage(advance())
   }
-  /** Directory refresh stops scheduling users without a mailbox; a partition queued earlier just completes. */
-  if (provider === 'gmail' && user.isMailboxSetup === false) return emptyPage(advance())
+  if (provider === 'gmail' && user.isMailboxSetup === false)
+    throw new GoogleWorkspaceMailboxNotSetup()
   const access: PageAccess = {
     provider,
     user,

apps/sim/connectors/listing-failures.ts

@@ -4,17 +4,17 @@ import { CONNECTOR_SOURCE_REASON_STATES } from '@/connectors/source-error'
 export const MAX_LISTING_FAILURE_SAMPLES = 10
 
 /** Persist only bounded scope identifiers and provider-owned codes, never raw errors or content. */
+export const listingFailureSampleSchema = z.object({
+  scope: z.string().min(1).max(254),
+  operation: z.string().min(1).max(96),
+  status: z.number().int().min(100).max(599).optional(),
+  reasons: z.array(z.string().min(1).max(64)).max(16),
+  reasonState: z.enum(CONNECTOR_SOURCE_REASON_STATES).optional(),
+  /** When a standing account condition was first observed, bounding how long it is retained. */
+  since: z.string().datetime().optional(),
+})
+
 export const listingFailuresSchema = z.object({
   count: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
-  samples: z
-    .array(
-      z.object({
-        scope: z.string().min(1).max(254),
-        operation: z.string().min(1).max(96),
-        status: z.number().int().min(100).max(599).optional(),
-        reasons: z.array(z.string().min(1).max(64)).max(16),
-        reasonState: z.enum(CONNECTOR_SOURCE_REASON_STATES).optional(),
-      })
-    )
-    .max(MAX_LISTING_FAILURE_SAMPLES),
+  samples: z.array(listingFailureSampleSchema).max(MAX_LISTING_FAILURE_SAMPLES),
 })

apps/sim/connectors/types.ts

@@ -205,6 +205,8 @@ export interface ExternalListingFailures {
     status?: number
     reasons: string[]
     reasonState?: ConnectorSourceReasonState
+    /** When a standing account condition was first observed, bounding how long it is retained. */
+    since?: string
   }[]
 }
 

apps/sim/lib/knowledge/connectors/google-company-scheduler.test.ts

@@ -15,7 +15,10 @@ import type {
 } from '@/lib/knowledge/connectors/partition-work'
 import { GoogleDriveApiError } from '@/connectors/google-drive/google-drive-errors'
 import { GoogleApiError } from '@/connectors/google-workspace/api-errors'
-import { listGoogleWorkspaceDocuments } from '@/connectors/google-workspace/company-crawl'
+import {
+  GoogleWorkspaceMailboxNotSetup,
+  listGoogleWorkspaceDocuments,
+} from '@/connectors/google-workspace/company-crawl'
 import { googleCompanyUserContextSchema } from '@/connectors/google-workspace/company-work'
 import type { GoogleWorkspaceUser } from '@/connectors/google-workspace/users'
 import { ConnectorSourceError } from '@/connectors/source-error'
@@ -142,10 +145,12 @@ function fixture(provider = 'google_calendar', syncIntervalMinutes = 60) {
     documents: [document],
     hasMore: false,
   }))
+  const visible = vi.fn(async (_user: GoogleWorkspaceUser) => false)
   let scheduler = createGoogleCompanyScheduler({
     provider,
     store,
     listDocuments: list,
+    hasVisibleDocuments: visible,
     syncIntervalMinutes,
     isListingCursorInvalidError: (error) => error === expiredCursor,
     now: () => clock,
@@ -182,6 +187,7 @@ function fixture(provider = 'google_calendar', syncIntervalMinutes = 60) {
   return {
     rows,
     list,
+    visible,
     process,
     step,
     saved: () => saved,
@@ -196,6 +202,7 @@ function fixture(provider = 'google_calendar', syncIntervalMinutes = 60) {
         provider,
         store,
         listDocuments: list,
+        hasVisibleDocuments: visible,
         syncIntervalMinutes,
         isListingCursorInvalidError: (error) => error === expiredCursor,
         now: () => clock,
@@ -324,6 +331,8 @@ describe('durable Google company user scheduling', () => {
   it('refreshes permissions for a user without the Calendar service at the permission cadence', async () => {
     mocks.directory.mockResolvedValue({ users: [user('a'), user('z')] })
     const f = fixture()
+    /** Skipping a permission pass hides nothing a retained failure would keep visible. */
+    f.visible.mockResolvedValue(true)
     f.list.mockResolvedValue({ documents: [document], hasMore: true, nextCursor: 'next-page' })
     await f.step(2)
     f.advance(13 * 60 * 60 * 1000)
@@ -342,22 +351,101 @@ describe('durable Google company user scheduling', () => {
   })
 
   it.each([
-    ['gmail', false],
-    ['google_calendar', true],
+    ['gmail', false, false],
+    ['gmail', true, true],
+    ['google_calendar', false, true],
   ] as const)(
-    'for %s, schedules a Directory user without a mailbox: %s',
-    async (provider, scheduled) => {
+    'for %s with visible documents %s, schedules a Directory user without a mailbox: %s',
+    async (provider, hasVisibleDocuments, scheduled) => {
       mocks.directory.mockResolvedValue({
         users: [user('a', { isMailboxSetup: false }), user('z')],
       })
       const f = fixture(provider)
+      f.visible.mockResolvedValue(hasVisibleDocuments)
       await f.step(4)
       expect(f.rows.has('a:content')).toBe(scheduled)
       expect(f.rows.get('z:content')?.complete).toBe(true)
       expect(f.saved()).toMatchObject({ unsafe: false, listingFailures: null })
     }
   )
 
+  it.each([
+    {
+      provider: 'google_calendar',
+      error: () => new GoogleApiError('calendar.events.list', 403, ['notACalendarUser']),
+      reason: { operation: 'calendar.events.list', status: 403, reasons: ['notACalendarUser'] },
+    },
+    {
+      provider: 'gmail',
+      error: () => new GoogleWorkspaceMailboxNotSetup(),
+      reason: { operation: 'directory.users.get', reasons: ['mailboxNotSetup'] },
+    },
+  ])(
+    'retains a $provider user whose documents readers still see until the condition outlasts propagation',
+    async ({ provider, error, reason }) => {
+      mocks.directory.mockResolvedValue({ users: [user('a')] })
+      const f = fixture(provider, 15)
+      f.visible.mockResolvedValue(true)
+      f.list.mockImplementation(async () => {
+        throw error()
+      })
+      await f.step(5)
+      const retained = {
+        complete: false,
+        failure: { scope: 'a@fixture.test', ...reason, since: '2026-09-17T00:00:00.000Z' },
+      }
+      expect(f.rows.get('a:content')).toMatchObject({ ...retained, attempts: 1 })
+      expect(f.visible).toHaveBeenCalledWith(expect.objectContaining({ id: 'a' }))
+      expect(f.saved()).toMatchObject({
+        complete: false,
+        unsafe: true,
+        listingFailures: { count: 1 },
+      })
+
+      f.advance(23 * 60 * 60 * 1000)
+      f.restart()
+      await f.step(5)
+      expect(f.rows.get('a:content')).toMatchObject({ ...retained, attempts: 2 })
+      expect(f.visible).toHaveBeenCalledOnce()
+      expect(f.saved().complete).toBe(false)
+
+      f.advance(60 * 60 * 1000)
+      f.restart()
+      await f.step(5)
+      expect(f.rows.get('a:content')).toMatchObject({ complete: true, attempts: 0 })
+      expect(f.rows.get('a:content')?.failure).toBeUndefined()
+      expect(f.saved()).toMatchObject({ complete: true, listingFailures: null })
+    }
+  )
+
+  it('retains a service-not-enabled answer after the first page and restarts the user once it persists', async () => {
+    mocks.directory.mockResolvedValue({ users: [user('a')] })
+    const f = fixture()
+    f.list
+      .mockResolvedValueOnce({ documents: [document], hasMore: true, nextCursor: 'page-2' })
+      .mockRejectedValue(new GoogleApiError('calendar.events.list', 403, ['notACalendarUser']))
+    await f.step(5)
+    expect(f.list.mock.calls.at(-1)?.[2]).toBe('page-2')
+    expect(f.rows.get('a:content')).toMatchObject({
+      complete: false,
+      cursor: 'page-2',
+      failure: { reasons: ['notACalendarUser'], since: '2026-09-17T00:00:00.000Z' },
+    })
+    expect(f.visible).not.toHaveBeenCalled()
+
+    f.advance(25 * 60 * 60 * 1000)
+    f.restart()
+    await f.step(5)
+    expect(f.rows.get('a:content')).toMatchObject({ complete: false })
+    expect(f.rows.get('a:content')?.cursor).toContain('google-workspace:v1:')
+
+    f.advance(60 * 60 * 1000)
+    f.restart()
+    await f.step(5)
+    expect(f.rows.get('a:content')).toMatchObject({ complete: true })
+    expect(f.rows.get('a:content')?.failure).toBeUndefined()
+  })
+
   it('picks up a Gmail user on the Directory refresh after their mailbox is provisioned', async () => {
     mocks.directory
       .mockResolvedValueOnce({ users: [user('a', { isMailboxSetup: false }), user('z')] })