From ed6f665224ba9829479d3a82cc8c304ee1e7bcfe Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 27 Sep 2026 18:38:37 -0700 Subject: [PATCH 1/2] test(knowledge): accept either ordered index in the scale window check and bound discussion case text --- .../knowledge/__integration__/scale.integration.ts | 11 ++++++++--- apps/sim/lib/sim-search/live/github.ts | 6 +++--- apps/sim/scripts/test-search-discussions-live.ts | 10 ++++++++-- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/apps/sim/lib/knowledge/__integration__/scale.integration.ts b/apps/sim/lib/knowledge/__integration__/scale.integration.ts index 72321f0fe94..a7bc7271952 100644 --- a/apps/sim/lib/knowledge/__integration__/scale.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/scale.integration.ts @@ -368,9 +368,14 @@ describe.skipIf(!enabled)('knowledge scale: isolated real PostgreSQL, no provide ) report['reconciliation.window.scan.plan'] = scanned['QUERY PLAN'] saveReport() - expect(planIndexNames('reconciliation.window.scan')).toContain( - 'doc_connector_reconciliation_v2_idx' - ) + /** + * Every keyset step is an ordered index probe: through the v2 index, or the primary key when + * this fixture's single connector is nearly the whole table. Either is a valid plan. + */ + const walkIndexes = new Set(planIndexNames('reconciliation.window.scan')) + expect(walkIndexes.size).toBeGreaterThan(0) + for (const name of walkIndexes) + expect(['doc_connector_reconciliation_v2_idx', 'document_pkey']).toContain(name) const [removed] = await db.execute( sql`SELECT count(*)::int AS count FROM document WHERE connector_id = ${ids.connectorId} AND external_id::integer > ${rows - absentCount} AND deleted_at IS NOT NULL AND cardinality(acl) = 0` ) diff --git a/apps/sim/lib/sim-search/live/github.ts b/apps/sim/lib/sim-search/live/github.ts index a5f69b4654a..124f46b079c 100644 --- a/apps/sim/lib/sim-search/live/github.ts +++ b/apps/sim/lib/sim-search/live/github.ts @@ -175,10 +175,10 @@ function groupGitHubText(query: string): string { /** GitHub rejects more than 256 characters of search text; qualifiers do not count toward it. */ const GITHUB_TEXT_CHARACTERS = 256 -const githubTextLength = (query: string) => +export const exceedsGitHubTextLimit = (query: string) => githubTokens(query) .filter((token) => !GITHUB_QUALIFIER.test(token)) - .join(' ').length + .join(' ').length > GITHUB_TEXT_CHARACTERS export async function searchGitHub( client: NativeClient, @@ -318,7 +318,7 @@ export async function searchGitHub( ? `${dateField}:<=${dates.end}` : '' const datedQuery = dateRange ? [groupGitHubText(text), dateRange].filter(Boolean).join(' ') : text - if (githubTextLength(text) > GITHUB_TEXT_CHARACTERS) + if (exceedsGitHubTextLimit(text)) throw new NativeSearchError( 'unavailable', 'GitHub search text is limited to 256 characters. Shorten the query.' diff --git a/apps/sim/scripts/test-search-discussions-live.ts b/apps/sim/scripts/test-search-discussions-live.ts index 310c1372875..8e708bf4565 100644 --- a/apps/sim/scripts/test-search-discussions-live.ts +++ b/apps/sim/scripts/test-search-discussions-live.ts @@ -11,7 +11,7 @@ import { type WorkspaceSearchFilters, workspaceSearchFiltersSchema, } from '@/lib/api/contracts/knowledge' -import { readGitHub, searchGitHub } from '@/lib/sim-search/live/github' +import { exceedsGitHubTextLimit, readGitHub, searchGitHub } from '@/lib/sim-search/live/github' import { array, object, string } from '@/lib/sim-search/live/http' import { defaultLiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' import type { NativeClient, NativePage } from '@/lib/sim-search/live/types' @@ -478,7 +478,13 @@ try { const payload: unknown = JSON.parse(buffer.toString('utf8', 0, length)) cases = queryCasesSchema.parse(payload) for (const test of cases) - for (const query of [test.query, ...test.oracleQueries]) assertRepositoryScope(query) + for (const query of [test.query, ...test.oracleQueries]) { + assertRepositoryScope(query) + assert.ok( + !exceedsGitHubTextLimit(query), + `Case "${test.name}" exceeds GitHub's 256-character search text limit` + ) + } } finally { await file.close() } From ec3efcb5765d253486763a18aca6b262505b4b37 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 27 Sep 2026 18:45:50 -0700 Subject: [PATCH 2/2] chore: document the GitHub text-limit helper and brace the scale plan assertion --- apps/sim/lib/knowledge/__integration__/scale.integration.ts | 3 ++- apps/sim/lib/sim-search/live/github.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/sim/lib/knowledge/__integration__/scale.integration.ts b/apps/sim/lib/knowledge/__integration__/scale.integration.ts index a7bc7271952..a9bebc84d4b 100644 --- a/apps/sim/lib/knowledge/__integration__/scale.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/scale.integration.ts @@ -374,8 +374,9 @@ describe.skipIf(!enabled)('knowledge scale: isolated real PostgreSQL, no provide */ const walkIndexes = new Set(planIndexNames('reconciliation.window.scan')) expect(walkIndexes.size).toBeGreaterThan(0) - for (const name of walkIndexes) + for (const name of walkIndexes) { expect(['doc_connector_reconciliation_v2_idx', 'document_pkey']).toContain(name) + } const [removed] = await db.execute( sql`SELECT count(*)::int AS count FROM document WHERE connector_id = ${ids.connectorId} AND external_id::integer > ${rows - absentCount} AND deleted_at IS NOT NULL AND cardinality(acl) = 0` ) diff --git a/apps/sim/lib/sim-search/live/github.ts b/apps/sim/lib/sim-search/live/github.ts index 124f46b079c..92efd385e48 100644 --- a/apps/sim/lib/sim-search/live/github.ts +++ b/apps/sim/lib/sim-search/live/github.ts @@ -173,8 +173,9 @@ function groupGitHubText(query: string): string { return [text ? `(${text})` : '', ...qualifiers].filter(Boolean).join(' ') } -/** GitHub rejects more than 256 characters of search text; qualifiers do not count toward it. */ const GITHUB_TEXT_CHARACTERS = 256 + +/** Whether a query's search text exceeds GitHub's 256-character limit; qualifiers do not count toward it. */ export const exceedsGitHubTextLimit = (query: string) => githubTokens(query) .filter((token) => !GITHUB_QUALIFIER.test(token))