Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions apps/sim/lib/knowledge/__integration__/scale.integration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -368,9 +368,15 @@ describe.skipIf(!enabled)('knowledge scale: isolated real PostgreSQL, no provide
)
report['reconciliation.window.scan.plan'] = scanned['QUERY PLAN']
saveReport()
expect(planIndexNames('reconciliation.window.scan')).toContain(
'doc_connector_reconciliation_v2_idx'
)
/**
* Every keyset step is an ordered index probe: through the v2 index, or the primary key when
* this fixture's single connector is nearly the whole table. Either is a valid plan.
*/
const walkIndexes = new Set(planIndexNames('reconciliation.window.scan'))
expect(walkIndexes.size).toBeGreaterThan(0)
for (const name of walkIndexes) {
expect(['doc_connector_reconciliation_v2_idx', 'document_pkey']).toContain(name)
}
const [removed] = await db.execute(
sql`SELECT count(*)::int AS count FROM document WHERE connector_id = ${ids.connectorId} AND external_id::integer > ${rows - absentCount} AND deleted_at IS NOT NULL AND cardinality(acl) = 0`
)
Expand Down
9 changes: 5 additions & 4 deletions apps/sim/lib/sim-search/live/github.ts
Original file line number Diff line number Diff line change
Expand Up @@ -173,12 +173,13 @@ function groupGitHubText(query: string): string {
return [text ? `(${text})` : '', ...qualifiers].filter(Boolean).join(' ')
}

/** GitHub rejects more than 256 characters of search text; qualifiers do not count toward it. */
const GITHUB_TEXT_CHARACTERS = 256
const githubTextLength = (query: string) =>

/** Whether a query's search text exceeds GitHub's 256-character limit; qualifiers do not count toward it. */
export const exceedsGitHubTextLimit = (query: string) =>
githubTokens(query)
.filter((token) => !GITHUB_QUALIFIER.test(token))
.join(' ').length
.join(' ').length > GITHUB_TEXT_CHARACTERS

export async function searchGitHub(
client: NativeClient,
Expand Down Expand Up @@ -318,7 +319,7 @@ export async function searchGitHub(
? `${dateField}:<=${dates.end}`
: ''
const datedQuery = dateRange ? [groupGitHubText(text), dateRange].filter(Boolean).join(' ') : text
if (githubTextLength(text) > GITHUB_TEXT_CHARACTERS)
if (exceedsGitHubTextLimit(text))
throw new NativeSearchError(
'unavailable',
'GitHub search text is limited to 256 characters. Shorten the query.'
Expand Down
10 changes: 8 additions & 2 deletions apps/sim/scripts/test-search-discussions-live.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import {
type WorkspaceSearchFilters,
workspaceSearchFiltersSchema,
} from '@/lib/api/contracts/knowledge'
import { readGitHub, searchGitHub } from '@/lib/sim-search/live/github'
import { exceedsGitHubTextLimit, readGitHub, searchGitHub } from '@/lib/sim-search/live/github'
import { array, object, string } from '@/lib/sim-search/live/http'
import { defaultLiveSearchPolicy } from '@/lib/sim-search/live/policy-schema'
import type { NativeClient, NativePage } from '@/lib/sim-search/live/types'
Expand Down Expand Up @@ -478,7 +478,13 @@ try {
const payload: unknown = JSON.parse(buffer.toString('utf8', 0, length))
cases = queryCasesSchema.parse(payload)
for (const test of cases)
for (const query of [test.query, ...test.oracleQueries]) assertRepositoryScope(query)
for (const query of [test.query, ...test.oracleQueries]) {
assertRepositoryScope(query)
assert.ok(
!exceedsGitHubTextLimit(query),
`Case "${test.name}" exceeds GitHub's 256-character search text limit`
)
}
} finally {
await file.close()
}
Expand Down
Loading