From 6f955ff446793ee1ae2b4edcabd9b802abb29fa9 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 10 Sep 2026 12:26:21 -0400 Subject: [PATCH 1/5] Give up a from-scratch index after consecutive idle render timeouts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A render that times out while its page is idle — main thread responsive, no script running, nothing fetching, no module import or document load in flight — was waiting on nothing, and nothing about the next file changes that. In a host CI shard whose prerender pages could not reach the icons server, every base module render timed out that way at about 72 s apiece; the 267-file pass would have needed five hours, the job would have been rejected at its 3600 s limit with nothing written, and until then nothing said what was wrong. The from-scratch visit loop now counts consecutive idle timeouts, recognised from the diagnostics the prerender server captured as the timer fired, and after three throws past the per-file error isolation with a message naming the files and the likely cause: an origin the page needs is unreachable from the browser. The job is rejected within minutes carrying that reason. Incremental passes are unchanged, since their rows are the only record of a user's write. INDEX_IDLE_RENDER_TIMEOUT_ABORT_AFTER overrides the threshold; 0 disables the guard. Co-Authored-By: Claude Fable 5.1 --- packages/runtime-common/index-runner.ts | 96 +++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 5810a359ecd..6359dbbdf25 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -63,6 +63,65 @@ type VisitRenderOutcome = | { status: 'skipped' } | { status: 'error'; error: unknown }; +// A render that times out while its page is idle — main thread responsive, no +// script running, nothing fetching, no module import or document load in +// flight — was waiting on nothing, and nothing about the next file changes +// that. Several in a row therefore describe the stack rather than the files: +// an origin the page needs (the host bundle, the realm-server, the icons +// server) is unreachable from the browser, and every remaining file would +// cost the full render timeout to fail the same way. A from-scratch pass gives +// up after this many consecutive idle timeouts, so the job fails within +// minutes naming the cause instead of grinding to its wall-clock limit and +// being rejected with nothing written. 0 disables the guard. +const DEFAULT_IDLE_RENDER_TIMEOUT_ABORT_AFTER = 3; +const envIdleRenderTimeoutAbortAfter = Number( + ( + globalThis as { + process?: { env?: Record }; + } + ).process?.env?.INDEX_IDLE_RENDER_TIMEOUT_ABORT_AFTER, +); +export const IDLE_RENDER_TIMEOUT_ABORT_AFTER = + Number.isFinite(envIdleRenderTimeoutAbortAfter) && + envIdleRenderTimeoutAbortAfter >= 0 + ? envIdleRenderTimeoutAbortAfter + : DEFAULT_IDLE_RENDER_TIMEOUT_ABORT_AFTER; + +// Script-busy fraction below which the page counts as idle. The CPU sample +// covers a short window, so a page that is doing nothing useful can still show +// a few percent of housekeeping. +const IDLE_SCRIPT_BUSY_MAX = 0.05; + +const RENDER_TIMEOUT_TITLE = 'Render timeout'; + +// Whether a visit's render timed out with nothing in flight. The timeout error +// carries the diagnostics the prerender server captured as it fired (see +// RenderTimeoutDiagnostics), flattened onto the visit's `diagnostics`. A +// timeout whose diagnostics are missing, or show work in progress, is a slow +// or stuck render rather than an idle one and does not count. +export function isIdleRenderTimeout(result: IndexVisitRenderResult): boolean { + let errors = [ + result.pageUnusableError, + result.card?.error, + result.fileExtract?.error, + result.fileRender?.error, + ]; + if (!errors.some((e) => e?.error?.title === RENDER_TIMEOUT_TITLE)) { + return false; + } + let d = result.diagnostics; + if (!d || d.mainThreadResponsive !== true) { + return false; + } + return ( + (d.scriptBusyFraction ?? 0) < IDLE_SCRIPT_BUSY_MAX && + (d.pendingNetworkRequests?.length ?? 0) === 0 && + (d.inFlightModuleImports?.length ?? 0) === 0 && + (d.cardDocsInFlight?.length ?? 0) === 0 && + (d.fileMetaDocsInFlight?.length ?? 0) === 0 + ); +} + export class IndexRunner { #indexingInstances = new Map>(); #reader: Reader; @@ -123,6 +182,7 @@ export class IndexRunner { // warm loader ownership — intended). Populated in the constructor after // jobInfo is known so the id is easy to correlate with a job in logs. #batchId!: string; + #idleRenderTimeoutAbortAfter: number; constructor({ realmURL, @@ -140,6 +200,7 @@ export class IndexRunner { auth, fetch, realmOwnerUserId, + idleRenderTimeoutAbortAfter = IDLE_RENDER_TIMEOUT_ABORT_AFTER, }: { realmURL: URL; reader: Reader; @@ -151,6 +212,9 @@ export class IndexRunner { auth: string; fetch: typeof globalThis.fetch; realmOwnerUserId: string; + // Consecutive idle render timeouts after which a from-scratch pass gives + // up; see IDLE_RENDER_TIMEOUT_ABORT_AFTER. 0 disables the guard. + idleRenderTimeoutAbortAfter?: number; jobInfo?: JobInfo; // Optional override of `jobInfo.priority`. When both are present, // `jobPriority` wins — this is the path the worker handler takes @@ -189,6 +253,7 @@ export class IndexRunner { this.#auth = auth; this.#fetch = fetch; this.#realmOwnerUserId = realmOwnerUserId; + this.#idleRenderTimeoutAbortAfter = idleRenderTimeoutAbortAfter; this.#definitionLookup = definitionLookup; this.#dependencyResolver = new IndexRunnerDependencyManager({ realmURL: this.#realmURL, @@ -300,6 +365,7 @@ export class IndexRunner { let resumedSkipped = 0; try { await current.#runVisitLoop(invalidations, { + abortAfterIdleRenderTimeouts: current.#idleRenderTimeoutAbortAfter, // Resume guard. If a previous attempt of this same job already wrote // URL_X to the working table AND the EFS mtime hasn't changed since, // skip the visit — the existing working row is still authoritative @@ -709,14 +775,21 @@ export class IndexRunner { skipReason, onSkip, onVisited, + abortAfterIdleRenderTimeouts = 0, }: { skipReason: (url: URL) => 'resumed' | 'delete' | undefined; onSkip: (url: URL, reason: 'resumed' | 'delete') => void; onVisited: (url: URL) => void; + // When set, the pass throws — abandoning the batch — once this many + // consecutive visits time out idle (see isIdleRenderTimeout). A + // from-scratch pass sets it; an incremental pass, whose rows are the + // only record of a write, does not. + abortAfterIdleRenderTimeouts?: number; }, ): Promise { let n = invalidations.length; let renders = new Map>(); + let idleTimeouts: string[] = []; // Start the render for the next non-skipped URL at or after `from`, // keeping exactly one render in flight ahead of the finish cursor. let prefetch = (from: number) => { @@ -757,6 +830,29 @@ export class IndexRunner { await this.#handleVisitError(url, err); } onVisited(url); + if (abortAfterIdleRenderTimeouts > 0) { + if ( + outcome.status === 'rendered' && + isIdleRenderTimeout(outcome.result) + ) { + idleTimeouts.push(url.href); + } else { + idleTimeouts = []; + } + if (idleTimeouts.length >= abortAfterIdleRenderTimeouts) { + // Thrown past #handleVisitError on purpose: this is not one file's + // failure to isolate but the pass's inability to render anything, + // and the job's rejection is what carries that to whoever awaits + // it (readiness, a publish, an operator). + let message = + `${jobIdentity(this.#jobInfo)} giving up: the last ${idleTimeouts.length} renders each timed out with the page idle ` + + `(main thread responsive, no script running, nothing fetching, no module import in flight), so the page was ` + + `waiting on nothing and no later file can render either. Check that the prerender's browser can reach the ` + + `host, realm-server and icons origins. Files: ${idleTimeouts.join(', ')}`; + this.#log.error(message); + throw new Error(message); + } + } } } From a2fbe6e9e5acc693415fa0a34e998187aeb9505d Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 10 Sep 2026 12:26:28 -0400 Subject: [PATCH 2/5] Report a failed boot index as a terminal readiness stage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A brand-new realm awaits its first from-scratch index at startup, and when that job fails the failure is logged and swallowed: startup completes, the index lane is clear, and `_readiness-check` answers 200 over an empty index. A caller waiting on the realm — a publish, the CI realm wait — is handed a realm that serves nothing, and every test that follows fails for a reason two steps removed from the cause. The realm now keeps the failure the awaited boot index returned and answers readiness with 503 `X-Boxel-Not-Ready: index-failed`, the failure in the body and no `Retry-After`, since no amount of polling cures it; a later full index that completes clears the state. The publish flow's poll and the CI wait task both stop on that stage and report the reason. A test drives a realm whose every render times out idle through startup and checks that the job gave up after three visits, was rejected naming the idle timeouts, and that readiness reports the terminal stage with the failure in its body. Co-Authored-By: Claude Fable 5.1 --- docs/realm-server-health-signals.md | 18 +- mise-tasks/ci/wait-for-realms | 15 +- .../realm-endpoints/readiness-check-test.ts | 191 ++++++++++++++++++ .../runtime-common/realm-index-updater.ts | 13 +- packages/runtime-common/realm-operations.ts | 13 ++ packages/runtime-common/realm.ts | 32 ++- 6 files changed, 267 insertions(+), 15 deletions(-) diff --git a/docs/realm-server-health-signals.md b/docs/realm-server-health-signals.md index f12a83845ce..c168f897361 100644 --- a/docs/realm-server-health-signals.md +++ b/docs/realm-server-health-signals.md @@ -95,11 +95,19 @@ no per-request timeout of their own. Blocking is therefore the correct behavior, not a symptom. The endpoint holds the request open while work is outstanding and answers 503 with `Retry-After` plus an -`X-Boxel-Not-Ready: index | prerender-html` header naming the stage. Every caller -retries on a non-ok status, so a 503 costs a poll rather than an error — which is -also why the hold is short by design: a hold that outlives a caller's own deadline -converts its poll loop into a single failed attempt, so the budget stays under the -shortest deadline any caller brings. +`X-Boxel-Not-Ready: startup | index | prerender-html` header naming the stage. +Every caller retries on a non-ok status, so a 503 costs a poll rather than an +error — which is also why the hold is short by design: a hold that outlives a +caller's own deadline converts its poll loop into a single failed attempt, so the +budget stays under the shortest deadline any caller brings. + +One stage is terminal. A brand-new realm whose first from-scratch index failed — +the worker gave up, or the job hit its wall-clock limit — is mounted over an empty +index. Readiness answers 503 with `X-Boxel-Not-Ready: index-failed`, the failure +in the body, and no `Retry-After`: a false ready would hand the caller a realm that +serves nothing, and `index` would keep it polling for work that is not coming. +The publish flow's poll and the CI realm wait both stop on it. A later full index +that completes clears the state. The gating has to read shared state, because in a multi-replica deployment the poll need not reach the replica that did the work. In-process indexing state is diff --git a/mise-tasks/ci/wait-for-realms b/mise-tasks/ci/wait-for-realms index c60070e72ab..24a355dc46c 100755 --- a/mise-tasks/ci/wait-for-realms +++ b/mise-tasks/ci/wait-for-realms @@ -32,12 +32,14 @@ # body — which is what tells a Traefik-originated `404 page not found` apart # from a realm-server 404. # -# Three conditions end the wait early instead of running the whole budget +# Four conditions end the wait early instead of running the whole budget # down against an environment that can no longer become ready: # - the service stack process has exited; # - a host that the realm-server had been answering for now gets Traefik's # own 404, i.e. its route file was removed (the realm-server deregisters # every route for the environment on shutdown); +# - the realm reports that its boot index failed (`X-Boxel-Not-Ready: +# index-failed`), which no amount of waiting cures; # - the time budget is spent. # Each failure prints the failing URL's response headers and the Traefik # diagnostics (ci:traefik-diagnostics) before exiting, while the stack is @@ -140,7 +142,13 @@ probe() { describe() { case "$code" in 200) printf '200' ;; - 503) printf '503 not-ready=%s' "${not_ready:-unknown}" ;; + 503) + if [ "$not_ready" = "index-failed" ]; then + printf '503 not-ready=index-failed (%s)' "$body_line" + else + printf '503 not-ready=%s' "${not_ready:-unknown}" + fi + ;; 000) printf 'no response (%s)' "$body_line" ;; *) printf '%s %s "%s"' "$code" "${ctype:-no-content-type}" "$body_line" ;; esac @@ -230,6 +238,9 @@ wait_for_200() { return 0 fi echo "t+$(elapsed)s ${label} -> $(describe)" + if [ "$code" = "503" ] && [ "$not_ready" = "index-failed" ]; then + fail "$url" "${label}: the realm's boot index failed, so it cannot become ready without a restart — ${body_line}" + fi if realm_server_answered; then mark_routed "$host" elif is_traefik_404 && was_routed "$host"; then diff --git a/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts b/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts index f7403c59b3a..099a93f621d 100644 --- a/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts +++ b/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts @@ -6,12 +6,17 @@ import fsExtra from 'fs-extra'; const { ensureDirSync, writeJSONSync } = fsExtra; import { dirSync } from 'tmp'; import type { + LooseSingleCardDocument, + Prerenderer, Realm, + RenderError, + RenderVisitResponse, QueuePublisher, QueueRunner, } from '@cardstack/runtime-common'; import { CachingDefinitionLookup, + rri, SupportedMimeType, } from '@cardstack/runtime-common'; import { indexingConcurrencyGroup } from '@cardstack/runtime-common/jobs/indexing'; @@ -209,4 +214,190 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { ); }); }); + + // A brand-new realm whose first from-scratch index cannot complete is the + // other way a readiness poll could run forever. Here every render times out + // with the page idle — the shape a prerender takes when its browser cannot + // reach an origin it needs — so the job gives up after a few files instead + // of paying the full render timeout for every file, and the realm reports + // that as a terminal not-ready rather than a ready over an empty index. + module('boot index whose renders all time out idle', function (hooks) { + let dbAdapter: PgAdapter; + let publisher: QueuePublisher; + let runner: QueueRunner; + + setupDB(hooks, { + beforeEach: async (adapter, pub, run) => { + dbAdapter = adapter; + publisher = pub; + runner = run; + }, + }); + + const realmURL = 'http://127.0.0.1:6678/idle-timeouts/'; + const cardCount = 6; + + // The response the prerender server sends for a render that hit its + // timeout while waiting on nothing: a `Render timeout` error on the pass + // and on `pageUnusableError`, with the diagnostics captured as the timer + // fired showing a responsive, idle page. + function idleTimeoutVisitResponse(url: string): RenderVisitResponse { + let timeout: RenderError = { + type: 'instance-error', + error: { + id: url, + status: 504, + title: 'Render timeout', + message: 'Render timed-out after 60000 ms', + additionalErrors: null, + }, + evict: true, + }; + return { + card: { + serialized: null, + searchDoc: null, + displayNames: null, + deps: null, + types: null, + isolatedHTML: null, + headHTML: null, + atomHTML: null, + embeddedHTML: null, + fittedHTML: null, + iconHTML: null, + markdown: null, + error: timeout, + }, + pageUnusableError: timeout, + meta: { + diagnostics: { + mainThreadResponsive: true, + scriptBusyFraction: 0, + pendingNetworkRequests: [], + inFlightModuleImports: [], + cardDocsInFlight: [], + fileMetaDocsInFlight: [], + }, + }, + }; + } + + test('gives up after a few idle timeouts and reports the failure as terminal', async function (assert) { + let real = await getTestPrerenderer(); + let indexVisits: string[] = []; + let prerenderer: Prerenderer = { + prerenderModule: (args) => real.prerenderModule(args), + runCommand: (args) => real.runCommand(args), + releaseBatch: async () => {}, + prerenderVisit: async (args) => { + if (args.visitType === 'index') { + indexVisits.push(args.url); + } + return idleTimeoutVisitResponse(args.url); + }, + }; + + let fileSystem: Record = { + 'realm.json': { + data: { + type: 'card', + attributes: { cardInfo: { name: 'Idle Timeouts Realm' } }, + meta: { + adoptsFrom: { + module: rri('@cardstack/base/realm-config'), + name: 'RealmConfig', + }, + }, + }, + }, + }; + for (let i = 0; i < cardCount; i++) { + fileSystem[`card-${i}.json`] = { + data: { + type: 'card', + attributes: { title: `Card ${i}` }, + meta: { + adoptsFrom: { + module: rri('https://cardstack.com/base/card-api'), + name: 'CardDef', + }, + }, + }, + }; + } + + let virtualNetwork = createVirtualNetwork(); + let definitionLookup = new CachingDefinitionLookup( + dbAdapter, + real, + virtualNetwork, + testCreatePrerenderAuth, + ); + let dir = join(dirSync().name, 'idle-timeouts'); + ensureDirSync(dir); + let { realm } = await createRealm({ + dir, + fileSystem, + definitionLookup, + realmURL, + permissions: { '*': ['read'] }, + virtualNetwork, + publisher, + runner, + dbAdapter, + withWorker: true, + prerenderer, + }); + virtualNetwork.mount(realm.handle); + + // Startup awaits a brand-new index's from-scratch job; the job's failure + // is swallowed there, so this resolves either way. + await realm.start(); + + // Three consecutive idle timeouts end the pass; the render-ahead loop + // may have started one more visit before the third was finished. + assert.true( + indexVisits.length >= 3, + `made at least the three visits it takes to give up (made ${indexVisits.length})`, + ); + assert.true( + indexVisits.length <= 4, + `gave up after ${indexVisits.length} index visits rather than visiting all ${cardCount} files`, + ); + + let [job] = (await dbAdapter.execute( + `SELECT status, result FROM jobs + WHERE job_type = 'from-scratch-index' AND concurrency_group = $1`, + { bind: [indexingConcurrencyGroup(realm.url)] }, + )) as { status: string; result: unknown }[]; + assert.strictEqual(job.status, 'rejected', 'the job was rejected'); + assert.true( + JSON.stringify(job.result).includes('timed out with the page idle'), + 'the rejection names the idle timeouts as the reason', + ); + + let response = await realm.handle( + new Request(`${realmURL}_readiness-check`, { + headers: { Accept: SupportedMimeType.RealmInfo }, + }), + ); + assert.strictEqual(response!.status, 503, 'reports not-ready'); + assert.strictEqual( + response!.headers.get('X-Boxel-Not-Ready'), + 'index-failed', + 'names the failed boot index as the stage', + ); + assert.strictEqual( + response!.headers.get('Retry-After'), + null, + 'carries no retry hint: the state is terminal', + ); + let body = await response!.text(); + assert.true( + body.includes('timed out with the page idle'), + `the body carries the failure: ${body}`, + ); + }); + }); }); diff --git a/packages/runtime-common/realm-index-updater.ts b/packages/runtime-common/realm-index-updater.ts index 947b0c0cd2e..b51834ab186 100644 --- a/packages/runtime-common/realm-index-updater.ts +++ b/packages/runtime-common/realm-index-updater.ts @@ -201,14 +201,21 @@ export class RealmIndexUpdater { }; } - async fullIndex(priority = systemInitiatedPriority) { + // Resolves to the failure when the job rejected and to undefined when it + // completed. The failure is returned rather than thrown so a caller that + // does not wait on the result — a bootstrap realm's per-boot reindex — is + // unaffected, while a startup that awaits a brand-new index can record that + // the index was never built. + async fullIndex( + priority = systemInitiatedPriority, + ): Promise { let { completed } = this.publishFullIndex(priority); try { await completed; + return undefined; } catch (e: any) { this.#log.error(`Error running from-scratch-index: ${e.message}`); - // Preserve the historical fullIndex() behavior for fire-and-forget - // callers such as startup. + return e instanceof Error ? e : new Error(String(e)); } } diff --git a/packages/runtime-common/realm-operations.ts b/packages/runtime-common/realm-operations.ts index b6d17e5271b..ae9ecadfd6a 100644 --- a/packages/runtime-common/realm-operations.ts +++ b/packages/runtime-common/realm-operations.ts @@ -409,6 +409,7 @@ export const waitForReady: RealmOperation = async ( let readinessUrl = readinessUrlObj.href; let startedAt = Date.now(); let lastError: string | undefined; + let terminalFailure: string | undefined; let stopSamplingProgress = input.onProgress ? sampleProgress( @@ -431,6 +432,15 @@ export const waitForReady: RealmOperation = async ( // `X-Boxel-Not-Ready` names the outstanding stage (index vs // prerender-html); it's a header because pollers discard the body. let stage = response.headers.get('X-Boxel-Not-Ready'); + if (stage === 'index-failed') { + // Terminal: the realm's boot index failed, so no amount of polling + // makes it ready. This response carries the reason in its body. + let detail = await response.text().catch(() => ''); + terminalFailure = `${publishedRealmURL} cannot become ready: its boot index failed${ + detail ? ` — ${detail}` : '' + }`; + break; + } lastError = `HTTP ${response.status}${stage ? ` (not ready: ${stage})` : ''}`; } catch (error) { // Node's fetch reports transport failures as a bare "fetch failed" and @@ -458,6 +468,9 @@ export const waitForReady: RealmOperation = async ( stopSamplingProgress?.(); } + if (terminalFailure) { + throw new Error(terminalFailure); + } throw new Error( `Timed out after ${timeoutMs}ms waiting for ${publishedRealmURL} to pass readiness check${ lastError ? `: ${lastError}` : '' diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index afb551ec6e5..341e91754cc 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -968,6 +968,12 @@ export class Realm { #disableModuleCaching = false; #fullIndexOnStartup = false; #skipBootIndex = false; + // The from-scratch index a brand-new realm's startup awaited, when it + // failed. Such a realm is mounted and answers requests, but its index holds + // nothing, so readiness reports the failure (`X-Boxel-Not-Ready: + // index-failed`) rather than a false ready. Cleared by a later full index + // that completes. + #bootIndexFailure: Error | undefined; #fromScratchIndexPriority = systemInitiatedPriority; #definitionLookup: DefinitionLookup; #copiedFromRealm: URL | undefined; @@ -1421,13 +1427,18 @@ export class Realm { // names which stage is outstanding — each has a different cause and a // different remedy, and the poll loops that consume this discard the // body, so the header is the only place an operator can read it from. - let notReady = (stage: 'startup' | 'index' | 'prerender-html') => + let notReady = ( + stage: 'startup' | 'index' | 'index-failed' | 'prerender-html', + detail?: string, + ) => createResponse({ - body: null, + body: detail ?? null, init: { headers: { - 'content-type': 'text/html', - 'Retry-After': '1', + 'content-type': detail ? 'text/plain' : 'text/html', + // `index-failed` is terminal — the realm cannot become ready + // without a reindex or a restart — so it carries no retry hint. + ...(stage === 'index-failed' ? {} : { 'Retry-After': '1' }), 'X-Boxel-Not-Ready': stage, }, status: 503, @@ -1467,6 +1478,16 @@ export class Realm { ); return notReady('startup'); } + // A brand-new realm whose first index failed is mounted over an empty + // index. Reporting ready would hand the caller a realm that serves + // nothing; reporting `index` would keep it polling for work that is not + // coming. The body carries the failure, since that is where the cause is. + if (this.#bootIndexFailure) { + return notReady( + 'index-failed', + `The boot index of ${this.url} failed: ${this.#bootIndexFailure.message}`, + ); + } let startupSettledAt = Date.now(); let inflight = this.indexing(); if (inflight && !(await settledBy(inflight, requestDeadline))) { @@ -1927,6 +1948,7 @@ export class Realm { }, ); await completed; + this.#bootIndexFailure = undefined; // The from-scratch swap has landed in boxel_index: drop searchCards // in-flight entries + the cached RealmInfo (which may have been // re-parsed from /realm.json during the pass), and broadcast the @@ -3207,7 +3229,7 @@ export class Realm { let promise = this.#realmIndexUpdater.fullIndex(priority); if (isNewIndex) { // we only await the full indexing at boot if this is a brand new index - await promise; + this.#bootIndexFailure = await promise; } // not sure how useful this event is--nothing is currently listening for // it, and it may happen during or after the full index... From c9c592cf19fc65515e4c537671507e21551a8696 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 10 Sep 2026 12:53:48 -0400 Subject: [PATCH 3/5] Ask a base module to render before giving up on an index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consecutive idle render timeouts are not proof of a broken stack on their own. The visit order groups a definition's instances together, so one card type that hangs on an untracked promise or timer times out idle instance after instance, and the pass would have abandoned the whole realm over one bad card. Nor were the idle signals themselves airtight: absent diagnostics counted as zero, so a timeout whose CDP sampling or in-page hook had failed read as idle. When the streak reaches the threshold the loop now renders a canary — the base card-api module, which every realm depends on and none owns — on the pass's own affinity and loader epoch. If it renders, the timeouts are the cards' own: the streak resets and the pass continues, recording them as the files' errors. If it cannot render either, the pass gives up as before, naming the canary's failure alongside the files. Idleness now requires each outside-the-page signal — the responsiveness probe, the CPU sample, the CDP request list — to be present and idle; the in-page counters disqualify when present and non-empty and say nothing when absent. The readiness test exercises both outcomes: a failing canary ends the pass after three or four of six visits and rejects the job naming both causes, and a rendering canary lets the pass visit every file, complete, and leave the realm ready. Co-Authored-By: Claude Fable 5.1 --- .../realm-endpoints/readiness-check-test.ts | 232 ++++++++++++------ packages/runtime-common/index-runner.ts | 94 ++++++- 2 files changed, 237 insertions(+), 89 deletions(-) diff --git a/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts b/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts index 099a93f621d..7bca38d46d3 100644 --- a/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts +++ b/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts @@ -7,6 +7,7 @@ const { ensureDirSync, writeJSONSync } = fsExtra; import { dirSync } from 'tmp'; import type { LooseSingleCardDocument, + ModuleRenderResponse, Prerenderer, Realm, RenderError, @@ -218,9 +219,11 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { // A brand-new realm whose first from-scratch index cannot complete is the // other way a readiness poll could run forever. Here every render times out // with the page idle — the shape a prerender takes when its browser cannot - // reach an origin it needs — so the job gives up after a few files instead - // of paying the full render timeout for every file, and the realm reports - // that as a terminal not-ready rather than a ready over an empty index. + // reach an origin it needs. Whether a base module renders decides what that + // means: when it cannot, the job gives up after a few files instead of + // paying the full render timeout for every file, and the realm reports that + // as a terminal not-ready rather than a ready over an empty index; when it + // can, the idle timeouts are the cards' own and the pass completes. module('boot index whose renders all time out idle', function (hooks) { let dbAdapter: PgAdapter; let publisher: QueuePublisher; @@ -237,67 +240,39 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { const realmURL = 'http://127.0.0.1:6678/idle-timeouts/'; const cardCount = 6; - // The response the prerender server sends for a render that hit its - // timeout while waiting on nothing: a `Render timeout` error on the pass - // and on `pageUnusableError`, with the diagnostics captured as the timer - // fired showing a responsive, idle page. - function idleTimeoutVisitResponse(url: string): RenderVisitResponse { - let timeout: RenderError = { - type: 'instance-error', - error: { - id: url, - status: 504, - title: 'Render timeout', - message: 'Render timed-out after 60000 ms', - additionalErrors: null, - }, - evict: true, + function moduleResponse( + url: string, + outcome: 'ready' | 'timeout', + ): ModuleRenderResponse { + let base = { + id: url, + nonce: 'canary', + isShimmed: false, + lastModified: 0, + createdAt: 0, + deps: [], + definitions: {}, }; + if (outcome === 'ready') { + return { ...base, status: 'ready' }; + } return { - card: { - serialized: null, - searchDoc: null, - displayNames: null, - deps: null, - types: null, - isolatedHTML: null, - headHTML: null, - atomHTML: null, - embeddedHTML: null, - fittedHTML: null, - iconHTML: null, - markdown: null, - error: timeout, - }, - pageUnusableError: timeout, - meta: { - diagnostics: { - mainThreadResponsive: true, - scriptBusyFraction: 0, - pendingNetworkRequests: [], - inFlightModuleImports: [], - cardDocsInFlight: [], - fileMetaDocsInFlight: [], + ...base, + status: 'error', + error: { + type: 'module-error', + error: { + id: url, + status: 504, + title: 'Render timeout', + message: 'Render timed-out after 60000 ms', + additionalErrors: null, }, }, }; } - test('gives up after a few idle timeouts and reports the failure as terminal', async function (assert) { - let real = await getTestPrerenderer(); - let indexVisits: string[] = []; - let prerenderer: Prerenderer = { - prerenderModule: (args) => real.prerenderModule(args), - runCommand: (args) => real.runCommand(args), - releaseBatch: async () => {}, - prerenderVisit: async (args) => { - if (args.visitType === 'index') { - indexVisits.push(args.url); - } - return idleTimeoutVisitResponse(args.url); - }, - }; - + function idleTimeoutFileSystem(): Record { let fileSystem: Record = { 'realm.json': { data: { @@ -326,7 +301,33 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { }, }; } + return fileSystem; + } + // A realm whose every index visit times out idle, with the canary module + // render behaving as `canary` says; started, so its boot index has run. + async function startRealmWithIdleTimeouts(canary: 'ready' | 'timeout') { + let real = await getTestPrerenderer(); + let indexVisits: string[] = []; + let canaryRenders = 0; + let prerenderer: Prerenderer = { + prerenderModule: async (args) => { + // The prerender-html job that follows a completed pass may render + // modules too; only the canary's module counts here. + if (args.url === 'https://cardstack.com/base/card-api') { + canaryRenders++; + } + return moduleResponse(args.url, canary); + }, + runCommand: (args) => real.runCommand(args), + releaseBatch: async () => {}, + prerenderVisit: async (args) => { + if (args.visitType === 'index') { + indexVisits.push(args.url); + } + return idleTimeoutVisitResponse(args.url); + }, + }; let virtualNetwork = createVirtualNetwork(); let definitionLookup = new CachingDefinitionLookup( dbAdapter, @@ -338,7 +339,7 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { ensureDirSync(dir); let { realm } = await createRealm({ dir, - fileSystem, + fileSystem: idleTimeoutFileSystem(), definitionLookup, realmURL, permissions: { '*': ['read'] }, @@ -350,13 +351,79 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { prerenderer, }); virtualNetwork.mount(realm.handle); - // Startup awaits a brand-new index's from-scratch job; the job's failure // is swallowed there, so this resolves either way. await realm.start(); + let [job] = (await dbAdapter.execute( + `SELECT status, result FROM jobs + WHERE job_type = 'from-scratch-index' AND concurrency_group = $1`, + { bind: [indexingConcurrencyGroup(realm.url)] }, + )) as { status: string; result: unknown }[]; + let readiness = await realm.handle( + new Request(`${realmURL}_readiness-check`, { + headers: { Accept: SupportedMimeType.RealmInfo }, + }), + ); + return { + indexVisits, + canaryRenders: () => canaryRenders, + job, + readiness: readiness!, + }; + } + + // The response the prerender server sends for a render that hit its + // timeout while waiting on nothing: a `Render timeout` error on the pass + // and on `pageUnusableError`, with the diagnostics captured as the timer + // fired showing a responsive, idle page. + function idleTimeoutVisitResponse(url: string): RenderVisitResponse { + let timeout: RenderError = { + type: 'instance-error', + error: { + id: url, + status: 504, + title: 'Render timeout', + message: 'Render timed-out after 60000 ms', + additionalErrors: null, + }, + evict: true, + }; + return { + card: { + serialized: null, + searchDoc: null, + displayNames: null, + deps: null, + types: null, + isolatedHTML: null, + headHTML: null, + atomHTML: null, + embeddedHTML: null, + fittedHTML: null, + iconHTML: null, + markdown: null, + error: timeout, + }, + pageUnusableError: timeout, + meta: { + diagnostics: { + mainThreadResponsive: true, + scriptBusyFraction: 0, + pendingNetworkRequests: [], + inFlightModuleImports: [], + cardDocsInFlight: [], + fileMetaDocsInFlight: [], + }, + }, + }; + } - // Three consecutive idle timeouts end the pass; the render-ahead loop - // may have started one more visit before the third was finished. + test('gives up when a base module cannot render either, and reports the failure as terminal', async function (assert) { + let { indexVisits, canaryRenders, job, readiness } = + await startRealmWithIdleTimeouts('timeout'); + + // Three consecutive idle timeouts trigger the canary; the render-ahead + // loop may have started one more visit before the third was finished. assert.true( indexVisits.length >= 3, `made at least the three visits it takes to give up (made ${indexVisits.length})`, @@ -365,39 +432,50 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { indexVisits.length <= 4, `gave up after ${indexVisits.length} index visits rather than visiting all ${cardCount} files`, ); + assert.strictEqual(canaryRenders(), 1, 'rendered the canary once'); - let [job] = (await dbAdapter.execute( - `SELECT status, result FROM jobs - WHERE job_type = 'from-scratch-index' AND concurrency_group = $1`, - { bind: [indexingConcurrencyGroup(realm.url)] }, - )) as { status: string; result: unknown }[]; assert.strictEqual(job.status, 'rejected', 'the job was rejected'); + let reason = JSON.stringify(job.result); assert.true( - JSON.stringify(job.result).includes('timed out with the page idle'), - 'the rejection names the idle timeouts as the reason', + reason.includes('timed out with the page idle'), + 'the rejection names the idle timeouts', ); - - let response = await realm.handle( - new Request(`${realmURL}_readiness-check`, { - headers: { Accept: SupportedMimeType.RealmInfo }, - }), + assert.true( + reason.includes('a base module could not render either'), + 'the rejection names the failed canary', ); - assert.strictEqual(response!.status, 503, 'reports not-ready'); + + assert.strictEqual(readiness.status, 503, 'reports not-ready'); assert.strictEqual( - response!.headers.get('X-Boxel-Not-Ready'), + readiness.headers.get('X-Boxel-Not-Ready'), 'index-failed', 'names the failed boot index as the stage', ); assert.strictEqual( - response!.headers.get('Retry-After'), + readiness.headers.get('Retry-After'), null, 'carries no retry hint: the state is terminal', ); - let body = await response!.text(); + let body = await readiness.text(); assert.true( body.includes('timed out with the page idle'), `the body carries the failure: ${body}`, ); }); + + test("keeps going when a base module renders: the idle timeouts are the cards' own", async function (assert) { + let { indexVisits, canaryRenders, job, readiness } = + await startRealmWithIdleTimeouts('ready'); + + assert.strictEqual(indexVisits.length, cardCount, 'visited every file'); + // One canary per run of three idle timeouts: six files, two canaries. + assert.strictEqual(canaryRenders(), 2, 'rendered the canary per streak'); + assert.strictEqual(job.status, 'resolved', 'the job completed'); + assert.strictEqual( + readiness.status, + 200, + 'the realm is ready: its index exists, with the files recorded as errors', + ); + }); }); }); diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 6359dbbdf25..39ce8b1c62d 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -30,6 +30,7 @@ import { type SearchIndexEntry, } from './index.ts'; import { moduleFrom } from './code-ref.ts'; +import { baseRealm } from './constants.ts'; import type { RealmResourceIdentifier } from './realm-identifiers.ts'; import type { CacheScope, DefinitionLookup } from './definition-lookup.ts'; import type { VirtualNetwork } from './virtual-network.ts'; @@ -94,11 +95,21 @@ const IDLE_SCRIPT_BUSY_MAX = 0.05; const RENDER_TIMEOUT_TITLE = 'Render timeout'; +// A module every realm depends on and none owns. When consecutive renders +// have timed out idle, whether this renders decides between a stack that +// cannot render anything and cards that happen to hang on their own. +const CANARY_MODULE_URL = new URL('card-api', baseRealm.url).href; + // Whether a visit's render timed out with nothing in flight. The timeout error // carries the diagnostics the prerender server captured as it fired (see -// RenderTimeoutDiagnostics), flattened onto the visit's `diagnostics`. A -// timeout whose diagnostics are missing, or show work in progress, is a slow -// or stuck render rather than an idle one and does not count. +// RenderTimeoutDiagnostics), flattened onto the visit's `diagnostics`. The +// signals that decide idleness are the ones captured from outside the page — +// the responsiveness probe, the CPU sample, the CDP request list — and each +// must be present and idle: a timeout whose diagnostics are missing, or show +// work in progress, is a slow or stuck render rather than an idle one and does +// not count. The in-page counters are captured only once the page has reached +// a render stage; when present they can show work the outside view cannot, and +// then disqualify, but their absence says nothing. export function isIdleRenderTimeout(result: IndexVisitRenderResult): boolean { let errors = [ result.pageUnusableError, @@ -113,12 +124,26 @@ export function isIdleRenderTimeout(result: IndexVisitRenderResult): boolean { if (!d || d.mainThreadResponsive !== true) { return false; } - return ( - (d.scriptBusyFraction ?? 0) < IDLE_SCRIPT_BUSY_MAX && - (d.pendingNetworkRequests?.length ?? 0) === 0 && - (d.inFlightModuleImports?.length ?? 0) === 0 && - (d.cardDocsInFlight?.length ?? 0) === 0 && - (d.fileMetaDocsInFlight?.length ?? 0) === 0 + if ( + typeof d.scriptBusyFraction !== 'number' || + d.scriptBusyFraction >= IDLE_SCRIPT_BUSY_MAX + ) { + return false; + } + if ( + !Array.isArray(d.pendingNetworkRequests) || + d.pendingNetworkRequests.length > 0 + ) { + return false; + } + let busy = (list: unknown[] | undefined) => + Array.isArray(list) && list.length > 0; + return !( + busy(d.inFlightModuleImports) || + busy(d.cardDocsInFlight) || + busy(d.fileMetaDocsInFlight) || + busy(d.cardDocLoadsInFlight) || + busy(d.fileMetaDocLoadsInFlight) ); } @@ -840,15 +865,29 @@ export class IndexRunner { idleTimeouts = []; } if (idleTimeouts.length >= abortAfterIdleRenderTimeouts) { + // Consecutive idle timeouts are not proof of a broken stack on their + // own: the visit order groups a definition's instances together, so + // one card type that hangs on an untracked promise produces the same + // run. A module the realm does not own settles it — if that cannot + // render either, nothing in this pass can. + let canary = await this.#canaryRenderFailure(); + if (!canary) { + this.#log.warn( + `${jobIdentity(this.#jobInfo)}: ${idleTimeouts.length} consecutive renders timed out idle (${idleTimeouts.join(', ')}) ` + + `but ${CANARY_MODULE_URL} renders, so the stack is healthy and these are recorded as the files' own errors`, + ); + idleTimeouts = []; + continue; + } // Thrown past #handleVisitError on purpose: this is not one file's // failure to isolate but the pass's inability to render anything, // and the job's rejection is what carries that to whoever awaits // it (readiness, a publish, an operator). let message = `${jobIdentity(this.#jobInfo)} giving up: the last ${idleTimeouts.length} renders each timed out with the page idle ` + - `(main thread responsive, no script running, nothing fetching, no module import in flight), so the page was ` + - `waiting on nothing and no later file can render either. Check that the prerender's browser can reach the ` + - `host, realm-server and icons origins. Files: ${idleTimeouts.join(', ')}`; + `(main thread responsive, no script running, nothing fetching, no module import in flight), and a base module ` + + `could not render either (${canary}), so the page is waiting on nothing and no file in this pass can render. ` + + `Check that the prerender's browser can reach the host, realm-server and icons origins. Files: ${idleTimeouts.join(', ')}`; this.#log.error(message); throw new Error(message); } @@ -856,6 +895,37 @@ export class IndexRunner { } } + // Renders CANARY_MODULE_URL on this pass's affinity and reports why it did + // not render, or undefined when it did. A module render rather than a card + // render: it needs nothing from this realm but the loader epoch, and it goes + // through the same page, host bundle, realm-server and icons origins every + // visit in the pass depends on. + async #canaryRenderFailure(): Promise { + try { + let response = await this.#prerenderer.prerenderModule({ + affinityType: 'realm', + affinityValue: this.#realmURL.href, + realm: this.#realmURL.href, + url: CANARY_MODULE_URL, + auth: this.#auth, + priority: this.#jobPriority, + renderOptions: { loaderEpoch: this.batch.loaderEpoch }, + }); + if (response.status === 'ready') { + return undefined; + } + return ( + response.error?.error?.message ?? + `module render reported status ${response.status}` + ); + } catch (err) { + return coerceErrorMessage( + err, + 'module render threw with no error message', + ); + } + } + async #handleVisitError(url: URL, err: any): Promise { if (isCardError(err) && err.status === 404) { this.#log.info( From 5953870d268101d895deee8a3f879ec93c3fc432 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 10 Sep 2026 12:53:56 -0400 Subject: [PATCH 4/5] Read a failed boot index from shared state, not a flag The terminal readiness stage was held in a field set when the awaited boot index failed and cleared only in Realm.fullIndex. The realm's own _reindex and _full-reindex endpoints go through startReindex, and the reindex handler enqueues the job directly, so a successful reindex from any of those paths left readiness reporting index-failed until the process restarted or the realm was republished. The field was also per-replica, so a peer that had not run the boot could not report it. Readiness now derives the stage from the rows every replica reads: once the index lane is clear, a realm that still has no index whose newest from-scratch job was rejected is reported as index-failed, with that job's failure in the body. A reindex from any path that completes gives the realm an index and clears the state the moment it lands. The in-memory flag and the return value that fed it are gone. Co-Authored-By: Claude Fable 5.1 --- docs/realm-server-health-signals.md | 6 ++- packages/runtime-common/jobs/indexing.ts | 30 ++++++++++++++ .../runtime-common/realm-index-updater.ts | 13 ++---- packages/runtime-common/realm.ts | 41 +++++++++++-------- 4 files changed, 60 insertions(+), 30 deletions(-) diff --git a/docs/realm-server-health-signals.md b/docs/realm-server-health-signals.md index c168f897361..42ad262ba8a 100644 --- a/docs/realm-server-health-signals.md +++ b/docs/realm-server-health-signals.md @@ -106,8 +106,10 @@ the worker gave up, or the job hit its wall-clock limit — is mounted over an e index. Readiness answers 503 with `X-Boxel-Not-Ready: index-failed`, the failure in the body, and no `Retry-After`: a false ready would hand the caller a realm that serves nothing, and `index` would keep it polling for work that is not coming. -The publish flow's poll and the CI realm wait both stop on it. A later full index -that completes clears the state. +The publish flow's poll and the CI realm wait both stop on it. The state is read +from the same rows every replica reads — the realm has no index, and the newest +from-scratch job for it was rejected — so a reindex from any path clears it the +moment it lands. The gating has to read shared state, because in a multi-replica deployment the poll need not reach the replica that did the work. In-process indexing state is diff --git a/packages/runtime-common/jobs/indexing.ts b/packages/runtime-common/jobs/indexing.ts index d1912f2d850..5023f3e036d 100644 --- a/packages/runtime-common/jobs/indexing.ts +++ b/packages/runtime-common/jobs/indexing.ts @@ -157,6 +157,36 @@ export function prerenderSpawnedPriority({ // deadline only between requests, so an attempt started just under the wire // overshoots by up to the length of the hold. A shorter budget bounds that // overshoot; no budget removes it. +// The failure of the newest from-scratch job in a realm's index lane, when +// that job was rejected; undefined when there is no such job or it completed. +// Meaningful alongside the realm's index state: over an index that has since +// been built a rejected job is history, but over a realm that has never had an +// index it is the reason the realm serves nothing. Read from the same rows +// every replica reads, so it holds whichever replica ran the job and whichever +// path enqueued it — a realm's own reindex endpoints, a publish, a system-wide +// reindex — and a later job that completes supersedes it the moment it lands. +export async function latestFromScratchIndexRejection( + dbAdapter: DBAdapter, + realmURL: string, +): Promise { + if (dbAdapter.kind !== 'pg') { + return undefined; + } + let [row] = (await query(dbAdapter, [ + `SELECT status, result FROM jobs WHERE job_type = 'from-scratch-index' AND concurrency_group =`, + param(indexingConcurrencyGroup(realmURL)), + 'ORDER BY id DESC LIMIT 1', + ])) as { status: string; result: unknown }[]; + if (!row || row.status !== 'rejected') { + return undefined; + } + let { result } = row; + if (isObjectLike(result) && typeof (result as any).message === 'string') { + return (result as any).message; + } + return typeof result === 'string' ? result : JSON.stringify(result); +} + export async function awaitRealmIndexSettled( dbAdapter: DBAdapter, realmURL: string, diff --git a/packages/runtime-common/realm-index-updater.ts b/packages/runtime-common/realm-index-updater.ts index b51834ab186..947b0c0cd2e 100644 --- a/packages/runtime-common/realm-index-updater.ts +++ b/packages/runtime-common/realm-index-updater.ts @@ -201,21 +201,14 @@ export class RealmIndexUpdater { }; } - // Resolves to the failure when the job rejected and to undefined when it - // completed. The failure is returned rather than thrown so a caller that - // does not wait on the result — a bootstrap realm's per-boot reindex — is - // unaffected, while a startup that awaits a brand-new index can record that - // the index was never built. - async fullIndex( - priority = systemInitiatedPriority, - ): Promise { + async fullIndex(priority = systemInitiatedPriority) { let { completed } = this.publishFullIndex(priority); try { await completed; - return undefined; } catch (e: any) { this.#log.error(`Error running from-scratch-index: ${e.message}`); - return e instanceof Error ? e : new Error(String(e)); + // Preserve the historical fullIndex() behavior for fire-and-forget + // callers such as startup. } } diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 341e91754cc..a4ba993f816 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -3,6 +3,7 @@ import { resolveRangeHeader } from './http-range.ts'; import { awaitRealmIndexSettled, indexingConcurrencyGroup, + latestFromScratchIndexRejection, } from './jobs/indexing.ts'; import { awaitPublishedHtmlReady } from './jobs/prerender-html.ts'; import { settledBy } from './settled-by.ts'; @@ -968,12 +969,6 @@ export class Realm { #disableModuleCaching = false; #fullIndexOnStartup = false; #skipBootIndex = false; - // The from-scratch index a brand-new realm's startup awaited, when it - // failed. Such a realm is mounted and answers requests, but its index holds - // nothing, so readiness reports the failure (`X-Boxel-Not-Ready: - // index-failed`) rather than a false ready. Cleared by a later full index - // that completes. - #bootIndexFailure: Error | undefined; #fromScratchIndexPriority = systemInitiatedPriority; #definitionLookup: DefinitionLookup; #copiedFromRealm: URL | undefined; @@ -1478,16 +1473,6 @@ export class Realm { ); return notReady('startup'); } - // A brand-new realm whose first index failed is mounted over an empty - // index. Reporting ready would hand the caller a realm that serves - // nothing; reporting `index` would keep it polling for work that is not - // coming. The body carries the failure, since that is where the cause is. - if (this.#bootIndexFailure) { - return notReady( - 'index-failed', - `The boot index of ${this.url} failed: ${this.#bootIndexFailure.message}`, - ); - } let startupSettledAt = Date.now(); let inflight = this.indexing(); if (inflight && !(await settledBy(inflight, requestDeadline))) { @@ -1528,6 +1513,27 @@ export class Realm { return notReady('index'); } + // The lane is clear, so every from-scratch job for this realm has run. A + // realm that still has no index whose newest such job was rejected never + // had its index built: reporting ready would hand the caller a realm that + // serves nothing, and `index` would keep it polling for work that is not + // coming. Both facts are read from shared state, so every replica answers + // alike, and a reindex from any path — this realm's own endpoints, a + // publish, the system-wide reindex — clears it as soon as it lands. The + // body carries the failure, since that is where the cause is. + if (await this.#realmIndexUpdater.isNewIndex()) { + let rejection = await latestFromScratchIndexRejection( + this.#dbAdapter, + this.url, + ); + if (rejection) { + return notReady( + 'index-failed', + `The boot index of ${this.url} failed: ${rejection}`, + ); + } + } + // Opt-in: also await the published HTML being live for the current // generation. Indexing makes a realm searchable; prerendering makes it // viewable — and for a published realm the HTML is the deliverable. That @@ -1948,7 +1954,6 @@ export class Realm { }, ); await completed; - this.#bootIndexFailure = undefined; // The from-scratch swap has landed in boxel_index: drop searchCards // in-flight entries + the cached RealmInfo (which may have been // re-parsed from /realm.json during the pass), and broadcast the @@ -3229,7 +3234,7 @@ export class Realm { let promise = this.#realmIndexUpdater.fullIndex(priority); if (isNewIndex) { // we only await the full indexing at boot if this is a brand new index - this.#bootIndexFailure = await promise; + await promise; } // not sure how useful this event is--nothing is currently listening for // it, and it may happen during or after the full index... From 770f945281e53293643a2e4aba89938cfc662291 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 10 Sep 2026 14:03:16 -0400 Subject: [PATCH 5/5] Read "never had an index built" from the generation, not the row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Readiness took the absence of a realm_generations row as "the realm has no index". A from-scratch pass inserts that row at generation 0 before it visits anything and advances the generation only when it completes, so after a pass that was rejected the row exists at 0 and the check read a realm that serves nothing as one with an index; the terminal stage never fired. The generation is the signal: 0, or no row, means no pass has ever promoted rows. The two shared-state reads now live in one helper. The fixture also has seven files, not six — realm.json is visited too — so the canary test expected one visit too few. Co-Authored-By: Claude Fable 5.1 --- .../realm-endpoints/readiness-check-test.ts | 9 +++-- packages/runtime-common/jobs/indexing.ts | 36 ++++++++++++------- packages/runtime-common/realm.ts | 32 ++++++++--------- 3 files changed, 45 insertions(+), 32 deletions(-) diff --git a/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts b/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts index 7bca38d46d3..bcc8db55763 100644 --- a/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts +++ b/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts @@ -467,8 +467,13 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { let { indexVisits, canaryRenders, job, readiness } = await startRealmWithIdleTimeouts('ready'); - assert.strictEqual(indexVisits.length, cardCount, 'visited every file'); - // One canary per run of three idle timeouts: six files, two canaries. + // Every card plus realm.json. + assert.strictEqual( + indexVisits.length, + cardCount + 1, + 'visited every file', + ); + // One canary per run of three idle timeouts: seven files, two canaries. assert.strictEqual(canaryRenders(), 2, 'rendered the canary per streak'); assert.strictEqual(job.status, 'resolved', 'the job completed'); assert.strictEqual( diff --git a/packages/runtime-common/jobs/indexing.ts b/packages/runtime-common/jobs/indexing.ts index 5023f3e036d..b52457bb7d6 100644 --- a/packages/runtime-common/jobs/indexing.ts +++ b/packages/runtime-common/jobs/indexing.ts @@ -157,30 +157,42 @@ export function prerenderSpawnedPriority({ // deadline only between requests, so an attempt started just under the wire // overshoots by up to the length of the hold. A shorter budget bounds that // overshoot; no budget removes it. -// The failure of the newest from-scratch job in a realm's index lane, when -// that job was rejected; undefined when there is no such job or it completed. -// Meaningful alongside the realm's index state: over an index that has since -// been built a rejected job is history, but over a realm that has never had an -// index it is the reason the realm serves nothing. Read from the same rows -// every replica reads, so it holds whichever replica ran the job and whichever -// path enqueued it — a realm's own reindex endpoints, a publish, a system-wide -// reindex — and a later job that completes supersedes it the moment it lands. -export async function latestFromScratchIndexRejection( +// Why a realm that has never had an index built has none: the failure of the +// newest from-scratch job in its index lane, when that job was rejected. +// Undefined when the realm has an index, when no from-scratch job has run, or +// when the newest one completed. +// +// "Never had an index built" is read from `realm_generations`: a pass inserts +// the realm's row at generation 0 before it visits anything and advances the +// generation only when it completes, so a row at 0 — or none — means no pass +// has ever promoted rows. A rejected job over an index that a later pass built +// is history and does not count. Both facts come from the rows every replica +// reads, so the answer holds whichever replica ran the job and whichever path +// enqueued it — a realm's own reindex endpoints, a publish, a system-wide +// reindex — and a pass that completes from any of them clears it as it lands. +export async function unbuiltIndexFailure( dbAdapter: DBAdapter, realmURL: string, ): Promise { if (dbAdapter.kind !== 'pg') { return undefined; } - let [row] = (await query(dbAdapter, [ + let [generation] = (await query(dbAdapter, [ + 'SELECT current_generation FROM realm_generations WHERE realm_url =', + param(realmURL), + ])) as { current_generation: number | string }[]; + if (generation && Number(generation.current_generation) > 0) { + return undefined; + } + let [job] = (await query(dbAdapter, [ `SELECT status, result FROM jobs WHERE job_type = 'from-scratch-index' AND concurrency_group =`, param(indexingConcurrencyGroup(realmURL)), 'ORDER BY id DESC LIMIT 1', ])) as { status: string; result: unknown }[]; - if (!row || row.status !== 'rejected') { + if (!job || job.status !== 'rejected') { return undefined; } - let { result } = row; + let { result } = job; if (isObjectLike(result) && typeof (result as any).message === 'string') { return (result as any).message; } diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index a4ba993f816..7f7be7bca4f 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -3,7 +3,7 @@ import { resolveRangeHeader } from './http-range.ts'; import { awaitRealmIndexSettled, indexingConcurrencyGroup, - latestFromScratchIndexRejection, + unbuiltIndexFailure, } from './jobs/indexing.ts'; import { awaitPublishedHtmlReady } from './jobs/prerender-html.ts'; import { settledBy } from './settled-by.ts'; @@ -1514,24 +1514,20 @@ export class Realm { } // The lane is clear, so every from-scratch job for this realm has run. A - // realm that still has no index whose newest such job was rejected never - // had its index built: reporting ready would hand the caller a realm that - // serves nothing, and `index` would keep it polling for work that is not - // coming. Both facts are read from shared state, so every replica answers - // alike, and a reindex from any path — this realm's own endpoints, a - // publish, the system-wide reindex — clears it as soon as it lands. The - // body carries the failure, since that is where the cause is. - if (await this.#realmIndexUpdater.isNewIndex()) { - let rejection = await latestFromScratchIndexRejection( - this.#dbAdapter, - this.url, + // realm that has never had an index built, whose newest such job was + // rejected, is mounted over nothing: reporting ready would hand the caller + // a realm that serves nothing, and `index` would keep it polling for work + // that is not coming. Both facts are read from shared state (see + // unbuiltIndexFailure), so every replica answers alike, and a pass that + // completes from any path — this realm's own endpoints, a publish, the + // system-wide reindex — clears it as soon as it lands. The body carries + // the failure, since that is where the cause is. + let unbuilt = await unbuiltIndexFailure(this.#dbAdapter, this.url); + if (unbuilt) { + return notReady( + 'index-failed', + `The boot index of ${this.url} failed: ${unbuilt}`, ); - if (rejection) { - return notReady( - 'index-failed', - `The boot index of ${this.url} failed: ${rejection}`, - ); - } } // Opt-in: also await the published HTML being live for the current