diff --git a/docs/realm-server-health-signals.md b/docs/realm-server-health-signals.md index f12a83845ce..42ad262ba8a 100644 --- a/docs/realm-server-health-signals.md +++ b/docs/realm-server-health-signals.md @@ -95,11 +95,21 @@ no per-request timeout of their own. Blocking is therefore the correct behavior, not a symptom. The endpoint holds the request open while work is outstanding and answers 503 with `Retry-After` plus an -`X-Boxel-Not-Ready: index | prerender-html` header naming the stage. Every caller -retries on a non-ok status, so a 503 costs a poll rather than an error — which is -also why the hold is short by design: a hold that outlives a caller's own deadline -converts its poll loop into a single failed attempt, so the budget stays under the -shortest deadline any caller brings. +`X-Boxel-Not-Ready: startup | index | prerender-html` header naming the stage. +Every caller retries on a non-ok status, so a 503 costs a poll rather than an +error — which is also why the hold is short by design: a hold that outlives a +caller's own deadline converts its poll loop into a single failed attempt, so the +budget stays under the shortest deadline any caller brings. + +One stage is terminal. A brand-new realm whose first from-scratch index failed — +the worker gave up, or the job hit its wall-clock limit — is mounted over an empty +index. Readiness answers 503 with `X-Boxel-Not-Ready: index-failed`, the failure +in the body, and no `Retry-After`: a false ready would hand the caller a realm that +serves nothing, and `index` would keep it polling for work that is not coming. +The publish flow's poll and the CI realm wait both stop on it. The state is read +from the same rows every replica reads — the realm has no index, and the newest +from-scratch job for it was rejected — so a reindex from any path clears it the +moment it lands. The gating has to read shared state, because in a multi-replica deployment the poll need not reach the replica that did the work. In-process indexing state is diff --git a/mise-tasks/ci/wait-for-realms b/mise-tasks/ci/wait-for-realms index c60070e72ab..24a355dc46c 100755 --- a/mise-tasks/ci/wait-for-realms +++ b/mise-tasks/ci/wait-for-realms @@ -32,12 +32,14 @@ # body — which is what tells a Traefik-originated `404 page not found` apart # from a realm-server 404. # -# Three conditions end the wait early instead of running the whole budget +# Four conditions end the wait early instead of running the whole budget # down against an environment that can no longer become ready: # - the service stack process has exited; # - a host that the realm-server had been answering for now gets Traefik's # own 404, i.e. its route file was removed (the realm-server deregisters # every route for the environment on shutdown); +# - the realm reports that its boot index failed (`X-Boxel-Not-Ready: +# index-failed`), which no amount of waiting cures; # - the time budget is spent. # Each failure prints the failing URL's response headers and the Traefik # diagnostics (ci:traefik-diagnostics) before exiting, while the stack is @@ -140,7 +142,13 @@ probe() { describe() { case "$code" in 200) printf '200' ;; - 503) printf '503 not-ready=%s' "${not_ready:-unknown}" ;; + 503) + if [ "$not_ready" = "index-failed" ]; then + printf '503 not-ready=index-failed (%s)' "$body_line" + else + printf '503 not-ready=%s' "${not_ready:-unknown}" + fi + ;; 000) printf 'no response (%s)' "$body_line" ;; *) printf '%s %s "%s"' "$code" "${ctype:-no-content-type}" "$body_line" ;; esac @@ -230,6 +238,9 @@ wait_for_200() { return 0 fi echo "t+$(elapsed)s ${label} -> $(describe)" + if [ "$code" = "503" ] && [ "$not_ready" = "index-failed" ]; then + fail "$url" "${label}: the realm's boot index failed, so it cannot become ready without a restart — ${body_line}" + fi if realm_server_answered; then mark_routed "$host" elif is_traefik_404 && was_routed "$host"; then diff --git a/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts b/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts index f7403c59b3a..bcc8db55763 100644 --- a/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts +++ b/packages/realm-server/tests/realm-endpoints/readiness-check-test.ts @@ -6,12 +6,18 @@ import fsExtra from 'fs-extra'; const { ensureDirSync, writeJSONSync } = fsExtra; import { dirSync } from 'tmp'; import type { + LooseSingleCardDocument, + ModuleRenderResponse, + Prerenderer, Realm, + RenderError, + RenderVisitResponse, QueuePublisher, QueueRunner, } from '@cardstack/runtime-common'; import { CachingDefinitionLookup, + rri, SupportedMimeType, } from '@cardstack/runtime-common'; import { indexingConcurrencyGroup } from '@cardstack/runtime-common/jobs/indexing'; @@ -209,4 +215,272 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { ); }); }); + + // A brand-new realm whose first from-scratch index cannot complete is the + // other way a readiness poll could run forever. Here every render times out + // with the page idle — the shape a prerender takes when its browser cannot + // reach an origin it needs. Whether a base module renders decides what that + // means: when it cannot, the job gives up after a few files instead of + // paying the full render timeout for every file, and the realm reports that + // as a terminal not-ready rather than a ready over an empty index; when it + // can, the idle timeouts are the cards' own and the pass completes. + module('boot index whose renders all time out idle', function (hooks) { + let dbAdapter: PgAdapter; + let publisher: QueuePublisher; + let runner: QueueRunner; + + setupDB(hooks, { + beforeEach: async (adapter, pub, run) => { + dbAdapter = adapter; + publisher = pub; + runner = run; + }, + }); + + const realmURL = 'http://127.0.0.1:6678/idle-timeouts/'; + const cardCount = 6; + + function moduleResponse( + url: string, + outcome: 'ready' | 'timeout', + ): ModuleRenderResponse { + let base = { + id: url, + nonce: 'canary', + isShimmed: false, + lastModified: 0, + createdAt: 0, + deps: [], + definitions: {}, + }; + if (outcome === 'ready') { + return { ...base, status: 'ready' }; + } + return { + ...base, + status: 'error', + error: { + type: 'module-error', + error: { + id: url, + status: 504, + title: 'Render timeout', + message: 'Render timed-out after 60000 ms', + additionalErrors: null, + }, + }, + }; + } + + function idleTimeoutFileSystem(): Record { + let fileSystem: Record = { + 'realm.json': { + data: { + type: 'card', + attributes: { cardInfo: { name: 'Idle Timeouts Realm' } }, + meta: { + adoptsFrom: { + module: rri('@cardstack/base/realm-config'), + name: 'RealmConfig', + }, + }, + }, + }, + }; + for (let i = 0; i < cardCount; i++) { + fileSystem[`card-${i}.json`] = { + data: { + type: 'card', + attributes: { title: `Card ${i}` }, + meta: { + adoptsFrom: { + module: rri('https://cardstack.com/base/card-api'), + name: 'CardDef', + }, + }, + }, + }; + } + return fileSystem; + } + + // A realm whose every index visit times out idle, with the canary module + // render behaving as `canary` says; started, so its boot index has run. + async function startRealmWithIdleTimeouts(canary: 'ready' | 'timeout') { + let real = await getTestPrerenderer(); + let indexVisits: string[] = []; + let canaryRenders = 0; + let prerenderer: Prerenderer = { + prerenderModule: async (args) => { + // The prerender-html job that follows a completed pass may render + // modules too; only the canary's module counts here. + if (args.url === 'https://cardstack.com/base/card-api') { + canaryRenders++; + } + return moduleResponse(args.url, canary); + }, + runCommand: (args) => real.runCommand(args), + releaseBatch: async () => {}, + prerenderVisit: async (args) => { + if (args.visitType === 'index') { + indexVisits.push(args.url); + } + return idleTimeoutVisitResponse(args.url); + }, + }; + let virtualNetwork = createVirtualNetwork(); + let definitionLookup = new CachingDefinitionLookup( + dbAdapter, + real, + virtualNetwork, + testCreatePrerenderAuth, + ); + let dir = join(dirSync().name, 'idle-timeouts'); + ensureDirSync(dir); + let { realm } = await createRealm({ + dir, + fileSystem: idleTimeoutFileSystem(), + definitionLookup, + realmURL, + permissions: { '*': ['read'] }, + virtualNetwork, + publisher, + runner, + dbAdapter, + withWorker: true, + prerenderer, + }); + virtualNetwork.mount(realm.handle); + // Startup awaits a brand-new index's from-scratch job; the job's failure + // is swallowed there, so this resolves either way. + await realm.start(); + let [job] = (await dbAdapter.execute( + `SELECT status, result FROM jobs + WHERE job_type = 'from-scratch-index' AND concurrency_group = $1`, + { bind: [indexingConcurrencyGroup(realm.url)] }, + )) as { status: string; result: unknown }[]; + let readiness = await realm.handle( + new Request(`${realmURL}_readiness-check`, { + headers: { Accept: SupportedMimeType.RealmInfo }, + }), + ); + return { + indexVisits, + canaryRenders: () => canaryRenders, + job, + readiness: readiness!, + }; + } + + // The response the prerender server sends for a render that hit its + // timeout while waiting on nothing: a `Render timeout` error on the pass + // and on `pageUnusableError`, with the diagnostics captured as the timer + // fired showing a responsive, idle page. + function idleTimeoutVisitResponse(url: string): RenderVisitResponse { + let timeout: RenderError = { + type: 'instance-error', + error: { + id: url, + status: 504, + title: 'Render timeout', + message: 'Render timed-out after 60000 ms', + additionalErrors: null, + }, + evict: true, + }; + return { + card: { + serialized: null, + searchDoc: null, + displayNames: null, + deps: null, + types: null, + isolatedHTML: null, + headHTML: null, + atomHTML: null, + embeddedHTML: null, + fittedHTML: null, + iconHTML: null, + markdown: null, + error: timeout, + }, + pageUnusableError: timeout, + meta: { + diagnostics: { + mainThreadResponsive: true, + scriptBusyFraction: 0, + pendingNetworkRequests: [], + inFlightModuleImports: [], + cardDocsInFlight: [], + fileMetaDocsInFlight: [], + }, + }, + }; + } + + test('gives up when a base module cannot render either, and reports the failure as terminal', async function (assert) { + let { indexVisits, canaryRenders, job, readiness } = + await startRealmWithIdleTimeouts('timeout'); + + // Three consecutive idle timeouts trigger the canary; the render-ahead + // loop may have started one more visit before the third was finished. + assert.true( + indexVisits.length >= 3, + `made at least the three visits it takes to give up (made ${indexVisits.length})`, + ); + assert.true( + indexVisits.length <= 4, + `gave up after ${indexVisits.length} index visits rather than visiting all ${cardCount} files`, + ); + assert.strictEqual(canaryRenders(), 1, 'rendered the canary once'); + + assert.strictEqual(job.status, 'rejected', 'the job was rejected'); + let reason = JSON.stringify(job.result); + assert.true( + reason.includes('timed out with the page idle'), + 'the rejection names the idle timeouts', + ); + assert.true( + reason.includes('a base module could not render either'), + 'the rejection names the failed canary', + ); + + assert.strictEqual(readiness.status, 503, 'reports not-ready'); + assert.strictEqual( + readiness.headers.get('X-Boxel-Not-Ready'), + 'index-failed', + 'names the failed boot index as the stage', + ); + assert.strictEqual( + readiness.headers.get('Retry-After'), + null, + 'carries no retry hint: the state is terminal', + ); + let body = await readiness.text(); + assert.true( + body.includes('timed out with the page idle'), + `the body carries the failure: ${body}`, + ); + }); + + test("keeps going when a base module renders: the idle timeouts are the cards' own", async function (assert) { + let { indexVisits, canaryRenders, job, readiness } = + await startRealmWithIdleTimeouts('ready'); + + // Every card plus realm.json. + assert.strictEqual( + indexVisits.length, + cardCount + 1, + 'visited every file', + ); + // One canary per run of three idle timeouts: seven files, two canaries. + assert.strictEqual(canaryRenders(), 2, 'rendered the canary per streak'); + assert.strictEqual(job.status, 'resolved', 'the job completed'); + assert.strictEqual( + readiness.status, + 200, + 'the realm is ready: its index exists, with the files recorded as errors', + ); + }); + }); }); diff --git a/packages/runtime-common/index-runner.ts b/packages/runtime-common/index-runner.ts index 5810a359ecd..39ce8b1c62d 100644 --- a/packages/runtime-common/index-runner.ts +++ b/packages/runtime-common/index-runner.ts @@ -30,6 +30,7 @@ import { type SearchIndexEntry, } from './index.ts'; import { moduleFrom } from './code-ref.ts'; +import { baseRealm } from './constants.ts'; import type { RealmResourceIdentifier } from './realm-identifiers.ts'; import type { CacheScope, DefinitionLookup } from './definition-lookup.ts'; import type { VirtualNetwork } from './virtual-network.ts'; @@ -63,6 +64,89 @@ type VisitRenderOutcome = | { status: 'skipped' } | { status: 'error'; error: unknown }; +// A render that times out while its page is idle — main thread responsive, no +// script running, nothing fetching, no module import or document load in +// flight — was waiting on nothing, and nothing about the next file changes +// that. Several in a row therefore describe the stack rather than the files: +// an origin the page needs (the host bundle, the realm-server, the icons +// server) is unreachable from the browser, and every remaining file would +// cost the full render timeout to fail the same way. A from-scratch pass gives +// up after this many consecutive idle timeouts, so the job fails within +// minutes naming the cause instead of grinding to its wall-clock limit and +// being rejected with nothing written. 0 disables the guard. +const DEFAULT_IDLE_RENDER_TIMEOUT_ABORT_AFTER = 3; +const envIdleRenderTimeoutAbortAfter = Number( + ( + globalThis as { + process?: { env?: Record }; + } + ).process?.env?.INDEX_IDLE_RENDER_TIMEOUT_ABORT_AFTER, +); +export const IDLE_RENDER_TIMEOUT_ABORT_AFTER = + Number.isFinite(envIdleRenderTimeoutAbortAfter) && + envIdleRenderTimeoutAbortAfter >= 0 + ? envIdleRenderTimeoutAbortAfter + : DEFAULT_IDLE_RENDER_TIMEOUT_ABORT_AFTER; + +// Script-busy fraction below which the page counts as idle. The CPU sample +// covers a short window, so a page that is doing nothing useful can still show +// a few percent of housekeeping. +const IDLE_SCRIPT_BUSY_MAX = 0.05; + +const RENDER_TIMEOUT_TITLE = 'Render timeout'; + +// A module every realm depends on and none owns. When consecutive renders +// have timed out idle, whether this renders decides between a stack that +// cannot render anything and cards that happen to hang on their own. +const CANARY_MODULE_URL = new URL('card-api', baseRealm.url).href; + +// Whether a visit's render timed out with nothing in flight. The timeout error +// carries the diagnostics the prerender server captured as it fired (see +// RenderTimeoutDiagnostics), flattened onto the visit's `diagnostics`. The +// signals that decide idleness are the ones captured from outside the page — +// the responsiveness probe, the CPU sample, the CDP request list — and each +// must be present and idle: a timeout whose diagnostics are missing, or show +// work in progress, is a slow or stuck render rather than an idle one and does +// not count. The in-page counters are captured only once the page has reached +// a render stage; when present they can show work the outside view cannot, and +// then disqualify, but their absence says nothing. +export function isIdleRenderTimeout(result: IndexVisitRenderResult): boolean { + let errors = [ + result.pageUnusableError, + result.card?.error, + result.fileExtract?.error, + result.fileRender?.error, + ]; + if (!errors.some((e) => e?.error?.title === RENDER_TIMEOUT_TITLE)) { + return false; + } + let d = result.diagnostics; + if (!d || d.mainThreadResponsive !== true) { + return false; + } + if ( + typeof d.scriptBusyFraction !== 'number' || + d.scriptBusyFraction >= IDLE_SCRIPT_BUSY_MAX + ) { + return false; + } + if ( + !Array.isArray(d.pendingNetworkRequests) || + d.pendingNetworkRequests.length > 0 + ) { + return false; + } + let busy = (list: unknown[] | undefined) => + Array.isArray(list) && list.length > 0; + return !( + busy(d.inFlightModuleImports) || + busy(d.cardDocsInFlight) || + busy(d.fileMetaDocsInFlight) || + busy(d.cardDocLoadsInFlight) || + busy(d.fileMetaDocLoadsInFlight) + ); +} + export class IndexRunner { #indexingInstances = new Map>(); #reader: Reader; @@ -123,6 +207,7 @@ export class IndexRunner { // warm loader ownership — intended). Populated in the constructor after // jobInfo is known so the id is easy to correlate with a job in logs. #batchId!: string; + #idleRenderTimeoutAbortAfter: number; constructor({ realmURL, @@ -140,6 +225,7 @@ export class IndexRunner { auth, fetch, realmOwnerUserId, + idleRenderTimeoutAbortAfter = IDLE_RENDER_TIMEOUT_ABORT_AFTER, }: { realmURL: URL; reader: Reader; @@ -151,6 +237,9 @@ export class IndexRunner { auth: string; fetch: typeof globalThis.fetch; realmOwnerUserId: string; + // Consecutive idle render timeouts after which a from-scratch pass gives + // up; see IDLE_RENDER_TIMEOUT_ABORT_AFTER. 0 disables the guard. + idleRenderTimeoutAbortAfter?: number; jobInfo?: JobInfo; // Optional override of `jobInfo.priority`. When both are present, // `jobPriority` wins — this is the path the worker handler takes @@ -189,6 +278,7 @@ export class IndexRunner { this.#auth = auth; this.#fetch = fetch; this.#realmOwnerUserId = realmOwnerUserId; + this.#idleRenderTimeoutAbortAfter = idleRenderTimeoutAbortAfter; this.#definitionLookup = definitionLookup; this.#dependencyResolver = new IndexRunnerDependencyManager({ realmURL: this.#realmURL, @@ -300,6 +390,7 @@ export class IndexRunner { let resumedSkipped = 0; try { await current.#runVisitLoop(invalidations, { + abortAfterIdleRenderTimeouts: current.#idleRenderTimeoutAbortAfter, // Resume guard. If a previous attempt of this same job already wrote // URL_X to the working table AND the EFS mtime hasn't changed since, // skip the visit — the existing working row is still authoritative @@ -709,14 +800,21 @@ export class IndexRunner { skipReason, onSkip, onVisited, + abortAfterIdleRenderTimeouts = 0, }: { skipReason: (url: URL) => 'resumed' | 'delete' | undefined; onSkip: (url: URL, reason: 'resumed' | 'delete') => void; onVisited: (url: URL) => void; + // When set, the pass throws — abandoning the batch — once this many + // consecutive visits time out idle (see isIdleRenderTimeout). A + // from-scratch pass sets it; an incremental pass, whose rows are the + // only record of a write, does not. + abortAfterIdleRenderTimeouts?: number; }, ): Promise { let n = invalidations.length; let renders = new Map>(); + let idleTimeouts: string[] = []; // Start the render for the next non-skipped URL at or after `from`, // keeping exactly one render in flight ahead of the finish cursor. let prefetch = (from: number) => { @@ -757,6 +855,74 @@ export class IndexRunner { await this.#handleVisitError(url, err); } onVisited(url); + if (abortAfterIdleRenderTimeouts > 0) { + if ( + outcome.status === 'rendered' && + isIdleRenderTimeout(outcome.result) + ) { + idleTimeouts.push(url.href); + } else { + idleTimeouts = []; + } + if (idleTimeouts.length >= abortAfterIdleRenderTimeouts) { + // Consecutive idle timeouts are not proof of a broken stack on their + // own: the visit order groups a definition's instances together, so + // one card type that hangs on an untracked promise produces the same + // run. A module the realm does not own settles it — if that cannot + // render either, nothing in this pass can. + let canary = await this.#canaryRenderFailure(); + if (!canary) { + this.#log.warn( + `${jobIdentity(this.#jobInfo)}: ${idleTimeouts.length} consecutive renders timed out idle (${idleTimeouts.join(', ')}) ` + + `but ${CANARY_MODULE_URL} renders, so the stack is healthy and these are recorded as the files' own errors`, + ); + idleTimeouts = []; + continue; + } + // Thrown past #handleVisitError on purpose: this is not one file's + // failure to isolate but the pass's inability to render anything, + // and the job's rejection is what carries that to whoever awaits + // it (readiness, a publish, an operator). + let message = + `${jobIdentity(this.#jobInfo)} giving up: the last ${idleTimeouts.length} renders each timed out with the page idle ` + + `(main thread responsive, no script running, nothing fetching, no module import in flight), and a base module ` + + `could not render either (${canary}), so the page is waiting on nothing and no file in this pass can render. ` + + `Check that the prerender's browser can reach the host, realm-server and icons origins. Files: ${idleTimeouts.join(', ')}`; + this.#log.error(message); + throw new Error(message); + } + } + } + } + + // Renders CANARY_MODULE_URL on this pass's affinity and reports why it did + // not render, or undefined when it did. A module render rather than a card + // render: it needs nothing from this realm but the loader epoch, and it goes + // through the same page, host bundle, realm-server and icons origins every + // visit in the pass depends on. + async #canaryRenderFailure(): Promise { + try { + let response = await this.#prerenderer.prerenderModule({ + affinityType: 'realm', + affinityValue: this.#realmURL.href, + realm: this.#realmURL.href, + url: CANARY_MODULE_URL, + auth: this.#auth, + priority: this.#jobPriority, + renderOptions: { loaderEpoch: this.batch.loaderEpoch }, + }); + if (response.status === 'ready') { + return undefined; + } + return ( + response.error?.error?.message ?? + `module render reported status ${response.status}` + ); + } catch (err) { + return coerceErrorMessage( + err, + 'module render threw with no error message', + ); } } diff --git a/packages/runtime-common/jobs/indexing.ts b/packages/runtime-common/jobs/indexing.ts index d1912f2d850..b52457bb7d6 100644 --- a/packages/runtime-common/jobs/indexing.ts +++ b/packages/runtime-common/jobs/indexing.ts @@ -157,6 +157,48 @@ export function prerenderSpawnedPriority({ // deadline only between requests, so an attempt started just under the wire // overshoots by up to the length of the hold. A shorter budget bounds that // overshoot; no budget removes it. +// Why a realm that has never had an index built has none: the failure of the +// newest from-scratch job in its index lane, when that job was rejected. +// Undefined when the realm has an index, when no from-scratch job has run, or +// when the newest one completed. +// +// "Never had an index built" is read from `realm_generations`: a pass inserts +// the realm's row at generation 0 before it visits anything and advances the +// generation only when it completes, so a row at 0 — or none — means no pass +// has ever promoted rows. A rejected job over an index that a later pass built +// is history and does not count. Both facts come from the rows every replica +// reads, so the answer holds whichever replica ran the job and whichever path +// enqueued it — a realm's own reindex endpoints, a publish, a system-wide +// reindex — and a pass that completes from any of them clears it as it lands. +export async function unbuiltIndexFailure( + dbAdapter: DBAdapter, + realmURL: string, +): Promise { + if (dbAdapter.kind !== 'pg') { + return undefined; + } + let [generation] = (await query(dbAdapter, [ + 'SELECT current_generation FROM realm_generations WHERE realm_url =', + param(realmURL), + ])) as { current_generation: number | string }[]; + if (generation && Number(generation.current_generation) > 0) { + return undefined; + } + let [job] = (await query(dbAdapter, [ + `SELECT status, result FROM jobs WHERE job_type = 'from-scratch-index' AND concurrency_group =`, + param(indexingConcurrencyGroup(realmURL)), + 'ORDER BY id DESC LIMIT 1', + ])) as { status: string; result: unknown }[]; + if (!job || job.status !== 'rejected') { + return undefined; + } + let { result } = job; + if (isObjectLike(result) && typeof (result as any).message === 'string') { + return (result as any).message; + } + return typeof result === 'string' ? result : JSON.stringify(result); +} + export async function awaitRealmIndexSettled( dbAdapter: DBAdapter, realmURL: string, diff --git a/packages/runtime-common/realm-operations.ts b/packages/runtime-common/realm-operations.ts index b6d17e5271b..ae9ecadfd6a 100644 --- a/packages/runtime-common/realm-operations.ts +++ b/packages/runtime-common/realm-operations.ts @@ -409,6 +409,7 @@ export const waitForReady: RealmOperation = async ( let readinessUrl = readinessUrlObj.href; let startedAt = Date.now(); let lastError: string | undefined; + let terminalFailure: string | undefined; let stopSamplingProgress = input.onProgress ? sampleProgress( @@ -431,6 +432,15 @@ export const waitForReady: RealmOperation = async ( // `X-Boxel-Not-Ready` names the outstanding stage (index vs // prerender-html); it's a header because pollers discard the body. let stage = response.headers.get('X-Boxel-Not-Ready'); + if (stage === 'index-failed') { + // Terminal: the realm's boot index failed, so no amount of polling + // makes it ready. This response carries the reason in its body. + let detail = await response.text().catch(() => ''); + terminalFailure = `${publishedRealmURL} cannot become ready: its boot index failed${ + detail ? ` — ${detail}` : '' + }`; + break; + } lastError = `HTTP ${response.status}${stage ? ` (not ready: ${stage})` : ''}`; } catch (error) { // Node's fetch reports transport failures as a bare "fetch failed" and @@ -458,6 +468,9 @@ export const waitForReady: RealmOperation = async ( stopSamplingProgress?.(); } + if (terminalFailure) { + throw new Error(terminalFailure); + } throw new Error( `Timed out after ${timeoutMs}ms waiting for ${publishedRealmURL} to pass readiness check${ lastError ? `: ${lastError}` : '' diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index afb551ec6e5..7f7be7bca4f 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -3,6 +3,7 @@ import { resolveRangeHeader } from './http-range.ts'; import { awaitRealmIndexSettled, indexingConcurrencyGroup, + unbuiltIndexFailure, } from './jobs/indexing.ts'; import { awaitPublishedHtmlReady } from './jobs/prerender-html.ts'; import { settledBy } from './settled-by.ts'; @@ -1421,13 +1422,18 @@ export class Realm { // names which stage is outstanding — each has a different cause and a // different remedy, and the poll loops that consume this discard the // body, so the header is the only place an operator can read it from. - let notReady = (stage: 'startup' | 'index' | 'prerender-html') => + let notReady = ( + stage: 'startup' | 'index' | 'index-failed' | 'prerender-html', + detail?: string, + ) => createResponse({ - body: null, + body: detail ?? null, init: { headers: { - 'content-type': 'text/html', - 'Retry-After': '1', + 'content-type': detail ? 'text/plain' : 'text/html', + // `index-failed` is terminal — the realm cannot become ready + // without a reindex or a restart — so it carries no retry hint. + ...(stage === 'index-failed' ? {} : { 'Retry-After': '1' }), 'X-Boxel-Not-Ready': stage, }, status: 503, @@ -1507,6 +1513,23 @@ export class Realm { return notReady('index'); } + // The lane is clear, so every from-scratch job for this realm has run. A + // realm that has never had an index built, whose newest such job was + // rejected, is mounted over nothing: reporting ready would hand the caller + // a realm that serves nothing, and `index` would keep it polling for work + // that is not coming. Both facts are read from shared state (see + // unbuiltIndexFailure), so every replica answers alike, and a pass that + // completes from any path — this realm's own endpoints, a publish, the + // system-wide reindex — clears it as soon as it lands. The body carries + // the failure, since that is where the cause is. + let unbuilt = await unbuiltIndexFailure(this.#dbAdapter, this.url); + if (unbuilt) { + return notReady( + 'index-failed', + `The boot index of ${this.url} failed: ${unbuilt}`, + ); + } + // Opt-in: also await the published HTML being live for the current // generation. Indexing makes a realm searchable; prerendering makes it // viewable — and for a published realm the HTML is the deliverable. That