Fail a boot index that cannot render within minutes, and report it as terminal

docs/realm-server-health-signals.md

@@ -95,11 +95,21 @@ no per-request timeout of their own.
 
 Blocking is therefore the correct behavior, not a symptom. The endpoint holds the
 request open while work is outstanding and answers 503 with `Retry-After` plus an
-`X-Boxel-Not-Ready: index | prerender-html` header naming the stage. Every caller
-retries on a non-ok status, so a 503 costs a poll rather than an error — which is
-also why the hold is short by design: a hold that outlives a caller's own deadline
-converts its poll loop into a single failed attempt, so the budget stays under the
-shortest deadline any caller brings.
+`X-Boxel-Not-Ready: startup | index | prerender-html` header naming the stage.
+Every caller retries on a non-ok status, so a 503 costs a poll rather than an
+error — which is also why the hold is short by design: a hold that outlives a
+caller's own deadline converts its poll loop into a single failed attempt, so the
+budget stays under the shortest deadline any caller brings.
+
+One stage is terminal. A brand-new realm whose first from-scratch index failed —
+the worker gave up, or the job hit its wall-clock limit — is mounted over an empty
+index. Readiness answers 503 with `X-Boxel-Not-Ready: index-failed`, the failure
+in the body, and no `Retry-After`: a false ready would hand the caller a realm that
+serves nothing, and `index` would keep it polling for work that is not coming.
+The publish flow's poll and the CI realm wait both stop on it. The state is read
+from the same rows every replica reads — the realm has no index, and the newest
+from-scratch job for it was rejected — so a reindex from any path clears it the
+moment it lands.
 
 The gating has to read shared state, because in a multi-replica deployment the poll
 need not reach the replica that did the work. In-process indexing state is

mise-tasks/ci/wait-for-realms

@@ -32,12 +32,14 @@
 # body — which is what tells a Traefik-originated `404 page not found` apart
 # from a realm-server 404.
 #
-# Three conditions end the wait early instead of running the whole budget
+# Four conditions end the wait early instead of running the whole budget
 # down against an environment that can no longer become ready:
 #   - the service stack process has exited;
 #   - a host that the realm-server had been answering for now gets Traefik's
 #     own 404, i.e. its route file was removed (the realm-server deregisters
 #     every route for the environment on shutdown);
+#   - the realm reports that its boot index failed (`X-Boxel-Not-Ready:
+#     index-failed`), which no amount of waiting cures;
 #   - the time budget is spent.
 # Each failure prints the failing URL's response headers and the Traefik
 # diagnostics (ci:traefik-diagnostics) before exiting, while the stack is
@@ -140,7 +142,13 @@ probe() {
 describe() {
   case "$code" in
     200) printf '200' ;;
-    503) printf '503 not-ready=%s' "${not_ready:-unknown}" ;;
+    503)
+      if [ "$not_ready" = "index-failed" ]; then
+        printf '503 not-ready=index-failed (%s)' "$body_line"
+      else
+        printf '503 not-ready=%s' "${not_ready:-unknown}"
+      fi
+      ;;
     000) printf 'no response (%s)' "$body_line" ;;
     *) printf '%s %s "%s"' "$code" "${ctype:-no-content-type}" "$body_line" ;;
   esac
@@ -230,6 +238,9 @@ wait_for_200() {
       return 0
     fi
     echo "t+$(elapsed)s  ${label} -> $(describe)"
+    if [ "$code" = "503" ] && [ "$not_ready" = "index-failed" ]; then
+      fail "$url" "${label}: the realm's boot index failed, so it cannot become ready without a restart — ${body_line}"
+    fi
     if realm_server_answered; then
       mark_routed "$host"
     elif is_traefik_404 && was_routed "$host"; then

packages/realm-server/tests/realm-endpoints/readiness-check-test.ts

@@ -6,12 +6,18 @@ import fsExtra from 'fs-extra';
 const { ensureDirSync, writeJSONSync } = fsExtra;
 import { dirSync } from 'tmp';
 import type {
+  LooseSingleCardDocument,
+  ModuleRenderResponse,
+  Prerenderer,
   Realm,
+  RenderError,
+  RenderVisitResponse,
   QueuePublisher,
   QueueRunner,
 } from '@cardstack/runtime-common';
 import {
   CachingDefinitionLookup,
+  rri,
   SupportedMimeType,
 } from '@cardstack/runtime-common';
 import { indexingConcurrencyGroup } from '@cardstack/runtime-common/jobs/indexing';
@@ -209,4 +215,272 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () {
       );
     });
   });
+
+  // A brand-new realm whose first from-scratch index cannot complete is the
+  // other way a readiness poll could run forever. Here every render times out
+  // with the page idle — the shape a prerender takes when its browser cannot
+  // reach an origin it needs. Whether a base module renders decides what that
+  // means: when it cannot, the job gives up after a few files instead of
+  // paying the full render timeout for every file, and the realm reports that
+  // as a terminal not-ready rather than a ready over an empty index; when it
+  // can, the idle timeouts are the cards' own and the pass completes.
+  module('boot index whose renders all time out idle', function (hooks) {
+    let dbAdapter: PgAdapter;
+    let publisher: QueuePublisher;
+    let runner: QueueRunner;
+
+    setupDB(hooks, {
+      beforeEach: async (adapter, pub, run) => {
+        dbAdapter = adapter;
+        publisher = pub;
+        runner = run;
+      },
+    });
+
+    const realmURL = 'http://127.0.0.1:6678/idle-timeouts/';
+    const cardCount = 6;
+
+    function moduleResponse(
+      url: string,
+      outcome: 'ready' | 'timeout',
+    ): ModuleRenderResponse {
+      let base = {
+        id: url,
+        nonce: 'canary',
+        isShimmed: false,
+        lastModified: 0,
+        createdAt: 0,
+        deps: [],
+        definitions: {},
+      };
+      if (outcome === 'ready') {
+        return { ...base, status: 'ready' };
+      }
+      return {
+        ...base,
+        status: 'error',
+        error: {
+          type: 'module-error',
+          error: {
+            id: url,
+            status: 504,
+            title: 'Render timeout',
+            message: 'Render timed-out after 60000 ms',
+            additionalErrors: null,
+          },
+        },
+      };
+    }
+
+    function idleTimeoutFileSystem(): Record<string, LooseSingleCardDocument> {
+      let fileSystem: Record<string, LooseSingleCardDocument> = {
+        'realm.json': {
+          data: {
+            type: 'card',
+            attributes: { cardInfo: { name: 'Idle Timeouts Realm' } },
+            meta: {
+              adoptsFrom: {
+                module: rri('@cardstack/base/realm-config'),
+                name: 'RealmConfig',
+              },
+            },
+          },
+        },
+      };
+      for (let i = 0; i < cardCount; i++) {
+        fileSystem[`card-${i}.json`] = {
+          data: {
+            type: 'card',
+            attributes: { title: `Card ${i}` },
+            meta: {
+              adoptsFrom: {
+                module: rri('https://cardstack.com/base/card-api'),
+                name: 'CardDef',
+              },
+            },
+          },
+        };
+      }
+      return fileSystem;
+    }
+
+    // A realm whose every index visit times out idle, with the canary module
+    // render behaving as `canary` says; started, so its boot index has run.
+    async function startRealmWithIdleTimeouts(canary: 'ready' | 'timeout') {
+      let real = await getTestPrerenderer();
+      let indexVisits: string[] = [];
+      let canaryRenders = 0;
+      let prerenderer: Prerenderer = {
+        prerenderModule: async (args) => {
+          // The prerender-html job that follows a completed pass may render
+          // modules too; only the canary's module counts here.
+          if (args.url === 'https://cardstack.com/base/card-api') {
+            canaryRenders++;
+          }
+          return moduleResponse(args.url, canary);
+        },
+        runCommand: (args) => real.runCommand(args),
+        releaseBatch: async () => {},
+        prerenderVisit: async (args) => {
+          if (args.visitType === 'index') {
+            indexVisits.push(args.url);
+          }
+          return idleTimeoutVisitResponse(args.url);
+        },
+      };
+      let virtualNetwork = createVirtualNetwork();
+      let definitionLookup = new CachingDefinitionLookup(
+        dbAdapter,
+        real,
+        virtualNetwork,
+        testCreatePrerenderAuth,
+      );
+      let dir = join(dirSync().name, 'idle-timeouts');
+      ensureDirSync(dir);
+      let { realm } = await createRealm({
+        dir,
+        fileSystem: idleTimeoutFileSystem(),
+        definitionLookup,
+        realmURL,
+        permissions: { '*': ['read'] },
+        virtualNetwork,
+        publisher,
+        runner,
+        dbAdapter,
+        withWorker: true,
+        prerenderer,
+      });
+      virtualNetwork.mount(realm.handle);
+      // Startup awaits a brand-new index's from-scratch job; the job's failure
+      // is swallowed there, so this resolves either way.
+      await realm.start();
+      let [job] = (await dbAdapter.execute(
+        `SELECT status, result FROM jobs
+           WHERE job_type = 'from-scratch-index' AND concurrency_group = $1`,
+        { bind: [indexingConcurrencyGroup(realm.url)] },
+      )) as { status: string; result: unknown }[];
+      let readiness = await realm.handle(
+        new Request(`${realmURL}_readiness-check`, {
+          headers: { Accept: SupportedMimeType.RealmInfo },
+        }),
+      );
+      return {
+        indexVisits,
+        canaryRenders: () => canaryRenders,
+        job,
+        readiness: readiness!,
+      };
+    }
+
+    // The response the prerender server sends for a render that hit its
+    // timeout while waiting on nothing: a `Render timeout` error on the pass
+    // and on `pageUnusableError`, with the diagnostics captured as the timer
+    // fired showing a responsive, idle page.
+    function idleTimeoutVisitResponse(url: string): RenderVisitResponse {
+      let timeout: RenderError = {
+        type: 'instance-error',
+        error: {
+          id: url,
+          status: 504,
+          title: 'Render timeout',
+          message: 'Render timed-out after 60000 ms',
+          additionalErrors: null,
+        },
+        evict: true,
+      };
+      return {
+        card: {
+          serialized: null,
+          searchDoc: null,
+          displayNames: null,
+          deps: null,
+          types: null,
+          isolatedHTML: null,
+          headHTML: null,
+          atomHTML: null,
+          embeddedHTML: null,
+          fittedHTML: null,
+          iconHTML: null,
+          markdown: null,
+          error: timeout,
+        },
+        pageUnusableError: timeout,
+        meta: {
+          diagnostics: {
+            mainThreadResponsive: true,
+            scriptBusyFraction: 0,
+            pendingNetworkRequests: [],
+            inFlightModuleImports: [],
+            cardDocsInFlight: [],
+            fileMetaDocsInFlight: [],
+          },
+        },
+      };
+    }
+
+    test('gives up when a base module cannot render either, and reports the failure as terminal', async function (assert) {
+      let { indexVisits, canaryRenders, job, readiness } =
+        await startRealmWithIdleTimeouts('timeout');
+
+      // Three consecutive idle timeouts trigger the canary; the render-ahead
+      // loop may have started one more visit before the third was finished.
+      assert.true(
+        indexVisits.length >= 3,
+        `made at least the three visits it takes to give up (made ${indexVisits.length})`,
+      );
+      assert.true(
+        indexVisits.length <= 4,
+        `gave up after ${indexVisits.length} index visits rather than visiting all ${cardCount} files`,
+      );
+      assert.strictEqual(canaryRenders(), 1, 'rendered the canary once');
+
+      assert.strictEqual(job.status, 'rejected', 'the job was rejected');
+      let reason = JSON.stringify(job.result);
+      assert.true(
+        reason.includes('timed out with the page idle'),
+        'the rejection names the idle timeouts',
+      );
+      assert.true(
+        reason.includes('a base module could not render either'),
+        'the rejection names the failed canary',
+      );
+
+      assert.strictEqual(readiness.status, 503, 'reports not-ready');
+      assert.strictEqual(
+        readiness.headers.get('X-Boxel-Not-Ready'),
+        'index-failed',
+        'names the failed boot index as the stage',
+      );
+      assert.strictEqual(
+        readiness.headers.get('Retry-After'),
+        null,
+        'carries no retry hint: the state is terminal',
+      );
+      let body = await readiness.text();
+      assert.true(
+        body.includes('timed out with the page idle'),
+        `the body carries the failure: ${body}`,
+      );
+    });
+
+    test("keeps going when a base module renders: the idle timeouts are the cards' own", async function (assert) {
+      let { indexVisits, canaryRenders, job, readiness } =
+        await startRealmWithIdleTimeouts('ready');
+
+      // Every card plus realm.json.
+      assert.strictEqual(
+        indexVisits.length,
+        cardCount + 1,
+        'visited every file',
+      );
+      // One canary per run of three idle timeouts: seven files, two canaries.
+      assert.strictEqual(canaryRenders(), 2, 'rendered the canary per streak');
+      assert.strictEqual(job.status, 'resolved', 'the job completed');
+      assert.strictEqual(
+        readiness.status,
+        200,
+        'the realm is ready: its index exists, with the files recorded as errors',
+      );
+    });
+  });
 });