From 8a6fa4bbb1ce4495962e383c3806453974668670 Mon Sep 17 00:00:00 2001 From: Stanley471 Date: Fri, 16 Oct 2026 09:25:02 +0100 Subject: [PATCH 1/2] fix(indexer): mark reads stale and surface per-network status when indexer is unconfigured (#238) --- README.md | 61 +++++++++-------- src/__tests__/multiNetworkIndexer.test.ts | 23 +++++++ src/__tests__/staleReads.test.ts | 79 +++++++++++++++++++++++ src/api.ts | 30 +++++++-- src/indexer.ts | 31 ++++++++- src/openapi/schemas.ts | 24 +++++-- 6 files changed, 204 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index d6028b71..7a69b71d 100644 --- a/README.md +++ b/README.md @@ -72,8 +72,9 @@ cp .env.example .env DATABASE_URL="postgresql://wraith:wraith@localhost:5432/wraith" DIRECT_DATABASE_URL="postgresql://wraith:wraith@localhost:5432/wraith" STELLAR_NETWORK="testnet" -# SOROBAN_RPC_URL is optional on testnet — the default public endpoint is used automatically -SOROBAN_RPC_URL= +# SOROBAN_RPC_URL_TESTNET (or legacy SOROBAN_RPC_URL) is optional on testnet — +# the default public endpoint (https://soroban-testnet.stellar.org) is used automatically +SOROBAN_RPC_URL_TESTNET= START_LEDGER= SAC_CONTRACT_IDS= @@ -86,8 +87,10 @@ PORT=3000 DATABASE_URL="postgresql://wraith:wraith@localhost:5432/wraith" DIRECT_DATABASE_URL="postgresql://wraith:wraith@localhost:5432/wraith" STELLAR_NETWORK="mainnet" -# Required on mainnet — no free public Soroban RPC exists -SOROBAN_RPC_URL="https://mainnet.stellar.validationcloud.io/v1/" +# Required on mainnet — no free public Soroban RPC exists: +# In multi-network setups (`NETWORKS=testnet,mainnet`), set SOROBAN_RPC_URL_MAINNET. +# In single-network deployments (`STELLAR_NETWORK=mainnet`), SOROBAN_RPC_URL may also be used. +SOROBAN_RPC_URL_MAINNET="https://mainnet.stellar.validationcloud.io/v1/" # Strongly recommended on mainnet: filter to specific contracts to reduce load SAC_CONTRACT_IDS="CTOKEN1...,CTOKEN2..." @@ -95,7 +98,7 @@ START_LEDGER= PORT=3000 ``` -> **Tip:** If you omit both `SOROBAN_RPC_URL` and `STELLAR_NETWORK`, Wraith will exit immediately with a clear error explaining what to set. +> **Tip:** In multi-network mode (`NETWORKS=testnet,mainnet`), `SOROBAN_RPC_URL_MAINNET` is required to index mainnet. If omitted, the mainnet indexer will not start and mainnet reads will be marked stale with an explicit staleness indicator (`stale: true`, `indexer_running: false`, header `X-Data-Stale: true`). Single-network deployments may still use `SOROBAN_RPC_URL` with `STELLAR_NETWORK=mainnet`. ### 3. Start Postgres @@ -686,33 +689,35 @@ curl -OJ "http://localhost:3000/transfers.parquet?fromLedger=5840000&toLedger=58 ## Environment Variables -| Variable | Default | Description | -| --------------------- | ------------- | --------------------------------------------------------------------------------------------- | -| `DATABASE_URL` | — | Postgres connection string (required) | -| `DIRECT_DATABASE_URL` | — | Direct (non-pooled) Postgres URL — required for Prisma migrations on Supabase | -| `STELLAR_NETWORK` | — | `testnet` or `mainnet`. Testnet auto-configures the default RPC URL. | -| `SOROBAN_RPC_URL` | *(see below)* | Soroban RPC endpoint. Overrides any network default. Required when `STELLAR_NETWORK=mainnet`. | -| `STELLAR_RPC_URL` | — | Backward-compat alias for `SOROBAN_RPC_URL`. Used when `SOROBAN_RPC_URL` is unset. | -| `HORIZON_URL` | — | Optional Horizon endpoint used as a fallback source when RPC is unhealthy. | -| `HORIZON_EVENTS_PATH` | `/events` | Horizon contract-events path used by the fallback source. | -| `START_LEDGER` | *(tip)* | Ledger to start indexing from. Leave blank to resume from DB state or start near the tip. | -| `POLL_INTERVAL_MS` | `6000` | Polling interval in ms (\~1 ledger ≈ 6 s) | -| `CONTRACT_IDS` | *(all)* | Comma-separated token contract IDs to watch. Empty = watch all (very heavy on mainnet) | -| `EVENTS_BATCH_SIZE` | `10000` | Max events per RPC call (Stellar RPC hard-cap is 10 000) | -| `RETENTION_DAYS` | `30` | Delete transfers older than N days (keeps DB within free-tier limits) | -| `NETWORKS` | *(`STELLAR_NETWORK`)* | Comma-separated networks to index in one process, e.g. `testnet,mainnet`. Also the set the API's `?network=` selector accepts. | -| `PORT` | `3000` | REST API port | -| `EXPORT_MAX_ROWS` | `50000` | Default row cap for `/transfers.csv` and `/transfers.parquet`. Clamped to 500 000; a caller may ask for less with `?maxRows=`. | +| Variable | Default | Description | +| -------------------------- | ------------- | --------------------------------------------------------------------------------------------- | +| `DATABASE_URL` | — | Postgres connection string (required) | +| `DIRECT_DATABASE_URL` | — | Direct (non-pooled) Postgres URL — required for Prisma migrations on Supabase | +| `STELLAR_NETWORK` | — | `testnet` or `mainnet`. Default single-network mode. | +| `SOROBAN_RPC_URL_TESTNET` | `https://soroban-testnet.stellar.org` | Explicit Soroban RPC endpoint for testnet. | +| `SOROBAN_RPC_URL_MAINNET` | — | Explicit Soroban RPC endpoint for mainnet. Required to index mainnet in multi-network setups. | +| `SOROBAN_RPC_URL` | *(see below)* | Legacy Soroban RPC endpoint. Scoped to the configured `STELLAR_NETWORK`. Required when `STELLAR_NETWORK=mainnet` if `SOROBAN_RPC_URL_MAINNET` is unset. | +| `STELLAR_RPC_URL` | — | Backward-compat alias for `SOROBAN_RPC_URL`. Used when `SOROBAN_RPC_URL` is unset. | +| `HORIZON_URL` | — | Optional Horizon endpoint used as a fallback source when RPC is unhealthy. | +| `HORIZON_EVENTS_PATH` | `/events` | Horizon contract-events path used by the fallback source. | +| `START_LEDGER` | *(tip)* | Ledger to start indexing from. Leave blank to resume from DB state or start near the tip. | +| `POLL_INTERVAL_MS` | `6000` | Polling interval in ms (\~1 ledger ≈ 6 s) | +| `CONTRACT_IDS` | *(all)* | Comma-separated token contract IDs to watch. Empty = watch all (very heavy on mainnet) | +| `EVENTS_BATCH_SIZE` | `10000` | Max events per RPC call (Stellar RPC hard-cap is 10 000) | +| `RETENTION_DAYS` | `30` | Delete transfers older than N days (keeps DB within free-tier limits) | +| `NETWORKS` | *(`STELLAR_NETWORK`)* | Comma-separated networks to index in one process, e.g. `testnet,mainnet`. Also the set the API's `?network=` selector accepts. | +| `PORT` | `3000` | REST API port | +| `EXPORT_MAX_ROWS` | `50000` | Default row cap for `/transfers.csv` and `/transfers.parquet`. Clamped to 500 000; a caller may ask for less with `?maxRows=`. | ### RPC URL Resolution -Wraith resolves the RPC endpoint in this order and fails fast at startup if nothing is configured: +Wraith resolves the RPC endpoint per network: -1. `SOROBAN_RPC_URL` — explicit; always wins -2. `STELLAR_RPC_URL` — backward-compat alias -3. `STELLAR_NETWORK=testnet` → `https://soroban-testnet.stellar.org` (free public endpoint) -4. `STELLAR_NETWORK=mainnet` → **error**: requires explicit `SOROBAN_RPC_URL` -5. Nothing set → **error**: clear message explaining what to configure +1. `SOROBAN_RPC_URL_` (or `STELLAR_RPC_URL_`) — explicit per-network variable (`SOROBAN_RPC_URL_TESTNET`, `SOROBAN_RPC_URL_MAINNET`). Always wins for that network. +2. `SOROBAN_RPC_URL` / `STELLAR_RPC_URL` — legacy variable; honoured **only for the configured network** (`STELLAR_NETWORK`), preventing accidental cross-network configuration. +3. `testnet` default → `https://soroban-testnet.stellar.org` (free public endpoint). +4. `mainnet` default → **error**: requires explicit `SOROBAN_RPC_URL_MAINNET` (or `SOROBAN_RPC_URL` with `STELLAR_NETWORK=mainnet`). There is no free public mainnet Soroban RPC. +5. If `startAllIndexers` encounters an unconfigured RPC for an enabled network (e.g. mainnet missing `SOROBAN_RPC_URL_MAINNET`), that network loop is marked not started, `/status` reports `running: false` with the exact error, and read endpoints for that network serve existing indexed data with explicit staleness markers (`stale: true`, `indexer_running: false`, `X-Data-Stale: true`). ### Indexer Source Fallback diff --git a/src/__tests__/multiNetworkIndexer.test.ts b/src/__tests__/multiNetworkIndexer.test.ts index 28968385..6e3334f3 100644 --- a/src/__tests__/multiNetworkIndexer.test.ts +++ b/src/__tests__/multiNetworkIndexer.test.ts @@ -15,6 +15,9 @@ import { resolveNftContractIds, resolveSacContractIds, runningNetworks, + startAllIndexers, + isIndexerRunning, + getIndexerUnstartedReason, _resetIndexerLoops, } from "../indexer"; import { getRpc, validateNetworkConfig, _resetRpcClients } from "../rpc"; @@ -208,4 +211,24 @@ describe("loop state isolation", () => { process.env.STELLAR_NETWORK = "testnet"; expect(currentNetwork()).toBe("testnet"); }); + + it("handles unconfigured RPC path gracefully and tracks unstarted status (#238)", () => { + // testnet has public default; mainnet has no SOROBAN_RPC_URL_MAINNET + const consoleError = jest.spyOn(console, "error").mockImplementation(() => {}); + const consoleLog = jest.spyOn(console, "log").mockImplementation(() => {}); + + try { + // Pass both networks where mainnet is missing its RPC endpoint + const runnable = startAllIndexers(["testnet", "mainnet"]); + + expect(runnable).toContain("testnet"); + expect(runnable).not.toContain("mainnet"); + expect(isIndexerRunning("mainnet")).toBe(false); + expect(getIndexerUnstartedReason("mainnet")).toMatch(/SOROBAN_RPC_URL_MAINNET is required/); + } finally { + consoleError.mockRestore(); + consoleLog.mockRestore(); + } + }); }); + diff --git a/src/__tests__/staleReads.test.ts b/src/__tests__/staleReads.test.ts index 2d65d0ec..4325b0ab 100644 --- a/src/__tests__/staleReads.test.ts +++ b/src/__tests__/staleReads.test.ts @@ -23,15 +23,22 @@ jest.mock("../indexer", () => ({ getIndexerStats: jest .fn() .mockReturnValue({ startedAt: "2024-01-01T00:00:00.000Z", uptimeSeconds: 100, totalIndexed: 50 }), + isIndexerRunning: jest.fn().mockReturnValue(true), + isIndexerStopped: jest.fn().mockReturnValue(false), + getIndexerUnstartedReason: jest.fn().mockReturnValue(undefined), })); import { queryTransfers, getLastIndexedLedger, prisma } from "../db"; import { getLatestLedger } from "../rpc"; +import { isIndexerRunning, isIndexerStopped, getIndexerUnstartedReason } from "../indexer"; const mockQueryTransfers = queryTransfers as jest.MockedFunction; const mockGetLastIndexedLedger = getLastIndexedLedger as jest.MockedFunction; const mockGetLatestLedger = getLatestLedger as jest.MockedFunction; const mockQueryRaw = prisma.$queryRaw as jest.MockedFunction; +const mockIsIndexerRunning = isIndexerRunning as jest.MockedFunction; +const mockIsIndexerStopped = isIndexerStopped as jest.MockedFunction; +const mockGetIndexerUnstartedReason = getIndexerUnstartedReason as jest.MockedFunction; const ALICE = "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWHF"; @@ -95,6 +102,41 @@ describe("Graceful stale reads during RPC outage (#164)", () => { expect(res2.headers["x-data-stale"]).toBeUndefined(); expect(res2.body.stale).toBeUndefined(); }); + + it("returns DB data with staleness and indexer_running=false when indexer is stopped/unconfigured (#238)", async () => { + mockIsIndexerRunning.mockReturnValue(false); + mockIsIndexerStopped.mockReturnValue(true); + mockQueryTransfers.mockResolvedValue({ + total: 1, + transfers: [ + { + id: 1, + contractId: "C123", + eventType: "transfer", + fromAddress: "GBOB", + toAddress: ALICE, + amount: "10000000", + ledger: 999, + ledgerClosedAt: new Date("2025-01-01T00:00:00Z"), + txHash: "tx123", + eventId: "evt123", + }, + ], + hasMore: false, + nextCursor: null, + }); + + const res = await request(app).get(`/transfers/incoming/${ALICE}`); + + expect(res.status).toBe(200); + expect(res.headers["x-data-stale"]).toBe("true"); + expect(res.headers["x-indexer-running"]).toBe("false"); + expect(res.headers["x-as-of-ledger"]).toBe("1000"); + expect(res.body.stale).toBe(true); + expect(res.body.indexer_running).toBe(false); + expect(res.body.as_of_ledger).toBe(1000); + expect(res.body.transfers).toHaveLength(1); + }); }); describe("GET /status during outages", () => { @@ -140,6 +182,43 @@ describe("Graceful stale reads during RPC outage (#164)", () => { expect(res.headers["x-data-stale"]).toBe("true"); }); + it("reports running state and lag per network when an indexer is stopped (#238)", async () => { + const origNetworks = process.env.NETWORKS; + try { + process.env.NETWORKS = "testnet,mainnet"; + mockIsIndexerRunning.mockImplementation((net) => net !== "mainnet"); + mockGetIndexerUnstartedReason.mockImplementation((net) => + net === "mainnet" ? "SOROBAN_RPC_URL_MAINNET is required" : undefined + ); + mockGetLastIndexedLedger.mockImplementation(async (net) => (net === "mainnet" ? 500 : 1000)); + mockGetLatestLedger.mockImplementation(async (net) => (net === "mainnet" ? 800 : 1050)); + + const res = await request(app).get("/status?network=testnet"); + + expect(res.status).toBe(200); + expect(res.body.networks).toBeDefined(); + expect(res.body.networks.testnet).toMatchObject({ + running: true, + lastIndexedLedger: 1000, + latestLedger: 1050, + lagLedgers: 50, + }); + expect(res.body.networks.mainnet).toMatchObject({ + running: false, + reason: "SOROBAN_RPC_URL_MAINNET is required", + lastIndexedLedger: 500, + latestLedger: 800, + lagLedgers: 300, + }); + } finally { + if (origNetworks !== undefined) { + process.env.NETWORKS = origNetworks; + } else { + delete process.env.NETWORKS; + } + } + }); + it("returns status 'down' (HTTP 503) when DB is down", async () => { mockGetLastIndexedLedger.mockRejectedValue(new Error("DB connection lost")); diff --git a/src/api.ts b/src/api.ts index e1ff0568..43856614 100644 --- a/src/api.ts +++ b/src/api.ts @@ -5,7 +5,7 @@ import { jsonApiMiddleware } from "./middleware/jsonapi"; import { queryHostFnLogs } from "./indexer/host-fn-log"; import { queryTransfers, queryAllTransfers, queryByTxHash, querySummary, queryNftTransfers, getNftOwner, getNftMetadata, getLastIndexedLedger, prisma } from "./db"; import { getLatestLedger } from "./rpc"; -import { getIndexerStats, getAllIndexerStats, runningNetworks } from "./indexer"; +import { getIndexerStats, getAllIndexerStats, runningNetworks, isIndexerRunning, isIndexerStopped, getIndexerUnstartedReason } from "./indexer"; import { currentNetwork, enabledNetworks, type Network } from "./network"; import { createAccountsRouter } from "./api/accounts"; import { createWebhooksRouter } from "./api/webhooks"; @@ -219,10 +219,17 @@ export function createApp(): express.Application { try { const net = requestNetwork(req); + const indexerStopped = + typeof isIndexerStopped === "function" + ? isIndexerStopped(net) + : false; const rpcHealthy = await checkRpcHealth(net); - if (!rpcHealthy) { + if (indexerStopped || !rpcHealthy) { const lastIndexed = await getLastIndexedLedger(net).catch(() => null); res.setHeader("X-Data-Stale", "true"); + if (indexerStopped) { + res.setHeader("X-Indexer-Running", "false"); + } if (lastIndexed !== null) { res.setHeader("X-As-Of-Ledger", String(lastIndexed)); } @@ -233,6 +240,9 @@ export function createApp(): express.Application { if (body.stale === undefined) { body.stale = true; } + if (body.indexer_running === undefined && indexerStopped) { + body.indexer_running = false; + } if (body.as_of_ledger === undefined && lastIndexed !== null) { body.as_of_ledger = lastIndexed; } @@ -459,8 +469,7 @@ export function createApp(): express.Application { // were so existing consumers are unaffected; this reports each loop // separately, which is the only way to see one chain falling behind // while the other is healthy. - const active = runningNetworks(); - const networks: Network[] = active.length > 0 ? active : enabledNetworks(); + const networks = enabledNetworks(); const loopStats = getAllIndexerStats(); const perNetwork = await Promise.all( networks.map(async (net) => { @@ -471,6 +480,14 @@ export function createApp(): express.Application { const indexedLedger = indexed.status === "fulfilled" ? indexed.value : null; const tipLedger = tip.status === "fulfilled" && typeof tip.value === "number" ? tip.value : null; + const isRunning = + typeof isIndexerRunning === "function" + ? isIndexerRunning(net) + : (loopStats[net] !== undefined || runningNetworks().includes(net)); + const unstartedReason = + !isRunning && typeof getIndexerUnstartedReason === "function" + ? getIndexerUnstartedReason(net) + : undefined; return [ net, { @@ -478,9 +495,8 @@ export function createApp(): express.Application { latestLedger: tipLedger, lagLedgers: tipLedger !== null ? tipLedger - (indexedLedger ?? tipLedger) : null, - // A loop that has not started reports no stats rather than zeroes - // pretending to be progress. - running: loopStats[net] !== undefined, + running: isRunning, + ...(unstartedReason ? { reason: unstartedReason } : {}), ...(loopStats[net] ?? {}), }, ] as const; diff --git a/src/indexer.ts b/src/indexer.ts index 03363be7..4dde673b 100644 --- a/src/indexer.ts +++ b/src/indexer.ts @@ -211,9 +211,33 @@ export function runningNetworks(): Network[] { return [...loops.keys()]; } -/** Test-only: drops loop state between cases. */ +/** Networks that were enabled but failed to start, with the reason. */ +const unstartedNetworks = new Map(); + +export function isIndexerRunning(network: Network): boolean { + return loops.has(network); +} + +export function isIndexerStopped(network: Network): boolean { + return unstartedNetworks.has(network); +} + +export function getIndexerUnstartedReason(network: Network): string | undefined { + return unstartedNetworks.get(network); +} + +export function getUnstartedNetworks(): Record { + return Object.fromEntries(unstartedNetworks); +} + +export function setIndexerUnstarted(network: Network, reason: string): void { + unstartedNetworks.set(network, reason); +} + +/** Test-only: drops loop state and unstarted records between cases. */ export function _resetIndexerLoops(): void { loops.clear(); + unstartedNetworks.clear(); } /** @@ -545,9 +569,10 @@ export function startAllIndexers(networks: Network[] = enabledNetworks()): Netwo validateNetworkConfig([net]); runnable.push(net); } catch (err) { + const message = (err as Error).message; + setIndexerUnstarted(net, message); console.error( - `[indexer/${net}] NOT STARTED — ${(err as Error).message} -` + + `[indexer/${net}] NOT STARTED — ${message}\n` + `[indexer/${net}] Reads for this network will serve whatever is already ` + `indexed and fall behind until it is configured. Other networks and the ` + `HTTP API are unaffected.`, diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index 6e7bdc17..9ae4c010 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -215,12 +215,24 @@ export const statusResponseSchema = z.object({ /** The network the top-level fields describe; `networks` reports every loop. */ network: z.enum(["testnet", "mainnet"]).optional(), lastIndexedLedger: z.number().int().nullable(), - latestLedger: z.number().int(), - lagLedgers: z.number().int(), - startedAt: z.string().datetime({ offset: true }), - uptimeSeconds: z.number().int(), - totalIndexed: z.number().int(), -}); + latestLedger: z.number().int().nullable(), + lagLedgers: z.number().int().nullable(), + startedAt: z.string().datetime({ offset: true }).optional(), + uptimeSeconds: z.number().int().optional(), + totalIndexed: z.number().int().optional(), + networks: z + .record( + z.string(), + z.object({ + lastIndexedLedger: z.number().int().nullable(), + latestLedger: z.number().int().nullable(), + lagLedgers: z.number().int().nullable(), + running: z.boolean(), + reason: z.string().optional(), + }).passthrough() + ) + .optional(), +}).passthrough(); export const transferSchema = z.object({ id: z.number().int(), From e5cdeb12d0b01506b430fcf8bdb289704ad8b54b Mon Sep 17 00:00:00 2001 From: Miracle656 Date: Sun, 18 Oct 2026 11:02:08 +0100 Subject: [PATCH 2/2] docs(openapi): regenerate after the status response gained staleness fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `the committed openapi.json matches the generator output` was the only red: this PR adds fields to the status schema in src/openapi/schemas.ts without re-running the generator, so the committed document described the previous shape. npm run docs:openapi The generator is deterministic, so this is mechanical — but the test exists because a spec that silently lags the code is worse than no spec: callers build against the document, not the handler. openapiCoverage.test.ts: 7 passed. Claude-Session: https://claude.ai/code/session_01USgemLt4Rnz4SGB1Srf3GB --- openapi.json | 51 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/openapi.json b/openapi.json index 055b139c..ed1cb141 100644 --- a/openapi.json +++ b/openapi.json @@ -545,10 +545,12 @@ "nullable": true }, "latestLedger": { - "type": "integer" + "type": "integer", + "nullable": true }, "lagLedgers": { - "type": "integer" + "type": "integer", + "nullable": true }, "startedAt": { "type": "string", @@ -559,17 +561,52 @@ }, "totalIndexed": { "type": "integer" + }, + "networks": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "lastIndexedLedger": { + "type": "integer", + "nullable": true + }, + "latestLedger": { + "type": "integer", + "nullable": true + }, + "lagLedgers": { + "type": "integer", + "nullable": true + }, + "running": { + "type": "boolean" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "lastIndexedLedger", + "latestLedger", + "lagLedgers", + "running" + ], + "additionalProperties": { + "nullable": true + } + } } }, "required": [ "ok", "lastIndexedLedger", "latestLedger", - "lagLedgers", - "startedAt", - "uptimeSeconds", - "totalIndexed" - ] + "lagLedgers" + ], + "additionalProperties": { + "nullable": true + } } } }