From 68dd7a7035d685aeab07c85b51a73f8a0928e3d7 Mon Sep 17 00:00:00 2001 From: Hayden Shively Date: Thu, 6 Aug 2026 10:39:40 -0500 Subject: [PATCH 1/2] feat(midnight-liquidation): union the market whitelist across sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Staging and prod pointed at different Midnight markets APIs (api.morpho.dev vs api.morpho.org), so the two deployments disagreed about which external service defines the market whitelist. The dev list is a strict superset of prod's — the same 6 markets plus 13 daily-maturity test markets — and the liquidation-candidates endpoint is byte-identical across both hosts, so the whitelist was the only real difference. `MARKETS_API_URL` now accepts a comma-separated list of endpoints, and the whitelist is the union across them. Both deployments can then hold the same value: the difference becomes which markets are listed, not which API is trusted, and staging finally exercises the endpoint prod depends on. The max-age staleness rule is applied PER SOURCE, which is what makes reading two endpoints safe in both directions: a source that goes down or goes stale drops out of the union (markets.source_expired) instead of either emptying the whitelist and halting all liquidations, or letting a stale set keep a since-delisted market in scope. Only when every source is stale is the whitelist empty (markets.whitelist_expired, fail-closed). `refresh` fans out concurrently and never throws — a per-source failure keeps that source's last-known-good and still lands its healthy peers, because a partial refresh must not read as a total one. The default stays the single public endpoint: an additional source widens what the bot will spend real capital on, so it is opt-in per deployment rather than shipped in the default. A single-URL value parses exactly as before. Co-Authored-By: Claude Opus 5 (1M context) --- bots/midnight-liquidation/README.md | 90 +++++++----- bots/midnight-liquidation/src/config.ts | 47 ++++-- .../src/discovery/markets.ts | 123 ++++++++++++++-- bots/midnight-liquidation/src/index.ts | 55 +++---- bots/midnight-liquidation/test/config.test.ts | 32 ++++- .../test/discovery/markets.test.ts | 136 +++++++++++++++++- 6 files changed, 389 insertions(+), 94 deletions(-) diff --git a/bots/midnight-liquidation/README.md b/bots/midnight-liquidation/README.md index cd1c89ab..1af9dbfb 100644 --- a/bots/midnight-liquidation/README.md +++ b/bots/midnight-liquidation/README.md @@ -35,7 +35,8 @@ This package is operational code, but it is still intentionally narrow: - A deployed permissionless Executor contract. If `EXECUTOOOR_ADDRESS` is unset, the bot uses the deterministic address derived by `@repo/contracts`; startup still requires code to exist there. - Network access to the markets liquidation-candidates API and the Midnight markets API (both public - by default; override with `LIQUIDATION_CANDIDATES_API_URL` / `MARKETS_API_URL`). + by default; override with `LIQUIDATION_CANDIDATES_API_URL` / `MARKETS_API_URL`, the latter accepting + a comma-separated list of endpoints whose whitelists are unioned). - At least one enabled venue to actually swap-liquidate: `ENABLE_LIFI=true` (or a `LIFI_API_KEY`), `ZEROX_API_KEY`, and/or `ONEINCH_API_KEY`. With none enabled the bot can only discover positions and realize bad debt, and refuses to start unless `ALLOW_BAD_DEBT_ONLY=true` is set. @@ -46,42 +47,42 @@ Never commit `.env` files, private keys, or RPC credentials. Environment variables: -| Var | Required | Default | Purpose | -| --------------------------------------------------------- | -------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `CHAIN_ID` | yes | — | Must be `8453` for Base. | -| `RPC_URL` | yes | — | Base RPC used for reads, simulation, and sends. Must be a full RPC that relays `eth_sendRawTransaction` — a read-only relay that acks sends without forwarding them to the sequencer would leave every tx unmined. | -| `RPC_URL_FALLBACK` | no | — | Optional fallback RPC for the signer's transport. | -| `LIQUIDATOR_PRIVATE_KEY` | yes | — | `0x`-prefixed 32-byte private key for the sender EOA. | -| `EXECUTOOOR_ADDRESS` | no | derived | Override for the shared Executor address. | -| `LIQUIDATION_CANDIDATES_API_URL` | no | public | Liquidation-candidates endpoint polled for borrower discovery. Defaults to the public Morpho markets API; validated as a URL at startup (fail-loud). | -| `HEALTH_FACTOR_LTE` | no | `1.02` | Health-factor cutoff sent to discovery (`health_factor_lte`); matured positions are always included regardless. Floored at `1.0`. Over-inclusive by design — the on-chain lens is the source of truth. | -| `MARKETS_API_URL` | no | public | Midnight markets endpoint used as the market whitelist (`listed=true`). Defaults to the public Morpho markets API; validated as a URL at startup. | -| `MARKETS_REFRESH_MS` | no | `60000` | How often the whitelist is refreshed. The endpoint is Morpho's own (not rate-limited); last-known-good is served on a transient failure. | -| `ZEROX_API_KEY` | cond. | — | Enables the `0x` venue when set. Read at point of use; never stored on config or logged. | -| `ONEINCH_API_KEY` | cond. | — | Enables the `1inch` venue when set. Read at point of use; never stored on config or logged. | -| `ENABLE_LIFI` | no | `false` | Enables the keyless `lifi` venue. Also implicitly enabled when `LIFI_API_KEY` is set. | -| `LIFI_API_KEY` | no | — | Optional; LiFi routes keyless, a key only raises its rate limits (and enables the venue). Read at point of use; never logged. | -| `ALLOW_BAD_DEBT_ONLY` | no | `false` | When no venue is enabled, the bot refuses to start unless this is `true` (then it runs bad-debt-only: discovers positions, realizes bad debt, never swap-liquidates). | -| `SLIPPAGE_BPS` | no | `100` | Global max oracle-to-DEX output discount passed to every venue (bakes the on-chain min-out into calldata). Replaces the old per-collateral `slippageBps`. | -| `ZEROX_BASE_URL` / `ONEINCH_BASE_URL` / `LIFI_BASE_URL` | no | public | Optional venue API host overrides. | -| `EXCLUDE_COLLATERALS` | no | — | Comma-separated collateral addresses the bot must never seize/hold — skipped (no quote) even in a listed market. | -| `MAX_FEE_GWEI` | no | `300` | Hard max fee cap used by the pending transaction queue. | -| `PRIORITY_FEE_GWEI` | no | `0.1` | First-send tip. The bump path adds at most 1.42x (3 attempts × 12.5%) over ~15 blocks, so this value, not the ceiling, sets what the bot actually pays for inclusion. Must leave room for one bump under `MAX_FEE_GWEI`. | -| `LOG_LEVEL` | no | `info` | One of `debug`, `info`, `warn`, `error`. | -| `CACHE_DIR` | no | `.cache` | Soltag/deployless cache directory. | -| `QUOTE_TIMEOUT_MS` | no | `2500` | Per-quote HTTP deadline (the firm quote runs inside the per-block tick). | -| `HTTP_RPS` / `HTTP_BURST` | no | `2` / `5` | Per-venue token-bucket refill rate and burst for FIRM quotes. The 1inch free tier is 1 RPS — set `HTTP_RPS=1` if you only use 1inch. | -| `PROBE_HTTP_RPS` | no | `1` | Per-venue token-bucket rate for BACKGROUND probes, on a separate client so probe bursts never queue ahead of a live firm quote. | -| `PROBE_STALE_MS` | no | `600000` | Probe-cache TTL per pair. A pair is re-probed only when a liquidatable position touches it after the cache goes stale — no probe traffic on quiet markets. | -| `PROBE_LADDER` | no | `0.01,0.1,1,10,100` | Comma-separated log-scaled probe sizes in whole collateral tokens; converted per-collateral to base units. Venue rankings are cached per size bucket. | -| `HTTP_MAX_RETRIES` | no | `2` | Retries on 429/5xx/network (honoring `Retry-After`) before a quote fails. | -| `MAX_ROUTE_IMPACT_BPS` | no | `500` | Reject a venue's quoted output more than this far below the oracle reference (route-quality guard). | -| `SEIZE_CAP_MARGIN_BPS` | no | `30` | Headroom shaved off the on-chain repay cap when sizing a cap-binding seize, so a one-block oracle move can't trip the contract's RCF/debt check. `0` sizes right at the cap. | -| `PENDLE_SLIPPAGE_BPS` | no | `50` | Slippage for the Pendle PT → underlying unwrap hop (before the downstream venue sells). | -| `BACKOFF_BASE_BLOCKS` / `BACKOFF_MAX_BLOCKS` | no | `2` / `64` | Exponential per-position cooldown (in blocks) after a failed quote/simulate, bounding API + RPC usage under a backlog. | -| `POSITION_LIQUIDATION_COOLDOWN_MS` | no | `0` | Opt-in per-position cooldown (ms) after a failed liquidation attempt; `0` disables it (re-attempt every tick). | -| `BETTERSTACK_SOURCE_TOKEN` / `BETTERSTACK_INGESTING_HOST` | no | — | Opt-in log shipping; when both are set the bot's in-process loglayer transport ships structured logs to BetterStack (inert otherwise). | -| `BETTERSTACK_HEARTBEAT_URL` | no | — | Optional Better Stack Uptime heartbeat URL, pinged every minute; failures only log a warning and never interrupt liquidations. | +| Var | Required | Default | Purpose | +| --------------------------------------------------------- | -------- | ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `CHAIN_ID` | yes | — | Must be `8453` for Base. | +| `RPC_URL` | yes | — | Base RPC used for reads, simulation, and sends. Must be a full RPC that relays `eth_sendRawTransaction` — a read-only relay that acks sends without forwarding them to the sequencer would leave every tx unmined. | +| `RPC_URL_FALLBACK` | no | — | Optional fallback RPC for the signer's transport. | +| `LIQUIDATOR_PRIVATE_KEY` | yes | — | `0x`-prefixed 32-byte private key for the sender EOA. | +| `EXECUTOOOR_ADDRESS` | no | derived | Override for the shared Executor address. | +| `LIQUIDATION_CANDIDATES_API_URL` | no | public | Liquidation-candidates endpoint polled for borrower discovery. Defaults to the public Morpho markets API; validated as a URL at startup (fail-loud). | +| `HEALTH_FACTOR_LTE` | no | `1.02` | Health-factor cutoff sent to discovery (`health_factor_lte`); matured positions are always included regardless. Floored at `1.0`. Over-inclusive by design — the on-chain lens is the source of truth. | +| `MARKETS_API_URL` | no | public | Midnight markets endpoint(s) used as the market whitelist (`listed=true`). Accepts a comma-separated list, whose whitelists are unioned per-source (see below). Defaults to the public Morpho markets API; every entry is validated as a URL at startup (fail-loud). | +| `MARKETS_REFRESH_MS` | no | `60000` | How often the whitelist is refreshed. The endpoint is Morpho's own (not rate-limited); last-known-good is served on a transient failure. | +| `ZEROX_API_KEY` | cond. | — | Enables the `0x` venue when set. Read at point of use; never stored on config or logged. | +| `ONEINCH_API_KEY` | cond. | — | Enables the `1inch` venue when set. Read at point of use; never stored on config or logged. | +| `ENABLE_LIFI` | no | `false` | Enables the keyless `lifi` venue. Also implicitly enabled when `LIFI_API_KEY` is set. | +| `LIFI_API_KEY` | no | — | Optional; LiFi routes keyless, a key only raises its rate limits (and enables the venue). Read at point of use; never logged. | +| `ALLOW_BAD_DEBT_ONLY` | no | `false` | When no venue is enabled, the bot refuses to start unless this is `true` (then it runs bad-debt-only: discovers positions, realizes bad debt, never swap-liquidates). | +| `SLIPPAGE_BPS` | no | `100` | Global max oracle-to-DEX output discount passed to every venue (bakes the on-chain min-out into calldata). Replaces the old per-collateral `slippageBps`. | +| `ZEROX_BASE_URL` / `ONEINCH_BASE_URL` / `LIFI_BASE_URL` | no | public | Optional venue API host overrides. | +| `EXCLUDE_COLLATERALS` | no | — | Comma-separated collateral addresses the bot must never seize/hold — skipped (no quote) even in a listed market. | +| `MAX_FEE_GWEI` | no | `300` | Hard max fee cap used by the pending transaction queue. | +| `PRIORITY_FEE_GWEI` | no | `0.1` | First-send tip. The bump path adds at most 1.42x (3 attempts × 12.5%) over ~15 blocks, so this value, not the ceiling, sets what the bot actually pays for inclusion. Must leave room for one bump under `MAX_FEE_GWEI`. | +| `LOG_LEVEL` | no | `info` | One of `debug`, `info`, `warn`, `error`. | +| `CACHE_DIR` | no | `.cache` | Soltag/deployless cache directory. | +| `QUOTE_TIMEOUT_MS` | no | `2500` | Per-quote HTTP deadline (the firm quote runs inside the per-block tick). | +| `HTTP_RPS` / `HTTP_BURST` | no | `2` / `5` | Per-venue token-bucket refill rate and burst for FIRM quotes. The 1inch free tier is 1 RPS — set `HTTP_RPS=1` if you only use 1inch. | +| `PROBE_HTTP_RPS` | no | `1` | Per-venue token-bucket rate for BACKGROUND probes, on a separate client so probe bursts never queue ahead of a live firm quote. | +| `PROBE_STALE_MS` | no | `600000` | Probe-cache TTL per pair. A pair is re-probed only when a liquidatable position touches it after the cache goes stale — no probe traffic on quiet markets. | +| `PROBE_LADDER` | no | `0.01,0.1,1,10,100` | Comma-separated log-scaled probe sizes in whole collateral tokens; converted per-collateral to base units. Venue rankings are cached per size bucket. | +| `HTTP_MAX_RETRIES` | no | `2` | Retries on 429/5xx/network (honoring `Retry-After`) before a quote fails. | +| `MAX_ROUTE_IMPACT_BPS` | no | `500` | Reject a venue's quoted output more than this far below the oracle reference (route-quality guard). | +| `SEIZE_CAP_MARGIN_BPS` | no | `30` | Headroom shaved off the on-chain repay cap when sizing a cap-binding seize, so a one-block oracle move can't trip the contract's RCF/debt check. `0` sizes right at the cap. | +| `PENDLE_SLIPPAGE_BPS` | no | `50` | Slippage for the Pendle PT → underlying unwrap hop (before the downstream venue sells). | +| `BACKOFF_BASE_BLOCKS` / `BACKOFF_MAX_BLOCKS` | no | `2` / `64` | Exponential per-position cooldown (in blocks) after a failed quote/simulate, bounding API + RPC usage under a backlog. | +| `POSITION_LIQUIDATION_COOLDOWN_MS` | no | `0` | Opt-in per-position cooldown (ms) after a failed liquidation attempt; `0` disables it (re-attempt every tick). | +| `BETTERSTACK_SOURCE_TOKEN` / `BETTERSTACK_INGESTING_HOST` | no | — | Opt-in log shipping; when both are set the bot's in-process loglayer transport ships structured logs to BetterStack (inert otherwise). | +| `BETTERSTACK_HEARTBEAT_URL` | no | — | Optional Better Stack Uptime heartbeat URL, pinged every minute; failures only log a warning and never interrupt liquidations. | The bot **refuses to start** if no venue is enabled, unless `ALLOW_BAD_DEBT_ONLY=true` — a rotated or forgotten key (or a missing `ENABLE_LIFI`) must not silently disable liquidations. @@ -108,6 +109,21 @@ There is no swap config file. Instead: `listed=true`: it is a hard **whitelist** — a market not in the listed set is never discovered, probed, or liquidated (fail-closed). This shapes only what the bot acts on; the on-chain lens remains the correctness boundary, and a delisted-but-underwater position simply falls out of scope. + + `MARKETS_API_URL` accepts **more than one endpoint**, comma-separated, and the whitelist is the union + across them. This is how a deployment reads an additional list (e.g. one carrying extra + shorter-maturity markets for testing) without the two environments diverging in _which_ API they + trust: set the same list everywhere. The `LISTED_MARKETS_MAX_AGE_MS` staleness rule is applied **per + source**, so one endpoint going down or going stale narrows the whitelist to its still-fresh peers + (logged as `markets.source_expired`) rather than emptying it and halting all liquidations. Only when + _every_ source is stale is the whitelist empty — `markets.whitelist_expired`, fail-closed. Each + source's own set survives its own transient failures (last-known-good, `markets.refresh_failed`), and + a cold start with no successful fetch lists nothing. + + Note that adding a source **widens what the bot will spend real capital on**, and it is only as + trustworthy as the endpoint serving it — anyone who can list a market there can direct this bot's key + at it. `EXCLUDE_COLLATERALS` remains the operator-side veto. + - **Which venue** clears a given liquidation is chosen automatically. Aggregators are enabled by the mere presence of their API key (LiFi also via `ENABLE_LIFI`, since it works keyless). For each collateral→loan pair, a background job requests diff --git a/bots/midnight-liquidation/src/config.ts b/bots/midnight-liquidation/src/config.ts index c3987edf..d33fe00b 100644 --- a/bots/midnight-liquidation/src/config.ts +++ b/bots/midnight-liquidation/src/config.ts @@ -66,7 +66,10 @@ const DEFAULT_POSITION_LIQUIDATION_COOLDOWN_MS = 0 // bursts never queue ahead of a time-sensitive firm quote; log-scaled ladder sizes are whole // collateral tokens (converted per-collateral to base units). `PROBE_STALE_MS` caps probe cadence per // pair; a pair is re-probed only when a liquidatable position touches it after the cache goes stale. -const DEFAULT_MARKETS_API_URL = 'https://api.morpho.org/v0/midnight/markets' +// The whitelist may read MORE THAN ONE markets endpoint (comma-separated), unioned per-source — see +// `MarketsConfig`. The default is the single public endpoint: an additional source widens what the bot +// will touch, so it must be opted into explicitly per deployment rather than shipped in the default. +const DEFAULT_MARKETS_API_URLS = ['https://api.morpho.org/v0/midnight/markets'] const DEFAULT_MARKETS_REFRESH_MS = 60_000 const DEFAULT_SLIPPAGE_BPS = 100 const DEFAULT_PROBE_STALE_MS = 600_000 @@ -126,12 +129,18 @@ export type VenueConfig = { } /** - * The Midnight markets API used as the market WHITELIST: only listed markets are discovered, probed, + * The Midnight markets API(s) used as the market WHITELIST: only listed markets are discovered, probed, * and liquidated. Over-inclusion is impossible (fail-closed); the on-chain lens remains the - * correctness boundary. `refreshMs` caps how often the (cheap, non-rate-limited) endpoint is polled. + * correctness boundary. `refreshMs` caps how often the (cheap, non-rate-limited) endpoints are polled. */ export type MarketsConfig = { - apiUrl: string + /** + * One or more markets endpoints, in `MARKETS_API_URL` order and de-duplicated. The effective + * whitelist is the union across the sources that are still fresh, so a deployment can read an + * additional list (e.g. one carrying extra shorter-maturity markets) without either endpoint + * becoming a single point of failure. Each is validated as a URL at load (fail-loud). + */ + apiUrls: string[] refreshMs: number } @@ -261,6 +270,27 @@ function ladderEnv(env: Env, name: string, def: string[]): string[] { return sizes } +// Parses an optional comma-separated list of endpoint URLs, with a default, de-duplicating while +// preserving order. Fails loud on an all-empty value or any malformed element rather than silently +// dropping it — a dropped whitelist source would narrow the market set with no signal at all. +function urlListEnv(env: Env, name: string, def: string[]): string[] { + const raw = env[name]?.trim() + if (!raw) return def + const urls = raw + .split(',') + .map(part => part.trim()) + .filter(part => part.length > 0) + if (urls.length === 0) { + throw new Error(`${name} must contain at least one URL, got: ${env[name]}`) + } + for (const url of urls) { + if (tryCatch(() => new URL(url)).error) { + throw new Error(`${name} is not a valid URL: ${url}`) + } + } + return [...new Set(urls)] +} + // Parses an optional comma-separated list of addresses into checksummed `Address`es, with `[]` as the // default. Fails loud on any malformed element (operator error). function addressListEnv(env: Env, name: string): Address[] { @@ -390,13 +420,10 @@ export function loadConfig( excludeCollaterals: addressListEnv(env, 'EXCLUDE_COLLATERALS') } - // Market whitelist endpoint. Default to the public markets API; fail loud on a malformed override. - const marketsApiUrl = env.MARKETS_API_URL?.trim() || DEFAULT_MARKETS_API_URL - if (tryCatch(() => new URL(marketsApiUrl)).error) { - throw new Error(`MARKETS_API_URL is not a valid URL: ${marketsApiUrl}`) - } + // Market whitelist endpoint(s) — one, or a comma-separated list that is unioned per-source. Default + // to the public markets API; fail loud on any malformed entry. const markets: MarketsConfig = { - apiUrl: marketsApiUrl, + apiUrls: urlListEnv(env, 'MARKETS_API_URL', DEFAULT_MARKETS_API_URLS), refreshMs: intEnv(env, 'MARKETS_REFRESH_MS', DEFAULT_MARKETS_REFRESH_MS, { min: 1 }) } diff --git a/bots/midnight-liquidation/src/discovery/markets.ts b/bots/midnight-liquidation/src/discovery/markets.ts index e679fcf0..2ff2a921 100644 --- a/bots/midnight-liquidation/src/discovery/markets.ts +++ b/bots/midnight-liquidation/src/discovery/markets.ts @@ -1,7 +1,7 @@ import type { Logger } from '@repo/bot-kit' import type { Address, Hex } from 'viem' -import { delay, fetchWithRetry } from '@repo/utils' +import { delay, fetchWithRetry, tryCatch } from '@repo/utils' import createClient from 'openapi-fetch' import { isAddress, isHex } from 'viem' @@ -23,18 +23,19 @@ export type ApiMarket = Omit boolean refresh: () => Promise - snapshot: () => { markets: number; updatedAt: number | null } + snapshot: () => { source: string; markets: number; updatedAt: number | null } } /** The `fetch` shape `openapi-fetch` calls — a single `Request`. The global `fetch` satisfies it. */ @@ -72,6 +73,9 @@ export function createListedMarketFilter(deps: { ? deps.apiUrl.slice(0, -PATH.length) : new URL(deps.apiUrl).origin const client = createClient({ baseUrl, fetch: deps.fetchImpl ?? fetch }) + // Log/snapshot label for this source. The HOST only: enough to tell two sources apart in logs, with + // no room for a query string or credential to ride along (these endpoints are public either way). + const source = new URL(deps.apiUrl).host // Last-known-good: only replaced by a fully-successful refresh, so a transient failure keeps serving // the prior set rather than emptying the whitelist. @@ -101,12 +105,109 @@ export function createListedMarketFilter(deps: { } listed = next updatedAt = now() - deps.logger.info('markets.listed', { chainId: deps.chainId, markets: listed.size }) + deps.logger.info('markets.listed', { chainId: deps.chainId, source, markets: listed.size }) } return { isListed: marketId => listed.has(marketId.toLowerCase()), refresh, - snapshot: () => ({ markets: listed.size, updatedAt }) + snapshot: () => ({ source, markets: listed.size, updatedAt }) + } +} + +/** One source's contribution to the union, as reported by {@link UnionListedMarketFilter.snapshot}. */ +type UnionSourceSnapshot = { + /** Host of the endpoint this source reads (see the single-source `source` label). */ + source: string + markets: number + updatedAt: number | null + /** `true` when this source is past `maxAgeMs` and therefore contributes nothing to the union. */ + expired: boolean +} + +/** + * The composed whitelist across every configured markets source. `isListed` is the UNION over sources + * that are still fresh, so deployments can read more than one endpoint (e.g. the public list plus an + * additional list carrying extra markets) without either endpoint becoming a single point of failure. + */ +type UnionListedMarketFilter = { + isListed: (marketId: Hex) => boolean + refresh: () => Promise + snapshot: () => { sources: UnionSourceSnapshot[]; fresh: number } +} + +/** + * Composes single-source {@link ListedMarketFilter}s into one union filter. + * + * The staleness rule is applied PER SOURCE: a source older than `maxAgeMs` (or never successfully + * fetched) contributes nothing, while its still-fresh peers keep working. That is what makes reading + * two endpoints safe in both directions — one endpoint going down or going stale narrows the whitelist + * to the sources that are still trustworthy instead of either emptying it (halting all liquidations) or + * letting a stale set keep a since-delisted market in scope. + * + * Union semantics are additive, so the whitelist is only ever as wide as the sources the operator + * configured; it stays fail-closed on a cold start (every source has `updatedAt === null` → nothing is + * listed). + * + * `refresh` fans out to every source concurrently and NEVER throws: each source's failure is logged + * (`markets.refresh_failed`, with its host) and the others still land, because a partial refresh must + * not read as a total one. After the fan-out it re-evaluates freshness and warns — `markets.source_expired` + * when some sources are stale, `markets.whitelist_expired` when all of them are (the whitelist is then + * empty and no liquidation can proceed). `now` is injectable for tests. + */ +export function createUnionListedMarketFilter(deps: { + filters: ListedMarketFilter[] + maxAgeMs: number + logger: Logger + now?: () => number +}): UnionListedMarketFilter { + const now = deps.now ?? (() => Date.now()) + const ageOf = (filter: ListedMarketFilter): number => { + const { updatedAt } = filter.snapshot() + return updatedAt === null ? Infinity : now() - updatedAt + } + const isFresh = (filter: ListedMarketFilter): boolean => ageOf(filter) <= deps.maxAgeMs + const snapshot = () => { + const sources = deps.filters.map(filter => ({ + ...filter.snapshot(), + expired: !isFresh(filter) + })) + return { sources, fresh: sources.filter(source => !source.expired).length } + } + + return { + isListed: marketId => deps.filters.some(filter => isFresh(filter) && filter.isListed(marketId)), + refresh: async () => { + await Promise.all( + deps.filters.map(async filter => { + const { error } = await tryCatch(filter.refresh()) + if (error) { + deps.logger.warn('markets.refresh_failed', { + source: filter.snapshot().source, + detail: error.message + }) + } + }) + ) + const { sources, fresh } = snapshot() + const expired = sources.filter(source => source.expired).map(source => source.source) + if (expired.length === 0) return + if (fresh === 0) { + deps.logger.warn('markets.whitelist_expired', { + expired, + maxAgeMs: deps.maxAgeMs, + detail: + 'every markets source is older than max age — whitelist is empty (fail-closed) until a refresh lands' + }) + return + } + deps.logger.warn('markets.source_expired', { + expired, + maxAgeMs: deps.maxAgeMs, + detail: + 'markets source older than max age — excluded from the whitelist until a refresh lands' + }) + }, + snapshot } } diff --git a/bots/midnight-liquidation/src/index.ts b/bots/midnight-liquidation/src/index.ts index 9a0c5d08..078c30d7 100644 --- a/bots/midnight-liquidation/src/index.ts +++ b/bots/midnight-liquidation/src/index.ts @@ -40,7 +40,7 @@ import { discoverBorrowers, MAX_DISCOVERY_PAGES } from './discovery/borrowers' -import { createListedMarketFilter } from './discovery/markets' +import { createListedMarketFilter, createUnionListedMarketFilter } from './discovery/markets' import { encodeLiquidationExec } from './execution/encode-call' import { composeQuoting } from './quotes' import { runTick } from './runner/tick' @@ -148,15 +148,19 @@ async function main() { logger }) - // Market whitelist: only listed markets are discovered / probed / liquidated. Refresh once at - // startup (non-fatal — a failed first fetch leaves the set empty = fail-closed, and the timer below - // retries), then poll on an interval. - const listedMarkets = createListedMarketFilter({ - apiUrl: config.markets.apiUrl, - chainId: config.chainId, + // Market whitelist: only listed markets are discovered / probed / liquidated. One filter per + // configured markets source, unioned — the union applies the max-age rule PER SOURCE, so a source + // that goes down or goes stale drops out of the whitelist instead of emptying it. Refresh once at + // startup (non-fatal by construction — `refresh` never throws, and a failed first fetch leaves the + // set empty = fail-closed), then poll on an interval. + const listedMarkets = createUnionListedMarketFilter({ + filters: config.markets.apiUrls.map(apiUrl => + createListedMarketFilter({ apiUrl, chainId: config.chainId, logger }) + ), + maxAgeMs: LISTED_MARKETS_MAX_AGE_MS, logger }) - await tryCatch(listedMarkets.refresh()) + await listedMarkets.refresh() // Pre-swap converters for exotic collateral (ERC4626 shares, Pendle PTs → underlying). // Auto-detecting with per-process memoization. erc4626 first: a memoized eth_call beats consulting @@ -224,29 +228,14 @@ async function main() { chainId: config.chainId, healthFactorLte: config.discovery.healthFactorLte }) - // Age of the whitelist since its last successful refresh — the caller's staleness signal. `Infinity` - // before the first successful fetch (never-refreshed = fail-closed). - const whitelistAge = () => { - const { updatedAt } = listedMarkets.snapshot() - return updatedAt === null ? Infinity : Date.now() - updatedAt - } // Filter candidates to the market whitelist BEFORE the lens read — a non-listed market is never - // touched (fail-closed), and this also shrinks the lens batch. Past the fail-closed max-age (a - // sustained markets-API outage the refresh loop could not recover from) the whitelist is treated as - // EMPTY so a since-delisted market can never linger in scope on the back of a stale set. + // touched (fail-closed), and this also shrinks the lens batch. `isListed` already excludes any source + // past the fail-closed max-age (a sustained markets-API outage the refresh loop could not recover + // from), so a since-delisted market can never linger in scope on the back of a stale set; the union + // warns (`markets.source_expired` / `markets.whitelist_expired`) when that happens. const discover = async () => { const candidates = await discoverBorrowers(fetchPage, { logger, maxPages: MAX_DISCOVERY_PAGES }) - const whitelistExpired = whitelistAge() > LISTED_MARKETS_MAX_AGE_MS - if (whitelistExpired) { - logger.warn('markets.whitelist_expired', { - ageMs: whitelistAge(), - detail: - 'whitelist older than max age — treating as empty (fail-closed) until a refresh lands' - }) - } - const listed = whitelistExpired - ? [] - : candidates.filter(candidate => listedMarkets.isListed(candidate.marketId)) + const listed = candidates.filter(candidate => listedMarkets.isListed(candidate.marketId)) if (listed.length < candidates.length) { logger.info('discover.filtered', { total: candidates.length, listed: listed.length }) } @@ -354,15 +343,17 @@ async function main() { runner.start() // Refresh the market whitelist on an interval, independent of the block loop, via a delay-spaced - // self-reschedule (no busy loop). Each round is wrapped so a transient markets-API failure logs and - // keeps last-known-good rather than killing the schedule (or emptying the whitelist). Also re-emits - // the bad-debt-only health signal while no venue is keyed. + // self-reschedule (no busy loop). The union refreshes every source concurrently and reports each + // source's failure itself (`markets.refresh_failed`), keeping that source's last-known-good rather + // than emptying the whitelist; the tryCatch here is belt-and-braces so nothing can kill the schedule. + // Also re-emits the bad-debt-only health signal while no venue is keyed. let stopped = false const refreshMarketsLoop = async () => { await delay(config.markets.refreshMs) if (stopped) return const { error } = await tryCatch(listedMarkets.refresh()) - if (error) logger.warn('markets.refresh_failed', { detail: error.message }) + // `refresh` is contractually non-throwing, so reaching this is a bug, not an API blip. + if (error) logger.warn('markets.refresh_error', { detail: error.message }) if (venues.length === 0) { logger.warn('quoting.no_routes', { detail: 'still no venue API keys — bad-debt-only' }) } diff --git a/bots/midnight-liquidation/test/config.test.ts b/bots/midnight-liquidation/test/config.test.ts index 2b670a06..957f3198 100644 --- a/bots/midnight-liquidation/test/config.test.ts +++ b/bots/midnight-liquidation/test/config.test.ts @@ -57,7 +57,7 @@ describe('loadConfig', () => { expect(config.venues.zeroxBaseUrl).toBeUndefined() // Market whitelist + probe defaults. - expect(config.markets.apiUrl).toBe('https://api.morpho.org/v0/midnight/markets') + expect(config.markets.apiUrls).toEqual(['https://api.morpho.org/v0/midnight/markets']) expect(config.markets.refreshMs).toBe(60_000) expect(config.probe.staleMs).toBe(600_000) expect(config.probe.httpRps).toBe(1) @@ -267,16 +267,44 @@ describe('loadConfig', () => { baseEnv({ MARKETS_API_URL: 'https://custom.example/markets', MARKETS_REFRESH_MS: '5000' }), deps ) - expect(config.markets.apiUrl).toBe('https://custom.example/markets') + expect(config.markets.apiUrls).toEqual(['https://custom.example/markets']) expect(config.markets.refreshMs).toBe(5000) }) + it('parses a comma-separated MARKETS_API_URL into ordered, de-duplicated sources', () => { + const config = loadConfig( + baseEnv({ + MARKETS_API_URL: + 'https://a.example/v0/midnight/markets, https://b.example/v0/midnight/markets ,https://a.example/v0/midnight/markets' + }), + deps + ) + expect(config.markets.apiUrls).toEqual([ + 'https://a.example/v0/midnight/markets', + 'https://b.example/v0/midnight/markets' + ]) + }) + it('throws on a malformed MARKETS_API_URL', () => { expect(() => loadConfig(baseEnv({ MARKETS_API_URL: 'not a url' }), deps)).toThrow( /MARKETS_API_URL is not a valid URL/ ) }) + // A malformed entry must fail loud rather than leaving the valid sources behind: silently dropping + // one would narrow the whitelist with no signal. + it('throws when any entry of a MARKETS_API_URL list is malformed', () => { + expect(() => + loadConfig(baseEnv({ MARKETS_API_URL: 'https://a.example/markets,not a url' }), deps) + ).toThrow(/MARKETS_API_URL is not a valid URL: not a url/) + }) + + it('throws when MARKETS_API_URL holds only separators', () => { + expect(() => loadConfig(baseEnv({ MARKETS_API_URL: ' , ' }), deps)).toThrow( + /MARKETS_API_URL must contain at least one URL/ + ) + }) + it('parses PROBE_LADDER into raw string sizes and rejects a malformed element', () => { expect( loadConfig(baseEnv({ PROBE_LADDER: '0.5, 5, 50' }), deps).probe.ladderWholeTokens diff --git a/bots/midnight-liquidation/test/discovery/markets.test.ts b/bots/midnight-liquidation/test/discovery/markets.test.ts index a5ccf7fe..72b78045 100644 --- a/bots/midnight-liquidation/test/discovery/markets.test.ts +++ b/bots/midnight-liquidation/test/discovery/markets.test.ts @@ -3,7 +3,10 @@ import type { Hex } from 'viem' import { describe, expect, it } from 'bun:test' -import { createListedMarketFilter } from '../../src/discovery/markets' +import { + createListedMarketFilter, + createUnionListedMarketFilter +} from '../../src/discovery/markets' const API_URL = 'https://api.example/v0/midnight/markets' const LISTED: Hex = `0x${'a'.repeat(64)}` @@ -109,7 +112,7 @@ describe('createListedMarketFilter', () => { now: () => 111 }) await filter.refresh() - expect(filter.snapshot()).toEqual({ markets: 1, updatedAt: 111 }) + expect(filter.snapshot()).toEqual({ source: 'api.example', markets: 1, updatedAt: 111 }) }) it('retries a 429 honoring Retry-After', async () => { @@ -131,3 +134,132 @@ describe('createListedMarketFilter', () => { expect(filter.isListed(LISTED)).toBe(true) }) }) + +// A capturing logger so the union's operator-visible warnings are assertable. +const capturingLogger = () => { + const warns: { event: string; fields: Record }[] = [] + return { + warns, + logger: { + debug: () => {}, + info: () => {}, + error: () => {}, + warn: (event: string, fields?: Record) => { + warns.push({ event, fields: fields ?? {} }) + } + } satisfies Logger + } +} + +// One real single-source filter, so the union is exercised against the actual factory. `now` sets this +// source's `updatedAt`, which is what the union's per-source staleness rule reads. +const sourceFilter = (opts: { + host: string + markets?: Hex[] + now?: () => number + fail?: boolean +}) => + createListedMarketFilter({ + apiUrl: `https://${opts.host}/v0/midnight/markets`, + chainId: 8453, + logger: NOOP_LOGGER, + fetchImpl: async () => + opts.fail + ? jsonResponse({}, 500) + : jsonResponse({ data: (opts.markets ?? []).map(id => market(id)) }), + sleep: async () => {}, + now: opts.now + }) + +describe('createUnionListedMarketFilter', () => { + it('whitelists the union of every fresh source', async () => { + const union = createUnionListedMarketFilter({ + filters: [ + sourceFilter({ host: 'a.example', markets: [LISTED], now: () => 0 }), + sourceFilter({ host: 'b.example', markets: [OTHER_CHAIN], now: () => 0 }) + ], + maxAgeMs: 1_000, + logger: NOOP_LOGGER, + now: () => 0 + }) + await union.refresh() + + expect(union.isListed(LISTED)).toBe(true) // only in source a + expect(union.isListed(OTHER_CHAIN)).toBe(true) // only in source b + expect(union.isListed(UNLISTED)).toBe(false) // in neither + expect(union.snapshot().fresh).toBe(2) + expect(union.snapshot().sources.map(s => s.source)).toEqual(['a.example', 'b.example']) + }) + + it('is fail-closed before the first refresh (no source has a set yet)', () => { + const union = createUnionListedMarketFilter({ + filters: [sourceFilter({ host: 'a.example', markets: [LISTED] })], + maxAgeMs: 1_000, + logger: NOOP_LOGGER, + now: () => 0 + }) + expect(union.isListed(LISTED)).toBe(false) + expect(union.snapshot().fresh).toBe(0) + expect(union.snapshot().sources[0]?.expired).toBe(true) + }) + + // The core safety property of reading more than one source: staleness is judged PER SOURCE, so one + // endpoint going stale narrows the whitelist instead of emptying it. + it('drops an expired source from the union while a fresh peer keeps working', async () => { + const { logger, warns } = capturingLogger() + const union = createUnionListedMarketFilter({ + filters: [ + sourceFilter({ host: 'stale.example', markets: [LISTED], now: () => 0 }), + sourceFilter({ host: 'fresh.example', markets: [OTHER_CHAIN], now: () => 1_000 }) + ], + maxAgeMs: 100, + logger, + now: () => 1_000 + }) + await union.refresh() + + expect(union.isListed(LISTED)).toBe(false) // stale source contributes nothing + expect(union.isListed(OTHER_CHAIN)).toBe(true) // fresh peer still whitelists + expect(union.snapshot().fresh).toBe(1) + expect(warns.map(w => w.event)).toContain('markets.source_expired') + expect(warns.find(w => w.event === 'markets.source_expired')?.fields.expired).toEqual([ + 'stale.example' + ]) + }) + + it('treats an all-expired whitelist as empty and warns loud', async () => { + const { logger, warns } = capturingLogger() + const union = createUnionListedMarketFilter({ + filters: [sourceFilter({ host: 'stale.example', markets: [LISTED], now: () => 0 })], + maxAgeMs: 100, + logger, + now: () => 1_000 + }) + await union.refresh() + + expect(union.isListed(LISTED)).toBe(false) + expect(union.snapshot().fresh).toBe(0) + expect(warns.map(w => w.event)).toContain('markets.whitelist_expired') + }) + + // A partial refresh must not read as a total one: the failing source is reported and skipped, and the + // healthy source's set still lands. + it('never throws when a source fails, and still lands the healthy sources', async () => { + const { logger, warns } = capturingLogger() + const union = createUnionListedMarketFilter({ + filters: [ + sourceFilter({ host: 'down.example', fail: true, now: () => 0 }), + sourceFilter({ host: 'up.example', markets: [LISTED], now: () => 0 }) + ], + maxAgeMs: 1_000, + logger, + now: () => 0 + }) + + expect(await union.refresh()).toBeUndefined() + expect(union.isListed(LISTED)).toBe(true) + const failure = warns.find(w => w.event === 'markets.refresh_failed') + expect(failure?.fields.source).toBe('down.example') + expect(union.snapshot().fresh).toBe(1) + }) +}) From 13f31f0704f091daeae91c35ac320f83efbe73b9 Mon Sep 17 00:00:00 2001 From: Hayden Shively Date: Thu, 6 Aug 2026 15:07:57 -0500 Subject: [PATCH 2/2] fix(midnight-liquidation): address review findings on the whitelist union MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keeps production single-source (the public endpoint) and lets staging hold the superset — staging then exercises the endpoint production depends on without production spending real capital on markets whitelisted for testing. The union mechanism stays; only the intended deployment differs, and the README no longer tells operators to set the same value everywhere. Silent-failure fixes: - Report `markets.whitelist_expired` per tick from `discover` again, instead of once per refresh. The whitelist expires on LISTED_MARKETS_MAX_AGE_MS but was only re-checked on MARKETS_REFRESH_MS, which is unbounded — a longer interval (or a wedged refresh loop) left a total liquidation halt unreported for most of each interval, visible only as `discover.filtered { listed: 0 }`. - Throw when no markets source is configured. An empty union lists nothing, logs nothing, and is indistinguishable from a working fail-closed whitelist. - Warn `markets.listed_empty` when a source goes from some listed markets to none. A successful-but-empty response is authoritative, so it replaces last-known-good — and in a union a healthy peer would mask it entirely. - Restore the startup `tryCatch`: a first-fetch failure must not be fatal, and the comment claiming "non-fatal by construction" contradicted the code. - Emit `markets.refresh_error` at error, not warn — it means the union's non-throwing contract broke, which is not an API blip. Correctness and observability: - `current()` freezes the fresh-source set for one discovery pass, so a pass is judged against one staleness reading rather than re-deriving it per candidate. - Log the deduplicated union size as `markets.whitelist`. Per-source `markets.listed` counts overlap, so they can be neither summed nor maxed into the combined number. - Label a source by host AND path, so two sources on one host stay distinguishable; de-duplicate `MARKETS_API_URL` on the parsed URL so trivially different spellings of one endpoint are not polled and counted twice. Docs: correct the claim that EXCLUDE_COLLATERALS vetoes an added source (it is collateral-scoped, and every listed market shares one collateral), note that LISTED_MARKETS_MAX_AGE_MS is a build-time constant rather than a knob, and warn that rolling back past this release with a list-valued var will crash-loop. Co-Authored-By: Claude Opus 5 (1M context) --- bots/midnight-liquidation/README.md | 111 +++++---- bots/midnight-liquidation/src/config.ts | 12 +- .../src/discovery/markets.ts | 110 ++++++--- bots/midnight-liquidation/src/index.ts | 36 ++- bots/midnight-liquidation/test/config.test.ts | 10 + .../test/discovery/markets.test.ts | 231 +++++++++++++----- 6 files changed, 349 insertions(+), 161 deletions(-) diff --git a/bots/midnight-liquidation/README.md b/bots/midnight-liquidation/README.md index 1af9dbfb..2911ba0e 100644 --- a/bots/midnight-liquidation/README.md +++ b/bots/midnight-liquidation/README.md @@ -47,42 +47,42 @@ Never commit `.env` files, private keys, or RPC credentials. Environment variables: -| Var | Required | Default | Purpose | -| --------------------------------------------------------- | -------- | ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `CHAIN_ID` | yes | — | Must be `8453` for Base. | -| `RPC_URL` | yes | — | Base RPC used for reads, simulation, and sends. Must be a full RPC that relays `eth_sendRawTransaction` — a read-only relay that acks sends without forwarding them to the sequencer would leave every tx unmined. | -| `RPC_URL_FALLBACK` | no | — | Optional fallback RPC for the signer's transport. | -| `LIQUIDATOR_PRIVATE_KEY` | yes | — | `0x`-prefixed 32-byte private key for the sender EOA. | -| `EXECUTOOOR_ADDRESS` | no | derived | Override for the shared Executor address. | -| `LIQUIDATION_CANDIDATES_API_URL` | no | public | Liquidation-candidates endpoint polled for borrower discovery. Defaults to the public Morpho markets API; validated as a URL at startup (fail-loud). | -| `HEALTH_FACTOR_LTE` | no | `1.02` | Health-factor cutoff sent to discovery (`health_factor_lte`); matured positions are always included regardless. Floored at `1.0`. Over-inclusive by design — the on-chain lens is the source of truth. | -| `MARKETS_API_URL` | no | public | Midnight markets endpoint(s) used as the market whitelist (`listed=true`). Accepts a comma-separated list, whose whitelists are unioned per-source (see below). Defaults to the public Morpho markets API; every entry is validated as a URL at startup (fail-loud). | -| `MARKETS_REFRESH_MS` | no | `60000` | How often the whitelist is refreshed. The endpoint is Morpho's own (not rate-limited); last-known-good is served on a transient failure. | -| `ZEROX_API_KEY` | cond. | — | Enables the `0x` venue when set. Read at point of use; never stored on config or logged. | -| `ONEINCH_API_KEY` | cond. | — | Enables the `1inch` venue when set. Read at point of use; never stored on config or logged. | -| `ENABLE_LIFI` | no | `false` | Enables the keyless `lifi` venue. Also implicitly enabled when `LIFI_API_KEY` is set. | -| `LIFI_API_KEY` | no | — | Optional; LiFi routes keyless, a key only raises its rate limits (and enables the venue). Read at point of use; never logged. | -| `ALLOW_BAD_DEBT_ONLY` | no | `false` | When no venue is enabled, the bot refuses to start unless this is `true` (then it runs bad-debt-only: discovers positions, realizes bad debt, never swap-liquidates). | -| `SLIPPAGE_BPS` | no | `100` | Global max oracle-to-DEX output discount passed to every venue (bakes the on-chain min-out into calldata). Replaces the old per-collateral `slippageBps`. | -| `ZEROX_BASE_URL` / `ONEINCH_BASE_URL` / `LIFI_BASE_URL` | no | public | Optional venue API host overrides. | -| `EXCLUDE_COLLATERALS` | no | — | Comma-separated collateral addresses the bot must never seize/hold — skipped (no quote) even in a listed market. | -| `MAX_FEE_GWEI` | no | `300` | Hard max fee cap used by the pending transaction queue. | -| `PRIORITY_FEE_GWEI` | no | `0.1` | First-send tip. The bump path adds at most 1.42x (3 attempts × 12.5%) over ~15 blocks, so this value, not the ceiling, sets what the bot actually pays for inclusion. Must leave room for one bump under `MAX_FEE_GWEI`. | -| `LOG_LEVEL` | no | `info` | One of `debug`, `info`, `warn`, `error`. | -| `CACHE_DIR` | no | `.cache` | Soltag/deployless cache directory. | -| `QUOTE_TIMEOUT_MS` | no | `2500` | Per-quote HTTP deadline (the firm quote runs inside the per-block tick). | -| `HTTP_RPS` / `HTTP_BURST` | no | `2` / `5` | Per-venue token-bucket refill rate and burst for FIRM quotes. The 1inch free tier is 1 RPS — set `HTTP_RPS=1` if you only use 1inch. | -| `PROBE_HTTP_RPS` | no | `1` | Per-venue token-bucket rate for BACKGROUND probes, on a separate client so probe bursts never queue ahead of a live firm quote. | -| `PROBE_STALE_MS` | no | `600000` | Probe-cache TTL per pair. A pair is re-probed only when a liquidatable position touches it after the cache goes stale — no probe traffic on quiet markets. | -| `PROBE_LADDER` | no | `0.01,0.1,1,10,100` | Comma-separated log-scaled probe sizes in whole collateral tokens; converted per-collateral to base units. Venue rankings are cached per size bucket. | -| `HTTP_MAX_RETRIES` | no | `2` | Retries on 429/5xx/network (honoring `Retry-After`) before a quote fails. | -| `MAX_ROUTE_IMPACT_BPS` | no | `500` | Reject a venue's quoted output more than this far below the oracle reference (route-quality guard). | -| `SEIZE_CAP_MARGIN_BPS` | no | `30` | Headroom shaved off the on-chain repay cap when sizing a cap-binding seize, so a one-block oracle move can't trip the contract's RCF/debt check. `0` sizes right at the cap. | -| `PENDLE_SLIPPAGE_BPS` | no | `50` | Slippage for the Pendle PT → underlying unwrap hop (before the downstream venue sells). | -| `BACKOFF_BASE_BLOCKS` / `BACKOFF_MAX_BLOCKS` | no | `2` / `64` | Exponential per-position cooldown (in blocks) after a failed quote/simulate, bounding API + RPC usage under a backlog. | -| `POSITION_LIQUIDATION_COOLDOWN_MS` | no | `0` | Opt-in per-position cooldown (ms) after a failed liquidation attempt; `0` disables it (re-attempt every tick). | -| `BETTERSTACK_SOURCE_TOKEN` / `BETTERSTACK_INGESTING_HOST` | no | — | Opt-in log shipping; when both are set the bot's in-process loglayer transport ships structured logs to BetterStack (inert otherwise). | -| `BETTERSTACK_HEARTBEAT_URL` | no | — | Optional Better Stack Uptime heartbeat URL, pinged every minute; failures only log a warning and never interrupt liquidations. | +| Var | Required | Default | Purpose | +| --------------------------------------------------------- | -------- | ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `CHAIN_ID` | yes | — | Must be `8453` for Base. | +| `RPC_URL` | yes | — | Base RPC used for reads, simulation, and sends. Must be a full RPC that relays `eth_sendRawTransaction` — a read-only relay that acks sends without forwarding them to the sequencer would leave every tx unmined. | +| `RPC_URL_FALLBACK` | no | — | Optional fallback RPC for the signer's transport. | +| `LIQUIDATOR_PRIVATE_KEY` | yes | — | `0x`-prefixed 32-byte private key for the sender EOA. | +| `EXECUTOOOR_ADDRESS` | no | derived | Override for the shared Executor address. | +| `LIQUIDATION_CANDIDATES_API_URL` | no | public | Liquidation-candidates endpoint polled for borrower discovery. Defaults to the public Morpho markets API; validated as a URL at startup (fail-loud). | +| `HEALTH_FACTOR_LTE` | no | `1.02` | Health-factor cutoff sent to discovery (`health_factor_lte`); matured positions are always included regardless. Floored at `1.0`. Over-inclusive by design — the on-chain lens is the source of truth. | +| `MARKETS_API_URL` | no | public | Midnight markets endpoint(s) used as the market whitelist (`listed=true`). Accepts a comma-separated list, whose whitelists are unioned per-source (see below). Defaults to the public Morpho markets API; every entry is validated as a URL at startup (fail-loud). ⚠️ Set a list only after an image that supports it is live, and clear it back to one URL before rolling back — older images reject a list. | +| `MARKETS_REFRESH_MS` | no | `60000` | How often the whitelist is refreshed. The endpoint is Morpho's own (not rate-limited); last-known-good is served on a transient failure. | +| `ZEROX_API_KEY` | cond. | — | Enables the `0x` venue when set. Read at point of use; never stored on config or logged. | +| `ONEINCH_API_KEY` | cond. | — | Enables the `1inch` venue when set. Read at point of use; never stored on config or logged. | +| `ENABLE_LIFI` | no | `false` | Enables the keyless `lifi` venue. Also implicitly enabled when `LIFI_API_KEY` is set. | +| `LIFI_API_KEY` | no | — | Optional; LiFi routes keyless, a key only raises its rate limits (and enables the venue). Read at point of use; never logged. | +| `ALLOW_BAD_DEBT_ONLY` | no | `false` | When no venue is enabled, the bot refuses to start unless this is `true` (then it runs bad-debt-only: discovers positions, realizes bad debt, never swap-liquidates). | +| `SLIPPAGE_BPS` | no | `100` | Global max oracle-to-DEX output discount passed to every venue (bakes the on-chain min-out into calldata). Replaces the old per-collateral `slippageBps`. | +| `ZEROX_BASE_URL` / `ONEINCH_BASE_URL` / `LIFI_BASE_URL` | no | public | Optional venue API host overrides. | +| `EXCLUDE_COLLATERALS` | no | — | Comma-separated collateral addresses the bot must never seize/hold — skipped (no quote) even in a listed market. | +| `MAX_FEE_GWEI` | no | `300` | Hard max fee cap used by the pending transaction queue. | +| `PRIORITY_FEE_GWEI` | no | `0.1` | First-send tip. The bump path adds at most 1.42x (3 attempts × 12.5%) over ~15 blocks, so this value, not the ceiling, sets what the bot actually pays for inclusion. Must leave room for one bump under `MAX_FEE_GWEI`. | +| `LOG_LEVEL` | no | `info` | One of `debug`, `info`, `warn`, `error`. | +| `CACHE_DIR` | no | `.cache` | Soltag/deployless cache directory. | +| `QUOTE_TIMEOUT_MS` | no | `2500` | Per-quote HTTP deadline (the firm quote runs inside the per-block tick). | +| `HTTP_RPS` / `HTTP_BURST` | no | `2` / `5` | Per-venue token-bucket refill rate and burst for FIRM quotes. The 1inch free tier is 1 RPS — set `HTTP_RPS=1` if you only use 1inch. | +| `PROBE_HTTP_RPS` | no | `1` | Per-venue token-bucket rate for BACKGROUND probes, on a separate client so probe bursts never queue ahead of a live firm quote. | +| `PROBE_STALE_MS` | no | `600000` | Probe-cache TTL per pair. A pair is re-probed only when a liquidatable position touches it after the cache goes stale — no probe traffic on quiet markets. | +| `PROBE_LADDER` | no | `0.01,0.1,1,10,100` | Comma-separated log-scaled probe sizes in whole collateral tokens; converted per-collateral to base units. Venue rankings are cached per size bucket. | +| `HTTP_MAX_RETRIES` | no | `2` | Retries on 429/5xx/network (honoring `Retry-After`) before a quote fails. | +| `MAX_ROUTE_IMPACT_BPS` | no | `500` | Reject a venue's quoted output more than this far below the oracle reference (route-quality guard). | +| `SEIZE_CAP_MARGIN_BPS` | no | `30` | Headroom shaved off the on-chain repay cap when sizing a cap-binding seize, so a one-block oracle move can't trip the contract's RCF/debt check. `0` sizes right at the cap. | +| `PENDLE_SLIPPAGE_BPS` | no | `50` | Slippage for the Pendle PT → underlying unwrap hop (before the downstream venue sells). | +| `BACKOFF_BASE_BLOCKS` / `BACKOFF_MAX_BLOCKS` | no | `2` / `64` | Exponential per-position cooldown (in blocks) after a failed quote/simulate, bounding API + RPC usage under a backlog. | +| `POSITION_LIQUIDATION_COOLDOWN_MS` | no | `0` | Opt-in per-position cooldown (ms) after a failed liquidation attempt; `0` disables it (re-attempt every tick). | +| `BETTERSTACK_SOURCE_TOKEN` / `BETTERSTACK_INGESTING_HOST` | no | — | Opt-in log shipping; when both are set the bot's in-process loglayer transport ships structured logs to BetterStack (inert otherwise). | +| `BETTERSTACK_HEARTBEAT_URL` | no | — | Optional Better Stack Uptime heartbeat URL, pinged every minute; failures only log a warning and never interrupt liquidations. | The bot **refuses to start** if no venue is enabled, unless `ALLOW_BAD_DEBT_ONLY=true` — a rotated or forgotten key (or a missing `ENABLE_LIFI`) must not silently disable liquidations. @@ -111,18 +111,33 @@ There is no swap config file. Instead: the correctness boundary, and a delisted-but-underwater position simply falls out of scope. `MARKETS_API_URL` accepts **more than one endpoint**, comma-separated, and the whitelist is the union - across them. This is how a deployment reads an additional list (e.g. one carrying extra - shorter-maturity markets for testing) without the two environments diverging in _which_ API they - trust: set the same list everywhere. The `LISTED_MARKETS_MAX_AGE_MS` staleness rule is applied **per - source**, so one endpoint going down or going stale narrows the whitelist to its still-fresh peers - (logged as `markets.source_expired`) rather than emptying it and halting all liquidations. Only when - _every_ source is stale is the whitelist empty — `markets.whitelist_expired`, fail-closed. Each - source's own set survives its own transient failures (last-known-good, `markets.refresh_failed`), and - a cold start with no successful fetch lists nothing. - - Note that adding a source **widens what the bot will spend real capital on**, and it is only as - trustworthy as the endpoint serving it — anyone who can list a market there can direct this bot's key - at it. `EXCLUDE_COLLATERALS` remains the operator-side veto. + across them. This lets a deployment read an additional list (e.g. one carrying extra shorter-maturity + markets for testing) without that list becoming the single source of truth for the whole whitelist. + The combined size is logged once per refresh as `markets.whitelist`; the per-source `markets.listed` + lines report each endpoint separately and must not be summed (sources overlap). + + The max-age staleness rule (`LISTED_MARKETS_MAX_AGE_MS` — a build-time constant of 10 minutes, not an + environment variable) is applied **per source**, so one endpoint going down or going stale narrows the + whitelist to its still-fresh peers (`markets.source_expired`) rather than emptying it and halting all + liquidations. Only when _every_ source is stale is the whitelist empty, reported every tick as + `markets.whitelist_expired` — fail-closed. Each source's own set survives its own transient failures + (last-known-good, `markets.refresh_failed`); a source that returns a successful but empty list is + authoritative, and because a healthy peer would otherwise mask it, that transition is called out as + `markets.listed_empty`. A cold start with no successful fetch lists nothing. + + **Environments deliberately differ here.** Staging reads both the public endpoint and the test-market + endpoint, so its whitelist is a strict superset of production's and it exercises the same API + production depends on. Production reads the public endpoint only. Adding a source **widens what the + bot will spend real capital on**, and the whitelist is only as trustworthy as the endpoints serving + it — anyone who can list a market on any configured endpoint can direct that deployment's key at it. + Note that `EXCLUDE_COLLATERALS` is a poor veto for this: it is collateral-scoped, and every currently + listed market shares one collateral, so excluding it would disable the real markets too. There is no + per-market denylist — the endpoint list _is_ the gate. + + ⚠️ **Rolling back past the release that added multi-source support will crash-loop the service** if + `MARKETS_API_URL` still holds a comma-separated value: the older image validates the variable as a + single URL and `loadConfig` rejects it at startup. Clear the variable to a single endpoint before + rolling back. This is also why the list must only be set _after_ the new image is live. - **Which venue** clears a given liquidation is chosen automatically. Aggregators are enabled by the mere presence of their API key (LiFi also via `ENABLE_LIFI`, since it works keyless). For each diff --git a/bots/midnight-liquidation/src/config.ts b/bots/midnight-liquidation/src/config.ts index d33fe00b..b4ddf91d 100644 --- a/bots/midnight-liquidation/src/config.ts +++ b/bots/midnight-liquidation/src/config.ts @@ -273,6 +273,8 @@ function ladderEnv(env: Env, name: string, def: string[]): string[] { // Parses an optional comma-separated list of endpoint URLs, with a default, de-duplicating while // preserving order. Fails loud on an all-empty value or any malformed element rather than silently // dropping it — a dropped whitelist source would narrow the market set with no signal at all. +// De-duplication is on the PARSED URL, so trivially different spellings of one endpoint (a trailing +// slash, a default port) collapse instead of being polled twice and counted twice. function urlListEnv(env: Env, name: string, def: string[]): string[] { const raw = env[name]?.trim() if (!raw) return def @@ -283,12 +285,14 @@ function urlListEnv(env: Env, name: string, def: string[]): string[] { if (urls.length === 0) { throw new Error(`${name} must contain at least one URL, got: ${env[name]}`) } - for (const url of urls) { - if (tryCatch(() => new URL(url)).error) { + const normalized = urls.map(url => { + const parsed = tryCatch(() => new URL(url)) + if (parsed.error) { throw new Error(`${name} is not a valid URL: ${url}`) } - } - return [...new Set(urls)] + return parsed.data.toString() + }) + return [...new Set(normalized)] } // Parses an optional comma-separated list of addresses into checksummed `Address`es, with `[]` as the diff --git a/bots/midnight-liquidation/src/discovery/markets.ts b/bots/midnight-liquidation/src/discovery/markets.ts index 2ff2a921..36ecc33b 100644 --- a/bots/midnight-liquidation/src/discovery/markets.ts +++ b/bots/midnight-liquidation/src/discovery/markets.ts @@ -34,6 +34,8 @@ export type ApiMarket = Omit boolean + /** This source's listed ids (lowercased), so the union can size the combined whitelist. */ + ids: () => ReadonlySet refresh: () => Promise snapshot: () => { source: string; markets: number; updatedAt: number | null } } @@ -69,13 +71,13 @@ export function createListedMarketFilter(deps: { }): ListedMarketFilter { const sleep = deps.sleep ?? delay const now = deps.now ?? (() => Date.now()) - const baseUrl = deps.apiUrl.endsWith(PATH) - ? deps.apiUrl.slice(0, -PATH.length) - : new URL(deps.apiUrl).origin + const url = new URL(deps.apiUrl) + const baseUrl = deps.apiUrl.endsWith(PATH) ? deps.apiUrl.slice(0, -PATH.length) : url.origin const client = createClient({ baseUrl, fetch: deps.fetchImpl ?? fetch }) - // Log/snapshot label for this source. The HOST only: enough to tell two sources apart in logs, with - // no room for a query string or credential to ride along (these endpoints are public either way). - const source = new URL(deps.apiUrl).host + // Log/snapshot label for this source. Host + path, so two sources sharing a host but differing by + // path prefix stay distinguishable in `markets.refresh_failed` / `markets.source_expired`; the query + // string is excluded so no credential can ride along (these endpoints are public either way). + const source = `${url.host}${url.pathname}` // Last-known-good: only replaced by a fully-successful refresh, so a transient failure keeps serving // the prior set rather than emptying the whitelist. @@ -103,6 +105,19 @@ export function createListedMarketFilter(deps: { if (!isAddress(market.loan_token, { strict: false })) continue next.add(market.market_id.toLowerCase()) } + // A successful-but-empty response is NOT a transient failure, so it legitimately replaces + // last-known-good and stamps `updatedAt` — but it silently drops this source to zero markets, and + // in a union a healthy peer would mask that entirely. Schema drift and an empty upstream database + // both look like this, so a nonempty→empty transition is called out loud rather than left to be + // inferred from `markets.listed { markets: 0 }` at info level. + if (listed.size > 0 && next.size === 0) { + deps.logger.warn('markets.listed_empty', { + chainId: deps.chainId, + source, + previous: listed.size, + detail: 'markets source returned zero listed markets where it previously returned some' + }) + } listed = next updatedAt = now() deps.logger.info('markets.listed', { chainId: deps.chainId, source, markets: listed.size }) @@ -110,6 +125,7 @@ export function createListedMarketFilter(deps: { return { isListed: marketId => listed.has(marketId.toLowerCase()), + ids: () => listed, refresh, snapshot: () => ({ source, markets: listed.size, updatedAt }) } @@ -126,12 +142,17 @@ type UnionSourceSnapshot = { } /** - * The composed whitelist across every configured markets source. `isListed` is the UNION over sources - * that are still fresh, so deployments can read more than one endpoint (e.g. the public list plus an - * additional list carrying extra markets) without either endpoint becoming a single point of failure. + * The composed whitelist across every configured markets source: the UNION over sources that are still + * fresh, so deployments can read more than one endpoint (e.g. the public list plus an additional list + * carrying extra markets) without either endpoint becoming a single point of failure. */ type UnionListedMarketFilter = { - isListed: (marketId: Hex) => boolean + /** + * Freezes the currently-fresh source set and returns a predicate over it, so ONE discovery pass is + * judged against ONE staleness reading. Re-deriving freshness per candidate would let a source cross + * the max-age boundary mid-pass, splitting a single pass across two different whitelists. + */ + current: () => { isListed: (marketId: Hex) => boolean; fresh: number } refresh: () => Promise snapshot: () => { sources: UnionSourceSnapshot[]; fresh: number } } @@ -150,10 +171,18 @@ type UnionListedMarketFilter = { * listed). * * `refresh` fans out to every source concurrently and NEVER throws: each source's failure is logged - * (`markets.refresh_failed`, with its host) and the others still land, because a partial refresh must - * not read as a total one. After the fan-out it re-evaluates freshness and warns — `markets.source_expired` - * when some sources are stale, `markets.whitelist_expired` when all of them are (the whitelist is then - * empty and no liquidation can proceed). `now` is injectable for tests. + * (`markets.refresh_failed`, with its source label) and the others still land, because a partial + * refresh must not read as a total one. It then emits `markets.whitelist` with the size of the combined + * whitelist — the per-source `markets.listed` lines cannot be summed or maxed into that number — and + * warns `markets.source_expired` when SOME sources are stale. The all-stale case is deliberately NOT + * warned here: the whitelist being empty is a per-tick condition the caller reports every block (see + * `markets.whitelist_expired` in the bot's `discover`), because a refresh interval longer than + * `maxAgeMs` would otherwise leave the halt unreported for most of each interval. + * + * Throws if `filters` is empty: an empty union lists nothing, which is indistinguishable from a working + * fail-closed whitelist and would halt every liquidation in silence. + * + * `now` is injectable for tests. */ export function createUnionListedMarketFilter(deps: { filters: ListedMarketFilter[] @@ -161,12 +190,16 @@ export function createUnionListedMarketFilter(deps: { logger: Logger now?: () => number }): UnionListedMarketFilter { + if (deps.filters.length === 0) { + throw new Error('createUnionListedMarketFilter requires at least one markets source') + } const now = deps.now ?? (() => Date.now()) - const ageOf = (filter: ListedMarketFilter): number => { + const ageOf = (filter: ListedMarketFilter) => { const { updatedAt } = filter.snapshot() return updatedAt === null ? Infinity : now() - updatedAt } - const isFresh = (filter: ListedMarketFilter): boolean => ageOf(filter) <= deps.maxAgeMs + const isFresh = (filter: ListedMarketFilter) => ageOf(filter) <= deps.maxAgeMs + const freshFilters = () => deps.filters.filter(isFresh) const snapshot = () => { const sources = deps.filters.map(filter => ({ ...filter.snapshot(), @@ -174,9 +207,31 @@ export function createUnionListedMarketFilter(deps: { })) return { sources, fresh: sources.filter(source => !source.expired).length } } + // Size of the combined whitelist. Derived from the id sets rather than the per-source counts, which + // overlap — summing them double-counts a market both sources list, and taking the max understates a + // union of two partially-overlapping sets. + const unionSize = () => new Set(freshFilters().flatMap(filter => [...filter.ids()])).size + const warnExpiry = () => { + const { sources, fresh } = snapshot() + const expired = sources.filter(source => source.expired).map(source => source.source) + // fresh === 0 is the caller's per-tick signal, not ours — see this factory's JSDoc. + if (expired.length === 0 || fresh === 0) return + deps.logger.warn('markets.source_expired', { + expired, + maxAgeMs: deps.maxAgeMs, + detail: + 'markets source older than max age — excluded from the whitelist until a refresh lands' + }) + } return { - isListed: marketId => deps.filters.some(filter => isFresh(filter) && filter.isListed(marketId)), + current: () => { + const fresh = freshFilters() + return { + isListed: marketId => fresh.some(filter => filter.isListed(marketId)), + fresh: fresh.length + } + }, refresh: async () => { await Promise.all( deps.filters.map(async filter => { @@ -190,23 +245,12 @@ export function createUnionListedMarketFilter(deps: { }) ) const { sources, fresh } = snapshot() - const expired = sources.filter(source => source.expired).map(source => source.source) - if (expired.length === 0) return - if (fresh === 0) { - deps.logger.warn('markets.whitelist_expired', { - expired, - maxAgeMs: deps.maxAgeMs, - detail: - 'every markets source is older than max age — whitelist is empty (fail-closed) until a refresh lands' - }) - return - } - deps.logger.warn('markets.source_expired', { - expired, - maxAgeMs: deps.maxAgeMs, - detail: - 'markets source older than max age — excluded from the whitelist until a refresh lands' + deps.logger.info('markets.whitelist', { + markets: unionSize(), + sources: sources.length, + fresh }) + warnExpiry() }, snapshot } diff --git a/bots/midnight-liquidation/src/index.ts b/bots/midnight-liquidation/src/index.ts index 078c30d7..301de36d 100644 --- a/bots/midnight-liquidation/src/index.ts +++ b/bots/midnight-liquidation/src/index.ts @@ -151,8 +151,8 @@ async function main() { // Market whitelist: only listed markets are discovered / probed / liquidated. One filter per // configured markets source, unioned — the union applies the max-age rule PER SOURCE, so a source // that goes down or goes stale drops out of the whitelist instead of emptying it. Refresh once at - // startup (non-fatal by construction — `refresh` never throws, and a failed first fetch leaves the - // set empty = fail-closed), then poll on an interval. + // startup (non-fatal — a failed first fetch leaves the set empty = fail-closed, and the timer below + // retries), then poll on an interval. const listedMarkets = createUnionListedMarketFilter({ filters: config.markets.apiUrls.map(apiUrl => createListedMarketFilter({ apiUrl, chainId: config.chainId, logger }) @@ -160,7 +160,7 @@ async function main() { maxAgeMs: LISTED_MARKETS_MAX_AGE_MS, logger }) - await listedMarkets.refresh() + await tryCatch(listedMarkets.refresh()) // Pre-swap converters for exotic collateral (ERC4626 shares, Pendle PTs → underlying). // Auto-detecting with per-process memoization. erc4626 first: a memoized eth_call beats consulting @@ -229,13 +229,28 @@ async function main() { healthFactorLte: config.discovery.healthFactorLte }) // Filter candidates to the market whitelist BEFORE the lens read — a non-listed market is never - // touched (fail-closed), and this also shrinks the lens batch. `isListed` already excludes any source - // past the fail-closed max-age (a sustained markets-API outage the refresh loop could not recover - // from), so a since-delisted market can never linger in scope on the back of a stale set; the union - // warns (`markets.source_expired` / `markets.whitelist_expired`) when that happens. + // touched (fail-closed), and this also shrinks the lens batch. `current()` freezes the fresh-source + // set for the whole pass, excluding any source past the fail-closed max-age (a sustained markets-API + // outage the refresh loop could not recover from), so a since-delisted market can never linger in + // scope on the back of a stale set. + // + // The all-sources-expired case is reported HERE, every tick, rather than from the refresh loop: the + // whitelist expires on `LISTED_MARKETS_MAX_AGE_MS` but is only re-checked on `MARKETS_REFRESH_MS`, so + // a longer refresh interval (or a wedged refresh loop) would otherwise leave a total liquidation halt + // unreported for most of each interval — visible only as `discover.filtered` reading `listed: 0`, + // which is indistinguishable from "nothing to do". const discover = async () => { const candidates = await discoverBorrowers(fetchPage, { logger, maxPages: MAX_DISCOVERY_PAGES }) - const listed = candidates.filter(candidate => listedMarkets.isListed(candidate.marketId)) + const whitelist = listedMarkets.current() + if (whitelist.fresh === 0) { + logger.warn('markets.whitelist_expired', { + maxAgeMs: LISTED_MARKETS_MAX_AGE_MS, + sources: listedMarkets.snapshot().sources, + detail: + 'every markets source is older than max age — whitelist is empty (fail-closed) until a refresh lands' + }) + } + const listed = candidates.filter(candidate => whitelist.isListed(candidate.marketId)) if (listed.length < candidates.length) { logger.info('discover.filtered', { total: candidates.length, listed: listed.length }) } @@ -352,8 +367,9 @@ async function main() { await delay(config.markets.refreshMs) if (stopped) return const { error } = await tryCatch(listedMarkets.refresh()) - // `refresh` is contractually non-throwing, so reaching this is a bug, not an API blip. - if (error) logger.warn('markets.refresh_error', { detail: error.message }) + // `refresh` is contractually non-throwing (it reports each source's failure itself), so reaching + // this is a bug in the union, not an API blip — hence `error`, not `warn`. + if (error) logger.error('markets.refresh_error', { detail: error.message }) if (venues.length === 0) { logger.warn('quoting.no_routes', { detail: 'still no venue API keys — bad-debt-only' }) } diff --git a/bots/midnight-liquidation/test/config.test.ts b/bots/midnight-liquidation/test/config.test.ts index 957f3198..f158d214 100644 --- a/bots/midnight-liquidation/test/config.test.ts +++ b/bots/midnight-liquidation/test/config.test.ts @@ -285,6 +285,16 @@ describe('loadConfig', () => { ]) }) + // Two spellings of one endpoint must collapse — otherwise the same source is polled twice and + // counted twice in the union's freshness bookkeeping. + it('de-duplicates MARKETS_API_URL entries that differ only in spelling', () => { + const config = loadConfig( + baseEnv({ MARKETS_API_URL: 'https://a.example/markets,https://a.example:443/markets' }), + deps + ) + expect(config.markets.apiUrls).toEqual(['https://a.example/markets']) + }) + it('throws on a malformed MARKETS_API_URL', () => { expect(() => loadConfig(baseEnv({ MARKETS_API_URL: 'not a url' }), deps)).toThrow( /MARKETS_API_URL is not a valid URL/ diff --git a/bots/midnight-liquidation/test/discovery/markets.test.ts b/bots/midnight-liquidation/test/discovery/markets.test.ts index 72b78045..c4955e22 100644 --- a/bots/midnight-liquidation/test/discovery/markets.test.ts +++ b/bots/midnight-liquidation/test/discovery/markets.test.ts @@ -1,4 +1,4 @@ -import type { Logger } from '@repo/bot-kit' +import type { LogLevel, Logger } from '@repo/bot-kit' import type { Hex } from 'viem' import { describe, expect, it } from 'bun:test' @@ -12,6 +12,7 @@ const API_URL = 'https://api.example/v0/midnight/markets' const LISTED: Hex = `0x${'a'.repeat(64)}` const UNLISTED: Hex = `0x${'b'.repeat(64)}` const OTHER_CHAIN: Hex = `0x${'c'.repeat(64)}` +const SHARED: Hex = `0x${'d'.repeat(64)}` const LOAN = '0x6666666666666666666666666666666666666666' const COLLATERAL = '0x7777777777777777777777777777777777777777' const ORACLE = '0x8888888888888888888888888888888888888888' @@ -34,6 +35,47 @@ const jsonResponse = (body: unknown, status = 200, headers: Record { + const events: { level: LogLevel; event: string; fields: Record }[] = [] + const record = (level: LogLevel) => (event: string, fields?: Record) => { + events.push({ level, event, fields: fields ?? {} }) + } + return { + names: () => events.map(entry => entry.event), + find: (event: string) => events.find(entry => entry.event === event), + logger: { + debug: record('debug'), + info: record('info'), + warn: record('warn'), + error: record('error') + } satisfies Logger + } +} + +// One real single-source filter, so the union is exercised against the actual factory. `now` sets this +// source's `updatedAt`, which is what the union's per-source staleness rule reads. +const sourceFilter = (opts: { + host: string + markets?: Hex[] + now?: () => number + fail?: boolean +}) => + createListedMarketFilter({ + apiUrl: `https://${opts.host}/v0/midnight/markets`, + chainId: 8453, + logger: NOOP_LOGGER, + fetchImpl: async () => + opts.fail + ? jsonResponse({}, 500) + : jsonResponse({ data: (opts.markets ?? []).map(id => market(id)) }), + sleep: async () => {}, + now: opts.now + }) + +// Source labels carry the path, so `https:///v0/midnight/markets` labels as `/v0/...`. +const label = (host: string) => `${host}/v0/midnight/markets` + describe('createListedMarketFilter', () => { it('requests listed=true and whitelists only listed markets on the configured chain', async () => { let requested = '' @@ -112,7 +154,44 @@ describe('createListedMarketFilter', () => { now: () => 111 }) await filter.refresh() - expect(filter.snapshot()).toEqual({ source: 'api.example', markets: 1, updatedAt: 111 }) + expect(filter.snapshot()).toEqual({ + source: 'api.example/v0/midnight/markets', + markets: 1, + updatedAt: 111 + }) + }) + + // The label carries the path so two sources on one host stay distinguishable in logs. + it('labels a source by host and path, excluding the query string', () => { + const filter = createListedMarketFilter({ + apiUrl: 'https://api.example/staging/v0/midnight/markets?token=secret', + chainId: 8453, + logger: NOOP_LOGGER, + fetchImpl: async () => jsonResponse({ data: [] }) + }) + expect(filter.snapshot().source).toBe('api.example/staging/v0/midnight/markets') + }) + + // An empty 200 is not a transient failure, so it replaces last-known-good — but in a union a healthy + // peer would mask it, so the nonempty→empty transition has to be loud on its own. + it('warns when a source drops from some listed markets to none', async () => { + const logs = capturingLogger() + let call = 0 + const filter = createListedMarketFilter({ + apiUrl: API_URL, + chainId: 8453, + logger: logs.logger, + fetchImpl: async () => { + call += 1 + return jsonResponse({ data: call === 1 ? [market(LISTED)] : [] }) + } + }) + await filter.refresh() + expect(logs.names()).not.toContain('markets.listed_empty') + + await filter.refresh() + expect(filter.isListed(LISTED)).toBe(false) // the empty response is authoritative + expect(logs.find('markets.listed_empty')?.fields.previous).toBe(1) }) it('retries a 429 honoring Retry-After', async () => { @@ -135,42 +214,6 @@ describe('createListedMarketFilter', () => { }) }) -// A capturing logger so the union's operator-visible warnings are assertable. -const capturingLogger = () => { - const warns: { event: string; fields: Record }[] = [] - return { - warns, - logger: { - debug: () => {}, - info: () => {}, - error: () => {}, - warn: (event: string, fields?: Record) => { - warns.push({ event, fields: fields ?? {} }) - } - } satisfies Logger - } -} - -// One real single-source filter, so the union is exercised against the actual factory. `now` sets this -// source's `updatedAt`, which is what the union's per-source staleness rule reads. -const sourceFilter = (opts: { - host: string - markets?: Hex[] - now?: () => number - fail?: boolean -}) => - createListedMarketFilter({ - apiUrl: `https://${opts.host}/v0/midnight/markets`, - chainId: 8453, - logger: NOOP_LOGGER, - fetchImpl: async () => - opts.fail - ? jsonResponse({}, 500) - : jsonResponse({ data: (opts.markets ?? []).map(id => market(id)) }), - sleep: async () => {}, - now: opts.now - }) - describe('createUnionListedMarketFilter', () => { it('whitelists the union of every fresh source', async () => { const union = createUnionListedMarketFilter({ @@ -183,12 +226,40 @@ describe('createUnionListedMarketFilter', () => { now: () => 0 }) await union.refresh() + const whitelist = union.current() + + expect(whitelist.isListed(LISTED)).toBe(true) // only in source a + expect(whitelist.isListed(OTHER_CHAIN)).toBe(true) // only in source b + expect(whitelist.isListed(UNLISTED)).toBe(false) // in neither + expect(whitelist.fresh).toBe(2) + expect(union.snapshot().sources.map(source => source.source)).toEqual([ + label('a.example'), + label('b.example') + ]) + }) + + // The combined size cannot be recovered from the per-source counts, so it is emitted on its own + // event: summing would double-count the shared market, and taking the max would report 2, not 3. + it('reports the deduplicated union size on markets.whitelist', async () => { + const logs = capturingLogger() + const union = createUnionListedMarketFilter({ + filters: [ + sourceFilter({ host: 'a.example', markets: [LISTED, SHARED], now: () => 0 }), + sourceFilter({ host: 'b.example', markets: [OTHER_CHAIN, SHARED], now: () => 0 }) + ], + maxAgeMs: 1_000, + logger: logs.logger, + now: () => 0 + }) + await union.refresh() + + expect(logs.find('markets.whitelist')?.fields).toEqual({ markets: 3, sources: 2, fresh: 2 }) + }) - expect(union.isListed(LISTED)).toBe(true) // only in source a - expect(union.isListed(OTHER_CHAIN)).toBe(true) // only in source b - expect(union.isListed(UNLISTED)).toBe(false) // in neither - expect(union.snapshot().fresh).toBe(2) - expect(union.snapshot().sources.map(s => s.source)).toEqual(['a.example', 'b.example']) + it('throws rather than silently listing nothing when no source is configured', () => { + expect(() => + createUnionListedMarketFilter({ filters: [], maxAgeMs: 1_000, logger: NOOP_LOGGER }) + ).toThrow(/requires at least one markets source/) }) it('is fail-closed before the first refresh (no source has a set yet)', () => { @@ -198,68 +269,96 @@ describe('createUnionListedMarketFilter', () => { logger: NOOP_LOGGER, now: () => 0 }) - expect(union.isListed(LISTED)).toBe(false) - expect(union.snapshot().fresh).toBe(0) + expect(union.current().isListed(LISTED)).toBe(false) + expect(union.current().fresh).toBe(0) expect(union.snapshot().sources[0]?.expired).toBe(true) }) // The core safety property of reading more than one source: staleness is judged PER SOURCE, so one // endpoint going stale narrows the whitelist instead of emptying it. it('drops an expired source from the union while a fresh peer keeps working', async () => { - const { logger, warns } = capturingLogger() + const logs = capturingLogger() const union = createUnionListedMarketFilter({ filters: [ sourceFilter({ host: 'stale.example', markets: [LISTED], now: () => 0 }), sourceFilter({ host: 'fresh.example', markets: [OTHER_CHAIN], now: () => 1_000 }) ], maxAgeMs: 100, - logger, + logger: logs.logger, now: () => 1_000 }) await union.refresh() + const whitelist = union.current() - expect(union.isListed(LISTED)).toBe(false) // stale source contributes nothing - expect(union.isListed(OTHER_CHAIN)).toBe(true) // fresh peer still whitelists - expect(union.snapshot().fresh).toBe(1) - expect(warns.map(w => w.event)).toContain('markets.source_expired') - expect(warns.find(w => w.event === 'markets.source_expired')?.fields.expired).toEqual([ - 'stale.example' - ]) + expect(whitelist.isListed(LISTED)).toBe(false) // stale source contributes nothing + expect(whitelist.isListed(OTHER_CHAIN)).toBe(true) // fresh peer still whitelists + expect(whitelist.fresh).toBe(1) + expect(logs.find('markets.source_expired')?.fields).toEqual({ + expired: [label('stale.example')], + maxAgeMs: 100, + detail: + 'markets source older than max age — excluded from the whitelist until a refresh lands' + }) + // Only the fresh peer's markets count toward the combined size. + expect(logs.find('markets.whitelist')?.fields).toEqual({ markets: 1, sources: 2, fresh: 1 }) }) - it('treats an all-expired whitelist as empty and warns loud', async () => { - const { logger, warns } = capturingLogger() + // When EVERY source is stale the union stays silent by design: the caller reports that per tick, so + // a refresh interval longer than maxAgeMs cannot leave the halt unreported between refreshes. + it('leaves the all-expired case to the caller rather than warning per refresh', async () => { + const logs = capturingLogger() const union = createUnionListedMarketFilter({ filters: [sourceFilter({ host: 'stale.example', markets: [LISTED], now: () => 0 })], maxAgeMs: 100, - logger, + logger: logs.logger, now: () => 1_000 }) await union.refresh() - expect(union.isListed(LISTED)).toBe(false) - expect(union.snapshot().fresh).toBe(0) - expect(warns.map(w => w.event)).toContain('markets.whitelist_expired') + expect(union.current().isListed(LISTED)).toBe(false) + expect(union.current().fresh).toBe(0) + expect(logs.names()).not.toContain('markets.source_expired') + expect(logs.names()).not.toContain('markets.whitelist_expired') }) // A partial refresh must not read as a total one: the failing source is reported and skipped, and the // healthy source's set still lands. it('never throws when a source fails, and still lands the healthy sources', async () => { - const { logger, warns } = capturingLogger() + const logs = capturingLogger() const union = createUnionListedMarketFilter({ filters: [ sourceFilter({ host: 'down.example', fail: true, now: () => 0 }), sourceFilter({ host: 'up.example', markets: [LISTED], now: () => 0 }) ], maxAgeMs: 1_000, - logger, + logger: logs.logger, now: () => 0 }) expect(await union.refresh()).toBeUndefined() - expect(union.isListed(LISTED)).toBe(true) - const failure = warns.find(w => w.event === 'markets.refresh_failed') - expect(failure?.fields.source).toBe('down.example') - expect(union.snapshot().fresh).toBe(1) + expect(union.current().isListed(LISTED)).toBe(true) + expect(logs.find('markets.refresh_failed')?.fields.source).toBe(label('down.example')) + expect(union.current().fresh).toBe(1) + }) + + // A source that never succeeds is excluded forever; the union must keep serving its fresh peer and + // must not let the dead source's absence widen or empty the whitelist. + it('keeps serving a fresh peer across repeated failures of another source', async () => { + const logs = capturingLogger() + const union = createUnionListedMarketFilter({ + filters: [ + sourceFilter({ host: 'down.example', fail: true, now: () => 0 }), + sourceFilter({ host: 'up.example', markets: [LISTED], now: () => 0 }) + ], + maxAgeMs: 1_000, + logger: logs.logger, + now: () => 0 + }) + await union.refresh() + await union.refresh() + + expect(union.current().isListed(LISTED)).toBe(true) + expect(union.current().fresh).toBe(1) + expect(logs.find('markets.source_expired')?.fields.expired).toEqual([label('down.example')]) }) })