feat(midnight-liquidation): union the market whitelist across sources

bots/midnight-liquidation/src/config.ts

@@ -66,7 +66,10 @@ const DEFAULT_POSITION_LIQUIDATION_COOLDOWN_MS = 0
 // bursts never queue ahead of a time-sensitive firm quote; log-scaled ladder sizes are whole
 // collateral tokens (converted per-collateral to base units). `PROBE_STALE_MS` caps probe cadence per
 // pair; a pair is re-probed only when a liquidatable position touches it after the cache goes stale.
-const DEFAULT_MARKETS_API_URL = 'https://api.morpho.org/v0/midnight/markets'
+// The whitelist may read MORE THAN ONE markets endpoint (comma-separated), unioned per-source — see
+// `MarketsConfig`. The default is the single public endpoint: an additional source widens what the bot
+// will touch, so it must be opted into explicitly per deployment rather than shipped in the default.
+const DEFAULT_MARKETS_API_URLS = ['https://api.morpho.org/v0/midnight/markets']
 const DEFAULT_MARKETS_REFRESH_MS = 60_000
 const DEFAULT_SLIPPAGE_BPS = 100
 const DEFAULT_PROBE_STALE_MS = 600_000
@@ -126,12 +129,18 @@ export type VenueConfig = {
 }
 
 /**
- * The Midnight markets API used as the market WHITELIST: only listed markets are discovered, probed,
+ * The Midnight markets API(s) used as the market WHITELIST: only listed markets are discovered, probed,
  * and liquidated. Over-inclusion is impossible (fail-closed); the on-chain lens remains the
- * correctness boundary. `refreshMs` caps how often the (cheap, non-rate-limited) endpoint is polled.
+ * correctness boundary. `refreshMs` caps how often the (cheap, non-rate-limited) endpoints are polled.
  */
 export type MarketsConfig = {
-  apiUrl: string
+  /**
+   * One or more markets endpoints, in `MARKETS_API_URL` order and de-duplicated. The effective
+   * whitelist is the union across the sources that are still fresh, so a deployment can read an
+   * additional list (e.g. one carrying extra shorter-maturity markets) without either endpoint
+   * becoming a single point of failure. Each is validated as a URL at load (fail-loud).
+   */
+  apiUrls: string[]
   refreshMs: number
 }
 
@@ -261,6 +270,31 @@ function ladderEnv(env: Env, name: string, def: string[]): string[] {
   return sizes
 }
 
+// Parses an optional comma-separated list of endpoint URLs, with a default, de-duplicating while
+// preserving order. Fails loud on an all-empty value or any malformed element rather than silently
+// dropping it — a dropped whitelist source would narrow the market set with no signal at all.
+// De-duplication is on the PARSED URL, so trivially different spellings of one endpoint (a trailing
+// slash, a default port) collapse instead of being polled twice and counted twice.
+function urlListEnv(env: Env, name: string, def: string[]): string[] {
+  const raw = env[name]?.trim()
+  if (!raw) return def
+  const urls = raw
+    .split(',')
+    .map(part => part.trim())
+    .filter(part => part.length > 0)
+  if (urls.length === 0) {
+    throw new Error(`${name} must contain at least one URL, got: ${env[name]}`)
+  }
+  const normalized = urls.map(url => {
+    const parsed = tryCatch(() => new URL(url))
+    if (parsed.error) {
+      throw new Error(`${name} is not a valid URL: ${url}`)
+    }
+    return parsed.data.toString()
+  })
+  return [...new Set(normalized)]
+}
+
 // Parses an optional comma-separated list of addresses into checksummed `Address`es, with `[]` as the
 // default. Fails loud on any malformed element (operator error).
 function addressListEnv(env: Env, name: string): Address[] {
@@ -390,13 +424,10 @@ export function loadConfig(
     excludeCollaterals: addressListEnv(env, 'EXCLUDE_COLLATERALS')
   }
 
-  // Market whitelist endpoint. Default to the public markets API; fail loud on a malformed override.
-  const marketsApiUrl = env.MARKETS_API_URL?.trim() || DEFAULT_MARKETS_API_URL
-  if (tryCatch(() => new URL(marketsApiUrl)).error) {
-    throw new Error(`MARKETS_API_URL is not a valid URL: ${marketsApiUrl}`)
-  }
+  // Market whitelist endpoint(s) — one, or a comma-separated list that is unioned per-source. Default
+  // to the public markets API; fail loud on any malformed entry.
   const markets: MarketsConfig = {
-    apiUrl: marketsApiUrl,
+    apiUrls: urlListEnv(env, 'MARKETS_API_URL', DEFAULT_MARKETS_API_URLS),
     refreshMs: intEnv(env, 'MARKETS_REFRESH_MS', DEFAULT_MARKETS_REFRESH_MS, { min: 1 })
   }
 

bots/midnight-liquidation/src/discovery/markets.ts

@@ -1,7 +1,7 @@
 import type { Logger } from '@repo/bot-kit'
 import type { Address, Hex } from 'viem'
 
-import { delay, fetchWithRetry } from '@repo/utils'
+import { delay, fetchWithRetry, tryCatch } from '@repo/utils'
 import createClient from 'openapi-fetch'
 import { isAddress, isHex } from 'viem'
 
@@ -23,18 +23,21 @@ export type ApiMarket = Omit<ApiMarketRow, 'market_id' | 'loan_token' | 'collate
 }
 
 /**
- * The market whitelist: `GET /v0/midnight/markets?listed=true` defines the set of markets the bot is
- * allowed to touch. `isListed` gates borrower candidates before the lens read, probing, and
- * liquidation — a market not in the listed set is never acted on (fail-closed). The set is refreshed
- * on a timer and serves last-known-good on a transient API failure, so a blip never silently widens
- * or empties the whitelist mid-flight; only a never-successful first fetch yields an empty set (safe).
- * `snapshot().updatedAt` is the caller's staleness signal — past `LISTED_MARKETS_MAX_AGE_MS` the
- * caller must treat the set as empty (fail-closed) so a since-delisted market can never linger.
+ * ONE source's view of the market whitelist: `GET /v0/midnight/markets?listed=true` defines the set of
+ * markets the bot is allowed to touch. `isListed` gates borrower candidates before the lens read,
+ * probing, and liquidation — a market not in the listed set is never acted on (fail-closed). The set is
+ * refreshed on a timer and serves last-known-good on a transient API failure, so a blip never silently
+ * widens or empties the whitelist mid-flight; only a never-successful first fetch yields an empty set
+ * (safe). `snapshot().updatedAt` is the staleness signal — past `LISTED_MARKETS_MAX_AGE_MS` the set
+ * must be treated as empty so a since-delisted market can never linger. Deployments read one or more
+ * sources; {@link createUnionListedMarketFilter} composes them and applies that staleness rule.
  */
 type ListedMarketFilter = {
   isListed: (marketId: Hex) => boolean
+  /** This source's listed ids (lowercased), so the union can size the combined whitelist. */
+  ids: () => ReadonlySet<string>
   refresh: () => Promise<void>
-  snapshot: () => { markets: number; updatedAt: number | null }
+  snapshot: () => { source: string; markets: number; updatedAt: number | null }
 }
 
 /** The `fetch` shape `openapi-fetch` calls — a single `Request`. The global `fetch` satisfies it. */
@@ -68,10 +71,13 @@ export function createListedMarketFilter(deps: {
 }): ListedMarketFilter {
   const sleep = deps.sleep ?? delay
   const now = deps.now ?? (() => Date.now())
-  const baseUrl = deps.apiUrl.endsWith(PATH)
-    ? deps.apiUrl.slice(0, -PATH.length)
-    : new URL(deps.apiUrl).origin
+  const url = new URL(deps.apiUrl)
+  const baseUrl = deps.apiUrl.endsWith(PATH) ? deps.apiUrl.slice(0, -PATH.length) : url.origin
   const client = createClient<paths>({ baseUrl, fetch: deps.fetchImpl ?? fetch })
+  // Log/snapshot label for this source. Host + path, so two sources sharing a host but differing by
+  // path prefix stay distinguishable in `markets.refresh_failed` / `markets.source_expired`; the query
+  // string is excluded so no credential can ride along (these endpoints are public either way).
+  const source = `${url.host}${url.pathname}`
 
   // Last-known-good: only replaced by a fully-successful refresh, so a transient failure keeps serving
   // the prior set rather than emptying the whitelist.
@@ -99,14 +105,153 @@ export function createListedMarketFilter(deps: {
       if (!isAddress(market.loan_token, { strict: false })) continue
       next.add(market.market_id.toLowerCase())
     }
+    // A successful-but-empty response is NOT a transient failure, so it legitimately replaces
+    // last-known-good and stamps `updatedAt` — but it silently drops this source to zero markets, and
+    // in a union a healthy peer would mask that entirely. Schema drift and an empty upstream database
+    // both look like this, so a nonempty→empty transition is called out loud rather than left to be
+    // inferred from `markets.listed { markets: 0 }` at info level.
+    if (listed.size > 0 && next.size === 0) {
+      deps.logger.warn('markets.listed_empty', {
+        chainId: deps.chainId,
+        source,
+        previous: listed.size,
+        detail: 'markets source returned zero listed markets where it previously returned some'
+      })
+    }
     listed = next
     updatedAt = now()
-    deps.logger.info('markets.listed', { chainId: deps.chainId, markets: listed.size })
+    deps.logger.info('markets.listed', { chainId: deps.chainId, source, markets: listed.size })
   }
 
   return {
     isListed: marketId => listed.has(marketId.toLowerCase()),
+    ids: () => listed,
     refresh,
-    snapshot: () => ({ markets: listed.size, updatedAt })
+    snapshot: () => ({ source, markets: listed.size, updatedAt })
+  }
+}
+
+/** One source's contribution to the union, as reported by {@link UnionListedMarketFilter.snapshot}. */
+type UnionSourceSnapshot = {
+  /** Host of the endpoint this source reads (see the single-source `source` label). */
+  source: string
+  markets: number
+  updatedAt: number | null
+  /** `true` when this source is past `maxAgeMs` and therefore contributes nothing to the union. */
+  expired: boolean
+}
+
+/**
+ * The composed whitelist across every configured markets source: the UNION over sources that are still
+ * fresh, so deployments can read more than one endpoint (e.g. the public list plus an additional list
+ * carrying extra markets) without either endpoint becoming a single point of failure.
+ */
+type UnionListedMarketFilter = {
+  /**
+   * Freezes the currently-fresh source set and returns a predicate over it, so ONE discovery pass is
+   * judged against ONE staleness reading. Re-deriving freshness per candidate would let a source cross
+   * the max-age boundary mid-pass, splitting a single pass across two different whitelists.
+   */
+  current: () => { isListed: (marketId: Hex) => boolean; fresh: number }
+  refresh: () => Promise<void>
+  snapshot: () => { sources: UnionSourceSnapshot[]; fresh: number }
+}
+
+/**
+ * Composes single-source {@link ListedMarketFilter}s into one union filter.
+ *
+ * The staleness rule is applied PER SOURCE: a source older than `maxAgeMs` (or never successfully
+ * fetched) contributes nothing, while its still-fresh peers keep working. That is what makes reading
+ * two endpoints safe in both directions — one endpoint going down or going stale narrows the whitelist
+ * to the sources that are still trustworthy instead of either emptying it (halting all liquidations) or
+ * letting a stale set keep a since-delisted market in scope.
+ *
+ * Union semantics are additive, so the whitelist is only ever as wide as the sources the operator
+ * configured; it stays fail-closed on a cold start (every source has `updatedAt === null` → nothing is
+ * listed).
+ *
+ * `refresh` fans out to every source concurrently and NEVER throws: each source's failure is logged
+ * (`markets.refresh_failed`, with its source label) and the others still land, because a partial
+ * refresh must not read as a total one. It then emits `markets.whitelist` with the size of the combined
+ * whitelist — the per-source `markets.listed` lines cannot be summed or maxed into that number — and
+ * warns `markets.source_expired` when SOME sources are stale. The all-stale case is deliberately NOT
+ * warned here: the whitelist being empty is a per-tick condition the caller reports every block (see
+ * `markets.whitelist_expired` in the bot's `discover`), because a refresh interval longer than
+ * `maxAgeMs` would otherwise leave the halt unreported for most of each interval.
+ *
+ * Throws if `filters` is empty: an empty union lists nothing, which is indistinguishable from a working
+ * fail-closed whitelist and would halt every liquidation in silence.
+ *
+ * `now` is injectable for tests.
+ */
+export function createUnionListedMarketFilter(deps: {
+  filters: ListedMarketFilter[]
+  maxAgeMs: number
+  logger: Logger
+  now?: () => number
+}): UnionListedMarketFilter {
+  if (deps.filters.length === 0) {
+    throw new Error('createUnionListedMarketFilter requires at least one markets source')
+  }
+  const now = deps.now ?? (() => Date.now())
+  const ageOf = (filter: ListedMarketFilter) => {
+    const { updatedAt } = filter.snapshot()
+    return updatedAt === null ? Infinity : now() - updatedAt
+  }
+  const isFresh = (filter: ListedMarketFilter) => ageOf(filter) <= deps.maxAgeMs
+  const freshFilters = () => deps.filters.filter(isFresh)
+  const snapshot = () => {
+    const sources = deps.filters.map(filter => ({
+      ...filter.snapshot(),
+      expired: !isFresh(filter)
+    }))
+    return { sources, fresh: sources.filter(source => !source.expired).length }
+  }
+  // Size of the combined whitelist. Derived from the id sets rather than the per-source counts, which
+  // overlap — summing them double-counts a market both sources list, and taking the max understates a
+  // union of two partially-overlapping sets.
+  const unionSize = () => new Set(freshFilters().flatMap(filter => [...filter.ids()])).size
+  const warnExpiry = () => {
+    const { sources, fresh } = snapshot()
+    const expired = sources.filter(source => source.expired).map(source => source.source)
+    // fresh === 0 is the caller's per-tick signal, not ours — see this factory's JSDoc.
+    if (expired.length === 0 || fresh === 0) return
+    deps.logger.warn('markets.source_expired', {
+      expired,
+      maxAgeMs: deps.maxAgeMs,
+      detail:
+        'markets source older than max age — excluded from the whitelist until a refresh lands'
+    })
+  }
+
+  return {
+    current: () => {
+      const fresh = freshFilters()
+      return {
+        isListed: marketId => fresh.some(filter => filter.isListed(marketId)),
+        fresh: fresh.length
+      }
+    },
+    refresh: async () => {
+      await Promise.all(
+        deps.filters.map(async filter => {
+          const { error } = await tryCatch(filter.refresh())
+          if (error) {
+            deps.logger.warn('markets.refresh_failed', {
+              source: filter.snapshot().source,
+              detail: error.message
+            })
+          }
+        })
+      )
+      const { sources, fresh } = snapshot()
+      deps.logger.info('markets.whitelist', {
+        markets: unionSize(),
+        sources: sources.length,
+        fresh
+      })
+      warnExpiry()
+    },
+    snapshot
   }
 }