From 2c62b33232e256961022bb6007d31f833b862eec Mon Sep 17 00:00:00 2001 From: Hayden Shively Date: Wed, 5 Aug 2026 12:44:00 -0400 Subject: [PATCH 1/4] fix(bots): count only broadcast submits and explain skipped plans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A BetterStack review of bot.liquidation.midnight found the bot looking healthy while doing nothing: since 2026-07-31 every tick reported `liquidatable: 12, planned: 0` with every skip counter at zero, for ~215,000 ticks, and no way to learn why. Three defects, all silent, both liquidators affected identically: 1. Two `continue`s (position in flight, sizing refused) incremented no counter and logged nothing. Now `inflightSkipped` / `planSkipped`, plus a sampled `plan.skipped` carrying the sizing inputs AND the derived numbers so the decision replays from one log line. 2. `submitted` counted submit *calls*, not broadcasts: on 2026-07-30 it summed 2,425 while tx.sent was 0 and tx.submit_failed was 2,666. `PendingQueue.submit` now returns a `SubmitOutcome`, and only a real broadcast counts. 3. `backoff.clear` ran unconditionally after submit, wiping the attempt count so the delay never grew — a sim-ok/send-fail position was re-quoted, re-simulated and re-sent every block forever. Only a broadcast clears it now; only the per-position failure (tx.submit_failed) records it, since the other three queue exits are queue-wide refusals that would otherwise suppress healthy positions. Also fixes a latent sizing bug found while planning: when `debt - maxDebt < badDebt < debt` the RCF numerator goes negative, and `capBoundPlan`'s `=== 0n` guard let a plan through with a NEGATIVE `seizedAssets` (verified: -8146) that reverts opaquely once abi-encoded. Sizing gains `planWithReason`, with `plan()` kept as a thin facade so the existing exact-bigint sizing tests are untouched. `tick.end` now carries `complete`, and is emitted even when a submit aborts the tick, so partial counters can never read as a genuinely idle tick. Counter identities are asserted in every tick test. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/commands/railway.md | 7 +- bots/blue-liquidation/README.md | 12 +- bots/blue-liquidation/src/constants.ts | 9 + bots/blue-liquidation/src/index.ts | 11 +- bots/blue-liquidation/src/runner/tick.ts | 298 +++++++++++----- bots/blue-liquidation/src/sizing/plan.ts | 71 +++- .../blue-liquidation/test/runner/tick.test.ts | 281 +++++++++++++-- .../blue-liquidation/test/sizing/plan.test.ts | 58 +++- bots/midnight-liquidation/README.md | 41 ++- bots/midnight-liquidation/src/constants.ts | 9 + bots/midnight-liquidation/src/index.ts | 15 +- bots/midnight-liquidation/src/runner/tick.ts | 318 +++++++++++------ bots/midnight-liquidation/src/sizing/plan.ts | 316 ++++++++++++----- .../test/runner/tick.test.ts | 326 ++++++++++++++++-- .../test/sizing/plan.test.ts | 130 ++++++- ...TIB-2026-05-28-midnight-liquidation-bot.md | 60 ++++ .../TIB-2026-06-30-blue-liquidation-bot.md | 59 ++++ packages/bot-kit/src/balance.ts | 9 +- packages/bot-kit/src/index.ts | 1 + packages/bot-kit/src/queue/backoff.ts | 6 +- packages/bot-kit/src/queue/cooldown.ts | 5 +- packages/bot-kit/src/queue/pending-queue.ts | 41 ++- packages/bot-kit/src/runner/block-cadence.ts | 36 ++ .../bot-kit/test/queue/pending-queue.test.ts | 20 +- .../bot-kit/test/runner/block-cadence.test.ts | 45 +++ 25 files changed, 1820 insertions(+), 364 deletions(-) create mode 100644 packages/bot-kit/src/runner/block-cadence.ts create mode 100644 packages/bot-kit/test/runner/block-cadence.test.ts diff --git a/.claude/commands/railway.md b/.claude/commands/railway.md index 65aeaa40..b9a64886 100644 --- a/.claude/commands/railway.md +++ b/.claude/commands/railway.md @@ -42,8 +42,11 @@ railway status # lists services + Online/offline ``` `midnight-liquidation` services: `bot`, `rindexer`, `Postgres`. Healthy `bot` ticks each block: -`event="block.new"` → `event="lens.read"` → `event="tick.end"` (`liquidatable=`/`submitted=` -counters), plus one `event="daemon.start"` at boot. Trouble = `event="tick.error"` or `[ERROR]`; +`event="block.new"` → `event="lens.read"` → `event="tick.end"`, plus one `event="daemon.start"` at +boot. In `tick.end`, `submitted=` counts only real broadcasts and `notSent=` counts submits that sent +nothing; `liquidatable=` is a market gauge, so `liquidatable>0` with `planSkipped=` equal to it means +the bot sees positions it cannot size — grep `plan.skipped` for the reason and the sizing numbers. +`complete=false` marks a tick that aborted part-way, so its counters are partial. Trouble = `event="tick.error"` or `[ERROR]`; for a reverting tx grep `tx.dropped` / `tx.replace_failed` / `tx.submit_failed`. Triage by event, not the raw line — error lines historically carried a multi-KB calldata dump that shippers truncate. diff --git a/bots/blue-liquidation/README.md b/bots/blue-liquidation/README.md index e9739cdc..17043f44 100644 --- a/bots/blue-liquidation/README.md +++ b/bots/blue-liquidation/README.md @@ -240,7 +240,17 @@ per-`(id, borrower)` exponential backoff suppresses repeated failures. ### Simulation The exact `Executor.exec_606BaXt(...)` bytes are `eth_call`-simulated from the EOA. Only an `ok` -result is broadcast (`simulate.ok` gate). The exec is `liquidate` + an in-callback queue (via +result is broadcast (`simulate.ok` gate). + +`tick.end`'s `submitted` counts only real broadcasts: `submit` reports whether a transaction went out, +and only that clears the position's backoff. A send that failed for this position (`tx.submit_failed`) +counts `notSent` and backs the position off; the three queue-wide refusals (`tx.send_aborted`, +`nonce.sync_failed`, `queue.nonce_hole`) count `notSent` only, since they reject every send that tick. +A tick aborted by a hashless nonce-bearing send still emits `tick.end` with `complete: false`. + +A position the chain reports liquidatable can still fail to size; the tick counts it as `planSkipped` +and emits a sampled `plan.skipped` with the reason (`no_collateral`, `zero_price`, +`seize_rounds_to_zero`) plus the sizing inputs and derived numbers. The exec is `liquidate` + an in-callback queue (via `onMorphoLiquidate`) that runs the plan's steps — a plain collateral is one venue swap; exotic collateral is unwrap step(s) then usually a venue swap — followed by the repay-token approval. After `liquidate` returns, trailing `skim` sweeps drain both market tokens **plus every intermediate token diff --git a/bots/blue-liquidation/src/constants.ts b/bots/blue-liquidation/src/constants.ts index a93a972b..b45ef528 100644 --- a/bots/blue-liquidation/src/constants.ts +++ b/bots/blue-liquidation/src/constants.ts @@ -39,3 +39,12 @@ export const VIRTUAL_ASSETS = 1n * terminal, so a brief suppression of an already-acted position is harmless. */ export const SETTLED_COOLDOWN_BLOCKS = 20n + +/** + * Block cadence bounding the per-position `plan.skipped` diagnostic. A liquidatable position that + * cannot be sized recurs every tick, so logging one line per position per block would dominate the + * log source while repeating one fact (~12 positions × ~1780 ticks/hour on Base). At ~2s blocks this + * is roughly every 5 minutes. The sampler is asked only when a skip actually happens, so a quiet + * stretch never consumes the window and the first skip after any gap is always reported. + */ +export const PLAN_SKIP_SAMPLE_EVERY_BLOCKS = 150n diff --git a/bots/blue-liquidation/src/index.ts b/bots/blue-liquidation/src/index.ts index 422ece56..76806d66 100644 --- a/bots/blue-liquidation/src/index.ts +++ b/bots/blue-liquidation/src/index.ts @@ -5,6 +5,7 @@ import { assertContractDeployed, createBalanceMonitor, createBackoff, + createBlockSampler, createCooldownStore, createDeploylessClient, createHeartbeatMonitor, @@ -34,7 +35,7 @@ import type { MarketParams } from './market' import type { LiquidationPlan } from './sizing/plan' import { loadConfig } from './config' -import { SETTLED_COOLDOWN_BLOCKS } from './constants' +import { PLAN_SKIP_SAMPLE_EVERY_BLOCKS, SETTLED_COOLDOWN_BLOCKS } from './constants' import { createGraphqlCandidateSource, discoverCandidates } from './discovery/borrowers' import { encodeLiquidationExec } from './execution/encode-call' import { composeQuoting } from './quotes' @@ -179,6 +180,9 @@ async function main() { // Opt-in per-position cooldown (default disabled): one in-memory store for the process lifetime, // complementary to `backoff` (see POSITION_LIQUIDATION_COOLDOWN_MS). const cooldown = createCooldownStore({ cooldownMs: config.positionCooldownMs }) + // Bounds the `plan.skipped` diagnostic to one burst per cadence; process-lifetime state, like the + // two stores above. + const planSkipSampler = createBlockSampler(PLAN_SKIP_SAMPLE_EVERY_BLOCKS) // The exec calldata for one liquidation — the same bytes the simulate gate checks and the queue // broadcasts, so a sim-ok plan and its broadcast can't drift. @@ -273,7 +277,9 @@ async function main() { }), submit: async ({ market, borrower, plan, swapPlan, blockNumber, label }) => { const fees = initialFees(await signer.getBaseFee(), config.maxFeeWei) - await queue.submit({ + // Returned, not awaited-and-discarded: the tick needs to know whether a tx actually went out + // before it clears this position's backoff. + return queue.submit({ request: { to: config.executooorAddress, data: encodeExec(market, borrower, plan, swapPlan) @@ -286,6 +292,7 @@ async function main() { }, backoff, cooldown, + planSkipSampler, inflightLabels: () => queue.inflightLabels(), logger }) diff --git a/bots/blue-liquidation/src/runner/tick.ts b/bots/blue-liquidation/src/runner/tick.ts index ade54323..d078226f 100644 --- a/bots/blue-liquidation/src/runner/tick.ts +++ b/bots/blue-liquidation/src/runner/tick.ts @@ -1,4 +1,11 @@ -import type { Backoff, CooldownStore, Logger, SimulateResult } from '@repo/bot-kit' +import type { + Backoff, + BlockSampler, + CooldownStore, + Logger, + SimulateResult, + SubmitOutcome +} from '@repo/bot-kit' import type { QuoteOutcome, SwapPlan } from '@repo/swaps' import type { Address } from 'viem' @@ -6,24 +13,63 @@ import { assertNever, lensKey, tryCatch } from '@repo/utils' import type { BorrowerCandidate } from '../discovery/borrowers' import type { MarketParams } from '../market' -import type { LiquidationPlan } from '../sizing/plan' +import type { LiquidationPlan, PlanSkipReason } from '../sizing/plan' import type { LensInput, LensOut } from '../state/lens.sol' import { marketId } from '../market' -import { plan } from '../sizing/plan' +import { planWithReason } from '../sizing/plan' import { isLiquidatable, planInputFromLens } from './eligibility' +/** + * Per-tick outcome tally, emitted as `tick.end`. Ordered as the pipeline runs, and exhaustive by + * construction: for a tick that finished (`complete: true` on the event) these identities hold, so a + * future stage added without a counter shows up as a broken sum rather than a silent drop. + * + * pairs >= liquidatable + * liquidatable === inflightSkipped + planSkipped + planned + * planned === cooledDown + backoffSkipped + noSwapPath + quoteFailed + ok + reverted + * ok === submitted + notSent + * + * They do NOT hold when `complete` is `false`: an aborting `submit` throws after `ok` was counted but + * before `submitted`/`notSent`, so the last identity is short by one. + */ type TickCounters = { + /** Lens inputs read this tick — the discovery universe. */ pairs: number + /** Positions the chain says are liquidatable at this block. A market gauge, not a bot decision. */ liquidatable: number + /** + * Skipped because a tx for this label is already in flight. Note the queue's backpressure set also + * holds labels whose tx SETTLED within `settledCooldownBlocks`, so this can be non-zero while the + * queue itself is empty. + */ + inflightSkipped: number + /** Skipped because sizing produced no plan — see the `plan.skipped` event for the reason. */ + planSkipped: number planned: number + cooledDown: number + backoffSkipped: number noSwapPath: number quoteFailed: number - backoffSkipped: number - cooledDown: number ok: number reverted: number + /** Broadcast: the queue reported a transaction actually went out. */ submitted: number + /** The queue returned without broadcasting (a send failure, or a queue-wide refusal). */ + notSent: number +} + +/** + * `warn` marks a reason that should be impossible, so it reads as a live assertion rather than noise: + * `no_debt`/`healthy` are the exact negation of `isLiquidatable`, and a non-reverting zero oracle + * price is a market anomaly. The dust reasons are ordinary and stay `info`. + */ +const LEVEL_BY_REASON: Record = { + no_debt: 'warn', + healthy: 'warn', + zero_price: 'warn', + no_collateral: 'info', + seize_rounds_to_zero: 'info' } /** @@ -38,6 +84,10 @@ type TickCounters = { * Discovery failure is tolerated: a transient error is logged (`discover.error`) and the tick proceeds * with zero new candidates. The lens reads every candidate fresh on-chain, so discovery is a coverage * source, never a correctness dependency — API indexing lag is coverage latency only. + * + * Every exit from the per-position loop increments exactly one counter, and `tick.end` is emitted even + * when a position aborts the tick — with `complete: false`, so partial counters can never be mistaken + * for a genuinely idle tick. */ export async function runTick(deps: { discover: () => Promise @@ -56,7 +106,12 @@ export async function runTick(deps: { plan: LiquidationPlan swapPlan: SwapPlan }) => Promise - /** Broadcasts a plan via the pending queue (builds the exec tx, derives fees, tracks the nonce). */ + /** + * Broadcasts a plan via the pending queue (builds the exec tx, derives fees, tracks the nonce). + * Reports whether a transaction actually went out: ONLY `kind: 'sent'` may clear the position's + * backoff. Throws when a send fails after claiming a nonce — the tick aborts by design, so the + * signer's cursor rollback is not raced. + */ submit: (args: { market: MarketParams borrower: Address @@ -64,15 +119,22 @@ export async function runTick(deps: { swapPlan: SwapPlan blockNumber: bigint label: string - }) => Promise - /** Per-position exponential backoff suppressing repeated quote/simulate failures (rate-limit defense). */ + }) => Promise + /** Per-position exponential backoff suppressing repeated quote/simulate/send failures (rate-limit defense). */ backoff: Backoff /** * Opt-in per-position cooldown, complementary to `backoff`: a fixed wall-clock window suppressing - * re-quoting a position whose last attempt produced no submittable tx. Disabled by default + * re-quoting a position whose last attempt produced no broadcast tx. Disabled by default * (`POSITION_LIQUIDATION_COOLDOWN_MS=0`), in which case `shouldSkip` is always false. */ cooldown: CooldownStore + /** + * Bounds how often the per-position `plan.skipped` diagnostic is emitted. Asked at most once per + * tick and only when there is something to explain, so a quiet stretch never consumes the window: + * the first skip after any gap is always reported, and a persistent one settles to a single burst + * per cadence. All lines in a burst share one `chainHead`, so they read as one coherent snapshot. + */ + planSkipSampler: BlockSampler /** Labels (`${id}:${borrower}`) already in flight — skipped to avoid re-submitting each block. */ inflightLabels: () => ReadonlySet logger: Logger @@ -86,6 +148,7 @@ export async function runTick(deps: { submit, backoff, cooldown, + planSkipSampler, inflightLabels, logger } = deps @@ -103,101 +166,138 @@ export async function runTick(deps: { // 2. Read the lens fresh for the whole batch in one deployless eth_call. const lensOut = await readLens(pairs) - logger.info('lens.read', { pairs: pairs.length, returned: lensOut.size }) + // `returned` is always `pairs` (the batch lens maps every input row); `invalid` is the informative + // one — it counts rows the lens zeroed (unknown market, reverting oracle), which would otherwise be + // indistinguishable from a healthy position in `pairs - liquidatable`. + let invalid = 0 + for (const out of lensOut.values()) if (!out.valid) invalid += 1 + logger.info('lens.read', { pairs: pairs.length, returned: lensOut.size, invalid }) const counters: TickCounters = { pairs: pairs.length, liquidatable: 0, + inflightSkipped: 0, + planSkipped: 0, planned: 0, + cooledDown: 0, + backoffSkipped: 0, noSwapPath: 0, quoteFailed: 0, - backoffSkipped: 0, - cooledDown: 0, ok: 0, reverted: 0, - submitted: 0 + submitted: 0, + notSent: 0 } // 3. Compose liquidatability off-chain → plan → quote → simulate → submit. `inflight` is captured // once; discovery yields distinct (market, borrower) pairs, so no label repeats within a tick. const inflight = inflightLabels() - for (const pair of pairs) { - const id = marketId(pair.params) - const label = lensKey(id, pair.borrower) - const out = lensOut.get(label) - if (!out || !isLiquidatable(out)) continue - counters.liquidatable += 1 - - // Backpressure: a tx for this position is already pending — don't re-plan/simulate/submit it - // every block while it confirms. - if (inflight.has(label)) continue - - const liquidationPlan = plan(planInputFromLens(out)) - if (!liquidationPlan) continue - counters.planned += 1 - logger.info('plan.built', { - marketId: id, - borrower: pair.borrower, - seizedAssets: liquidationPlan.seizedAssets - }) - - // Opt-in cooldown (complementary to backoff): a position whose last attempt produced no - // submittable tx is skipped without re-quoting until its wall-clock window elapses. No-op when - // disabled (POSITION_LIQUIDATION_COOLDOWN_MS=0). - if (cooldown.shouldSkip(label)) { - counters.cooledDown += 1 - logger.info('cooldown.skip', { marketId: id, borrower: pair.borrower }) - continue - } - // Suppress positions that keep failing to quote/simulate — bounds API + RPC usage under a - // backlog, since executable quotes are spent only on positions not currently backed off. - if (backoff.shouldSkip(label, chainHead)) { - counters.backoffSkipped += 1 - continue - } - const outcome = await quoteFor(liquidationPlan, out, label) - if (outcome.kind === 'no_config') { - counters.noSwapPath += 1 - cooldown.mark(label) - logger.info('config.no_swap_path', { marketId: id, borrower: pair.borrower }) - continue - } - if (outcome.kind === 'failed') { - counters.quoteFailed += 1 - backoff.record(label, chainHead) - cooldown.mark(label) - continue - } - const swapPlan = outcome.plan - - const result = await simulate({ - market: out.params, - borrower: pair.borrower, - plan: liquidationPlan, - swapPlan - }) - const fields = { marketId: id, borrower: pair.borrower } - switch (result.status) { - case 'ok': - counters.ok += 1 - logger.info('simulate.ok', fields) - break - case 'revert': - counters.reverted += 1 - // Back off: a sim revert (stale quote, transient unliquidatability, oracle drift) shouldn't - // re-quote + re-simulate this position every block. + // Resolved lazily on the first skip of the tick and reused for the rest, so one tick emits either a + // full snapshot of the blocker set or nothing — never a staggered subset. + let explainSkips: boolean | null = null + + const processPairs = async () => { + for (const pair of pairs) { + const id = marketId(pair.params) + const label = lensKey(id, pair.borrower) + const out = lensOut.get(label) + if (!out || !isLiquidatable(out)) continue + counters.liquidatable += 1 + + // Backpressure: a tx for this position is already pending — don't re-plan/simulate/submit it + // every block while it confirms. No log: the queue already narrates this label's whole + // lifecycle (`tx.sent` → `tx.confirmed`/`tx.reverted`/`tx.dropped`) under the same key. + if (inflight.has(label)) { + counters.inflightSkipped += 1 + continue + } + + const planInput = planInputFromLens(out) + const planOutcome = planWithReason(planInput) + if (planOutcome.kind === 'skip') { + counters.planSkipped += 1 + explainSkips ??= planSkipSampler.claim(chainHead) + if (explainSkips) { + // Spread the inputs AND the derived trace so the line is a closed causal chain — an + // operator can replay sizing from it without re-deriving anything. `marketId`/`borrower` + // last so a future `PlanInput` field can never shadow them. + logger[LEVEL_BY_REASON[planOutcome.reason]]('plan.skipped', { + ...planInput, + ...planOutcome.trace, + reason: planOutcome.reason, + marketId: id, + borrower: pair.borrower + }) + } + continue + } + + const liquidationPlan = planOutcome.plan + counters.planned += 1 + logger.info('plan.built', { + marketId: id, + borrower: pair.borrower, + seizedAssets: liquidationPlan.seizedAssets + }) + + // Opt-in cooldown (complementary to backoff): a position whose last attempt produced no + // broadcast tx is skipped without re-quoting until its wall-clock window elapses. No-op when + // disabled (POSITION_LIQUIDATION_COOLDOWN_MS=0). + if (cooldown.shouldSkip(label)) { + counters.cooledDown += 1 + logger.info('cooldown.skip', { marketId: id, borrower: pair.borrower }) + continue + } + // Suppress positions that keep failing to quote/simulate/send — bounds API + RPC usage under a + // backlog, since executable quotes are spent only on positions not currently backed off. + if (backoff.shouldSkip(label, chainHead)) { + counters.backoffSkipped += 1 + continue + } + const quote = await quoteFor(liquidationPlan, out, label) + if (quote.kind === 'no_config') { + counters.noSwapPath += 1 + cooldown.mark(label) + logger.info('config.no_swap_path', { marketId: id, borrower: pair.borrower }) + continue + } + if (quote.kind === 'failed') { + counters.quoteFailed += 1 backoff.record(label, chainHead) cooldown.mark(label) - logger.warn('simulate.revert', { ...fields, reason: result.reason }) - break - default: - assertNever(result.status) - } + continue + } + const swapPlan = quote.plan - // ok-only gate: broadcast only a fully-simulated, swap-funded liquidation. Any revert — not - // liquidatable, swap slippage, repay shortfall — isn't a fundable plan, so skip it. - if (result.status === 'ok') { - await submit({ + const result = await simulate({ + market: out.params, + borrower: pair.borrower, + plan: liquidationPlan, + swapPlan + }) + const fields = { marketId: id, borrower: pair.borrower } + switch (result.status) { + case 'ok': + counters.ok += 1 + logger.info('simulate.ok', fields) + break + case 'revert': + counters.reverted += 1 + // Back off: a sim revert (stale quote, transient unliquidatability, oracle drift) shouldn't + // re-quote + re-simulate this position every block. + backoff.record(label, chainHead) + cooldown.mark(label) + logger.warn('simulate.revert', { ...fields, reason: result.reason }) + break + default: + assertNever(result.status) + } + + // ok-only gate: broadcast only a fully-simulated, swap-funded liquidation. Any revert — not + // liquidatable, swap slippage, repay shortfall — isn't a fundable plan, so skip it. + if (result.status !== 'ok') continue + + const sendOutcome = await submit({ market: out.params, borrower: pair.borrower, plan: liquidationPlan, @@ -205,11 +305,29 @@ export async function runTick(deps: { blockNumber: chainHead, label }) - backoff.clear(label) - counters.submitted += 1 + if (sendOutcome.kind === 'sent') { + backoff.clear(label) + counters.submitted += 1 + continue + } + counters.notSent += 1 + // Only a per-position send failure earns a backoff. `send_aborted`, `nonce_hole` and + // `nonce_sync_failed` are queue-WIDE refusals that reject every send this tick, so backing off + // here would suppress positions that did nothing wrong — for 2, 4, 8… blocks after the latch + // itself has cleared. The queue has already logged which exit it took. + if (sendOutcome.reason === 'submit_failed') { + backoff.record(label, chainHead) + cooldown.mark(label) + } } } - logger.info('tick.end', { ...counters }) + // Emit counters even when a position aborts the tick (a hashless send after the nonce was claimed + // throws by design). Without `complete` an aborted tick's partial counters are indistinguishable + // from a genuinely idle one. `ensureError` preserves the instance, so rethrowing keeps `TxSendError` + // intact for the runner's `tick.error` decode. + const { error } = await tryCatch(processPairs()) + logger.info('tick.end', { ...counters, complete: !error }) + if (error) throw error return counters } diff --git a/bots/blue-liquidation/src/sizing/plan.ts b/bots/blue-liquidation/src/sizing/plan.ts index 7b88fcfa..ae4ef49e 100644 --- a/bots/blue-liquidation/src/sizing/plan.ts +++ b/bots/blue-liquidation/src/sizing/plan.ts @@ -25,8 +25,46 @@ export type PlanInput = { export type LiquidationPlan = { seizedAssets: bigint } /** - * Turns a fresh lens reading into a seize-exact liquidation plan, or `null` when the position is not - * liquidatable or has nothing to seize. + * Why {@link planWithReason} produced no plan. Reasons discriminate SEVERITY (and hence log level); + * the numbers on {@link SizingTrace} discriminate cause. + * + * `no_debt` and `healthy` are UNREACHABLE from the tick — `isLiquidatable` tests exactly + * `hasDebt && !healthy` — so observing one means the eligibility gate and the sizing module have + * diverged, a correctness bug. They are logged at `warn`, as a live assertion. + */ +export type PlanSkipReason = + | 'no_debt' + | 'healthy' + /** Pure residual bad debt: shares outstanding but no collateral left to seize. */ + | 'no_collateral' + /** A non-reverting zero oracle price — unsizable, and a divide-by-zero if used. */ + | 'zero_price' + /** Dust, or a price so high the full-debt seize floors to zero collateral. */ + | 'seize_rounds_to_zero' + +/** + * Every value on the causal chain from a lens reading to a refused seize, so an operator can replay + * the decision from one log line instead of re-deriving it. Absent for the two pre-sizing refusals + * (`no_debt`, `healthy`), which are decided before any arithmetic runs. + */ +type SizingTrace = { + lif: bigint + /** Full debt in loan assets, floored from `borrowShares`. */ + repaidAssetsFull: bigint + /** Collateral the full-debt repay would seize, before the 100%-of-collateral clamp. */ + seizeForFullDebt: bigint + /** The final seize; `0n` is what refused the plan. */ + seizedAssets: bigint +} + +/** A plan, or the reason there is none plus the numbers that explain it. */ +type PlanOutcome = + | { kind: 'plan'; plan: LiquidationPlan } + | { kind: 'skip'; reason: PlanSkipReason; trace?: SizingTrace } + +/** + * Turns a fresh lens reading into a seize-exact liquidation plan, or a {@link PlanSkipReason} + * explaining why there is none. * * Blue liquidation is permissionless and time-independent, so the only gate is `hasDebt && !healthy` * (no maturity, no liquidator gate, no RCF cap). The single sizing rule pins the seize at the amount @@ -41,17 +79,22 @@ export type LiquidationPlan = { seizedAssets: bigint } * `plan.test.ts`). When collateral binds, seizing all collateral socializes the residual as bad debt * in the same call. If a later oracle move makes the derived repay too large, simulation catches the * revert before broadcast. + * + * @param input - Fresh lens-derived position state, all read at one block. + * @returns `{ kind: 'plan' }` with a submittable plan, or `{ kind: 'skip' }` with the reason and — + * for every refusal reached after sizing began — a {@link SizingTrace}. */ -export function plan(input: PlanInput): LiquidationPlan | null { - if (!input.hasDebt || input.healthy) return null +export const planWithReason = (input: PlanInput): PlanOutcome => { + if (!input.hasDebt) return { kind: 'skip', reason: 'no_debt' } + if (input.healthy) return { kind: 'skip', reason: 'healthy' } // Degenerate: pure residual bad debt (borrowShares > 0 but collateral == 0). Nothing to seize; a // backstop bot does not perform an uncompensated loan-token repay, and Blue rejects a (0, 0) // liquidate anyway (`exactlyOneZero`). - if (input.collateral === 0n) return null + if (input.collateral === 0n) return { kind: 'skip', reason: 'no_collateral' } // A zero (non-reverting) oracle price would divide-by-zero below. It also makes maxBorrow 0, so the // lens reports the position unhealthy — but we cannot size against it; skip rather than throw and // abort the whole tick. (A reverting oracle is already dropped by the lens as valid=false.) - if (input.collateralPrice === 0n) return null + if (input.collateralPrice === 0n) return { kind: 'skip', reason: 'zero_price' } const lif = lifFromLltv(input.lltv) const repaidAssetsFull = toAssetsDown( @@ -65,7 +108,19 @@ export function plan(input: PlanInput): LiquidationPlan | null { input.collateralPrice ) const seizedAssets = min(input.collateral, seizeForFullDebt) + const trace: SizingTrace = { lif, repaidAssetsFull, seizeForFullDebt, seizedAssets } // Rounds to nothing (dust position, or price ≫ debt): can't pass 0 to `liquidate`, so skip it. - if (seizedAssets === 0n) return null - return { seizedAssets } + if (seizedAssets === 0n) return { kind: 'skip', reason: 'seize_rounds_to_zero', trace } + return { kind: 'plan', plan: { seizedAssets } } +} + +/** + * Bare-plan facade over {@link planWithReason} for callers that do not report a reason. + * + * @param input - Fresh lens-derived position state. + * @returns The plan, or `null` when the position is not liquidatable or has nothing to seize. + */ +export const plan = (input: PlanInput): LiquidationPlan | null => { + const outcome = planWithReason(input) + return outcome.kind === 'plan' ? outcome.plan : null } diff --git a/bots/blue-liquidation/test/runner/tick.test.ts b/bots/blue-liquidation/test/runner/tick.test.ts index b2f236e5..217110be 100644 --- a/bots/blue-liquidation/test/runner/tick.test.ts +++ b/bots/blue-liquidation/test/runner/tick.test.ts @@ -1,9 +1,9 @@ -import type { Logger, SimulateResult } from '@repo/bot-kit' -import type { CooldownStore } from '@repo/bot-kit' +import type { Logger, SimulateResult, SubmitOutcome } from '@repo/bot-kit' +import type { Backoff, BlockSampler, CooldownStore } from '@repo/bot-kit' import type { QuoteOutcome, SwapPlan } from '@repo/swaps' -import type { Address } from 'viem' +import type { Address, Hex } from 'viem' -import { createBackoff, createCooldownStore } from '@repo/bot-kit' +import { createBackoff, createBlockSampler, createCooldownStore } from '@repo/bot-kit' import { lensKey } from '@repo/utils' import { getAddress } from 'viem' import { describe, expect, it } from 'vitest' @@ -44,6 +44,25 @@ const PARAMS: MarketParams = { lltv: 86n * 10n ** 16n } const LABEL = lensKey(marketId(PARAMS), BORROWER) +const MARKET_ID = marketId(PARAMS) +const TX_HASH: Hex = `0x${'b'.repeat(64)}` +const SENT: SubmitOutcome = { kind: 'sent', nonce: 7, txHash: TX_HASH } + +type TickCounters = Awaited> + +/** + * Every counter identity the tick promises for a completed tick. Asserted for EVERY case built by + * `runWith`, so a stage added without a counter breaks the sums instead of silently dropping a + * position — the exact class of bug these counters exist to catch. + */ +function expectCountersConsistent(c: TickCounters) { + expect(c.pairs).toBeGreaterThanOrEqual(c.liquidatable) + expect(c.liquidatable).toBe(c.inflightSkipped + c.planSkipped + c.planned) + expect(c.planned).toBe( + c.cooledDown + c.backoffSkipped + c.noSwapPath + c.quoteFailed + c.ok + c.reverted + ) + expect(c.ok).toBe(c.submitted + c.notSent) +} const SWAP_PLAN: SwapPlan = { steps: [ @@ -90,7 +109,7 @@ function stubReadLens(out: LensOut | null) { } } -function runWith(opts: { +type RunOpts = { out?: LensOut | null simulateResult?: SimulateResult quoteOutcome?: QuoteOutcome @@ -101,20 +120,32 @@ function runWith(opts: { noSwap?: boolean seedBackoffAt?: bigint cooldown?: CooldownStore -}) { + /** What the queue reports; defaults to a real broadcast. */ + submitOutcome?: SubmitOutcome + /** Makes `submit` throw, as a hashless send after a nonce was claimed does. */ + submitThrows?: Error + /** Reuse a store/sampler across two `runTick` calls, so cross-tick behavior can be asserted. */ + backoff?: Backoff + planSkipSampler?: BlockSampler +} + +// Shared dep construction so the throwing case exercises exactly the same wiring as `runWith`. +function buildDeps(opts: RunOpts) { const { logger, events } = spyLogger() let simulateCalls = 0 let submitCalls = 0 let quoteCalls = 0 const chainHead = opts.chainHead ?? 100n - const backoff = createBackoff({ baseBlocks: 2n, maxBlocks: 64n }) + const backoff = opts.backoff ?? createBackoff({ baseBlocks: 2n, maxBlocks: 64n }) if (opts.seedBackoffAt !== undefined) backoff.record(LABEL, opts.seedBackoffAt) // Default disabled (0) so existing cases are unaffected; opt-in cases pass an enabled store. const cooldown = opts.cooldown ?? createCooldownStore({ cooldownMs: 0 }) + // 0n so every skip explains itself by default; throttling cases pass a real cadence. + const planSkipSampler = opts.planSkipSampler ?? createBlockSampler(0n) const defaultOutcome: QuoteOutcome = opts.noSwap ? { kind: 'no_config' } : { kind: 'swap', plan: SWAP_PLAN } - const result = runTick({ + const deps = { discover: async () => { if (opts.discoverError) throw opts.discoverError return candidates(...(opts.borrowers ?? [BORROWER])) @@ -127,25 +158,46 @@ function runWith(opts: { }, simulate: async () => { simulateCalls += 1 - return opts.simulateResult ?? { status: 'ok' } + return opts.simulateResult ?? ({ status: 'ok' } as SimulateResult) }, submit: async () => { submitCalls += 1 + if (opts.submitThrows) throw opts.submitThrows + return opts.submitOutcome ?? SENT }, backoff, cooldown, - inflightLabels: () => opts.inflight ?? new Set(), + planSkipSampler, + inflightLabels: () => opts.inflight ?? new Set(), logger + } + return { + deps, + probes: { + backoff, + cooldown, + planSkipSampler, + simulateCalls: () => simulateCalls, + submitCalls: () => submitCalls, + quoteCalls: () => quoteCalls, + events + } + } +} + +function runWith(opts: RunOpts) { + const { deps, probes } = buildDeps(opts) + return runTick(deps).then(counters => { + expectCountersConsistent(counters) + return { counters, ...probes } }) - return result.then(counters => ({ - counters, - backoff, - cooldown, - simulateCalls: () => simulateCalls, - submitCalls: () => submitCalls, - quoteCalls: () => quoteCalls, - events - })) +} + +/** For the abort path: `runTick` rejects, so counters come from the emitted `tick.end` instead. */ +async function runExpectingThrow(opts: RunOpts) { + const { deps, probes } = buildDeps(opts) + await expect(runTick(deps)).rejects.toThrow() + return probes } describe('runTick', () => { @@ -156,14 +208,17 @@ describe('runTick', () => { expect(counters).toEqual({ pairs: 1, liquidatable: 1, + inflightSkipped: 0, + planSkipped: 0, planned: 1, + cooledDown: 0, + backoffSkipped: 0, noSwapPath: 0, quoteFailed: 0, - backoffSkipped: 0, - cooledDown: 0, ok: 1, reverted: 0, - submitted: 1 + submitted: 1, + notSent: 0 }) expect(simulateCalls()).toBe(1) expect(submitCalls()).toBe(1) @@ -227,7 +282,13 @@ describe('runTick', () => { const { counters, quoteCalls, simulateCalls, submitCalls } = await runWith({ inflight: new Set([LABEL]) }) - expect(counters).toMatchObject({ liquidatable: 1, planned: 0, submitted: 0 }) + expect(counters).toMatchObject({ + liquidatable: 1, + inflightSkipped: 1, + planSkipped: 0, + planned: 0, + submitted: 0 + }) expect(quoteCalls()).toBe(0) expect(simulateCalls()).toBe(0) expect(submitCalls()).toBe(0) @@ -248,13 +309,21 @@ describe('runTick', () => { expect(submitCalls()).toBe(0) }) - it('skips a degenerate collateral-less position (plan returns null)', async () => { - const { counters, quoteCalls, submitCalls } = await runWith({ + it('counts a degenerate collateral-less position as planSkipped', async () => { + const { counters, quoteCalls, submitCalls, events } = await runWith({ out: lensOut({ collateral: 0n }) }) - expect(counters).toMatchObject({ liquidatable: 1, planned: 0, submitted: 0 }) + expect(counters).toMatchObject({ + liquidatable: 1, + planSkipped: 1, + planned: 0, + submitted: 0 + }) expect(quoteCalls()).toBe(0) expect(submitCalls()).toBe(0) + const skipped = events.find(e => e.event === 'plan.skipped') + expect(skipped?.level).toBe('info') // documented residual-bad-debt degenerate + expect(skipped?.fields).toMatchObject({ reason: 'no_collateral' }) }) it('tolerates a discovery failure: logs discover.error and submits nothing', async () => { @@ -266,6 +335,166 @@ describe('runTick', () => { expect(submitCalls()).toBe(0) }) + describe('unplannable positions (plan.skipped)', () => { + it('logs plan.skipped with a closed causal chain: sizing inputs AND the derived trace', async () => { + // Dust: a tiny debt against an expensive slot floors the full-debt seize to zero collateral. + const { counters, events } = await runWith({ + out: lensOut({ borrowShares: 1n, collateralPrice: ORACLE_PRICE_SCALE * 10n ** 6n }) + }) + expect(counters).toMatchObject({ liquidatable: 1, planSkipped: 1, planned: 0 }) + const skipped = events.find(e => e.event === 'plan.skipped') + expect(skipped?.level).toBe('info') + expect(skipped?.fields).toMatchObject({ + reason: 'seize_rounds_to_zero', + marketId: MARKET_ID, + borrower: BORROWER, + // derived + seizedAssets: 0n, + // inputs, so the line replays offline + borrowShares: 1n, + collateral: 5000n * WAD, + lltv: PARAMS.lltv + }) + expect(skipped?.fields).toHaveProperty('lif') + expect(skipped?.fields).toHaveProperty('repaidAssetsFull') + expect(skipped?.fields).toHaveProperty('seizeForFullDebt') + }) + + it('warns on a non-reverting zero oracle price', async () => { + const { counters, events } = await runWith({ out: lensOut({ collateralPrice: 0n }) }) + expect(counters).toMatchObject({ liquidatable: 1, planSkipped: 1, planned: 0 }) + const skipped = events.find(e => e.event === 'plan.skipped') + expect(skipped?.level).toBe('warn') // a market anomaly, not routine dust + expect(skipped?.fields).toMatchObject({ reason: 'zero_price' }) + }) + + it('emits one snapshot per sampler window, covering every offender in the tick', async () => { + const planSkipSampler = createBlockSampler(150n) + const other: Address = getAddress('0x9999999999999999999999999999999999999999') + const first = await runWith({ + out: lensOut({ collateral: 0n }), + borrowers: [BORROWER, other], + planSkipSampler, + chainHead: 100n + }) + expect(first.counters).toMatchObject({ liquidatable: 2, planSkipped: 2 }) + expect(first.events.filter(e => e.event === 'plan.skipped')).toHaveLength(2) + + const second = await runWith({ + out: lensOut({ collateral: 0n }), + borrowers: [BORROWER, other], + planSkipSampler, + chainHead: 101n + }) + expect(second.counters).toMatchObject({ planSkipped: 2 }) // still counted at full fidelity + expect(second.events.filter(e => e.event === 'plan.skipped')).toHaveLength(0) + }) + + it('does not consume the sampler window on a tick with nothing to explain', async () => { + const planSkipSampler = createBlockSampler(150n) + const clean = await runWith({ planSkipSampler, chainHead: 100n }) + expect(clean.counters.planSkipped).toBe(0) + const dirty = await runWith({ + out: lensOut({ collateral: 0n }), + planSkipSampler, + chainHead: 101n + }) + expect(dirty.events.filter(e => e.event === 'plan.skipped')).toHaveLength(1) + }) + }) + + describe('submit outcomes', () => { + it('counts a broadcast as submitted and clears the backoff', async () => { + const { counters, backoff } = await runWith({ + seedBackoffAt: 1n, + submitOutcome: { kind: 'sent', nonce: 7, txHash: TX_HASH } + }) + expect(counters).toMatchObject({ ok: 1, submitted: 1, notSent: 0 }) + expect(backoff.shouldSkip(LABEL, 1n)).toBe(false) + }) + + it('counts notSent and ACCUMULATES the backoff when the send failed for this position', async () => { + const cooldown = createCooldownStore({ cooldownMs: 60_000 }) + const { counters, backoff } = await runWith({ + cooldown, + submitOutcome: { kind: 'failed', reason: 'submit_failed' } + }) + expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) + expect(backoff.shouldSkip(LABEL, 100n)).toBe(true) + expect(cooldown.shouldSkip(LABEL)).toBe(true) + }) + + it('lets the backoff delay GROW across repeated send failures', async () => { + // The precise mechanism of bug 3: `clear` deletes the attempt count, so clearing on a + // non-broadcast resets the delay to `baseBlocks` forever and the exponential never accrues. + // Seeded at block 1 (attempts=1) and failing again at 100 must reach attempts=2 → a 4-block + // wait (until 104), not the 2-block wait a reset would give. + const { backoff } = await runWith({ + seedBackoffAt: 1n, + submitOutcome: { kind: 'failed', reason: 'submit_failed' } + }) + expect(backoff.shouldSkip(LABEL, 103n)).toBe(true) + expect(backoff.shouldSkip(LABEL, 104n)).toBe(false) + }) + + it.each(['send_aborted', 'nonce_hole', 'nonce_sync_failed'] as const)( + 'counts notSent but does NOT back off a queue-wide refusal (%s)', + async reason => { + const cooldown = createCooldownStore({ cooldownMs: 60_000 }) + const { counters, backoff } = await runWith({ + cooldown, + submitOutcome: { kind: 'failed', reason } + }) + expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) + expect(backoff.shouldSkip(LABEL, 100n)).toBe(false) + expect(cooldown.shouldSkip(LABEL)).toBe(false) + } + ) + }) + + describe('tick.end completeness', () => { + it('marks a clean tick complete', async () => { + const { events } = await runWith({}) + expect(events.find(e => e.event === 'tick.end')?.fields).toMatchObject({ + complete: true, + submitted: 1 + }) + }) + + it('still emits tick.end with complete:false when a submit aborts the tick', async () => { + const { events } = await runExpectingThrow({ submitThrows: new Error('rpc timeout') }) + expect(events.find(e => e.event === 'tick.end')?.fields).toMatchObject({ + complete: false, + liquidatable: 1, + planned: 1, + ok: 1, + submitted: 0, + notSent: 0 + }) + }) + }) + + describe('lens.read', () => { + it('counts rows the lens returned as invalid', async () => { + const { counters, events } = await runWith({ out: lensOut({ valid: false }) }) + expect(events.find(e => e.event === 'lens.read')?.fields).toEqual({ + pairs: 1, + returned: 1, + invalid: 1 + }) + expect(counters).toMatchObject({ pairs: 1, liquidatable: 0 }) + }) + + it('reports zero invalid for a healthy batch', async () => { + const { events } = await runWith({}) + expect(events.find(e => e.event === 'lens.read')?.fields).toEqual({ + pairs: 1, + returned: 1, + invalid: 0 + }) + }) + }) + describe('position cooldown (opt-in)', () => { it('skips a cooled-down position without quoting or simulating, counting cooledDown', async () => { const cooldown = createCooldownStore({ cooldownMs: 60_000 }) diff --git a/bots/blue-liquidation/test/sizing/plan.test.ts b/bots/blue-liquidation/test/sizing/plan.test.ts index 50d797a4..9175c9ee 100644 --- a/bots/blue-liquidation/test/sizing/plan.test.ts +++ b/bots/blue-liquidation/test/sizing/plan.test.ts @@ -12,7 +12,7 @@ import { wMulDown, mulDivUp } from '../../src/sizing/math' -import { plan } from '../../src/sizing/plan' +import { plan, planWithReason } from '../../src/sizing/plan' const LLTV = 86n * 10n ** 16n // 0.86e18 @@ -141,3 +141,59 @@ describe('plan — repaidShares ≤ borrowShares (no on-chain underflow)', () => expect(checked).toBeGreaterThan(100) }) }) + +describe('planWithReason', () => { + it.each([ + ['no_debt', { hasDebt: false }], + ['healthy', { healthy: true }], + ['no_collateral', { collateral: 0n }], + ['zero_price', { collateralPrice: 0n }] + ] as const)('reports %s without a trace (decided before any arithmetic)', (reason, overrides) => { + expect(planWithReason(baseInput(overrides))).toEqual({ kind: 'skip', reason }) + }) + + it('reports seize_rounds_to_zero with the full sizing trace', () => { + // Dust debt against a 1e6x-priced slot: the full-debt seize floors to zero collateral. + const outcome = planWithReason( + baseInput({ borrowShares: 1n, collateralPrice: ORACLE_PRICE_SCALE * 10n ** 6n }) + ) + expect(outcome).toEqual({ + kind: 'skip', + reason: 'seize_rounds_to_zero', + trace: { + lif: lifFromLltv(LLTV), + repaidAssetsFull: 0n, + seizeForFullDebt: 0n, + seizedAssets: 0n + } + }) + }) + + it('returns a plan whose seize matches the documented rule', () => { + const input = baseInput() + const outcome = planWithReason(input) + expect(outcome.kind).toBe('plan') + expect(outcome).toMatchObject({ plan: { seizedAssets: plan(input)?.seizedAssets } }) + }) + + describe('plan() facade', () => { + // Pins the facade to the implementation so a later edit to planWithReason cannot silently change + // plan()'s contract (which the whole suite above still asserts through plan()). + const cases: PlanInput[] = [ + baseInput(), + baseInput({ hasDebt: false }), + baseInput({ healthy: true }), + baseInput({ collateral: 0n }), + baseInput({ collateralPrice: 0n }), + baseInput({ borrowShares: 1n, collateralPrice: ORACLE_PRICE_SCALE * 10n ** 6n }) + ] + + it.each(cases.map((input, i) => [i, input] as const))( + 'case %i returns the outcome plan, or null for a skip', + (_i, input) => { + const outcome = planWithReason(input) + expect(plan(input)).toEqual(outcome.kind === 'plan' ? outcome.plan : null) + } + ) + }) +}) diff --git a/bots/midnight-liquidation/README.md b/bots/midnight-liquidation/README.md index 19b3ee36..8b2dc887 100644 --- a/bots/midnight-liquidation/README.md +++ b/bots/midnight-liquidation/README.md @@ -400,7 +400,8 @@ The bot computes the oracle-priced reference output for free (no extra API call) route more than `MAX_ROUTE_IMPACT_BPS` below it (`quote.route_quality_failed`). Quote failures (no route, timeout, rate-limited, API error) log `quote.failed`; once every ranked venue is exhausted the position is backed off — an exponential per-position cooldown that bounds API + RPC usage when many -positions fail (the rate-limit defense). A successful submit clears the backoff. +positions fail (the rate-limit defense). Only a successful **broadcast** clears the backoff; a submit +that returned without sending accumulates it instead, so the delay actually grows. If no venue is enabled (bad-debt-only mode) or the collateral is on `EXCLUDE_COLLATERALS`, the tick logs `config.no_swap_path` and skips the candidate (no API call, no backoff). Pure bad-debt @@ -430,14 +431,44 @@ On simulation success, `@repo/bot-kit`'s shared pending queue sends the transaction through the signer client and tracks it by nonce and `(marketId, borrower)` label. -While a label is pending, later ticks skip that position. On each block the queue checks receipts, +While a label is pending, later ticks skip that position and count `inflightSkipped` (that set also +holds labels whose transaction settled within the last `SETTLED_COOLDOWN_BLOCKS`, so the counter can +be non-zero while the queue is empty). On each block the queue checks receipts, logs confirmed or reverted transactions, and fee-bumps stuck transactions until either they confirm, hit the fee ceiling, or exhaust bump attempts. Queue state is in-memory. On restart, chain truth wins: the bot rediscovers live candidates and the -signer nonce cursor starts from the pending chain nonce. If the initial raw broadcast fails after a -nonce is claimed but before a hash is returned, the signer rolls the cursor back and the queue aborts -that tick instead of counting a hashless transaction as submitted. +signer nonce cursor starts from the pending chain nonce. + +`submit` reports whether a transaction actually went out, and `tick.end`'s `submitted` counts only +real broadcasts. The queue has five exits: + +| outcome | logged event | tick effect | +| ------------------------------ | ------------------- | -------------------------------------------- | +| broadcast | `tx.sent` | `submitted`, clears the position's backoff | +| send failed (no nonce claimed) | `tx.submit_failed` | `notSent`, backs the position off + cools it | +| send latched after an abort | `tx.send_aborted` | `notSent` only — a queue-wide refusal | +| nonce cursor unusable | `nonce.sync_failed` | `notSent` only — a queue-wide refusal | +| nonce hole below the cursor | `queue.nonce_hole` | `notSent` only — a queue-wide refusal | + +The three queue-wide refusals reject _every_ send that tick, so they deliberately do NOT back off the +position that happened to be in hand — otherwise a single latch would suppress every healthy position +for several blocks after the latch itself cleared. + +If the initial raw broadcast fails after a nonce is claimed but before a hash is returned, the signer +rolls the cursor back and the queue aborts that tick instead of counting a hashless transaction as +submitted. The tick still emits `tick.end`, with `complete: false` — so partial counters can never be +mistaken for a genuinely idle tick. + +### Why a liquidatable position may not be planned + +A position the chain reports liquidatable can still fail to size. The tick counts every such case as +`planSkipped` and emits a sampled `plan.skipped` carrying the sizing inputs AND the derived numbers +(`lif`, `effectiveDebt`, `cap`, `capEff`, `seizedAssets`), so the decision can be replayed from one +log line. Reasons: `seize_rounds_to_zero` (dust — the largest in-cap seize floors to zero collateral), +`nothing_to_seize` (the best slot is empty), and `cap_not_positive` (a negative RCF numerator, which +should be impossible and therefore logs at `warn`). The diagnostic is sampled at most once per +`PLAN_SKIP_SAMPLE_EVERY_BLOCKS`; the counter is exact on every tick. ## Important Operational Notes diff --git a/bots/midnight-liquidation/src/constants.ts b/bots/midnight-liquidation/src/constants.ts index 9ee04a60..3bd1f46d 100644 --- a/bots/midnight-liquidation/src/constants.ts +++ b/bots/midnight-liquidation/src/constants.ts @@ -50,3 +50,12 @@ export const LISTED_MARKETS_MAX_AGE_MS = 10 * 60_000 * sizing never depends on the swap-quoting package. */ export const BPS = 10_000n + +/** + * Block cadence bounding the per-position `plan.skipped` diagnostic. A liquidatable position that + * cannot be sized recurs every tick, so logging one line per position per block would dominate the + * log source while repeating one fact (~12 positions × ~1780 ticks/hour on Base). At ~2s blocks this + * is roughly every 5 minutes. The sampler is asked only when a skip actually happens, so a quiet + * stretch never consumes the window and the first skip after any gap is always reported. + */ +export const PLAN_SKIP_SAMPLE_EVERY_BLOCKS = 150n diff --git a/bots/midnight-liquidation/src/index.ts b/bots/midnight-liquidation/src/index.ts index f26da88c..d2407bf1 100644 --- a/bots/midnight-liquidation/src/index.ts +++ b/bots/midnight-liquidation/src/index.ts @@ -5,6 +5,7 @@ import { assertContractDeployed, createBalanceMonitor, createBackoff, + createBlockSampler, createCooldownStore, createDeploylessClient, createHeartbeatMonitor, @@ -34,7 +35,11 @@ import type { Market } from './execution/encode-call' import type { LiquidationPlan } from './sizing/plan' import { loadConfig } from './config' -import { LISTED_MARKETS_MAX_AGE_MS, SETTLED_COOLDOWN_BLOCKS } from './constants' +import { + LISTED_MARKETS_MAX_AGE_MS, + PLAN_SKIP_SAMPLE_EVERY_BLOCKS, + SETTLED_COOLDOWN_BLOCKS +} from './constants' import { createApiCandidateSource, discoverBorrowers, @@ -194,6 +199,9 @@ async function main() { // Opt-in per-position cooldown (default disabled): one in-memory store for the process lifetime, // complementary to `backoff` (see POSITION_LIQUIDATION_COOLDOWN_MS). const cooldown = createCooldownStore({ cooldownMs: config.positionCooldownMs }) + // Bounds the `plan.skipped` diagnostic to one burst per cadence; process-lifetime state, like the + // two stores above. + const planSkipSampler = createBlockSampler(PLAN_SKIP_SAMPLE_EVERY_BLOCKS) // The exec calldata for one liquidation — the same bytes the simulate gate checks and the queue // broadcasts, so a sim-ok plan and its broadcast can't drift. @@ -318,7 +326,9 @@ async function main() { }), submit: async ({ market, borrower, plan, swapPlan, blockNumber, label }) => { const fees = initialFees(await signer.getBaseFee(), config.maxFeeWei, config.priorityFeeWei) - await queue.submit({ + // Returned, not awaited-and-discarded: the tick needs to know whether a tx actually went out + // before it clears this position's backoff. + return queue.submit({ request: { to: config.executooorAddress, data: encodeExec(market, borrower, plan, swapPlan) @@ -331,6 +341,7 @@ async function main() { }, backoff, cooldown, + planSkipSampler, inflightLabels: () => queue.inflightLabels(), logger }) diff --git a/bots/midnight-liquidation/src/runner/tick.ts b/bots/midnight-liquidation/src/runner/tick.ts index a271d4c1..886269a1 100644 --- a/bots/midnight-liquidation/src/runner/tick.ts +++ b/bots/midnight-liquidation/src/runner/tick.ts @@ -1,4 +1,11 @@ -import type { Backoff, CooldownStore, Logger, SimulateResult } from '@repo/bot-kit' +import type { + Backoff, + BlockSampler, + CooldownStore, + Logger, + SimulateResult, + SubmitOutcome +} from '@repo/bot-kit' import type { QuoteOutcome, SwapPlan } from '@repo/swaps' import type { Address } from 'viem' @@ -6,23 +13,63 @@ import { assertNever, lensKey, tryCatch } from '@repo/utils' import type { BorrowerCandidate } from '../discovery/borrowers' import type { Market } from '../execution/encode-call' -import type { LiquidationPlan } from '../sizing/plan' +import type { LiquidationPlan, PlanSkipReason } from '../sizing/plan' import type { LensInput, LensOut } from '../state/lens.sol' -import { isBadDebtRealization, plan } from '../sizing/plan' +import { isBadDebtRealization, planWithReason } from '../sizing/plan' import { isLiquidatable, planInputFromLens } from './eligibility' +/** + * Per-tick outcome tally, emitted as `tick.end`. Ordered as the pipeline runs, and exhaustive by + * construction: for a tick that finished (`complete: true` on the event) these identities hold, so a + * future stage added without a counter shows up as a broken sum rather than a silent drop. + * + * pairs >= liquidatable + * liquidatable === inflightSkipped + planSkipped + planned + * planned === cooledDown + backoffSkipped + noSwapPath + quoteFailed + ok + reverted + * ok === submitted + notSent + * + * They do NOT hold when `complete` is `false`: an aborting `submit` throws after `ok` was counted but + * before `submitted`/`notSent`, so the last identity is short by one. + */ type TickCounters = { + /** Lens inputs read this tick — the post-whitelist discovery universe. */ pairs: number + /** Positions the chain says are liquidatable at this block. A market gauge, not a bot decision. */ liquidatable: number + /** + * Skipped because a tx for this label is already in flight. Note the queue's backpressure set also + * holds labels whose tx SETTLED within `settledCooldownBlocks`, so this can be non-zero while the + * queue itself is empty. + */ + inflightSkipped: number + /** Skipped because sizing produced no plan — see the `plan.skipped` event for the reason. */ + planSkipped: number planned: number + cooledDown: number + backoffSkipped: number noSwapPath: number quoteFailed: number - backoffSkipped: number - cooledDown: number ok: number reverted: number + /** Broadcast: the queue reported a transaction actually went out. */ submitted: number + /** The queue returned without broadcasting (a send failure, or a queue-wide refusal). */ + notSent: number +} + +/** + * `warn` marks a reason that should be impossible, so it reads as a live assertion rather than noise: + * the first three are the exact negation of `isLiquidatable`, and `cap_not_positive` means the RCF + * numerator went negative. The dust reasons are ordinary and stay `info`. + */ +const LEVEL_BY_REASON: Record = { + no_debt: 'warn', + locked: 'warn', + healthy_pre_maturity: 'warn', + cap_not_positive: 'warn', + nothing_to_seize: 'info', + seize_rounds_to_zero: 'info' } /** @@ -37,6 +84,10 @@ type TickCounters = { * Discovery failure is tolerated: a transient error is logged (`discover.error`) and the tick proceeds * with zero new candidates. The lens reads every candidate fresh on-chain, so discovery is a coverage * source, never a correctness dependency. + * + * Every exit from the per-position loop increments exactly one counter, and `tick.end` is emitted even + * when a position aborts the tick — with `complete: false`, so partial counters can never be mistaken + * for a genuinely idle tick. */ export async function runTick(deps: { discover: () => Promise @@ -44,7 +95,7 @@ export async function runTick(deps: { chainHead: bigint /** The Executor singleton — the `liquidate` msg.sender whose gate the lens checks. */ caller: Address - /** Headroom (bps) shaved off a cap-binding seize for one-block oracle-drift; passed to `plan()`. */ + /** Headroom (bps) shaved off a cap-binding seize for one-block oracle-drift; passed to sizing. */ seizeCapMarginBps: number readLens: (pairs: LensInput[]) => Promise> /** @@ -59,7 +110,12 @@ export async function runTick(deps: { plan: LiquidationPlan swapPlan: SwapPlan | null }) => Promise - /** Broadcasts a plan via the pending queue (builds the exec tx, derives fees, tracks the nonce). */ + /** + * Broadcasts a plan via the pending queue (builds the exec tx, derives fees, tracks the nonce). + * Reports whether a transaction actually went out: ONLY `kind: 'sent'` may clear the position's + * backoff. Throws when a send fails after claiming a nonce — the tick aborts by design, so the + * signer's cursor rollback is not raced. + */ submit: (args: { market: Market borrower: Address @@ -67,15 +123,22 @@ export async function runTick(deps: { swapPlan: SwapPlan | null blockNumber: bigint label: string - }) => Promise - /** Per-position exponential backoff suppressing repeated quote/simulate failures (rate-limit defense). */ + }) => Promise + /** Per-position exponential backoff suppressing repeated quote/simulate/send failures (rate-limit defense). */ backoff: Backoff /** * Opt-in per-position cooldown, complementary to `backoff`: a fixed wall-clock window suppressing - * re-attempting a position whose last attempt produced no submittable tx (bad-debt realizations + * re-attempting a position whose last attempt produced no broadcast tx (bad-debt realizations * included). Disabled by default (`POSITION_LIQUIDATION_COOLDOWN_MS=0`) — `shouldSkip` always false. */ cooldown: CooldownStore + /** + * Bounds how often the per-position `plan.skipped` diagnostic is emitted. Asked at most once per + * tick and only when there is something to explain, so a quiet stretch never consumes the window: + * the first skip after any gap is always reported, and a persistent one settles to a single burst + * per cadence. All lines in a burst share one `chainHead`, so they read as one coherent snapshot. + */ + planSkipSampler: BlockSampler /** Labels (`${id}:${borrower}`) already in flight — skipped to avoid re-submitting each block. */ inflightLabels: () => ReadonlySet logger: Logger @@ -91,6 +154,7 @@ export async function runTick(deps: { submit, backoff, cooldown, + planSkipSampler, inflightLabels, logger } = deps @@ -108,114 +172,154 @@ export async function runTick(deps: { // 2. Read the lens fresh for the whole batch in one deployless eth_call. const lensOut = await readLens(pairs) - logger.info('lens.read', { pairs: pairs.length, returned: lensOut.size }) + // `returned` is always `pairs` (the batch lens maps every input row); `invalid` is the informative + // one — it counts rows the lens zeroed (unknown market, reverting oracle), which would otherwise be + // indistinguishable from a healthy position in `pairs - liquidatable`. + let invalid = 0 + for (const out of lensOut.values()) if (!out.valid) invalid += 1 + logger.info('lens.read', { pairs: pairs.length, returned: lensOut.size, invalid }) const counters: TickCounters = { pairs: pairs.length, liquidatable: 0, + inflightSkipped: 0, + planSkipped: 0, planned: 0, + cooledDown: 0, + backoffSkipped: 0, noSwapPath: 0, quoteFailed: 0, - backoffSkipped: 0, - cooledDown: 0, ok: 0, reverted: 0, - submitted: 0 + submitted: 0, + notSent: 0 } // 3. Compose liquidatability off-chain → plan → simulate → submit. `inflight` is captured once; // discovery yields distinct (id, borrower) pairs, so no label repeats within a single tick. const inflight = inflightLabels() - for (const pair of pairs) { - const label = lensKey(pair.id, pair.borrower) - const out = lensOut.get(label) - if (!out || !isLiquidatable(out)) continue - counters.liquidatable += 1 - - // Backpressure: a tx for this position is already pending — don't re-plan/simulate/submit it - // every block while it confirms. - if (inflight.has(label)) continue - - const liquidationPlan = plan(planInputFromLens(out), { seizeCapMarginBps }) - if (!liquidationPlan) continue - counters.planned += 1 - logger.info('plan.built', { - marketId: pair.id, - borrower: pair.borrower, - collateralIndex: liquidationPlan.collateralIndex, - seizedAssets: liquidationPlan.seizedAssets, - repaidUnits: liquidationPlan.repaidUnits, - postMaturityMode: liquidationPlan.postMaturityMode - }) - - // Opt-in cooldown (complementary to backoff): a position whose last attempt produced no - // submittable tx is skipped until its wall-clock window elapses — bad-debt realizations included, - // so a repeatedly-reverting one also backs off. No-op when disabled - // (POSITION_LIQUIDATION_COOLDOWN_MS=0). - if (cooldown.shouldSkip(label)) { - counters.cooledDown += 1 - logger.info('cooldown.skip', { marketId: pair.id, borrower: pair.borrower }) - continue - } + // Resolved lazily on the first skip of the tick and reused for the rest, so one tick emits either a + // full snapshot of the blocker set or nothing — never a staggered subset. + let explainSkips: boolean | null = null - // The swap funds repay/seize liquidations. Pure bad-debt realization transfers no assets, so it - // deliberately skips quoting and executes as a no-callback `liquidate`. - let swapPlan: SwapPlan | null = null - if (!isBadDebtRealization(liquidationPlan)) { - // Suppress positions that keep failing to quote/simulate — bounds API + RPC usage under a - // backlog, since executable quotes are spent only on positions not currently backed off. - if (backoff.shouldSkip(label, chainHead)) { - counters.backoffSkipped += 1 + const processPairs = async () => { + for (const pair of pairs) { + const label = lensKey(pair.id, pair.borrower) + const out = lensOut.get(label) + if (!out || !isLiquidatable(out)) continue + counters.liquidatable += 1 + + // Backpressure: a tx for this position is already pending — don't re-plan/simulate/submit it + // every block while it confirms. No log: the queue already narrates this label's whole + // lifecycle (`tx.sent` → `tx.confirmed`/`tx.reverted`/`tx.dropped`) under the same key. + if (inflight.has(label)) { + counters.inflightSkipped += 1 continue } - const outcome = await quoteFor(liquidationPlan, out, label) - if (outcome.kind === 'no_config') { - counters.noSwapPath += 1 - cooldown.mark(label) - logger.info('config.no_swap_path', { - marketId: pair.id, - borrower: pair.borrower, - collateralIndex: liquidationPlan.collateralIndex - }) + + const planInput = planInputFromLens(out) + const outcome = planWithReason(planInput, { seizeCapMarginBps }) + if (outcome.kind === 'skip') { + counters.planSkipped += 1 + explainSkips ??= planSkipSampler.claim(chainHead) + if (explainSkips) { + // Spread the inputs AND the derived trace so the line is a closed causal chain — an + // operator can replay sizing from it without re-deriving anything. `marketId`/`borrower` + // last so a future `PlanInput` field can never shadow them. + logger[LEVEL_BY_REASON[outcome.reason]]('plan.skipped', { + ...planInput, + ...outcome.trace, + marginBps: seizeCapMarginBps, + reason: outcome.reason, + marketId: pair.id, + borrower: pair.borrower + }) + } continue } - if (outcome.kind === 'failed') { - counters.quoteFailed += 1 - backoff.record(label, chainHead) - cooldown.mark(label) + + const liquidationPlan = outcome.plan + counters.planned += 1 + logger.info('plan.built', { + marketId: pair.id, + borrower: pair.borrower, + collateralIndex: liquidationPlan.collateralIndex, + seizedAssets: liquidationPlan.seizedAssets, + repaidUnits: liquidationPlan.repaidUnits, + postMaturityMode: liquidationPlan.postMaturityMode + }) + + // Opt-in cooldown (complementary to backoff): a position whose last attempt produced no + // broadcast tx is skipped until its wall-clock window elapses — bad-debt realizations included, + // so a repeatedly-reverting one also backs off. No-op when disabled + // (POSITION_LIQUIDATION_COOLDOWN_MS=0). + if (cooldown.shouldSkip(label)) { + counters.cooledDown += 1 + logger.info('cooldown.skip', { marketId: pair.id, borrower: pair.borrower }) continue } - swapPlan = outcome.plan - } - const result = await simulate({ - market: out.market, - borrower: pair.borrower, - plan: liquidationPlan, - swapPlan - }) - const fields = { marketId: pair.id, borrower: pair.borrower } - switch (result.status) { - case 'ok': - counters.ok += 1 - logger.info('simulate.ok', fields) - break - case 'revert': - counters.reverted += 1 - // Back off: a sim revert (stale quote, transient unliquidatability) shouldn't re-quote + - // re-simulate this position every block. - backoff.record(label, chainHead) - cooldown.mark(label) - logger.warn('simulate.revert', { ...fields, reason: result.reason }) - break - default: - assertNever(result.status) - } + // Suppress positions that keep failing to quote/simulate/send — bounds API + RPC usage under a + // backlog. Checked for EVERY plan, bad-debt realizations included: they skip quoting but still + // cost a simulation and a send, so a repeatedly-failing one must back off too. + if (backoff.shouldSkip(label, chainHead)) { + counters.backoffSkipped += 1 + continue + } - // ok-only gate (Amendment §10): broadcast only a fully-simulated, swap-funded liquidation. Any - // revert — not-liquidatable, swap slippage, repay shortfall — isn't a fundable plan, so skip it. - if (result.status === 'ok') { - await submit({ + // The swap funds repay/seize liquidations. Pure bad-debt realization transfers no assets, so it + // deliberately skips quoting and executes as a no-callback `liquidate`. + let swapPlan: SwapPlan | null = null + if (!isBadDebtRealization(liquidationPlan)) { + const quote = await quoteFor(liquidationPlan, out, label) + if (quote.kind === 'no_config') { + counters.noSwapPath += 1 + cooldown.mark(label) + logger.info('config.no_swap_path', { + marketId: pair.id, + borrower: pair.borrower, + collateralIndex: liquidationPlan.collateralIndex + }) + continue + } + if (quote.kind === 'failed') { + counters.quoteFailed += 1 + backoff.record(label, chainHead) + cooldown.mark(label) + continue + } + swapPlan = quote.plan + } + + const result = await simulate({ + market: out.market, + borrower: pair.borrower, + plan: liquidationPlan, + swapPlan + }) + const fields = { marketId: pair.id, borrower: pair.borrower } + switch (result.status) { + case 'ok': + counters.ok += 1 + logger.info('simulate.ok', fields) + break + case 'revert': + counters.reverted += 1 + // Back off: a sim revert (stale quote, transient unliquidatability) shouldn't re-quote + + // re-simulate this position every block. + backoff.record(label, chainHead) + cooldown.mark(label) + logger.warn('simulate.revert', { ...fields, reason: result.reason }) + break + default: + assertNever(result.status) + } + + // ok-only gate: broadcast only a fully-simulated, swap-funded liquidation. Any revert — not + // liquidatable, swap slippage, repay shortfall — isn't a fundable plan, so skip it. + if (result.status !== 'ok') continue + + const sendOutcome = await submit({ market: out.market, borrower: pair.borrower, plan: liquidationPlan, @@ -223,11 +327,29 @@ export async function runTick(deps: { blockNumber: chainHead, label }) - backoff.clear(label) - counters.submitted += 1 + if (sendOutcome.kind === 'sent') { + backoff.clear(label) + counters.submitted += 1 + continue + } + counters.notSent += 1 + // Only a per-position send failure earns a backoff. `send_aborted`, `nonce_hole` and + // `nonce_sync_failed` are queue-WIDE refusals that reject every send this tick, so backing off + // here would suppress positions that did nothing wrong — for 2, 4, 8… blocks after the latch + // itself has cleared. The queue has already logged which exit it took. + if (sendOutcome.reason === 'submit_failed') { + backoff.record(label, chainHead) + cooldown.mark(label) + } } } - logger.info('tick.end', { ...counters }) + // Emit counters even when a position aborts the tick (a hashless send after the nonce was claimed + // throws by design). Without `complete` an aborted tick's partial counters are indistinguishable + // from a genuinely idle one. `ensureError` preserves the instance, so rethrowing keeps `TxSendError` + // intact for the runner's `tick.error` decode. + const { error } = await tryCatch(processPairs()) + logger.info('tick.end', { ...counters, complete: !error }) + if (error) throw error return counters } diff --git a/bots/midnight-liquidation/src/sizing/plan.ts b/bots/midnight-liquidation/src/sizing/plan.ts index 28bbd4b5..26bf3eb4 100644 --- a/bots/midnight-liquidation/src/sizing/plan.ts +++ b/bots/midnight-liquidation/src/sizing/plan.ts @@ -52,15 +52,66 @@ type PlanOptions = { seizeCapMarginBps?: number } -export function isBadDebtRealization(plan: LiquidationPlan): boolean { - return plan.seizedAssets === 0n && plan.repaidUnits === 0n +/** + * Why {@link planWithReason} produced no plan. Reasons discriminate SEVERITY (and hence log level); + * the numbers on {@link SizingTrace} discriminate cause. That is why there is no `margin_ate_cap` or + * `price_too_high` — those are one continuous arithmetic that `cap` vs `capEff` vs `seizedAssets` + * pin exactly. + * + * `no_debt`, `locked` and `healthy_pre_maturity` are UNREACHABLE from the tick: `isLiquidatable` + * tests the same lens fields `planWithReason` re-tests, and `!postMaturityMode && healthy` is the + * exact negation of its third clause. A caller that observes one has diverged from the sizing + * module, which is a correctness bug — hence they are logged at `warn`, as a live assertion. + */ +export type PlanSkipReason = + | 'no_debt' + | 'locked' + | 'healthy_pre_maturity' + /** Best slot holds nothing, and the position is not a full bad-debt write-off. */ + | 'nothing_to_seize' + /** The repay cap is zero or negative — see the guard in {@link capBoundOutcome}. */ + | 'cap_not_positive' + /** The expected dust case: a positive cap whose largest in-cap seize floors to zero collateral. */ + | 'seize_rounds_to_zero' + +/** + * Every value on the causal chain from a lens reading to a refused seize, so an operator can replay + * the decision from one log line instead of re-deriving it. `maxRepaid`/`rcfExempt`/`rcfDisabled` + * apply to normal mode only and are `undefined` post-maturity — the logger drops `undefined`, so a + * post-maturity line self-describes its mode. + */ +type SizingTrace = { + postMaturityMode: boolean + lif: bigint + /** `debt - badDebt`: the post-writeoff debt every cap is taken against. */ + effectiveDebt: bigint + /** The seize that was refused — `0n`, or negative when the cap went negative. */ + seizedAssets: bigint + /** The repay bound before `seizeCapMarginBps`. Absent when the cap stage never ran. */ + cap?: bigint + /** `cap` after the margin — separates "the margin ate the cap" from "the division floored". */ + capEff?: bigint + maxRepaid?: bigint + rcfExempt?: boolean + /** `true` when `lltv >= WAD` waives the RCF cap entirely (so `maxRepaid` is omitted, not huge). */ + rcfDisabled?: boolean } +/** A plan, or the reason there is none plus the numbers that explain it. */ +type PlanOutcome = + | { kind: 'plan'; plan: LiquidationPlan } + | { kind: 'skip'; reason: PlanSkipReason; trace?: SizingTrace } + +/** Trace fields fixed once the mode is chosen, before the cap stage runs. */ +type TraceBase = Omit + +export const isBadDebtRealization = (plan: LiquidationPlan): boolean => + plan.seizedAssets === 0n && plan.repaidUnits === 0n + // Repaid units the contract derives when the caller passes `seizedAssets` (midnight-contracts.txt:2369): // two chained ceil-divisions, collateral → loan units → repaid units. -function impliedRepaidUnits(seizedAssets: bigint, price: bigint, lif: bigint): bigint { - return mulDivUp(mulDivUp(seizedAssets, price, ORACLE_PRICE_SCALE), WAD, lif) -} +const impliedRepaidUnits = (seizedAssets: bigint, price: bigint, lif: bigint): bigint => + mulDivUp(mulDivUp(seizedAssets, price, ORACLE_PRICE_SCALE), WAD, lif) /** * The largest seize `S` whose contract-derived repaid (`impliedRepaidUnits(S, price, lif)`) stays @@ -72,65 +123,182 @@ function impliedRepaidUnits(seizedAssets: bigint, price: bigint, lif: bigint): b * overshoots: `impliedRepaidUnits(maxSeizeForCap(cap, …)) <= cap` always holds, and the result is the * largest such seize (`impliedRepaidUnits(result + 1) > cap`). */ -export function maxSeizeForCap(cap: bigint, price: bigint, lif: bigint): bigint { +export const maxSeizeForCap = (cap: bigint, price: bigint, lif: bigint): bigint => { if (cap === 0n || price === 0n) return 0n return mulDivDown(mulDivDown(cap, lif, WAD), ORACLE_PRICE_SCALE, price) } -// Builds a cap-binding seize-exact plan: seize the largest amount whose contract-derived repaid stays -// within `cap`, after shaving `marginBps` off the cap for one-block drift headroom. Returns null when -// that rounds to zero — never a `(0, 0)` plan, which `isBadDebtRealization` would misclassify as a -// bad-debt write-off against a solvent position. -function capBoundPlan( - input: PlanInput, - cap: bigint, - lif: bigint, - marginBps: number, - postMaturityMode: boolean -): LiquidationPlan | null { +/** + * Sizes a cap-binding seize-exact plan: seize the largest amount whose contract-derived repaid stays + * within `cap`, after shaving `marginBps` off the cap for one-block drift headroom. + */ +const capBoundOutcome = ({ + input, + cap, + marginBps, + base +}: { + input: PlanInput + cap: bigint + marginBps: number + base: TraceBase +}): PlanOutcome => { const capEff = mulDivDown(cap, BPS - BigInt(marginBps), BPS) - const seizedAssets = maxSeizeForCap(capEff, input.bestCollateralPrice, lif) - if (seizedAssets === 0n) return null + const seizedAssets = maxSeizeForCap(capEff, input.bestCollateralPrice, base.lif) + const trace: SizingTrace = { ...base, cap, capEff, seizedAssets } + // Discriminate on the RAW cap, not `capEff`: a legitimate 1-wei cap with any margin > 0 floors to + // capEff 0, which is ordinary dust, not an impossible state. A non-positive RAW cap means the RCF + // numerator went negative (`debt - maxDebt < badDebt < debt`), which breaks the module's invariant + // AND would otherwise produce a NEGATIVE seize that slips past an `=== 0n` guard and reverts + // opaquely when abi-encoded as uint256. + if (cap <= 0n) return { kind: 'skip', reason: 'cap_not_positive', trace } + // Rounds to nothing. Never emit a `(0, 0)` plan, which `isBadDebtRealization` would misclassify as + // a bad-debt write-off against a solvent position. `<= 0n` rather than `=== 0n` is defense in + // depth: a positive cap can only floor to a non-negative seize, so the negative case is already + // caught by the guard above — this simply cannot be the branch that lets one through. + if (seizedAssets <= 0n) return { kind: 'skip', reason: 'seize_rounds_to_zero', trace } return { + kind: 'plan', + plan: { + collateralIndex: input.bestCollateralIndex, + seizedAssets, + repaidUnits: 0n, + postMaturityMode: base.postMaturityMode + } + } +} + +const seizeWholeSlot = (input: PlanInput, postMaturityMode: boolean): PlanOutcome => ({ + kind: 'plan', + plan: { collateralIndex: input.bestCollateralIndex, - seizedAssets, + seizedAssets: input.bestCollateralAmt, repaidUnits: 0n, postMaturityMode } +}) + +/** + * Post-maturity mode: the RCF cap does not apply, but the contract still subtracts `repaidUnits` + * from the (post-writeoff) debt with no clamp, so over-repaying reverts (Panic 0x11 underflow). + * Seizing the whole slot is correct only while its implied repaid units fit within the debt — the + * underwater case. When the slot is worth more than the debt (the common case: a solvent borrower + * who simply missed maturity), seize the largest amount whose contract-derived repaid stays within + * that debt. `badDebt` is written off before the repay, so the cap is `debt - badDebt`. + */ +const postMaturityOutcome = ({ + input, + lif, + marginBps +}: { + input: PlanInput + lif: bigint + marginBps: number +}): PlanOutcome => { + const effectiveDebt = input.debt - input.badDebt + const base: TraceBase = { postMaturityMode: true, lif, effectiveDebt } + const wholeSlotRepaid = impliedRepaidUnits( + input.bestCollateralAmt, + input.bestCollateralPrice, + lif + ) + if (wholeSlotRepaid <= effectiveDebt) return seizeWholeSlot(input, true) + return capBoundOutcome({ input, cap: effectiveDebt, marginBps, base }) +} + +/** + * Normal mode: like post-maturity, the contract subtracts `repaidUnits` from the post-writeoff debt + * with no clamp, so an implied repay above it reverts (Panic 0x11). The repay is bounded by the RCF + * cap (waived when the slot is rcf-exempt) AND never exceeds that debt. Seize the whole slot only + * when its implied repaid units fit within the bound; otherwise seize the largest amount whose + * contract-derived repaid stays within that bound. + */ +const normalModeOutcome = ({ + input, + lif, + marginBps +}: { + input: PlanInput + lif: bigint + marginBps: number +}): PlanOutcome => { + const effectiveDebt = input.debt - input.badDebt + const maxRepaid = maxRepaidPreMaturity({ + debt: input.debt, + badDebt: input.badDebt, + maxDebt: input.maxDebt, + lif, + lltv: input.bestCollateralLltv + }) + const exempt = isRcfExempt({ + collateralAmt: input.bestCollateralAmt, + price: input.bestCollateralPrice, + lif, + maxRepaid, + rcfThreshold: input.rcfThreshold + }) + // `lltv >= WAD` waives the cap and returns maxUint256; report that as a flag rather than logging a + // 78-digit number that reads like a real bound. + const rcfDisabled = input.bestCollateralLltv >= WAD + const base: TraceBase = { + postMaturityMode: false, + lif, + effectiveDebt, + rcfExempt: exempt, + ...(rcfDisabled ? { rcfDisabled } : { maxRepaid }) + } + const wholeSlotRepaid = impliedRepaidUnits( + input.bestCollateralAmt, + input.bestCollateralPrice, + lif + ) + const repayCap = exempt ? effectiveDebt : min(maxRepaid, effectiveDebt) + if (wholeSlotRepaid <= repayCap) return seizeWholeSlot(input, false) + return capBoundOutcome({ input, cap: repayCap, marginBps, base }) } /** - * Turns a fresh lens reading into a liquidation plan, or `null` when the position is not - * liquidatable. Mirrors the mode and amount policy of `liquidate(...)`: + * Turns a fresh lens reading into a liquidation plan, or a {@link PlanSkipReason} explaining why + * there is none. Mirrors the mode and amount policy of `liquidate(...)`: * * - past maturity → post-maturity mode: no RCF cap; seize 100% of the best slot if its implied - * repaid units fit within the (post-writeoff) debt, else seize the largest amount whose - * contract-derived repaid stays within that debt; - * - pre-maturity & unhealthy → normal mode: seize 100% of the slot when its implied repaid units - * fit within the repay bound — the RCF cap (waived when rcf-exempt) clamped to the post-writeoff - * debt — otherwise seize the largest amount whose contract-derived repaid stays within that bound; + * repaid units fit within the (post-writeoff) debt, else the largest amount that does; + * - pre-maturity & unhealthy → normal mode: the same, bounded by the RCF cap (waived when + * rcf-exempt) clamped to the post-writeoff debt; * - otherwise (no debt, locked, or healthy-and-pre-maturity) → skip. * * Every non-bad-debt plan is **seize-exact**: it pins `seizedAssets` (with `repaidUnits = 0`) and the * contract ceil-derives `repaidUnits` (:2369). Pinning the seize means the Executor holds exactly what * every venue (Uniswap or aggregator) sells, so there is no sell-side drift. Bad-debt realization is * the only `(0, 0)` plan. A cap-binding seize is sized against `cap·(1 - seizeCapMarginBps)` to keep - * headroom for a one-block oracle move; any residual drift fails closed in `simulate()`, never on-chain. + * headroom for a one-block oracle move; any residual drift fails closed in `simulate()`, never + * on-chain. + * + * @param input - Fresh lens-derived position state, all read at one `blockTimestamp`. + * @param options - Non-lens sizing knobs; `seizeCapMarginBps` defaults to `0` (unmargined cap). + * @returns `{ kind: 'plan' }` with a submittable plan, or `{ kind: 'skip' }` with the reason and — + * for every cap-stage refusal — a {@link SizingTrace} sufficient to replay the arithmetic. */ -export function plan(input: PlanInput, options: PlanOptions = {}): LiquidationPlan | null { - const { seizeCapMarginBps = 0 } = options +export const planWithReason = (input: PlanInput, options: PlanOptions = {}): PlanOutcome => { + const { seizeCapMarginBps: marginBps = 0 } = options - if (!input.hasDebt || input.locked) return null + if (!input.hasDebt) return { kind: 'skip', reason: 'no_debt' } + if (input.locked) return { kind: 'skip', reason: 'locked' } const postMaturityMode = input.blockTimestamp > input.maturity - if (!postMaturityMode && input.healthy) return null + if (!postMaturityMode && input.healthy) return { kind: 'skip', reason: 'healthy_pre_maturity' } + // Full write-off: the only legitimate `(0, 0)` plan. Must precede the `nothing_to_seize` guard, + // which would otherwise steal it (a fully-bad-debt position often holds no collateral either). if (input.badDebt >= input.debt) { return { - collateralIndex: input.bestCollateralIndex, - seizedAssets: 0n, - repaidUnits: 0n, - postMaturityMode + kind: 'plan', + plan: { + collateralIndex: input.bestCollateralIndex, + seizedAssets: 0n, + repaidUnits: 0n, + postMaturityMode + } } } @@ -141,58 +309,32 @@ export function plan(input: PlanInput, options: PlanOptions = {}): LiquidationPl postMaturityMode }) - const seizeWholeSlot: LiquidationPlan = { - collateralIndex: input.bestCollateralIndex, - seizedAssets: input.bestCollateralAmt, - repaidUnits: 0n, - postMaturityMode - } - - // Post-maturity mode: the RCF cap does not apply, but the contract still subtracts `repaidUnits` - // from the (post-writeoff) debt with no clamp, so over-repaying reverts (Panic 0x11 underflow). - // Seizing the whole slot is correct only while its implied repaid units fit within the debt — the - // underwater case. When the slot is worth more than the debt (the common case: a solvent borrower - // who simply missed maturity), seize the largest amount whose contract-derived repaid stays within - // that debt. `badDebt` is written off before the repay, so the cap is the post-writeoff debt - // (`debt - badDebt`). - if (postMaturityMode) { - const effectiveDebt = input.debt - input.badDebt - const wholeSlotRepaid = impliedRepaidUnits( - input.bestCollateralAmt, - input.bestCollateralPrice, - lif - ) - if (wholeSlotRepaid <= effectiveDebt) return seizeWholeSlot - return capBoundPlan(input, effectiveDebt, lif, seizeCapMarginBps, postMaturityMode) + // An empty best slot would make `wholeSlotRepaid` 0, pass every cap comparison, and return a + // `(0, 0)` whole-slot plan that `isBadDebtRealization` reads as a write-off against a position we + // just established is NOT fully bad debt. Refuse it explicitly instead. + if (input.bestCollateralAmt === 0n) { + return { + kind: 'skip', + reason: 'nothing_to_seize', + // No cap stage ran — the slot is empty, so `cap`/`capEff` are deliberately absent rather than + // fabricated (the logger drops `undefined`, so the line self-describes). + trace: { postMaturityMode, lif, effectiveDebt: input.debt - input.badDebt, seizedAssets: 0n } + } } - // Normal mode: like post-maturity above, the contract subtracts `repaidUnits` from the - // post-writeoff debt with no clamp, so an implied repay above it reverts (Panic 0x11). The repay is - // bounded by the RCF cap (waived when the slot is rcf-exempt) AND never exceeds that debt. Seize - // the whole slot only when its implied repaid units fit within the bound; otherwise seize the - // largest amount whose contract-derived repaid stays within that bound. - const effectiveDebt = input.debt - input.badDebt - const maxRepaid = maxRepaidPreMaturity({ - debt: input.debt, - badDebt: input.badDebt, - maxDebt: input.maxDebt, - lif, - lltv: input.bestCollateralLltv - }) - const wholeSlotRepaid = impliedRepaidUnits( - input.bestCollateralAmt, - input.bestCollateralPrice, - lif - ) - const exempt = isRcfExempt({ - collateralAmt: input.bestCollateralAmt, - price: input.bestCollateralPrice, - lif, - maxRepaid, - rcfThreshold: input.rcfThreshold - }) - const repayCap = exempt ? effectiveDebt : min(maxRepaid, effectiveDebt) + return postMaturityMode + ? postMaturityOutcome({ input, lif, marginBps }) + : normalModeOutcome({ input, lif, marginBps }) +} - if (wholeSlotRepaid <= repayCap) return seizeWholeSlot - return capBoundPlan(input, repayCap, lif, seizeCapMarginBps, postMaturityMode) +/** + * Bare-plan facade over {@link planWithReason} for callers that do not report a reason. + * + * @param input - Fresh lens-derived position state. + * @param options - Non-lens sizing knobs. + * @returns The plan, or `null` when the position is not liquidatable or cannot be sized. + */ +export const plan = (input: PlanInput, options: PlanOptions = {}): LiquidationPlan | null => { + const outcome = planWithReason(input, options) + return outcome.kind === 'plan' ? outcome.plan : null } diff --git a/bots/midnight-liquidation/test/runner/tick.test.ts b/bots/midnight-liquidation/test/runner/tick.test.ts index ee826b01..95341956 100644 --- a/bots/midnight-liquidation/test/runner/tick.test.ts +++ b/bots/midnight-liquidation/test/runner/tick.test.ts @@ -1,9 +1,9 @@ -import type { Logger, SimulateResult } from '@repo/bot-kit' -import type { CooldownStore } from '@repo/bot-kit' +import type { Logger, SimulateResult, SubmitOutcome } from '@repo/bot-kit' +import type { Backoff, BlockSampler, CooldownStore } from '@repo/bot-kit' import type { QuoteOutcome, SwapPlan } from '@repo/swaps' import type { Address, Hex } from 'viem' -import { createBackoff, createCooldownStore } from '@repo/bot-kit' +import { createBackoff, createBlockSampler, createCooldownStore } from '@repo/bot-kit' import { lensKey } from '@repo/utils' import { getAddress } from 'viem' import { describe, expect, it } from 'vitest' @@ -34,6 +34,24 @@ const ROUTER: Address = getAddress('0x5555555555555555555555555555555555555555') const ZERO = '0x0000000000000000000000000000000000000000' as const const MARKET: Hex = `0x${'a'.repeat(64)}` const LABEL = lensKey(MARKET, BORROWER) +const TX_HASH: Hex = `0x${'b'.repeat(64)}` +const SENT: SubmitOutcome = { kind: 'sent', nonce: 7, txHash: TX_HASH } + +type TickCounters = Awaited> + +/** + * Every counter identity the tick promises for a completed tick. Asserted for EVERY case built by + * `runWith`, so a stage added without a counter breaks the sums instead of silently dropping a + * position — the exact class of bug these counters exist to catch. + */ +function expectCountersConsistent(c: TickCounters) { + expect(c.pairs).toBeGreaterThanOrEqual(c.liquidatable) + expect(c.liquidatable).toBe(c.inflightSkipped + c.planSkipped + c.planned) + expect(c.planned).toBe( + c.cooledDown + c.backoffSkipped + c.noSwapPath + c.quoteFailed + c.ok + c.reverted + ) + expect(c.ok).toBe(c.submitted + c.notSent) +} const SWAP_PLAN: SwapPlan = { steps: [ @@ -101,7 +119,7 @@ function stubReadLens(out: LensOut | null) { } } -function runWith(opts: { +type RunOpts = { out?: LensOut | null simulateResult?: SimulateResult quoteOutcome?: QuoteOutcome @@ -112,27 +130,40 @@ function runWith(opts: { noSwap?: boolean seedBackoffAt?: bigint cooldown?: CooldownStore -}) { + seizeCapMarginBps?: number + /** What the queue reports; defaults to a real broadcast. */ + submitOutcome?: SubmitOutcome + /** Makes `submit` throw, as a hashless send after a nonce was claimed does. */ + submitThrows?: Error + /** Reuse a store/sampler across two `runTick` calls, so cross-tick behavior can be asserted. */ + backoff?: Backoff + planSkipSampler?: BlockSampler +} + +// Shared dep construction so the throwing case exercises exactly the same wiring as `runWith`. +function buildDeps(opts: RunOpts) { const { logger, events } = spyLogger() let simulateCalls = 0 let submitCalls = 0 let quoteCalls = 0 const chainHead = opts.chainHead ?? 100n - const backoff = createBackoff({ baseBlocks: 2n, maxBlocks: 64n }) + const backoff = opts.backoff ?? createBackoff({ baseBlocks: 2n, maxBlocks: 64n }) if (opts.seedBackoffAt !== undefined) backoff.record(LABEL, opts.seedBackoffAt) // Default disabled (0) so existing cases are unaffected; opt-in cases pass an enabled store. const cooldown = opts.cooldown ?? createCooldownStore({ cooldownMs: 0 }) + // 0n so every skip explains itself by default; throttling cases pass a real cadence. + const planSkipSampler = opts.planSkipSampler ?? createBlockSampler(0n) const defaultOutcome: QuoteOutcome = opts.noSwap ? { kind: 'no_config' } : { kind: 'swap', plan: SWAP_PLAN } - const result = runTick({ + const deps = { discover: async () => { if (opts.discoverError) throw opts.discoverError return candidates(...(opts.borrowers ?? [BORROWER])) }, chainHead, caller: CALLER, - seizeCapMarginBps: 0, + seizeCapMarginBps: opts.seizeCapMarginBps ?? 0, readLens: stubReadLens(opts.out === undefined ? lensOut() : opts.out), quoteFor: async () => { quoteCalls += 1 @@ -140,25 +171,46 @@ function runWith(opts: { }, simulate: async () => { simulateCalls += 1 - return opts.simulateResult ?? { status: 'ok' } + return opts.simulateResult ?? ({ status: 'ok' } as SimulateResult) }, submit: async () => { submitCalls += 1 + if (opts.submitThrows) throw opts.submitThrows + return opts.submitOutcome ?? SENT }, backoff, cooldown, - inflightLabels: () => opts.inflight ?? new Set(), + planSkipSampler, + inflightLabels: () => opts.inflight ?? new Set(), logger + } + return { + deps, + probes: { + backoff, + cooldown, + planSkipSampler, + simulateCalls: () => simulateCalls, + submitCalls: () => submitCalls, + quoteCalls: () => quoteCalls, + events + } + } +} + +function runWith(opts: RunOpts) { + const { deps, probes } = buildDeps(opts) + return runTick(deps).then(counters => { + expectCountersConsistent(counters) + return { counters, ...probes } }) - return result.then(counters => ({ - counters, - backoff, - cooldown, - simulateCalls: () => simulateCalls, - submitCalls: () => submitCalls, - quoteCalls: () => quoteCalls, - events - })) +} + +/** For the abort path: `runTick` rejects, so counters come from the emitted `tick.end` instead. */ +async function runExpectingThrow(opts: RunOpts) { + const { deps, probes } = buildDeps(opts) + await expect(runTick(deps)).rejects.toThrow() + return probes } describe('runTick', () => { @@ -169,14 +221,17 @@ describe('runTick', () => { expect(counters).toEqual({ pairs: 1, liquidatable: 1, + inflightSkipped: 0, + planSkipped: 0, planned: 1, + cooledDown: 0, + backoffSkipped: 0, noSwapPath: 0, quoteFailed: 0, - backoffSkipped: 0, - cooledDown: 0, ok: 1, reverted: 0, - submitted: 1 + submitted: 1, + notSent: 0 }) expect(simulateCalls()).toBe(1) expect(submitCalls()).toBe(1) @@ -262,7 +317,13 @@ describe('runTick', () => { const { counters, quoteCalls, simulateCalls, submitCalls } = await runWith({ inflight: new Set([LABEL]) }) - expect(counters).toMatchObject({ liquidatable: 1, planned: 0, submitted: 0 }) + expect(counters).toMatchObject({ + liquidatable: 1, + inflightSkipped: 1, + planSkipped: 0, + planned: 0, + submitted: 0 + }) expect(quoteCalls()).toBe(0) expect(simulateCalls()).toBe(0) expect(submitCalls()).toBe(0) @@ -292,6 +353,225 @@ describe('runTick', () => { expect(submitCalls()).toBe(0) }) + describe('unplannable positions (plan.skipped)', () => { + // Post-maturity dust: 1 wei of debt against a high-priced slot. The cap binds, but the largest + // seize whose derived repaid fits in 1 wei rounds to 0 collateral — the production signature that + // left 12 liquidatable positions silently unplanned. + const DUST = () => + lensOut({ + healthy: true, + blockTimestamp: 3000n, + debt: 1n, + badDebt: 0n, + bestCollateralAmt: 10n ** 18n, + bestCollateralPrice: 10n ** 37n + }) + + it('counts planSkipped instead of silently dropping a liquidatable position', async () => { + const { counters, quoteCalls, simulateCalls, submitCalls } = await runWith({ out: DUST() }) + expect(counters).toMatchObject({ + liquidatable: 1, + inflightSkipped: 0, + planSkipped: 1, + planned: 0, + submitted: 0 + }) + expect(quoteCalls()).toBe(0) + expect(simulateCalls()).toBe(0) + expect(submitCalls()).toBe(0) + }) + + it('logs plan.skipped with a closed causal chain: sizing inputs AND the derived trace', async () => { + const { events } = await runWith({ out: DUST() }) + const skipped = events.find(e => e.event === 'plan.skipped') + expect(skipped?.level).toBe('info') // ordinary dust, not an invariant violation + expect(skipped?.fields).toMatchObject({ + reason: 'seize_rounds_to_zero', + marketId: MARKET, + borrower: BORROWER, + // derived (unknowable without replaying plan()) + postMaturityMode: true, + lif: 1027777777777777777n, + effectiveDebt: 1n, + cap: 1n, + capEff: 1n, + seizedAssets: 0n, + marginBps: 0, + // inputs (so the line can be replayed offline) + debt: 1n, + badDebt: 0n, + maturity: 2000n, + bestCollateralAmt: 10n ** 18n, + bestCollateralPrice: 10n ** 37n + }) + }) + + it('distinguishes the margin eating the cap from the division flooring', async () => { + // Same 1-wei cap, but a 30bps margin floors capEff to 0. Same reason, different numbers — this + // is what proves the field set discriminates the sub-causes. + const { events } = await runWith({ out: DUST(), seizeCapMarginBps: 30 }) + expect(events.find(e => e.event === 'plan.skipped')?.fields).toMatchObject({ + reason: 'seize_rounds_to_zero', + cap: 1n, + capEff: 0n, + marginBps: 30 + }) + }) + + it('warns on a non-positive cap and refuses the negative-seize plan', async () => { + // debt - maxDebt (100) < badDebt (500) < debt (1000) makes the RCF numerator negative, so the + // cap and the derived seize both go negative. Pre-fix this returned a plan with a NEGATIVE + // seizedAssets that slipped past an `=== 0n` guard. + // rcfThreshold 0 so the slot is NOT rcf-exempt and the negative RCF cap actually binds. + const { counters, events, submitCalls } = await runWith({ + out: lensOut({ + debt: 1000n, + badDebt: 500n, + maxDebt: 900n, + market: { ...lensOut().market, rcfThreshold: 0n } + }), + chainHead: 100n + }) + expect(counters).toMatchObject({ liquidatable: 1, planSkipped: 1, planned: 0 }) + const skipped = events.find(e => e.event === 'plan.skipped') + expect(skipped?.level).toBe('warn') // should be impossible → alertable + expect(skipped?.fields).toMatchObject({ reason: 'cap_not_positive', cap: -7406n }) + expect(submitCalls()).toBe(0) + }) + + it('emits one snapshot per sampler window, covering every offender in the tick', async () => { + // Cadence of 150 blocks and two offenders: both explain on the first tick, neither on the next. + const planSkipSampler = createBlockSampler(150n) + const other: Address = getAddress('0x9999999999999999999999999999999999999999') + const first = await runWith({ + out: DUST(), + borrowers: [BORROWER, other], + planSkipSampler, + chainHead: 100n + }) + expect(first.counters).toMatchObject({ liquidatable: 2, planSkipped: 2 }) + expect(first.events.filter(e => e.event === 'plan.skipped')).toHaveLength(2) + + const second = await runWith({ + out: DUST(), + borrowers: [BORROWER, other], + planSkipSampler, + chainHead: 101n + }) + expect(second.counters).toMatchObject({ planSkipped: 2 }) // still counted at full fidelity + expect(second.events.filter(e => e.event === 'plan.skipped')).toHaveLength(0) + }) + + it('does not consume the sampler window on a tick with nothing to explain', async () => { + // The edge-trigger guarantee: a clean tick must not spend the window, so the first skip after + // any quiet stretch is always reported. + const planSkipSampler = createBlockSampler(150n) + const clean = await runWith({ planSkipSampler, chainHead: 100n }) + expect(clean.counters.planSkipped).toBe(0) + const dirty = await runWith({ out: DUST(), planSkipSampler, chainHead: 101n }) + expect(dirty.events.filter(e => e.event === 'plan.skipped')).toHaveLength(1) + }) + }) + + describe('submit outcomes', () => { + it('counts a broadcast as submitted and clears the backoff', async () => { + const { counters, backoff } = await runWith({ + seedBackoffAt: 1n, + submitOutcome: { kind: 'sent', nonce: 7, txHash: TX_HASH } + }) + expect(counters).toMatchObject({ ok: 1, submitted: 1, notSent: 0 }) + expect(backoff.shouldSkip(LABEL, 1n)).toBe(false) + }) + + it('counts notSent and ACCUMULATES the backoff when the send failed for this position', async () => { + // The bug-3 regression: pre-fix `backoff.clear` ran unconditionally after submit, so a + // send-failing position was re-quoted + re-simulated + re-sent every block forever. + const cooldown = createCooldownStore({ cooldownMs: 60_000 }) + const { counters, backoff } = await runWith({ + cooldown, + submitOutcome: { kind: 'failed', reason: 'submit_failed' } + }) + expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) + expect(backoff.shouldSkip(LABEL, 100n)).toBe(true) + expect(cooldown.shouldSkip(LABEL)).toBe(true) + }) + + it('lets the backoff delay GROW across repeated send failures', async () => { + // The precise mechanism of bug 3: `clear` deletes the attempt count, so clearing on a + // non-broadcast resets the delay to `baseBlocks` forever and the exponential never accrues. + // Seeded at block 1 (attempts=1) and failing again at 100 must reach attempts=2 → a 4-block + // wait (until 104), not the 2-block wait a reset would give. + const { backoff } = await runWith({ + seedBackoffAt: 1n, + submitOutcome: { kind: 'failed', reason: 'submit_failed' } + }) + expect(backoff.shouldSkip(LABEL, 103n)).toBe(true) + expect(backoff.shouldSkip(LABEL, 104n)).toBe(false) + }) + + it.each(['send_aborted', 'nonce_hole', 'nonce_sync_failed'] as const)( + 'counts notSent but does NOT back off a queue-wide refusal (%s)', + async reason => { + // These refuse EVERY send this tick, so attributing them to whichever position was in hand + // would suppress innocent positions after the latch itself has cleared. + const cooldown = createCooldownStore({ cooldownMs: 60_000 }) + const { counters, backoff } = await runWith({ + cooldown, + submitOutcome: { kind: 'failed', reason } + }) + expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) + expect(backoff.shouldSkip(LABEL, 100n)).toBe(false) + expect(cooldown.shouldSkip(LABEL)).toBe(false) + } + ) + }) + + describe('tick.end completeness', () => { + it('marks a clean tick complete', async () => { + const { events } = await runWith({}) + expect(events.find(e => e.event === 'tick.end')?.fields).toMatchObject({ + complete: true, + submitted: 1 + }) + }) + + it('still emits tick.end with complete:false when a submit aborts the tick', async () => { + // A hashless send after the nonce was claimed throws by design. Pre-fix the throw escaped + // before `tick.end`, so the whole tick's counters vanished. + const { events } = await runExpectingThrow({ submitThrows: new Error('rpc timeout') }) + const end = events.find(e => e.event === 'tick.end') + expect(end?.fields).toMatchObject({ + complete: false, + liquidatable: 1, + planned: 1, + ok: 1, + submitted: 0, + notSent: 0 + }) + }) + }) + + describe('lens.read', () => { + it('counts rows the lens returned as invalid', async () => { + const { counters, events } = await runWith({ out: lensOut({ valid: false }) }) + expect(events.find(e => e.event === 'lens.read')?.fields).toEqual({ + pairs: 1, + returned: 1, + invalid: 1 + }) + expect(counters).toMatchObject({ pairs: 1, liquidatable: 0 }) + }) + + it('reports zero invalid for a healthy batch', async () => { + const { events } = await runWith({}) + expect(events.find(e => e.event === 'lens.read')?.fields).toEqual({ + pairs: 1, + returned: 1, + invalid: 0 + }) + }) + }) + describe('position cooldown (opt-in)', () => { it('skips a cooled-down position without quoting or simulating, counting cooledDown', async () => { const cooldown = createCooldownStore({ cooldownMs: 60_000 }) diff --git a/bots/midnight-liquidation/test/sizing/plan.test.ts b/bots/midnight-liquidation/test/sizing/plan.test.ts index 126e8649..2a829232 100644 --- a/bots/midnight-liquidation/test/sizing/plan.test.ts +++ b/bots/midnight-liquidation/test/sizing/plan.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from 'vitest' import type { LiquidationPlan, PlanInput } from '../../src/sizing/plan' import { ORACLE_PRICE_SCALE, WAD } from '../../src/constants' -import { maxSeizeForCap, plan } from '../../src/sizing/plan' +import { maxSeizeForCap, plan, planWithReason } from '../../src/sizing/plan' const MAX_LIF = 1036269430051813471n const LLTV = 860000000000000000n @@ -234,3 +234,131 @@ describe('maxSeizeForCap', () => { } }) }) + +describe('planWithReason', () => { + // The reachable-from-the-tick skips. `no_debt`/`locked`/`healthy_pre_maturity` are excluded by + // `isLiquidatable` upstream, so they are asserted only for their reason, not their trace. + it.each([ + ['no_debt', { hasDebt: false }], + ['locked', { locked: true }], + ['healthy_pre_maturity', { healthy: true }] + ] as const)('reports %s without a trace (decided before any arithmetic)', (reason, overrides) => { + const outcome = planWithReason(baseInput(overrides)) + expect(outcome).toEqual({ kind: 'skip', reason }) + }) + + it('reports seize_rounds_to_zero with the full cap-stage trace', () => { + // Post-maturity dust: a 1-wei cap against a 10x-priced slot floors the seize to zero. + const outcome = planWithReason( + baseInput({ + blockTimestamp: 3000n, + healthy: true, + debt: 1n, + badDebt: 0n, + bestCollateralAmt: 1n * WAD, + bestCollateralPrice: ORACLE_PRICE_SCALE * 10n + }) + ) + expect(outcome).toEqual({ + kind: 'skip', + reason: 'seize_rounds_to_zero', + trace: { + postMaturityMode: true, + lif: 1010074841681059297n, + effectiveDebt: 1n, + cap: 1n, + capEff: 1n, + seizedAssets: 0n + } + }) + }) + + it('reports capEff 0 when the margin alone eats a 1-wei cap', () => { + const outcome = planWithReason( + baseInput({ + blockTimestamp: 3000n, + healthy: true, + debt: 1n, + badDebt: 0n, + bestCollateralAmt: 1n * WAD, + bestCollateralPrice: ORACLE_PRICE_SCALE * 10n + }), + { seizeCapMarginBps: 30 } + ) + expect(outcome.kind).toBe('skip') + expect(outcome).toMatchObject({ + reason: 'seize_rounds_to_zero', + trace: { cap: 1n, capEff: 0n, seizedAssets: 0n } + }) + }) + + it('reports nothing_to_seize for an empty best slot, omitting the cap stage', () => { + const outcome = planWithReason(baseInput({ bestCollateralAmt: 0n })) + expect(outcome).toEqual({ + kind: 'skip', + reason: 'nothing_to_seize', + trace: { + postMaturityMode: false, + lif: MAX_LIF, + effectiveDebt: 1000n * WAD, + seizedAssets: 0n + } + }) + }) + + it('reports cap_not_positive — and no longer returns a NEGATIVE-seize plan', () => { + // `debt - maxDebt < badDebt < debt` makes the RCF numerator negative, so maxRepaid, the cap and + // the derived seize all go negative. Before the `<= 0n` guard this returned a plan whose + // `seizedAssets` was negative, which reverts opaquely once abi-encoded as uint256. + const input = baseInput({ + debt: 1000n, + badDebt: 500n, + maxDebt: 900n, + rcfThreshold: 0n, // not rcf-exempt, so the negative cap actually binds + bestCollateralAmt: 5000n, + bestCollateralPrice: ORACLE_PRICE_SCALE, + bestCollateralMaxLif: 1100000000000000000n + }) + const outcome = planWithReason(input) + expect(outcome).toMatchObject({ kind: 'skip', reason: 'cap_not_positive' }) + expect(outcome.kind === 'skip' && outcome.trace?.cap).toBe(-7406n) + expect(outcome.kind === 'skip' && outcome.trace?.seizedAssets).toBe(-8146n) + expect(plan(input)).toBeNull() + }) + + it('omits maxRepaid and flags rcfDisabled when lltv waives the cap', () => { + const outcome = planWithReason( + baseInput({ bestCollateralLltv: WAD, bestCollateralAmt: 0n, bestCollateralMaxLif: WAD }) + ) + // Reached via nothing_to_seize, so only the mode fields are present — the point is that a + // maxUint256 maxRepaid is never logged as if it were a real bound. + expect(outcome.kind === 'skip' && outcome.trace).not.toHaveProperty('maxRepaid') + }) + + describe('plan() facade', () => { + // Pins the facade to the implementation so a later edit to planWithReason cannot silently change + // plan()'s contract (which the whole suite above still asserts through plan()). + const cases: PlanInput[] = [ + baseInput(), // plans + baseInput({ hasDebt: false }), + baseInput({ locked: true }), + baseInput({ healthy: true }), + baseInput({ bestCollateralAmt: 0n }), + baseInput({ + blockTimestamp: 3000n, + healthy: true, + debt: 1n, + bestCollateralAmt: 1n * WAD, + bestCollateralPrice: ORACLE_PRICE_SCALE * 10n + }) + ] + + it.each(cases.map((input, i) => [i, input] as const))( + 'case %i returns the outcome plan, or null for a skip', + (_i, input) => { + const outcome = planWithReason(input) + expect(plan(input)).toEqual(outcome.kind === 'plan' ? outcome.plan : null) + } + ) + }) +}) diff --git a/docs/decisions/TIB-2026-05-28-midnight-liquidation-bot.md b/docs/decisions/TIB-2026-05-28-midnight-liquidation-bot.md index 308ff4c2..45782a63 100644 --- a/docs/decisions/TIB-2026-05-28-midnight-liquidation-bot.md +++ b/docs/decisions/TIB-2026-05-28-midnight-liquidation-bot.md @@ -926,6 +926,8 @@ After each phase: a few comments here and in `sizing/plan.ts` point at that snapshot and are approximate; for the authoritative surface, trust the vendored interface above and the deployed contract. +## Addenda + ### 2026-07-10 — persistent runner and Alternative 5 superseded by the pipeline architecture [TIB-2026-07-13 (bot architecture)](./TIB-2026-07-13-bot-architecture.md) supersedes two of this @@ -935,3 +937,61 @@ Alternative 5 ("Persistent queue state across runner restarts", rejected here) i required by the one-shot model**: a per-run process with no memory of its pending txs could never fee-bump a stuck one. The spirit survives — persisted state is a hint reconciled against chain truth, and losing the file degrades to this TIB's restart semantics. + +### 2026-08-05 — `tick.end` counters reconciled; `submitted` now means broadcast + +Three production defects in the tick, found by a BetterStack review of `bot.liquidation.midnight` +(source 2607569), changed the observability contract this TIB documents. This addendum records the +new shape; the body above is left as written. + +**The defects.** (1) Two `continue`s — a position already in flight, and a position sizing refused — +incremented no counter and logged nothing, so from 2026-07-31 15:37 the bot reported +`liquidatable: 12, planned: 0` for ~215,000 consecutive ticks with every skip counter at zero and no +way to learn why. (2) `submitted` counted `submit` _calls_, not broadcasts: on 2026-07-30 it summed +2,425 while `tx.sent` was 0 and `tx.submit_failed` was 2,666. (3) `backoff.clear` ran unconditionally +after `submit`, wiping the accumulated attempt count, so a position that simulated ok and then failed +to send was re-quoted, re-simulated and re-sent every block indefinitely. + +**The counter set** (identical in both liquidators, ordered as the pipeline runs): + +```text +tick.end { pairs, liquidatable, inflightSkipped, planSkipped, planned, + cooledDown, backoffSkipped, noSwapPath, quoteFailed, + ok, reverted, submitted, notSent, complete } +``` + +This reconciles drift in both directions: `backoffSkipped`/`cooledDown` shipped without ever being +documented, and `badRoute` was documented here but never implemented (route-quality rejection surfaces +as the `quote.route_quality_failed` event, not a counter). + +For a tick with `complete: true` the set is exhaustive by construction, and the identities are +asserted in every tick test — so a stage added without a counter breaks a sum instead of silently +dropping a position: + +```text +pairs >= liquidatable +liquidatable === inflightSkipped + planSkipped + planned +planned === cooledDown + backoffSkipped + noSwapPath + quoteFailed + ok + reverted +ok === submitted + notSent +``` + +`complete: false` marks a tick aborted part-way (a hashless send after a nonce was claimed still +throws by design); its counters are partial and the last identity is short by one. Previously such a +tick emitted no `tick.end` at all. + +**Semantics that changed.** `submitted` counts only real broadcasts. `notSent` counts a `submit` that +returned without sending, and the queue now reports which of its four non-sending exits it took +(`tx.submit_failed`, `tx.send_aborted`, `nonce.sync_failed`, `queue.nonce_hole`). Only the +per-position one (`tx.submit_failed`) backs the position off; the other three are queue-wide refusals +that reject every send that tick, so attributing them to one position would suppress healthy ones. +Only a broadcast clears a backoff. + +**New event.** `plan.skipped` explains a liquidatable-but-unsizable position, carrying the sizing +inputs and the derived numbers (`lif`, `effectiveDebt`, `cap`, `capEff`, `seizedAssets`) so the +decision replays from one line. `info` for the ordinary dust reasons; `warn` for reasons that should +be unreachable, which makes it a live assertion that the eligibility gate and the sizing module have +not diverged. It is sampled at most once per `PLAN_SKIP_SAMPLE_EVERY_BLOCKS` (the counter stays exact +every tick), because one line per position per block would have been ~21k lines/hour/bot. + +**Any BetterStack chart or alert built on `submitted` changes meaning** and needs review; the new +fields also need the dashboard metrics-collection step before they are queryable. diff --git a/docs/decisions/TIB-2026-06-30-blue-liquidation-bot.md b/docs/decisions/TIB-2026-06-30-blue-liquidation-bot.md index bed48733..05369459 100644 --- a/docs/decisions/TIB-2026-06-30-blue-liquidation-bot.md +++ b/docs/decisions/TIB-2026-06-30-blue-liquidation-bot.md @@ -651,3 +651,62 @@ TIB conventions: how the TIB is applied without changing the decision itself. - TIB identifiers use CalVer (YYYY-MM-DD) based on the date the TIB was first drafted. --> + +## Addenda + +### 2026-08-05 — `tick.end` counters reconciled; `submitted` now means broadcast + +Three production defects, found by a BetterStack review of `bot.liquidation.midnight` (source 2607569) and present verbatim in this bot too, changed the observability contract this TIB documents. This addendum records the +new shape; the body above is left as written. + +**The defects.** (1) Two `continue`s — a position already in flight, and a position sizing refused — +incremented no counter and logged nothing, so from 2026-07-31 15:37 the bot reported +`liquidatable: 12, planned: 0` for ~215,000 consecutive ticks with every skip counter at zero and no +way to learn why. (2) `submitted` counted `submit` _calls_, not broadcasts: on 2026-07-30 it summed +2,425 while `tx.sent` was 0 and `tx.submit_failed` was 2,666. (3) `backoff.clear` ran unconditionally +after `submit`, wiping the accumulated attempt count, so a position that simulated ok and then failed +to send was re-quoted, re-simulated and re-sent every block indefinitely. + +**The counter set** (identical in both liquidators, ordered as the pipeline runs): + +```text +tick.end { pairs, liquidatable, inflightSkipped, planSkipped, planned, + cooledDown, backoffSkipped, noSwapPath, quoteFailed, + ok, reverted, submitted, notSent, complete } +``` + +This reconciles drift in both directions: `backoffSkipped`/`cooledDown` shipped without ever being +documented, and `badRoute` was documented here but never implemented (route-quality rejection surfaces +as the `quote.route_quality_failed` event, not a counter). + +For a tick with `complete: true` the set is exhaustive by construction, and the identities are +asserted in every tick test — so a stage added without a counter breaks a sum instead of silently +dropping a position: + +```text +pairs >= liquidatable +liquidatable === inflightSkipped + planSkipped + planned +planned === cooledDown + backoffSkipped + noSwapPath + quoteFailed + ok + reverted +ok === submitted + notSent +``` + +`complete: false` marks a tick aborted part-way (a hashless send after a nonce was claimed still +throws by design); its counters are partial and the last identity is short by one. Previously such a +tick emitted no `tick.end` at all. + +**Semantics that changed.** `submitted` counts only real broadcasts. `notSent` counts a `submit` that +returned without sending, and the queue now reports which of its four non-sending exits it took +(`tx.submit_failed`, `tx.send_aborted`, `nonce.sync_failed`, `queue.nonce_hole`). Only the +per-position one (`tx.submit_failed`) backs the position off; the other three are queue-wide refusals +that reject every send that tick, so attributing them to one position would suppress healthy ones. +Only a broadcast clears a backoff. + +**New event.** `plan.skipped` explains a liquidatable-but-unsizable position, carrying the sizing +inputs and the derived numbers (`lif`, `repaidAssetsFull`, `seizeForFullDebt`, `seizedAssets`) so the +decision replays from one line. `info` for the ordinary dust reasons; `warn` for reasons that should +be unreachable, which makes it a live assertion that the eligibility gate and the sizing module have +not diverged. It is sampled at most once per `PLAN_SKIP_SAMPLE_EVERY_BLOCKS` (the counter stays exact +every tick), because one line per position per block would have been ~21k lines/hour/bot. + +**Any BetterStack chart or alert built on `submitted` changes meaning** and needs review; the new +fields also need the dashboard metrics-collection step before they are queryable. diff --git a/packages/bot-kit/src/balance.ts b/packages/bot-kit/src/balance.ts index 87f5d2bf..467cf340 100644 --- a/packages/bot-kit/src/balance.ts +++ b/packages/bot-kit/src/balance.ts @@ -5,6 +5,8 @@ import { formatEther } from 'viem' import type { Logger } from './logger' +import { createBlockSampler } from './runner/block-cadence' + /** * Default block cadence for the signer-balance metric. On ~2s Base blocks this ships roughly every * 60s — matching the daemon-era wall-clock cadence — so operators see EOA gas drain at a steady rate. @@ -23,12 +25,11 @@ export function createBalanceMonitor(deps: { logger: Logger everyBlocks?: bigint }): { maybeLog: (blockNumber: bigint) => Promise } { - const everyBlocks = deps.everyBlocks ?? BALANCE_EVERY_BLOCKS - let lastAt: bigint | null = null + // Always asks, so the sampler's edge-triggering never engages — this stays a plain fixed cadence. + const sampler = createBlockSampler(deps.everyBlocks ?? BALANCE_EVERY_BLOCKS) return { async maybeLog(blockNumber) { - if (lastAt !== null && blockNumber - lastAt < everyBlocks) return - lastAt = blockNumber + if (!sampler.claim(blockNumber)) return const balance = await tryCatch(deps.read()) if (balance.error) { deps.logger.warn('signer.balance_failed', { diff --git a/packages/bot-kit/src/index.ts b/packages/bot-kit/src/index.ts index 5875a74e..aa0e2b7a 100644 --- a/packages/bot-kit/src/index.ts +++ b/packages/bot-kit/src/index.ts @@ -8,6 +8,7 @@ export * from './queue/backoff' export * from './queue/cooldown' export * from './queue/fee-policy' export * from './queue/pending-queue' +export * from './runner/block-cadence' export * from './runner/runner' export * from './runner/watcher' export * from './shipping-config' diff --git a/packages/bot-kit/src/queue/backoff.ts b/packages/bot-kit/src/queue/backoff.ts index 78918e7c..abd216b8 100644 --- a/packages/bot-kit/src/queue/backoff.ts +++ b/packages/bot-kit/src/queue/backoff.ts @@ -10,7 +10,11 @@ export type Backoff = { shouldSkip: (label: string, block: bigint) => boolean /** Record a failure at `block`; the next attempt is allowed after an exponentially growing delay. */ record: (label: string, block: bigint) => void - /** Clear the position's backoff (call on a successful submit). */ + /** + * Clear the position's backoff. Call ONLY on a real broadcast — a `submit` that returned without + * sending must `record` instead. Clearing deletes the accumulated attempt count, so clearing on a + * non-broadcast resets the delay to `baseBlocks` and the exponential growth never accrues. + */ clear: (label: string) => void /** Number of positions currently tracked. */ readonly size: number diff --git a/packages/bot-kit/src/queue/cooldown.ts b/packages/bot-kit/src/queue/cooldown.ts index 5283a274..2690de6d 100644 --- a/packages/bot-kit/src/queue/cooldown.ts +++ b/packages/bot-kit/src/queue/cooldown.ts @@ -1,7 +1,8 @@ /** * Opt-in per-position liquidation cooldown, keyed by the `${id}:${borrower}` label. A COMPLEMENTARY * rate-limit defense to {@link Backoff}, not a replacement: after a liquidation attempt fails to - * produce a submittable transaction (no swap route, quote failure, or sim revert), the caller + * produce a broadcast transaction (no swap route, quote failure, sim revert, or a per-position send + * failure), the caller * {@link CooldownStore.mark}s the position and {@link CooldownStore.shouldSkip} suppresses re-quoting * the rate-limited venue APIs for a fixed `cooldownMs` window. Unlike {@link Backoff}'s block-based * exponential growth, this is a flat wall-clock window and is DISABLED by default — an operator opts @@ -21,7 +22,7 @@ export type CooldownStore = { /** True if `label` was marked within the cooldown window and should be skipped this tick. */ shouldSkip: (label: string) => boolean - /** Record that `label` just failed to produce a submittable tx (starts/refreshes its cooldown). */ + /** Record that `label` just failed to produce a broadcast tx (starts/refreshes its cooldown). */ mark: (label: string) => void } diff --git a/packages/bot-kit/src/queue/pending-queue.ts b/packages/bot-kit/src/queue/pending-queue.ts index 01a5b5da..a3c57672 100644 --- a/packages/bot-kit/src/queue/pending-queue.ts +++ b/packages/bot-kit/src/queue/pending-queue.ts @@ -51,6 +51,34 @@ type Pending = { attempt: number } +/** + * What one {@link PendingQueue.submit} call did. `sent` carries the nonce + hash of a real broadcast; + * `failed` means NO transaction left this process. Callers must treat only `sent` as a submission — + * it is the sole outcome that may clear a per-position backoff. + * + * Each `reason` names the exit the queue took, and pairs with the warn line it already logged: + * + * | `reason` | logged event | scope | + * | ------------------- | ------------------- | -------------------------------------------- | + * | `send_aborted` | `tx.send_aborted` | queue-wide latch (clears next `onBlock`) | + * | `nonce_sync_failed` | `nonce.sync_failed` | queue-wide (cursor unusable this tick) | + * | `nonce_hole` | `queue.nonce_hole` | queue-wide latch (clears when chain consumes) | + * | `submit_failed` | `tx.submit_failed` | THIS position's send failed | + * + * The scope column is load-bearing: the three queue-wide reasons refuse *every* send this tick, so a + * caller must not attribute them to the position it happened to be holding. Only `submit_failed` is + * per-position and therefore the only reason that should back a position off. + * + * A first send that fails hashless AFTER claiming a nonce is NOT a `failed` outcome — it throws + * `TxSendError` so the caller aborts the tick rather than racing the signer's cursor rollback. + */ +export type SubmitOutcome = + | { kind: 'sent'; nonce: number; txHash: Hex } + | { + kind: 'failed' + reason: 'send_aborted' | 'nonce_sync_failed' | 'nonce_hole' | 'submit_failed' + } + export type PendingQueue = { submit(args: { request: TxRequest @@ -58,7 +86,7 @@ export type PendingQueue = { maxFeePerGas: bigint maxPriorityFeePerGas: bigint blockNumber: bigint - }): Promise + }): Promise onBlock(blockNumber: bigint): Promise readonly size: number snapshot(): { nonce: number; txHash: Hex; attempt: number }[] @@ -189,12 +217,12 @@ export function createPendingQueue({ maxFeePerGas: bigint maxPriorityFeePerGas: bigint blockNumber: bigint - }): Promise { + }): Promise { // Latched by a prior hashless send: skip until the next `onBlock` clears it. The signer has // rolled its cursor back, so broadcasting again now would race that rollback. if (sendAborted) { logger.warn('tx.send_aborted', { label: args.label }) - return + return { kind: 'failed', reason: 'send_aborted' } } // Nothing in flight → reconcile the cursor with chain before claiming a nonce. A failed sync // would leave a stale (possibly runaway) cursor, so skip the send this tick rather than risk a @@ -206,7 +234,7 @@ export function createPendingQueue({ const synced = await tryCatch(syncNonce()) if (synced.error) { logger.warn('nonce.sync_failed', { label: args.label, reason: revertReason(synced.error) }) - return + return { kind: 'failed', reason: 'nonce_sync_failed' } } if (nonceHoleLow !== null) clearNonceHole('sync') } @@ -216,7 +244,7 @@ export function createPendingQueue({ // sweep clears it once the chain consumes past the hole. if (nonceHoleLow !== null) { logger.warn('queue.nonce_hole', { label: args.label, nonce: nonceHoleLow }) - return + return { kind: 'failed', reason: 'nonce_hole' } } const sent = await tryCatch( send({ @@ -240,7 +268,7 @@ export function createPendingQueue({ sendAborted = true throw sent.error } - return + return { kind: 'failed', reason: 'submit_failed' } } const { nonce, txHash } = sent.data pending.set(nonce, { @@ -260,6 +288,7 @@ export function createPendingQueue({ maxFee: args.maxFeePerGas, priority: args.maxPriorityFeePerGas }) + return { kind: 'sent', nonce, txHash } } async function replaceStuck(entry: Pending, blockNumber: bigint, baseFee: bigint): Promise { diff --git a/packages/bot-kit/src/runner/block-cadence.ts b/packages/bot-kit/src/runner/block-cadence.ts new file mode 100644 index 00000000..ec1a2b3d --- /dev/null +++ b/packages/bot-kit/src/runner/block-cadence.ts @@ -0,0 +1,36 @@ +/** + * Block-height cadence gate: lets a caller act at most once per `everyBlocks`. Used to bound the log + * volume of a condition that recurs every tick — a per-position line emitted on every block would + * dominate the source (12 positions × ~1780 ticks/hour ≈ 21k lines/hour/bot) while repeating one + * fact. O(1) state regardless of how many distinct positions pass through it, so unlike a per-label + * map there is nothing to evict and nothing to leak. + */ +export type BlockSampler = { + /** + * True when `block` is at least `everyBlocks` past the last granted claim (the first call always + * grants). STAMPS the cadence on true, hence `claim` and not a predicate name. + * + * A caller that asks ONLY when it has something to say therefore gets edge-triggering for free: a + * quiet stretch never consumes the window, so the first occurrence after any gap is always + * reported, and a sustained condition settles to one report per `everyBlocks`. + */ + claim: (block: bigint) => boolean +} + +/** + * Builds a {@link BlockSampler} that grants a claim at most once per `everyBlocks` blocks. + * + * @param everyBlocks - Minimum block distance between granted claims. `0n` grants every call. + * @returns A sampler holding one block height; not shared or reset — construct one per call site and + * hold it for the process lifetime. + */ +export const createBlockSampler = (everyBlocks: bigint): BlockSampler => { + let lastAt: bigint | null = null + return { + claim(block) { + if (lastAt !== null && block - lastAt < everyBlocks) return false + lastAt = block + return true + } + } +} diff --git a/packages/bot-kit/test/queue/pending-queue.test.ts b/packages/bot-kit/test/queue/pending-queue.test.ts index d965a58f..be39d8b3 100644 --- a/packages/bot-kit/test/queue/pending-queue.test.ts +++ b/packages/bot-kit/test/queue/pending-queue.test.ts @@ -112,10 +112,12 @@ function submitOne(queue: PendingQueue, blockNumber = 0n) { describe('createPendingQueue', () => { it('records a submitted tx with the signer-assigned nonce', async () => { const { queue, sends } = setup() - await submitOne(queue) + const outcome = await submitOne(queue) expect(queue.size).toBe(1) expect(sends[0]?.nonce).toBeUndefined() // first send leaves nonce assignment to the signer expect(queue.snapshot()[0]).toEqual({ nonce: 7, txHash: hashOf(1), attempt: 0 }) + // The caller's broadcast signal — only this outcome may clear a per-position backoff. + expect(outcome).toEqual({ kind: 'sent', nonce: 7, txHash: hashOf(1) }) }) it('removes a tx once its receipt confirms', async () => { @@ -168,9 +170,11 @@ describe('createPendingQueue', () => { throw new Error('rpc down') } const { queue } = setup({ send, logger }) - await submitOne(queue) // must not throw + const outcome = await submitOne(queue) // must not throw expect(queue.size).toBe(0) expect(events.find(e => e.event === 'tx.submit_failed')?.level).toBe('warn') + // Per-position: this is the one reason a caller may attribute to the position it was holding. + expect(outcome).toEqual({ kind: 'failed', reason: 'submit_failed' }) }) it('rethrows a first-send failure after a nonce was claimed but no hash was returned', async () => { @@ -179,6 +183,8 @@ describe('createPendingQueue', () => { throw new TxSendError(new Error('rpc timeout after broadcast'), 7) } const { queue } = setup({ send, logger }) + // Still a THROW, deliberately not a `failed` outcome: the caller must abort the tick rather than + // race the signer's cursor rollback. await expect(submitOne(queue)).rejects.toThrow(/rpc timeout after broadcast/) expect(queue.size).toBe(0) expect(events.find(e => e.event === 'tx.submit_failed')?.fields?.nonce).toBe(7) @@ -301,8 +307,9 @@ describe('createPendingQueue', () => { }, logger }) - await submitOne(ctx.queue) // must not throw + const outcome = await submitOne(ctx.queue) // must not throw expect(ctx.queue.size).toBe(0) // nothing broadcast on a stale cursor + expect(outcome).toEqual({ kind: 'failed', reason: 'nonce_sync_failed' }) expect(ctx.sends).toHaveLength(0) expect(events.find(e => e.event === 'nonce.sync_failed')?.level).toBe('warn') }) @@ -485,8 +492,10 @@ describe('send-aborted latch', () => { } const { queue } = setup({ send, logger }) await expect(submitOne(queue, 0n)).rejects.toThrow() - await submitOne(queue, 0n) // latched → skipped + const outcome = await submitOne(queue, 0n) // latched → skipped expect(events.find(e => e.event === 'tx.send_aborted')?.level).toBe('warn') + // Queue-WIDE: refuses every send this tick, so a caller must not back the position off for it. + expect(outcome).toEqual({ kind: 'failed', reason: 'send_aborted' }) }) }) @@ -554,8 +563,9 @@ describe('nonce-hole latch', () => { const sendsAfterDrop = ctx.sends.length // A NEW first-send is refused while the hole is latched (nonce 8 still pending → queue not empty, // so the empty-queue sync can't clear it). - await ctx.submit('c', 6n) + const refused = await ctx.submit('c', 6n) expect(ctx.sends.length).toBe(sendsAfterDrop) // no new broadcast + expect(refused).toEqual({ kind: 'failed', reason: 'nonce_hole' }) expect(ctx.queue.size).toBe(1) expect(ctx.events.some(e => e.event === 'queue.nonce_hole' && e.fields?.label === 'c')).toBe( true diff --git a/packages/bot-kit/test/runner/block-cadence.test.ts b/packages/bot-kit/test/runner/block-cadence.test.ts new file mode 100644 index 00000000..d1ed41a4 --- /dev/null +++ b/packages/bot-kit/test/runner/block-cadence.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' + +import { createBlockSampler } from '../../src/runner/block-cadence' + +describe('createBlockSampler', () => { + it('grants the first claim', () => { + expect(createBlockSampler(150n).claim(100n)).toBe(true) + }) + + it('refuses a claim inside the window', () => { + const sampler = createBlockSampler(150n) + sampler.claim(100n) + expect(sampler.claim(101n)).toBe(false) + expect(sampler.claim(249n)).toBe(false) + }) + + it('grants again at exactly everyBlocks past the last grant', () => { + // The boundary `createBalanceMonitor` depends on: a delta of exactly `everyBlocks` logs. + const sampler = createBlockSampler(150n) + sampler.claim(100n) + expect(sampler.claim(250n)).toBe(true) + }) + + it('measures the window from the last GRANT, not the last call', () => { + const sampler = createBlockSampler(10n) + expect(sampler.claim(100n)).toBe(true) + expect(sampler.claim(105n)).toBe(false) // refused, so it must not re-stamp + expect(sampler.claim(110n)).toBe(true) + }) + + it('does not consume the window when it is never asked', () => { + // The edge-trigger guarantee: a caller that asks only when it has something to say gets an + // immediate grant after any quiet stretch, with no per-caller state. + const sampler = createBlockSampler(150n) + expect(sampler.claim(100n)).toBe(true) + // ...400 quiet blocks during which the caller never asks... + expect(sampler.claim(500n)).toBe(true) + }) + + it('grants every claim at a zero cadence', () => { + const sampler = createBlockSampler(0n) + expect(sampler.claim(1n)).toBe(true) + expect(sampler.claim(1n)).toBe(true) + }) +}) From f0293a77a92f5fc27697ac03def0eaad18de3dae Mon Sep 17 00:00:00 2001 From: Hayden Shively Date: Wed, 5 Aug 2026 13:00:20 -0400 Subject: [PATCH 2/4] refactor(bots): trim duplicated rationale and hoist shared sizing math Self-review pass for simplicity and comment density. No behavior change: the same 1,475 tests pass, and every bug-reintroducing mutant is still killed. - The sampler's edge-triggering rationale was explained in four places (the primitive, both tick deps, both constants). Now stated once on `BlockSampler.claim`; the call sites just say what they bound. - `planWithReason` computed `effectiveDebt` and `wholeSlotRepaid` in both mode branches. Hoisted into the dispatcher and passed via one shared `ModeStage` type, which also removes the duplicated inline param types and shrinks `postMaturityOutcome` to two lines. - Trimmed the `SubmitOutcome`, `TickCounters` and `PlanSkipReason` docs to the load-bearing parts, dropping prose that restated an adjacent table or field comment. Same for the README paragraph under the queue-exit table. - `invalid` counting reads as one filter instead of a mutable loop. - Dropped editorialising test comments; kept the ones that explain a non-obvious fixture. Closes a coverage gap the mutation run surfaced: nothing pinned that a post-maturity cap uses the post-writeoff debt (`debt - badDebt`) rather than the gross debt, so mutating that sign survived. Now asserted by equivalence, so the test does not re-derive the arithmetic it checks. Co-Authored-By: Claude Opus 5 (1M context) --- bots/blue-liquidation/src/constants.ts | 8 +- bots/blue-liquidation/src/runner/tick.ts | 57 ++++---- .../blue-liquidation/test/runner/tick.test.ts | 16 +-- bots/midnight-liquidation/README.md | 5 +- bots/midnight-liquidation/src/constants.ts | 8 +- bots/midnight-liquidation/src/runner/tick.ts | 57 ++++---- bots/midnight-liquidation/src/sizing/plan.ts | 124 ++++++++---------- .../test/runner/tick.test.ts | 34 ++--- .../test/sizing/plan.test.ts | 17 +++ packages/bot-kit/src/queue/pending-queue.ts | 29 ++-- packages/bot-kit/src/runner/block-cadence.ts | 22 ++-- 11 files changed, 171 insertions(+), 206 deletions(-) diff --git a/bots/blue-liquidation/src/constants.ts b/bots/blue-liquidation/src/constants.ts index b45ef528..fb86e5e7 100644 --- a/bots/blue-liquidation/src/constants.ts +++ b/bots/blue-liquidation/src/constants.ts @@ -41,10 +41,8 @@ export const VIRTUAL_ASSETS = 1n export const SETTLED_COOLDOWN_BLOCKS = 20n /** - * Block cadence bounding the per-position `plan.skipped` diagnostic. A liquidatable position that - * cannot be sized recurs every tick, so logging one line per position per block would dominate the - * log source while repeating one fact (~12 positions × ~1780 ticks/hour on Base). At ~2s blocks this - * is roughly every 5 minutes. The sampler is asked only when a skip actually happens, so a quiet - * stretch never consumes the window and the first skip after any gap is always reported. + * Block cadence bounding the per-position `plan.skipped` diagnostic (~5 min at ~2s Base blocks). An + * unsizable position recurs every tick, so one line per position per block would be ~21k + * lines/hour/bot — all repeating one fact. The `planSkipped` counter stays exact regardless. */ export const PLAN_SKIP_SAMPLE_EVERY_BLOCKS = 150n diff --git a/bots/blue-liquidation/src/runner/tick.ts b/bots/blue-liquidation/src/runner/tick.ts index d078226f..ce5f0b3a 100644 --- a/bots/blue-liquidation/src/runner/tick.ts +++ b/bots/blue-liquidation/src/runner/tick.ts @@ -21,17 +21,16 @@ import { planWithReason } from '../sizing/plan' import { isLiquidatable, planInputFromLens } from './eligibility' /** - * Per-tick outcome tally, emitted as `tick.end`. Ordered as the pipeline runs, and exhaustive by - * construction: for a tick that finished (`complete: true` on the event) these identities hold, so a - * future stage added without a counter shows up as a broken sum rather than a silent drop. + * Per-tick outcome tally, emitted as `tick.end`, ordered as the pipeline runs. For a tick that + * finished (`complete: true`) these identities hold, so a stage added without a counter breaks a sum + * instead of silently dropping a position: * * pairs >= liquidatable * liquidatable === inflightSkipped + planSkipped + planned * planned === cooledDown + backoffSkipped + noSwapPath + quoteFailed + ok + reverted * ok === submitted + notSent * - * They do NOT hold when `complete` is `false`: an aborting `submit` throws after `ok` was counted but - * before `submitted`/`notSent`, so the last identity is short by one. + * On `complete: false` the last is short by one: an aborting `submit` throws after `ok` was counted. */ type TickCounters = { /** Lens inputs read this tick — the discovery universe. */ @@ -39,9 +38,8 @@ type TickCounters = { /** Positions the chain says are liquidatable at this block. A market gauge, not a bot decision. */ liquidatable: number /** - * Skipped because a tx for this label is already in flight. Note the queue's backpressure set also - * holds labels whose tx SETTLED within `settledCooldownBlocks`, so this can be non-zero while the - * queue itself is empty. + * Skipped because a tx for this label is in flight. The queue's backpressure set also holds labels + * that SETTLED within `settledCooldownBlocks`, so this can be non-zero while the queue is empty. */ inflightSkipped: number /** Skipped because sizing produced no plan — see the `plan.skipped` event for the reason. */ @@ -60,9 +58,8 @@ type TickCounters = { } /** - * `warn` marks a reason that should be impossible, so it reads as a live assertion rather than noise: - * `no_debt`/`healthy` are the exact negation of `isLiquidatable`, and a non-reverting zero oracle - * price is a market anomaly. The dust reasons are ordinary and stay `info`. + * `warn` marks a reason that should not happen — `no_debt`/`healthy` negate `isLiquidatable`, and a + * non-reverting zero oracle price is a market anomaly — so the line reads as a live assertion. */ const LEVEL_BY_REASON: Record = { no_debt: 'warn', @@ -130,9 +127,7 @@ export async function runTick(deps: { cooldown: CooldownStore /** * Bounds how often the per-position `plan.skipped` diagnostic is emitted. Asked at most once per - * tick and only when there is something to explain, so a quiet stretch never consumes the window: - * the first skip after any gap is always reported, and a persistent one settles to a single burst - * per cadence. All lines in a burst share one `chainHead`, so they read as one coherent snapshot. + * tick, so every line in a burst shares one `chainHead` and reads as a single snapshot. */ planSkipSampler: BlockSampler /** Labels (`${id}:${borrower}`) already in flight — skipped to avoid re-submitting each block. */ @@ -166,11 +161,10 @@ export async function runTick(deps: { // 2. Read the lens fresh for the whole batch in one deployless eth_call. const lensOut = await readLens(pairs) - // `returned` is always `pairs` (the batch lens maps every input row); `invalid` is the informative - // one — it counts rows the lens zeroed (unknown market, reverting oracle), which would otherwise be - // indistinguishable from a healthy position in `pairs - liquidatable`. - let invalid = 0 - for (const out of lensOut.values()) if (!out.valid) invalid += 1 + // `returned` always equals `pairs` (the batch lens maps every input row); `invalid` is the + // informative one — a row the lens zeroed (unknown market, reverting oracle) would otherwise be + // indistinguishable from a healthy position inside `pairs - liquidatable`. + const invalid = [...lensOut.values()].filter(out => !out.valid).length logger.info('lens.read', { pairs: pairs.length, returned: lensOut.size, invalid }) const counters: TickCounters = { @@ -192,8 +186,7 @@ export async function runTick(deps: { // 3. Compose liquidatability off-chain → plan → quote → simulate → submit. `inflight` is captured // once; discovery yields distinct (market, borrower) pairs, so no label repeats within a tick. const inflight = inflightLabels() - // Resolved lazily on the first skip of the tick and reused for the rest, so one tick emits either a - // full snapshot of the blocker set or nothing — never a staggered subset. + // Resolved on the first skip and reused, so a tick emits the whole blocker set or none of it. let explainSkips: boolean | null = null const processPairs = async () => { @@ -204,9 +197,8 @@ export async function runTick(deps: { if (!out || !isLiquidatable(out)) continue counters.liquidatable += 1 - // Backpressure: a tx for this position is already pending — don't re-plan/simulate/submit it - // every block while it confirms. No log: the queue already narrates this label's whole - // lifecycle (`tx.sent` → `tx.confirmed`/`tx.reverted`/`tx.dropped`) under the same key. + // Backpressure: don't re-plan/simulate/submit while a tx confirms. No log — the queue already + // narrates this label end to end (`tx.sent` → `tx.confirmed`/`tx.reverted`/`tx.dropped`). if (inflight.has(label)) { counters.inflightSkipped += 1 continue @@ -218,9 +210,8 @@ export async function runTick(deps: { counters.planSkipped += 1 explainSkips ??= planSkipSampler.claim(chainHead) if (explainSkips) { - // Spread the inputs AND the derived trace so the line is a closed causal chain — an - // operator can replay sizing from it without re-deriving anything. `marketId`/`borrower` - // last so a future `PlanInput` field can never shadow them. + // Inputs AND derived trace, so sizing replays from this one line. `marketId`/`borrower` + // last, so a future `PlanInput` field can never shadow them. logger[LEVEL_BY_REASON[planOutcome.reason]]('plan.skipped', { ...planInput, ...planOutcome.trace, @@ -311,10 +302,9 @@ export async function runTick(deps: { continue } counters.notSent += 1 - // Only a per-position send failure earns a backoff. `send_aborted`, `nonce_hole` and - // `nonce_sync_failed` are queue-WIDE refusals that reject every send this tick, so backing off - // here would suppress positions that did nothing wrong — for 2, 4, 8… blocks after the latch - // itself has cleared. The queue has already logged which exit it took. + // Only a per-position send failure earns a backoff. The other reasons are queue-WIDE refusals + // that reject every send this tick, so backing off here would suppress positions that did + // nothing wrong, for 2, 4, 8… blocks after the latch itself cleared. if (sendOutcome.reason === 'submit_failed') { backoff.record(label, chainHead) cooldown.mark(label) @@ -323,9 +313,8 @@ export async function runTick(deps: { } // Emit counters even when a position aborts the tick (a hashless send after the nonce was claimed - // throws by design). Without `complete` an aborted tick's partial counters are indistinguishable - // from a genuinely idle one. `ensureError` preserves the instance, so rethrowing keeps `TxSendError` - // intact for the runner's `tick.error` decode. + // throws by design) — without `complete`, partial counters look exactly like an idle tick. + // `tryCatch` preserves the error instance, so the rethrow keeps `TxSendError` intact downstream. const { error } = await tryCatch(processPairs()) logger.info('tick.end', { ...counters, complete: !error }) if (error) throw error diff --git a/bots/blue-liquidation/test/runner/tick.test.ts b/bots/blue-liquidation/test/runner/tick.test.ts index 217110be..b889106a 100644 --- a/bots/blue-liquidation/test/runner/tick.test.ts +++ b/bots/blue-liquidation/test/runner/tick.test.ts @@ -43,17 +43,16 @@ const PARAMS: MarketParams = { irm: IRM, lltv: 86n * 10n ** 16n } -const LABEL = lensKey(marketId(PARAMS), BORROWER) const MARKET_ID = marketId(PARAMS) +const LABEL = lensKey(MARKET_ID, BORROWER) const TX_HASH: Hex = `0x${'b'.repeat(64)}` const SENT: SubmitOutcome = { kind: 'sent', nonce: 7, txHash: TX_HASH } type TickCounters = Awaited> /** - * Every counter identity the tick promises for a completed tick. Asserted for EVERY case built by - * `runWith`, so a stage added without a counter breaks the sums instead of silently dropping a - * position — the exact class of bug these counters exist to catch. + * Asserted for EVERY case built by `runWith`, so a stage added without a counter breaks a sum rather + * than silently dropping a position — the exact class of bug these counters exist to catch. */ function expectCountersConsistent(c: TickCounters) { expect(c.pairs).toBeGreaterThanOrEqual(c.liquidatable) @@ -337,7 +336,7 @@ describe('runTick', () => { describe('unplannable positions (plan.skipped)', () => { it('logs plan.skipped with a closed causal chain: sizing inputs AND the derived trace', async () => { - // Dust: a tiny debt against an expensive slot floors the full-debt seize to zero collateral. + // Dust: a tiny debt against an expensive slot floors the full-debt seize to zero. const { counters, events } = await runWith({ out: lensOut({ borrowShares: 1n, collateralPrice: ORACLE_PRICE_SCALE * 10n ** 6n }) }) @@ -425,10 +424,9 @@ describe('runTick', () => { }) it('lets the backoff delay GROW across repeated send failures', async () => { - // The precise mechanism of bug 3: `clear` deletes the attempt count, so clearing on a - // non-broadcast resets the delay to `baseBlocks` forever and the exponential never accrues. - // Seeded at block 1 (attempts=1) and failing again at 100 must reach attempts=2 → a 4-block - // wait (until 104), not the 2-block wait a reset would give. + // `clear` deletes the attempt count, so clearing on a non-broadcast pins the delay at + // `baseBlocks` forever. Seeded at block 1 (attempts=1) then failing at 100 must reach + // attempts=2 → a 4-block wait (until 104), not the 2-block wait a reset would give. const { backoff } = await runWith({ seedBackoffAt: 1n, submitOutcome: { kind: 'failed', reason: 'submit_failed' } diff --git a/bots/midnight-liquidation/README.md b/bots/midnight-liquidation/README.md index 8b2dc887..11b131ab 100644 --- a/bots/midnight-liquidation/README.md +++ b/bots/midnight-liquidation/README.md @@ -451,9 +451,8 @@ real broadcasts. The queue has five exits: | nonce cursor unusable | `nonce.sync_failed` | `notSent` only — a queue-wide refusal | | nonce hole below the cursor | `queue.nonce_hole` | `notSent` only — a queue-wide refusal | -The three queue-wide refusals reject _every_ send that tick, so they deliberately do NOT back off the -position that happened to be in hand — otherwise a single latch would suppress every healthy position -for several blocks after the latch itself cleared. +A queue-wide refusal rejects _every_ send that tick, so backing off the position that happened to be +in hand would suppress healthy positions for several blocks after the latch itself cleared. If the initial raw broadcast fails after a nonce is claimed but before a hash is returned, the signer rolls the cursor back and the queue aborts that tick instead of counting a hashless transaction as diff --git a/bots/midnight-liquidation/src/constants.ts b/bots/midnight-liquidation/src/constants.ts index 3bd1f46d..d007b31d 100644 --- a/bots/midnight-liquidation/src/constants.ts +++ b/bots/midnight-liquidation/src/constants.ts @@ -52,10 +52,8 @@ export const LISTED_MARKETS_MAX_AGE_MS = 10 * 60_000 export const BPS = 10_000n /** - * Block cadence bounding the per-position `plan.skipped` diagnostic. A liquidatable position that - * cannot be sized recurs every tick, so logging one line per position per block would dominate the - * log source while repeating one fact (~12 positions × ~1780 ticks/hour on Base). At ~2s blocks this - * is roughly every 5 minutes. The sampler is asked only when a skip actually happens, so a quiet - * stretch never consumes the window and the first skip after any gap is always reported. + * Block cadence bounding the per-position `plan.skipped` diagnostic (~5 min at ~2s Base blocks). An + * unsizable position recurs every tick, so one line per position per block would be ~21k + * lines/hour/bot — all repeating one fact. The `planSkipped` counter stays exact regardless. */ export const PLAN_SKIP_SAMPLE_EVERY_BLOCKS = 150n diff --git a/bots/midnight-liquidation/src/runner/tick.ts b/bots/midnight-liquidation/src/runner/tick.ts index 886269a1..832650b5 100644 --- a/bots/midnight-liquidation/src/runner/tick.ts +++ b/bots/midnight-liquidation/src/runner/tick.ts @@ -20,17 +20,16 @@ import { isBadDebtRealization, planWithReason } from '../sizing/plan' import { isLiquidatable, planInputFromLens } from './eligibility' /** - * Per-tick outcome tally, emitted as `tick.end`. Ordered as the pipeline runs, and exhaustive by - * construction: for a tick that finished (`complete: true` on the event) these identities hold, so a - * future stage added without a counter shows up as a broken sum rather than a silent drop. + * Per-tick outcome tally, emitted as `tick.end`, ordered as the pipeline runs. For a tick that + * finished (`complete: true`) these identities hold, so a stage added without a counter breaks a sum + * instead of silently dropping a position: * * pairs >= liquidatable * liquidatable === inflightSkipped + planSkipped + planned * planned === cooledDown + backoffSkipped + noSwapPath + quoteFailed + ok + reverted * ok === submitted + notSent * - * They do NOT hold when `complete` is `false`: an aborting `submit` throws after `ok` was counted but - * before `submitted`/`notSent`, so the last identity is short by one. + * On `complete: false` the last is short by one: an aborting `submit` throws after `ok` was counted. */ type TickCounters = { /** Lens inputs read this tick — the post-whitelist discovery universe. */ @@ -38,9 +37,8 @@ type TickCounters = { /** Positions the chain says are liquidatable at this block. A market gauge, not a bot decision. */ liquidatable: number /** - * Skipped because a tx for this label is already in flight. Note the queue's backpressure set also - * holds labels whose tx SETTLED within `settledCooldownBlocks`, so this can be non-zero while the - * queue itself is empty. + * Skipped because a tx for this label is in flight. The queue's backpressure set also holds labels + * that SETTLED within `settledCooldownBlocks`, so this can be non-zero while the queue is empty. */ inflightSkipped: number /** Skipped because sizing produced no plan — see the `plan.skipped` event for the reason. */ @@ -59,9 +57,8 @@ type TickCounters = { } /** - * `warn` marks a reason that should be impossible, so it reads as a live assertion rather than noise: - * the first three are the exact negation of `isLiquidatable`, and `cap_not_positive` means the RCF - * numerator went negative. The dust reasons are ordinary and stay `info`. + * `warn` marks a reason that should be unreachable — the first three negate `isLiquidatable`, and + * `cap_not_positive` means the RCF numerator went negative — so the line reads as a live assertion. */ const LEVEL_BY_REASON: Record = { no_debt: 'warn', @@ -134,9 +131,7 @@ export async function runTick(deps: { cooldown: CooldownStore /** * Bounds how often the per-position `plan.skipped` diagnostic is emitted. Asked at most once per - * tick and only when there is something to explain, so a quiet stretch never consumes the window: - * the first skip after any gap is always reported, and a persistent one settles to a single burst - * per cadence. All lines in a burst share one `chainHead`, so they read as one coherent snapshot. + * tick, so every line in a burst shares one `chainHead` and reads as a single snapshot. */ planSkipSampler: BlockSampler /** Labels (`${id}:${borrower}`) already in flight — skipped to avoid re-submitting each block. */ @@ -172,11 +167,10 @@ export async function runTick(deps: { // 2. Read the lens fresh for the whole batch in one deployless eth_call. const lensOut = await readLens(pairs) - // `returned` is always `pairs` (the batch lens maps every input row); `invalid` is the informative - // one — it counts rows the lens zeroed (unknown market, reverting oracle), which would otherwise be - // indistinguishable from a healthy position in `pairs - liquidatable`. - let invalid = 0 - for (const out of lensOut.values()) if (!out.valid) invalid += 1 + // `returned` always equals `pairs` (the batch lens maps every input row); `invalid` is the + // informative one — a row the lens zeroed (unknown market, reverting oracle) would otherwise be + // indistinguishable from a healthy position inside `pairs - liquidatable`. + const invalid = [...lensOut.values()].filter(out => !out.valid).length logger.info('lens.read', { pairs: pairs.length, returned: lensOut.size, invalid }) const counters: TickCounters = { @@ -198,8 +192,7 @@ export async function runTick(deps: { // 3. Compose liquidatability off-chain → plan → simulate → submit. `inflight` is captured once; // discovery yields distinct (id, borrower) pairs, so no label repeats within a single tick. const inflight = inflightLabels() - // Resolved lazily on the first skip of the tick and reused for the rest, so one tick emits either a - // full snapshot of the blocker set or nothing — never a staggered subset. + // Resolved on the first skip and reused, so a tick emits the whole blocker set or none of it. let explainSkips: boolean | null = null const processPairs = async () => { @@ -209,9 +202,8 @@ export async function runTick(deps: { if (!out || !isLiquidatable(out)) continue counters.liquidatable += 1 - // Backpressure: a tx for this position is already pending — don't re-plan/simulate/submit it - // every block while it confirms. No log: the queue already narrates this label's whole - // lifecycle (`tx.sent` → `tx.confirmed`/`tx.reverted`/`tx.dropped`) under the same key. + // Backpressure: don't re-plan/simulate/submit while a tx confirms. No log — the queue already + // narrates this label end to end (`tx.sent` → `tx.confirmed`/`tx.reverted`/`tx.dropped`). if (inflight.has(label)) { counters.inflightSkipped += 1 continue @@ -223,9 +215,8 @@ export async function runTick(deps: { counters.planSkipped += 1 explainSkips ??= planSkipSampler.claim(chainHead) if (explainSkips) { - // Spread the inputs AND the derived trace so the line is a closed causal chain — an - // operator can replay sizing from it without re-deriving anything. `marketId`/`borrower` - // last so a future `PlanInput` field can never shadow them. + // Inputs AND derived trace, so sizing replays from this one line. `marketId`/`borrower` + // last, so a future `PlanInput` field can never shadow them. logger[LEVEL_BY_REASON[outcome.reason]]('plan.skipped', { ...planInput, ...outcome.trace, @@ -333,10 +324,9 @@ export async function runTick(deps: { continue } counters.notSent += 1 - // Only a per-position send failure earns a backoff. `send_aborted`, `nonce_hole` and - // `nonce_sync_failed` are queue-WIDE refusals that reject every send this tick, so backing off - // here would suppress positions that did nothing wrong — for 2, 4, 8… blocks after the latch - // itself has cleared. The queue has already logged which exit it took. + // Only a per-position send failure earns a backoff. The other reasons are queue-WIDE refusals + // that reject every send this tick, so backing off here would suppress positions that did + // nothing wrong, for 2, 4, 8… blocks after the latch itself cleared. if (sendOutcome.reason === 'submit_failed') { backoff.record(label, chainHead) cooldown.mark(label) @@ -345,9 +335,8 @@ export async function runTick(deps: { } // Emit counters even when a position aborts the tick (a hashless send after the nonce was claimed - // throws by design). Without `complete` an aborted tick's partial counters are indistinguishable - // from a genuinely idle one. `ensureError` preserves the instance, so rethrowing keeps `TxSendError` - // intact for the runner's `tick.error` decode. + // throws by design) — without `complete`, partial counters look exactly like an idle tick. + // `tryCatch` preserves the error instance, so the rethrow keeps `TxSendError` intact downstream. const { error } = await tryCatch(processPairs()) logger.info('tick.end', { ...counters, complete: !error }) if (error) throw error diff --git a/bots/midnight-liquidation/src/sizing/plan.ts b/bots/midnight-liquidation/src/sizing/plan.ts index 26bf3eb4..b920adba 100644 --- a/bots/midnight-liquidation/src/sizing/plan.ts +++ b/bots/midnight-liquidation/src/sizing/plan.ts @@ -53,15 +53,13 @@ type PlanOptions = { } /** - * Why {@link planWithReason} produced no plan. Reasons discriminate SEVERITY (and hence log level); - * the numbers on {@link SizingTrace} discriminate cause. That is why there is no `margin_ate_cap` or - * `price_too_high` — those are one continuous arithmetic that `cap` vs `capEff` vs `seizedAssets` - * pin exactly. + * Why {@link planWithReason} produced no plan. Reasons discriminate SEVERITY (hence log level); the + * numbers on {@link SizingTrace} discriminate cause — so there is no `margin_ate_cap` reason, because + * `cap` vs `capEff` vs `seizedAssets` already pin that. * - * `no_debt`, `locked` and `healthy_pre_maturity` are UNREACHABLE from the tick: `isLiquidatable` - * tests the same lens fields `planWithReason` re-tests, and `!postMaturityMode && healthy` is the - * exact negation of its third clause. A caller that observes one has diverged from the sizing - * module, which is a correctness bug — hence they are logged at `warn`, as a live assertion. + * The first three are UNREACHABLE from the tick: `isLiquidatable` tests the same lens fields, and + * `!postMaturityMode && healthy` is the exact negation of its third clause. Observing one means the + * eligibility gate and this module have diverged, so callers log them at `warn`. */ export type PlanSkipReason = | 'no_debt' @@ -75,10 +73,9 @@ export type PlanSkipReason = | 'seize_rounds_to_zero' /** - * Every value on the causal chain from a lens reading to a refused seize, so an operator can replay - * the decision from one log line instead of re-deriving it. `maxRepaid`/`rcfExempt`/`rcfDisabled` - * apply to normal mode only and are `undefined` post-maturity — the logger drops `undefined`, so a - * post-maturity line self-describes its mode. + * The derived values between a lens reading and a refused seize, so an operator can replay the + * decision from one log line. The normal-mode-only fields are `undefined` post-maturity, and the + * logger drops `undefined`, so a line self-describes its mode. */ type SizingTrace = { postMaturityMode: boolean @@ -93,7 +90,7 @@ type SizingTrace = { capEff?: bigint maxRepaid?: bigint rcfExempt?: boolean - /** `true` when `lltv >= WAD` waives the RCF cap entirely (so `maxRepaid` is omitted, not huge). */ + /** `lltv >= WAD` waives the RCF cap, so `maxRepaid` is omitted rather than logged as maxUint256. */ rcfDisabled?: boolean } @@ -146,16 +143,14 @@ const capBoundOutcome = ({ const capEff = mulDivDown(cap, BPS - BigInt(marginBps), BPS) const seizedAssets = maxSeizeForCap(capEff, input.bestCollateralPrice, base.lif) const trace: SizingTrace = { ...base, cap, capEff, seizedAssets } - // Discriminate on the RAW cap, not `capEff`: a legitimate 1-wei cap with any margin > 0 floors to - // capEff 0, which is ordinary dust, not an impossible state. A non-positive RAW cap means the RCF - // numerator went negative (`debt - maxDebt < badDebt < debt`), which breaks the module's invariant - // AND would otherwise produce a NEGATIVE seize that slips past an `=== 0n` guard and reverts - // opaquely when abi-encoded as uint256. + // Guard the RAW cap, not `capEff`: a legitimate 1-wei cap with any margin floors capEff to 0, which + // is ordinary dust. A non-positive RAW cap means the RCF numerator went negative + // (`debt - maxDebt < badDebt < debt`), which also yields a NEGATIVE seize that would slip past an + // `=== 0n` check and revert opaquely once abi-encoded as uint256. if (cap <= 0n) return { kind: 'skip', reason: 'cap_not_positive', trace } - // Rounds to nothing. Never emit a `(0, 0)` plan, which `isBadDebtRealization` would misclassify as - // a bad-debt write-off against a solvent position. `<= 0n` rather than `=== 0n` is defense in - // depth: a positive cap can only floor to a non-negative seize, so the negative case is already - // caught by the guard above — this simply cannot be the branch that lets one through. + // Rounds to nothing. Never emit a `(0, 0)` plan, which `isBadDebtRealization` would misread as a + // write-off against a solvent position. (`<= 0n` is belt-and-braces: a positive cap cannot floor to + // a negative seize, so the guard above already covers that.) if (seizedAssets <= 0n) return { kind: 'skip', reason: 'seize_rounds_to_zero', trace } return { kind: 'plan', @@ -178,51 +173,46 @@ const seizeWholeSlot = (input: PlanInput, postMaturityMode: boolean): PlanOutcom } }) +/** Everything both mode branches need, derived once in {@link planWithReason}. */ +type ModeStage = { + input: PlanInput + marginBps: number + lif: bigint + /** `debt - badDebt`: the post-writeoff debt every cap is taken against. */ + effectiveDebt: bigint + /** Implied repaid units if the whole best slot were seized. */ + wholeSlotRepaid: bigint +} + /** - * Post-maturity mode: the RCF cap does not apply, but the contract still subtracts `repaidUnits` - * from the (post-writeoff) debt with no clamp, so over-repaying reverts (Panic 0x11 underflow). - * Seizing the whole slot is correct only while its implied repaid units fit within the debt — the - * underwater case. When the slot is worth more than the debt (the common case: a solvent borrower - * who simply missed maturity), seize the largest amount whose contract-derived repaid stays within - * that debt. `badDebt` is written off before the repay, so the cap is `debt - badDebt`. + * Post-maturity mode: the RCF cap does not apply, but the contract still subtracts `repaidUnits` from + * the post-writeoff debt with no clamp, so over-repaying reverts (Panic 0x11 underflow). Seizing the + * whole slot is therefore correct only while its implied repaid fits within that debt — the underwater + * case. Otherwise (a solvent borrower who simply missed maturity) fall back to the cap. */ const postMaturityOutcome = ({ input, + marginBps, lif, - marginBps -}: { - input: PlanInput - lif: bigint - marginBps: number -}): PlanOutcome => { - const effectiveDebt = input.debt - input.badDebt - const base: TraceBase = { postMaturityMode: true, lif, effectiveDebt } - const wholeSlotRepaid = impliedRepaidUnits( - input.bestCollateralAmt, - input.bestCollateralPrice, - lif - ) + effectiveDebt, + wholeSlotRepaid +}: ModeStage): PlanOutcome => { if (wholeSlotRepaid <= effectiveDebt) return seizeWholeSlot(input, true) + const base: TraceBase = { postMaturityMode: true, lif, effectiveDebt } return capBoundOutcome({ input, cap: effectiveDebt, marginBps, base }) } /** - * Normal mode: like post-maturity, the contract subtracts `repaidUnits` from the post-writeoff debt - * with no clamp, so an implied repay above it reverts (Panic 0x11). The repay is bounded by the RCF - * cap (waived when the slot is rcf-exempt) AND never exceeds that debt. Seize the whole slot only - * when its implied repaid units fit within the bound; otherwise seize the largest amount whose - * contract-derived repaid stays within that bound. + * Normal mode: same no-clamp subtraction as post-maturity, but the repay is additionally bounded by + * the RCF cap (waived when the slot is rcf-exempt), never exceeding the post-writeoff debt. */ const normalModeOutcome = ({ input, + marginBps, lif, - marginBps -}: { - input: PlanInput - lif: bigint - marginBps: number -}): PlanOutcome => { - const effectiveDebt = input.debt - input.badDebt + effectiveDebt, + wholeSlotRepaid +}: ModeStage): PlanOutcome => { const maxRepaid = maxRepaidPreMaturity({ debt: input.debt, badDebt: input.badDebt, @@ -237,21 +227,15 @@ const normalModeOutcome = ({ maxRepaid, rcfThreshold: input.rcfThreshold }) - // `lltv >= WAD` waives the cap and returns maxUint256; report that as a flag rather than logging a - // 78-digit number that reads like a real bound. - const rcfDisabled = input.bestCollateralLltv >= WAD const base: TraceBase = { postMaturityMode: false, lif, effectiveDebt, rcfExempt: exempt, - ...(rcfDisabled ? { rcfDisabled } : { maxRepaid }) + // `lltv >= WAD` waives the cap and returns maxUint256; flag that instead of logging a 78-digit + // number that reads like a real bound. + ...(input.bestCollateralLltv >= WAD ? { rcfDisabled: true } : { maxRepaid }) } - const wholeSlotRepaid = impliedRepaidUnits( - input.bestCollateralAmt, - input.bestCollateralPrice, - lif - ) const repayCap = exempt ? effectiveDebt : min(maxRepaid, effectiveDebt) if (wholeSlotRepaid <= repayCap) return seizeWholeSlot(input, false) return capBoundOutcome({ input, cap: repayCap, marginBps, base }) @@ -308,6 +292,7 @@ export const planWithReason = (input: PlanInput, options: PlanOptions = {}): Pla maxLif: input.bestCollateralMaxLif, postMaturityMode }) + const effectiveDebt = input.debt - input.badDebt // An empty best slot would make `wholeSlotRepaid` 0, pass every cap comparison, and return a // `(0, 0)` whole-slot plan that `isBadDebtRealization` reads as a write-off against a position we @@ -316,15 +301,18 @@ export const planWithReason = (input: PlanInput, options: PlanOptions = {}): Pla return { kind: 'skip', reason: 'nothing_to_seize', - // No cap stage ran — the slot is empty, so `cap`/`capEff` are deliberately absent rather than - // fabricated (the logger drops `undefined`, so the line self-describes). - trace: { postMaturityMode, lif, effectiveDebt: input.debt - input.badDebt, seizedAssets: 0n } + // No cap stage ran, so `cap`/`capEff` are absent rather than fabricated. + trace: { postMaturityMode, lif, effectiveDebt, seizedAssets: 0n } } } - return postMaturityMode - ? postMaturityOutcome({ input, lif, marginBps }) - : normalModeOutcome({ input, lif, marginBps }) + const wholeSlotRepaid = impliedRepaidUnits( + input.bestCollateralAmt, + input.bestCollateralPrice, + lif + ) + const stage: ModeStage = { input, marginBps, lif, effectiveDebt, wholeSlotRepaid } + return postMaturityMode ? postMaturityOutcome(stage) : normalModeOutcome(stage) } /** diff --git a/bots/midnight-liquidation/test/runner/tick.test.ts b/bots/midnight-liquidation/test/runner/tick.test.ts index 95341956..bdf3a6a4 100644 --- a/bots/midnight-liquidation/test/runner/tick.test.ts +++ b/bots/midnight-liquidation/test/runner/tick.test.ts @@ -40,9 +40,8 @@ const SENT: SubmitOutcome = { kind: 'sent', nonce: 7, txHash: TX_HASH } type TickCounters = Awaited> /** - * Every counter identity the tick promises for a completed tick. Asserted for EVERY case built by - * `runWith`, so a stage added without a counter breaks the sums instead of silently dropping a - * position — the exact class of bug these counters exist to catch. + * Asserted for EVERY case built by `runWith`, so a stage added without a counter breaks a sum rather + * than silently dropping a position — the exact class of bug these counters exist to catch. */ function expectCountersConsistent(c: TickCounters) { expect(c.pairs).toBeGreaterThanOrEqual(c.liquidatable) @@ -354,9 +353,8 @@ describe('runTick', () => { }) describe('unplannable positions (plan.skipped)', () => { - // Post-maturity dust: 1 wei of debt against a high-priced slot. The cap binds, but the largest - // seize whose derived repaid fits in 1 wei rounds to 0 collateral — the production signature that - // left 12 liquidatable positions silently unplanned. + // Post-maturity dust: 1 wei of debt against a high-priced slot, so the largest in-cap seize + // rounds to 0 collateral. This is the production signature that went unexplained for 5 days. const DUST = () => lensOut({ healthy: true, @@ -407,8 +405,8 @@ describe('runTick', () => { }) it('distinguishes the margin eating the cap from the division flooring', async () => { - // Same 1-wei cap, but a 30bps margin floors capEff to 0. Same reason, different numbers — this - // is what proves the field set discriminates the sub-causes. + // Same 1-wei cap, but a 30bps margin floors capEff to 0: same reason, different numbers. Shows + // the field set discriminates the sub-causes. const { events } = await runWith({ out: DUST(), seizeCapMarginBps: 30 }) expect(events.find(e => e.event === 'plan.skipped')?.fields).toMatchObject({ reason: 'seize_rounds_to_zero', @@ -420,8 +418,7 @@ describe('runTick', () => { it('warns on a non-positive cap and refuses the negative-seize plan', async () => { // debt - maxDebt (100) < badDebt (500) < debt (1000) makes the RCF numerator negative, so the - // cap and the derived seize both go negative. Pre-fix this returned a plan with a NEGATIVE - // seizedAssets that slipped past an `=== 0n` guard. + // cap and the seize both go negative. Pre-fix this returned a NEGATIVE-seize plan. // rcfThreshold 0 so the slot is NOT rcf-exempt and the negative RCF cap actually binds. const { counters, events, submitCalls } = await runWith({ out: lensOut({ @@ -484,8 +481,6 @@ describe('runTick', () => { }) it('counts notSent and ACCUMULATES the backoff when the send failed for this position', async () => { - // The bug-3 regression: pre-fix `backoff.clear` ran unconditionally after submit, so a - // send-failing position was re-quoted + re-simulated + re-sent every block forever. const cooldown = createCooldownStore({ cooldownMs: 60_000 }) const { counters, backoff } = await runWith({ cooldown, @@ -497,10 +492,9 @@ describe('runTick', () => { }) it('lets the backoff delay GROW across repeated send failures', async () => { - // The precise mechanism of bug 3: `clear` deletes the attempt count, so clearing on a - // non-broadcast resets the delay to `baseBlocks` forever and the exponential never accrues. - // Seeded at block 1 (attempts=1) and failing again at 100 must reach attempts=2 → a 4-block - // wait (until 104), not the 2-block wait a reset would give. + // `clear` deletes the attempt count, so clearing on a non-broadcast pins the delay at + // `baseBlocks` forever. Seeded at block 1 (attempts=1) then failing at 100 must reach + // attempts=2 → a 4-block wait (until 104), not the 2-block wait a reset would give. const { backoff } = await runWith({ seedBackoffAt: 1n, submitOutcome: { kind: 'failed', reason: 'submit_failed' } @@ -512,8 +506,8 @@ describe('runTick', () => { it.each(['send_aborted', 'nonce_hole', 'nonce_sync_failed'] as const)( 'counts notSent but does NOT back off a queue-wide refusal (%s)', async reason => { - // These refuse EVERY send this tick, so attributing them to whichever position was in hand - // would suppress innocent positions after the latch itself has cleared. + // These refuse EVERY send this tick, so attributing one to the position in hand would + // suppress positions that did nothing wrong. const cooldown = createCooldownStore({ cooldownMs: 60_000 }) const { counters, backoff } = await runWith({ cooldown, @@ -536,8 +530,8 @@ describe('runTick', () => { }) it('still emits tick.end with complete:false when a submit aborts the tick', async () => { - // A hashless send after the nonce was claimed throws by design. Pre-fix the throw escaped - // before `tick.end`, so the whole tick's counters vanished. + // A hashless send after the nonce was claimed throws by design; pre-fix the throw escaped + // before `tick.end`, so the tick's counters vanished. const { events } = await runExpectingThrow({ submitThrows: new Error('rpc timeout') }) const end = events.find(e => e.event === 'tick.end') expect(end?.fields).toMatchObject({ diff --git a/bots/midnight-liquidation/test/sizing/plan.test.ts b/bots/midnight-liquidation/test/sizing/plan.test.ts index 2a829232..49ce3b77 100644 --- a/bots/midnight-liquidation/test/sizing/plan.test.ts +++ b/bots/midnight-liquidation/test/sizing/plan.test.ts @@ -292,6 +292,23 @@ describe('planWithReason', () => { }) }) + it('caps a post-maturity seize at the post-writeoff debt, not the gross debt', () => { + // `badDebt` is written off before the repay, so the cap is `debt - badDebt`. Sizing against the + // gross debt would over-repay and revert on-chain (Panic 0x11). Asserted by equivalence rather + // than a literal, so the test does not re-derive the sizing arithmetic it is checking. + const post = { + blockTimestamp: 3000n, + maturity: 2000n, + healthy: true, + bestCollateralAmt: 1000n * WAD + } + const withWriteoff = plan(baseInput({ ...post, debt: 1000n * WAD, badDebt: 400n * WAD })) + const equivalent = plan(baseInput({ ...post, debt: 600n * WAD, badDebt: 0n })) + const grossDebt = plan(baseInput({ ...post, debt: 1000n * WAD, badDebt: 0n })) + expect(withWriteoff).toEqual(equivalent) + expect(withWriteoff).not.toEqual(grossDebt) + }) + it('reports nothing_to_seize for an empty best slot, omitting the cap stage', () => { const outcome = planWithReason(baseInput({ bestCollateralAmt: 0n })) expect(outcome).toEqual({ diff --git a/packages/bot-kit/src/queue/pending-queue.ts b/packages/bot-kit/src/queue/pending-queue.ts index a3c57672..5125d120 100644 --- a/packages/bot-kit/src/queue/pending-queue.ts +++ b/packages/bot-kit/src/queue/pending-queue.ts @@ -52,25 +52,22 @@ type Pending = { } /** - * What one {@link PendingQueue.submit} call did. `sent` carries the nonce + hash of a real broadcast; - * `failed` means NO transaction left this process. Callers must treat only `sent` as a submission — - * it is the sole outcome that may clear a per-position backoff. + * What one {@link PendingQueue.submit} call did. Only `sent` is a submission — the sole outcome that + * may clear a per-position backoff. `failed` means no transaction left this process, and its `reason` + * pairs with the warn line the queue already logged: * - * Each `reason` names the exit the queue took, and pairs with the warn line it already logged: + * | `reason` | logged event | scope | + * | ------------------- | ------------------- | -------------------- | + * | `submit_failed` | `tx.submit_failed` | THIS position's send | + * | `send_aborted` | `tx.send_aborted` | queue-wide refusal | + * | `nonce_sync_failed` | `nonce.sync_failed` | queue-wide refusal | + * | `nonce_hole` | `queue.nonce_hole` | queue-wide refusal | * - * | `reason` | logged event | scope | - * | ------------------- | ------------------- | -------------------------------------------- | - * | `send_aborted` | `tx.send_aborted` | queue-wide latch (clears next `onBlock`) | - * | `nonce_sync_failed` | `nonce.sync_failed` | queue-wide (cursor unusable this tick) | - * | `nonce_hole` | `queue.nonce_hole` | queue-wide latch (clears when chain consumes) | - * | `submit_failed` | `tx.submit_failed` | THIS position's send failed | + * Scope matters: a queue-wide reason refuses EVERY send this tick, so a caller must not back off the + * position it happened to be holding. Only `submit_failed` is that position's own failure. * - * The scope column is load-bearing: the three queue-wide reasons refuse *every* send this tick, so a - * caller must not attribute them to the position it happened to be holding. Only `submit_failed` is - * per-position and therefore the only reason that should back a position off. - * - * A first send that fails hashless AFTER claiming a nonce is NOT a `failed` outcome — it throws - * `TxSendError` so the caller aborts the tick rather than racing the signer's cursor rollback. + * A hashless send that already claimed a nonce is not a `failed` outcome — it throws `TxSendError` so + * the caller aborts the tick instead of racing the signer's cursor rollback. */ export type SubmitOutcome = | { kind: 'sent'; nonce: number; txHash: Hex } diff --git a/packages/bot-kit/src/runner/block-cadence.ts b/packages/bot-kit/src/runner/block-cadence.ts index ec1a2b3d..b8d51a24 100644 --- a/packages/bot-kit/src/runner/block-cadence.ts +++ b/packages/bot-kit/src/runner/block-cadence.ts @@ -1,28 +1,26 @@ /** - * Block-height cadence gate: lets a caller act at most once per `everyBlocks`. Used to bound the log - * volume of a condition that recurs every tick — a per-position line emitted on every block would - * dominate the source (12 positions × ~1780 ticks/hour ≈ 21k lines/hour/bot) while repeating one - * fact. O(1) state regardless of how many distinct positions pass through it, so unlike a per-label - * map there is nothing to evict and nothing to leak. + * Block-height cadence gate: lets a caller act at most once per `everyBlocks`. Bounds the log volume + * of a condition that recurs every tick, in O(1) state — unlike a per-label map, there is nothing to + * evict and nothing to leak. */ export type BlockSampler = { /** - * True when `block` is at least `everyBlocks` past the last granted claim (the first call always - * grants). STAMPS the cadence on true, hence `claim` and not a predicate name. + * True when `block` is at least `everyBlocks` past the last granted claim; the first call always + * grants. STAMPS the cadence on true, hence `claim` rather than a predicate name. * * A caller that asks ONLY when it has something to say therefore gets edge-triggering for free: a - * quiet stretch never consumes the window, so the first occurrence after any gap is always - * reported, and a sustained condition settles to one report per `everyBlocks`. + * quiet stretch never consumes the window, so the first occurrence after any gap is always reported + * and a sustained condition settles to one report per `everyBlocks`. */ claim: (block: bigint) => boolean } /** - * Builds a {@link BlockSampler} that grants a claim at most once per `everyBlocks` blocks. + * Builds a {@link BlockSampler}. * * @param everyBlocks - Minimum block distance between granted claims. `0n` grants every call. - * @returns A sampler holding one block height; not shared or reset — construct one per call site and - * hold it for the process lifetime. + * @returns A sampler holding one block height — construct one per call site and hold it for the + * process lifetime. */ export const createBlockSampler = (everyBlocks: bigint): BlockSampler => { let lastAt: bigint | null = null From 2d3ee9ed3974202a4be9e4f5bf139e0b7b182a60 Mon Sep 17 00:00:00 2001 From: Hayden Shively Date: Fri, 7 Aug 2026 11:15:54 -0500 Subject: [PATCH 3/4] fix(bots): address telemetry review findings --- bots/blue-liquidation/src/runner/tick.ts | 1 + bots/blue-liquidation/test/runner/tick.test.ts | 14 +++++++------- bots/midnight-liquidation/src/runner/tick.ts | 1 + bots/midnight-liquidation/src/sizing/plan.ts | 18 +++++++++++------- .../test/sizing/plan.test.ts | 2 +- packages/bot-kit/src/runner/block-cadence.ts | 5 +++++ 6 files changed, 26 insertions(+), 15 deletions(-) diff --git a/bots/blue-liquidation/src/runner/tick.ts b/bots/blue-liquidation/src/runner/tick.ts index ce5f0b3a..ac78c5e6 100644 --- a/bots/blue-liquidation/src/runner/tick.ts +++ b/bots/blue-liquidation/src/runner/tick.ts @@ -316,6 +316,7 @@ export async function runTick(deps: { // throws by design) — without `complete`, partial counters look exactly like an idle tick. // `tryCatch` preserves the error instance, so the rethrow keeps `TxSendError` intact downstream. const { error } = await tryCatch(processPairs()) + if (counters.planSkipped === 0) planSkipSampler.reset() logger.info('tick.end', { ...counters, complete: !error }) if (error) throw error return counters diff --git a/bots/blue-liquidation/test/runner/tick.test.ts b/bots/blue-liquidation/test/runner/tick.test.ts index b889106a..7c962141 100644 --- a/bots/blue-liquidation/test/runner/tick.test.ts +++ b/bots/blue-liquidation/test/runner/tick.test.ts @@ -16,7 +16,7 @@ import { ORACLE_PRICE_SCALE, WAD } from '../../src/constants' import { marketId } from '../../src/market' import { runTick } from '../../src/runner/tick' -function spyLogger() { +const spyLogger = () => { const events: { level: string; event: string; fields?: Record }[] = [] const make = (level: string) => (event: string, fields?: Record) => events.push({ level, event, fields }) @@ -54,7 +54,7 @@ type TickCounters = Awaited> * Asserted for EVERY case built by `runWith`, so a stage added without a counter breaks a sum rather * than silently dropping a position — the exact class of bug these counters exist to catch. */ -function expectCountersConsistent(c: TickCounters) { +const expectCountersConsistent = (c: TickCounters) => { expect(c.pairs).toBeGreaterThanOrEqual(c.liquidatable) expect(c.liquidatable).toBe(c.inflightSkipped + c.planSkipped + c.planned) expect(c.planned).toBe( @@ -80,7 +80,7 @@ const SWAP_PLAN: SwapPlan = { } // A liquidatable reading: valid, has debt, unhealthy, ample collateral (debt-binds → seize > 0). -function lensOut(overrides: Partial = {}): LensOut { +const lensOut = (overrides: Partial = {}): LensOut => { return { params: PARAMS, valid: true, @@ -100,7 +100,7 @@ function lensOut(overrides: Partial = {}): LensOut { const candidates = (...borrowers: Address[]): BorrowerCandidate[] => borrowers.map(borrower => ({ marketParams: PARAMS, borrower })) -function stubReadLens(out: LensOut | null) { +const stubReadLens = (out: LensOut | null) => { return async (pairs: LensInput[]) => { const map = new Map() if (out) for (const pair of pairs) map.set(lensKey(marketId(pair.params), pair.borrower), out) @@ -129,7 +129,7 @@ type RunOpts = { } // Shared dep construction so the throwing case exercises exactly the same wiring as `runWith`. -function buildDeps(opts: RunOpts) { +const buildDeps = (opts: RunOpts) => { const { logger, events } = spyLogger() let simulateCalls = 0 let submitCalls = 0 @@ -184,7 +184,7 @@ function buildDeps(opts: RunOpts) { } } -function runWith(opts: RunOpts) { +const runWith = (opts: RunOpts) => { const { deps, probes } = buildDeps(opts) return runTick(deps).then(counters => { expectCountersConsistent(counters) @@ -193,7 +193,7 @@ function runWith(opts: RunOpts) { } /** For the abort path: `runTick` rejects, so counters come from the emitted `tick.end` instead. */ -async function runExpectingThrow(opts: RunOpts) { +const runExpectingThrow = async (opts: RunOpts) => { const { deps, probes } = buildDeps(opts) await expect(runTick(deps)).rejects.toThrow() return probes diff --git a/bots/midnight-liquidation/src/runner/tick.ts b/bots/midnight-liquidation/src/runner/tick.ts index 832650b5..c7ff567d 100644 --- a/bots/midnight-liquidation/src/runner/tick.ts +++ b/bots/midnight-liquidation/src/runner/tick.ts @@ -338,6 +338,7 @@ export async function runTick(deps: { // throws by design) — without `complete`, partial counters look exactly like an idle tick. // `tryCatch` preserves the error instance, so the rethrow keeps `TxSendError` intact downstream. const { error } = await tryCatch(processPairs()) + if (counters.planSkipped === 0) planSkipSampler.reset() logger.info('tick.end', { ...counters, complete: !error }) if (error) throw error return counters diff --git a/bots/midnight-liquidation/src/sizing/plan.ts b/bots/midnight-liquidation/src/sizing/plan.ts index b920adba..b49ab421 100644 --- a/bots/midnight-liquidation/src/sizing/plan.ts +++ b/bots/midnight-liquidation/src/sizing/plan.ts @@ -220,6 +220,13 @@ const normalModeOutcome = ({ lif, lltv: input.bestCollateralLltv }) + const base: TraceBase = { + postMaturityMode: false, + lif, + effectiveDebt, + ...(input.bestCollateralLltv >= WAD ? { rcfDisabled: true } : { maxRepaid }) + } + if (maxRepaid <= 0n) return capBoundOutcome({ input, cap: maxRepaid, marginBps, base }) const exempt = isRcfExempt({ collateralAmt: input.bestCollateralAmt, price: input.bestCollateralPrice, @@ -227,18 +234,15 @@ const normalModeOutcome = ({ maxRepaid, rcfThreshold: input.rcfThreshold }) - const base: TraceBase = { - postMaturityMode: false, - lif, - effectiveDebt, - rcfExempt: exempt, + const exemptBase: TraceBase = { + ...base, + rcfExempt: exempt // `lltv >= WAD` waives the cap and returns maxUint256; flag that instead of logging a 78-digit // number that reads like a real bound. - ...(input.bestCollateralLltv >= WAD ? { rcfDisabled: true } : { maxRepaid }) } const repayCap = exempt ? effectiveDebt : min(maxRepaid, effectiveDebt) if (wholeSlotRepaid <= repayCap) return seizeWholeSlot(input, false) - return capBoundOutcome({ input, cap: repayCap, marginBps, base }) + return capBoundOutcome({ input, cap: repayCap, marginBps, base: exemptBase }) } /** diff --git a/bots/midnight-liquidation/test/sizing/plan.test.ts b/bots/midnight-liquidation/test/sizing/plan.test.ts index 49ce3b77..48717161 100644 --- a/bots/midnight-liquidation/test/sizing/plan.test.ts +++ b/bots/midnight-liquidation/test/sizing/plan.test.ts @@ -331,7 +331,7 @@ describe('planWithReason', () => { debt: 1000n, badDebt: 500n, maxDebt: 900n, - rcfThreshold: 0n, // not rcf-exempt, so the negative cap actually binds + rcfThreshold: 10n ** 30n, // even an otherwise exempt slot must reject a negative RCF cap bestCollateralAmt: 5000n, bestCollateralPrice: ORACLE_PRICE_SCALE, bestCollateralMaxLif: 1100000000000000000n diff --git a/packages/bot-kit/src/runner/block-cadence.ts b/packages/bot-kit/src/runner/block-cadence.ts index b8d51a24..c89dd835 100644 --- a/packages/bot-kit/src/runner/block-cadence.ts +++ b/packages/bot-kit/src/runner/block-cadence.ts @@ -13,6 +13,8 @@ export type BlockSampler = { * and a sustained condition settles to one report per `everyBlocks`. */ claim: (block: bigint) => boolean + /** Clears the cadence after a quiet observation so the next active observation is reported. */ + reset: () => void } /** @@ -29,6 +31,9 @@ export const createBlockSampler = (everyBlocks: bigint): BlockSampler => { if (lastAt !== null && block - lastAt < everyBlocks) return false lastAt = block return true + }, + reset() { + lastAt = null } } } From b8f032ecd7fdb84dc22cde962ec8e5df96271b44 Mon Sep 17 00:00:00 2001 From: Hayden Shively Date: Tue, 11 Aug 2026 15:07:01 -0500 Subject: [PATCH 4/4] refactor(bots): encode submit-failure scope in the type and trim tick overhead Apply the /simplify pass: SubmitOutcome carries scope ('position' | 'queue') so callers branch on it instead of string-matching reasons, and drops the never-read sent payload; the tick epilogue uses try/finally instead of a closure + tryCatch; the plan-skip sampler is claimed once eagerly per tick; invalid lens rows are counted without throwaway arrays; sizing builds trace objects only on refusal paths. Also convert midnight's tick-test helpers to arrow constants per the repo convention (codex review thread). Co-Authored-By: Claude Fable 5 --- bots/blue-liquidation/src/runner/tick.ts | 36 ++++++++-------- bots/blue-liquidation/src/sizing/plan.ts | 9 ++-- .../blue-liquidation/test/runner/tick.test.ts | 17 ++++---- bots/midnight-liquidation/src/runner/tick.ts | 36 ++++++++-------- bots/midnight-liquidation/src/sizing/plan.ts | 42 +++++++++---------- .../test/runner/tick.test.ts | 29 +++++++------ packages/bot-kit/src/queue/pending-queue.ts | 36 ++++++++-------- packages/bot-kit/src/runner/block-cadence.ts | 4 +- .../bot-kit/test/queue/pending-queue.test.ts | 10 ++--- 9 files changed, 112 insertions(+), 107 deletions(-) diff --git a/bots/blue-liquidation/src/runner/tick.ts b/bots/blue-liquidation/src/runner/tick.ts index ac78c5e6..ea4b0104 100644 --- a/bots/blue-liquidation/src/runner/tick.ts +++ b/bots/blue-liquidation/src/runner/tick.ts @@ -164,7 +164,8 @@ export async function runTick(deps: { // `returned` always equals `pairs` (the batch lens maps every input row); `invalid` is the // informative one — a row the lens zeroed (unknown market, reverting oracle) would otherwise be // indistinguishable from a healthy position inside `pairs - liquidatable`. - const invalid = [...lensOut.values()].filter(out => !out.valid).length + let invalid = 0 + for (const out of lensOut.values()) if (!out.valid) invalid += 1 logger.info('lens.read', { pairs: pairs.length, returned: lensOut.size, invalid }) const counters: TickCounters = { @@ -186,10 +187,14 @@ export async function runTick(deps: { // 3. Compose liquidatability off-chain → plan → quote → simulate → submit. `inflight` is captured // once; discovery yields distinct (market, borrower) pairs, so no label repeats within a tick. const inflight = inflightLabels() - // Resolved on the first skip and reused, so a tick emits the whole blocker set or none of it. - let explainSkips: boolean | null = null + // Claimed once per tick, so a tick emits the whole blocker set or none of it, all sharing one + // `chainHead`. A claim on a skip-free tick is undone by the `reset` in the epilogue below. + const explainSkips = planSkipSampler.claim(chainHead) - const processPairs = async () => { + // Emit counters even when a position aborts the tick (a hashless send after the nonce was claimed + // throws by design) — without `complete`, partial counters look exactly like an idle tick. + let complete = false + try { for (const pair of pairs) { const id = marketId(pair.params) const label = lensKey(id, pair.borrower) @@ -208,7 +213,6 @@ export async function runTick(deps: { const planOutcome = planWithReason(planInput) if (planOutcome.kind === 'skip') { counters.planSkipped += 1 - explainSkips ??= planSkipSampler.claim(chainHead) if (explainSkips) { // Inputs AND derived trace, so sizing replays from this one line. `marketId`/`borrower` // last, so a future `PlanInput` field can never shadow them. @@ -302,22 +306,20 @@ export async function runTick(deps: { continue } counters.notSent += 1 - // Only a per-position send failure earns a backoff. The other reasons are queue-WIDE refusals - // that reject every send this tick, so backing off here would suppress positions that did - // nothing wrong, for 2, 4, 8… blocks after the latch itself cleared. - if (sendOutcome.reason === 'submit_failed') { + // Only a failure scoped to THIS position earns a backoff. A queue-scoped refusal rejects every + // send this tick, so backing off here would suppress positions that did nothing wrong, for 2, + // 4, 8… blocks after the latch itself cleared. + if (sendOutcome.scope === 'position') { backoff.record(label, chainHead) cooldown.mark(label) } } + complete = true + } finally { + // Undo the eager claim when nothing was skipped, so the first skip after any quiet stretch is + // always explained (edge-triggering). + if (counters.planSkipped === 0) planSkipSampler.reset() + logger.info('tick.end', { ...counters, complete }) } - - // Emit counters even when a position aborts the tick (a hashless send after the nonce was claimed - // throws by design) — without `complete`, partial counters look exactly like an idle tick. - // `tryCatch` preserves the error instance, so the rethrow keeps `TxSendError` intact downstream. - const { error } = await tryCatch(processPairs()) - if (counters.planSkipped === 0) planSkipSampler.reset() - logger.info('tick.end', { ...counters, complete: !error }) - if (error) throw error return counters } diff --git a/bots/blue-liquidation/src/sizing/plan.ts b/bots/blue-liquidation/src/sizing/plan.ts index ae4ef49e..8b570586 100644 --- a/bots/blue-liquidation/src/sizing/plan.ts +++ b/bots/blue-liquidation/src/sizing/plan.ts @@ -108,9 +108,12 @@ export const planWithReason = (input: PlanInput): PlanOutcome => { input.collateralPrice ) const seizedAssets = min(input.collateral, seizeForFullDebt) - const trace: SizingTrace = { lif, repaidAssetsFull, seizeForFullDebt, seizedAssets } - // Rounds to nothing (dust position, or price ≫ debt): can't pass 0 to `liquidate`, so skip it. - if (seizedAssets === 0n) return { kind: 'skip', reason: 'seize_rounds_to_zero', trace } + // Rounds to nothing (dust position, or price ≫ debt): can't pass 0 to `liquidate`, so skip it. The + // trace is built only here, so the plan-success path allocates nothing extra. + if (seizedAssets === 0n) { + const trace: SizingTrace = { lif, repaidAssetsFull, seizeForFullDebt, seizedAssets } + return { kind: 'skip', reason: 'seize_rounds_to_zero', trace } + } return { kind: 'plan', plan: { seizedAssets } } } diff --git a/bots/blue-liquidation/test/runner/tick.test.ts b/bots/blue-liquidation/test/runner/tick.test.ts index 7c962141..d57a0ac5 100644 --- a/bots/blue-liquidation/test/runner/tick.test.ts +++ b/bots/blue-liquidation/test/runner/tick.test.ts @@ -1,7 +1,7 @@ import type { Logger, SimulateResult, SubmitOutcome } from '@repo/bot-kit' import type { Backoff, BlockSampler, CooldownStore } from '@repo/bot-kit' import type { QuoteOutcome, SwapPlan } from '@repo/swaps' -import type { Address, Hex } from 'viem' +import type { Address } from 'viem' import { createBackoff, createBlockSampler, createCooldownStore } from '@repo/bot-kit' import { lensKey } from '@repo/utils' @@ -45,8 +45,7 @@ const PARAMS: MarketParams = { } const MARKET_ID = marketId(PARAMS) const LABEL = lensKey(MARKET_ID, BORROWER) -const TX_HASH: Hex = `0x${'b'.repeat(64)}` -const SENT: SubmitOutcome = { kind: 'sent', nonce: 7, txHash: TX_HASH } +const SENT: SubmitOutcome = { kind: 'sent' } type TickCounters = Awaited> @@ -155,9 +154,9 @@ const buildDeps = (opts: RunOpts) => { quoteCalls += 1 return opts.quoteOutcome ?? defaultOutcome }, - simulate: async () => { + simulate: async (): Promise => { simulateCalls += 1 - return opts.simulateResult ?? ({ status: 'ok' } as SimulateResult) + return opts.simulateResult ?? { status: 'ok' } }, submit: async () => { submitCalls += 1 @@ -406,7 +405,7 @@ describe('runTick', () => { it('counts a broadcast as submitted and clears the backoff', async () => { const { counters, backoff } = await runWith({ seedBackoffAt: 1n, - submitOutcome: { kind: 'sent', nonce: 7, txHash: TX_HASH } + submitOutcome: { kind: 'sent' } }) expect(counters).toMatchObject({ ok: 1, submitted: 1, notSent: 0 }) expect(backoff.shouldSkip(LABEL, 1n)).toBe(false) @@ -416,7 +415,7 @@ describe('runTick', () => { const cooldown = createCooldownStore({ cooldownMs: 60_000 }) const { counters, backoff } = await runWith({ cooldown, - submitOutcome: { kind: 'failed', reason: 'submit_failed' } + submitOutcome: { kind: 'failed', scope: 'position', reason: 'submit_failed' } }) expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) expect(backoff.shouldSkip(LABEL, 100n)).toBe(true) @@ -429,7 +428,7 @@ describe('runTick', () => { // attempts=2 → a 4-block wait (until 104), not the 2-block wait a reset would give. const { backoff } = await runWith({ seedBackoffAt: 1n, - submitOutcome: { kind: 'failed', reason: 'submit_failed' } + submitOutcome: { kind: 'failed', scope: 'position', reason: 'submit_failed' } }) expect(backoff.shouldSkip(LABEL, 103n)).toBe(true) expect(backoff.shouldSkip(LABEL, 104n)).toBe(false) @@ -441,7 +440,7 @@ describe('runTick', () => { const cooldown = createCooldownStore({ cooldownMs: 60_000 }) const { counters, backoff } = await runWith({ cooldown, - submitOutcome: { kind: 'failed', reason } + submitOutcome: { kind: 'failed', scope: 'queue', reason } }) expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) expect(backoff.shouldSkip(LABEL, 100n)).toBe(false) diff --git a/bots/midnight-liquidation/src/runner/tick.ts b/bots/midnight-liquidation/src/runner/tick.ts index c7ff567d..59d985d5 100644 --- a/bots/midnight-liquidation/src/runner/tick.ts +++ b/bots/midnight-liquidation/src/runner/tick.ts @@ -170,7 +170,8 @@ export async function runTick(deps: { // `returned` always equals `pairs` (the batch lens maps every input row); `invalid` is the // informative one — a row the lens zeroed (unknown market, reverting oracle) would otherwise be // indistinguishable from a healthy position inside `pairs - liquidatable`. - const invalid = [...lensOut.values()].filter(out => !out.valid).length + let invalid = 0 + for (const out of lensOut.values()) if (!out.valid) invalid += 1 logger.info('lens.read', { pairs: pairs.length, returned: lensOut.size, invalid }) const counters: TickCounters = { @@ -192,10 +193,14 @@ export async function runTick(deps: { // 3. Compose liquidatability off-chain → plan → simulate → submit. `inflight` is captured once; // discovery yields distinct (id, borrower) pairs, so no label repeats within a single tick. const inflight = inflightLabels() - // Resolved on the first skip and reused, so a tick emits the whole blocker set or none of it. - let explainSkips: boolean | null = null + // Claimed once per tick, so a tick emits the whole blocker set or none of it, all sharing one + // `chainHead`. A claim on a skip-free tick is undone by the `reset` in the epilogue below. + const explainSkips = planSkipSampler.claim(chainHead) - const processPairs = async () => { + // Emit counters even when a position aborts the tick (a hashless send after the nonce was claimed + // throws by design) — without `complete`, partial counters look exactly like an idle tick. + let complete = false + try { for (const pair of pairs) { const label = lensKey(pair.id, pair.borrower) const out = lensOut.get(label) @@ -213,7 +218,6 @@ export async function runTick(deps: { const outcome = planWithReason(planInput, { seizeCapMarginBps }) if (outcome.kind === 'skip') { counters.planSkipped += 1 - explainSkips ??= planSkipSampler.claim(chainHead) if (explainSkips) { // Inputs AND derived trace, so sizing replays from this one line. `marketId`/`borrower` // last, so a future `PlanInput` field can never shadow them. @@ -324,22 +328,20 @@ export async function runTick(deps: { continue } counters.notSent += 1 - // Only a per-position send failure earns a backoff. The other reasons are queue-WIDE refusals - // that reject every send this tick, so backing off here would suppress positions that did - // nothing wrong, for 2, 4, 8… blocks after the latch itself cleared. - if (sendOutcome.reason === 'submit_failed') { + // Only a failure scoped to THIS position earns a backoff. A queue-scoped refusal rejects every + // send this tick, so backing off here would suppress positions that did nothing wrong, for 2, + // 4, 8… blocks after the latch itself cleared. + if (sendOutcome.scope === 'position') { backoff.record(label, chainHead) cooldown.mark(label) } } + complete = true + } finally { + // Undo the eager claim when nothing was skipped, so the first skip after any quiet stretch is + // always explained (edge-triggering). + if (counters.planSkipped === 0) planSkipSampler.reset() + logger.info('tick.end', { ...counters, complete }) } - - // Emit counters even when a position aborts the tick (a hashless send after the nonce was claimed - // throws by design) — without `complete`, partial counters look exactly like an idle tick. - // `tryCatch` preserves the error instance, so the rethrow keeps `TxSendError` intact downstream. - const { error } = await tryCatch(processPairs()) - if (counters.planSkipped === 0) planSkipSampler.reset() - logger.info('tick.end', { ...counters, complete: !error }) - if (error) throw error return counters } diff --git a/bots/midnight-liquidation/src/sizing/plan.ts b/bots/midnight-liquidation/src/sizing/plan.ts index b49ab421..91a023db 100644 --- a/bots/midnight-liquidation/src/sizing/plan.ts +++ b/bots/midnight-liquidation/src/sizing/plan.ts @@ -142,25 +142,27 @@ const capBoundOutcome = ({ }): PlanOutcome => { const capEff = mulDivDown(cap, BPS - BigInt(marginBps), BPS) const seizedAssets = maxSeizeForCap(capEff, input.bestCollateralPrice, base.lif) + if (cap > 0n && seizedAssets > 0n) { + return { + kind: 'plan', + plan: { + collateralIndex: input.bestCollateralIndex, + seizedAssets, + repaidUnits: 0n, + postMaturityMode: base.postMaturityMode + } + } + } + // The trace is built only on the refusal paths, so the plan-success path allocates nothing extra. const trace: SizingTrace = { ...base, cap, capEff, seizedAssets } - // Guard the RAW cap, not `capEff`: a legitimate 1-wei cap with any margin floors capEff to 0, which - // is ordinary dust. A non-positive RAW cap means the RCF numerator went negative + // Discriminate on the RAW cap, not `capEff`: a legitimate 1-wei cap with any margin floors capEff + // to 0, which is ordinary dust. A non-positive RAW cap means the RCF numerator went negative // (`debt - maxDebt < badDebt < debt`), which also yields a NEGATIVE seize that would slip past an - // `=== 0n` check and revert opaquely once abi-encoded as uint256. + // `=== 0n` check and revert opaquely once abi-encoded as uint256 — hence the `> 0n` gate above. if (cap <= 0n) return { kind: 'skip', reason: 'cap_not_positive', trace } // Rounds to nothing. Never emit a `(0, 0)` plan, which `isBadDebtRealization` would misread as a - // write-off against a solvent position. (`<= 0n` is belt-and-braces: a positive cap cannot floor to - // a negative seize, so the guard above already covers that.) - if (seizedAssets <= 0n) return { kind: 'skip', reason: 'seize_rounds_to_zero', trace } - return { - kind: 'plan', - plan: { - collateralIndex: input.bestCollateralIndex, - seizedAssets, - repaidUnits: 0n, - postMaturityMode: base.postMaturityMode - } - } + // write-off against a solvent position. + return { kind: 'skip', reason: 'seize_rounds_to_zero', trace } } const seizeWholeSlot = (input: PlanInput, postMaturityMode: boolean): PlanOutcome => ({ @@ -224,6 +226,8 @@ const normalModeOutcome = ({ postMaturityMode: false, lif, effectiveDebt, + // `lltv >= WAD` waives the cap and `maxRepaidPreMaturity` returns maxUint256; flag that instead + // of logging a 78-digit number that reads like a real bound. ...(input.bestCollateralLltv >= WAD ? { rcfDisabled: true } : { maxRepaid }) } if (maxRepaid <= 0n) return capBoundOutcome({ input, cap: maxRepaid, marginBps, base }) @@ -234,15 +238,9 @@ const normalModeOutcome = ({ maxRepaid, rcfThreshold: input.rcfThreshold }) - const exemptBase: TraceBase = { - ...base, - rcfExempt: exempt - // `lltv >= WAD` waives the cap and returns maxUint256; flag that instead of logging a 78-digit - // number that reads like a real bound. - } const repayCap = exempt ? effectiveDebt : min(maxRepaid, effectiveDebt) if (wholeSlotRepaid <= repayCap) return seizeWholeSlot(input, false) - return capBoundOutcome({ input, cap: repayCap, marginBps, base: exemptBase }) + return capBoundOutcome({ input, cap: repayCap, marginBps, base: { ...base, rcfExempt: exempt } }) } /** diff --git a/bots/midnight-liquidation/test/runner/tick.test.ts b/bots/midnight-liquidation/test/runner/tick.test.ts index bdf3a6a4..9a03fc62 100644 --- a/bots/midnight-liquidation/test/runner/tick.test.ts +++ b/bots/midnight-liquidation/test/runner/tick.test.ts @@ -13,7 +13,7 @@ import type { LensInput, LensOut } from '../../src/state/lens.sol' import { runTick } from '../../src/runner/tick' -function spyLogger() { +const spyLogger = () => { const events: { level: string; event: string; fields?: Record }[] = [] const make = (level: string) => (event: string, fields?: Record) => events.push({ level, event, fields }) @@ -34,8 +34,7 @@ const ROUTER: Address = getAddress('0x5555555555555555555555555555555555555555') const ZERO = '0x0000000000000000000000000000000000000000' as const const MARKET: Hex = `0x${'a'.repeat(64)}` const LABEL = lensKey(MARKET, BORROWER) -const TX_HASH: Hex = `0x${'b'.repeat(64)}` -const SENT: SubmitOutcome = { kind: 'sent', nonce: 7, txHash: TX_HASH } +const SENT: SubmitOutcome = { kind: 'sent' } type TickCounters = Awaited> @@ -43,7 +42,7 @@ type TickCounters = Awaited> * Asserted for EVERY case built by `runWith`, so a stage added without a counter breaks a sum rather * than silently dropping a position — the exact class of bug these counters exist to catch. */ -function expectCountersConsistent(c: TickCounters) { +const expectCountersConsistent = (c: TickCounters) => { expect(c.pairs).toBeGreaterThanOrEqual(c.liquidatable) expect(c.liquidatable).toBe(c.inflightSkipped + c.planSkipped + c.planned) expect(c.planned).toBe( @@ -69,7 +68,7 @@ const SWAP_PLAN: SwapPlan = { } // A liquidatable reading: valid, gate open, has debt, unlocked, unhealthy, pre-maturity. -function lensOut(overrides: Partial = {}): LensOut { +const lensOut = (overrides: Partial = {}): LensOut => { return { valid: true, hasDebt: true, @@ -110,7 +109,7 @@ function lensOut(overrides: Partial = {}): LensOut { const candidates = (...borrowers: Address[]): BorrowerCandidate[] => borrowers.map(borrower => ({ marketId: MARKET, borrower })) -function stubReadLens(out: LensOut | null) { +const stubReadLens = (out: LensOut | null) => { return async (pairs: LensInput[]) => { const map = new Map() if (out) for (const pair of pairs) map.set(lensKey(pair.id, pair.borrower), out) @@ -140,7 +139,7 @@ type RunOpts = { } // Shared dep construction so the throwing case exercises exactly the same wiring as `runWith`. -function buildDeps(opts: RunOpts) { +const buildDeps = (opts: RunOpts) => { const { logger, events } = spyLogger() let simulateCalls = 0 let submitCalls = 0 @@ -168,9 +167,9 @@ function buildDeps(opts: RunOpts) { quoteCalls += 1 return opts.quoteOutcome ?? defaultOutcome }, - simulate: async () => { + simulate: async (): Promise => { simulateCalls += 1 - return opts.simulateResult ?? ({ status: 'ok' } as SimulateResult) + return opts.simulateResult ?? { status: 'ok' } }, submit: async () => { submitCalls += 1 @@ -197,7 +196,7 @@ function buildDeps(opts: RunOpts) { } } -function runWith(opts: RunOpts) { +const runWith = (opts: RunOpts) => { const { deps, probes } = buildDeps(opts) return runTick(deps).then(counters => { expectCountersConsistent(counters) @@ -206,7 +205,7 @@ function runWith(opts: RunOpts) { } /** For the abort path: `runTick` rejects, so counters come from the emitted `tick.end` instead. */ -async function runExpectingThrow(opts: RunOpts) { +const runExpectingThrow = async (opts: RunOpts) => { const { deps, probes } = buildDeps(opts) await expect(runTick(deps)).rejects.toThrow() return probes @@ -474,7 +473,7 @@ describe('runTick', () => { it('counts a broadcast as submitted and clears the backoff', async () => { const { counters, backoff } = await runWith({ seedBackoffAt: 1n, - submitOutcome: { kind: 'sent', nonce: 7, txHash: TX_HASH } + submitOutcome: { kind: 'sent' } }) expect(counters).toMatchObject({ ok: 1, submitted: 1, notSent: 0 }) expect(backoff.shouldSkip(LABEL, 1n)).toBe(false) @@ -484,7 +483,7 @@ describe('runTick', () => { const cooldown = createCooldownStore({ cooldownMs: 60_000 }) const { counters, backoff } = await runWith({ cooldown, - submitOutcome: { kind: 'failed', reason: 'submit_failed' } + submitOutcome: { kind: 'failed', scope: 'position', reason: 'submit_failed' } }) expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) expect(backoff.shouldSkip(LABEL, 100n)).toBe(true) @@ -497,7 +496,7 @@ describe('runTick', () => { // attempts=2 → a 4-block wait (until 104), not the 2-block wait a reset would give. const { backoff } = await runWith({ seedBackoffAt: 1n, - submitOutcome: { kind: 'failed', reason: 'submit_failed' } + submitOutcome: { kind: 'failed', scope: 'position', reason: 'submit_failed' } }) expect(backoff.shouldSkip(LABEL, 103n)).toBe(true) expect(backoff.shouldSkip(LABEL, 104n)).toBe(false) @@ -511,7 +510,7 @@ describe('runTick', () => { const cooldown = createCooldownStore({ cooldownMs: 60_000 }) const { counters, backoff } = await runWith({ cooldown, - submitOutcome: { kind: 'failed', reason } + submitOutcome: { kind: 'failed', scope: 'queue', reason } }) expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) expect(backoff.shouldSkip(LABEL, 100n)).toBe(false) diff --git a/packages/bot-kit/src/queue/pending-queue.ts b/packages/bot-kit/src/queue/pending-queue.ts index 5125d120..032bddeb 100644 --- a/packages/bot-kit/src/queue/pending-queue.ts +++ b/packages/bot-kit/src/queue/pending-queue.ts @@ -56,25 +56,25 @@ type Pending = { * may clear a per-position backoff. `failed` means no transaction left this process, and its `reason` * pairs with the warn line the queue already logged: * - * | `reason` | logged event | scope | - * | ------------------- | ------------------- | -------------------- | - * | `submit_failed` | `tx.submit_failed` | THIS position's send | - * | `send_aborted` | `tx.send_aborted` | queue-wide refusal | - * | `nonce_sync_failed` | `nonce.sync_failed` | queue-wide refusal | - * | `nonce_hole` | `queue.nonce_hole` | queue-wide refusal | + * | `reason` | logged event | `scope` | + * | ------------------- | ------------------- | ------------ | + * | `submit_failed` | `tx.submit_failed` | `'position'` | + * | `send_aborted` | `tx.send_aborted` | `'queue'` | + * | `nonce_sync_failed` | `nonce.sync_failed` | `'queue'` | + * | `nonce_hole` | `queue.nonce_hole` | `'queue'` | * - * Scope matters: a queue-wide reason refuses EVERY send this tick, so a caller must not back off the - * position it happened to be holding. Only `submit_failed` is that position's own failure. + * `scope` is the field callers should branch on: a `'queue'` refusal rejects EVERY send this tick, + * so a caller must not back off the position it happened to be holding — only a `'position'` failure + * is that position's own. Keeping the classification here means a new reason cannot be added without + * the type forcing a scope decision. * * A hashless send that already claimed a nonce is not a `failed` outcome — it throws `TxSendError` so * the caller aborts the tick instead of racing the signer's cursor rollback. */ export type SubmitOutcome = - | { kind: 'sent'; nonce: number; txHash: Hex } - | { - kind: 'failed' - reason: 'send_aborted' | 'nonce_sync_failed' | 'nonce_hole' | 'submit_failed' - } + | { kind: 'sent' } + | { kind: 'failed'; scope: 'position'; reason: 'submit_failed' } + | { kind: 'failed'; scope: 'queue'; reason: 'send_aborted' | 'nonce_sync_failed' | 'nonce_hole' } export type PendingQueue = { submit(args: { @@ -219,7 +219,7 @@ export function createPendingQueue({ // rolled its cursor back, so broadcasting again now would race that rollback. if (sendAborted) { logger.warn('tx.send_aborted', { label: args.label }) - return { kind: 'failed', reason: 'send_aborted' } + return { kind: 'failed', scope: 'queue', reason: 'send_aborted' } } // Nothing in flight → reconcile the cursor with chain before claiming a nonce. A failed sync // would leave a stale (possibly runaway) cursor, so skip the send this tick rather than risk a @@ -231,7 +231,7 @@ export function createPendingQueue({ const synced = await tryCatch(syncNonce()) if (synced.error) { logger.warn('nonce.sync_failed', { label: args.label, reason: revertReason(synced.error) }) - return { kind: 'failed', reason: 'nonce_sync_failed' } + return { kind: 'failed', scope: 'queue', reason: 'nonce_sync_failed' } } if (nonceHoleLow !== null) clearNonceHole('sync') } @@ -241,7 +241,7 @@ export function createPendingQueue({ // sweep clears it once the chain consumes past the hole. if (nonceHoleLow !== null) { logger.warn('queue.nonce_hole', { label: args.label, nonce: nonceHoleLow }) - return { kind: 'failed', reason: 'nonce_hole' } + return { kind: 'failed', scope: 'queue', reason: 'nonce_hole' } } const sent = await tryCatch( send({ @@ -265,7 +265,7 @@ export function createPendingQueue({ sendAborted = true throw sent.error } - return { kind: 'failed', reason: 'submit_failed' } + return { kind: 'failed', scope: 'position', reason: 'submit_failed' } } const { nonce, txHash } = sent.data pending.set(nonce, { @@ -285,7 +285,7 @@ export function createPendingQueue({ maxFee: args.maxFeePerGas, priority: args.maxPriorityFeePerGas }) - return { kind: 'sent', nonce, txHash } + return { kind: 'sent' } } async function replaceStuck(entry: Pending, blockNumber: bigint, baseFee: bigint): Promise { diff --git a/packages/bot-kit/src/runner/block-cadence.ts b/packages/bot-kit/src/runner/block-cadence.ts index c89dd835..3d5ff156 100644 --- a/packages/bot-kit/src/runner/block-cadence.ts +++ b/packages/bot-kit/src/runner/block-cadence.ts @@ -10,7 +10,9 @@ export type BlockSampler = { * * A caller that asks ONLY when it has something to say therefore gets edge-triggering for free: a * quiet stretch never consumes the window, so the first occurrence after any gap is always reported - * and a sustained condition settles to one report per `everyBlocks`. + * and a sustained condition settles to one report per `everyBlocks`. A caller that claims eagerly + * (once per tick regardless) gets the same guarantee by calling {@link BlockSampler.reset} after a + * quiet tick. */ claim: (block: bigint) => boolean /** Clears the cadence after a quiet observation so the next active observation is reported. */ diff --git a/packages/bot-kit/test/queue/pending-queue.test.ts b/packages/bot-kit/test/queue/pending-queue.test.ts index be39d8b3..1421a46c 100644 --- a/packages/bot-kit/test/queue/pending-queue.test.ts +++ b/packages/bot-kit/test/queue/pending-queue.test.ts @@ -117,7 +117,7 @@ describe('createPendingQueue', () => { expect(sends[0]?.nonce).toBeUndefined() // first send leaves nonce assignment to the signer expect(queue.snapshot()[0]).toEqual({ nonce: 7, txHash: hashOf(1), attempt: 0 }) // The caller's broadcast signal — only this outcome may clear a per-position backoff. - expect(outcome).toEqual({ kind: 'sent', nonce: 7, txHash: hashOf(1) }) + expect(outcome).toEqual({ kind: 'sent' }) }) it('removes a tx once its receipt confirms', async () => { @@ -174,7 +174,7 @@ describe('createPendingQueue', () => { expect(queue.size).toBe(0) expect(events.find(e => e.event === 'tx.submit_failed')?.level).toBe('warn') // Per-position: this is the one reason a caller may attribute to the position it was holding. - expect(outcome).toEqual({ kind: 'failed', reason: 'submit_failed' }) + expect(outcome).toEqual({ kind: 'failed', scope: 'position', reason: 'submit_failed' }) }) it('rethrows a first-send failure after a nonce was claimed but no hash was returned', async () => { @@ -309,7 +309,7 @@ describe('createPendingQueue', () => { }) const outcome = await submitOne(ctx.queue) // must not throw expect(ctx.queue.size).toBe(0) // nothing broadcast on a stale cursor - expect(outcome).toEqual({ kind: 'failed', reason: 'nonce_sync_failed' }) + expect(outcome).toEqual({ kind: 'failed', scope: 'queue', reason: 'nonce_sync_failed' }) expect(ctx.sends).toHaveLength(0) expect(events.find(e => e.event === 'nonce.sync_failed')?.level).toBe('warn') }) @@ -495,7 +495,7 @@ describe('send-aborted latch', () => { const outcome = await submitOne(queue, 0n) // latched → skipped expect(events.find(e => e.event === 'tx.send_aborted')?.level).toBe('warn') // Queue-WIDE: refuses every send this tick, so a caller must not back the position off for it. - expect(outcome).toEqual({ kind: 'failed', reason: 'send_aborted' }) + expect(outcome).toEqual({ kind: 'failed', scope: 'queue', reason: 'send_aborted' }) }) }) @@ -565,7 +565,7 @@ describe('nonce-hole latch', () => { // so the empty-queue sync can't clear it). const refused = await ctx.submit('c', 6n) expect(ctx.sends.length).toBe(sendsAfterDrop) // no new broadcast - expect(refused).toEqual({ kind: 'failed', reason: 'nonce_hole' }) + expect(refused).toEqual({ kind: 'failed', scope: 'queue', reason: 'nonce_hole' }) expect(ctx.queue.size).toBe(1) expect(ctx.events.some(e => e.event === 'queue.nonce_hole' && e.fields?.label === 'c')).toBe( true