diff --git a/.claude/commands/railway.md b/.claude/commands/railway.md index 65aeaa40..b9a64886 100644 --- a/.claude/commands/railway.md +++ b/.claude/commands/railway.md @@ -42,8 +42,11 @@ railway status # lists services + Online/offline ``` `midnight-liquidation` services: `bot`, `rindexer`, `Postgres`. Healthy `bot` ticks each block: -`event="block.new"` → `event="lens.read"` → `event="tick.end"` (`liquidatable=`/`submitted=` -counters), plus one `event="daemon.start"` at boot. Trouble = `event="tick.error"` or `[ERROR]`; +`event="block.new"` → `event="lens.read"` → `event="tick.end"`, plus one `event="daemon.start"` at +boot. In `tick.end`, `submitted=` counts only real broadcasts and `notSent=` counts submits that sent +nothing; `liquidatable=` is a market gauge, so `liquidatable>0` with `planSkipped=` equal to it means +the bot sees positions it cannot size — grep `plan.skipped` for the reason and the sizing numbers. +`complete=false` marks a tick that aborted part-way, so its counters are partial. Trouble = `event="tick.error"` or `[ERROR]`; for a reverting tx grep `tx.dropped` / `tx.replace_failed` / `tx.submit_failed`. Triage by event, not the raw line — error lines historically carried a multi-KB calldata dump that shippers truncate. diff --git a/bots/blue-liquidation/README.md b/bots/blue-liquidation/README.md index e9739cdc..17043f44 100644 --- a/bots/blue-liquidation/README.md +++ b/bots/blue-liquidation/README.md @@ -240,7 +240,17 @@ per-`(id, borrower)` exponential backoff suppresses repeated failures. ### Simulation The exact `Executor.exec_606BaXt(...)` bytes are `eth_call`-simulated from the EOA. Only an `ok` -result is broadcast (`simulate.ok` gate). The exec is `liquidate` + an in-callback queue (via +result is broadcast (`simulate.ok` gate). + +`tick.end`'s `submitted` counts only real broadcasts: `submit` reports whether a transaction went out, +and only that clears the position's backoff. A send that failed for this position (`tx.submit_failed`) +counts `notSent` and backs the position off; the three queue-wide refusals (`tx.send_aborted`, +`nonce.sync_failed`, `queue.nonce_hole`) count `notSent` only, since they reject every send that tick. +A tick aborted by a hashless nonce-bearing send still emits `tick.end` with `complete: false`. + +A position the chain reports liquidatable can still fail to size; the tick counts it as `planSkipped` +and emits a sampled `plan.skipped` with the reason (`no_collateral`, `zero_price`, +`seize_rounds_to_zero`) plus the sizing inputs and derived numbers. The exec is `liquidate` + an in-callback queue (via `onMorphoLiquidate`) that runs the plan's steps — a plain collateral is one venue swap; exotic collateral is unwrap step(s) then usually a venue swap — followed by the repay-token approval. After `liquidate` returns, trailing `skim` sweeps drain both market tokens **plus every intermediate token diff --git a/bots/blue-liquidation/src/constants.ts b/bots/blue-liquidation/src/constants.ts index a93a972b..fb86e5e7 100644 --- a/bots/blue-liquidation/src/constants.ts +++ b/bots/blue-liquidation/src/constants.ts @@ -39,3 +39,10 @@ export const VIRTUAL_ASSETS = 1n * terminal, so a brief suppression of an already-acted position is harmless. */ export const SETTLED_COOLDOWN_BLOCKS = 20n + +/** + * Block cadence bounding the per-position `plan.skipped` diagnostic (~5 min at ~2s Base blocks). An + * unsizable position recurs every tick, so one line per position per block would be ~21k + * lines/hour/bot — all repeating one fact. The `planSkipped` counter stays exact regardless. + */ +export const PLAN_SKIP_SAMPLE_EVERY_BLOCKS = 150n diff --git a/bots/blue-liquidation/src/index.ts b/bots/blue-liquidation/src/index.ts index 422ece56..76806d66 100644 --- a/bots/blue-liquidation/src/index.ts +++ b/bots/blue-liquidation/src/index.ts @@ -5,6 +5,7 @@ import { assertContractDeployed, createBalanceMonitor, createBackoff, + createBlockSampler, createCooldownStore, createDeploylessClient, createHeartbeatMonitor, @@ -34,7 +35,7 @@ import type { MarketParams } from './market' import type { LiquidationPlan } from './sizing/plan' import { loadConfig } from './config' -import { SETTLED_COOLDOWN_BLOCKS } from './constants' +import { PLAN_SKIP_SAMPLE_EVERY_BLOCKS, SETTLED_COOLDOWN_BLOCKS } from './constants' import { createGraphqlCandidateSource, discoverCandidates } from './discovery/borrowers' import { encodeLiquidationExec } from './execution/encode-call' import { composeQuoting } from './quotes' @@ -179,6 +180,9 @@ async function main() { // Opt-in per-position cooldown (default disabled): one in-memory store for the process lifetime, // complementary to `backoff` (see POSITION_LIQUIDATION_COOLDOWN_MS). const cooldown = createCooldownStore({ cooldownMs: config.positionCooldownMs }) + // Bounds the `plan.skipped` diagnostic to one burst per cadence; process-lifetime state, like the + // two stores above. + const planSkipSampler = createBlockSampler(PLAN_SKIP_SAMPLE_EVERY_BLOCKS) // The exec calldata for one liquidation — the same bytes the simulate gate checks and the queue // broadcasts, so a sim-ok plan and its broadcast can't drift. @@ -273,7 +277,9 @@ async function main() { }), submit: async ({ market, borrower, plan, swapPlan, blockNumber, label }) => { const fees = initialFees(await signer.getBaseFee(), config.maxFeeWei) - await queue.submit({ + // Returned, not awaited-and-discarded: the tick needs to know whether a tx actually went out + // before it clears this position's backoff. + return queue.submit({ request: { to: config.executooorAddress, data: encodeExec(market, borrower, plan, swapPlan) @@ -286,6 +292,7 @@ async function main() { }, backoff, cooldown, + planSkipSampler, inflightLabels: () => queue.inflightLabels(), logger }) diff --git a/bots/blue-liquidation/src/runner/tick.ts b/bots/blue-liquidation/src/runner/tick.ts index ade54323..ea4b0104 100644 --- a/bots/blue-liquidation/src/runner/tick.ts +++ b/bots/blue-liquidation/src/runner/tick.ts @@ -1,4 +1,11 @@ -import type { Backoff, CooldownStore, Logger, SimulateResult } from '@repo/bot-kit' +import type { + Backoff, + BlockSampler, + CooldownStore, + Logger, + SimulateResult, + SubmitOutcome +} from '@repo/bot-kit' import type { QuoteOutcome, SwapPlan } from '@repo/swaps' import type { Address } from 'viem' @@ -6,24 +13,60 @@ import { assertNever, lensKey, tryCatch } from '@repo/utils' import type { BorrowerCandidate } from '../discovery/borrowers' import type { MarketParams } from '../market' -import type { LiquidationPlan } from '../sizing/plan' +import type { LiquidationPlan, PlanSkipReason } from '../sizing/plan' import type { LensInput, LensOut } from '../state/lens.sol' import { marketId } from '../market' -import { plan } from '../sizing/plan' +import { planWithReason } from '../sizing/plan' import { isLiquidatable, planInputFromLens } from './eligibility' +/** + * Per-tick outcome tally, emitted as `tick.end`, ordered as the pipeline runs. For a tick that + * finished (`complete: true`) these identities hold, so a stage added without a counter breaks a sum + * instead of silently dropping a position: + * + * pairs >= liquidatable + * liquidatable === inflightSkipped + planSkipped + planned + * planned === cooledDown + backoffSkipped + noSwapPath + quoteFailed + ok + reverted + * ok === submitted + notSent + * + * On `complete: false` the last is short by one: an aborting `submit` throws after `ok` was counted. + */ type TickCounters = { + /** Lens inputs read this tick — the discovery universe. */ pairs: number + /** Positions the chain says are liquidatable at this block. A market gauge, not a bot decision. */ liquidatable: number + /** + * Skipped because a tx for this label is in flight. The queue's backpressure set also holds labels + * that SETTLED within `settledCooldownBlocks`, so this can be non-zero while the queue is empty. + */ + inflightSkipped: number + /** Skipped because sizing produced no plan — see the `plan.skipped` event for the reason. */ + planSkipped: number planned: number + cooledDown: number + backoffSkipped: number noSwapPath: number quoteFailed: number - backoffSkipped: number - cooledDown: number ok: number reverted: number + /** Broadcast: the queue reported a transaction actually went out. */ submitted: number + /** The queue returned without broadcasting (a send failure, or a queue-wide refusal). */ + notSent: number +} + +/** + * `warn` marks a reason that should not happen — `no_debt`/`healthy` negate `isLiquidatable`, and a + * non-reverting zero oracle price is a market anomaly — so the line reads as a live assertion. + */ +const LEVEL_BY_REASON: Record = { + no_debt: 'warn', + healthy: 'warn', + zero_price: 'warn', + no_collateral: 'info', + seize_rounds_to_zero: 'info' } /** @@ -38,6 +81,10 @@ type TickCounters = { * Discovery failure is tolerated: a transient error is logged (`discover.error`) and the tick proceeds * with zero new candidates. The lens reads every candidate fresh on-chain, so discovery is a coverage * source, never a correctness dependency — API indexing lag is coverage latency only. + * + * Every exit from the per-position loop increments exactly one counter, and `tick.end` is emitted even + * when a position aborts the tick — with `complete: false`, so partial counters can never be mistaken + * for a genuinely idle tick. */ export async function runTick(deps: { discover: () => Promise @@ -56,7 +103,12 @@ export async function runTick(deps: { plan: LiquidationPlan swapPlan: SwapPlan }) => Promise - /** Broadcasts a plan via the pending queue (builds the exec tx, derives fees, tracks the nonce). */ + /** + * Broadcasts a plan via the pending queue (builds the exec tx, derives fees, tracks the nonce). + * Reports whether a transaction actually went out: ONLY `kind: 'sent'` may clear the position's + * backoff. Throws when a send fails after claiming a nonce — the tick aborts by design, so the + * signer's cursor rollback is not raced. + */ submit: (args: { market: MarketParams borrower: Address @@ -64,15 +116,20 @@ export async function runTick(deps: { swapPlan: SwapPlan blockNumber: bigint label: string - }) => Promise - /** Per-position exponential backoff suppressing repeated quote/simulate failures (rate-limit defense). */ + }) => Promise + /** Per-position exponential backoff suppressing repeated quote/simulate/send failures (rate-limit defense). */ backoff: Backoff /** * Opt-in per-position cooldown, complementary to `backoff`: a fixed wall-clock window suppressing - * re-quoting a position whose last attempt produced no submittable tx. Disabled by default + * re-quoting a position whose last attempt produced no broadcast tx. Disabled by default * (`POSITION_LIQUIDATION_COOLDOWN_MS=0`), in which case `shouldSkip` is always false. */ cooldown: CooldownStore + /** + * Bounds how often the per-position `plan.skipped` diagnostic is emitted. Asked at most once per + * tick, so every line in a burst shares one `chainHead` and reads as a single snapshot. + */ + planSkipSampler: BlockSampler /** Labels (`${id}:${borrower}`) already in flight — skipped to avoid re-submitting each block. */ inflightLabels: () => ReadonlySet logger: Logger @@ -86,6 +143,7 @@ export async function runTick(deps: { submit, backoff, cooldown, + planSkipSampler, inflightLabels, logger } = deps @@ -103,101 +161,138 @@ export async function runTick(deps: { // 2. Read the lens fresh for the whole batch in one deployless eth_call. const lensOut = await readLens(pairs) - logger.info('lens.read', { pairs: pairs.length, returned: lensOut.size }) + // `returned` always equals `pairs` (the batch lens maps every input row); `invalid` is the + // informative one — a row the lens zeroed (unknown market, reverting oracle) would otherwise be + // indistinguishable from a healthy position inside `pairs - liquidatable`. + let invalid = 0 + for (const out of lensOut.values()) if (!out.valid) invalid += 1 + logger.info('lens.read', { pairs: pairs.length, returned: lensOut.size, invalid }) const counters: TickCounters = { pairs: pairs.length, liquidatable: 0, + inflightSkipped: 0, + planSkipped: 0, planned: 0, + cooledDown: 0, + backoffSkipped: 0, noSwapPath: 0, quoteFailed: 0, - backoffSkipped: 0, - cooledDown: 0, ok: 0, reverted: 0, - submitted: 0 + submitted: 0, + notSent: 0 } // 3. Compose liquidatability off-chain → plan → quote → simulate → submit. `inflight` is captured // once; discovery yields distinct (market, borrower) pairs, so no label repeats within a tick. const inflight = inflightLabels() - for (const pair of pairs) { - const id = marketId(pair.params) - const label = lensKey(id, pair.borrower) - const out = lensOut.get(label) - if (!out || !isLiquidatable(out)) continue - counters.liquidatable += 1 - - // Backpressure: a tx for this position is already pending — don't re-plan/simulate/submit it - // every block while it confirms. - if (inflight.has(label)) continue - - const liquidationPlan = plan(planInputFromLens(out)) - if (!liquidationPlan) continue - counters.planned += 1 - logger.info('plan.built', { - marketId: id, - borrower: pair.borrower, - seizedAssets: liquidationPlan.seizedAssets - }) - - // Opt-in cooldown (complementary to backoff): a position whose last attempt produced no - // submittable tx is skipped without re-quoting until its wall-clock window elapses. No-op when - // disabled (POSITION_LIQUIDATION_COOLDOWN_MS=0). - if (cooldown.shouldSkip(label)) { - counters.cooledDown += 1 - logger.info('cooldown.skip', { marketId: id, borrower: pair.borrower }) - continue - } - // Suppress positions that keep failing to quote/simulate — bounds API + RPC usage under a - // backlog, since executable quotes are spent only on positions not currently backed off. - if (backoff.shouldSkip(label, chainHead)) { - counters.backoffSkipped += 1 - continue - } - const outcome = await quoteFor(liquidationPlan, out, label) - if (outcome.kind === 'no_config') { - counters.noSwapPath += 1 - cooldown.mark(label) - logger.info('config.no_swap_path', { marketId: id, borrower: pair.borrower }) - continue - } - if (outcome.kind === 'failed') { - counters.quoteFailed += 1 - backoff.record(label, chainHead) - cooldown.mark(label) - continue - } - const swapPlan = outcome.plan - - const result = await simulate({ - market: out.params, - borrower: pair.borrower, - plan: liquidationPlan, - swapPlan - }) - const fields = { marketId: id, borrower: pair.borrower } - switch (result.status) { - case 'ok': - counters.ok += 1 - logger.info('simulate.ok', fields) - break - case 'revert': - counters.reverted += 1 - // Back off: a sim revert (stale quote, transient unliquidatability, oracle drift) shouldn't - // re-quote + re-simulate this position every block. + // Claimed once per tick, so a tick emits the whole blocker set or none of it, all sharing one + // `chainHead`. A claim on a skip-free tick is undone by the `reset` in the epilogue below. + const explainSkips = planSkipSampler.claim(chainHead) + + // Emit counters even when a position aborts the tick (a hashless send after the nonce was claimed + // throws by design) — without `complete`, partial counters look exactly like an idle tick. + let complete = false + try { + for (const pair of pairs) { + const id = marketId(pair.params) + const label = lensKey(id, pair.borrower) + const out = lensOut.get(label) + if (!out || !isLiquidatable(out)) continue + counters.liquidatable += 1 + + // Backpressure: don't re-plan/simulate/submit while a tx confirms. No log — the queue already + // narrates this label end to end (`tx.sent` → `tx.confirmed`/`tx.reverted`/`tx.dropped`). + if (inflight.has(label)) { + counters.inflightSkipped += 1 + continue + } + + const planInput = planInputFromLens(out) + const planOutcome = planWithReason(planInput) + if (planOutcome.kind === 'skip') { + counters.planSkipped += 1 + if (explainSkips) { + // Inputs AND derived trace, so sizing replays from this one line. `marketId`/`borrower` + // last, so a future `PlanInput` field can never shadow them. + logger[LEVEL_BY_REASON[planOutcome.reason]]('plan.skipped', { + ...planInput, + ...planOutcome.trace, + reason: planOutcome.reason, + marketId: id, + borrower: pair.borrower + }) + } + continue + } + + const liquidationPlan = planOutcome.plan + counters.planned += 1 + logger.info('plan.built', { + marketId: id, + borrower: pair.borrower, + seizedAssets: liquidationPlan.seizedAssets + }) + + // Opt-in cooldown (complementary to backoff): a position whose last attempt produced no + // broadcast tx is skipped without re-quoting until its wall-clock window elapses. No-op when + // disabled (POSITION_LIQUIDATION_COOLDOWN_MS=0). + if (cooldown.shouldSkip(label)) { + counters.cooledDown += 1 + logger.info('cooldown.skip', { marketId: id, borrower: pair.borrower }) + continue + } + // Suppress positions that keep failing to quote/simulate/send — bounds API + RPC usage under a + // backlog, since executable quotes are spent only on positions not currently backed off. + if (backoff.shouldSkip(label, chainHead)) { + counters.backoffSkipped += 1 + continue + } + const quote = await quoteFor(liquidationPlan, out, label) + if (quote.kind === 'no_config') { + counters.noSwapPath += 1 + cooldown.mark(label) + logger.info('config.no_swap_path', { marketId: id, borrower: pair.borrower }) + continue + } + if (quote.kind === 'failed') { + counters.quoteFailed += 1 backoff.record(label, chainHead) cooldown.mark(label) - logger.warn('simulate.revert', { ...fields, reason: result.reason }) - break - default: - assertNever(result.status) - } + continue + } + const swapPlan = quote.plan - // ok-only gate: broadcast only a fully-simulated, swap-funded liquidation. Any revert — not - // liquidatable, swap slippage, repay shortfall — isn't a fundable plan, so skip it. - if (result.status === 'ok') { - await submit({ + const result = await simulate({ + market: out.params, + borrower: pair.borrower, + plan: liquidationPlan, + swapPlan + }) + const fields = { marketId: id, borrower: pair.borrower } + switch (result.status) { + case 'ok': + counters.ok += 1 + logger.info('simulate.ok', fields) + break + case 'revert': + counters.reverted += 1 + // Back off: a sim revert (stale quote, transient unliquidatability, oracle drift) shouldn't + // re-quote + re-simulate this position every block. + backoff.record(label, chainHead) + cooldown.mark(label) + logger.warn('simulate.revert', { ...fields, reason: result.reason }) + break + default: + assertNever(result.status) + } + + // ok-only gate: broadcast only a fully-simulated, swap-funded liquidation. Any revert — not + // liquidatable, swap slippage, repay shortfall — isn't a fundable plan, so skip it. + if (result.status !== 'ok') continue + + const sendOutcome = await submit({ market: out.params, borrower: pair.borrower, plan: liquidationPlan, @@ -205,11 +300,26 @@ export async function runTick(deps: { blockNumber: chainHead, label }) - backoff.clear(label) - counters.submitted += 1 + if (sendOutcome.kind === 'sent') { + backoff.clear(label) + counters.submitted += 1 + continue + } + counters.notSent += 1 + // Only a failure scoped to THIS position earns a backoff. A queue-scoped refusal rejects every + // send this tick, so backing off here would suppress positions that did nothing wrong, for 2, + // 4, 8… blocks after the latch itself cleared. + if (sendOutcome.scope === 'position') { + backoff.record(label, chainHead) + cooldown.mark(label) + } } + complete = true + } finally { + // Undo the eager claim when nothing was skipped, so the first skip after any quiet stretch is + // always explained (edge-triggering). + if (counters.planSkipped === 0) planSkipSampler.reset() + logger.info('tick.end', { ...counters, complete }) } - - logger.info('tick.end', { ...counters }) return counters } diff --git a/bots/blue-liquidation/src/sizing/plan.ts b/bots/blue-liquidation/src/sizing/plan.ts index 7b88fcfa..8b570586 100644 --- a/bots/blue-liquidation/src/sizing/plan.ts +++ b/bots/blue-liquidation/src/sizing/plan.ts @@ -25,8 +25,46 @@ export type PlanInput = { export type LiquidationPlan = { seizedAssets: bigint } /** - * Turns a fresh lens reading into a seize-exact liquidation plan, or `null` when the position is not - * liquidatable or has nothing to seize. + * Why {@link planWithReason} produced no plan. Reasons discriminate SEVERITY (and hence log level); + * the numbers on {@link SizingTrace} discriminate cause. + * + * `no_debt` and `healthy` are UNREACHABLE from the tick — `isLiquidatable` tests exactly + * `hasDebt && !healthy` — so observing one means the eligibility gate and the sizing module have + * diverged, a correctness bug. They are logged at `warn`, as a live assertion. + */ +export type PlanSkipReason = + | 'no_debt' + | 'healthy' + /** Pure residual bad debt: shares outstanding but no collateral left to seize. */ + | 'no_collateral' + /** A non-reverting zero oracle price — unsizable, and a divide-by-zero if used. */ + | 'zero_price' + /** Dust, or a price so high the full-debt seize floors to zero collateral. */ + | 'seize_rounds_to_zero' + +/** + * Every value on the causal chain from a lens reading to a refused seize, so an operator can replay + * the decision from one log line instead of re-deriving it. Absent for the two pre-sizing refusals + * (`no_debt`, `healthy`), which are decided before any arithmetic runs. + */ +type SizingTrace = { + lif: bigint + /** Full debt in loan assets, floored from `borrowShares`. */ + repaidAssetsFull: bigint + /** Collateral the full-debt repay would seize, before the 100%-of-collateral clamp. */ + seizeForFullDebt: bigint + /** The final seize; `0n` is what refused the plan. */ + seizedAssets: bigint +} + +/** A plan, or the reason there is none plus the numbers that explain it. */ +type PlanOutcome = + | { kind: 'plan'; plan: LiquidationPlan } + | { kind: 'skip'; reason: PlanSkipReason; trace?: SizingTrace } + +/** + * Turns a fresh lens reading into a seize-exact liquidation plan, or a {@link PlanSkipReason} + * explaining why there is none. * * Blue liquidation is permissionless and time-independent, so the only gate is `hasDebt && !healthy` * (no maturity, no liquidator gate, no RCF cap). The single sizing rule pins the seize at the amount @@ -41,17 +79,22 @@ export type LiquidationPlan = { seizedAssets: bigint } * `plan.test.ts`). When collateral binds, seizing all collateral socializes the residual as bad debt * in the same call. If a later oracle move makes the derived repay too large, simulation catches the * revert before broadcast. + * + * @param input - Fresh lens-derived position state, all read at one block. + * @returns `{ kind: 'plan' }` with a submittable plan, or `{ kind: 'skip' }` with the reason and — + * for every refusal reached after sizing began — a {@link SizingTrace}. */ -export function plan(input: PlanInput): LiquidationPlan | null { - if (!input.hasDebt || input.healthy) return null +export const planWithReason = (input: PlanInput): PlanOutcome => { + if (!input.hasDebt) return { kind: 'skip', reason: 'no_debt' } + if (input.healthy) return { kind: 'skip', reason: 'healthy' } // Degenerate: pure residual bad debt (borrowShares > 0 but collateral == 0). Nothing to seize; a // backstop bot does not perform an uncompensated loan-token repay, and Blue rejects a (0, 0) // liquidate anyway (`exactlyOneZero`). - if (input.collateral === 0n) return null + if (input.collateral === 0n) return { kind: 'skip', reason: 'no_collateral' } // A zero (non-reverting) oracle price would divide-by-zero below. It also makes maxBorrow 0, so the // lens reports the position unhealthy — but we cannot size against it; skip rather than throw and // abort the whole tick. (A reverting oracle is already dropped by the lens as valid=false.) - if (input.collateralPrice === 0n) return null + if (input.collateralPrice === 0n) return { kind: 'skip', reason: 'zero_price' } const lif = lifFromLltv(input.lltv) const repaidAssetsFull = toAssetsDown( @@ -65,7 +108,22 @@ export function plan(input: PlanInput): LiquidationPlan | null { input.collateralPrice ) const seizedAssets = min(input.collateral, seizeForFullDebt) - // Rounds to nothing (dust position, or price ≫ debt): can't pass 0 to `liquidate`, so skip it. - if (seizedAssets === 0n) return null - return { seizedAssets } + // Rounds to nothing (dust position, or price ≫ debt): can't pass 0 to `liquidate`, so skip it. The + // trace is built only here, so the plan-success path allocates nothing extra. + if (seizedAssets === 0n) { + const trace: SizingTrace = { lif, repaidAssetsFull, seizeForFullDebt, seizedAssets } + return { kind: 'skip', reason: 'seize_rounds_to_zero', trace } + } + return { kind: 'plan', plan: { seizedAssets } } +} + +/** + * Bare-plan facade over {@link planWithReason} for callers that do not report a reason. + * + * @param input - Fresh lens-derived position state. + * @returns The plan, or `null` when the position is not liquidatable or has nothing to seize. + */ +export const plan = (input: PlanInput): LiquidationPlan | null => { + const outcome = planWithReason(input) + return outcome.kind === 'plan' ? outcome.plan : null } diff --git a/bots/blue-liquidation/test/runner/tick.test.ts b/bots/blue-liquidation/test/runner/tick.test.ts index b2f236e5..d57a0ac5 100644 --- a/bots/blue-liquidation/test/runner/tick.test.ts +++ b/bots/blue-liquidation/test/runner/tick.test.ts @@ -1,9 +1,9 @@ -import type { Logger, SimulateResult } from '@repo/bot-kit' -import type { CooldownStore } from '@repo/bot-kit' +import type { Logger, SimulateResult, SubmitOutcome } from '@repo/bot-kit' +import type { Backoff, BlockSampler, CooldownStore } from '@repo/bot-kit' import type { QuoteOutcome, SwapPlan } from '@repo/swaps' import type { Address } from 'viem' -import { createBackoff, createCooldownStore } from '@repo/bot-kit' +import { createBackoff, createBlockSampler, createCooldownStore } from '@repo/bot-kit' import { lensKey } from '@repo/utils' import { getAddress } from 'viem' import { describe, expect, it } from 'vitest' @@ -16,7 +16,7 @@ import { ORACLE_PRICE_SCALE, WAD } from '../../src/constants' import { marketId } from '../../src/market' import { runTick } from '../../src/runner/tick' -function spyLogger() { +const spyLogger = () => { const events: { level: string; event: string; fields?: Record }[] = [] const make = (level: string) => (event: string, fields?: Record) => events.push({ level, event, fields }) @@ -43,7 +43,24 @@ const PARAMS: MarketParams = { irm: IRM, lltv: 86n * 10n ** 16n } -const LABEL = lensKey(marketId(PARAMS), BORROWER) +const MARKET_ID = marketId(PARAMS) +const LABEL = lensKey(MARKET_ID, BORROWER) +const SENT: SubmitOutcome = { kind: 'sent' } + +type TickCounters = Awaited> + +/** + * Asserted for EVERY case built by `runWith`, so a stage added without a counter breaks a sum rather + * than silently dropping a position — the exact class of bug these counters exist to catch. + */ +const expectCountersConsistent = (c: TickCounters) => { + expect(c.pairs).toBeGreaterThanOrEqual(c.liquidatable) + expect(c.liquidatable).toBe(c.inflightSkipped + c.planSkipped + c.planned) + expect(c.planned).toBe( + c.cooledDown + c.backoffSkipped + c.noSwapPath + c.quoteFailed + c.ok + c.reverted + ) + expect(c.ok).toBe(c.submitted + c.notSent) +} const SWAP_PLAN: SwapPlan = { steps: [ @@ -62,7 +79,7 @@ const SWAP_PLAN: SwapPlan = { } // A liquidatable reading: valid, has debt, unhealthy, ample collateral (debt-binds → seize > 0). -function lensOut(overrides: Partial = {}): LensOut { +const lensOut = (overrides: Partial = {}): LensOut => { return { params: PARAMS, valid: true, @@ -82,7 +99,7 @@ function lensOut(overrides: Partial = {}): LensOut { const candidates = (...borrowers: Address[]): BorrowerCandidate[] => borrowers.map(borrower => ({ marketParams: PARAMS, borrower })) -function stubReadLens(out: LensOut | null) { +const stubReadLens = (out: LensOut | null) => { return async (pairs: LensInput[]) => { const map = new Map() if (out) for (const pair of pairs) map.set(lensKey(marketId(pair.params), pair.borrower), out) @@ -90,7 +107,7 @@ function stubReadLens(out: LensOut | null) { } } -function runWith(opts: { +type RunOpts = { out?: LensOut | null simulateResult?: SimulateResult quoteOutcome?: QuoteOutcome @@ -101,20 +118,32 @@ function runWith(opts: { noSwap?: boolean seedBackoffAt?: bigint cooldown?: CooldownStore -}) { + /** What the queue reports; defaults to a real broadcast. */ + submitOutcome?: SubmitOutcome + /** Makes `submit` throw, as a hashless send after a nonce was claimed does. */ + submitThrows?: Error + /** Reuse a store/sampler across two `runTick` calls, so cross-tick behavior can be asserted. */ + backoff?: Backoff + planSkipSampler?: BlockSampler +} + +// Shared dep construction so the throwing case exercises exactly the same wiring as `runWith`. +const buildDeps = (opts: RunOpts) => { const { logger, events } = spyLogger() let simulateCalls = 0 let submitCalls = 0 let quoteCalls = 0 const chainHead = opts.chainHead ?? 100n - const backoff = createBackoff({ baseBlocks: 2n, maxBlocks: 64n }) + const backoff = opts.backoff ?? createBackoff({ baseBlocks: 2n, maxBlocks: 64n }) if (opts.seedBackoffAt !== undefined) backoff.record(LABEL, opts.seedBackoffAt) // Default disabled (0) so existing cases are unaffected; opt-in cases pass an enabled store. const cooldown = opts.cooldown ?? createCooldownStore({ cooldownMs: 0 }) + // 0n so every skip explains itself by default; throttling cases pass a real cadence. + const planSkipSampler = opts.planSkipSampler ?? createBlockSampler(0n) const defaultOutcome: QuoteOutcome = opts.noSwap ? { kind: 'no_config' } : { kind: 'swap', plan: SWAP_PLAN } - const result = runTick({ + const deps = { discover: async () => { if (opts.discoverError) throw opts.discoverError return candidates(...(opts.borrowers ?? [BORROWER])) @@ -125,27 +154,48 @@ function runWith(opts: { quoteCalls += 1 return opts.quoteOutcome ?? defaultOutcome }, - simulate: async () => { + simulate: async (): Promise => { simulateCalls += 1 return opts.simulateResult ?? { status: 'ok' } }, submit: async () => { submitCalls += 1 + if (opts.submitThrows) throw opts.submitThrows + return opts.submitOutcome ?? SENT }, backoff, cooldown, - inflightLabels: () => opts.inflight ?? new Set(), + planSkipSampler, + inflightLabels: () => opts.inflight ?? new Set(), logger + } + return { + deps, + probes: { + backoff, + cooldown, + planSkipSampler, + simulateCalls: () => simulateCalls, + submitCalls: () => submitCalls, + quoteCalls: () => quoteCalls, + events + } + } +} + +const runWith = (opts: RunOpts) => { + const { deps, probes } = buildDeps(opts) + return runTick(deps).then(counters => { + expectCountersConsistent(counters) + return { counters, ...probes } }) - return result.then(counters => ({ - counters, - backoff, - cooldown, - simulateCalls: () => simulateCalls, - submitCalls: () => submitCalls, - quoteCalls: () => quoteCalls, - events - })) +} + +/** For the abort path: `runTick` rejects, so counters come from the emitted `tick.end` instead. */ +const runExpectingThrow = async (opts: RunOpts) => { + const { deps, probes } = buildDeps(opts) + await expect(runTick(deps)).rejects.toThrow() + return probes } describe('runTick', () => { @@ -156,14 +206,17 @@ describe('runTick', () => { expect(counters).toEqual({ pairs: 1, liquidatable: 1, + inflightSkipped: 0, + planSkipped: 0, planned: 1, + cooledDown: 0, + backoffSkipped: 0, noSwapPath: 0, quoteFailed: 0, - backoffSkipped: 0, - cooledDown: 0, ok: 1, reverted: 0, - submitted: 1 + submitted: 1, + notSent: 0 }) expect(simulateCalls()).toBe(1) expect(submitCalls()).toBe(1) @@ -227,7 +280,13 @@ describe('runTick', () => { const { counters, quoteCalls, simulateCalls, submitCalls } = await runWith({ inflight: new Set([LABEL]) }) - expect(counters).toMatchObject({ liquidatable: 1, planned: 0, submitted: 0 }) + expect(counters).toMatchObject({ + liquidatable: 1, + inflightSkipped: 1, + planSkipped: 0, + planned: 0, + submitted: 0 + }) expect(quoteCalls()).toBe(0) expect(simulateCalls()).toBe(0) expect(submitCalls()).toBe(0) @@ -248,13 +307,21 @@ describe('runTick', () => { expect(submitCalls()).toBe(0) }) - it('skips a degenerate collateral-less position (plan returns null)', async () => { - const { counters, quoteCalls, submitCalls } = await runWith({ + it('counts a degenerate collateral-less position as planSkipped', async () => { + const { counters, quoteCalls, submitCalls, events } = await runWith({ out: lensOut({ collateral: 0n }) }) - expect(counters).toMatchObject({ liquidatable: 1, planned: 0, submitted: 0 }) + expect(counters).toMatchObject({ + liquidatable: 1, + planSkipped: 1, + planned: 0, + submitted: 0 + }) expect(quoteCalls()).toBe(0) expect(submitCalls()).toBe(0) + const skipped = events.find(e => e.event === 'plan.skipped') + expect(skipped?.level).toBe('info') // documented residual-bad-debt degenerate + expect(skipped?.fields).toMatchObject({ reason: 'no_collateral' }) }) it('tolerates a discovery failure: logs discover.error and submits nothing', async () => { @@ -266,6 +333,165 @@ describe('runTick', () => { expect(submitCalls()).toBe(0) }) + describe('unplannable positions (plan.skipped)', () => { + it('logs plan.skipped with a closed causal chain: sizing inputs AND the derived trace', async () => { + // Dust: a tiny debt against an expensive slot floors the full-debt seize to zero. + const { counters, events } = await runWith({ + out: lensOut({ borrowShares: 1n, collateralPrice: ORACLE_PRICE_SCALE * 10n ** 6n }) + }) + expect(counters).toMatchObject({ liquidatable: 1, planSkipped: 1, planned: 0 }) + const skipped = events.find(e => e.event === 'plan.skipped') + expect(skipped?.level).toBe('info') + expect(skipped?.fields).toMatchObject({ + reason: 'seize_rounds_to_zero', + marketId: MARKET_ID, + borrower: BORROWER, + // derived + seizedAssets: 0n, + // inputs, so the line replays offline + borrowShares: 1n, + collateral: 5000n * WAD, + lltv: PARAMS.lltv + }) + expect(skipped?.fields).toHaveProperty('lif') + expect(skipped?.fields).toHaveProperty('repaidAssetsFull') + expect(skipped?.fields).toHaveProperty('seizeForFullDebt') + }) + + it('warns on a non-reverting zero oracle price', async () => { + const { counters, events } = await runWith({ out: lensOut({ collateralPrice: 0n }) }) + expect(counters).toMatchObject({ liquidatable: 1, planSkipped: 1, planned: 0 }) + const skipped = events.find(e => e.event === 'plan.skipped') + expect(skipped?.level).toBe('warn') // a market anomaly, not routine dust + expect(skipped?.fields).toMatchObject({ reason: 'zero_price' }) + }) + + it('emits one snapshot per sampler window, covering every offender in the tick', async () => { + const planSkipSampler = createBlockSampler(150n) + const other: Address = getAddress('0x9999999999999999999999999999999999999999') + const first = await runWith({ + out: lensOut({ collateral: 0n }), + borrowers: [BORROWER, other], + planSkipSampler, + chainHead: 100n + }) + expect(first.counters).toMatchObject({ liquidatable: 2, planSkipped: 2 }) + expect(first.events.filter(e => e.event === 'plan.skipped')).toHaveLength(2) + + const second = await runWith({ + out: lensOut({ collateral: 0n }), + borrowers: [BORROWER, other], + planSkipSampler, + chainHead: 101n + }) + expect(second.counters).toMatchObject({ planSkipped: 2 }) // still counted at full fidelity + expect(second.events.filter(e => e.event === 'plan.skipped')).toHaveLength(0) + }) + + it('does not consume the sampler window on a tick with nothing to explain', async () => { + const planSkipSampler = createBlockSampler(150n) + const clean = await runWith({ planSkipSampler, chainHead: 100n }) + expect(clean.counters.planSkipped).toBe(0) + const dirty = await runWith({ + out: lensOut({ collateral: 0n }), + planSkipSampler, + chainHead: 101n + }) + expect(dirty.events.filter(e => e.event === 'plan.skipped')).toHaveLength(1) + }) + }) + + describe('submit outcomes', () => { + it('counts a broadcast as submitted and clears the backoff', async () => { + const { counters, backoff } = await runWith({ + seedBackoffAt: 1n, + submitOutcome: { kind: 'sent' } + }) + expect(counters).toMatchObject({ ok: 1, submitted: 1, notSent: 0 }) + expect(backoff.shouldSkip(LABEL, 1n)).toBe(false) + }) + + it('counts notSent and ACCUMULATES the backoff when the send failed for this position', async () => { + const cooldown = createCooldownStore({ cooldownMs: 60_000 }) + const { counters, backoff } = await runWith({ + cooldown, + submitOutcome: { kind: 'failed', scope: 'position', reason: 'submit_failed' } + }) + expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) + expect(backoff.shouldSkip(LABEL, 100n)).toBe(true) + expect(cooldown.shouldSkip(LABEL)).toBe(true) + }) + + it('lets the backoff delay GROW across repeated send failures', async () => { + // `clear` deletes the attempt count, so clearing on a non-broadcast pins the delay at + // `baseBlocks` forever. Seeded at block 1 (attempts=1) then failing at 100 must reach + // attempts=2 → a 4-block wait (until 104), not the 2-block wait a reset would give. + const { backoff } = await runWith({ + seedBackoffAt: 1n, + submitOutcome: { kind: 'failed', scope: 'position', reason: 'submit_failed' } + }) + expect(backoff.shouldSkip(LABEL, 103n)).toBe(true) + expect(backoff.shouldSkip(LABEL, 104n)).toBe(false) + }) + + it.each(['send_aborted', 'nonce_hole', 'nonce_sync_failed'] as const)( + 'counts notSent but does NOT back off a queue-wide refusal (%s)', + async reason => { + const cooldown = createCooldownStore({ cooldownMs: 60_000 }) + const { counters, backoff } = await runWith({ + cooldown, + submitOutcome: { kind: 'failed', scope: 'queue', reason } + }) + expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) + expect(backoff.shouldSkip(LABEL, 100n)).toBe(false) + expect(cooldown.shouldSkip(LABEL)).toBe(false) + } + ) + }) + + describe('tick.end completeness', () => { + it('marks a clean tick complete', async () => { + const { events } = await runWith({}) + expect(events.find(e => e.event === 'tick.end')?.fields).toMatchObject({ + complete: true, + submitted: 1 + }) + }) + + it('still emits tick.end with complete:false when a submit aborts the tick', async () => { + const { events } = await runExpectingThrow({ submitThrows: new Error('rpc timeout') }) + expect(events.find(e => e.event === 'tick.end')?.fields).toMatchObject({ + complete: false, + liquidatable: 1, + planned: 1, + ok: 1, + submitted: 0, + notSent: 0 + }) + }) + }) + + describe('lens.read', () => { + it('counts rows the lens returned as invalid', async () => { + const { counters, events } = await runWith({ out: lensOut({ valid: false }) }) + expect(events.find(e => e.event === 'lens.read')?.fields).toEqual({ + pairs: 1, + returned: 1, + invalid: 1 + }) + expect(counters).toMatchObject({ pairs: 1, liquidatable: 0 }) + }) + + it('reports zero invalid for a healthy batch', async () => { + const { events } = await runWith({}) + expect(events.find(e => e.event === 'lens.read')?.fields).toEqual({ + pairs: 1, + returned: 1, + invalid: 0 + }) + }) + }) + describe('position cooldown (opt-in)', () => { it('skips a cooled-down position without quoting or simulating, counting cooledDown', async () => { const cooldown = createCooldownStore({ cooldownMs: 60_000 }) diff --git a/bots/blue-liquidation/test/sizing/plan.test.ts b/bots/blue-liquidation/test/sizing/plan.test.ts index 50d797a4..9175c9ee 100644 --- a/bots/blue-liquidation/test/sizing/plan.test.ts +++ b/bots/blue-liquidation/test/sizing/plan.test.ts @@ -12,7 +12,7 @@ import { wMulDown, mulDivUp } from '../../src/sizing/math' -import { plan } from '../../src/sizing/plan' +import { plan, planWithReason } from '../../src/sizing/plan' const LLTV = 86n * 10n ** 16n // 0.86e18 @@ -141,3 +141,59 @@ describe('plan — repaidShares ≤ borrowShares (no on-chain underflow)', () => expect(checked).toBeGreaterThan(100) }) }) + +describe('planWithReason', () => { + it.each([ + ['no_debt', { hasDebt: false }], + ['healthy', { healthy: true }], + ['no_collateral', { collateral: 0n }], + ['zero_price', { collateralPrice: 0n }] + ] as const)('reports %s without a trace (decided before any arithmetic)', (reason, overrides) => { + expect(planWithReason(baseInput(overrides))).toEqual({ kind: 'skip', reason }) + }) + + it('reports seize_rounds_to_zero with the full sizing trace', () => { + // Dust debt against a 1e6x-priced slot: the full-debt seize floors to zero collateral. + const outcome = planWithReason( + baseInput({ borrowShares: 1n, collateralPrice: ORACLE_PRICE_SCALE * 10n ** 6n }) + ) + expect(outcome).toEqual({ + kind: 'skip', + reason: 'seize_rounds_to_zero', + trace: { + lif: lifFromLltv(LLTV), + repaidAssetsFull: 0n, + seizeForFullDebt: 0n, + seizedAssets: 0n + } + }) + }) + + it('returns a plan whose seize matches the documented rule', () => { + const input = baseInput() + const outcome = planWithReason(input) + expect(outcome.kind).toBe('plan') + expect(outcome).toMatchObject({ plan: { seizedAssets: plan(input)?.seizedAssets } }) + }) + + describe('plan() facade', () => { + // Pins the facade to the implementation so a later edit to planWithReason cannot silently change + // plan()'s contract (which the whole suite above still asserts through plan()). + const cases: PlanInput[] = [ + baseInput(), + baseInput({ hasDebt: false }), + baseInput({ healthy: true }), + baseInput({ collateral: 0n }), + baseInput({ collateralPrice: 0n }), + baseInput({ borrowShares: 1n, collateralPrice: ORACLE_PRICE_SCALE * 10n ** 6n }) + ] + + it.each(cases.map((input, i) => [i, input] as const))( + 'case %i returns the outcome plan, or null for a skip', + (_i, input) => { + const outcome = planWithReason(input) + expect(plan(input)).toEqual(outcome.kind === 'plan' ? outcome.plan : null) + } + ) + }) +}) diff --git a/bots/midnight-liquidation/README.md b/bots/midnight-liquidation/README.md index 19b3ee36..11b131ab 100644 --- a/bots/midnight-liquidation/README.md +++ b/bots/midnight-liquidation/README.md @@ -400,7 +400,8 @@ The bot computes the oracle-priced reference output for free (no extra API call) route more than `MAX_ROUTE_IMPACT_BPS` below it (`quote.route_quality_failed`). Quote failures (no route, timeout, rate-limited, API error) log `quote.failed`; once every ranked venue is exhausted the position is backed off — an exponential per-position cooldown that bounds API + RPC usage when many -positions fail (the rate-limit defense). A successful submit clears the backoff. +positions fail (the rate-limit defense). Only a successful **broadcast** clears the backoff; a submit +that returned without sending accumulates it instead, so the delay actually grows. If no venue is enabled (bad-debt-only mode) or the collateral is on `EXCLUDE_COLLATERALS`, the tick logs `config.no_swap_path` and skips the candidate (no API call, no backoff). Pure bad-debt @@ -430,14 +431,43 @@ On simulation success, `@repo/bot-kit`'s shared pending queue sends the transaction through the signer client and tracks it by nonce and `(marketId, borrower)` label. -While a label is pending, later ticks skip that position. On each block the queue checks receipts, +While a label is pending, later ticks skip that position and count `inflightSkipped` (that set also +holds labels whose transaction settled within the last `SETTLED_COOLDOWN_BLOCKS`, so the counter can +be non-zero while the queue is empty). On each block the queue checks receipts, logs confirmed or reverted transactions, and fee-bumps stuck transactions until either they confirm, hit the fee ceiling, or exhaust bump attempts. Queue state is in-memory. On restart, chain truth wins: the bot rediscovers live candidates and the -signer nonce cursor starts from the pending chain nonce. If the initial raw broadcast fails after a -nonce is claimed but before a hash is returned, the signer rolls the cursor back and the queue aborts -that tick instead of counting a hashless transaction as submitted. +signer nonce cursor starts from the pending chain nonce. + +`submit` reports whether a transaction actually went out, and `tick.end`'s `submitted` counts only +real broadcasts. The queue has five exits: + +| outcome | logged event | tick effect | +| ------------------------------ | ------------------- | -------------------------------------------- | +| broadcast | `tx.sent` | `submitted`, clears the position's backoff | +| send failed (no nonce claimed) | `tx.submit_failed` | `notSent`, backs the position off + cools it | +| send latched after an abort | `tx.send_aborted` | `notSent` only — a queue-wide refusal | +| nonce cursor unusable | `nonce.sync_failed` | `notSent` only — a queue-wide refusal | +| nonce hole below the cursor | `queue.nonce_hole` | `notSent` only — a queue-wide refusal | + +A queue-wide refusal rejects _every_ send that tick, so backing off the position that happened to be +in hand would suppress healthy positions for several blocks after the latch itself cleared. + +If the initial raw broadcast fails after a nonce is claimed but before a hash is returned, the signer +rolls the cursor back and the queue aborts that tick instead of counting a hashless transaction as +submitted. The tick still emits `tick.end`, with `complete: false` — so partial counters can never be +mistaken for a genuinely idle tick. + +### Why a liquidatable position may not be planned + +A position the chain reports liquidatable can still fail to size. The tick counts every such case as +`planSkipped` and emits a sampled `plan.skipped` carrying the sizing inputs AND the derived numbers +(`lif`, `effectiveDebt`, `cap`, `capEff`, `seizedAssets`), so the decision can be replayed from one +log line. Reasons: `seize_rounds_to_zero` (dust — the largest in-cap seize floors to zero collateral), +`nothing_to_seize` (the best slot is empty), and `cap_not_positive` (a negative RCF numerator, which +should be impossible and therefore logs at `warn`). The diagnostic is sampled at most once per +`PLAN_SKIP_SAMPLE_EVERY_BLOCKS`; the counter is exact on every tick. ## Important Operational Notes diff --git a/bots/midnight-liquidation/src/constants.ts b/bots/midnight-liquidation/src/constants.ts index 9ee04a60..d007b31d 100644 --- a/bots/midnight-liquidation/src/constants.ts +++ b/bots/midnight-liquidation/src/constants.ts @@ -50,3 +50,10 @@ export const LISTED_MARKETS_MAX_AGE_MS = 10 * 60_000 * sizing never depends on the swap-quoting package. */ export const BPS = 10_000n + +/** + * Block cadence bounding the per-position `plan.skipped` diagnostic (~5 min at ~2s Base blocks). An + * unsizable position recurs every tick, so one line per position per block would be ~21k + * lines/hour/bot — all repeating one fact. The `planSkipped` counter stays exact regardless. + */ +export const PLAN_SKIP_SAMPLE_EVERY_BLOCKS = 150n diff --git a/bots/midnight-liquidation/src/index.ts b/bots/midnight-liquidation/src/index.ts index f26da88c..d2407bf1 100644 --- a/bots/midnight-liquidation/src/index.ts +++ b/bots/midnight-liquidation/src/index.ts @@ -5,6 +5,7 @@ import { assertContractDeployed, createBalanceMonitor, createBackoff, + createBlockSampler, createCooldownStore, createDeploylessClient, createHeartbeatMonitor, @@ -34,7 +35,11 @@ import type { Market } from './execution/encode-call' import type { LiquidationPlan } from './sizing/plan' import { loadConfig } from './config' -import { LISTED_MARKETS_MAX_AGE_MS, SETTLED_COOLDOWN_BLOCKS } from './constants' +import { + LISTED_MARKETS_MAX_AGE_MS, + PLAN_SKIP_SAMPLE_EVERY_BLOCKS, + SETTLED_COOLDOWN_BLOCKS +} from './constants' import { createApiCandidateSource, discoverBorrowers, @@ -194,6 +199,9 @@ async function main() { // Opt-in per-position cooldown (default disabled): one in-memory store for the process lifetime, // complementary to `backoff` (see POSITION_LIQUIDATION_COOLDOWN_MS). const cooldown = createCooldownStore({ cooldownMs: config.positionCooldownMs }) + // Bounds the `plan.skipped` diagnostic to one burst per cadence; process-lifetime state, like the + // two stores above. + const planSkipSampler = createBlockSampler(PLAN_SKIP_SAMPLE_EVERY_BLOCKS) // The exec calldata for one liquidation — the same bytes the simulate gate checks and the queue // broadcasts, so a sim-ok plan and its broadcast can't drift. @@ -318,7 +326,9 @@ async function main() { }), submit: async ({ market, borrower, plan, swapPlan, blockNumber, label }) => { const fees = initialFees(await signer.getBaseFee(), config.maxFeeWei, config.priorityFeeWei) - await queue.submit({ + // Returned, not awaited-and-discarded: the tick needs to know whether a tx actually went out + // before it clears this position's backoff. + return queue.submit({ request: { to: config.executooorAddress, data: encodeExec(market, borrower, plan, swapPlan) @@ -331,6 +341,7 @@ async function main() { }, backoff, cooldown, + planSkipSampler, inflightLabels: () => queue.inflightLabels(), logger }) diff --git a/bots/midnight-liquidation/src/runner/tick.ts b/bots/midnight-liquidation/src/runner/tick.ts index a271d4c1..59d985d5 100644 --- a/bots/midnight-liquidation/src/runner/tick.ts +++ b/bots/midnight-liquidation/src/runner/tick.ts @@ -1,4 +1,11 @@ -import type { Backoff, CooldownStore, Logger, SimulateResult } from '@repo/bot-kit' +import type { + Backoff, + BlockSampler, + CooldownStore, + Logger, + SimulateResult, + SubmitOutcome +} from '@repo/bot-kit' import type { QuoteOutcome, SwapPlan } from '@repo/swaps' import type { Address } from 'viem' @@ -6,23 +13,60 @@ import { assertNever, lensKey, tryCatch } from '@repo/utils' import type { BorrowerCandidate } from '../discovery/borrowers' import type { Market } from '../execution/encode-call' -import type { LiquidationPlan } from '../sizing/plan' +import type { LiquidationPlan, PlanSkipReason } from '../sizing/plan' import type { LensInput, LensOut } from '../state/lens.sol' -import { isBadDebtRealization, plan } from '../sizing/plan' +import { isBadDebtRealization, planWithReason } from '../sizing/plan' import { isLiquidatable, planInputFromLens } from './eligibility' +/** + * Per-tick outcome tally, emitted as `tick.end`, ordered as the pipeline runs. For a tick that + * finished (`complete: true`) these identities hold, so a stage added without a counter breaks a sum + * instead of silently dropping a position: + * + * pairs >= liquidatable + * liquidatable === inflightSkipped + planSkipped + planned + * planned === cooledDown + backoffSkipped + noSwapPath + quoteFailed + ok + reverted + * ok === submitted + notSent + * + * On `complete: false` the last is short by one: an aborting `submit` throws after `ok` was counted. + */ type TickCounters = { + /** Lens inputs read this tick — the post-whitelist discovery universe. */ pairs: number + /** Positions the chain says are liquidatable at this block. A market gauge, not a bot decision. */ liquidatable: number + /** + * Skipped because a tx for this label is in flight. The queue's backpressure set also holds labels + * that SETTLED within `settledCooldownBlocks`, so this can be non-zero while the queue is empty. + */ + inflightSkipped: number + /** Skipped because sizing produced no plan — see the `plan.skipped` event for the reason. */ + planSkipped: number planned: number + cooledDown: number + backoffSkipped: number noSwapPath: number quoteFailed: number - backoffSkipped: number - cooledDown: number ok: number reverted: number + /** Broadcast: the queue reported a transaction actually went out. */ submitted: number + /** The queue returned without broadcasting (a send failure, or a queue-wide refusal). */ + notSent: number +} + +/** + * `warn` marks a reason that should be unreachable — the first three negate `isLiquidatable`, and + * `cap_not_positive` means the RCF numerator went negative — so the line reads as a live assertion. + */ +const LEVEL_BY_REASON: Record = { + no_debt: 'warn', + locked: 'warn', + healthy_pre_maturity: 'warn', + cap_not_positive: 'warn', + nothing_to_seize: 'info', + seize_rounds_to_zero: 'info' } /** @@ -37,6 +81,10 @@ type TickCounters = { * Discovery failure is tolerated: a transient error is logged (`discover.error`) and the tick proceeds * with zero new candidates. The lens reads every candidate fresh on-chain, so discovery is a coverage * source, never a correctness dependency. + * + * Every exit from the per-position loop increments exactly one counter, and `tick.end` is emitted even + * when a position aborts the tick — with `complete: false`, so partial counters can never be mistaken + * for a genuinely idle tick. */ export async function runTick(deps: { discover: () => Promise @@ -44,7 +92,7 @@ export async function runTick(deps: { chainHead: bigint /** The Executor singleton — the `liquidate` msg.sender whose gate the lens checks. */ caller: Address - /** Headroom (bps) shaved off a cap-binding seize for one-block oracle-drift; passed to `plan()`. */ + /** Headroom (bps) shaved off a cap-binding seize for one-block oracle-drift; passed to sizing. */ seizeCapMarginBps: number readLens: (pairs: LensInput[]) => Promise> /** @@ -59,7 +107,12 @@ export async function runTick(deps: { plan: LiquidationPlan swapPlan: SwapPlan | null }) => Promise - /** Broadcasts a plan via the pending queue (builds the exec tx, derives fees, tracks the nonce). */ + /** + * Broadcasts a plan via the pending queue (builds the exec tx, derives fees, tracks the nonce). + * Reports whether a transaction actually went out: ONLY `kind: 'sent'` may clear the position's + * backoff. Throws when a send fails after claiming a nonce — the tick aborts by design, so the + * signer's cursor rollback is not raced. + */ submit: (args: { market: Market borrower: Address @@ -67,15 +120,20 @@ export async function runTick(deps: { swapPlan: SwapPlan | null blockNumber: bigint label: string - }) => Promise - /** Per-position exponential backoff suppressing repeated quote/simulate failures (rate-limit defense). */ + }) => Promise + /** Per-position exponential backoff suppressing repeated quote/simulate/send failures (rate-limit defense). */ backoff: Backoff /** * Opt-in per-position cooldown, complementary to `backoff`: a fixed wall-clock window suppressing - * re-attempting a position whose last attempt produced no submittable tx (bad-debt realizations + * re-attempting a position whose last attempt produced no broadcast tx (bad-debt realizations * included). Disabled by default (`POSITION_LIQUIDATION_COOLDOWN_MS=0`) — `shouldSkip` always false. */ cooldown: CooldownStore + /** + * Bounds how often the per-position `plan.skipped` diagnostic is emitted. Asked at most once per + * tick, so every line in a burst shares one `chainHead` and reads as a single snapshot. + */ + planSkipSampler: BlockSampler /** Labels (`${id}:${borrower}`) already in flight — skipped to avoid re-submitting each block. */ inflightLabels: () => ReadonlySet logger: Logger @@ -91,6 +149,7 @@ export async function runTick(deps: { submit, backoff, cooldown, + planSkipSampler, inflightLabels, logger } = deps @@ -108,114 +167,154 @@ export async function runTick(deps: { // 2. Read the lens fresh for the whole batch in one deployless eth_call. const lensOut = await readLens(pairs) - logger.info('lens.read', { pairs: pairs.length, returned: lensOut.size }) + // `returned` always equals `pairs` (the batch lens maps every input row); `invalid` is the + // informative one — a row the lens zeroed (unknown market, reverting oracle) would otherwise be + // indistinguishable from a healthy position inside `pairs - liquidatable`. + let invalid = 0 + for (const out of lensOut.values()) if (!out.valid) invalid += 1 + logger.info('lens.read', { pairs: pairs.length, returned: lensOut.size, invalid }) const counters: TickCounters = { pairs: pairs.length, liquidatable: 0, + inflightSkipped: 0, + planSkipped: 0, planned: 0, + cooledDown: 0, + backoffSkipped: 0, noSwapPath: 0, quoteFailed: 0, - backoffSkipped: 0, - cooledDown: 0, ok: 0, reverted: 0, - submitted: 0 + submitted: 0, + notSent: 0 } // 3. Compose liquidatability off-chain → plan → simulate → submit. `inflight` is captured once; // discovery yields distinct (id, borrower) pairs, so no label repeats within a single tick. const inflight = inflightLabels() - for (const pair of pairs) { - const label = lensKey(pair.id, pair.borrower) - const out = lensOut.get(label) - if (!out || !isLiquidatable(out)) continue - counters.liquidatable += 1 - - // Backpressure: a tx for this position is already pending — don't re-plan/simulate/submit it - // every block while it confirms. - if (inflight.has(label)) continue - - const liquidationPlan = plan(planInputFromLens(out), { seizeCapMarginBps }) - if (!liquidationPlan) continue - counters.planned += 1 - logger.info('plan.built', { - marketId: pair.id, - borrower: pair.borrower, - collateralIndex: liquidationPlan.collateralIndex, - seizedAssets: liquidationPlan.seizedAssets, - repaidUnits: liquidationPlan.repaidUnits, - postMaturityMode: liquidationPlan.postMaturityMode - }) - - // Opt-in cooldown (complementary to backoff): a position whose last attempt produced no - // submittable tx is skipped until its wall-clock window elapses — bad-debt realizations included, - // so a repeatedly-reverting one also backs off. No-op when disabled - // (POSITION_LIQUIDATION_COOLDOWN_MS=0). - if (cooldown.shouldSkip(label)) { - counters.cooledDown += 1 - logger.info('cooldown.skip', { marketId: pair.id, borrower: pair.borrower }) - continue - } + // Claimed once per tick, so a tick emits the whole blocker set or none of it, all sharing one + // `chainHead`. A claim on a skip-free tick is undone by the `reset` in the epilogue below. + const explainSkips = planSkipSampler.claim(chainHead) - // The swap funds repay/seize liquidations. Pure bad-debt realization transfers no assets, so it - // deliberately skips quoting and executes as a no-callback `liquidate`. - let swapPlan: SwapPlan | null = null - if (!isBadDebtRealization(liquidationPlan)) { - // Suppress positions that keep failing to quote/simulate — bounds API + RPC usage under a - // backlog, since executable quotes are spent only on positions not currently backed off. - if (backoff.shouldSkip(label, chainHead)) { - counters.backoffSkipped += 1 + // Emit counters even when a position aborts the tick (a hashless send after the nonce was claimed + // throws by design) — without `complete`, partial counters look exactly like an idle tick. + let complete = false + try { + for (const pair of pairs) { + const label = lensKey(pair.id, pair.borrower) + const out = lensOut.get(label) + if (!out || !isLiquidatable(out)) continue + counters.liquidatable += 1 + + // Backpressure: don't re-plan/simulate/submit while a tx confirms. No log — the queue already + // narrates this label end to end (`tx.sent` → `tx.confirmed`/`tx.reverted`/`tx.dropped`). + if (inflight.has(label)) { + counters.inflightSkipped += 1 continue } - const outcome = await quoteFor(liquidationPlan, out, label) - if (outcome.kind === 'no_config') { - counters.noSwapPath += 1 - cooldown.mark(label) - logger.info('config.no_swap_path', { - marketId: pair.id, - borrower: pair.borrower, - collateralIndex: liquidationPlan.collateralIndex - }) + + const planInput = planInputFromLens(out) + const outcome = planWithReason(planInput, { seizeCapMarginBps }) + if (outcome.kind === 'skip') { + counters.planSkipped += 1 + if (explainSkips) { + // Inputs AND derived trace, so sizing replays from this one line. `marketId`/`borrower` + // last, so a future `PlanInput` field can never shadow them. + logger[LEVEL_BY_REASON[outcome.reason]]('plan.skipped', { + ...planInput, + ...outcome.trace, + marginBps: seizeCapMarginBps, + reason: outcome.reason, + marketId: pair.id, + borrower: pair.borrower + }) + } continue } - if (outcome.kind === 'failed') { - counters.quoteFailed += 1 - backoff.record(label, chainHead) - cooldown.mark(label) + + const liquidationPlan = outcome.plan + counters.planned += 1 + logger.info('plan.built', { + marketId: pair.id, + borrower: pair.borrower, + collateralIndex: liquidationPlan.collateralIndex, + seizedAssets: liquidationPlan.seizedAssets, + repaidUnits: liquidationPlan.repaidUnits, + postMaturityMode: liquidationPlan.postMaturityMode + }) + + // Opt-in cooldown (complementary to backoff): a position whose last attempt produced no + // broadcast tx is skipped until its wall-clock window elapses — bad-debt realizations included, + // so a repeatedly-reverting one also backs off. No-op when disabled + // (POSITION_LIQUIDATION_COOLDOWN_MS=0). + if (cooldown.shouldSkip(label)) { + counters.cooledDown += 1 + logger.info('cooldown.skip', { marketId: pair.id, borrower: pair.borrower }) continue } - swapPlan = outcome.plan - } - const result = await simulate({ - market: out.market, - borrower: pair.borrower, - plan: liquidationPlan, - swapPlan - }) - const fields = { marketId: pair.id, borrower: pair.borrower } - switch (result.status) { - case 'ok': - counters.ok += 1 - logger.info('simulate.ok', fields) - break - case 'revert': - counters.reverted += 1 - // Back off: a sim revert (stale quote, transient unliquidatability) shouldn't re-quote + - // re-simulate this position every block. - backoff.record(label, chainHead) - cooldown.mark(label) - logger.warn('simulate.revert', { ...fields, reason: result.reason }) - break - default: - assertNever(result.status) - } + // Suppress positions that keep failing to quote/simulate/send — bounds API + RPC usage under a + // backlog. Checked for EVERY plan, bad-debt realizations included: they skip quoting but still + // cost a simulation and a send, so a repeatedly-failing one must back off too. + if (backoff.shouldSkip(label, chainHead)) { + counters.backoffSkipped += 1 + continue + } - // ok-only gate (Amendment §10): broadcast only a fully-simulated, swap-funded liquidation. Any - // revert — not-liquidatable, swap slippage, repay shortfall — isn't a fundable plan, so skip it. - if (result.status === 'ok') { - await submit({ + // The swap funds repay/seize liquidations. Pure bad-debt realization transfers no assets, so it + // deliberately skips quoting and executes as a no-callback `liquidate`. + let swapPlan: SwapPlan | null = null + if (!isBadDebtRealization(liquidationPlan)) { + const quote = await quoteFor(liquidationPlan, out, label) + if (quote.kind === 'no_config') { + counters.noSwapPath += 1 + cooldown.mark(label) + logger.info('config.no_swap_path', { + marketId: pair.id, + borrower: pair.borrower, + collateralIndex: liquidationPlan.collateralIndex + }) + continue + } + if (quote.kind === 'failed') { + counters.quoteFailed += 1 + backoff.record(label, chainHead) + cooldown.mark(label) + continue + } + swapPlan = quote.plan + } + + const result = await simulate({ + market: out.market, + borrower: pair.borrower, + plan: liquidationPlan, + swapPlan + }) + const fields = { marketId: pair.id, borrower: pair.borrower } + switch (result.status) { + case 'ok': + counters.ok += 1 + logger.info('simulate.ok', fields) + break + case 'revert': + counters.reverted += 1 + // Back off: a sim revert (stale quote, transient unliquidatability) shouldn't re-quote + + // re-simulate this position every block. + backoff.record(label, chainHead) + cooldown.mark(label) + logger.warn('simulate.revert', { ...fields, reason: result.reason }) + break + default: + assertNever(result.status) + } + + // ok-only gate: broadcast only a fully-simulated, swap-funded liquidation. Any revert — not + // liquidatable, swap slippage, repay shortfall — isn't a fundable plan, so skip it. + if (result.status !== 'ok') continue + + const sendOutcome = await submit({ market: out.market, borrower: pair.borrower, plan: liquidationPlan, @@ -223,11 +322,26 @@ export async function runTick(deps: { blockNumber: chainHead, label }) - backoff.clear(label) - counters.submitted += 1 + if (sendOutcome.kind === 'sent') { + backoff.clear(label) + counters.submitted += 1 + continue + } + counters.notSent += 1 + // Only a failure scoped to THIS position earns a backoff. A queue-scoped refusal rejects every + // send this tick, so backing off here would suppress positions that did nothing wrong, for 2, + // 4, 8… blocks after the latch itself cleared. + if (sendOutcome.scope === 'position') { + backoff.record(label, chainHead) + cooldown.mark(label) + } } + complete = true + } finally { + // Undo the eager claim when nothing was skipped, so the first skip after any quiet stretch is + // always explained (edge-triggering). + if (counters.planSkipped === 0) planSkipSampler.reset() + logger.info('tick.end', { ...counters, complete }) } - - logger.info('tick.end', { ...counters }) return counters } diff --git a/bots/midnight-liquidation/src/sizing/plan.ts b/bots/midnight-liquidation/src/sizing/plan.ts index 28bbd4b5..91a023db 100644 --- a/bots/midnight-liquidation/src/sizing/plan.ts +++ b/bots/midnight-liquidation/src/sizing/plan.ts @@ -52,15 +52,63 @@ type PlanOptions = { seizeCapMarginBps?: number } -export function isBadDebtRealization(plan: LiquidationPlan): boolean { - return plan.seizedAssets === 0n && plan.repaidUnits === 0n +/** + * Why {@link planWithReason} produced no plan. Reasons discriminate SEVERITY (hence log level); the + * numbers on {@link SizingTrace} discriminate cause — so there is no `margin_ate_cap` reason, because + * `cap` vs `capEff` vs `seizedAssets` already pin that. + * + * The first three are UNREACHABLE from the tick: `isLiquidatable` tests the same lens fields, and + * `!postMaturityMode && healthy` is the exact negation of its third clause. Observing one means the + * eligibility gate and this module have diverged, so callers log them at `warn`. + */ +export type PlanSkipReason = + | 'no_debt' + | 'locked' + | 'healthy_pre_maturity' + /** Best slot holds nothing, and the position is not a full bad-debt write-off. */ + | 'nothing_to_seize' + /** The repay cap is zero or negative — see the guard in {@link capBoundOutcome}. */ + | 'cap_not_positive' + /** The expected dust case: a positive cap whose largest in-cap seize floors to zero collateral. */ + | 'seize_rounds_to_zero' + +/** + * The derived values between a lens reading and a refused seize, so an operator can replay the + * decision from one log line. The normal-mode-only fields are `undefined` post-maturity, and the + * logger drops `undefined`, so a line self-describes its mode. + */ +type SizingTrace = { + postMaturityMode: boolean + lif: bigint + /** `debt - badDebt`: the post-writeoff debt every cap is taken against. */ + effectiveDebt: bigint + /** The seize that was refused — `0n`, or negative when the cap went negative. */ + seizedAssets: bigint + /** The repay bound before `seizeCapMarginBps`. Absent when the cap stage never ran. */ + cap?: bigint + /** `cap` after the margin — separates "the margin ate the cap" from "the division floored". */ + capEff?: bigint + maxRepaid?: bigint + rcfExempt?: boolean + /** `lltv >= WAD` waives the RCF cap, so `maxRepaid` is omitted rather than logged as maxUint256. */ + rcfDisabled?: boolean } +/** A plan, or the reason there is none plus the numbers that explain it. */ +type PlanOutcome = + | { kind: 'plan'; plan: LiquidationPlan } + | { kind: 'skip'; reason: PlanSkipReason; trace?: SizingTrace } + +/** Trace fields fixed once the mode is chosen, before the cap stage runs. */ +type TraceBase = Omit + +export const isBadDebtRealization = (plan: LiquidationPlan): boolean => + plan.seizedAssets === 0n && plan.repaidUnits === 0n + // Repaid units the contract derives when the caller passes `seizedAssets` (midnight-contracts.txt:2369): // two chained ceil-divisions, collateral → loan units → repaid units. -function impliedRepaidUnits(seizedAssets: bigint, price: bigint, lif: bigint): bigint { - return mulDivUp(mulDivUp(seizedAssets, price, ORACLE_PRICE_SCALE), WAD, lif) -} +const impliedRepaidUnits = (seizedAssets: bigint, price: bigint, lif: bigint): bigint => + mulDivUp(mulDivUp(seizedAssets, price, ORACLE_PRICE_SCALE), WAD, lif) /** * The largest seize `S` whose contract-derived repaid (`impliedRepaidUnits(S, price, lif)`) stays @@ -72,65 +120,171 @@ function impliedRepaidUnits(seizedAssets: bigint, price: bigint, lif: bigint): b * overshoots: `impliedRepaidUnits(maxSeizeForCap(cap, …)) <= cap` always holds, and the result is the * largest such seize (`impliedRepaidUnits(result + 1) > cap`). */ -export function maxSeizeForCap(cap: bigint, price: bigint, lif: bigint): bigint { +export const maxSeizeForCap = (cap: bigint, price: bigint, lif: bigint): bigint => { if (cap === 0n || price === 0n) return 0n return mulDivDown(mulDivDown(cap, lif, WAD), ORACLE_PRICE_SCALE, price) } -// Builds a cap-binding seize-exact plan: seize the largest amount whose contract-derived repaid stays -// within `cap`, after shaving `marginBps` off the cap for one-block drift headroom. Returns null when -// that rounds to zero — never a `(0, 0)` plan, which `isBadDebtRealization` would misclassify as a -// bad-debt write-off against a solvent position. -function capBoundPlan( - input: PlanInput, - cap: bigint, - lif: bigint, - marginBps: number, - postMaturityMode: boolean -): LiquidationPlan | null { +/** + * Sizes a cap-binding seize-exact plan: seize the largest amount whose contract-derived repaid stays + * within `cap`, after shaving `marginBps` off the cap for one-block drift headroom. + */ +const capBoundOutcome = ({ + input, + cap, + marginBps, + base +}: { + input: PlanInput + cap: bigint + marginBps: number + base: TraceBase +}): PlanOutcome => { const capEff = mulDivDown(cap, BPS - BigInt(marginBps), BPS) - const seizedAssets = maxSeizeForCap(capEff, input.bestCollateralPrice, lif) - if (seizedAssets === 0n) return null - return { + const seizedAssets = maxSeizeForCap(capEff, input.bestCollateralPrice, base.lif) + if (cap > 0n && seizedAssets > 0n) { + return { + kind: 'plan', + plan: { + collateralIndex: input.bestCollateralIndex, + seizedAssets, + repaidUnits: 0n, + postMaturityMode: base.postMaturityMode + } + } + } + // The trace is built only on the refusal paths, so the plan-success path allocates nothing extra. + const trace: SizingTrace = { ...base, cap, capEff, seizedAssets } + // Discriminate on the RAW cap, not `capEff`: a legitimate 1-wei cap with any margin floors capEff + // to 0, which is ordinary dust. A non-positive RAW cap means the RCF numerator went negative + // (`debt - maxDebt < badDebt < debt`), which also yields a NEGATIVE seize that would slip past an + // `=== 0n` check and revert opaquely once abi-encoded as uint256 — hence the `> 0n` gate above. + if (cap <= 0n) return { kind: 'skip', reason: 'cap_not_positive', trace } + // Rounds to nothing. Never emit a `(0, 0)` plan, which `isBadDebtRealization` would misread as a + // write-off against a solvent position. + return { kind: 'skip', reason: 'seize_rounds_to_zero', trace } +} + +const seizeWholeSlot = (input: PlanInput, postMaturityMode: boolean): PlanOutcome => ({ + kind: 'plan', + plan: { collateralIndex: input.bestCollateralIndex, - seizedAssets, + seizedAssets: input.bestCollateralAmt, repaidUnits: 0n, postMaturityMode } +}) + +/** Everything both mode branches need, derived once in {@link planWithReason}. */ +type ModeStage = { + input: PlanInput + marginBps: number + lif: bigint + /** `debt - badDebt`: the post-writeoff debt every cap is taken against. */ + effectiveDebt: bigint + /** Implied repaid units if the whole best slot were seized. */ + wholeSlotRepaid: bigint +} + +/** + * Post-maturity mode: the RCF cap does not apply, but the contract still subtracts `repaidUnits` from + * the post-writeoff debt with no clamp, so over-repaying reverts (Panic 0x11 underflow). Seizing the + * whole slot is therefore correct only while its implied repaid fits within that debt — the underwater + * case. Otherwise (a solvent borrower who simply missed maturity) fall back to the cap. + */ +const postMaturityOutcome = ({ + input, + marginBps, + lif, + effectiveDebt, + wholeSlotRepaid +}: ModeStage): PlanOutcome => { + if (wholeSlotRepaid <= effectiveDebt) return seizeWholeSlot(input, true) + const base: TraceBase = { postMaturityMode: true, lif, effectiveDebt } + return capBoundOutcome({ input, cap: effectiveDebt, marginBps, base }) } /** - * Turns a fresh lens reading into a liquidation plan, or `null` when the position is not - * liquidatable. Mirrors the mode and amount policy of `liquidate(...)`: + * Normal mode: same no-clamp subtraction as post-maturity, but the repay is additionally bounded by + * the RCF cap (waived when the slot is rcf-exempt), never exceeding the post-writeoff debt. + */ +const normalModeOutcome = ({ + input, + marginBps, + lif, + effectiveDebt, + wholeSlotRepaid +}: ModeStage): PlanOutcome => { + const maxRepaid = maxRepaidPreMaturity({ + debt: input.debt, + badDebt: input.badDebt, + maxDebt: input.maxDebt, + lif, + lltv: input.bestCollateralLltv + }) + const base: TraceBase = { + postMaturityMode: false, + lif, + effectiveDebt, + // `lltv >= WAD` waives the cap and `maxRepaidPreMaturity` returns maxUint256; flag that instead + // of logging a 78-digit number that reads like a real bound. + ...(input.bestCollateralLltv >= WAD ? { rcfDisabled: true } : { maxRepaid }) + } + if (maxRepaid <= 0n) return capBoundOutcome({ input, cap: maxRepaid, marginBps, base }) + const exempt = isRcfExempt({ + collateralAmt: input.bestCollateralAmt, + price: input.bestCollateralPrice, + lif, + maxRepaid, + rcfThreshold: input.rcfThreshold + }) + const repayCap = exempt ? effectiveDebt : min(maxRepaid, effectiveDebt) + if (wholeSlotRepaid <= repayCap) return seizeWholeSlot(input, false) + return capBoundOutcome({ input, cap: repayCap, marginBps, base: { ...base, rcfExempt: exempt } }) +} + +/** + * Turns a fresh lens reading into a liquidation plan, or a {@link PlanSkipReason} explaining why + * there is none. Mirrors the mode and amount policy of `liquidate(...)`: * * - past maturity → post-maturity mode: no RCF cap; seize 100% of the best slot if its implied - * repaid units fit within the (post-writeoff) debt, else seize the largest amount whose - * contract-derived repaid stays within that debt; - * - pre-maturity & unhealthy → normal mode: seize 100% of the slot when its implied repaid units - * fit within the repay bound — the RCF cap (waived when rcf-exempt) clamped to the post-writeoff - * debt — otherwise seize the largest amount whose contract-derived repaid stays within that bound; + * repaid units fit within the (post-writeoff) debt, else the largest amount that does; + * - pre-maturity & unhealthy → normal mode: the same, bounded by the RCF cap (waived when + * rcf-exempt) clamped to the post-writeoff debt; * - otherwise (no debt, locked, or healthy-and-pre-maturity) → skip. * * Every non-bad-debt plan is **seize-exact**: it pins `seizedAssets` (with `repaidUnits = 0`) and the * contract ceil-derives `repaidUnits` (:2369). Pinning the seize means the Executor holds exactly what * every venue (Uniswap or aggregator) sells, so there is no sell-side drift. Bad-debt realization is * the only `(0, 0)` plan. A cap-binding seize is sized against `cap·(1 - seizeCapMarginBps)` to keep - * headroom for a one-block oracle move; any residual drift fails closed in `simulate()`, never on-chain. + * headroom for a one-block oracle move; any residual drift fails closed in `simulate()`, never + * on-chain. + * + * @param input - Fresh lens-derived position state, all read at one `blockTimestamp`. + * @param options - Non-lens sizing knobs; `seizeCapMarginBps` defaults to `0` (unmargined cap). + * @returns `{ kind: 'plan' }` with a submittable plan, or `{ kind: 'skip' }` with the reason and — + * for every cap-stage refusal — a {@link SizingTrace} sufficient to replay the arithmetic. */ -export function plan(input: PlanInput, options: PlanOptions = {}): LiquidationPlan | null { - const { seizeCapMarginBps = 0 } = options +export const planWithReason = (input: PlanInput, options: PlanOptions = {}): PlanOutcome => { + const { seizeCapMarginBps: marginBps = 0 } = options - if (!input.hasDebt || input.locked) return null + if (!input.hasDebt) return { kind: 'skip', reason: 'no_debt' } + if (input.locked) return { kind: 'skip', reason: 'locked' } const postMaturityMode = input.blockTimestamp > input.maturity - if (!postMaturityMode && input.healthy) return null + if (!postMaturityMode && input.healthy) return { kind: 'skip', reason: 'healthy_pre_maturity' } + // Full write-off: the only legitimate `(0, 0)` plan. Must precede the `nothing_to_seize` guard, + // which would otherwise steal it (a fully-bad-debt position often holds no collateral either). if (input.badDebt >= input.debt) { return { - collateralIndex: input.bestCollateralIndex, - seizedAssets: 0n, - repaidUnits: 0n, - postMaturityMode + kind: 'plan', + plan: { + collateralIndex: input.bestCollateralIndex, + seizedAssets: 0n, + repaidUnits: 0n, + postMaturityMode + } } } @@ -140,59 +294,37 @@ export function plan(input: PlanInput, options: PlanOptions = {}): LiquidationPl maxLif: input.bestCollateralMaxLif, postMaturityMode }) + const effectiveDebt = input.debt - input.badDebt - const seizeWholeSlot: LiquidationPlan = { - collateralIndex: input.bestCollateralIndex, - seizedAssets: input.bestCollateralAmt, - repaidUnits: 0n, - postMaturityMode - } - - // Post-maturity mode: the RCF cap does not apply, but the contract still subtracts `repaidUnits` - // from the (post-writeoff) debt with no clamp, so over-repaying reverts (Panic 0x11 underflow). - // Seizing the whole slot is correct only while its implied repaid units fit within the debt — the - // underwater case. When the slot is worth more than the debt (the common case: a solvent borrower - // who simply missed maturity), seize the largest amount whose contract-derived repaid stays within - // that debt. `badDebt` is written off before the repay, so the cap is the post-writeoff debt - // (`debt - badDebt`). - if (postMaturityMode) { - const effectiveDebt = input.debt - input.badDebt - const wholeSlotRepaid = impliedRepaidUnits( - input.bestCollateralAmt, - input.bestCollateralPrice, - lif - ) - if (wholeSlotRepaid <= effectiveDebt) return seizeWholeSlot - return capBoundPlan(input, effectiveDebt, lif, seizeCapMarginBps, postMaturityMode) + // An empty best slot would make `wholeSlotRepaid` 0, pass every cap comparison, and return a + // `(0, 0)` whole-slot plan that `isBadDebtRealization` reads as a write-off against a position we + // just established is NOT fully bad debt. Refuse it explicitly instead. + if (input.bestCollateralAmt === 0n) { + return { + kind: 'skip', + reason: 'nothing_to_seize', + // No cap stage ran, so `cap`/`capEff` are absent rather than fabricated. + trace: { postMaturityMode, lif, effectiveDebt, seizedAssets: 0n } + } } - // Normal mode: like post-maturity above, the contract subtracts `repaidUnits` from the - // post-writeoff debt with no clamp, so an implied repay above it reverts (Panic 0x11). The repay is - // bounded by the RCF cap (waived when the slot is rcf-exempt) AND never exceeds that debt. Seize - // the whole slot only when its implied repaid units fit within the bound; otherwise seize the - // largest amount whose contract-derived repaid stays within that bound. - const effectiveDebt = input.debt - input.badDebt - const maxRepaid = maxRepaidPreMaturity({ - debt: input.debt, - badDebt: input.badDebt, - maxDebt: input.maxDebt, - lif, - lltv: input.bestCollateralLltv - }) const wholeSlotRepaid = impliedRepaidUnits( input.bestCollateralAmt, input.bestCollateralPrice, lif ) - const exempt = isRcfExempt({ - collateralAmt: input.bestCollateralAmt, - price: input.bestCollateralPrice, - lif, - maxRepaid, - rcfThreshold: input.rcfThreshold - }) - const repayCap = exempt ? effectiveDebt : min(maxRepaid, effectiveDebt) + const stage: ModeStage = { input, marginBps, lif, effectiveDebt, wholeSlotRepaid } + return postMaturityMode ? postMaturityOutcome(stage) : normalModeOutcome(stage) +} - if (wholeSlotRepaid <= repayCap) return seizeWholeSlot - return capBoundPlan(input, repayCap, lif, seizeCapMarginBps, postMaturityMode) +/** + * Bare-plan facade over {@link planWithReason} for callers that do not report a reason. + * + * @param input - Fresh lens-derived position state. + * @param options - Non-lens sizing knobs. + * @returns The plan, or `null` when the position is not liquidatable or cannot be sized. + */ +export const plan = (input: PlanInput, options: PlanOptions = {}): LiquidationPlan | null => { + const outcome = planWithReason(input, options) + return outcome.kind === 'plan' ? outcome.plan : null } diff --git a/bots/midnight-liquidation/test/runner/tick.test.ts b/bots/midnight-liquidation/test/runner/tick.test.ts index ee826b01..9a03fc62 100644 --- a/bots/midnight-liquidation/test/runner/tick.test.ts +++ b/bots/midnight-liquidation/test/runner/tick.test.ts @@ -1,9 +1,9 @@ -import type { Logger, SimulateResult } from '@repo/bot-kit' -import type { CooldownStore } from '@repo/bot-kit' +import type { Logger, SimulateResult, SubmitOutcome } from '@repo/bot-kit' +import type { Backoff, BlockSampler, CooldownStore } from '@repo/bot-kit' import type { QuoteOutcome, SwapPlan } from '@repo/swaps' import type { Address, Hex } from 'viem' -import { createBackoff, createCooldownStore } from '@repo/bot-kit' +import { createBackoff, createBlockSampler, createCooldownStore } from '@repo/bot-kit' import { lensKey } from '@repo/utils' import { getAddress } from 'viem' import { describe, expect, it } from 'vitest' @@ -13,7 +13,7 @@ import type { LensInput, LensOut } from '../../src/state/lens.sol' import { runTick } from '../../src/runner/tick' -function spyLogger() { +const spyLogger = () => { const events: { level: string; event: string; fields?: Record }[] = [] const make = (level: string) => (event: string, fields?: Record) => events.push({ level, event, fields }) @@ -34,6 +34,22 @@ const ROUTER: Address = getAddress('0x5555555555555555555555555555555555555555') const ZERO = '0x0000000000000000000000000000000000000000' as const const MARKET: Hex = `0x${'a'.repeat(64)}` const LABEL = lensKey(MARKET, BORROWER) +const SENT: SubmitOutcome = { kind: 'sent' } + +type TickCounters = Awaited> + +/** + * Asserted for EVERY case built by `runWith`, so a stage added without a counter breaks a sum rather + * than silently dropping a position — the exact class of bug these counters exist to catch. + */ +const expectCountersConsistent = (c: TickCounters) => { + expect(c.pairs).toBeGreaterThanOrEqual(c.liquidatable) + expect(c.liquidatable).toBe(c.inflightSkipped + c.planSkipped + c.planned) + expect(c.planned).toBe( + c.cooledDown + c.backoffSkipped + c.noSwapPath + c.quoteFailed + c.ok + c.reverted + ) + expect(c.ok).toBe(c.submitted + c.notSent) +} const SWAP_PLAN: SwapPlan = { steps: [ @@ -52,7 +68,7 @@ const SWAP_PLAN: SwapPlan = { } // A liquidatable reading: valid, gate open, has debt, unlocked, unhealthy, pre-maturity. -function lensOut(overrides: Partial = {}): LensOut { +const lensOut = (overrides: Partial = {}): LensOut => { return { valid: true, hasDebt: true, @@ -93,7 +109,7 @@ function lensOut(overrides: Partial = {}): LensOut { const candidates = (...borrowers: Address[]): BorrowerCandidate[] => borrowers.map(borrower => ({ marketId: MARKET, borrower })) -function stubReadLens(out: LensOut | null) { +const stubReadLens = (out: LensOut | null) => { return async (pairs: LensInput[]) => { const map = new Map() if (out) for (const pair of pairs) map.set(lensKey(pair.id, pair.borrower), out) @@ -101,7 +117,7 @@ function stubReadLens(out: LensOut | null) { } } -function runWith(opts: { +type RunOpts = { out?: LensOut | null simulateResult?: SimulateResult quoteOutcome?: QuoteOutcome @@ -112,53 +128,87 @@ function runWith(opts: { noSwap?: boolean seedBackoffAt?: bigint cooldown?: CooldownStore -}) { + seizeCapMarginBps?: number + /** What the queue reports; defaults to a real broadcast. */ + submitOutcome?: SubmitOutcome + /** Makes `submit` throw, as a hashless send after a nonce was claimed does. */ + submitThrows?: Error + /** Reuse a store/sampler across two `runTick` calls, so cross-tick behavior can be asserted. */ + backoff?: Backoff + planSkipSampler?: BlockSampler +} + +// Shared dep construction so the throwing case exercises exactly the same wiring as `runWith`. +const buildDeps = (opts: RunOpts) => { const { logger, events } = spyLogger() let simulateCalls = 0 let submitCalls = 0 let quoteCalls = 0 const chainHead = opts.chainHead ?? 100n - const backoff = createBackoff({ baseBlocks: 2n, maxBlocks: 64n }) + const backoff = opts.backoff ?? createBackoff({ baseBlocks: 2n, maxBlocks: 64n }) if (opts.seedBackoffAt !== undefined) backoff.record(LABEL, opts.seedBackoffAt) // Default disabled (0) so existing cases are unaffected; opt-in cases pass an enabled store. const cooldown = opts.cooldown ?? createCooldownStore({ cooldownMs: 0 }) + // 0n so every skip explains itself by default; throttling cases pass a real cadence. + const planSkipSampler = opts.planSkipSampler ?? createBlockSampler(0n) const defaultOutcome: QuoteOutcome = opts.noSwap ? { kind: 'no_config' } : { kind: 'swap', plan: SWAP_PLAN } - const result = runTick({ + const deps = { discover: async () => { if (opts.discoverError) throw opts.discoverError return candidates(...(opts.borrowers ?? [BORROWER])) }, chainHead, caller: CALLER, - seizeCapMarginBps: 0, + seizeCapMarginBps: opts.seizeCapMarginBps ?? 0, readLens: stubReadLens(opts.out === undefined ? lensOut() : opts.out), quoteFor: async () => { quoteCalls += 1 return opts.quoteOutcome ?? defaultOutcome }, - simulate: async () => { + simulate: async (): Promise => { simulateCalls += 1 return opts.simulateResult ?? { status: 'ok' } }, submit: async () => { submitCalls += 1 + if (opts.submitThrows) throw opts.submitThrows + return opts.submitOutcome ?? SENT }, backoff, cooldown, - inflightLabels: () => opts.inflight ?? new Set(), + planSkipSampler, + inflightLabels: () => opts.inflight ?? new Set(), logger + } + return { + deps, + probes: { + backoff, + cooldown, + planSkipSampler, + simulateCalls: () => simulateCalls, + submitCalls: () => submitCalls, + quoteCalls: () => quoteCalls, + events + } + } +} + +const runWith = (opts: RunOpts) => { + const { deps, probes } = buildDeps(opts) + return runTick(deps).then(counters => { + expectCountersConsistent(counters) + return { counters, ...probes } }) - return result.then(counters => ({ - counters, - backoff, - cooldown, - simulateCalls: () => simulateCalls, - submitCalls: () => submitCalls, - quoteCalls: () => quoteCalls, - events - })) +} + +/** For the abort path: `runTick` rejects, so counters come from the emitted `tick.end` instead. */ +const runExpectingThrow = async (opts: RunOpts) => { + const { deps, probes } = buildDeps(opts) + await expect(runTick(deps)).rejects.toThrow() + return probes } describe('runTick', () => { @@ -169,14 +219,17 @@ describe('runTick', () => { expect(counters).toEqual({ pairs: 1, liquidatable: 1, + inflightSkipped: 0, + planSkipped: 0, planned: 1, + cooledDown: 0, + backoffSkipped: 0, noSwapPath: 0, quoteFailed: 0, - backoffSkipped: 0, - cooledDown: 0, ok: 1, reverted: 0, - submitted: 1 + submitted: 1, + notSent: 0 }) expect(simulateCalls()).toBe(1) expect(submitCalls()).toBe(1) @@ -262,7 +315,13 @@ describe('runTick', () => { const { counters, quoteCalls, simulateCalls, submitCalls } = await runWith({ inflight: new Set([LABEL]) }) - expect(counters).toMatchObject({ liquidatable: 1, planned: 0, submitted: 0 }) + expect(counters).toMatchObject({ + liquidatable: 1, + inflightSkipped: 1, + planSkipped: 0, + planned: 0, + submitted: 0 + }) expect(quoteCalls()).toBe(0) expect(simulateCalls()).toBe(0) expect(submitCalls()).toBe(0) @@ -292,6 +351,220 @@ describe('runTick', () => { expect(submitCalls()).toBe(0) }) + describe('unplannable positions (plan.skipped)', () => { + // Post-maturity dust: 1 wei of debt against a high-priced slot, so the largest in-cap seize + // rounds to 0 collateral. This is the production signature that went unexplained for 5 days. + const DUST = () => + lensOut({ + healthy: true, + blockTimestamp: 3000n, + debt: 1n, + badDebt: 0n, + bestCollateralAmt: 10n ** 18n, + bestCollateralPrice: 10n ** 37n + }) + + it('counts planSkipped instead of silently dropping a liquidatable position', async () => { + const { counters, quoteCalls, simulateCalls, submitCalls } = await runWith({ out: DUST() }) + expect(counters).toMatchObject({ + liquidatable: 1, + inflightSkipped: 0, + planSkipped: 1, + planned: 0, + submitted: 0 + }) + expect(quoteCalls()).toBe(0) + expect(simulateCalls()).toBe(0) + expect(submitCalls()).toBe(0) + }) + + it('logs plan.skipped with a closed causal chain: sizing inputs AND the derived trace', async () => { + const { events } = await runWith({ out: DUST() }) + const skipped = events.find(e => e.event === 'plan.skipped') + expect(skipped?.level).toBe('info') // ordinary dust, not an invariant violation + expect(skipped?.fields).toMatchObject({ + reason: 'seize_rounds_to_zero', + marketId: MARKET, + borrower: BORROWER, + // derived (unknowable without replaying plan()) + postMaturityMode: true, + lif: 1027777777777777777n, + effectiveDebt: 1n, + cap: 1n, + capEff: 1n, + seizedAssets: 0n, + marginBps: 0, + // inputs (so the line can be replayed offline) + debt: 1n, + badDebt: 0n, + maturity: 2000n, + bestCollateralAmt: 10n ** 18n, + bestCollateralPrice: 10n ** 37n + }) + }) + + it('distinguishes the margin eating the cap from the division flooring', async () => { + // Same 1-wei cap, but a 30bps margin floors capEff to 0: same reason, different numbers. Shows + // the field set discriminates the sub-causes. + const { events } = await runWith({ out: DUST(), seizeCapMarginBps: 30 }) + expect(events.find(e => e.event === 'plan.skipped')?.fields).toMatchObject({ + reason: 'seize_rounds_to_zero', + cap: 1n, + capEff: 0n, + marginBps: 30 + }) + }) + + it('warns on a non-positive cap and refuses the negative-seize plan', async () => { + // debt - maxDebt (100) < badDebt (500) < debt (1000) makes the RCF numerator negative, so the + // cap and the seize both go negative. Pre-fix this returned a NEGATIVE-seize plan. + // rcfThreshold 0 so the slot is NOT rcf-exempt and the negative RCF cap actually binds. + const { counters, events, submitCalls } = await runWith({ + out: lensOut({ + debt: 1000n, + badDebt: 500n, + maxDebt: 900n, + market: { ...lensOut().market, rcfThreshold: 0n } + }), + chainHead: 100n + }) + expect(counters).toMatchObject({ liquidatable: 1, planSkipped: 1, planned: 0 }) + const skipped = events.find(e => e.event === 'plan.skipped') + expect(skipped?.level).toBe('warn') // should be impossible → alertable + expect(skipped?.fields).toMatchObject({ reason: 'cap_not_positive', cap: -7406n }) + expect(submitCalls()).toBe(0) + }) + + it('emits one snapshot per sampler window, covering every offender in the tick', async () => { + // Cadence of 150 blocks and two offenders: both explain on the first tick, neither on the next. + const planSkipSampler = createBlockSampler(150n) + const other: Address = getAddress('0x9999999999999999999999999999999999999999') + const first = await runWith({ + out: DUST(), + borrowers: [BORROWER, other], + planSkipSampler, + chainHead: 100n + }) + expect(first.counters).toMatchObject({ liquidatable: 2, planSkipped: 2 }) + expect(first.events.filter(e => e.event === 'plan.skipped')).toHaveLength(2) + + const second = await runWith({ + out: DUST(), + borrowers: [BORROWER, other], + planSkipSampler, + chainHead: 101n + }) + expect(second.counters).toMatchObject({ planSkipped: 2 }) // still counted at full fidelity + expect(second.events.filter(e => e.event === 'plan.skipped')).toHaveLength(0) + }) + + it('does not consume the sampler window on a tick with nothing to explain', async () => { + // The edge-trigger guarantee: a clean tick must not spend the window, so the first skip after + // any quiet stretch is always reported. + const planSkipSampler = createBlockSampler(150n) + const clean = await runWith({ planSkipSampler, chainHead: 100n }) + expect(clean.counters.planSkipped).toBe(0) + const dirty = await runWith({ out: DUST(), planSkipSampler, chainHead: 101n }) + expect(dirty.events.filter(e => e.event === 'plan.skipped')).toHaveLength(1) + }) + }) + + describe('submit outcomes', () => { + it('counts a broadcast as submitted and clears the backoff', async () => { + const { counters, backoff } = await runWith({ + seedBackoffAt: 1n, + submitOutcome: { kind: 'sent' } + }) + expect(counters).toMatchObject({ ok: 1, submitted: 1, notSent: 0 }) + expect(backoff.shouldSkip(LABEL, 1n)).toBe(false) + }) + + it('counts notSent and ACCUMULATES the backoff when the send failed for this position', async () => { + const cooldown = createCooldownStore({ cooldownMs: 60_000 }) + const { counters, backoff } = await runWith({ + cooldown, + submitOutcome: { kind: 'failed', scope: 'position', reason: 'submit_failed' } + }) + expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) + expect(backoff.shouldSkip(LABEL, 100n)).toBe(true) + expect(cooldown.shouldSkip(LABEL)).toBe(true) + }) + + it('lets the backoff delay GROW across repeated send failures', async () => { + // `clear` deletes the attempt count, so clearing on a non-broadcast pins the delay at + // `baseBlocks` forever. Seeded at block 1 (attempts=1) then failing at 100 must reach + // attempts=2 → a 4-block wait (until 104), not the 2-block wait a reset would give. + const { backoff } = await runWith({ + seedBackoffAt: 1n, + submitOutcome: { kind: 'failed', scope: 'position', reason: 'submit_failed' } + }) + expect(backoff.shouldSkip(LABEL, 103n)).toBe(true) + expect(backoff.shouldSkip(LABEL, 104n)).toBe(false) + }) + + it.each(['send_aborted', 'nonce_hole', 'nonce_sync_failed'] as const)( + 'counts notSent but does NOT back off a queue-wide refusal (%s)', + async reason => { + // These refuse EVERY send this tick, so attributing one to the position in hand would + // suppress positions that did nothing wrong. + const cooldown = createCooldownStore({ cooldownMs: 60_000 }) + const { counters, backoff } = await runWith({ + cooldown, + submitOutcome: { kind: 'failed', scope: 'queue', reason } + }) + expect(counters).toMatchObject({ ok: 1, submitted: 0, notSent: 1 }) + expect(backoff.shouldSkip(LABEL, 100n)).toBe(false) + expect(cooldown.shouldSkip(LABEL)).toBe(false) + } + ) + }) + + describe('tick.end completeness', () => { + it('marks a clean tick complete', async () => { + const { events } = await runWith({}) + expect(events.find(e => e.event === 'tick.end')?.fields).toMatchObject({ + complete: true, + submitted: 1 + }) + }) + + it('still emits tick.end with complete:false when a submit aborts the tick', async () => { + // A hashless send after the nonce was claimed throws by design; pre-fix the throw escaped + // before `tick.end`, so the tick's counters vanished. + const { events } = await runExpectingThrow({ submitThrows: new Error('rpc timeout') }) + const end = events.find(e => e.event === 'tick.end') + expect(end?.fields).toMatchObject({ + complete: false, + liquidatable: 1, + planned: 1, + ok: 1, + submitted: 0, + notSent: 0 + }) + }) + }) + + describe('lens.read', () => { + it('counts rows the lens returned as invalid', async () => { + const { counters, events } = await runWith({ out: lensOut({ valid: false }) }) + expect(events.find(e => e.event === 'lens.read')?.fields).toEqual({ + pairs: 1, + returned: 1, + invalid: 1 + }) + expect(counters).toMatchObject({ pairs: 1, liquidatable: 0 }) + }) + + it('reports zero invalid for a healthy batch', async () => { + const { events } = await runWith({}) + expect(events.find(e => e.event === 'lens.read')?.fields).toEqual({ + pairs: 1, + returned: 1, + invalid: 0 + }) + }) + }) + describe('position cooldown (opt-in)', () => { it('skips a cooled-down position without quoting or simulating, counting cooledDown', async () => { const cooldown = createCooldownStore({ cooldownMs: 60_000 }) diff --git a/bots/midnight-liquidation/test/sizing/plan.test.ts b/bots/midnight-liquidation/test/sizing/plan.test.ts index 126e8649..48717161 100644 --- a/bots/midnight-liquidation/test/sizing/plan.test.ts +++ b/bots/midnight-liquidation/test/sizing/plan.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from 'vitest' import type { LiquidationPlan, PlanInput } from '../../src/sizing/plan' import { ORACLE_PRICE_SCALE, WAD } from '../../src/constants' -import { maxSeizeForCap, plan } from '../../src/sizing/plan' +import { maxSeizeForCap, plan, planWithReason } from '../../src/sizing/plan' const MAX_LIF = 1036269430051813471n const LLTV = 860000000000000000n @@ -234,3 +234,148 @@ describe('maxSeizeForCap', () => { } }) }) + +describe('planWithReason', () => { + // The reachable-from-the-tick skips. `no_debt`/`locked`/`healthy_pre_maturity` are excluded by + // `isLiquidatable` upstream, so they are asserted only for their reason, not their trace. + it.each([ + ['no_debt', { hasDebt: false }], + ['locked', { locked: true }], + ['healthy_pre_maturity', { healthy: true }] + ] as const)('reports %s without a trace (decided before any arithmetic)', (reason, overrides) => { + const outcome = planWithReason(baseInput(overrides)) + expect(outcome).toEqual({ kind: 'skip', reason }) + }) + + it('reports seize_rounds_to_zero with the full cap-stage trace', () => { + // Post-maturity dust: a 1-wei cap against a 10x-priced slot floors the seize to zero. + const outcome = planWithReason( + baseInput({ + blockTimestamp: 3000n, + healthy: true, + debt: 1n, + badDebt: 0n, + bestCollateralAmt: 1n * WAD, + bestCollateralPrice: ORACLE_PRICE_SCALE * 10n + }) + ) + expect(outcome).toEqual({ + kind: 'skip', + reason: 'seize_rounds_to_zero', + trace: { + postMaturityMode: true, + lif: 1010074841681059297n, + effectiveDebt: 1n, + cap: 1n, + capEff: 1n, + seizedAssets: 0n + } + }) + }) + + it('reports capEff 0 when the margin alone eats a 1-wei cap', () => { + const outcome = planWithReason( + baseInput({ + blockTimestamp: 3000n, + healthy: true, + debt: 1n, + badDebt: 0n, + bestCollateralAmt: 1n * WAD, + bestCollateralPrice: ORACLE_PRICE_SCALE * 10n + }), + { seizeCapMarginBps: 30 } + ) + expect(outcome.kind).toBe('skip') + expect(outcome).toMatchObject({ + reason: 'seize_rounds_to_zero', + trace: { cap: 1n, capEff: 0n, seizedAssets: 0n } + }) + }) + + it('caps a post-maturity seize at the post-writeoff debt, not the gross debt', () => { + // `badDebt` is written off before the repay, so the cap is `debt - badDebt`. Sizing against the + // gross debt would over-repay and revert on-chain (Panic 0x11). Asserted by equivalence rather + // than a literal, so the test does not re-derive the sizing arithmetic it is checking. + const post = { + blockTimestamp: 3000n, + maturity: 2000n, + healthy: true, + bestCollateralAmt: 1000n * WAD + } + const withWriteoff = plan(baseInput({ ...post, debt: 1000n * WAD, badDebt: 400n * WAD })) + const equivalent = plan(baseInput({ ...post, debt: 600n * WAD, badDebt: 0n })) + const grossDebt = plan(baseInput({ ...post, debt: 1000n * WAD, badDebt: 0n })) + expect(withWriteoff).toEqual(equivalent) + expect(withWriteoff).not.toEqual(grossDebt) + }) + + it('reports nothing_to_seize for an empty best slot, omitting the cap stage', () => { + const outcome = planWithReason(baseInput({ bestCollateralAmt: 0n })) + expect(outcome).toEqual({ + kind: 'skip', + reason: 'nothing_to_seize', + trace: { + postMaturityMode: false, + lif: MAX_LIF, + effectiveDebt: 1000n * WAD, + seizedAssets: 0n + } + }) + }) + + it('reports cap_not_positive — and no longer returns a NEGATIVE-seize plan', () => { + // `debt - maxDebt < badDebt < debt` makes the RCF numerator negative, so maxRepaid, the cap and + // the derived seize all go negative. Before the `<= 0n` guard this returned a plan whose + // `seizedAssets` was negative, which reverts opaquely once abi-encoded as uint256. + const input = baseInput({ + debt: 1000n, + badDebt: 500n, + maxDebt: 900n, + rcfThreshold: 10n ** 30n, // even an otherwise exempt slot must reject a negative RCF cap + bestCollateralAmt: 5000n, + bestCollateralPrice: ORACLE_PRICE_SCALE, + bestCollateralMaxLif: 1100000000000000000n + }) + const outcome = planWithReason(input) + expect(outcome).toMatchObject({ kind: 'skip', reason: 'cap_not_positive' }) + expect(outcome.kind === 'skip' && outcome.trace?.cap).toBe(-7406n) + expect(outcome.kind === 'skip' && outcome.trace?.seizedAssets).toBe(-8146n) + expect(plan(input)).toBeNull() + }) + + it('omits maxRepaid and flags rcfDisabled when lltv waives the cap', () => { + const outcome = planWithReason( + baseInput({ bestCollateralLltv: WAD, bestCollateralAmt: 0n, bestCollateralMaxLif: WAD }) + ) + // Reached via nothing_to_seize, so only the mode fields are present — the point is that a + // maxUint256 maxRepaid is never logged as if it were a real bound. + expect(outcome.kind === 'skip' && outcome.trace).not.toHaveProperty('maxRepaid') + }) + + describe('plan() facade', () => { + // Pins the facade to the implementation so a later edit to planWithReason cannot silently change + // plan()'s contract (which the whole suite above still asserts through plan()). + const cases: PlanInput[] = [ + baseInput(), // plans + baseInput({ hasDebt: false }), + baseInput({ locked: true }), + baseInput({ healthy: true }), + baseInput({ bestCollateralAmt: 0n }), + baseInput({ + blockTimestamp: 3000n, + healthy: true, + debt: 1n, + bestCollateralAmt: 1n * WAD, + bestCollateralPrice: ORACLE_PRICE_SCALE * 10n + }) + ] + + it.each(cases.map((input, i) => [i, input] as const))( + 'case %i returns the outcome plan, or null for a skip', + (_i, input) => { + const outcome = planWithReason(input) + expect(plan(input)).toEqual(outcome.kind === 'plan' ? outcome.plan : null) + } + ) + }) +}) diff --git a/docs/decisions/TIB-2026-05-28-midnight-liquidation-bot.md b/docs/decisions/TIB-2026-05-28-midnight-liquidation-bot.md index 308ff4c2..45782a63 100644 --- a/docs/decisions/TIB-2026-05-28-midnight-liquidation-bot.md +++ b/docs/decisions/TIB-2026-05-28-midnight-liquidation-bot.md @@ -926,6 +926,8 @@ After each phase: a few comments here and in `sizing/plan.ts` point at that snapshot and are approximate; for the authoritative surface, trust the vendored interface above and the deployed contract. +## Addenda + ### 2026-07-10 — persistent runner and Alternative 5 superseded by the pipeline architecture [TIB-2026-07-13 (bot architecture)](./TIB-2026-07-13-bot-architecture.md) supersedes two of this @@ -935,3 +937,61 @@ Alternative 5 ("Persistent queue state across runner restarts", rejected here) i required by the one-shot model**: a per-run process with no memory of its pending txs could never fee-bump a stuck one. The spirit survives — persisted state is a hint reconciled against chain truth, and losing the file degrades to this TIB's restart semantics. + +### 2026-08-05 — `tick.end` counters reconciled; `submitted` now means broadcast + +Three production defects in the tick, found by a BetterStack review of `bot.liquidation.midnight` +(source 2607569), changed the observability contract this TIB documents. This addendum records the +new shape; the body above is left as written. + +**The defects.** (1) Two `continue`s — a position already in flight, and a position sizing refused — +incremented no counter and logged nothing, so from 2026-07-31 15:37 the bot reported +`liquidatable: 12, planned: 0` for ~215,000 consecutive ticks with every skip counter at zero and no +way to learn why. (2) `submitted` counted `submit` _calls_, not broadcasts: on 2026-07-30 it summed +2,425 while `tx.sent` was 0 and `tx.submit_failed` was 2,666. (3) `backoff.clear` ran unconditionally +after `submit`, wiping the accumulated attempt count, so a position that simulated ok and then failed +to send was re-quoted, re-simulated and re-sent every block indefinitely. + +**The counter set** (identical in both liquidators, ordered as the pipeline runs): + +```text +tick.end { pairs, liquidatable, inflightSkipped, planSkipped, planned, + cooledDown, backoffSkipped, noSwapPath, quoteFailed, + ok, reverted, submitted, notSent, complete } +``` + +This reconciles drift in both directions: `backoffSkipped`/`cooledDown` shipped without ever being +documented, and `badRoute` was documented here but never implemented (route-quality rejection surfaces +as the `quote.route_quality_failed` event, not a counter). + +For a tick with `complete: true` the set is exhaustive by construction, and the identities are +asserted in every tick test — so a stage added without a counter breaks a sum instead of silently +dropping a position: + +```text +pairs >= liquidatable +liquidatable === inflightSkipped + planSkipped + planned +planned === cooledDown + backoffSkipped + noSwapPath + quoteFailed + ok + reverted +ok === submitted + notSent +``` + +`complete: false` marks a tick aborted part-way (a hashless send after a nonce was claimed still +throws by design); its counters are partial and the last identity is short by one. Previously such a +tick emitted no `tick.end` at all. + +**Semantics that changed.** `submitted` counts only real broadcasts. `notSent` counts a `submit` that +returned without sending, and the queue now reports which of its four non-sending exits it took +(`tx.submit_failed`, `tx.send_aborted`, `nonce.sync_failed`, `queue.nonce_hole`). Only the +per-position one (`tx.submit_failed`) backs the position off; the other three are queue-wide refusals +that reject every send that tick, so attributing them to one position would suppress healthy ones. +Only a broadcast clears a backoff. + +**New event.** `plan.skipped` explains a liquidatable-but-unsizable position, carrying the sizing +inputs and the derived numbers (`lif`, `effectiveDebt`, `cap`, `capEff`, `seizedAssets`) so the +decision replays from one line. `info` for the ordinary dust reasons; `warn` for reasons that should +be unreachable, which makes it a live assertion that the eligibility gate and the sizing module have +not diverged. It is sampled at most once per `PLAN_SKIP_SAMPLE_EVERY_BLOCKS` (the counter stays exact +every tick), because one line per position per block would have been ~21k lines/hour/bot. + +**Any BetterStack chart or alert built on `submitted` changes meaning** and needs review; the new +fields also need the dashboard metrics-collection step before they are queryable. diff --git a/docs/decisions/TIB-2026-06-30-blue-liquidation-bot.md b/docs/decisions/TIB-2026-06-30-blue-liquidation-bot.md index bed48733..05369459 100644 --- a/docs/decisions/TIB-2026-06-30-blue-liquidation-bot.md +++ b/docs/decisions/TIB-2026-06-30-blue-liquidation-bot.md @@ -651,3 +651,62 @@ TIB conventions: how the TIB is applied without changing the decision itself. - TIB identifiers use CalVer (YYYY-MM-DD) based on the date the TIB was first drafted. --> + +## Addenda + +### 2026-08-05 — `tick.end` counters reconciled; `submitted` now means broadcast + +Three production defects, found by a BetterStack review of `bot.liquidation.midnight` (source 2607569) and present verbatim in this bot too, changed the observability contract this TIB documents. This addendum records the +new shape; the body above is left as written. + +**The defects.** (1) Two `continue`s — a position already in flight, and a position sizing refused — +incremented no counter and logged nothing, so from 2026-07-31 15:37 the bot reported +`liquidatable: 12, planned: 0` for ~215,000 consecutive ticks with every skip counter at zero and no +way to learn why. (2) `submitted` counted `submit` _calls_, not broadcasts: on 2026-07-30 it summed +2,425 while `tx.sent` was 0 and `tx.submit_failed` was 2,666. (3) `backoff.clear` ran unconditionally +after `submit`, wiping the accumulated attempt count, so a position that simulated ok and then failed +to send was re-quoted, re-simulated and re-sent every block indefinitely. + +**The counter set** (identical in both liquidators, ordered as the pipeline runs): + +```text +tick.end { pairs, liquidatable, inflightSkipped, planSkipped, planned, + cooledDown, backoffSkipped, noSwapPath, quoteFailed, + ok, reverted, submitted, notSent, complete } +``` + +This reconciles drift in both directions: `backoffSkipped`/`cooledDown` shipped without ever being +documented, and `badRoute` was documented here but never implemented (route-quality rejection surfaces +as the `quote.route_quality_failed` event, not a counter). + +For a tick with `complete: true` the set is exhaustive by construction, and the identities are +asserted in every tick test — so a stage added without a counter breaks a sum instead of silently +dropping a position: + +```text +pairs >= liquidatable +liquidatable === inflightSkipped + planSkipped + planned +planned === cooledDown + backoffSkipped + noSwapPath + quoteFailed + ok + reverted +ok === submitted + notSent +``` + +`complete: false` marks a tick aborted part-way (a hashless send after a nonce was claimed still +throws by design); its counters are partial and the last identity is short by one. Previously such a +tick emitted no `tick.end` at all. + +**Semantics that changed.** `submitted` counts only real broadcasts. `notSent` counts a `submit` that +returned without sending, and the queue now reports which of its four non-sending exits it took +(`tx.submit_failed`, `tx.send_aborted`, `nonce.sync_failed`, `queue.nonce_hole`). Only the +per-position one (`tx.submit_failed`) backs the position off; the other three are queue-wide refusals +that reject every send that tick, so attributing them to one position would suppress healthy ones. +Only a broadcast clears a backoff. + +**New event.** `plan.skipped` explains a liquidatable-but-unsizable position, carrying the sizing +inputs and the derived numbers (`lif`, `repaidAssetsFull`, `seizeForFullDebt`, `seizedAssets`) so the +decision replays from one line. `info` for the ordinary dust reasons; `warn` for reasons that should +be unreachable, which makes it a live assertion that the eligibility gate and the sizing module have +not diverged. It is sampled at most once per `PLAN_SKIP_SAMPLE_EVERY_BLOCKS` (the counter stays exact +every tick), because one line per position per block would have been ~21k lines/hour/bot. + +**Any BetterStack chart or alert built on `submitted` changes meaning** and needs review; the new +fields also need the dashboard metrics-collection step before they are queryable. diff --git a/packages/bot-kit/src/balance.ts b/packages/bot-kit/src/balance.ts index 87f5d2bf..467cf340 100644 --- a/packages/bot-kit/src/balance.ts +++ b/packages/bot-kit/src/balance.ts @@ -5,6 +5,8 @@ import { formatEther } from 'viem' import type { Logger } from './logger' +import { createBlockSampler } from './runner/block-cadence' + /** * Default block cadence for the signer-balance metric. On ~2s Base blocks this ships roughly every * 60s — matching the daemon-era wall-clock cadence — so operators see EOA gas drain at a steady rate. @@ -23,12 +25,11 @@ export function createBalanceMonitor(deps: { logger: Logger everyBlocks?: bigint }): { maybeLog: (blockNumber: bigint) => Promise } { - const everyBlocks = deps.everyBlocks ?? BALANCE_EVERY_BLOCKS - let lastAt: bigint | null = null + // Always asks, so the sampler's edge-triggering never engages — this stays a plain fixed cadence. + const sampler = createBlockSampler(deps.everyBlocks ?? BALANCE_EVERY_BLOCKS) return { async maybeLog(blockNumber) { - if (lastAt !== null && blockNumber - lastAt < everyBlocks) return - lastAt = blockNumber + if (!sampler.claim(blockNumber)) return const balance = await tryCatch(deps.read()) if (balance.error) { deps.logger.warn('signer.balance_failed', { diff --git a/packages/bot-kit/src/index.ts b/packages/bot-kit/src/index.ts index 5875a74e..aa0e2b7a 100644 --- a/packages/bot-kit/src/index.ts +++ b/packages/bot-kit/src/index.ts @@ -8,6 +8,7 @@ export * from './queue/backoff' export * from './queue/cooldown' export * from './queue/fee-policy' export * from './queue/pending-queue' +export * from './runner/block-cadence' export * from './runner/runner' export * from './runner/watcher' export * from './shipping-config' diff --git a/packages/bot-kit/src/queue/backoff.ts b/packages/bot-kit/src/queue/backoff.ts index 78918e7c..abd216b8 100644 --- a/packages/bot-kit/src/queue/backoff.ts +++ b/packages/bot-kit/src/queue/backoff.ts @@ -10,7 +10,11 @@ export type Backoff = { shouldSkip: (label: string, block: bigint) => boolean /** Record a failure at `block`; the next attempt is allowed after an exponentially growing delay. */ record: (label: string, block: bigint) => void - /** Clear the position's backoff (call on a successful submit). */ + /** + * Clear the position's backoff. Call ONLY on a real broadcast — a `submit` that returned without + * sending must `record` instead. Clearing deletes the accumulated attempt count, so clearing on a + * non-broadcast resets the delay to `baseBlocks` and the exponential growth never accrues. + */ clear: (label: string) => void /** Number of positions currently tracked. */ readonly size: number diff --git a/packages/bot-kit/src/queue/cooldown.ts b/packages/bot-kit/src/queue/cooldown.ts index 5283a274..2690de6d 100644 --- a/packages/bot-kit/src/queue/cooldown.ts +++ b/packages/bot-kit/src/queue/cooldown.ts @@ -1,7 +1,8 @@ /** * Opt-in per-position liquidation cooldown, keyed by the `${id}:${borrower}` label. A COMPLEMENTARY * rate-limit defense to {@link Backoff}, not a replacement: after a liquidation attempt fails to - * produce a submittable transaction (no swap route, quote failure, or sim revert), the caller + * produce a broadcast transaction (no swap route, quote failure, sim revert, or a per-position send + * failure), the caller * {@link CooldownStore.mark}s the position and {@link CooldownStore.shouldSkip} suppresses re-quoting * the rate-limited venue APIs for a fixed `cooldownMs` window. Unlike {@link Backoff}'s block-based * exponential growth, this is a flat wall-clock window and is DISABLED by default — an operator opts @@ -21,7 +22,7 @@ export type CooldownStore = { /** True if `label` was marked within the cooldown window and should be skipped this tick. */ shouldSkip: (label: string) => boolean - /** Record that `label` just failed to produce a submittable tx (starts/refreshes its cooldown). */ + /** Record that `label` just failed to produce a broadcast tx (starts/refreshes its cooldown). */ mark: (label: string) => void } diff --git a/packages/bot-kit/src/queue/pending-queue.ts b/packages/bot-kit/src/queue/pending-queue.ts index 01a5b5da..032bddeb 100644 --- a/packages/bot-kit/src/queue/pending-queue.ts +++ b/packages/bot-kit/src/queue/pending-queue.ts @@ -51,6 +51,31 @@ type Pending = { attempt: number } +/** + * What one {@link PendingQueue.submit} call did. Only `sent` is a submission — the sole outcome that + * may clear a per-position backoff. `failed` means no transaction left this process, and its `reason` + * pairs with the warn line the queue already logged: + * + * | `reason` | logged event | `scope` | + * | ------------------- | ------------------- | ------------ | + * | `submit_failed` | `tx.submit_failed` | `'position'` | + * | `send_aborted` | `tx.send_aborted` | `'queue'` | + * | `nonce_sync_failed` | `nonce.sync_failed` | `'queue'` | + * | `nonce_hole` | `queue.nonce_hole` | `'queue'` | + * + * `scope` is the field callers should branch on: a `'queue'` refusal rejects EVERY send this tick, + * so a caller must not back off the position it happened to be holding — only a `'position'` failure + * is that position's own. Keeping the classification here means a new reason cannot be added without + * the type forcing a scope decision. + * + * A hashless send that already claimed a nonce is not a `failed` outcome — it throws `TxSendError` so + * the caller aborts the tick instead of racing the signer's cursor rollback. + */ +export type SubmitOutcome = + | { kind: 'sent' } + | { kind: 'failed'; scope: 'position'; reason: 'submit_failed' } + | { kind: 'failed'; scope: 'queue'; reason: 'send_aborted' | 'nonce_sync_failed' | 'nonce_hole' } + export type PendingQueue = { submit(args: { request: TxRequest @@ -58,7 +83,7 @@ export type PendingQueue = { maxFeePerGas: bigint maxPriorityFeePerGas: bigint blockNumber: bigint - }): Promise + }): Promise onBlock(blockNumber: bigint): Promise readonly size: number snapshot(): { nonce: number; txHash: Hex; attempt: number }[] @@ -189,12 +214,12 @@ export function createPendingQueue({ maxFeePerGas: bigint maxPriorityFeePerGas: bigint blockNumber: bigint - }): Promise { + }): Promise { // Latched by a prior hashless send: skip until the next `onBlock` clears it. The signer has // rolled its cursor back, so broadcasting again now would race that rollback. if (sendAborted) { logger.warn('tx.send_aborted', { label: args.label }) - return + return { kind: 'failed', scope: 'queue', reason: 'send_aborted' } } // Nothing in flight → reconcile the cursor with chain before claiming a nonce. A failed sync // would leave a stale (possibly runaway) cursor, so skip the send this tick rather than risk a @@ -206,7 +231,7 @@ export function createPendingQueue({ const synced = await tryCatch(syncNonce()) if (synced.error) { logger.warn('nonce.sync_failed', { label: args.label, reason: revertReason(synced.error) }) - return + return { kind: 'failed', scope: 'queue', reason: 'nonce_sync_failed' } } if (nonceHoleLow !== null) clearNonceHole('sync') } @@ -216,7 +241,7 @@ export function createPendingQueue({ // sweep clears it once the chain consumes past the hole. if (nonceHoleLow !== null) { logger.warn('queue.nonce_hole', { label: args.label, nonce: nonceHoleLow }) - return + return { kind: 'failed', scope: 'queue', reason: 'nonce_hole' } } const sent = await tryCatch( send({ @@ -240,7 +265,7 @@ export function createPendingQueue({ sendAborted = true throw sent.error } - return + return { kind: 'failed', scope: 'position', reason: 'submit_failed' } } const { nonce, txHash } = sent.data pending.set(nonce, { @@ -260,6 +285,7 @@ export function createPendingQueue({ maxFee: args.maxFeePerGas, priority: args.maxPriorityFeePerGas }) + return { kind: 'sent' } } async function replaceStuck(entry: Pending, blockNumber: bigint, baseFee: bigint): Promise { diff --git a/packages/bot-kit/src/runner/block-cadence.ts b/packages/bot-kit/src/runner/block-cadence.ts new file mode 100644 index 00000000..3d5ff156 --- /dev/null +++ b/packages/bot-kit/src/runner/block-cadence.ts @@ -0,0 +1,41 @@ +/** + * Block-height cadence gate: lets a caller act at most once per `everyBlocks`. Bounds the log volume + * of a condition that recurs every tick, in O(1) state — unlike a per-label map, there is nothing to + * evict and nothing to leak. + */ +export type BlockSampler = { + /** + * True when `block` is at least `everyBlocks` past the last granted claim; the first call always + * grants. STAMPS the cadence on true, hence `claim` rather than a predicate name. + * + * A caller that asks ONLY when it has something to say therefore gets edge-triggering for free: a + * quiet stretch never consumes the window, so the first occurrence after any gap is always reported + * and a sustained condition settles to one report per `everyBlocks`. A caller that claims eagerly + * (once per tick regardless) gets the same guarantee by calling {@link BlockSampler.reset} after a + * quiet tick. + */ + claim: (block: bigint) => boolean + /** Clears the cadence after a quiet observation so the next active observation is reported. */ + reset: () => void +} + +/** + * Builds a {@link BlockSampler}. + * + * @param everyBlocks - Minimum block distance between granted claims. `0n` grants every call. + * @returns A sampler holding one block height — construct one per call site and hold it for the + * process lifetime. + */ +export const createBlockSampler = (everyBlocks: bigint): BlockSampler => { + let lastAt: bigint | null = null + return { + claim(block) { + if (lastAt !== null && block - lastAt < everyBlocks) return false + lastAt = block + return true + }, + reset() { + lastAt = null + } + } +} diff --git a/packages/bot-kit/test/queue/pending-queue.test.ts b/packages/bot-kit/test/queue/pending-queue.test.ts index d965a58f..1421a46c 100644 --- a/packages/bot-kit/test/queue/pending-queue.test.ts +++ b/packages/bot-kit/test/queue/pending-queue.test.ts @@ -112,10 +112,12 @@ function submitOne(queue: PendingQueue, blockNumber = 0n) { describe('createPendingQueue', () => { it('records a submitted tx with the signer-assigned nonce', async () => { const { queue, sends } = setup() - await submitOne(queue) + const outcome = await submitOne(queue) expect(queue.size).toBe(1) expect(sends[0]?.nonce).toBeUndefined() // first send leaves nonce assignment to the signer expect(queue.snapshot()[0]).toEqual({ nonce: 7, txHash: hashOf(1), attempt: 0 }) + // The caller's broadcast signal — only this outcome may clear a per-position backoff. + expect(outcome).toEqual({ kind: 'sent' }) }) it('removes a tx once its receipt confirms', async () => { @@ -168,9 +170,11 @@ describe('createPendingQueue', () => { throw new Error('rpc down') } const { queue } = setup({ send, logger }) - await submitOne(queue) // must not throw + const outcome = await submitOne(queue) // must not throw expect(queue.size).toBe(0) expect(events.find(e => e.event === 'tx.submit_failed')?.level).toBe('warn') + // Per-position: this is the one reason a caller may attribute to the position it was holding. + expect(outcome).toEqual({ kind: 'failed', scope: 'position', reason: 'submit_failed' }) }) it('rethrows a first-send failure after a nonce was claimed but no hash was returned', async () => { @@ -179,6 +183,8 @@ describe('createPendingQueue', () => { throw new TxSendError(new Error('rpc timeout after broadcast'), 7) } const { queue } = setup({ send, logger }) + // Still a THROW, deliberately not a `failed` outcome: the caller must abort the tick rather than + // race the signer's cursor rollback. await expect(submitOne(queue)).rejects.toThrow(/rpc timeout after broadcast/) expect(queue.size).toBe(0) expect(events.find(e => e.event === 'tx.submit_failed')?.fields?.nonce).toBe(7) @@ -301,8 +307,9 @@ describe('createPendingQueue', () => { }, logger }) - await submitOne(ctx.queue) // must not throw + const outcome = await submitOne(ctx.queue) // must not throw expect(ctx.queue.size).toBe(0) // nothing broadcast on a stale cursor + expect(outcome).toEqual({ kind: 'failed', scope: 'queue', reason: 'nonce_sync_failed' }) expect(ctx.sends).toHaveLength(0) expect(events.find(e => e.event === 'nonce.sync_failed')?.level).toBe('warn') }) @@ -485,8 +492,10 @@ describe('send-aborted latch', () => { } const { queue } = setup({ send, logger }) await expect(submitOne(queue, 0n)).rejects.toThrow() - await submitOne(queue, 0n) // latched → skipped + const outcome = await submitOne(queue, 0n) // latched → skipped expect(events.find(e => e.event === 'tx.send_aborted')?.level).toBe('warn') + // Queue-WIDE: refuses every send this tick, so a caller must not back the position off for it. + expect(outcome).toEqual({ kind: 'failed', scope: 'queue', reason: 'send_aborted' }) }) }) @@ -554,8 +563,9 @@ describe('nonce-hole latch', () => { const sendsAfterDrop = ctx.sends.length // A NEW first-send is refused while the hole is latched (nonce 8 still pending → queue not empty, // so the empty-queue sync can't clear it). - await ctx.submit('c', 6n) + const refused = await ctx.submit('c', 6n) expect(ctx.sends.length).toBe(sendsAfterDrop) // no new broadcast + expect(refused).toEqual({ kind: 'failed', scope: 'queue', reason: 'nonce_hole' }) expect(ctx.queue.size).toBe(1) expect(ctx.events.some(e => e.event === 'queue.nonce_hole' && e.fields?.label === 'c')).toBe( true diff --git a/packages/bot-kit/test/runner/block-cadence.test.ts b/packages/bot-kit/test/runner/block-cadence.test.ts new file mode 100644 index 00000000..d1ed41a4 --- /dev/null +++ b/packages/bot-kit/test/runner/block-cadence.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' + +import { createBlockSampler } from '../../src/runner/block-cadence' + +describe('createBlockSampler', () => { + it('grants the first claim', () => { + expect(createBlockSampler(150n).claim(100n)).toBe(true) + }) + + it('refuses a claim inside the window', () => { + const sampler = createBlockSampler(150n) + sampler.claim(100n) + expect(sampler.claim(101n)).toBe(false) + expect(sampler.claim(249n)).toBe(false) + }) + + it('grants again at exactly everyBlocks past the last grant', () => { + // The boundary `createBalanceMonitor` depends on: a delta of exactly `everyBlocks` logs. + const sampler = createBlockSampler(150n) + sampler.claim(100n) + expect(sampler.claim(250n)).toBe(true) + }) + + it('measures the window from the last GRANT, not the last call', () => { + const sampler = createBlockSampler(10n) + expect(sampler.claim(100n)).toBe(true) + expect(sampler.claim(105n)).toBe(false) // refused, so it must not re-stamp + expect(sampler.claim(110n)).toBe(true) + }) + + it('does not consume the window when it is never asked', () => { + // The edge-trigger guarantee: a caller that asks only when it has something to say gets an + // immediate grant after any quiet stretch, with no per-caller state. + const sampler = createBlockSampler(150n) + expect(sampler.claim(100n)).toBe(true) + // ...400 quiet blocks during which the caller never asks... + expect(sampler.claim(500n)).toBe(true) + }) + + it('grants every claim at a zero cadence', () => { + const sampler = createBlockSampler(0n) + expect(sampler.claim(1n)).toBe(true) + expect(sampler.claim(1n)).toBe(true) + }) +})