diff --git a/docker-compose.yml b/docker-compose.yml index b747961..100d9fb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,7 +83,7 @@ services: depends_on: - api - # RPC reliability study: polls public mainnet endpoints through @heightproof/web3 with + # RPC reliability study: polls public Ethereum, Base and Arbitrum endpoints through @heightproof/web3 with # cross-provider verification. Opt-in: docker compose --profile study up -d # Report: docker compose --profile study run --rm study npm run study:report study: diff --git a/scripts/study/collect.ts b/scripts/study/collect.ts index f657698..a502800 100644 --- a/scripts/study/collect.ts +++ b/scripts/study/collect.ts @@ -1,8 +1,9 @@ /** * RPC reliability study: collector. * - * Polls several Ethereum mainnet endpoints on a fixed interval through @heightproof/web3, with - * cross-provider verification on, so Heightproof records every answer and every detector finding. + * Polls public endpoints on Ethereum mainnet, Base and Arbitrum on a fixed interval through + * @heightproof/web3, with cross-provider verification on among the endpoints of each chain, so + * Heightproof records every answer and every detector finding. * Meant to run unattended for days (see the `study` compose profile); `report.ts` turns what it * collected into numbers. * @@ -14,36 +15,106 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname } from "node:path"; import { createHeightproofRpcClient } from "@heightproof/web3"; -type Endpoint = { name: string; url: string }; +type Chain = { + chainId: number; + /** USDC on this chain: a busy contract, for reads and for log queries. */ + usdc: string; + /** + * Blocks a log query spans. Free public tiers cap it (BlastAPI at 10 blocks) and treat ranges + * far from the head as archive requests, so it stays short and recent. + */ + logRange: number; + /** How far behind the head a log range ends, about 10 s, so a provider a block behind is not a mismatch. */ + logLag: number; +}; +type Endpoint = { name: string; url: string; chain: string }; type StudyState = { projectId: string; apiKey: string; startedAt: string }; +const CHAINS: Record = { + ethereum: { + chainId: 1, + usdc: "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + logRange: 10, + logLag: 3 + }, + base: { + chainId: 8453, + usdc: "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913", + logRange: 10, + logLag: 5 + }, + arbitrum: { + chainId: 42161, + usdc: "0xaf88d065e77c8cC2239327C5EDb3A432268e5831", + logRange: 10, + logLag: 40 + } +}; + +// `name=url` is an Ethereum mainnet endpoint; `chain/name=url` names its chain. const DEFAULT_ENDPOINTS = "publicnode=https://ethereum-rpc.publicnode.com," + "drpc=https://eth.drpc.org," + "blastapi=https://eth-mainnet.public.blastapi.io," + "mevblocker=https://rpc.mevblocker.io," + - "onfinality=https://eth.api.onfinality.io/public"; + "onfinality=https://eth.api.onfinality.io/public," + + "base/official=https://mainnet.base.org," + + "base/publicnode=https://base-rpc.publicnode.com," + + "base/drpc=https://base.drpc.org," + + "base/blastapi=https://base-mainnet.public.blastapi.io," + + "arbitrum/official=https://arb1.arbitrum.io/rpc," + + "arbitrum/publicnode=https://arbitrum-one-rpc.publicnode.com," + + "arbitrum/drpc=https://arbitrum.drpc.org," + + "arbitrum/blastapi=https://arbitrum-one.public.blastapi.io"; const api = process.env.HEIGHTPROOF_ENDPOINT ?? "http://localhost:8080"; const statePath = process.env.STUDY_STATE ?? ".study/state.json"; const intervalMs = Number(process.env.STUDY_INTERVAL_MS ?? 15_000); const sampleRate = Number(process.env.STUDY_VERIFY_SAMPLE_RATE ?? 0.2); const callTimeoutMs = Number(process.env.STUDY_CALL_TIMEOUT_MS ?? 10_000); +// Log queries return the most data by far, so they run on every Nth tick only. +const logsEveryTicks = Number(process.env.STUDY_LOGS_EVERY_TICKS ?? 4); const endpoints: Endpoint[] = (process.env.STUDY_ENDPOINTS ?? DEFAULT_ENDPOINTS) .split(",") .map((entry) => { - const [name, ...url] = entry.trim().split("="); - return { name, url: url.join("=") }; + const [label, ...url] = entry.trim().split("="); + const chain = label.includes("/") ? label.slice(0, label.indexOf("/")) : "ethereum"; + if (!CHAINS[chain]) throw new Error(`unknown chain "${chain}" in STUDY_ENDPOINTS`); + return { name: label, url: url.join("="), chain }; }); // Deterministic reads whose answers every honest provider must agree on at a given height. const WALLET = "0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045"; -const USDC = "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"; -const CALLS: Array<{ method: string; params: unknown[] }> = [ - { method: "eth_blockNumber", params: [] }, - { method: "eth_getBalance", params: [WALLET, "latest"] }, - { method: "eth_call", params: [{ to: USDC, data: "0x18160ddd" }, "latest"] } -]; +const TRANSFER_TOPIC = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; + +type Call = { method: string; params: unknown[] }; + +/** + * One tick's calls against one endpoint. The block at the tip is where reorgs and lagging nodes + * show; a log range over a busy contract is where providers silently truncate. + */ +function callsFor(chain: Chain, head: number | undefined, tick: number): Call[] { + const calls: Call[] = [ + { method: "eth_getBalance", params: [WALLET, "latest"] }, + { method: "eth_call", params: [{ to: chain.usdc, data: "0x18160ddd" }, "latest"] }, + { method: "eth_getBlockByNumber", params: ["latest", false] } + ]; + if (head !== undefined && tick % logsEveryTicks === 0) { + const toBlock = head - chain.logLag; + calls.push({ + method: "eth_getLogs", + params: [ + { + address: chain.usdc, + topics: [TRANSFER_TOPIC], + fromBlock: `0x${(toBlock - chain.logRange + 1).toString(16)}`, + toBlock: `0x${toBlock.toString(16)}` + } + ] + }); + } + return calls; +} const state = await loadOrCreateState(); console.log( @@ -58,12 +129,12 @@ const clients = endpoints.map((endpoint) => ({ endpoint: api, rpcUrl: endpoint.url, provider: endpoint.name, - chainId: 1, + chainId: CHAINS[endpoint.chain].chainId, serviceName: "rpc-study", environment: "study", verify: { endpoints: endpoints - .filter((other) => other !== endpoint) + .filter((other) => other !== endpoint && other.chain === endpoint.chain) .map((other) => ({ rpcUrl: other.url, provider: other.name })), sampleRate, maxCallsPerMinute: 20, @@ -81,7 +152,16 @@ for (let tick = 1; !stopping; tick++) { const started = Date.now(); await Promise.all( clients.map(async ({ endpoint, client }) => { - for (const call of CALLS) { + const head = await withTimeout( + client.request({ method: "eth_blockNumber", params: [] }), + callTimeoutMs + ).then( + (result) => (typeof result === "string" ? Number.parseInt(result, 16) : undefined), + () => undefined + ); + totals.get(endpoint.name)![head === undefined ? "failed" : "ok"] += 1; + + for (const call of callsFor(CHAINS[endpoint.chain], head, tick)) { const outcome = await withTimeout(client.request(call), callTimeoutMs).then( () => "ok" as const, () => "failed" as const diff --git a/scripts/study/report.ts b/scripts/study/report.ts index 67fb4b7..f7178d2 100644 --- a/scripts/study/report.ts +++ b/scripts/study/report.ts @@ -3,7 +3,7 @@ * * Reads what the collector recorded and prints a Markdown summary per provider: availability, * errors inside 200 responses, rate limiting, latency, how often each provider's chain head - * trailed the others, and how often providers disagreed at the same block height. + * trailed the others on its chain, and how often providers disagreed at the same block height. * * Ticks where most providers failed at once with a client-side network error are the * collector's own connection dropping, not the providers. They are reported on their own and @@ -48,6 +48,15 @@ const LOCAL_OUTAGE_TICKS = ` const MEASURED = `${PRIMARY} and ${TICK} not in (${LOCAL_OUTAGE_TICKS})`; const withInterval = [projectId, intervalSeconds]; +const CHAIN_NAMES: Record = { + "1": "Ethereum", + "8453": "Base", + "42161": "Arbitrum" +}; +// Block time in seconds, so head lag compares across chains: two blocks is 24 s on Ethereum +// but half a second on Arbitrum. +const BLOCK_SECONDS = `(case evm_chain_id when '8453' then 2 when '42161' then 0.25 else 12 end)`; + const [window] = await q<{ first: Date; last: Date; calls: number }>( `select min(timestamp) as first, max(timestamp) as last, count(*)::int as calls from api_events where ${PRIMARY}` ); @@ -84,49 +93,59 @@ const availability = await q<{ ); // Every provider is polled at the same moment each tick, so within one tick the highest head any -// provider reported is the reference. Lag of 1 block is mostly a block landing mid-tick; the -// stale-head detector's threshold is more than 3. +// provider on the same chain reported is the reference. Up to one Ethereum block is mostly a block +// landing mid-tick; the stale-head detector's threshold is more than 3 blocks (36 s). const headLag = await q<{ provider: string; samples: number; - lag_ge_2: number; - lag_gt_3: number; - max_lag: number; + lag_ge_24s: number; + lag_gt_36s: number; + max_lag_s: number; }>( `with heads as ( - select evm_provider as provider, evm_block_number::bigint as head, - ${TICK} as tick + select evm_provider as provider, evm_chain_id as chain, evm_block_number::bigint as head, + ${BLOCK_SECONDS} as block_seconds, ${TICK} as tick from api_events where ${MEASURED} and evm_rpc_method = 'eth_blockNumber' and evm_block_number is not null ), lagged as ( - select provider, max(head) over (partition by tick) - head as lag from heads + select provider, (max(head) over (partition by chain, tick) - head) * block_seconds as lag_s + from heads ) select provider, count(*)::int as samples, - count(*) filter (where lag >= 2)::int as lag_ge_2, - count(*) filter (where lag > 3)::int as lag_gt_3, - coalesce(max(lag), 0)::int as max_lag + count(*) filter (where lag_s >= 24)::int as lag_ge_24s, + count(*) filter (where lag_s > 36)::int as lag_gt_36s, + coalesce(max(lag_s), 0)::float as max_lag_s from lagged group by 1 order by 1`, withInterval ); -// One verification run = every endpoint answering the same call pinned to the same height. -const disagreements = await q<{ method: string; runs: number; disagreeing: number }>( +// One verification run = every endpoint answering the same call pinned to the same height. An +// error or a null is an endpoint that could not answer (often one not yet at that height), never +// a differing answer. +const disagreements = await q<{ + chain: string; + method: string; + runs: number; + disagreeing: number; +}>( `with runs as ( - select evm_shadow_of_trace_id as run, evm_rpc_method as method, evm_block_number as block, - count(distinct evm_result_hash) filter (where evm_result_shape <> 'error_in_body') as answers + select evm_shadow_of_trace_id as run, evm_chain_id as chain, evm_rpc_method as method, + evm_block_number as block, + count(distinct evm_result_hash) filter (where evm_result_shape not in ('error_in_body', 'null')) as answers from api_events where project_id = $1 and evm_shadow_of_trace_id is not null - group by 1, 2, 3 + group by 1, 2, 3, 4 ) - select method, count(*)::int as runs, count(*) filter (where answers > 1)::int as disagreeing - from runs group by 1 order by 1` + select chain, method, count(*)::int as runs, + count(*) filter (where answers > 1)::int as disagreeing + from runs group by 1, 2 order by 1, 2` ); const minority = await q<{ provider: string; times_in_minority: number }>( `with answers as ( select evm_shadow_of_trace_id as run, evm_block_number as block, evm_provider as provider, evm_result_hash as hash from api_events - where project_id = $1 and evm_shadow_of_trace_id is not null and evm_result_shape <> 'error_in_body' + where project_id = $1 and evm_shadow_of_trace_id is not null and evm_result_shape not in ('error_in_body', 'null') ), counted as ( select run, block, hash, count(*) as votes from answers group by 1, 2, 3 ), majority as ( @@ -164,7 +183,7 @@ const table = (head: string[], rows: Array>) => ...rows.map((r) => `| ${r.join(" | ")} |`) ].join("\n"); -const report = `# Ethereum mainnet RPC reliability +const report = `# EVM RPC reliability Project \`${projectId}\` · ${window.first.toISOString()} → ${window.last.toISOString()} (${hours.toFixed(1)} h) · ${window.calls.toLocaleString()} primary calls · polled every ${intervalSeconds}s @@ -195,24 +214,29 @@ ${table( ]) )} -## Chain head lag (vs. the highest head any provider reported in the same tick) +## Chain head lag (vs. the highest head any provider on the same chain reported in the same tick) ${table( - ["Provider", "Samples", "≥ 2 blocks behind", "> 3 blocks behind", "Worst"], + ["Provider", "Samples", "≥ 24 s behind", "> 36 s behind", "Worst"], headLag.map((r) => [ r.provider, r.samples, - pct(r.lag_ge_2, r.samples), - pct(r.lag_gt_3, r.samples), - `${r.max_lag} blocks` + pct(r.lag_ge_24s, r.samples), + pct(r.lag_gt_36s, r.samples), + `${r.max_lag_s} s` ]) )} ## Disagreement at equal height (verification runs) ${table( - ["Method", "Runs", "Runs with differing answers"], - disagreements.map((r) => [r.method, r.runs, `${r.disagreeing} (${pct(r.disagreeing, r.runs)})`]) + ["Chain", "Method", "Runs", "Runs with differing answers"], + disagreements.map((r) => [ + CHAIN_NAMES[r.chain] ?? r.chain ?? "—", + r.method, + r.runs, + `${r.disagreeing} (${pct(r.disagreeing, r.runs)})` + ]) )} ${