From cb0c078cf01aeb195fd262f4b68dbe49504c425f Mon Sep 17 00:00:00 2001 From: macjayz Date: Sun, 1 Nov 2026 10:50:37 +0100 Subject: [PATCH] study: add Base, Arbitrum, tip blocks and log ranges The study only asked Ethereum mainnet for a balance, a USDC call and the head, which is where providers almost never differ. It now also covers: - Base and Arbitrum, four public endpoints each, verified only against endpoints on the same chain. - eth_getBlockByNumber at the tip, where reorgs and lagging nodes show. - eth_getLogs over the last 10 blocks of USDC transfers (every 4th tick), where providers truncate. Ranges stay short and recent because free tiers cap them and treat older ranges as archive requests. The report groups disagreements by chain, measures head lag in seconds so chains with different block times compare, and stops counting a null answer (a node not yet at that height) as a differing one. A two-minute smoke run already found one real difference: the official Arbitrum RPC returns blockTimestamp "0x0" on every log, where the other three providers return the block's time. Co-Authored-By: Claude Opus 5.5 --- docker-compose.yml | 2 +- scripts/study/collect.ts | 110 +++++++++++++++++++++++++++++++++------ scripts/study/report.ts | 80 ++++++++++++++++++---------- 3 files changed, 148 insertions(+), 44 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b747961..100d9fb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,7 +83,7 @@ services: depends_on: - api - # RPC reliability study: polls public mainnet endpoints through @heightproof/web3 with + # RPC reliability study: polls public Ethereum, Base and Arbitrum endpoints through @heightproof/web3 with # cross-provider verification. Opt-in: docker compose --profile study up -d # Report: docker compose --profile study run --rm study npm run study:report study: diff --git a/scripts/study/collect.ts b/scripts/study/collect.ts index f657698..a502800 100644 --- a/scripts/study/collect.ts +++ b/scripts/study/collect.ts @@ -1,8 +1,9 @@ /** * RPC reliability study: collector. * - * Polls several Ethereum mainnet endpoints on a fixed interval through @heightproof/web3, with - * cross-provider verification on, so Heightproof records every answer and every detector finding. + * Polls public endpoints on Ethereum mainnet, Base and Arbitrum on a fixed interval through + * @heightproof/web3, with cross-provider verification on among the endpoints of each chain, so + * Heightproof records every answer and every detector finding. * Meant to run unattended for days (see the `study` compose profile); `report.ts` turns what it * collected into numbers. * @@ -14,36 +15,106 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname } from "node:path"; import { createHeightproofRpcClient } from "@heightproof/web3"; -type Endpoint = { name: string; url: string }; +type Chain = { + chainId: number; + /** USDC on this chain: a busy contract, for reads and for log queries. */ + usdc: string; + /** + * Blocks a log query spans. Free public tiers cap it (BlastAPI at 10 blocks) and treat ranges + * far from the head as archive requests, so it stays short and recent. + */ + logRange: number; + /** How far behind the head a log range ends, about 10 s, so a provider a block behind is not a mismatch. */ + logLag: number; +}; +type Endpoint = { name: string; url: string; chain: string }; type StudyState = { projectId: string; apiKey: string; startedAt: string }; +const CHAINS: Record = { + ethereum: { + chainId: 1, + usdc: "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", + logRange: 10, + logLag: 3 + }, + base: { + chainId: 8453, + usdc: "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913", + logRange: 10, + logLag: 5 + }, + arbitrum: { + chainId: 42161, + usdc: "0xaf88d065e77c8cC2239327C5EDb3A432268e5831", + logRange: 10, + logLag: 40 + } +}; + +// `name=url` is an Ethereum mainnet endpoint; `chain/name=url` names its chain. const DEFAULT_ENDPOINTS = "publicnode=https://ethereum-rpc.publicnode.com," + "drpc=https://eth.drpc.org," + "blastapi=https://eth-mainnet.public.blastapi.io," + "mevblocker=https://rpc.mevblocker.io," + - "onfinality=https://eth.api.onfinality.io/public"; + "onfinality=https://eth.api.onfinality.io/public," + + "base/official=https://mainnet.base.org," + + "base/publicnode=https://base-rpc.publicnode.com," + + "base/drpc=https://base.drpc.org," + + "base/blastapi=https://base-mainnet.public.blastapi.io," + + "arbitrum/official=https://arb1.arbitrum.io/rpc," + + "arbitrum/publicnode=https://arbitrum-one-rpc.publicnode.com," + + "arbitrum/drpc=https://arbitrum.drpc.org," + + "arbitrum/blastapi=https://arbitrum-one.public.blastapi.io"; const api = process.env.HEIGHTPROOF_ENDPOINT ?? "http://localhost:8080"; const statePath = process.env.STUDY_STATE ?? ".study/state.json"; const intervalMs = Number(process.env.STUDY_INTERVAL_MS ?? 15_000); const sampleRate = Number(process.env.STUDY_VERIFY_SAMPLE_RATE ?? 0.2); const callTimeoutMs = Number(process.env.STUDY_CALL_TIMEOUT_MS ?? 10_000); +// Log queries return the most data by far, so they run on every Nth tick only. +const logsEveryTicks = Number(process.env.STUDY_LOGS_EVERY_TICKS ?? 4); const endpoints: Endpoint[] = (process.env.STUDY_ENDPOINTS ?? DEFAULT_ENDPOINTS) .split(",") .map((entry) => { - const [name, ...url] = entry.trim().split("="); - return { name, url: url.join("=") }; + const [label, ...url] = entry.trim().split("="); + const chain = label.includes("/") ? label.slice(0, label.indexOf("/")) : "ethereum"; + if (!CHAINS[chain]) throw new Error(`unknown chain "${chain}" in STUDY_ENDPOINTS`); + return { name: label, url: url.join("="), chain }; }); // Deterministic reads whose answers every honest provider must agree on at a given height. const WALLET = "0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045"; -const USDC = "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"; -const CALLS: Array<{ method: string; params: unknown[] }> = [ - { method: "eth_blockNumber", params: [] }, - { method: "eth_getBalance", params: [WALLET, "latest"] }, - { method: "eth_call", params: [{ to: USDC, data: "0x18160ddd" }, "latest"] } -]; +const TRANSFER_TOPIC = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"; + +type Call = { method: string; params: unknown[] }; + +/** + * One tick's calls against one endpoint. The block at the tip is where reorgs and lagging nodes + * show; a log range over a busy contract is where providers silently truncate. + */ +function callsFor(chain: Chain, head: number | undefined, tick: number): Call[] { + const calls: Call[] = [ + { method: "eth_getBalance", params: [WALLET, "latest"] }, + { method: "eth_call", params: [{ to: chain.usdc, data: "0x18160ddd" }, "latest"] }, + { method: "eth_getBlockByNumber", params: ["latest", false] } + ]; + if (head !== undefined && tick % logsEveryTicks === 0) { + const toBlock = head - chain.logLag; + calls.push({ + method: "eth_getLogs", + params: [ + { + address: chain.usdc, + topics: [TRANSFER_TOPIC], + fromBlock: `0x${(toBlock - chain.logRange + 1).toString(16)}`, + toBlock: `0x${toBlock.toString(16)}` + } + ] + }); + } + return calls; +} const state = await loadOrCreateState(); console.log( @@ -58,12 +129,12 @@ const clients = endpoints.map((endpoint) => ({ endpoint: api, rpcUrl: endpoint.url, provider: endpoint.name, - chainId: 1, + chainId: CHAINS[endpoint.chain].chainId, serviceName: "rpc-study", environment: "study", verify: { endpoints: endpoints - .filter((other) => other !== endpoint) + .filter((other) => other !== endpoint && other.chain === endpoint.chain) .map((other) => ({ rpcUrl: other.url, provider: other.name })), sampleRate, maxCallsPerMinute: 20, @@ -81,7 +152,16 @@ for (let tick = 1; !stopping; tick++) { const started = Date.now(); await Promise.all( clients.map(async ({ endpoint, client }) => { - for (const call of CALLS) { + const head = await withTimeout( + client.request({ method: "eth_blockNumber", params: [] }), + callTimeoutMs + ).then( + (result) => (typeof result === "string" ? Number.parseInt(result, 16) : undefined), + () => undefined + ); + totals.get(endpoint.name)![head === undefined ? "failed" : "ok"] += 1; + + for (const call of callsFor(CHAINS[endpoint.chain], head, tick)) { const outcome = await withTimeout(client.request(call), callTimeoutMs).then( () => "ok" as const, () => "failed" as const diff --git a/scripts/study/report.ts b/scripts/study/report.ts index 67fb4b7..f7178d2 100644 --- a/scripts/study/report.ts +++ b/scripts/study/report.ts @@ -3,7 +3,7 @@ * * Reads what the collector recorded and prints a Markdown summary per provider: availability, * errors inside 200 responses, rate limiting, latency, how often each provider's chain head - * trailed the others, and how often providers disagreed at the same block height. + * trailed the others on its chain, and how often providers disagreed at the same block height. * * Ticks where most providers failed at once with a client-side network error are the * collector's own connection dropping, not the providers. They are reported on their own and @@ -48,6 +48,15 @@ const LOCAL_OUTAGE_TICKS = ` const MEASURED = `${PRIMARY} and ${TICK} not in (${LOCAL_OUTAGE_TICKS})`; const withInterval = [projectId, intervalSeconds]; +const CHAIN_NAMES: Record = { + "1": "Ethereum", + "8453": "Base", + "42161": "Arbitrum" +}; +// Block time in seconds, so head lag compares across chains: two blocks is 24 s on Ethereum +// but half a second on Arbitrum. +const BLOCK_SECONDS = `(case evm_chain_id when '8453' then 2 when '42161' then 0.25 else 12 end)`; + const [window] = await q<{ first: Date; last: Date; calls: number }>( `select min(timestamp) as first, max(timestamp) as last, count(*)::int as calls from api_events where ${PRIMARY}` ); @@ -84,49 +93,59 @@ const availability = await q<{ ); // Every provider is polled at the same moment each tick, so within one tick the highest head any -// provider reported is the reference. Lag of 1 block is mostly a block landing mid-tick; the -// stale-head detector's threshold is more than 3. +// provider on the same chain reported is the reference. Up to one Ethereum block is mostly a block +// landing mid-tick; the stale-head detector's threshold is more than 3 blocks (36 s). const headLag = await q<{ provider: string; samples: number; - lag_ge_2: number; - lag_gt_3: number; - max_lag: number; + lag_ge_24s: number; + lag_gt_36s: number; + max_lag_s: number; }>( `with heads as ( - select evm_provider as provider, evm_block_number::bigint as head, - ${TICK} as tick + select evm_provider as provider, evm_chain_id as chain, evm_block_number::bigint as head, + ${BLOCK_SECONDS} as block_seconds, ${TICK} as tick from api_events where ${MEASURED} and evm_rpc_method = 'eth_blockNumber' and evm_block_number is not null ), lagged as ( - select provider, max(head) over (partition by tick) - head as lag from heads + select provider, (max(head) over (partition by chain, tick) - head) * block_seconds as lag_s + from heads ) select provider, count(*)::int as samples, - count(*) filter (where lag >= 2)::int as lag_ge_2, - count(*) filter (where lag > 3)::int as lag_gt_3, - coalesce(max(lag), 0)::int as max_lag + count(*) filter (where lag_s >= 24)::int as lag_ge_24s, + count(*) filter (where lag_s > 36)::int as lag_gt_36s, + coalesce(max(lag_s), 0)::float as max_lag_s from lagged group by 1 order by 1`, withInterval ); -// One verification run = every endpoint answering the same call pinned to the same height. -const disagreements = await q<{ method: string; runs: number; disagreeing: number }>( +// One verification run = every endpoint answering the same call pinned to the same height. An +// error or a null is an endpoint that could not answer (often one not yet at that height), never +// a differing answer. +const disagreements = await q<{ + chain: string; + method: string; + runs: number; + disagreeing: number; +}>( `with runs as ( - select evm_shadow_of_trace_id as run, evm_rpc_method as method, evm_block_number as block, - count(distinct evm_result_hash) filter (where evm_result_shape <> 'error_in_body') as answers + select evm_shadow_of_trace_id as run, evm_chain_id as chain, evm_rpc_method as method, + evm_block_number as block, + count(distinct evm_result_hash) filter (where evm_result_shape not in ('error_in_body', 'null')) as answers from api_events where project_id = $1 and evm_shadow_of_trace_id is not null - group by 1, 2, 3 + group by 1, 2, 3, 4 ) - select method, count(*)::int as runs, count(*) filter (where answers > 1)::int as disagreeing - from runs group by 1 order by 1` + select chain, method, count(*)::int as runs, + count(*) filter (where answers > 1)::int as disagreeing + from runs group by 1, 2 order by 1, 2` ); const minority = await q<{ provider: string; times_in_minority: number }>( `with answers as ( select evm_shadow_of_trace_id as run, evm_block_number as block, evm_provider as provider, evm_result_hash as hash from api_events - where project_id = $1 and evm_shadow_of_trace_id is not null and evm_result_shape <> 'error_in_body' + where project_id = $1 and evm_shadow_of_trace_id is not null and evm_result_shape not in ('error_in_body', 'null') ), counted as ( select run, block, hash, count(*) as votes from answers group by 1, 2, 3 ), majority as ( @@ -164,7 +183,7 @@ const table = (head: string[], rows: Array>) => ...rows.map((r) => `| ${r.join(" | ")} |`) ].join("\n"); -const report = `# Ethereum mainnet RPC reliability +const report = `# EVM RPC reliability Project \`${projectId}\` · ${window.first.toISOString()} → ${window.last.toISOString()} (${hours.toFixed(1)} h) · ${window.calls.toLocaleString()} primary calls · polled every ${intervalSeconds}s @@ -195,24 +214,29 @@ ${table( ]) )} -## Chain head lag (vs. the highest head any provider reported in the same tick) +## Chain head lag (vs. the highest head any provider on the same chain reported in the same tick) ${table( - ["Provider", "Samples", "≥ 2 blocks behind", "> 3 blocks behind", "Worst"], + ["Provider", "Samples", "≥ 24 s behind", "> 36 s behind", "Worst"], headLag.map((r) => [ r.provider, r.samples, - pct(r.lag_ge_2, r.samples), - pct(r.lag_gt_3, r.samples), - `${r.max_lag} blocks` + pct(r.lag_ge_24s, r.samples), + pct(r.lag_gt_36s, r.samples), + `${r.max_lag_s} s` ]) )} ## Disagreement at equal height (verification runs) ${table( - ["Method", "Runs", "Runs with differing answers"], - disagreements.map((r) => [r.method, r.runs, `${r.disagreeing} (${pct(r.disagreeing, r.runs)})`]) + ["Chain", "Method", "Runs", "Runs with differing answers"], + disagreements.map((r) => [ + CHAIN_NAMES[r.chain] ?? r.chain ?? "—", + r.method, + r.runs, + `${r.disagreeing} (${pct(r.disagreeing, r.runs)})` + ]) )} ${