study: add Base, Arbitrum, tip blocks and log ranges

docker-compose.yml

@@ -83,7 +83,7 @@ services:
     depends_on:
       - api
 
-  # RPC reliability study: polls public mainnet endpoints through @heightproof/web3 with
+  # RPC reliability study: polls public Ethereum, Base and Arbitrum endpoints through @heightproof/web3 with
   # cross-provider verification. Opt-in: docker compose --profile study up -d
   # Report: docker compose --profile study run --rm study npm run study:report
   study:

scripts/study/collect.ts

@@ -1,8 +1,9 @@
 /**
  * RPC reliability study: collector.
  *
- * Polls several Ethereum mainnet endpoints on a fixed interval through @heightproof/web3, with
- * cross-provider verification on, so Heightproof records every answer and every detector finding.
+ * Polls public endpoints on Ethereum mainnet, Base and Arbitrum on a fixed interval through
+ * @heightproof/web3, with cross-provider verification on among the endpoints of each chain, so
+ * Heightproof records every answer and every detector finding.
  * Meant to run unattended for days (see the `study` compose profile); `report.ts` turns what it
  * collected into numbers.
  *
@@ -14,36 +15,106 @@ import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { dirname } from "node:path";
 import { createHeightproofRpcClient } from "@heightproof/web3";
 
-type Endpoint = { name: string; url: string };
+type Chain = {
+  chainId: number;
+  /** USDC on this chain: a busy contract, for reads and for log queries. */
+  usdc: string;
+  /**
+   * Blocks a log query spans. Free public tiers cap it (BlastAPI at 10 blocks) and treat ranges
+   * far from the head as archive requests, so it stays short and recent.
+   */
+  logRange: number;
+  /** How far behind the head a log range ends, about 10 s, so a provider a block behind is not a mismatch. */
+  logLag: number;
+};
+type Endpoint = { name: string; url: string; chain: string };
 type StudyState = { projectId: string; apiKey: string; startedAt: string };
 
+const CHAINS: Record<string, Chain> = {
+  ethereum: {
+    chainId: 1,
+    usdc: "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48",
+    logRange: 10,
+    logLag: 3
+  },
+  base: {
+    chainId: 8453,
+    usdc: "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
+    logRange: 10,
+    logLag: 5
+  },
+  arbitrum: {
+    chainId: 42161,
+    usdc: "0xaf88d065e77c8cC2239327C5EDb3A432268e5831",
+    logRange: 10,
+    logLag: 40
+  }
+};
+
+// `name=url` is an Ethereum mainnet endpoint; `chain/name=url` names its chain.
 const DEFAULT_ENDPOINTS =
   "publicnode=https://ethereum-rpc.publicnode.com," +
   "drpc=https://eth.drpc.org," +
   "blastapi=https://eth-mainnet.public.blastapi.io," +
   "mevblocker=https://rpc.mevblocker.io," +
-  "onfinality=https://eth.api.onfinality.io/public";
+  "onfinality=https://eth.api.onfinality.io/public," +
+  "base/official=https://mainnet.base.org," +
+  "base/publicnode=https://base-rpc.publicnode.com," +
+  "base/drpc=https://base.drpc.org," +
+  "base/blastapi=https://base-mainnet.public.blastapi.io," +
+  "arbitrum/official=https://arb1.arbitrum.io/rpc," +
+  "arbitrum/publicnode=https://arbitrum-one-rpc.publicnode.com," +
+  "arbitrum/drpc=https://arbitrum.drpc.org," +
+  "arbitrum/blastapi=https://arbitrum-one.public.blastapi.io";
 
 const api = process.env.HEIGHTPROOF_ENDPOINT ?? "http://localhost:8080";
 const statePath = process.env.STUDY_STATE ?? ".study/state.json";
 const intervalMs = Number(process.env.STUDY_INTERVAL_MS ?? 15_000);
 const sampleRate = Number(process.env.STUDY_VERIFY_SAMPLE_RATE ?? 0.2);
 const callTimeoutMs = Number(process.env.STUDY_CALL_TIMEOUT_MS ?? 10_000);
+// Log queries return the most data by far, so they run on every Nth tick only.
+const logsEveryTicks = Number(process.env.STUDY_LOGS_EVERY_TICKS ?? 4);
 const endpoints: Endpoint[] = (process.env.STUDY_ENDPOINTS ?? DEFAULT_ENDPOINTS)
   .split(",")
   .map((entry) => {
-    const [name, ...url] = entry.trim().split("=");
-    return { name, url: url.join("=") };
+    const [label, ...url] = entry.trim().split("=");
+    const chain = label.includes("/") ? label.slice(0, label.indexOf("/")) : "ethereum";
+    if (!CHAINS[chain]) throw new Error(`unknown chain "${chain}" in STUDY_ENDPOINTS`);
+    return { name: label, url: url.join("="), chain };
   });
 
 // Deterministic reads whose answers every honest provider must agree on at a given height.
 const WALLET = "0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045";
-const USDC = "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48";
-const CALLS: Array<{ method: string; params: unknown[] }> = [
-  { method: "eth_blockNumber", params: [] },
-  { method: "eth_getBalance", params: [WALLET, "latest"] },
-  { method: "eth_call", params: [{ to: USDC, data: "0x18160ddd" }, "latest"] }
-];
+const TRANSFER_TOPIC = "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef";
+
+type Call = { method: string; params: unknown[] };
+
+/**
+ * One tick's calls against one endpoint. The block at the tip is where reorgs and lagging nodes
+ * show; a log range over a busy contract is where providers silently truncate.
+ */
+function callsFor(chain: Chain, head: number | undefined, tick: number): Call[] {
+  const calls: Call[] = [
+    { method: "eth_getBalance", params: [WALLET, "latest"] },
+    { method: "eth_call", params: [{ to: chain.usdc, data: "0x18160ddd" }, "latest"] },
+    { method: "eth_getBlockByNumber", params: ["latest", false] }
+  ];
+  if (head !== undefined && tick % logsEveryTicks === 0) {
+    const toBlock = head - chain.logLag;
+    calls.push({
+      method: "eth_getLogs",
+      params: [
+        {
+          address: chain.usdc,
+          topics: [TRANSFER_TOPIC],
+          fromBlock: `0x${(toBlock - chain.logRange + 1).toString(16)}`,
+          toBlock: `0x${toBlock.toString(16)}`
+        }
+      ]
+    });
+  }
+  return calls;
+}
 
 const state = await loadOrCreateState();
 console.log(
@@ -58,12 +129,12 @@ const clients = endpoints.map((endpoint) => ({
     endpoint: api,
     rpcUrl: endpoint.url,
     provider: endpoint.name,
-    chainId: 1,
+    chainId: CHAINS[endpoint.chain].chainId,
     serviceName: "rpc-study",
     environment: "study",
     verify: {
       endpoints: endpoints
-        .filter((other) => other !== endpoint)
+        .filter((other) => other !== endpoint && other.chain === endpoint.chain)
         .map((other) => ({ rpcUrl: other.url, provider: other.name })),
       sampleRate,
       maxCallsPerMinute: 20,
@@ -81,7 +152,16 @@ for (let tick = 1; !stopping; tick++) {
   const started = Date.now();
   await Promise.all(
     clients.map(async ({ endpoint, client }) => {
-      for (const call of CALLS) {
+      const head = await withTimeout(
+        client.request({ method: "eth_blockNumber", params: [] }),
+        callTimeoutMs
+      ).then(
+        (result) => (typeof result === "string" ? Number.parseInt(result, 16) : undefined),
+        () => undefined
+      );
+      totals.get(endpoint.name)![head === undefined ? "failed" : "ok"] += 1;
+
+      for (const call of callsFor(CHAINS[endpoint.chain], head, tick)) {
         const outcome = await withTimeout(client.request(call), callTimeoutMs).then(
           () => "ok" as const,
           () => "failed" as const

scripts/study/report.ts

@@ -3,7 +3,7 @@
  *
  * Reads what the collector recorded and prints a Markdown summary per provider: availability,
  * errors inside 200 responses, rate limiting, latency, how often each provider's chain head
- * trailed the others, and how often providers disagreed at the same block height.
+ * trailed the others on its chain, and how often providers disagreed at the same block height.
  *
  * Ticks where most providers failed at once with a client-side network error are the
  * collector's own connection dropping, not the providers. They are reported on their own and
@@ -48,6 +48,15 @@ const LOCAL_OUTAGE_TICKS = `
 const MEASURED = `${PRIMARY} and ${TICK} not in (${LOCAL_OUTAGE_TICKS})`;
 const withInterval = [projectId, intervalSeconds];
 
+const CHAIN_NAMES: Record<string, string> = {
+  "1": "Ethereum",
+  "8453": "Base",
+  "42161": "Arbitrum"
+};
+// Block time in seconds, so head lag compares across chains: two blocks is 24 s on Ethereum
+// but half a second on Arbitrum.
+const BLOCK_SECONDS = `(case evm_chain_id when '8453' then 2 when '42161' then 0.25 else 12 end)`;
+
 const [window] = await q<{ first: Date; last: Date; calls: number }>(
   `select min(timestamp) as first, max(timestamp) as last, count(*)::int as calls from api_events where ${PRIMARY}`
 );
@@ -84,49 +93,59 @@ const availability = await q<{
 );
 
 // Every provider is polled at the same moment each tick, so within one tick the highest head any
-// provider reported is the reference. Lag of 1 block is mostly a block landing mid-tick; the
-// stale-head detector's threshold is more than 3.
+// provider on the same chain reported is the reference. Up to one Ethereum block is mostly a block
+// landing mid-tick; the stale-head detector's threshold is more than 3 blocks (36 s).
 const headLag = await q<{
   provider: string;
   samples: number;
-  lag_ge_2: number;
-  lag_gt_3: number;
-  max_lag: number;
+  lag_ge_24s: number;
+  lag_gt_36s: number;
+  max_lag_s: number;
 }>(
   `with heads as (
-     select evm_provider as provider, evm_block_number::bigint as head,
-            ${TICK} as tick
+     select evm_provider as provider, evm_chain_id as chain, evm_block_number::bigint as head,
+            ${BLOCK_SECONDS} as block_seconds, ${TICK} as tick
      from api_events
      where ${MEASURED} and evm_rpc_method = 'eth_blockNumber' and evm_block_number is not null
    ), lagged as (
-     select provider, max(head) over (partition by tick) - head as lag from heads
+     select provider, (max(head) over (partition by chain, tick) - head) * block_seconds as lag_s
+     from heads
    )
    select provider, count(*)::int as samples,
-     count(*) filter (where lag >= 2)::int as lag_ge_2,
-     count(*) filter (where lag > 3)::int as lag_gt_3,
-     coalesce(max(lag), 0)::int as max_lag
+     count(*) filter (where lag_s >= 24)::int as lag_ge_24s,
+     count(*) filter (where lag_s > 36)::int as lag_gt_36s,
+     coalesce(max(lag_s), 0)::float as max_lag_s
    from lagged group by 1 order by 1`,
   withInterval
 );
 
-// One verification run = every endpoint answering the same call pinned to the same height.
-const disagreements = await q<{ method: string; runs: number; disagreeing: number }>(
+// One verification run = every endpoint answering the same call pinned to the same height. An
+// error or a null is an endpoint that could not answer (often one not yet at that height), never
+// a differing answer.
+const disagreements = await q<{
+  chain: string;
+  method: string;
+  runs: number;
+  disagreeing: number;
+}>(
   `with runs as (
-     select evm_shadow_of_trace_id as run, evm_rpc_method as method, evm_block_number as block,
-            count(distinct evm_result_hash) filter (where evm_result_shape <> 'error_in_body') as answers
+     select evm_shadow_of_trace_id as run, evm_chain_id as chain, evm_rpc_method as method,
+            evm_block_number as block,
+            count(distinct evm_result_hash) filter (where evm_result_shape not in ('error_in_body', 'null')) as answers
      from api_events
      where project_id = $1 and evm_shadow_of_trace_id is not null
-     group by 1, 2, 3
+     group by 1, 2, 3, 4
    )
-   select method, count(*)::int as runs, count(*) filter (where answers > 1)::int as disagreeing
-   from runs group by 1 order by 1`
+   select chain, method, count(*)::int as runs,
+     count(*) filter (where answers > 1)::int as disagreeing
+   from runs group by 1, 2 order by 1, 2`
 );
 
 const minority = await q<{ provider: string; times_in_minority: number }>(
   `with answers as (
      select evm_shadow_of_trace_id as run, evm_block_number as block, evm_provider as provider, evm_result_hash as hash
      from api_events
-     where project_id = $1 and evm_shadow_of_trace_id is not null and evm_result_shape <> 'error_in_body'
+     where project_id = $1 and evm_shadow_of_trace_id is not null and evm_result_shape not in ('error_in_body', 'null')
    ), counted as (
      select run, block, hash, count(*) as votes from answers group by 1, 2, 3
    ), majority as (
@@ -164,7 +183,7 @@ const table = (head: string[], rows: Array<Array<string | number>>) =>
     ...rows.map((r) => `| ${r.join(" | ")} |`)
   ].join("\n");
 
-const report = `# Ethereum mainnet RPC reliability
+const report = `# EVM RPC reliability
 
 Project \`${projectId}\` · ${window.first.toISOString()} → ${window.last.toISOString()} (${hours.toFixed(1)} h) · ${window.calls.toLocaleString()} primary calls · polled every ${intervalSeconds}s
 
@@ -195,24 +214,29 @@ ${table(
   ])
 )}
 
-## Chain head lag (vs. the highest head any provider reported in the same tick)
+## Chain head lag (vs. the highest head any provider on the same chain reported in the same tick)
 
 ${table(
-  ["Provider", "Samples", "≥ 2 blocks behind", "> 3 blocks behind", "Worst"],
+  ["Provider", "Samples", "≥ 24 s behind", "> 36 s behind", "Worst"],
   headLag.map((r) => [
     r.provider,
     r.samples,
-    pct(r.lag_ge_2, r.samples),
-    pct(r.lag_gt_3, r.samples),
-    `${r.max_lag} blocks`
+    pct(r.lag_ge_24s, r.samples),
+    pct(r.lag_gt_36s, r.samples),
+    `${r.max_lag_s} s`
   ])
 )}
 
 ## Disagreement at equal height (verification runs)
 
 ${table(
-  ["Method", "Runs", "Runs with differing answers"],
-  disagreements.map((r) => [r.method, r.runs, `${r.disagreeing} (${pct(r.disagreeing, r.runs)})`])
+  ["Chain", "Method", "Runs", "Runs with differing answers"],
+  disagreements.map((r) => [
+    CHAIN_NAMES[r.chain] ?? r.chain ?? "—",
+    r.method,
+    r.runs,
+    `${r.disagreeing} (${pct(r.disagreeing, r.runs)})`
+  ])
 )}
 
 ${