From 8a50c02d68b2794d8350c363779e5bae14c7f721 Mon Sep 17 00:00:00 2001 From: macjayz Date: Sun, 1 Nov 2026 10:37:55 +0100 Subject: [PATCH] study: report local outages apart from provider failures The collector polls every provider in the same tick, so a client-side "fetch failed" on most of them at once is the collector's own connection dropping. On a 64.6 h laptop run that was 15.7% of ticks, and it showed up as a ~9.5% failure rate on every provider. Those ticks are now counted in their own section and left out of availability, latency and head lag. The "JSON-RPC error inside 200" column also counted errors on any status code, so it always matched transport failures. It now counts 200s only. Co-Authored-By: Claude Opus 5.5 --- scripts/study/report.ts | 42 +++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/scripts/study/report.ts b/scripts/study/report.ts index f94069b..67fb4b7 100644 --- a/scripts/study/report.ts +++ b/scripts/study/report.ts @@ -5,6 +5,10 @@ * errors inside 200 responses, rate limiting, latency, how often each provider's chain head * trailed the others, and how often providers disagreed at the same block height. * + * Ticks where most providers failed at once with a client-side network error are the + * collector's own connection dropping, not the providers. They are reported on their own and + * left out of availability, latency and head lag. + * * Usage: DATABASE_URL=... npx tsx scripts/study/report.ts [--out report.md] * The project comes from STUDY_PROJECT_ID, or from the collector's STUDY_STATE file. */ @@ -32,12 +36,33 @@ const q = async (sql: string, params: unknown[] = [projectId]) => // Primary calls only: verification (shadow) calls are reported separately as disagreements. const PRIMARY = `project_id = $1 and kind = 'evm_rpc' and evm_shadow_of_trace_id is null`; +// The collector polls every provider in the same tick. A client-side network error on most of +// them at once is the collector's own connection, and says nothing about any one provider. +const TICK = `floor(extract(epoch from timestamp) / $2)`; +const LOCAL_OUTAGE_TICKS = ` + select ${TICK} as tick from api_events + where ${PRIMARY} + group by 1 + having 2 * count(distinct evm_provider) filter (where evm_rpc_error_message = 'fetch failed') + > count(distinct evm_provider)`; +const MEASURED = `${PRIMARY} and ${TICK} not in (${LOCAL_OUTAGE_TICKS})`; +const withInterval = [projectId, intervalSeconds]; + const [window] = await q<{ first: Date; last: Date; calls: number }>( `select min(timestamp) as first, max(timestamp) as last, count(*)::int as calls from api_events where ${PRIMARY}` ); if (!window?.calls) throw new Error(`no study data for project ${projectId}`); const hours = (window.last.getTime() - window.first.getTime()) / 3_600_000; +const [outages] = await q<{ ticks: number; total_ticks: number; calls: number }>( + `select + (select count(*) from (${LOCAL_OUTAGE_TICKS}) o)::int as ticks, + count(distinct ${TICK})::int as total_ticks, + count(*) filter (where ${TICK} in (${LOCAL_OUTAGE_TICKS}))::int as calls + from api_events where ${PRIMARY}`, + withInterval +); + const availability = await q<{ provider: string; calls: number; @@ -49,12 +74,13 @@ const availability = await q<{ }>( `select evm_provider as provider, count(*)::int as calls, count(*) filter (where status_code >= 400)::int as transport_failed, - count(*) filter (where evm_result_shape = 'error_in_body')::int as error_in_body, + count(*) filter (where status_code < 400 and evm_result_shape = 'error_in_body')::int as error_in_body, count(*) filter (where evm_throttled)::int as throttled, percentile_cont(0.5) within group (order by latency_ms) filter (where status_code < 400) as p50, percentile_cont(0.95) within group (order by latency_ms) filter (where status_code < 400) as p95 - from api_events where ${PRIMARY} - group by 1 order by 1` + from api_events where ${MEASURED} + group by 1 order by 1`, + withInterval ); // Every provider is polled at the same moment each tick, so within one tick the highest head any @@ -69,9 +95,9 @@ const headLag = await q<{ }>( `with heads as ( select evm_provider as provider, evm_block_number::bigint as head, - floor(extract(epoch from timestamp) / $2) as tick + ${TICK} as tick from api_events - where ${PRIMARY} and evm_rpc_method = 'eth_blockNumber' and evm_block_number is not null + where ${MEASURED} and evm_rpc_method = 'eth_blockNumber' and evm_block_number is not null ), lagged as ( select provider, max(head) over (partition by tick) - head as lag from heads ) @@ -80,7 +106,7 @@ const headLag = await q<{ count(*) filter (where lag > 3)::int as lag_gt_3, coalesce(max(lag), 0)::int as max_lag from lagged group by 1 order by 1`, - [projectId, intervalSeconds] + withInterval ); // One verification run = every endpoint answering the same call pinned to the same height. @@ -142,6 +168,10 @@ const report = `# Ethereum mainnet RPC reliability Project \`${projectId}\` · ${window.first.toISOString()} → ${window.last.toISOString()} (${hours.toFixed(1)} h) · ${window.calls.toLocaleString()} primary calls · polled every ${intervalSeconds}s +## Local outages (excluded) + +${outages.ticks} of ${outages.total_ticks} ticks (${pct(outages.ticks, outages.total_ticks)}), ${outages.calls.toLocaleString()} calls: most providers failed at once with a client-side network error, which is the collector's own connection rather than any provider. Availability, latency and head lag below leave these ticks out; detector findings and incidents do not. + ## Availability and answers ${table(