From 940c04124586d5a11babb28fac3a8cd632309674 Mon Sep 17 00:00:00 2001 From: Pavel Volkov Date: Wed, 5 Aug 2026 13:53:19 +0300 Subject: [PATCH 1/2] feat: collect per-template deployment counts and failures workspaceTemplates only carries a rounded health percentage, which cannot answer how many deploys failed in a given period: health is computed over a rolling window, so it climbs back up as old failures age out of it. Collect templateMetrics alongside it. It reports the deploy counts behind that percentage, and storing them turns health into a number of failed deploys - both as failed_deployments per snapshot and, via the template_deployment_failures view, as the deploys and failures added between consecutive runs. The query takes one template id per call, so it is built with an alias per template and sent in batches of 40: Railway rejects a query whose breadth exceeds 500 and each template costs ten nodes. Deploy counts are a different population from the existing project counts - they include redeploys of an existing install - so they go in their own table rather than as columns on template_snapshots. --- ingest/README.md | 25 ++++++ ingest/db/client.ts | 87 +++++++++++++++++-- ingest/db/schemas/gauge.sql | 73 ++++++++++++++++ ingest/db/types.ts | 24 +++++ ingest/index.ts | 28 ++++++ ingest/railway/client.ts | 58 +++++++++++++ .../railway/queries/template-metrics.test.ts | 71 +++++++++++++++ ingest/railway/queries/template-metrics.ts | 52 +++++++++++ ingest/railway/types.ts | 24 +++++ 9 files changed, 436 insertions(+), 6 deletions(-) create mode 100644 ingest/railway/queries/template-metrics.test.ts create mode 100644 ingest/railway/queries/template-metrics.ts diff --git a/ingest/README.md b/ingest/README.md index c845cca..56c7ae6 100644 --- a/ingest/README.md +++ b/ingest/README.md @@ -2,6 +2,31 @@ Railway Template Metrics Ingestor. Collects earnings and template metrics from Railway API and persists them to PostgreSQL. +## What gets collected + +Each run writes three sets of rows: + +| Table | Source | Contents | +|-------|--------|----------| +| `earnings_snapshots` | `earningDetails` | Workspace earnings, in cents | +| `template_snapshots` | `workspaceTemplates` | Per-template metadata, project counts, payout | +| `template_deployment_snapshots` | `templateMetrics` | Per-template deploy counts, health, earnings in dollars | + +The two template tables count different things. `template_snapshots.projects` +counts projects created from a template; `template_deployment_snapshots.total_deployments` +counts deploys, redeploys of an existing install included, and that is the +population Railway's health percentage is calculated over. + +Health on its own cannot answer "how many deploys failed since yesterday" - it is +a percentage over a rolling window, so it climbs back up as old failures age out +of that window. `failed_deployments` stores the count the percentage implies, and +the `template_deployment_failures` view differences consecutive snapshots into +`deployments_added` and `failures_added`, which does answer it. + +`templateMetrics` accepts one template id per call, so the query is built with an +alias per template and sent in batches of 40 - Railway rejects a query whose +breadth exceeds 500, and each template costs ten nodes. + ## Environment Variables The following environment variables are required: diff --git a/ingest/db/client.ts b/ingest/db/client.ts index 1d6834e..9249a14 100644 --- a/ingest/db/client.ts +++ b/ingest/db/client.ts @@ -5,10 +5,22 @@ import { SQL } from 'bun'; import type { Logger } from 'pino'; -import type { EarningDetails, Template } from '../railway/types'; +import type { EarningDetails, Template, TemplateMetrics } from '../railway/types'; import { DatabaseError, SchemaError, InsertError } from './errors'; import path from 'path'; +/** + * Tables the schema file is expected to provide. Adding a table here makes + * ensureSchema re-run the schema file on databases created before it existed; + * every statement in that file is idempotent, so re-running is safe. + */ +const REQUIRED_TABLES = [ + 'earnings_snapshots', + 'template_snapshots', + 'template_metrics_derived', + 'template_deployment_snapshots', +] as const; + export interface GaugeDatabaseConfig { connectionString: string; logger?: Logger; @@ -37,7 +49,7 @@ export class GaugeDatabase { SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' - AND table_name IN ('earnings_snapshots', 'template_snapshots', 'template_metrics_derived') + AND table_name IN ${this.sql(REQUIRED_TABLES)} ORDER BY table_name `; @@ -58,7 +70,7 @@ export class GaugeDatabase { // Check if tables exist const existingTables = await this.checkTablesExist(); - if (existingTables.length === 3) { + if (existingTables.length === REQUIRED_TABLES.length) { this.logger?.info( { tableCount: existingTables.length }, 'Database schema already exists' @@ -67,7 +79,7 @@ export class GaugeDatabase { } this.logger?.info( - { existingTables: existingTables.length, required: 3 }, + { existingTables: existingTables.length, required: REQUIRED_TABLES.length }, 'Database schema incomplete, creating schema' ); @@ -84,9 +96,9 @@ export class GaugeDatabase { // Verify tables were created const tables = await this.checkTablesExist(); - if (tables.length < 3) { + if (tables.length < REQUIRED_TABLES.length) { throw new SchemaError( - `Failed to create all required tables (created ${tables.length}/3)` + `Failed to create all required tables (created ${tables.length}/${REQUIRED_TABLES.length})` ); } @@ -227,6 +239,69 @@ export class GaugeDatabase { } } + /** + * Insert per-template deployment snapshots into database + * + * Templates without metrics are skipped rather than stored as zeroes, so a + * gap in the series means "not reported" and never "no deploys". + */ + async insertTemplateDeploymentSnapshots( + templates: Template[], + metrics: Map, + collectedAt: Date + ): Promise { + const rows = templates + .filter((t) => metrics.has(t.id)) + .map((t) => { + const m = metrics.get(t.id)!; + + // Health is the share of deploys that succeeded, so the rest failed. + const failedDeployments = Math.round( + (m.totalDeployments * (100 - m.templateHealth)) / 100 + ); + + return { + collected_at: collectedAt, + template_id: t.id, + template_code: t.code ?? null, + template_name: t.name, + total_deployments: m.totalDeployments, + deployments_last_90d: m.deploymentsLast90Days, + active_deployments: m.activeDeployments, + template_health: m.templateHealth, + support_health: m.supportHealth, + total_earnings: m.totalEarnings, + earnings_last_30d: m.earningsLast30Days, + earnings_last_90d: m.earningsLast90Days, + eligible_for_support_bonus: m.eligibleForSupportBonus, + failed_deployments: failedDeployments, + }; + }); + + this.logger?.info({ count: rows.length }, 'Inserting template deployment snapshots'); + + if (rows.length === 0) { + this.logger?.warn('No template deployment metrics to insert'); + return 0; + } + + try { + await this.sql` + INSERT INTO template_deployment_snapshots ${this.sql(rows)} + ON CONFLICT (collected_at, template_id) DO NOTHING + `; + + this.logger?.info( + { count: rows.length }, + 'Template deployment snapshots inserted successfully' + ); + + return rows.length; + } catch (error) { + throw new InsertError('Failed to insert template deployment snapshots', error); + } + } + /** * Calculate and store derived metrics (growth rates, profitability scores) * Matches Python: lines 543-634 in collect_metrics.py diff --git a/ingest/db/schemas/gauge.sql b/ingest/db/schemas/gauge.sql index 7d0095e..b7c0f21 100644 --- a/ingest/db/schemas/gauge.sql +++ b/ingest/db/schemas/gauge.sql @@ -84,6 +84,49 @@ CREATE INDEX IF NOT EXISTS idx_template_id ON template_snapshots(template_id); CREATE INDEX IF NOT EXISTS idx_template_category ON template_snapshots(category); CREATE INDEX IF NOT EXISTS idx_template_total_payout ON template_snapshots(total_payout DESC); +-- Table: template_deployment_snapshots +-- Stores per-template deploy counters from the creator-private templateMetrics +-- query. These count deploys, including redeploys of an existing install, and +-- are therefore a different population from template_snapshots.projects, which +-- counts projects created from the template. +CREATE TABLE IF NOT EXISTS template_deployment_snapshots ( + id SERIAL PRIMARY KEY, + collected_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + + -- Template identification + template_id TEXT NOT NULL, + template_code TEXT, + template_name TEXT NOT NULL, + + -- Deploy counters + total_deployments INTEGER NOT NULL, + deployments_last_90d INTEGER NOT NULL, + active_deployments INTEGER NOT NULL, + + -- Health percentages over Railway's rolling window + template_health NUMERIC(5,2), + support_health NUMERIC(5,2), + + -- Earnings, in dollars: templateMetrics reports dollars where + -- earnings_snapshots reports cents + total_earnings NUMERIC(12,2), + earnings_last_30d NUMERIC(12,2), + earnings_last_90d NUMERIC(12,2), + eligible_for_support_bonus BOOLEAN, + + -- Failed deploys implied by health, stored so the count is queryable + -- directly instead of being re-derived in every query + failed_deployments INTEGER, + + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + + UNIQUE(collected_at, template_id) +); + +CREATE INDEX IF NOT EXISTS idx_deployment_collected_at ON template_deployment_snapshots(collected_at DESC); +CREATE INDEX IF NOT EXISTS idx_deployment_template_id ON template_deployment_snapshots(template_id); +CREATE INDEX IF NOT EXISTS idx_deployment_health ON template_deployment_snapshots(template_health); + -- Table: template_metrics_derived -- Stores calculated time-series metrics (revenue growth, etc.) -- This table is populated by comparing snapshots over time @@ -186,6 +229,33 @@ LEFT JOIN previous p ON l.template_id = p.template_id WHERE l.health < 70 OR l.active_projects < COALESCE(p.prev_active_projects, l.active_projects); +-- View: template_deployment_failures +-- Deploys and failures added between consecutive snapshots of a template. +-- health alone cannot answer "how many deploys failed since yesterday": it is a +-- percentage over a rolling window, so it recovers as old failures age out. The +-- deltas here do answer it. +CREATE OR REPLACE VIEW template_deployment_failures AS +SELECT + curr.collected_at, + curr.template_id, + curr.template_name, + curr.template_health, + curr.total_deployments, + curr.failed_deployments, + curr.total_deployments - prev.total_deployments AS deployments_added, + curr.failed_deployments - prev.failed_deployments AS failures_added, + prev.collected_at AS compared_to +FROM template_deployment_snapshots curr +LEFT JOIN LATERAL ( + SELECT total_deployments, failed_deployments, collected_at + FROM template_deployment_snapshots prev + WHERE prev.template_id = curr.template_id + AND prev.collected_at < curr.collected_at + ORDER BY prev.collected_at DESC + LIMIT 1 +) prev ON true +ORDER BY curr.collected_at DESC, failures_added DESC NULLS LAST; + -- Function: calculate_profitability_score -- Calculates a composite profitability score for a template -- Weights: Revenue 40%, Growth 30%, Retention 20%, Health 10% @@ -227,6 +297,9 @@ $$ LANGUAGE plpgsql; COMMENT ON TABLE earnings_snapshots IS 'Stores overall earnings data snapshots collected every 12 hours'; COMMENT ON TABLE template_snapshots IS 'Stores individual template performance metrics collected every 12 hours'; COMMENT ON TABLE template_metrics_derived IS 'Stores calculated metrics derived from comparing snapshots over time'; +COMMENT ON TABLE template_deployment_snapshots IS 'Stores per-template deploy counters and health from the creator-private templateMetrics query'; +COMMENT ON COLUMN template_deployment_snapshots.total_deployments IS 'Deploys of this template, including redeploys of an existing install - not the same population as template_snapshots.projects'; +COMMENT ON COLUMN template_deployment_snapshots.failed_deployments IS 'Deploys implied to have failed by health: total_deployments * (100 - template_health) / 100, rounded'; COMMENT ON COLUMN template_snapshots.retention_rate IS 'Percentage of total projects that are still active (active/total * 100)'; COMMENT ON COLUMN template_snapshots.revenue_per_active IS 'Average revenue per active project (total_payout / active_projects)'; COMMENT ON COLUMN template_snapshots.growth_momentum IS 'Recent projects as percentage of active projects (recent/active * 100)'; diff --git a/ingest/db/types.ts b/ingest/db/types.ts index 35264e9..4f7e337 100644 --- a/ingest/db/types.ts +++ b/ingest/db/types.ts @@ -55,6 +55,29 @@ export const TemplateSnapshotSchema = z.object({ created_at: z.date(), }); +/** + * Schema for template_deployment_snapshots table rows + * Deploy counters from templateMetrics, plus the failure count they imply + */ +export const TemplateDeploymentSnapshotSchema = z.object({ + id: z.number(), + collected_at: z.date(), + template_id: z.string(), + template_code: z.string().nullable(), + template_name: z.string(), + total_deployments: z.number(), + deployments_last_90d: z.number(), + active_deployments: z.number(), + template_health: z.number().nullable(), + support_health: z.number().nullable(), + total_earnings: z.number().nullable(), + earnings_last_30d: z.number().nullable(), + earnings_last_90d: z.number().nullable(), + eligible_for_support_bonus: z.boolean().nullable(), + failed_deployments: z.number().nullable(), + created_at: z.date(), +}); + /** * Schema for template_metrics_derived table rows */ @@ -78,4 +101,5 @@ export const TemplateMetricsDerivedSchema = z.object({ // Export inferred TypeScript types export type EarningsSnapshot = z.infer; export type TemplateSnapshot = z.infer; +export type TemplateDeploymentSnapshot = z.infer; export type TemplateMetricsDerived = z.infer; diff --git a/ingest/index.ts b/ingest/index.ts index 56ddc2e..07b4b76 100644 --- a/ingest/index.ts +++ b/ingest/index.ts @@ -78,6 +78,32 @@ async function collectMetrics() { await db.insertTemplateSnapshots(templates, startTime); logger.info({ count: templates.length }, 'Template snapshots persisted'); + // Fetch deployment metrics. workspaceTemplates only carries the rounded + // health percentage; the deploy counts behind it come from templateMetrics, + // and their difference between runs is what turns health into a number of + // failed deploys. + logger.info('Fetching template deployment metrics from Railway...'); + const deploymentMetrics = await railwayClient.getTemplateMetrics( + templates.map((t) => t.id), + ); + const totalDeployments = [...deploymentMetrics.values()] + .reduce((sum, m) => sum + m.totalDeployments, 0); + const failedDeployments = [...deploymentMetrics.values()] + .reduce((sum, m) => sum + Math.round((m.totalDeployments * (100 - m.templateHealth)) / 100), 0); + + logger.info( + { count: deploymentMetrics.size, totalDeployments, failedDeployments }, + 'Deployment metrics fetched', + ); + + logger.info({ count: deploymentMetrics.size }, 'Persisting deployment snapshots to database...'); + const deploymentRows = await db.insertTemplateDeploymentSnapshots( + templates, + deploymentMetrics, + startTime, + ); + logger.info({ count: deploymentRows }, 'Deployment snapshots persisted'); + // Calculate derived metrics logger.info('Calculating derived metrics...'); await db.calculateDerivedMetrics(startTime); @@ -92,6 +118,8 @@ async function collectMetrics() { executionTime: `${duration.toFixed(2)}s`, templatesProcessed: templates.length, templateRevenue: `$${(totalRevenue / 100).toFixed(2)}`, + deploymentsTracked: totalDeployments, + deploymentsFailed: failedDeployments, }, 'METRICS COLLECTION COMPLETED SUCCESSFULLY'); } catch (error) { diff --git a/ingest/railway/client.ts b/ingest/railway/client.ts index 94ac73d..1c95eda 100644 --- a/ingest/railway/client.ts +++ b/ingest/railway/client.ts @@ -8,13 +8,22 @@ import type { Logger } from 'pino'; import { EarningsResponseSchema, + TemplateMetricsResponseSchema, TemplatesResponseSchema, WorkspaceResponseSchema, type EarningDetails, type Template, + type TemplateMetrics, } from './types'; import { EARNINGS_QUERY, EARNINGS_OPERATION } from './queries/earnings'; import { TEMPLATES_QUERY, TEMPLATES_OPERATION } from './queries/templates'; +import { + buildTemplateMetricsQuery, + buildTemplateMetricsVariables, + metricsAlias, + TEMPLATE_METRICS_BATCH_SIZE, + TEMPLATE_METRICS_OPERATION, +} from './queries/template-metrics'; import { WORKSPACE_QUERY, WORKSPACE_OPERATION } from './queries/workspace'; export interface RailwayClientConfig { @@ -201,6 +210,55 @@ export class RailwayClient { return templates; } + /** + * Fetch deployment metrics for the given templates + * + * Returns deploy counts and the health percentage they imply, keyed by + * template id. Templates the API reports nothing for are simply absent from + * the map, so a missing entry is not an error. + */ + async getTemplateMetrics(templateIds: string[]): Promise> { + if (templateIds.length === 0) { + this.logger?.warn('No template ids given, skipping deployment metrics'); + return new Map(); + } + + this.logger?.info({ count: templateIds.length }, 'Fetching template deployment metrics from Railway'); + + const metrics = new Map(); + + // Batched because Railway caps query breadth; see TEMPLATE_METRICS_BATCH_SIZE. + for (let offset = 0; offset < templateIds.length; offset += TEMPLATE_METRICS_BATCH_SIZE) { + const batch = templateIds.slice(offset, offset + TEMPLATE_METRICS_BATCH_SIZE); + + const response = await this.executeQuery( + buildTemplateMetricsQuery(batch), + buildTemplateMetricsVariables(batch), + TEMPLATE_METRICS_OPERATION, + (data) => TemplateMetricsResponseSchema.parse(data), + ); + + batch.forEach((id, index) => { + const node = response[metricsAlias(index)]; + if (node) { + metrics.set(id, node); + } + }); + + this.logger?.debug( + { offset, batchSize: batch.length, collected: metrics.size }, + 'Fetched a batch of template deployment metrics', + ); + } + + this.logger?.info( + { requested: templateIds.length, returned: metrics.size }, + 'Successfully fetched template deployment metrics', + ); + + return metrics; + } + /** * Validate credentials by making a test API call * diff --git a/ingest/railway/queries/template-metrics.test.ts b/ingest/railway/queries/template-metrics.test.ts new file mode 100644 index 0000000..ee83e04 --- /dev/null +++ b/ingest/railway/queries/template-metrics.test.ts @@ -0,0 +1,71 @@ +/** + * Tests for the templateMetrics query builder + * + * The builder exists because templateMetrics takes one id at a time. The rules + * it has to keep are that aliases match between query and variables, and that a + * batch stays under Railway's query breadth cap. + */ + +import { describe, test, expect } from 'bun:test'; +import { + buildTemplateMetricsQuery, + buildTemplateMetricsVariables, + metricsAlias, + TEMPLATE_METRICS_BATCH_SIZE, +} from './template-metrics'; + +// Each template contributes its own node plus the fields selected on it. +const NODES_PER_TEMPLATE = 10; +const BREADTH_LIMIT = 500; + +describe('buildTemplateMetricsQuery', () => { + test('declares one variable and one alias per template', () => { + const query = buildTemplateMetricsQuery(['a', 'b', 'c']); + + expect(query).toContain('$t0: String!'); + expect(query).toContain('$t2: String!'); + expect(query).toContain('t0: templateMetrics(id: $t0)'); + expect(query).toContain('t2: templateMetrics(id: $t2)'); + expect(query.match(/templateMetrics\(id:/g)).toHaveLength(3); + }); + + test('selects the fields the snapshot table stores', () => { + const query = buildTemplateMetricsQuery(['a']); + + for (const field of [ + 'templateHealth', + 'supportHealth', + 'totalDeployments', + 'deploymentsLast90Days', + 'activeDeployments', + 'totalEarnings', + 'earningsLast30Days', + 'earningsLast90Days', + 'eligibleForSupportBonus', + ]) { + expect(query).toContain(field); + } + }); + + test('variables use the aliases the query declares', () => { + const ids = ['first', 'second']; + const variables = buildTemplateMetricsVariables(ids); + + expect(variables).toEqual({ t0: 'first', t1: 'second' }); + for (const [index, id] of ids.entries()) { + expect(variables[metricsAlias(index)]).toBe(id); + } + }); + + test('a full batch stays under the breadth cap', () => { + const ids = Array.from({ length: TEMPLATE_METRICS_BATCH_SIZE }, (_, i) => `id-${i}`); + const query = buildTemplateMetricsQuery(ids); + + expect(TEMPLATE_METRICS_BATCH_SIZE * NODES_PER_TEMPLATE).toBeLessThan(BREADTH_LIMIT); + expect(query.match(/templateMetrics\(id:/g)).toHaveLength(TEMPLATE_METRICS_BATCH_SIZE); + }); + + test('empty input produces no variables', () => { + expect(buildTemplateMetricsVariables([])).toEqual({}); + }); +}); diff --git a/ingest/railway/queries/template-metrics.ts b/ingest/railway/queries/template-metrics.ts new file mode 100644 index 0000000..2d2b659 --- /dev/null +++ b/ingest/railway/queries/template-metrics.ts @@ -0,0 +1,52 @@ +/** + * GraphQL query for per-template deployment metrics. + * + * These are the creator-private counters behind the number shown on a template + * card: how many times the template was deployed and what share of those deploys + * succeeded. `workspaceTemplates` only exposes the rounded percentage, so the + * failure count has to come from here. + * + * templateMetrics takes one template id, so the query is assembled with an alias + * per template and the whole workspace arrives in a single round trip. + */ + +export const TEMPLATE_METRICS_OPERATION = 'templateMetrics'; + +/** + * Templates per request. + * + * Railway rejects a query whose breadth exceeds 500, and each template costs its + * own node plus the nine fields below - ten. Fifty would sit exactly on the + * limit, so this leaves room for a field to be added later without the query + * starting to fail in production. + */ +export const TEMPLATE_METRICS_BATCH_SIZE = 40; + +const METRIC_FIELDS = ` + templateHealth + supportHealth + totalDeployments + deploymentsLast90Days + activeDeployments + totalEarnings + earningsLast30Days + earningsLast90Days + eligibleForSupportBonus`; + +/** Alias used for the template at `index`, shared by the query and its parser. */ +export const metricsAlias = (index: number): string => `t${index}`; + +export function buildTemplateMetricsQuery(templateIds: string[]): string { + const params = templateIds.map((_, i) => `$${metricsAlias(i)}: String!`).join(', '); + const selections = templateIds + .map((_, i) => ` ${metricsAlias(i)}: templateMetrics(id: $${metricsAlias(i)}) {${METRIC_FIELDS}\n }`) + .join('\n'); + + return `query ${TEMPLATE_METRICS_OPERATION}(${params}) {\n${selections}\n }`; +} + +export function buildTemplateMetricsVariables( + templateIds: string[], +): Record { + return Object.fromEntries(templateIds.map((id, i) => [metricsAlias(i), id])); +} diff --git a/ingest/railway/types.ts b/ingest/railway/types.ts index 4f91d4a..513e8ec 100644 --- a/ingest/railway/types.ts +++ b/ingest/railway/types.ts @@ -42,6 +42,20 @@ export const TemplateSchema = z.object({ totalPayout: z.number().nullable().optional().transform(v => v ?? 0), }); +// Per-template deployment metrics schema (creator-private counters) +export const TemplateMetricsSchema = z.object({ + templateHealth: z.number(), + supportHealth: z.number(), + totalDeployments: z.number(), + deploymentsLast90Days: z.number(), + activeDeployments: z.number(), + // Earnings here are dollars, unlike the cents used by earningDetails + totalEarnings: z.number(), + earningsLast30Days: z.number(), + earningsLast90Days: z.number(), + eligibleForSupportBonus: z.boolean(), +}); + // GraphQL error schema export const GraphQLErrorSchema = z.object({ message: z.string(), @@ -73,6 +87,14 @@ export const TemplatesResponseSchema = z.object({ }), }); +// Template metrics response schema +// One entry per alias in the generated query; a template the API declines to +// report on comes back as null rather than as an error. +export const TemplateMetricsResponseSchema = z.record( + z.string(), + TemplateMetricsSchema.nullable(), +); + // Workspace response schema (for fetching customer ID) export const WorkspaceResponseSchema = z.object({ workspace: z.object({ @@ -85,6 +107,8 @@ export const WorkspaceResponseSchema = z.object({ // Infer TypeScript types from schemas export type EarningDetails = z.infer; export type Template = z.infer; +export type TemplateMetrics = z.infer; +export type TemplateMetricsResponse = z.infer; export type GraphQLError = z.infer; export type EarningsResponse = z.infer; export type TemplatesResponse = z.infer; From 6d3d90fcaa24b724b51d102c8b4771d1bfcc00d0 Mon Sep 17 00:00:00 2001 From: Pavel Volkov Date: Thu, 6 Aug 2026 11:43:04 +0300 Subject: [PATCH 2/2] fix: drop the derived failure count, it cannot be computed Deploys are a lifetime counter; health is a percentage over a rolling window whose size Railway does not publish. Multiplying them is unsound, and the first two snapshots showed it: a template whose six deploys aged out of the window went from "6 failed" to "0 failed" with no deploy in between - minus six failures in a day. templateMetrics also answers templateHealth 100 when it has nothing to report, which reads exactly like a genuine 100%. The public health field is null in that case, so health_reported is derived from it and the view returns NULL health rather than a default dressed up as a measurement. The view keeps what holds up - deployments_added off the monotonic counter, and health_change beside it - and is renamed to template_deployment_activity since it no longer reports failures. --- ingest/README.md | 26 +++++++++++++++----- ingest/db/client.ts | 10 +++----- ingest/db/schemas/gauge.sql | 49 ++++++++++++++++++++++++------------- ingest/db/types.ts | 2 +- ingest/index.ts | 11 ++++++--- 5 files changed, 64 insertions(+), 34 deletions(-) diff --git a/ingest/README.md b/ingest/README.md index 56c7ae6..2c43666 100644 --- a/ingest/README.md +++ b/ingest/README.md @@ -10,18 +10,32 @@ Each run writes three sets of rows: |-------|--------|----------| | `earnings_snapshots` | `earningDetails` | Workspace earnings, in cents | | `template_snapshots` | `workspaceTemplates` | Per-template metadata, project counts, payout | -| `template_deployment_snapshots` | `templateMetrics` | Per-template deploy counts, health, earnings in dollars | +| `template_deployment_snapshots` | `templateMetrics` + public `health` | Per-template deploy counts, health, earnings in dollars | The two template tables count different things. `template_snapshots.projects` counts projects created from a template; `template_deployment_snapshots.total_deployments` counts deploys, redeploys of an existing install included, and that is the population Railway's health percentage is calculated over. -Health on its own cannot answer "how many deploys failed since yesterday" - it is -a percentage over a rolling window, so it climbs back up as old failures age out -of that window. `failed_deployments` stores the count the percentage implies, and -the `template_deployment_failures` view differences consecutive snapshots into -`deployments_added` and `failures_added`, which does answer it. +### Why there is no failure count + +It is tempting to turn health into failed deploys as +`total_deployments * (100 - health) / 100`. That is wrong, and the data says so +plainly: deploys are a lifetime counter while health is a percentage over a +rolling window whose size Railway does not publish. Once the window empties, the +formula reports zero failures for deploys that did fail - one template went from +"6 failed" to "0 failed" without a single deploy in between, a change of minus +six failures in a day. + +`templateMetrics` also answers `templateHealth: 100` when it has nothing to +report, which is indistinguishable from a real 100%. The public `health` field is +`null` in that case, so it is what the `health_reported` column is derived from; +read health without it and a template with no data looks perfectly healthy. + +What the `template_deployment_activity` view does report is what holds up: +`deployments_added` (the counter is monotonic, so the difference is real) and +`health_change` next to it. A health drop against a positive `deployments_added` +is the signal worth chasing. `templateMetrics` accepts one template id per call, so the query is built with an alias per template and sent in batches of 40 - Railway rejects a query whose diff --git a/ingest/db/client.ts b/ingest/db/client.ts index 9249a14..1f0b641 100644 --- a/ingest/db/client.ts +++ b/ingest/db/client.ts @@ -255,11 +255,6 @@ export class GaugeDatabase { .map((t) => { const m = metrics.get(t.id)!; - // Health is the share of deploys that succeeded, so the rest failed. - const failedDeployments = Math.round( - (m.totalDeployments * (100 - m.templateHealth)) / 100 - ); - return { collected_at: collectedAt, template_id: t.id, @@ -269,12 +264,15 @@ export class GaugeDatabase { deployments_last_90d: m.deploymentsLast90Days, active_deployments: m.activeDeployments, template_health: m.templateHealth, + // templateMetrics reports 100 when it has no data; the public + // health field is null in that case, so it is what tells the two + // apart. + health_reported: t.health !== null && t.health !== undefined, support_health: m.supportHealth, total_earnings: m.totalEarnings, earnings_last_30d: m.earningsLast30Days, earnings_last_90d: m.earningsLast90Days, eligible_for_support_bonus: m.eligibleForSupportBonus, - failed_deployments: failedDeployments, }; }); diff --git a/ingest/db/schemas/gauge.sql b/ingest/db/schemas/gauge.sql index b7c0f21..e870959 100644 --- a/ingest/db/schemas/gauge.sql +++ b/ingest/db/schemas/gauge.sql @@ -103,8 +103,14 @@ CREATE TABLE IF NOT EXISTS template_deployment_snapshots ( deployments_last_90d INTEGER NOT NULL, active_deployments INTEGER NOT NULL, - -- Health percentages over Railway's rolling window + -- Health percentages over Railway's rolling window. + -- templateMetrics reports 100 when it has nothing to report, which is + -- indistinguishable from a genuine 100. health_reported carries whether + -- workspaceTemplates returned a health for the same template in the same + -- run: it is NULL there when there is no data, so false here means + -- template_health is a default and not a measurement. template_health NUMERIC(5,2), + health_reported BOOLEAN NOT NULL DEFAULT false, support_health NUMERIC(5,2), -- Earnings, in dollars: templateMetrics reports dollars where @@ -114,10 +120,6 @@ CREATE TABLE IF NOT EXISTS template_deployment_snapshots ( earnings_last_90d NUMERIC(12,2), eligible_for_support_bonus BOOLEAN, - -- Failed deploys implied by health, stored so the count is queryable - -- directly instead of being re-derived in every query - failed_deployments INTEGER, - created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), UNIQUE(collected_at, template_id) @@ -229,32 +231,44 @@ LEFT JOIN previous p ON l.template_id = p.template_id WHERE l.health < 70 OR l.active_projects < COALESCE(p.prev_active_projects, l.active_projects); --- View: template_deployment_failures --- Deploys and failures added between consecutive snapshots of a template. --- health alone cannot answer "how many deploys failed since yesterday": it is a --- percentage over a rolling window, so it recovers as old failures age out. The --- deltas here do answer it. -CREATE OR REPLACE VIEW template_deployment_failures AS +-- View: template_deployment_activity +-- What changed for a template between consecutive snapshots. +-- +-- Deliberately does not report a failure count. Railway exposes deploys as a +-- lifetime counter and success as a percentage over a rolling window, and the +-- window's size is not published - so the two cannot be multiplied into a +-- number of failed deploys. Doing that produces nonsense as soon as the window +-- empties: a template whose six deploys aged out went from "6 failed" to +-- "0 failed" with no deploy in between, i.e. minus six failures in a day. +-- +-- What is sound: deployments_added (the counter is monotonic, so the difference +-- is real) and health_change alongside it. A drop in health against a positive +-- deployments_added is the signal worth looking at. +CREATE OR REPLACE VIEW template_deployment_activity AS SELECT curr.collected_at, curr.template_id, curr.template_name, - curr.template_health, curr.total_deployments, - curr.failed_deployments, + curr.active_deployments, + -- NULL rather than a number when Railway reported no health this run + CASE WHEN curr.health_reported THEN curr.template_health END AS template_health, curr.total_deployments - prev.total_deployments AS deployments_added, - curr.failed_deployments - prev.failed_deployments AS failures_added, + CASE + WHEN curr.health_reported AND prev.health_reported + THEN curr.template_health - prev.template_health + END AS health_change, prev.collected_at AS compared_to FROM template_deployment_snapshots curr LEFT JOIN LATERAL ( - SELECT total_deployments, failed_deployments, collected_at + SELECT total_deployments, template_health, health_reported, collected_at FROM template_deployment_snapshots prev WHERE prev.template_id = curr.template_id AND prev.collected_at < curr.collected_at ORDER BY prev.collected_at DESC LIMIT 1 ) prev ON true -ORDER BY curr.collected_at DESC, failures_added DESC NULLS LAST; +ORDER BY curr.collected_at DESC, deployments_added DESC NULLS LAST; -- Function: calculate_profitability_score -- Calculates a composite profitability score for a template @@ -299,7 +313,8 @@ COMMENT ON TABLE template_snapshots IS 'Stores individual template performance m COMMENT ON TABLE template_metrics_derived IS 'Stores calculated metrics derived from comparing snapshots over time'; COMMENT ON TABLE template_deployment_snapshots IS 'Stores per-template deploy counters and health from the creator-private templateMetrics query'; COMMENT ON COLUMN template_deployment_snapshots.total_deployments IS 'Deploys of this template, including redeploys of an existing install - not the same population as template_snapshots.projects'; -COMMENT ON COLUMN template_deployment_snapshots.failed_deployments IS 'Deploys implied to have failed by health: total_deployments * (100 - template_health) / 100, rounded'; +COMMENT ON COLUMN template_deployment_snapshots.template_health IS 'Percentage of deploys that succeeded over Railway rolling window; 100 is also what the API returns when it has no data, so read it together with health_reported'; +COMMENT ON COLUMN template_deployment_snapshots.health_reported IS 'False when workspaceTemplates returned no health for this template in the same run, meaning template_health is a default rather than a measurement'; COMMENT ON COLUMN template_snapshots.retention_rate IS 'Percentage of total projects that are still active (active/total * 100)'; COMMENT ON COLUMN template_snapshots.revenue_per_active IS 'Average revenue per active project (total_payout / active_projects)'; COMMENT ON COLUMN template_snapshots.growth_momentum IS 'Recent projects as percentage of active projects (recent/active * 100)'; diff --git a/ingest/db/types.ts b/ingest/db/types.ts index 4f7e337..6deb567 100644 --- a/ingest/db/types.ts +++ b/ingest/db/types.ts @@ -69,12 +69,12 @@ export const TemplateDeploymentSnapshotSchema = z.object({ deployments_last_90d: z.number(), active_deployments: z.number(), template_health: z.number().nullable(), + health_reported: z.boolean(), support_health: z.number().nullable(), total_earnings: z.number().nullable(), earnings_last_30d: z.number().nullable(), earnings_last_90d: z.number().nullable(), eligible_for_support_bonus: z.boolean().nullable(), - failed_deployments: z.number().nullable(), created_at: z.date(), }); diff --git a/ingest/index.ts b/ingest/index.ts index 07b4b76..6be81fe 100644 --- a/ingest/index.ts +++ b/ingest/index.ts @@ -88,11 +88,14 @@ async function collectMetrics() { ); const totalDeployments = [...deploymentMetrics.values()] .reduce((sum, m) => sum + m.totalDeployments, 0); - const failedDeployments = [...deploymentMetrics.values()] - .reduce((sum, m) => sum + Math.round((m.totalDeployments * (100 - m.templateHealth)) / 100), 0); + // Counted off the public health field: templateMetrics answers 100 when it + // has nothing to report, so it cannot be used to tell measured from missing. + const withHealth = templates.filter( + (t) => deploymentMetrics.has(t.id) && t.health !== null && t.health !== undefined, + ).length; logger.info( - { count: deploymentMetrics.size, totalDeployments, failedDeployments }, + { count: deploymentMetrics.size, totalDeployments, withReportedHealth: withHealth }, 'Deployment metrics fetched', ); @@ -119,7 +122,7 @@ async function collectMetrics() { templatesProcessed: templates.length, templateRevenue: `$${(totalRevenue / 100).toFixed(2)}`, deploymentsTracked: totalDeployments, - deploymentsFailed: failedDeployments, + templatesWithReportedHealth: withHealth, }, 'METRICS COLLECTION COMPLETED SUCCESSFULLY'); } catch (error) {