diff --git a/ingest/README.md b/ingest/README.md index c845cca..2c43666 100644 --- a/ingest/README.md +++ b/ingest/README.md @@ -2,6 +2,45 @@ Railway Template Metrics Ingestor. Collects earnings and template metrics from Railway API and persists them to PostgreSQL. +## What gets collected + +Each run writes three sets of rows: + +| Table | Source | Contents | +|-------|--------|----------| +| `earnings_snapshots` | `earningDetails` | Workspace earnings, in cents | +| `template_snapshots` | `workspaceTemplates` | Per-template metadata, project counts, payout | +| `template_deployment_snapshots` | `templateMetrics` + public `health` | Per-template deploy counts, health, earnings in dollars | + +The two template tables count different things. `template_snapshots.projects` +counts projects created from a template; `template_deployment_snapshots.total_deployments` +counts deploys, redeploys of an existing install included, and that is the +population Railway's health percentage is calculated over. + +### Why there is no failure count + +It is tempting to turn health into failed deploys as +`total_deployments * (100 - health) / 100`. That is wrong, and the data says so +plainly: deploys are a lifetime counter while health is a percentage over a +rolling window whose size Railway does not publish. Once the window empties, the +formula reports zero failures for deploys that did fail - one template went from +"6 failed" to "0 failed" without a single deploy in between, a change of minus +six failures in a day. + +`templateMetrics` also answers `templateHealth: 100` when it has nothing to +report, which is indistinguishable from a real 100%. The public `health` field is +`null` in that case, so it is what the `health_reported` column is derived from; +read health without it and a template with no data looks perfectly healthy. + +What the `template_deployment_activity` view does report is what holds up: +`deployments_added` (the counter is monotonic, so the difference is real) and +`health_change` next to it. A health drop against a positive `deployments_added` +is the signal worth chasing. + +`templateMetrics` accepts one template id per call, so the query is built with an +alias per template and sent in batches of 40 - Railway rejects a query whose +breadth exceeds 500, and each template costs ten nodes. + ## Environment Variables The following environment variables are required: diff --git a/ingest/db/client.ts b/ingest/db/client.ts index 1d6834e..1f0b641 100644 --- a/ingest/db/client.ts +++ b/ingest/db/client.ts @@ -5,10 +5,22 @@ import { SQL } from 'bun'; import type { Logger } from 'pino'; -import type { EarningDetails, Template } from '../railway/types'; +import type { EarningDetails, Template, TemplateMetrics } from '../railway/types'; import { DatabaseError, SchemaError, InsertError } from './errors'; import path from 'path'; +/** + * Tables the schema file is expected to provide. Adding a table here makes + * ensureSchema re-run the schema file on databases created before it existed; + * every statement in that file is idempotent, so re-running is safe. + */ +const REQUIRED_TABLES = [ + 'earnings_snapshots', + 'template_snapshots', + 'template_metrics_derived', + 'template_deployment_snapshots', +] as const; + export interface GaugeDatabaseConfig { connectionString: string; logger?: Logger; @@ -37,7 +49,7 @@ export class GaugeDatabase { SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' - AND table_name IN ('earnings_snapshots', 'template_snapshots', 'template_metrics_derived') + AND table_name IN ${this.sql(REQUIRED_TABLES)} ORDER BY table_name `; @@ -58,7 +70,7 @@ export class GaugeDatabase { // Check if tables exist const existingTables = await this.checkTablesExist(); - if (existingTables.length === 3) { + if (existingTables.length === REQUIRED_TABLES.length) { this.logger?.info( { tableCount: existingTables.length }, 'Database schema already exists' @@ -67,7 +79,7 @@ export class GaugeDatabase { } this.logger?.info( - { existingTables: existingTables.length, required: 3 }, + { existingTables: existingTables.length, required: REQUIRED_TABLES.length }, 'Database schema incomplete, creating schema' ); @@ -84,9 +96,9 @@ export class GaugeDatabase { // Verify tables were created const tables = await this.checkTablesExist(); - if (tables.length < 3) { + if (tables.length < REQUIRED_TABLES.length) { throw new SchemaError( - `Failed to create all required tables (created ${tables.length}/3)` + `Failed to create all required tables (created ${tables.length}/${REQUIRED_TABLES.length})` ); } @@ -227,6 +239,67 @@ export class GaugeDatabase { } } + /** + * Insert per-template deployment snapshots into database + * + * Templates without metrics are skipped rather than stored as zeroes, so a + * gap in the series means "not reported" and never "no deploys". + */ + async insertTemplateDeploymentSnapshots( + templates: Template[], + metrics: Map, + collectedAt: Date + ): Promise { + const rows = templates + .filter((t) => metrics.has(t.id)) + .map((t) => { + const m = metrics.get(t.id)!; + + return { + collected_at: collectedAt, + template_id: t.id, + template_code: t.code ?? null, + template_name: t.name, + total_deployments: m.totalDeployments, + deployments_last_90d: m.deploymentsLast90Days, + active_deployments: m.activeDeployments, + template_health: m.templateHealth, + // templateMetrics reports 100 when it has no data; the public + // health field is null in that case, so it is what tells the two + // apart. + health_reported: t.health !== null && t.health !== undefined, + support_health: m.supportHealth, + total_earnings: m.totalEarnings, + earnings_last_30d: m.earningsLast30Days, + earnings_last_90d: m.earningsLast90Days, + eligible_for_support_bonus: m.eligibleForSupportBonus, + }; + }); + + this.logger?.info({ count: rows.length }, 'Inserting template deployment snapshots'); + + if (rows.length === 0) { + this.logger?.warn('No template deployment metrics to insert'); + return 0; + } + + try { + await this.sql` + INSERT INTO template_deployment_snapshots ${this.sql(rows)} + ON CONFLICT (collected_at, template_id) DO NOTHING + `; + + this.logger?.info( + { count: rows.length }, + 'Template deployment snapshots inserted successfully' + ); + + return rows.length; + } catch (error) { + throw new InsertError('Failed to insert template deployment snapshots', error); + } + } + /** * Calculate and store derived metrics (growth rates, profitability scores) * Matches Python: lines 543-634 in collect_metrics.py diff --git a/ingest/db/schemas/gauge.sql b/ingest/db/schemas/gauge.sql index 7d0095e..e870959 100644 --- a/ingest/db/schemas/gauge.sql +++ b/ingest/db/schemas/gauge.sql @@ -84,6 +84,51 @@ CREATE INDEX IF NOT EXISTS idx_template_id ON template_snapshots(template_id); CREATE INDEX IF NOT EXISTS idx_template_category ON template_snapshots(category); CREATE INDEX IF NOT EXISTS idx_template_total_payout ON template_snapshots(total_payout DESC); +-- Table: template_deployment_snapshots +-- Stores per-template deploy counters from the creator-private templateMetrics +-- query. These count deploys, including redeploys of an existing install, and +-- are therefore a different population from template_snapshots.projects, which +-- counts projects created from the template. +CREATE TABLE IF NOT EXISTS template_deployment_snapshots ( + id SERIAL PRIMARY KEY, + collected_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + + -- Template identification + template_id TEXT NOT NULL, + template_code TEXT, + template_name TEXT NOT NULL, + + -- Deploy counters + total_deployments INTEGER NOT NULL, + deployments_last_90d INTEGER NOT NULL, + active_deployments INTEGER NOT NULL, + + -- Health percentages over Railway's rolling window. + -- templateMetrics reports 100 when it has nothing to report, which is + -- indistinguishable from a genuine 100. health_reported carries whether + -- workspaceTemplates returned a health for the same template in the same + -- run: it is NULL there when there is no data, so false here means + -- template_health is a default and not a measurement. + template_health NUMERIC(5,2), + health_reported BOOLEAN NOT NULL DEFAULT false, + support_health NUMERIC(5,2), + + -- Earnings, in dollars: templateMetrics reports dollars where + -- earnings_snapshots reports cents + total_earnings NUMERIC(12,2), + earnings_last_30d NUMERIC(12,2), + earnings_last_90d NUMERIC(12,2), + eligible_for_support_bonus BOOLEAN, + + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + + UNIQUE(collected_at, template_id) +); + +CREATE INDEX IF NOT EXISTS idx_deployment_collected_at ON template_deployment_snapshots(collected_at DESC); +CREATE INDEX IF NOT EXISTS idx_deployment_template_id ON template_deployment_snapshots(template_id); +CREATE INDEX IF NOT EXISTS idx_deployment_health ON template_deployment_snapshots(template_health); + -- Table: template_metrics_derived -- Stores calculated time-series metrics (revenue growth, etc.) -- This table is populated by comparing snapshots over time @@ -186,6 +231,45 @@ LEFT JOIN previous p ON l.template_id = p.template_id WHERE l.health < 70 OR l.active_projects < COALESCE(p.prev_active_projects, l.active_projects); +-- View: template_deployment_activity +-- What changed for a template between consecutive snapshots. +-- +-- Deliberately does not report a failure count. Railway exposes deploys as a +-- lifetime counter and success as a percentage over a rolling window, and the +-- window's size is not published - so the two cannot be multiplied into a +-- number of failed deploys. Doing that produces nonsense as soon as the window +-- empties: a template whose six deploys aged out went from "6 failed" to +-- "0 failed" with no deploy in between, i.e. minus six failures in a day. +-- +-- What is sound: deployments_added (the counter is monotonic, so the difference +-- is real) and health_change alongside it. A drop in health against a positive +-- deployments_added is the signal worth looking at. +CREATE OR REPLACE VIEW template_deployment_activity AS +SELECT + curr.collected_at, + curr.template_id, + curr.template_name, + curr.total_deployments, + curr.active_deployments, + -- NULL rather than a number when Railway reported no health this run + CASE WHEN curr.health_reported THEN curr.template_health END AS template_health, + curr.total_deployments - prev.total_deployments AS deployments_added, + CASE + WHEN curr.health_reported AND prev.health_reported + THEN curr.template_health - prev.template_health + END AS health_change, + prev.collected_at AS compared_to +FROM template_deployment_snapshots curr +LEFT JOIN LATERAL ( + SELECT total_deployments, template_health, health_reported, collected_at + FROM template_deployment_snapshots prev + WHERE prev.template_id = curr.template_id + AND prev.collected_at < curr.collected_at + ORDER BY prev.collected_at DESC + LIMIT 1 +) prev ON true +ORDER BY curr.collected_at DESC, deployments_added DESC NULLS LAST; + -- Function: calculate_profitability_score -- Calculates a composite profitability score for a template -- Weights: Revenue 40%, Growth 30%, Retention 20%, Health 10% @@ -227,6 +311,10 @@ $$ LANGUAGE plpgsql; COMMENT ON TABLE earnings_snapshots IS 'Stores overall earnings data snapshots collected every 12 hours'; COMMENT ON TABLE template_snapshots IS 'Stores individual template performance metrics collected every 12 hours'; COMMENT ON TABLE template_metrics_derived IS 'Stores calculated metrics derived from comparing snapshots over time'; +COMMENT ON TABLE template_deployment_snapshots IS 'Stores per-template deploy counters and health from the creator-private templateMetrics query'; +COMMENT ON COLUMN template_deployment_snapshots.total_deployments IS 'Deploys of this template, including redeploys of an existing install - not the same population as template_snapshots.projects'; +COMMENT ON COLUMN template_deployment_snapshots.template_health IS 'Percentage of deploys that succeeded over Railway rolling window; 100 is also what the API returns when it has no data, so read it together with health_reported'; +COMMENT ON COLUMN template_deployment_snapshots.health_reported IS 'False when workspaceTemplates returned no health for this template in the same run, meaning template_health is a default rather than a measurement'; COMMENT ON COLUMN template_snapshots.retention_rate IS 'Percentage of total projects that are still active (active/total * 100)'; COMMENT ON COLUMN template_snapshots.revenue_per_active IS 'Average revenue per active project (total_payout / active_projects)'; COMMENT ON COLUMN template_snapshots.growth_momentum IS 'Recent projects as percentage of active projects (recent/active * 100)'; diff --git a/ingest/db/types.ts b/ingest/db/types.ts index 35264e9..6deb567 100644 --- a/ingest/db/types.ts +++ b/ingest/db/types.ts @@ -55,6 +55,29 @@ export const TemplateSnapshotSchema = z.object({ created_at: z.date(), }); +/** + * Schema for template_deployment_snapshots table rows + * Deploy counters from templateMetrics, plus the failure count they imply + */ +export const TemplateDeploymentSnapshotSchema = z.object({ + id: z.number(), + collected_at: z.date(), + template_id: z.string(), + template_code: z.string().nullable(), + template_name: z.string(), + total_deployments: z.number(), + deployments_last_90d: z.number(), + active_deployments: z.number(), + template_health: z.number().nullable(), + health_reported: z.boolean(), + support_health: z.number().nullable(), + total_earnings: z.number().nullable(), + earnings_last_30d: z.number().nullable(), + earnings_last_90d: z.number().nullable(), + eligible_for_support_bonus: z.boolean().nullable(), + created_at: z.date(), +}); + /** * Schema for template_metrics_derived table rows */ @@ -78,4 +101,5 @@ export const TemplateMetricsDerivedSchema = z.object({ // Export inferred TypeScript types export type EarningsSnapshot = z.infer; export type TemplateSnapshot = z.infer; +export type TemplateDeploymentSnapshot = z.infer; export type TemplateMetricsDerived = z.infer; diff --git a/ingest/index.ts b/ingest/index.ts index 56ddc2e..6be81fe 100644 --- a/ingest/index.ts +++ b/ingest/index.ts @@ -78,6 +78,35 @@ async function collectMetrics() { await db.insertTemplateSnapshots(templates, startTime); logger.info({ count: templates.length }, 'Template snapshots persisted'); + // Fetch deployment metrics. workspaceTemplates only carries the rounded + // health percentage; the deploy counts behind it come from templateMetrics, + // and their difference between runs is what turns health into a number of + // failed deploys. + logger.info('Fetching template deployment metrics from Railway...'); + const deploymentMetrics = await railwayClient.getTemplateMetrics( + templates.map((t) => t.id), + ); + const totalDeployments = [...deploymentMetrics.values()] + .reduce((sum, m) => sum + m.totalDeployments, 0); + // Counted off the public health field: templateMetrics answers 100 when it + // has nothing to report, so it cannot be used to tell measured from missing. + const withHealth = templates.filter( + (t) => deploymentMetrics.has(t.id) && t.health !== null && t.health !== undefined, + ).length; + + logger.info( + { count: deploymentMetrics.size, totalDeployments, withReportedHealth: withHealth }, + 'Deployment metrics fetched', + ); + + logger.info({ count: deploymentMetrics.size }, 'Persisting deployment snapshots to database...'); + const deploymentRows = await db.insertTemplateDeploymentSnapshots( + templates, + deploymentMetrics, + startTime, + ); + logger.info({ count: deploymentRows }, 'Deployment snapshots persisted'); + // Calculate derived metrics logger.info('Calculating derived metrics...'); await db.calculateDerivedMetrics(startTime); @@ -92,6 +121,8 @@ async function collectMetrics() { executionTime: `${duration.toFixed(2)}s`, templatesProcessed: templates.length, templateRevenue: `$${(totalRevenue / 100).toFixed(2)}`, + deploymentsTracked: totalDeployments, + templatesWithReportedHealth: withHealth, }, 'METRICS COLLECTION COMPLETED SUCCESSFULLY'); } catch (error) { diff --git a/ingest/railway/client.ts b/ingest/railway/client.ts index 94ac73d..1c95eda 100644 --- a/ingest/railway/client.ts +++ b/ingest/railway/client.ts @@ -8,13 +8,22 @@ import type { Logger } from 'pino'; import { EarningsResponseSchema, + TemplateMetricsResponseSchema, TemplatesResponseSchema, WorkspaceResponseSchema, type EarningDetails, type Template, + type TemplateMetrics, } from './types'; import { EARNINGS_QUERY, EARNINGS_OPERATION } from './queries/earnings'; import { TEMPLATES_QUERY, TEMPLATES_OPERATION } from './queries/templates'; +import { + buildTemplateMetricsQuery, + buildTemplateMetricsVariables, + metricsAlias, + TEMPLATE_METRICS_BATCH_SIZE, + TEMPLATE_METRICS_OPERATION, +} from './queries/template-metrics'; import { WORKSPACE_QUERY, WORKSPACE_OPERATION } from './queries/workspace'; export interface RailwayClientConfig { @@ -201,6 +210,55 @@ export class RailwayClient { return templates; } + /** + * Fetch deployment metrics for the given templates + * + * Returns deploy counts and the health percentage they imply, keyed by + * template id. Templates the API reports nothing for are simply absent from + * the map, so a missing entry is not an error. + */ + async getTemplateMetrics(templateIds: string[]): Promise> { + if (templateIds.length === 0) { + this.logger?.warn('No template ids given, skipping deployment metrics'); + return new Map(); + } + + this.logger?.info({ count: templateIds.length }, 'Fetching template deployment metrics from Railway'); + + const metrics = new Map(); + + // Batched because Railway caps query breadth; see TEMPLATE_METRICS_BATCH_SIZE. + for (let offset = 0; offset < templateIds.length; offset += TEMPLATE_METRICS_BATCH_SIZE) { + const batch = templateIds.slice(offset, offset + TEMPLATE_METRICS_BATCH_SIZE); + + const response = await this.executeQuery( + buildTemplateMetricsQuery(batch), + buildTemplateMetricsVariables(batch), + TEMPLATE_METRICS_OPERATION, + (data) => TemplateMetricsResponseSchema.parse(data), + ); + + batch.forEach((id, index) => { + const node = response[metricsAlias(index)]; + if (node) { + metrics.set(id, node); + } + }); + + this.logger?.debug( + { offset, batchSize: batch.length, collected: metrics.size }, + 'Fetched a batch of template deployment metrics', + ); + } + + this.logger?.info( + { requested: templateIds.length, returned: metrics.size }, + 'Successfully fetched template deployment metrics', + ); + + return metrics; + } + /** * Validate credentials by making a test API call * diff --git a/ingest/railway/queries/template-metrics.test.ts b/ingest/railway/queries/template-metrics.test.ts new file mode 100644 index 0000000..ee83e04 --- /dev/null +++ b/ingest/railway/queries/template-metrics.test.ts @@ -0,0 +1,71 @@ +/** + * Tests for the templateMetrics query builder + * + * The builder exists because templateMetrics takes one id at a time. The rules + * it has to keep are that aliases match between query and variables, and that a + * batch stays under Railway's query breadth cap. + */ + +import { describe, test, expect } from 'bun:test'; +import { + buildTemplateMetricsQuery, + buildTemplateMetricsVariables, + metricsAlias, + TEMPLATE_METRICS_BATCH_SIZE, +} from './template-metrics'; + +// Each template contributes its own node plus the fields selected on it. +const NODES_PER_TEMPLATE = 10; +const BREADTH_LIMIT = 500; + +describe('buildTemplateMetricsQuery', () => { + test('declares one variable and one alias per template', () => { + const query = buildTemplateMetricsQuery(['a', 'b', 'c']); + + expect(query).toContain('$t0: String!'); + expect(query).toContain('$t2: String!'); + expect(query).toContain('t0: templateMetrics(id: $t0)'); + expect(query).toContain('t2: templateMetrics(id: $t2)'); + expect(query.match(/templateMetrics\(id:/g)).toHaveLength(3); + }); + + test('selects the fields the snapshot table stores', () => { + const query = buildTemplateMetricsQuery(['a']); + + for (const field of [ + 'templateHealth', + 'supportHealth', + 'totalDeployments', + 'deploymentsLast90Days', + 'activeDeployments', + 'totalEarnings', + 'earningsLast30Days', + 'earningsLast90Days', + 'eligibleForSupportBonus', + ]) { + expect(query).toContain(field); + } + }); + + test('variables use the aliases the query declares', () => { + const ids = ['first', 'second']; + const variables = buildTemplateMetricsVariables(ids); + + expect(variables).toEqual({ t0: 'first', t1: 'second' }); + for (const [index, id] of ids.entries()) { + expect(variables[metricsAlias(index)]).toBe(id); + } + }); + + test('a full batch stays under the breadth cap', () => { + const ids = Array.from({ length: TEMPLATE_METRICS_BATCH_SIZE }, (_, i) => `id-${i}`); + const query = buildTemplateMetricsQuery(ids); + + expect(TEMPLATE_METRICS_BATCH_SIZE * NODES_PER_TEMPLATE).toBeLessThan(BREADTH_LIMIT); + expect(query.match(/templateMetrics\(id:/g)).toHaveLength(TEMPLATE_METRICS_BATCH_SIZE); + }); + + test('empty input produces no variables', () => { + expect(buildTemplateMetricsVariables([])).toEqual({}); + }); +}); diff --git a/ingest/railway/queries/template-metrics.ts b/ingest/railway/queries/template-metrics.ts new file mode 100644 index 0000000..2d2b659 --- /dev/null +++ b/ingest/railway/queries/template-metrics.ts @@ -0,0 +1,52 @@ +/** + * GraphQL query for per-template deployment metrics. + * + * These are the creator-private counters behind the number shown on a template + * card: how many times the template was deployed and what share of those deploys + * succeeded. `workspaceTemplates` only exposes the rounded percentage, so the + * failure count has to come from here. + * + * templateMetrics takes one template id, so the query is assembled with an alias + * per template and the whole workspace arrives in a single round trip. + */ + +export const TEMPLATE_METRICS_OPERATION = 'templateMetrics'; + +/** + * Templates per request. + * + * Railway rejects a query whose breadth exceeds 500, and each template costs its + * own node plus the nine fields below - ten. Fifty would sit exactly on the + * limit, so this leaves room for a field to be added later without the query + * starting to fail in production. + */ +export const TEMPLATE_METRICS_BATCH_SIZE = 40; + +const METRIC_FIELDS = ` + templateHealth + supportHealth + totalDeployments + deploymentsLast90Days + activeDeployments + totalEarnings + earningsLast30Days + earningsLast90Days + eligibleForSupportBonus`; + +/** Alias used for the template at `index`, shared by the query and its parser. */ +export const metricsAlias = (index: number): string => `t${index}`; + +export function buildTemplateMetricsQuery(templateIds: string[]): string { + const params = templateIds.map((_, i) => `$${metricsAlias(i)}: String!`).join(', '); + const selections = templateIds + .map((_, i) => ` ${metricsAlias(i)}: templateMetrics(id: $${metricsAlias(i)}) {${METRIC_FIELDS}\n }`) + .join('\n'); + + return `query ${TEMPLATE_METRICS_OPERATION}(${params}) {\n${selections}\n }`; +} + +export function buildTemplateMetricsVariables( + templateIds: string[], +): Record { + return Object.fromEntries(templateIds.map((id, i) => [metricsAlias(i), id])); +} diff --git a/ingest/railway/types.ts b/ingest/railway/types.ts index 4f91d4a..513e8ec 100644 --- a/ingest/railway/types.ts +++ b/ingest/railway/types.ts @@ -42,6 +42,20 @@ export const TemplateSchema = z.object({ totalPayout: z.number().nullable().optional().transform(v => v ?? 0), }); +// Per-template deployment metrics schema (creator-private counters) +export const TemplateMetricsSchema = z.object({ + templateHealth: z.number(), + supportHealth: z.number(), + totalDeployments: z.number(), + deploymentsLast90Days: z.number(), + activeDeployments: z.number(), + // Earnings here are dollars, unlike the cents used by earningDetails + totalEarnings: z.number(), + earningsLast30Days: z.number(), + earningsLast90Days: z.number(), + eligibleForSupportBonus: z.boolean(), +}); + // GraphQL error schema export const GraphQLErrorSchema = z.object({ message: z.string(), @@ -73,6 +87,14 @@ export const TemplatesResponseSchema = z.object({ }), }); +// Template metrics response schema +// One entry per alias in the generated query; a template the API declines to +// report on comes back as null rather than as an error. +export const TemplateMetricsResponseSchema = z.record( + z.string(), + TemplateMetricsSchema.nullable(), +); + // Workspace response schema (for fetching customer ID) export const WorkspaceResponseSchema = z.object({ workspace: z.object({ @@ -85,6 +107,8 @@ export const WorkspaceResponseSchema = z.object({ // Infer TypeScript types from schemas export type EarningDetails = z.infer; export type Template = z.infer; +export type TemplateMetrics = z.infer; +export type TemplateMetricsResponse = z.infer; export type GraphQLError = z.infer; export type EarningsResponse = z.infer; export type TemplatesResponse = z.infer;