From c39a879ad1d3f65abd55dbd0625cd88e90636c8a Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 27 Aug 2026 11:08:37 +0200 Subject: [PATCH 1/5] chore(query-engine): drop redundant service_overview declared rowSchema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An audit of every surviving declared rowSchema against the builder-derived one found this site byte-identical — the builder derives the exact same struct from the SELECT's column types. The serviceOverviewRowSchema export stays: the compare path's period-wrapper still spreads its fields. --- packages/query-engine/src/ch/pipe-dispatch.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/query-engine/src/ch/pipe-dispatch.ts b/packages/query-engine/src/ch/pipe-dispatch.ts index 9b21ee56c..8bbb8a699 100644 --- a/packages/query-engine/src/ch/pipe-dispatch.ts +++ b/packages/query-engine/src/ch/pipe-dispatch.ts @@ -377,7 +377,6 @@ export function compilePipeQuery( commitShas: str("commit_shas")?.split(",").filter(Boolean), }), { orgId, startTime, endTime }, - { rowSchema: serviceOverviewRowSchema }, ), ), ), From d64553cc9bbee552db29604bd9551cb1c4459c4d Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 27 Aug 2026 11:19:27 +0200 Subject: [PATCH 2/5] feat(clickhouse-builder,query-engine): branded identity columns derive branded row schemas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comparisons on a column now widen a branded decoded type to its primitive (Widen), so $.OrgId.eq(param.string("orgId")) keeps compiling when the column's schema brands it — the type-level mirror of literalSchema. Maple's tables.ts declares OrgId (all tables) and TraceId/SpanId (error_events tables) via T.custom with their branded domain schemas, so derived row schemas carry the brands. That retires the declared rowSchemas at activeOrgsBy* and errorIssueSampleTraces; their output types now come off the derivation. The SQL-catalog synthetic zero-value row ("" for String columns) would fail a brand's minLength check, so the affected fixtures — and now pipe fixtures, which gained sampleValues support — override those fields. --- .../dashboards/ServiceMapRollupService.ts | 6 +- .../errors/ErrorIssueReadModelsService.ts | 1 - apps/api/src/services/errors/ErrorsService.ts | 8 +- lib/clickhouse-builder/src/ch/compile.test.ts | 26 +++++ lib/clickhouse-builder/src/ch/expr.test-d.ts | 15 +++ lib/clickhouse-builder/src/ch/expr.ts | 35 ++++-- .../query-engine/src/ch/builder-fixtures.ts | 8 ++ packages/query-engine/src/ch/index.ts | 2 - .../query-engine/src/ch/queries/activity.ts | 10 +- .../query-engine/src/ch/queries/errors.ts | 21 ++-- packages/query-engine/src/ch/tables.ts | 100 ++++++++++-------- packages/query-engine/src/sql-catalog.ts | 13 ++- 12 files changed, 164 insertions(+), 81 deletions(-) diff --git a/apps/api/src/services/dashboards/ServiceMapRollupService.ts b/apps/api/src/services/dashboards/ServiceMapRollupService.ts index 6f4efcef4..f52e61f8a 100644 --- a/apps/api/src/services/dashboards/ServiceMapRollupService.ts +++ b/apps/api/src/services/dashboards/ServiceMapRollupService.ts @@ -249,11 +249,7 @@ export class ServiceMapRollupService extends Context.Service< return yield* warehouse .crossOrgQuery( systemTenant(knownOrgs[0]!), - CH.compile( - CH.activeOrgsByTracesQuery(), - { startTime }, - { rowSchema: CH.ActiveOrgsOutputSchema }, - ), + CH.compile(CH.activeOrgsByTracesQuery(), { startTime }), { profile: "discovery", context: "serviceMapRollupActiveOrgs", diff --git a/apps/api/src/services/errors/ErrorIssueReadModelsService.ts b/apps/api/src/services/errors/ErrorIssueReadModelsService.ts index 0f7cd3948..2e4698291 100644 --- a/apps/api/src/services/errors/ErrorIssueReadModelsService.ts +++ b/apps/api/src/services/errors/ErrorIssueReadModelsService.ts @@ -360,7 +360,6 @@ const make: Effect.Effect< startTime: formatWarehouseDateTime(startMs), endTime: formatWarehouseDateTime(endMs), }, - { rowSchema: CH.ErrorIssueSampleTracesOutputSchema }, ) const samplesEffect = isErrorKind ? warehouse.compiledQuery(tenant, samplesCompiled, { diff --git a/apps/api/src/services/errors/ErrorsService.ts b/apps/api/src/services/errors/ErrorsService.ts index 6a60b7a48..a8828d2eb 100644 --- a/apps/api/src/services/errors/ErrorsService.ts +++ b/apps/api/src/services/errors/ErrorsService.ts @@ -302,11 +302,9 @@ const make: Effect.Effect< return byo as ReadonlySet } - const compiled = CH.compile( - CH.activeOrgsByErrorEventsQuery(), - { startTime: formatWarehouseDateTime(nowMs - ERROR_ACTIVE_DISCOVERY_WINDOW_MS) }, - { rowSchema: CH.ActiveOrgsOutputSchema }, - ) + const compiled = CH.compile(CH.activeOrgsByErrorEventsQuery(), { + startTime: formatWarehouseDateTime(nowMs - ERROR_ACTIVE_DISCOVERY_WINDOW_MS), + }) return yield* warehouse .crossOrgQuery(systemTenant(knownOrgs[0]!), compiled, { // Bound the one cross-org scan (no OrgId predicate ⇒ can't prune the diff --git a/lib/clickhouse-builder/src/ch/compile.test.ts b/lib/clickhouse-builder/src/ch/compile.test.ts index 602c2ccef..dd7370bde 100644 --- a/lib/clickhouse-builder/src/ch/compile.test.ts +++ b/lib/clickhouse-builder/src/ch/compile.test.ts @@ -35,6 +35,32 @@ describe("CompiledQuery.decodeRows", () => { }), ) + // `T.custom("String", branded)` is how a caller brands an id column. The + // brand must survive derivation — it is the whole reason to declare it — and + // the column must still compare against a plain-string param. + it.effect("a branded custom column derives a branded row schema", () => + Effect.gen(function* () { + const OrgId = Schema.String.check(Schema.isMinLength(1)).pipe(Schema.brand("OrgId")) + const table = CH.table( + "events", + { OrgId: T.custom("String", OrgId), Count: CH.uint64 }, + { tenantColumn: "OrgId" }, + ) + const compiled = compileCHUnsafe( + CH.from(table) + .select(($) => ({ orgId: $.OrgId })) + .where(($) => [$.OrgId.eq(CH.param.string("orgId"))]), + { orgId: "org_1" }, + ) + + expect(compiled.rowSchemaSource).toBe("derived") + expect(yield* compiled.decodeRows([{ orgId: "org_1" }])).toEqual([{ orgId: "org_1" }]) + // The brand's checks validate: an empty id is a decode failure. + const exit = yield* Effect.exit(compiled.decodeRows([{ orgId: "" }])) + expect(Exit.isFailure(exit)).toBe(true) + }), + ) + it.effect("has no schema when a selected expression has no type to read", () => Effect.gen(function* () { const table = CH.table("events", { OrgId: CH.string, Count: CH.uint64 }) diff --git a/lib/clickhouse-builder/src/ch/expr.test-d.ts b/lib/clickhouse-builder/src/ch/expr.test-d.ts index 7924badff..bc80e1799 100644 --- a/lib/clickhouse-builder/src/ch/expr.test-d.ts +++ b/lib/clickhouse-builder/src/ch/expr.test-d.ts @@ -166,3 +166,18 @@ expectTypeOf(CH.inList(strExpr, ["a", "b"])).toMatchTypeOf() expectTypeOf(CH.arrayOf(CH.lit("a"), CH.lit("b"))).toMatchTypeOf>>() expectTypeOf(CH.arrayOf(CH.lit(1), CH.lit(2))).toMatchTypeOf>>() + +// Branded column types — comparisons widen to the underlying primitive + +type BrandedId = string & { readonly __brand: "BrandedId" } +declare const brandedRef: Expr +declare const plainStringExpr: Expr + +// A branded column compares against a plain param/expr and a plain literal… +expectTypeOf(brandedRef.eq(CH.param.string("orgId"))).toMatchTypeOf() +expectTypeOf(brandedRef.eq(plainStringExpr)).toMatchTypeOf() +expectTypeOf(brandedRef.eq("org_123")).toMatchTypeOf() +expectTypeOf(brandedRef.in_("a", "b")).toMatchTypeOf() +// …and against another ref of its own branded type. +expectTypeOf(brandedRef.eq(brandedRef)).toMatchTypeOf() +expectTypeOf(CH.inList(brandedRef, ["a", "b"])).toMatchTypeOf() diff --git a/lib/clickhouse-builder/src/ch/expr.ts b/lib/clickhouse-builder/src/ch/expr.ts index a89e7e0bf..a0f4ce3e4 100644 --- a/lib/clickhouse-builder/src/ch/expr.ts +++ b/lib/clickhouse-builder/src/ch/expr.ts @@ -23,6 +23,19 @@ import { encodeColumnLiteral } from "./literal" */ export type Comparable = TSType extends DateTime.Utc ? DateTime.Utc | Date | string : TSType +/** + * A branded primitive compares as the primitive it brands. + * + * A column may decode to a branded type (`T.custom("String", OrgId)`), but the + * wire value it is compared against is the plain primitive — a param, another + * column, a literal. Without widening, `$.OrgId.eq(param.string("orgId"))` + * stops compiling the moment the column's schema brands its decoded type, + * which would make branding a breaking change instead of an annotation. The + * type-level mirror of `literalSchema`: comparisons may accept more than the + * column decodes to. + */ +export type Widen = TSType extends string ? string : TSType extends number ? number : TSType + export interface Expr { readonly _brand: "Expr" readonly _phantom?: TSType @@ -38,13 +51,15 @@ export interface Expr { readonly schema?: Schema.Codec toFragment(): SqlFragment - // Comparison — returns Condition - eq(other: Comparable | Expr): Condition - neq(other: Comparable | Expr): Condition - gt(other: Comparable | Expr): Condition - gte(other: Comparable | Expr): Condition - lt(other: Comparable | Expr): Condition - lte(other: Comparable | Expr): Condition + // Comparison — returns Condition. `Expr` is listed alongside the + // widened form because `Expr` is invariant: a branded column must accept + // both its own refs and plain-primitive exprs (params, other columns). + eq(other: Comparable> | Expr | Expr>): Condition + neq(other: Comparable> | Expr | Expr>): Condition + gt(other: Comparable> | Expr | Expr>): Condition + gte(other: Comparable> | Expr | Expr>): Condition + lt(other: Comparable> | Expr | Expr>): Condition + lte(other: Comparable> | Expr | Expr>): Condition // String operations like(this: Expr, pattern: string): Condition @@ -52,8 +67,8 @@ export interface Expr { ilike(this: Expr, pattern: string): Condition // IN / NOT IN - in_(...values: Array>): Condition - notIn(...values: Array>): Condition + in_(...values: Array>>): Condition + notIn(...values: Array>>): Condition // Arithmetic — only valid for number expressions div(this: Expr, n: number | Expr): Expr @@ -348,7 +363,7 @@ export function outerRef(name: string): Expr { return makeUntypedExpr(raw(name)) } -export function inList(expr: Expr, values: readonly string[]): Condition { +export function inList(expr: Expr, values: readonly string[]): Condition { const escaped = values.map((v) => compile(str(v))).join(", ") return makeCond(raw(`${compile(expr.toFragment())} IN (${escaped})`)) } diff --git a/packages/query-engine/src/ch/builder-fixtures.ts b/packages/query-engine/src/ch/builder-fixtures.ts index 7c88123f1..d20ace8eb 100644 --- a/packages/query-engine/src/ch/builder-fixtures.ts +++ b/packages/query-engine/src/ch/builder-fixtures.ts @@ -534,6 +534,9 @@ export const builderFixtures: ReadonlyArray = [ ...window, fingerprintHash: FINGERPRINT, }), + // TraceId/SpanId decode through their branded schemas (minLength 1), which + // the synthetic row's "" would fail. + sampleValues: { traceId: "0af7651916cd43dd8448eb211c80319c", spanId: "b7ad6b7169203331" }, }, { module: "errors", @@ -748,6 +751,7 @@ export const builderFixtures: ReadonlyArray = [ }).format("JSON"), { orgId: ORG_ID, hourStart: START_TIME, hourEnd: END_TIME }, ), + sampleValues: { OrgId: ORG_ID }, }, { // The service-scoped variant pushes the filter into the parent subquery. @@ -764,6 +768,7 @@ export const builderFixtures: ReadonlyArray = [ }).format("JSON"), { orgId: ORG_ID, hourStart: START_TIME, hourEnd: END_TIME }, ), + sampleValues: { OrgId: ORG_ID }, }, { module: "service-map-rollup", @@ -849,17 +854,20 @@ export const builderFixtures: ReadonlyArray = [ name: "activeOrgsByErrorEventsQuery", label: "default", compile: () => CH.compileUnsafe(CH.activeOrgsByErrorEventsQuery(), { startTime: START_TIME }), + sampleValues: { orgId: ORG_ID }, }, { module: "activity", name: "activeOrgsByTracesQuery", label: "default", compile: () => CH.compileUnsafe(CH.activeOrgsByTracesQuery(), { startTime: START_TIME }), + sampleValues: { orgId: ORG_ID }, }, { module: "activity", name: "activeOrgsByLogsQuery", label: "default", compile: () => CH.compileUnsafe(CH.activeOrgsByLogsQuery(), { startTime: START_TIME }), + sampleValues: { orgId: ORG_ID }, }, ] diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 64419e809..dc295ce88 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -259,7 +259,6 @@ export { errorFingerprintsQuery, errorIssueTimeseriesQuery, errorIssueSampleTracesQuery, - ErrorIssueSampleTracesOutputSchema, errorIssueEnvironmentsQuery, errorIssueVersionsSinceQuery, ErrorIssueVersionsSinceOutputSchema, @@ -319,7 +318,6 @@ export { activeOrgsByErrorEventsQuery, activeOrgsByTracesQuery, activeOrgsByLogsQuery, - ActiveOrgsOutputSchema, type ActiveOrgsOutput, } from "./queries/activity" diff --git a/packages/query-engine/src/ch/queries/activity.ts b/packages/query-engine/src/ch/queries/activity.ts index 72412191d..2d8caa30f 100644 --- a/packages/query-engine/src/ch/queries/activity.ts +++ b/packages/query-engine/src/ch/queries/activity.ts @@ -18,12 +18,14 @@ // active org is missed for the tick. import { from, param } from "@maple-dev/clickhouse-builder" -import { OrgId } from "@maple/domain" -import { Schema } from "effect" +import type { OrgId } from "@maple/domain" import { ErrorEventsByTime, LogsAggregatesHourly, TracesAggregatesHourly } from "../tables" -export const ActiveOrgsOutputSchema = Schema.Struct({ orgId: OrgId }) -export type ActiveOrgsOutput = Schema.Schema.Type +/** The `OrgId` brand comes off the tables' branded `OrgId` column — the + * derived row schema carries it, so no declared schema is needed. */ +export interface ActiveOrgsOutput { + readonly orgId: OrgId +} /** Orgs with any error events since `startTime` (gates the error-issue detector). */ export function activeOrgsByErrorEventsQuery() { diff --git a/packages/query-engine/src/ch/queries/errors.ts b/packages/query-engine/src/ch/queries/errors.ts index 472f84f02..2cf25fa45 100644 --- a/packages/query-engine/src/ch/queries/errors.ts +++ b/packages/query-engine/src/ch/queries/errors.ts @@ -11,7 +11,7 @@ import { from, fromQuery, type CHQuery, type ColumnAccessor } from "@maple-dev/c import type { ColumnDefs } from "@maple-dev/clickhouse-builder/types" import * as T from "@maple-dev/clickhouse-builder/types" import { unionAll, type CHUnionQuery } from "@maple-dev/clickhouse-builder" -import { SpanId, TraceId } from "@maple/domain" +import type { SpanId, TraceId } from "@maple/domain" import { Schema } from "effect" import { ErrorEvents, @@ -1131,15 +1131,16 @@ export function errorIssueTimeseriesQuery() { // Error Issue sample traces — most recent occurrences for one issue -export const ErrorIssueSampleTracesOutputSchema = Schema.Struct({ - traceId: TraceId, - spanId: SpanId, - serviceName: Schema.String, - timestamp: Schema.String, - exceptionMessage: Schema.String, - durationMicros: CHNumber, -}) -export type ErrorIssueSampleTracesOutput = Schema.Schema.Type +/** `TraceId`/`SpanId` brands come off `ErrorEvents`' branded columns — the + * derived row schema carries them, so no declared schema is needed. */ +export interface ErrorIssueSampleTracesOutput { + readonly traceId: TraceId + readonly spanId: SpanId + readonly serviceName: string + readonly timestamp: string + readonly exceptionMessage: string + readonly durationMicros: number +} export function errorIssueSampleTracesQuery(opts: { limit?: number }) { return from(ErrorEvents) diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index 18f6e6905..0c6a4ff72 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -4,8 +4,8 @@ // These define the ClickHouse table schemas used by the query DSL. import { type ColumnDefs, type Table, table as chTable } from "@maple-dev/clickhouse-builder" -import type { CHString } from "@maple-dev/clickhouse-builder/types" import * as T from "@maple-dev/clickhouse-builder/types" +import { OrgId, SpanId, TraceId } from "@maple/domain" /** * Maple's warehouse timestamps stay the strings ClickHouse sends. @@ -31,19 +31,33 @@ const dateTime64 = T.dateTime64String */ export type StringMap = T.CHMap +/** + * Identity columns carry their branded domain schemas, so a query that SELECTs + * one derives a row schema whose decoded type is the brand — no declared + * `rowSchema` needed just to keep `OrgId`/`TraceId`/`SpanId` in the output + * type. Comparisons still take plain strings/params: the builder widens a + * branded column's comparison type to its primitive. + */ +const orgId = T.custom("String", OrgId) +const traceId = T.custom("String", TraceId) +const spanId = T.custom("String", SpanId) + /** * Every Maple warehouse table is keyed by `OrgId`, so tenancy is declared once * here rather than at 37 call sites. Requiring the column in the signature is * the point: a new table without one is a type error, not a table that silently * compiles every query as `cross-tenant`. */ -const table = ( +const table = < + const Name extends string, + const Columns extends ColumnDefs & { OrgId: T.CHType<"String", any, any> }, +>( name: Name, columns: Columns, ): Table => chTable(name, columns, { tenantColumn: "OrgId" }) export const Traces = table("traces", { - OrgId: T.string, + OrgId: orgId, Timestamp: dateTime64, TraceId: T.string, SpanId: T.string, @@ -77,7 +91,7 @@ export const Traces = table("traces", { }) export const TraceDetailSpans = table("trace_detail_spans", { - OrgId: T.string, + OrgId: orgId, Timestamp: dateTime64, TraceId: T.string, SpanId: T.string, @@ -93,7 +107,7 @@ export const TraceDetailSpans = table("trace_detail_spans", { }) export const TraceListMv = table("trace_list_mv", { - OrgId: T.string, + OrgId: orgId, TraceId: T.string, Timestamp: dateTime, ServiceName: T.string, @@ -111,7 +125,7 @@ export const TraceListMv = table("trace_list_mv", { }) export const Logs = table("logs", { - OrgId: T.string, + OrgId: orgId, Timestamp: dateTime64, TimestampTime: dateTime, TraceId: T.string, @@ -134,7 +148,7 @@ export const Logs = table("logs", { }) export const ServiceOverviewSpans = table("service_overview_spans", { - OrgId: T.string, + OrgId: orgId, Timestamp: dateTime, ServiceName: T.string, Duration: T.uint64, @@ -147,7 +161,7 @@ export const ServiceOverviewSpans = table("service_overview_spans", { }) export const ServiceOverviewHourly = table("service_overview_hourly", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, ServiceName: T.string, DeploymentEnv: T.string, @@ -170,7 +184,7 @@ export const ServiceOverviewHourly = table("service_overview_hourly", { * bucket column, so the two can share a UNION ALL branch shape. */ export const ServiceOverviewMinutely = table("service_overview_minutely", { - OrgId: T.string, + OrgId: orgId, Minute: dateTime, ServiceName: T.string, DeploymentEnv: T.string, @@ -188,10 +202,10 @@ export const ServiceOverviewMinutely = table("service_overview_minutely", { }) export const ErrorEvents = table("error_events", { - OrgId: T.string, + OrgId: orgId, Timestamp: dateTime, - TraceId: T.string, - SpanId: T.string, + TraceId: traceId, + SpanId: spanId, ParentSpanId: T.string, ServiceName: T.string, DeploymentEnv: T.string, @@ -214,10 +228,10 @@ export const ErrorEvents = table("error_events", { * `packages/domain/src/tinybird/datasources.ts`. */ export const ErrorEventsByTime = table("error_events_by_time", { - OrgId: T.string, + OrgId: orgId, Timestamp: dateTime, - TraceId: T.string, - SpanId: T.string, + TraceId: traceId, + SpanId: spanId, ParentSpanId: T.string, ServiceName: T.string, DeploymentEnv: T.string, @@ -234,7 +248,7 @@ export const ErrorEventsByTime = table("error_events_by_time", { /** Minute-grain per-fingerprint rollup consumed by the error issue tick. */ export const ErrorFingerprintsMinutely = table("error_fingerprints_minutely", { - OrgId: T.string, + OrgId: orgId, Minute: dateTime, FingerprintHash: T.uint64, ServiceName: T.string, @@ -249,7 +263,7 @@ export const ErrorFingerprintsMinutely = table("error_fingerprints_minutely", { }) export const MetricsSum = table("metrics_sum", { - OrgId: T.string, + OrgId: orgId, ResourceAttributes: T.map(T.string, T.string), ServiceName: T.string, MetricName: T.string, @@ -265,7 +279,7 @@ export const MetricsSum = table("metrics_sum", { }) export const MetricsGauge = table("metrics_gauge", { - OrgId: T.string, + OrgId: orgId, ResourceAttributes: T.map(T.string, T.string), ServiceName: T.string, MetricName: T.string, @@ -279,7 +293,7 @@ export const MetricsGauge = table("metrics_gauge", { }) export const MetricsHistogram = table("metrics_histogram", { - OrgId: T.string, + OrgId: orgId, ResourceAttributes: T.map(T.string, T.string), ServiceName: T.string, MetricName: T.string, @@ -299,7 +313,7 @@ export const MetricsHistogram = table("metrics_histogram", { }) export const MetricCatalog = table("metric_catalog", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, MetricType: T.string, ServiceName: T.string, @@ -313,7 +327,7 @@ export const MetricCatalog = table("metric_catalog", { }) export const SpanMetricsCallsHourly = table("span_metrics_calls_hourly", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, ServiceName: T.string, MetricName: T.string, @@ -326,7 +340,7 @@ export const SpanMetricsCallsHourly = table("span_metrics_calls_hourly", { }) export const AttributeKeysHourly = table("attribute_keys_hourly", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, AttributeKey: T.string, AttributeScope: T.string, @@ -334,7 +348,7 @@ export const AttributeKeysHourly = table("attribute_keys_hourly", { }) export const AttributeValuesHourly = table("attribute_values_hourly", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, AttributeKey: T.string, AttributeValue: T.string, @@ -343,7 +357,7 @@ export const AttributeValuesHourly = table("attribute_values_hourly", { }) export const ServiceUsage = table("service_usage", { - OrgId: T.string, + OrgId: orgId, ServiceName: T.string, Hour: dateTime, LogCount: T.uint64, @@ -361,7 +375,7 @@ export const ServiceUsage = table("service_usage", { }) export const ServiceMapSpans = table("service_map_spans", { - OrgId: T.string, + OrgId: orgId, Timestamp: dateTime, TraceId: T.string, SpanId: T.string, @@ -375,7 +389,7 @@ export const ServiceMapSpans = table("service_map_spans", { }) export const ServiceMapChildren = table("service_map_children", { - OrgId: T.string, + OrgId: orgId, Timestamp: dateTime, TraceId: T.string, ParentSpanId: T.string, @@ -388,7 +402,7 @@ export const ServiceMapChildren = table("service_map_children", { }) export const TracesAggregatesHourly = table("traces_aggregates_hourly", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, ServiceName: T.string, SpanName: T.string, @@ -407,7 +421,7 @@ export const TracesAggregatesHourly = table("traces_aggregates_hourly", { }) export const ServiceOperationsMinutely = table("service_operations_minutely", { - OrgId: T.string, + OrgId: orgId, Minute: dateTime, ServiceName: T.string, DeploymentEnv: T.string, @@ -422,7 +436,7 @@ export const ServiceOperationsMinutely = table("service_operations_minutely", { }) export const LogsAggregatesHourly = table("logs_aggregates_hourly", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, ServiceName: T.string, SeverityText: T.string, @@ -433,7 +447,7 @@ export const LogsAggregatesHourly = table("logs_aggregates_hourly", { }) export const ServiceMapEdgesHourly = table("service_map_edges_hourly", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, SourceService: T.string, TargetService: T.string, @@ -452,7 +466,7 @@ export const ServiceMapEdgesHourly = table("service_map_edges_hourly", { // they emit are what the DSL can't express. Declared here anyway so // tables.test.ts drift-checks the columns those builders read. export const ServiceExternalEdgesHourly = table("service_external_edges_hourly", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, ServiceName: T.string, TargetType: T.string, @@ -467,7 +481,7 @@ export const ServiceExternalEdgesHourly = table("service_external_edges_hourly", }) export const ServiceAddressResolutionsHourly = table("service_address_resolutions_hourly", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, SourceService: T.string, ParentServerAddress: T.string, @@ -476,7 +490,7 @@ export const ServiceAddressResolutionsHourly = table("service_address_resolution }) export const ServiceMapDbEdgesHourly = table("service_map_db_edges_hourly", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, ServiceName: T.string, DbSystem: T.string, @@ -492,7 +506,7 @@ export const ServiceMapDbEdgesHourly = table("service_map_db_edges_hourly", { }) export const ServiceMapDbQuerySignaturesHourly = table("service_map_db_query_shapes_hourly", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, ServiceName: T.string, DbSystem: T.string, @@ -512,7 +526,7 @@ export const ServiceMapDbQuerySignaturesHourly = table("service_map_db_query_sha }) export const ServicePlatformsHourly = table("service_platforms_hourly", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, ServiceName: T.string, DeploymentEnv: T.string, @@ -531,7 +545,7 @@ export const ServicePlatformsHourly = table("service_platforms_hourly", { }) export const ServiceOperationsHourly = table("service_operations_hourly", { - OrgId: T.string, + OrgId: orgId, Hour: dateTime, ServiceName: T.string, DeploymentEnv: T.string, @@ -545,7 +559,7 @@ export const ServiceOperationsHourly = table("service_operations_hourly", { }) export const AlertChecks = table("alert_checks", { - OrgId: T.string, + OrgId: orgId, RuleId: T.string, GroupKey: T.string, Timestamp: dateTime64, @@ -568,7 +582,7 @@ export const AlertChecks = table("alert_checks", { }) export const SessionReplays = table("session_replays", { - OrgId: T.string, + OrgId: orgId, SessionId: T.string, StartTime: dateTime64, EndTime: T.nullable(dateTime64), @@ -627,7 +641,7 @@ export const SessionReplays = table("session_replays", { }) export const SessionReplayEvents = table("session_replay_events", { - OrgId: T.string, + OrgId: orgId, SessionId: T.string, ChunkSeq: T.uint32, // Gateway receipt time — partitioning, TTL, and the anchor the chunk index @@ -649,7 +663,7 @@ export const SessionReplayEvents = table("session_replay_events", { // panels, and the agent transcript. Sparse: only the fields relevant to a row's // Type are populated; the rest default empty. export const SessionEvents = table("session_events", { - OrgId: T.string, + OrgId: orgId, SessionId: T.string, Timestamp: dateTime64, // Monotonic per-session ordering tiebreaker (events can share a ms timestamp). @@ -698,7 +712,7 @@ export const SessionEvents = table("session_events", { // through `identity_links`. VisitorId is third in the sorting key so one // person's rows are contiguous inside a time range. export const ProductEvents = table("product_events", { - OrgId: T.string, + OrgId: orgId, Timestamp: dateTime64, // "browser" | "server" | "mobile". Source: T.string, @@ -736,13 +750,13 @@ export const ProductEvents = table("product_events", { // still hold several rows per pair, so always aggregate (`min(FirstSeen)` per // pair) or semi-join; never assume one row per pair on read. export const IdentityLinks = table("identity_links", { - OrgId: T.string, + OrgId: orgId, VisitorId: T.string, UserId: T.string, FirstSeen: dateTime64, }) export const MetricsExpHistogram = table("metrics_exponential_histogram", { - OrgId: T.string, + OrgId: orgId, ResourceAttributes: T.map(T.string, T.string), ServiceName: T.string, MetricName: T.string, diff --git a/packages/query-engine/src/sql-catalog.ts b/packages/query-engine/src/sql-catalog.ts index 0405cf1b0..2fafa96b9 100644 --- a/packages/query-engine/src/sql-catalog.ts +++ b/packages/query-engine/src/sql-catalog.ts @@ -95,6 +95,9 @@ export interface PipeFixture { readonly params: Record /** Run under every capability variant, not just the baseline. */ readonly allCapabilities?: boolean + /** Overrides for the synthetic zero-value row where the row schema demands + * more than the ClickHouse type — see `BuilderFixture.sampleValues`. */ + readonly sampleValues?: Readonly> } const TRACE_ID = "0af7651916cd43dd8448eb211c80319c" @@ -259,7 +262,14 @@ export const pipeFixtures: ReadonlyArray = [ { pipe: "error_detail_traces", label: "default", params: { fingerprint_hash: FINGERPRINT } }, { pipe: "error_issues", label: "default", params: {} }, { pipe: "error_issue_timeseries", label: "default", params: { fingerprint_hash: FINGERPRINT } }, - { pipe: "error_issue_sample_traces", label: "default", params: { fingerprint_hash: FINGERPRINT } }, + { + pipe: "error_issue_sample_traces", + label: "default", + params: { fingerprint_hash: FINGERPRINT }, + // TraceId/SpanId decode through their branded schemas (minLength 1), which + // the synthetic row's "" would fail. + sampleValues: { traceId: TRACE_ID, spanId: "b7ad6b7169203331" }, + }, { pipe: "error_issue_environments", label: "default", params: { fingerprint_hash: FINGERPRINT } }, { pipe: "list_metrics", label: "default", params: {} }, @@ -358,6 +368,7 @@ export function collectPipeCatalog(): ReadonlyArray { sql: compiled.sql, fingerprint: fingerprintSql(compiled.sql), compiled, + ...(fixture.sampleValues ? { sampleValues: fixture.sampleValues } : undefined), }) } } From 5dfa4a790e452ed64db9ba80848c3307749130c5 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 27 Aug 2026 11:33:20 +0200 Subject: [PATCH 3/5] =?UTF-8?q?feat(clickhouse-builder):=20CHStringLike=20?= =?UTF-8?q?=E2=80=94=20the=20constraint=20form=20of=20a=20String=20column?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tenant-column constraint like { OrgId: } has to accept a branded column, and CHType is invariant in its decoded type, so CHString can't say it. Name the concept in the lib instead of hand-rolling CHType<"String", any, any> at the consumer. --- lib/clickhouse-builder/src/ch/types.ts | 11 +++++++++++ lib/clickhouse-builder/src/types.ts | 1 + packages/query-engine/src/ch/tables.ts | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/clickhouse-builder/src/ch/types.ts b/lib/clickhouse-builder/src/ch/types.ts index 0b1098108..15dfcafb6 100644 --- a/lib/clickhouse-builder/src/ch/types.ts +++ b/lib/clickhouse-builder/src/ch/types.ts @@ -158,6 +158,17 @@ export type CHDateTime = CHType<"DateTime", DateTime.Utc, string> export type CHDateTime64 = CHType<"DateTime64", DateTime.Utc, string> export type CHBool = CHType<"Bool", boolean, boolean | number> +/** + * A `String` column whatever its decoded type — plain, branded, or narrowed. + * + * The constraint form of `CHString`, for "this table must carry this String + * column" (a tenant column, a join key) where how the value decodes is the + * table's own business. Spelling such a constraint as `CHString` would reject + * a branded column: `CHType` is invariant in its decoded type, so + * `custom("String", OrgId)` is not a `CHType<"String", string>`. + */ +export type CHStringLike = CHType<"String", any, any> + /** * The same columns left as the strings ClickHouse sends. * diff --git a/lib/clickhouse-builder/src/types.ts b/lib/clickhouse-builder/src/types.ts index e4c619d8d..6c0c74d01 100644 --- a/lib/clickhouse-builder/src/types.ts +++ b/lib/clickhouse-builder/src/types.ts @@ -17,6 +17,7 @@ export { type CHMap, type CHNullable, type CHString, + type CHStringLike, type CHType, type CHUInt8, type CHUInt16, diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index 0c6a4ff72..7339a7b91 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -50,7 +50,7 @@ const spanId = T.custom("String", SpanId) */ const table = < const Name extends string, - const Columns extends ColumnDefs & { OrgId: T.CHType<"String", any, any> }, + const Columns extends ColumnDefs & { OrgId: T.CHStringLike }, >( name: Name, columns: Columns, From ffb118d409b470cb6fb11d508ebfe33eceed38e9 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 27 Aug 2026 12:06:07 +0200 Subject: [PATCH 4/5] fix(clickhouse-builder): derive output types from the phantom, not structural inference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI caught what a stale local tsbuildinfo hid: InferOutput's `S[K] extends Expr` resolved T through the comparison methods' contravariant params — which are now widened for branded columns — so every branded column's derived output type collapsed to the bare primitive and the api call sites failed against their branded consumers. InferOutput now reads the expression's `_phantom` property directly, which is exact; Exclude strips only the optionality's undefined so Nullable columns keep their `| null`. The edges-rollup declared schema also has to say the brand now that the derived output carries it — declared schemas narrow, they don't contradict. --- lib/clickhouse-builder/src/ch/expr.ts | 3 +++ lib/clickhouse-builder/src/ch/query.ts | 11 ++++++++++- .../query-engine/src/ch/queries/service-map-rollup.ts | 9 ++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/clickhouse-builder/src/ch/expr.ts b/lib/clickhouse-builder/src/ch/expr.ts index a0f4ce3e4..d8c5d8223 100644 --- a/lib/clickhouse-builder/src/ch/expr.ts +++ b/lib/clickhouse-builder/src/ch/expr.ts @@ -54,6 +54,9 @@ export interface Expr { // Comparison — returns Condition. `Expr` is listed alongside the // widened form because `Expr` is invariant: a branded column must accept // both its own refs and plain-primitive exprs (params, other columns). + // The widened arms sit in contravariant positions, which TypeScript's + // `extends Expr` inference would prefer — the reason `InferOutput` + // reads the `_phantom` property instead of structurally inferring T. eq(other: Comparable> | Expr | Expr>): Condition neq(other: Comparable> | Expr | Expr>): Condition gt(other: Comparable> | Expr | Expr>): Condition diff --git a/lib/clickhouse-builder/src/ch/query.ts b/lib/clickhouse-builder/src/ch/query.ts index df06e9650..17fbdcd3b 100644 --- a/lib/clickhouse-builder/src/ch/query.ts +++ b/lib/clickhouse-builder/src/ch/query.ts @@ -46,8 +46,17 @@ export type JoinedColumnAccessor< type SelectRecord = Record> +/** + * Read each selected expression's output type off its `_phantom` property + * rather than `S[K] extends Expr`. Structural inference prefers the + * contravariant candidates in the comparison methods, and those are widened + * (`Widen`) so branded columns accept plain params — inferring through + * them resolved a branded column's output to the bare primitive. The indexed + * read is exact; `Exclude` only strips the `undefined` that `_phantom`'s + * optionality adds, so a `Nullable(...)` column's `| null` survives. + */ export type InferOutput = { - readonly [K in keyof S]: S[K] extends Expr ? T : never + readonly [K in keyof S]: S[K] extends Expr ? Exclude : never } type OrderBySpec = [keyof Output & string, "asc" | "desc"] diff --git a/packages/query-engine/src/ch/queries/service-map-rollup.ts b/packages/query-engine/src/ch/queries/service-map-rollup.ts index 6bce89bdf..17e247f5c 100644 --- a/packages/query-engine/src/ch/queries/service-map-rollup.ts +++ b/packages/query-engine/src/ch/queries/service-map-rollup.ts @@ -18,6 +18,7 @@ import { compile } from "@maple-dev/clickhouse-builder" import * as CH from "@maple-dev/clickhouse-builder/expr" import { param } from "@maple-dev/clickhouse-builder" import { from, fromQuery } from "@maple-dev/clickhouse-builder" +import { OrgId } from "@maple/domain" import { ServiceAddressResolutionsHourly, ServiceMapEdgesHourly, Traces } from "../tables" import { deploymentEnvExpr } from "@maple/domain/tinybird/semconv-renames" import { serviceMapEdgeJoinQuery } from "./service-map" @@ -27,7 +28,7 @@ import type { QueryBuilderError } from "@maple-dev/clickhouse-builder" /** One pre-aggregated service-to-service edge bucket — mirrors the columns of * the `service_map_edges_hourly` ClickHouse table. */ export interface ServiceMapEdgesHourlyOutput { - readonly OrgId: string + readonly OrgId: OrgId readonly Hour: string readonly SourceService: string readonly TargetService: string @@ -42,7 +43,9 @@ export interface ServiceMapEdgesHourlyOutput { } const ServiceMapEdgesHourlyOutputSchema: CompiledQueryRowSchema = Schema.Struct({ - OrgId: Schema.String, + // The tables' OrgId column is branded, so the derived output is too — a + // declared schema may only narrow, so it has to say the brand as well. + OrgId, Hour: Schema.String, SourceService: Schema.String, TargetService: Schema.String, @@ -173,7 +176,7 @@ export function serviceMapEdgesRollupSQL( /** One resolved address-to-service mapping bucket — mirrors the columns of * `service_address_resolutions_hourly`. */ export interface ServiceAddressResolutionsHourlyOutput { - readonly OrgId: string + readonly OrgId: OrgId readonly Hour: string readonly SourceService: string readonly ParentServerAddress: string From d54e3061ab0f0090ba37d6b9691ac108b5a9b7d5 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Thu, 27 Aug 2026 12:59:49 +0200 Subject: [PATCH 5/5] test(api): failed inserts now classify as WarehouseInvalidSqlError MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit main's invalid-sql tag added a caller-authored twin of the analyzer rule, and inserts deliberately keep the default caller authorship — so the syntax-shaped insert rejection takes the new tag. The api-side expectation was the one call-out that commit missed. --- .../src/services/warehouse/WarehouseQueryService.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/api/src/services/warehouse/WarehouseQueryService.test.ts b/apps/api/src/services/warehouse/WarehouseQueryService.test.ts index cdbf4a50b..0d8a43bf8 100644 --- a/apps/api/src/services/warehouse/WarehouseQueryService.test.ts +++ b/apps/api/src/services/warehouse/WarehouseQueryService.test.ts @@ -11,7 +11,7 @@ import { TinybirdOrgTokenConfigError, UserId, WarehouseConfigError, - WarehouseQueryError, + WarehouseInvalidSqlError, WarehouseResultDecodeError, WarehouseScopeError, WarehouseUpstreamError, @@ -661,7 +661,10 @@ describe("WarehouseQueryService.ingest writes through the SQL client", () => { }).pipe(Effect.provide(layer)) }) - it.effect("maps a failed insert to WarehouseQueryError", () => { + // Inserts classify with the read path's default "caller" authorship (the + // rows, not Maple's SQL, are what usually earned the rejection), so a + // syntax-shaped complaint takes the caller-authored invalid-SQL tag. + it.effect("maps a failed insert through the classifier", () => { __testables.setClientFactory(() => ({ sql: async () => ({ data: [] }), insert: async () => { @@ -679,7 +682,7 @@ describe("WarehouseQueryService.ingest writes through the SQL client", () => { assert.isTrue(Exit.isFailure(exit)) const failure = getError(exit) - assert.instanceOf(failure, WarehouseQueryError) + assert.instanceOf(failure, WarehouseInvalidSqlError) }).pipe(Effect.provide(layer)) }) })