diff --git a/LICENSE b/LICENSE index e42009d58edca..936bb0dc3db90 100644 --- a/LICENSE +++ b/LICENSE @@ -13,7 +13,7 @@ Business Source License 1.1 Licensor: Materialize, Inc. -Licensed Work: Materialize Version 20260826 +Licensed Work: Materialize Version 20260827 The Licensed Work is © 2026 Materialize, Inc. Additional Use Grant: Within a single installation of the Licensed Work, @@ -39,7 +39,7 @@ Additional Use Grant: Within a single installation of the Licensed Work, whose definitions are controlled by such third parties. -Change Date: August 26, 2030 +Change Date: August 27, 2030 Change License: Apache License, Version 2.0 diff --git a/console/src/api/materialize/maintained-objects/hydrationAggregate.test.sql.ts b/console/src/api/materialize/maintained-objects/hydrationAggregate.test.sql.ts index 5e91e4b963a67..2c0655847ea55 100644 --- a/console/src/api/materialize/maintained-objects/hydrationAggregate.test.sql.ts +++ b/console/src/api/materialize/maintained-objects/hydrationAggregate.test.sql.ts @@ -16,23 +16,76 @@ import { testdrive } from "~/test/sql/mzcompose"; import { SEARCH_PATH } from "../executeSql"; import { buildHydrationAggregateQuery } from "./hydrationAggregate"; +const TEST_SCHEMA = "test_hydration_aggregate"; + +/** Shadows the four relations the query reads. Empty value lists are elided. + * `replicas` defaults to the ids the fixtures use, so statistics rows count + * as coming from live replicas unless a test overrides it. */ +const seed = ({ + hydration = "", + statuses = "", + statistics = "", + replicas = "('r1'), ('r2'), ('r3')", + sources = "", +}: { + hydration?: string; + statuses?: string; + statistics?: string; + replicas?: string; + sources?: string; +}) => ` + > DROP SCHEMA IF EXISTS ${TEST_SCHEMA} CASCADE; + > CREATE SCHEMA ${TEST_SCHEMA}; + > CREATE TABLE ${TEST_SCHEMA}.mz_hydration_statuses ( + object_id TEXT NOT NULL, + replica_id TEXT, + hydrated BOOLEAN + ); + > CREATE TABLE ${TEST_SCHEMA}.mz_sources ( + id TEXT NOT NULL, + type TEXT NOT NULL + ); + > CREATE TABLE ${TEST_SCHEMA}.mz_source_statuses ( + id TEXT NOT NULL, + status TEXT NOT NULL, + error TEXT, + type TEXT NOT NULL + ); + > CREATE TABLE ${TEST_SCHEMA}.mz_source_statistics ( + id TEXT NOT NULL, + replica_id TEXT, + snapshot_committed BOOLEAN + ); + > CREATE TABLE ${TEST_SCHEMA}.mz_cluster_replicas ( + id TEXT NOT NULL + ); + ${hydration ? `> INSERT INTO ${TEST_SCHEMA}.mz_hydration_statuses VALUES ${hydration};` : ""} + ${statuses ? `> INSERT INTO ${TEST_SCHEMA}.mz_source_statuses VALUES ${statuses};` : ""} + ${statistics ? `> INSERT INTO ${TEST_SCHEMA}.mz_source_statistics VALUES ${statistics};` : ""} + ${replicas ? `> INSERT INTO ${TEST_SCHEMA}.mz_cluster_replicas VALUES ${replicas};` : ""} + ${sources ? `> INSERT INTO ${TEST_SCHEMA}.mz_sources VALUES ${sources};` : ""} +`; + +const run = async () => { + const compiled = buildHydrationAggregateQuery().compile(); + const result = await executeSqlHttp(compiled, { + sessionVariables: { + cluster: QUICKSTART_CLUSTER, + search_path: `${TEST_SCHEMA}, ${SEARCH_PATH}`, + }, + }); + return result.rows.sort((a, b) => a.object_id.localeCompare(b.object_id)); +}; + describe("buildHydrationAggregateQuery", () => { it("aggregates hydrated and total replicas per object", async () => { - const testSchema = "test_hydration_aggregate"; - // u1: single replica, hydrated → 1/1 // u2: 3 replicas, all hydrated → 3/3 // u3: 3 replicas, 1 hydrated → 1/3 // u4: 2 replicas, none hydrated → 0/2 - await testdrive(` - > DROP SCHEMA IF EXISTS ${testSchema} CASCADE; - > CREATE SCHEMA ${testSchema}; - > CREATE TABLE ${testSchema}.mz_hydration_statuses ( - object_id TEXT NOT NULL, - replica_id TEXT, - hydrated BOOLEAN - ); - > INSERT INTO ${testSchema}.mz_hydration_statuses VALUES + await testdrive( + seed({ + hydration: ` ('u1', 'r1', true), ('u2', 'r1', true), ('u2', 'r2', true), @@ -41,20 +94,11 @@ describe("buildHydrationAggregateQuery", () => { ('u3', 'r2', false), ('u3', 'r3', false), ('u4', 'r1', false), - ('u4', 'r2', false); - `); - - const compiled = buildHydrationAggregateQuery().compile(); - const result = await executeSqlHttp(compiled, { - sessionVariables: { - cluster: QUICKSTART_CLUSTER, - search_path: `${testSchema}, ${SEARCH_PATH}`, - }, - }); - - const rows = result.rows.sort((a, b) => - a.object_id.localeCompare(b.object_id), + ('u4', 'r2', false)`, + }), ); + + const rows = await run(); expect(rows).toHaveLength(4); expect(rows[0]).toMatchObject({ object_id: "u1" }); @@ -72,38 +116,149 @@ describe("buildHydrationAggregateQuery", () => { expect(rows[3]).toMatchObject({ object_id: "u4" }); expect(Number(rows[3].hydratedReplicas)).toBe(0); expect(Number(rows[3].totalReplicas)).toBe(2); + + // No source rows seeded, so the source columns are null throughout. + expect(rows[0].sourceStatus).toBeNull(); + expect(rows[0].snapshotCommitted).toBeNull(); }); it("treats null hydrated values as not-hydrated for the FILTER clause", async () => { - const testSchema = "test_hydration_aggregate"; - // A replica that hasn't reported a status yet shows up as a null row; // the FILTER (WHERE hydrated) should not count it as hydrated, but the // total count still includes it. - await testdrive(` - > DROP SCHEMA IF EXISTS ${testSchema} CASCADE; - > CREATE SCHEMA ${testSchema}; - > CREATE TABLE ${testSchema}.mz_hydration_statuses ( - object_id TEXT NOT NULL, - replica_id TEXT, - hydrated BOOLEAN - ); - > INSERT INTO ${testSchema}.mz_hydration_statuses VALUES - ('u1', 'r1', true), - ('u1', 'r2', NULL); - `); - - const compiled = buildHydrationAggregateQuery().compile(); - const result = await executeSqlHttp(compiled, { - sessionVariables: { - cluster: QUICKSTART_CLUSTER, - search_path: `${testSchema}, ${SEARCH_PATH}`, - }, + await testdrive( + seed({ hydration: `('u1', 'r1', true), ('u1', 'r2', NULL)` }), + ); + + const rows = await run(); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ object_id: "u1" }); + expect(Number(rows[0].hydratedReplicas)).toBe(1); + expect(Number(rows[0].totalReplicas)).toBe(2); + }); + + it("joins source status, error and the snapshot flag onto source rows", async () => { + await testdrive( + seed({ + hydration: `('u1', 'r1', true), ('u2', 'r1', false)`, + statuses: ` + ('u1', 'running', NULL, 'kafka'), + ('u2', 'stalled', 'broker unreachable', 'kafka')`, + statistics: `('u1', 'r1', true), ('u2', 'r1', false)`, + }), + ); + + const rows = await run(); + expect(rows).toHaveLength(2); + expect(rows[0]).toMatchObject({ + object_id: "u1", + sourceStatus: "running", + sourceError: null, + snapshotCommitted: true, }); + expect(rows[1]).toMatchObject({ + object_id: "u2", + sourceStatus: "stalled", + sourceError: "broker unreachable", + snapshotCommitted: false, + }); + }); + + it("only reports the snapshot as committed once every replica agrees", async () => { + // A replica that is still snapshotting holds the whole source at false, so + // the UI keeps showing Snapshotting rather than flipping to Running early. + await testdrive( + seed({ + hydration: `('u1', 'r1', true)`, + statuses: `('u1', 'running', NULL, 'kafka')`, + statistics: `('u1', 'r1', true), ('u1', 'r2', false)`, + }), + ); + + const rows = await run(); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + object_id: "u1", + snapshotCommitted: false, + }); + }); + + it("keeps sources that have a status but no hydration rows", async () => { + // Webhook sources are omitted from mz_hydration_statuses, so the FULL + // JOIN is what keeps their status visible. + await testdrive( + seed({ + hydration: `('u1', 'r1', true)`, + statuses: `('u1', 'running', NULL, 'kafka'), ('u7', 'running', NULL, 'webhook')`, + }), + ); + + const rows = await run(); + expect(rows).toHaveLength(2); + expect(rows[1]).toMatchObject({ + object_id: "u7", + sourceStatus: "running", + snapshotCommitted: null, + }); + expect(rows[1].hydratedReplicas).toBeNull(); + expect(rows[1].totalReplicas).toBeNull(); + }); - expect(result.rows).toHaveLength(1); - expect(result.rows[0]).toMatchObject({ object_id: "u1" }); - expect(Number(result.rows[0].hydratedReplicas)).toBe(1); - expect(Number(result.rows[0].totalReplicas)).toBe(2); + it("excludes subsources and progress collections from the feed", async () => { + // The UI hides them, and each source carries one progress collection and + // often several subsources, so keeping them out shrinks the subscribe. + // Subsources also have hydration rows of their own (they run on the + // parent's cluster), so the exclusion must hold on both join sides. + await testdrive( + seed({ + hydration: `('u1', 'r1', true), ('u2', 'r1', true)`, + statuses: ` + ('u1', 'running', NULL, 'postgres'), + ('u2', 'running', NULL, 'subsource'), + ('u3', 'running', NULL, 'progress')`, + sources: `('u1', 'postgres'), ('u2', 'subsource'), ('u3', 'progress')`, + }), + ); + + const rows = await run(); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ object_id: "u1", sourceStatus: "running" }); + }); + + it("ignores statistics rows from dropped replicas", async () => { + // Statistics rows can outlive their replica; a dropped replica's stale + // false must not pin a committed source back to snapshotting. + await testdrive( + seed({ + statuses: `('u1', 'running', NULL, 'kafka')`, + statistics: `('u1', 'r1', true), ('u1', 'r_dropped', false)`, + replicas: `('r1')`, + }), + ); + + const rows = await run(); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + object_id: "u1", + snapshotCommitted: true, + }); + }); + + it("keeps statistics rows that report no replica", async () => { + // Webhook sources report statistics with a null replica_id. + await testdrive( + seed({ + statuses: `('u1', 'running', NULL, 'webhook')`, + statistics: `('u1', NULL, true)`, + replicas: `('r1')`, + }), + ); + + const rows = await run(); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + object_id: "u1", + snapshotCommitted: true, + }); }); }); diff --git a/console/src/api/materialize/maintained-objects/hydrationAggregate.ts b/console/src/api/materialize/maintained-objects/hydrationAggregate.ts index 5856929f96d30..9e4c26c461241 100644 --- a/console/src/api/materialize/maintained-objects/hydrationAggregate.ts +++ b/console/src/api/materialize/maintained-objects/hydrationAggregate.ts @@ -13,23 +13,85 @@ import { queryBuilder } from "~/api/materialize"; export interface HydrationAggregateRow { object_id: string; - hydratedReplicas: bigint; - totalReplicas: bigint; + hydratedReplicas: bigint | null; + totalReplicas: bigint | null; + sourceStatus: string | null; + sourceError: string | null; + snapshotCommitted: boolean | null; } /** - * One row per object: how many of its replicas report `hydrated = true`, out - * of the total replicas recorded in `mz_hydration_statuses` + * One row per object: how many of its replicas report `hydrated = true` out of + * the total replicas recorded in `mz_hydration_statuses`, plus, for sources, + * their ingestion status, error, and snapshot flag. + * + * Source status rides on this feed rather than its own SUBSCRIBE: it is the + * same one-row-per-object grain, and `mz_hydration_statuses` already reads + * `mz_source_statistics`, so this adds no new frontier dependency. The join + * with `mz_source_statuses` is FULL so webhook sources, which have a status but + * no hydration rows, still get a row. + * + * `snapshot_committed` is aggregated with `bool_and`, so a source only reads as + * committed once every replica reporting statistics agrees. A replica that has + * not reported yet contributes no row, so a scale-up cannot flip a committed + * source back to snapshotting. Statistics rows can outlive their replica, so + * the aggregate is restricted to live replicas (keeping rows with no + * `replica_id`, which is how webhook sources report); a dropped replica's + * stale row must not pin the flag. + * + * Subsources and progress collections are excluded: the UI hides them, and + * they multiply the feed's cardinality several times over. */ export function buildHydrationAggregateQuery() { - return queryBuilder - .selectFrom("mz_hydration_statuses") + // Subsources have hydration rows of their own (they run on their parent's + // cluster), so both sides of the FULL JOIN need the exclusion. + const hydration = queryBuilder + .selectFrom("mz_hydration_statuses as hs") + .leftJoin("mz_sources as s", "s.id", "hs.object_id") + .where((eb) => + eb.or([ + eb("s.type", "is", null), + eb("s.type", "not in", ["subsource", "progress"]), + ]), + ) .select((eb) => [ - "object_id", - sql`count(*) FILTER (WHERE ${eb.ref("hydrated")})`.as( + "hs.object_id", + sql`count(*) FILTER (WHERE ${eb.ref("hs.hydrated")})`.as( "hydratedReplicas", ), sql`count(*)`.as("totalReplicas"), ]) - .groupBy("object_id"); + .groupBy("hs.object_id"); + + const statuses = queryBuilder + .selectFrom("mz_source_statuses") + .where("type", "not in", ["subsource", "progress"]) + .select(["id", "status", "error"]); + + const snapshotCommittedBySource = queryBuilder + .selectFrom("mz_source_statistics as st") + .leftJoin("mz_cluster_replicas as r", "r.id", "st.replica_id") + .where((eb) => + eb.or([eb("r.id", "is not", null), eb("st.replica_id", "is", null)]), + ) + .select((eb) => [ + "st.id", + sql`bool_and(${eb.ref("st.snapshot_committed")})`.as( + "snapshotCommitted", + ), + ]) + .groupBy("st.id"); + + return queryBuilder + .selectFrom(hydration.as("h")) + .fullJoin(statuses.as("ss"), "ss.id", "h.object_id") + .leftJoin(snapshotCommittedBySource.as("stats"), "stats.id", "ss.id") + .select([ + sql`coalesce(h.object_id, ss.id)`.as("object_id"), + "h.hydratedReplicas", + "h.totalReplicas", + "ss.status as sourceStatus", + "ss.error as sourceError", + "stats.snapshotCommitted", + ]); } diff --git a/console/src/platform/connectors/utils.ts b/console/src/platform/connectors/utils.ts index 957393bce79c7..94c2845f7023c 100644 --- a/console/src/platform/connectors/utils.ts +++ b/console/src/platform/connectors/utils.ts @@ -40,6 +40,28 @@ export const snapshotting = (connector: ConnectorStatusInfo) => { ); }; +/** + * Explains why a snapshot's record count is approximate, per source type. + * + * Postgres and MySQL read the count from table statistics (`reltuples` and + * equivalents), which drift until an ANALYZE. Kafka derives it from offset + * watermarks, which overcount compacted or transactional topics. Either way the + * staged/known ratio can plateau below 100% after the snapshot is done, so + * `snapshot_committed` decides completion and this note explains the number. + */ +export const snapshotEstimateNote = (type: string | null) => { + switch (type) { + case "postgres": + case "mysql": + case "sql-server": + return "Snapshot progress and row counts are estimates, based on table statistics."; + case "kafka": + return "Snapshot progress and message counts are estimates."; + default: + return "Snapshot progress and record counts are estimates."; + } +}; + /** * Derives a simple `healthy` | `unhealthy` status from the underlying connector status. */ diff --git a/console/src/platform/maintained-objects/CriticalPathClusterMetrics.tsx b/console/src/platform/maintained-objects/CriticalPathClusterMetrics.tsx index 6245791a9fa3e..ad9b24d093e84 100644 --- a/console/src/platform/maintained-objects/CriticalPathClusterMetrics.tsx +++ b/console/src/platform/maintained-objects/CriticalPathClusterMetrics.tsx @@ -317,22 +317,22 @@ const DroppedReplicasFooter = ({ ); }; -export interface CriticalPathClusterMetricsProps { - node: RenderableNode; - pageObjectId: string; +export interface ClusterReplicaMetricsProps { + cluster: Cluster | null; timestamp: Date | null; bucketSizeMs: number; lookbackMs: number; } -export const CriticalPathClusterMetrics = ({ - node, - pageObjectId, +/** Per-replica memory/CPU utilization for one cluster at a point in time, + * with any replicas dropped inside the window called out. */ +export const ClusterReplicaMetrics = ({ + cluster, timestamp, bucketSizeMs, lookbackMs, -}: CriticalPathClusterMetricsProps) => { - const clusterId = node.cluster?.id ?? null; +}: ClusterReplicaMetricsProps) => { + const clusterId = cluster?.id ?? null; const { data: bucketsByReplicaId, isLoading, @@ -348,9 +348,19 @@ export const CriticalPathClusterMetrics = ({ Math.floor((timestamp ?? new Date()).getTime() / bucketSizeMs) * bucketSizeMs; const buckets = Object.values(bucketsByReplicaId ?? {}); - const replicas = buckets.flatMap((bs) => + const atAnchor = buckets.flatMap((bs) => bs.filter((b) => b.bucketStart.getTime() === targetBucketStartMs), ); + // Right after a replica swap the current bucket can predate the new + // replica's first metrics; fall back to each replica's latest bucket so a + // live cluster is not reported as inactive. Only for a now-ish anchor: for + // a historical anchor an empty bucket means the cluster really had no + // replicas then, and present-time metrics must not impersonate it. + const anchorIsRecent = Date.now() - targetBucketStartMs < 2 * bucketSizeMs; + const replicas = + atAnchor.length > 0 || !anchorIsRecent + ? atAnchor + : buckets.flatMap((bs) => bs.slice(-1)); const replicasInWindow: ReplicaInWindow[] = buckets .flatMap((bs) => bs.slice(-1)) .map(({ replicaId, name, size, bucketEnd }) => ({ @@ -362,13 +372,8 @@ export const CriticalPathClusterMetrics = ({ return ( - ); }; + +export interface CriticalPathClusterMetricsProps { + node: RenderableNode; + pageObjectId: string; + timestamp: Date | null; + bucketSizeMs: number; + lookbackMs: number; +} + +export const CriticalPathClusterMetrics = ({ + node, + pageObjectId, + timestamp, + bucketSizeMs, + lookbackMs, +}: CriticalPathClusterMetricsProps) => ( + + + + +); diff --git a/console/src/platform/maintained-objects/FilterChips.tsx b/console/src/platform/maintained-objects/FilterChips.tsx index 498c518199050..66f3d7d7c2374 100644 --- a/console/src/platform/maintained-objects/FilterChips.tsx +++ b/console/src/platform/maintained-objects/FilterChips.tsx @@ -14,7 +14,7 @@ import React from "react"; import { MaintainedObjectType } from "~/api/materialize/maintained-objects/constants"; import { fromSeconds } from "~/utils/format"; -import { HYDRATION_LABELS, HydrationBucket } from "./filters"; +import { STATUS_FILTER_LABELS } from "./filters"; import { MaintainedObjectListItem } from "./queries"; interface Chip { @@ -80,12 +80,12 @@ const CHIP_SOURCES: ChipSource[] = [ }, }, { - columnId: "hydration", + columnId: "status", build: (column) => - multiSelectChips( + multiSelectChips( column, - "Hydration", - (b) => HYDRATION_LABELS[b], + "Status", + (b) => STATUS_FILTER_LABELS[b], ), }, ]; diff --git a/console/src/platform/maintained-objects/MaintainedObjects.test.tsx b/console/src/platform/maintained-objects/MaintainedObjects.test.tsx index d7793768dbbe2..44f0434d39408 100644 --- a/console/src/platform/maintained-objects/MaintainedObjects.test.tsx +++ b/console/src/platform/maintained-objects/MaintainedObjects.test.tsx @@ -7,7 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { screen, waitFor } from "@testing-library/react"; +import { screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import React from "react"; @@ -29,6 +29,7 @@ const buildObject = ( hydratedReplicas: 1, totalReplicas: 1, lag: null, + sourceStatus: null, ...overrides, }); @@ -54,9 +55,11 @@ const eventsSrc = buildObject({ id: "u103", name: "events_src", objectType: "source", + sourceType: "kafka", cluster: { id: "u200", name: "ingest_cluster" }, hydratedReplicas: 0, totalReplicas: 1, + sourceStatus: { status: "running", error: null, snapshotCommitted: true }, }); const buildProps = ( @@ -108,6 +111,63 @@ describe("MaintainedObjects", () => { expect(screen.getByText("Type: index")).toBeVisible(); }); + it("shows a source's ingestion status rather than replica hydration", async () => { + // A source mid-snapshot has no rehydration latency, so hydration reports 0 + // of 1 replicas hydrated. That must not surface as "Not Hydrated". + const snapshottingSrc = buildObject({ + id: "u104", + name: "snapshotting_src", + objectType: "source", + sourceType: "postgres", + hydratedReplicas: 0, + totalReplicas: 1, + sourceStatus: { + status: "running", + error: null, + snapshotCommitted: false, + }, + }); + await renderMaintainedObjects({ objects: [snapshottingSrc] }); + + const row = await screen.findByRole("row", { name: /snapshotting_src/ }); + expect(within(row).getByText("Snapshotting")).toBeVisible(); + expect(within(row).queryByText("Not Hydrated")).not.toBeInTheDocument(); + }); + + it("surfaces a stalled source's status", async () => { + const stalledSrc = buildObject({ + id: "u105", + name: "stalled_src", + objectType: "source", + sourceType: "kafka", + hydratedReplicas: 1, + totalReplicas: 1, + sourceStatus: { + status: "stalled", + error: "broker unreachable", + snapshotCommitted: true, + }, + }); + await renderMaintainedObjects({ objects: [stalledSrc] }); + + const row = await screen.findByRole("row", { name: /stalled_src/ }); + expect(within(row).getByText("Stalled")).toBeVisible(); + }); + + it("filters rows by status and reflects it in a chip", async () => { + const user = userEvent.setup(); + await renderMaintainedObjects(); + + await user.click( + await screen.findByRole("button", { name: "Filter status" }), + ); + await user.click(await screen.findByLabelText("Running")); + + expect(screen.getByText("events_src")).toBeVisible(); + expect(screen.queryByText("inventory_idx")).not.toBeInTheDocument(); + expect(screen.getByText("Status: Running")).toBeVisible(); + }); + it("filters rows by search term", async () => { const user = userEvent.setup(); await renderMaintainedObjects(); diff --git a/console/src/platform/maintained-objects/MaintainedObjects.tsx b/console/src/platform/maintained-objects/MaintainedObjects.tsx index 73acf8c33f0ff..7d1527e64c1f7 100644 --- a/console/src/platform/maintained-objects/MaintainedObjects.tsx +++ b/console/src/platform/maintained-objects/MaintainedObjects.tsx @@ -35,7 +35,7 @@ import { Link as RouterLink, useLocation, useNavigate } from "react-router-dom"; import { createNamespace } from "~/api/materialize"; import { OUTDATED_THRESHOLD_SECONDS } from "~/api/materialize/cluster/materializationLag"; -import StatusPill from "~/components/StatusPill"; +import StatusPill, { ConnectorStatusPill } from "~/components/StatusPill"; import { sortingFunctions } from "~/components/Table/tableColumnBuilders"; import { TablePagination } from "~/components/Table/TablePagination"; import { TableSearch } from "~/components/Table/TableSearch"; @@ -68,8 +68,8 @@ import { FilterChips } from "./FilterChips"; import { ClusterFilterPanel, FreshnessFilterPanel, - HydrationFilterPanel, ObjectTypeFilterPanel, + StatusFilterPanel, } from "./filterPanels"; import { bucketForHydration, @@ -77,10 +77,11 @@ import { FILTER_URL_SPECS, freshnessFilterFn, HYDRATION_LABELS, - hydrationFilterFn, initialColumnFiltersFromUrl, objectTypeFilterFn, STATUS_COLOR_SCHEMES, + statusBucketForRow, + statusFilterFn, } from "./filters"; import { MaintainedObjectListItem } from "./queries"; @@ -259,16 +260,47 @@ const FreshnessCell = ({ ); }; -const StatusCell = ({ +const pendingPill = ; + +/** + * Sources report their own ingestion status, so they don't go through replica + * hydration. For a source, `hydrated` only tracks whether `rehydration_latency` + * is set, which stays false for the whole initial snapshot and would render a + * healthy source as `Not Hydrated`. + */ +const SourceStatusCell = ({ row, ready, }: { row: MaintainedObjectListItem; ready: boolean; }) => { + // Status rides on the hydration-aggregate feed, so it can trail the row. + if (!row.sourceStatus) return ready ? "-" : pendingPill; + return ( + + ); +}; + +const StatusCell = ({ + row, + hydrationReady, +}: { + row: MaintainedObjectListItem; + hydrationReady: boolean; +}) => { + if (row.objectType === "source") { + return ; + } const { hydratedReplicas, totalReplicas } = row; if (totalReplicas === 0) { - return ready ? "-" : ; + return hydrationReady ? "-" : pendingPill; } const bucket = bucketForHydration(hydratedReplicas, totalReplicas); if (!bucket) return "-"; @@ -320,27 +352,26 @@ const columns = [ renderFilter: (column) => , }, }), - columnHelper.accessor( - (row) => - row.totalReplicas === 0 ? null : row.hydratedReplicas / row.totalReplicas, - { - id: "status", - header: "Status", - sortingFn: sortingFunctions.nullsLast, - filterFn: hydrationFilterFn, - cell: (info) => { - const meta = info.table.options.meta as MaintainedObjectsTableMeta; - return ( - - ); - }, - meta: { - tooltip: - "Object status derived from replica hydration across all clusters.", - renderFilter: (column) => , - }, + columnHelper.accessor((row) => statusBucketForRow(row) ?? null, { + id: "status", + header: "Status", + sortingFn: sortingFunctions.nullsLast, + filterFn: statusFilterFn, + cell: (info) => { + const meta = info.table.options.meta as MaintainedObjectsTableMeta; + return ( + + ); }, - ), + meta: { + tooltip: + "Ingestion status for sources, replica hydration across all clusters for everything else.", + renderFilter: (column) => , + }, + }), columnHelper.accessor((row) => row.cluster?.name ?? null, { id: "clusterName", header: "Cluster", diff --git a/console/src/platform/maintained-objects/ObjectDetailsCard.tsx b/console/src/platform/maintained-objects/ObjectDetailsCard.tsx index 878bf525f5517..cce9edf9534d9 100644 --- a/console/src/platform/maintained-objects/ObjectDetailsCard.tsx +++ b/console/src/platform/maintained-objects/ObjectDetailsCard.tsx @@ -21,7 +21,7 @@ import { Link as RouterLink } from "react-router-dom"; import { createNamespace } from "~/api/materialize"; import { OUTDATED_THRESHOLD_SECONDS } from "~/api/materialize/cluster/materializationLag"; -import StatusPill from "~/components/StatusPill"; +import StatusPill, { ConnectorStatusPill } from "~/components/StatusPill"; import TextLink from "~/components/TextLink"; import { DetailItem } from "~/platform/connectors/AsideBox"; import { absoluteClusterPath } from "~/platform/routeHelpers"; @@ -91,7 +91,7 @@ export const ObjectDetailsCard = ({ {item.name} - + @@ -143,7 +143,22 @@ export const ObjectDetailsCard = ({ ); }; -const HydrationBadge = ({ item }: { item: MaintainedObjectListItem }) => { +/** Mirrors the list's Status column: ingestion status for sources, replica + * hydration for everything else. */ +const StatusBadge = ({ item }: { item: MaintainedObjectListItem }) => { + if (item.objectType === "source") { + if (!item.sourceStatus) return null; + return ( + + ); + } const bucket = bucketForHydration(item.hydratedReplicas, item.totalReplicas); if (!bucket) return null; return ( diff --git a/console/src/platform/maintained-objects/ObjectFreshness.tsx b/console/src/platform/maintained-objects/ObjectFreshness.tsx index c8faab82587c9..2e96263ffd83a 100644 --- a/console/src/platform/maintained-objects/ObjectFreshness.tsx +++ b/console/src/platform/maintained-objects/ObjectFreshness.tsx @@ -24,6 +24,7 @@ import { MaterializeTheme } from "~/theme"; import { formatDate } from "~/utils/dateFormat"; import { LOOKBACK_OPTIONS } from "./constants"; +import { ClusterReplicaMetrics } from "./CriticalPathClusterMetrics"; import { CriticalPathGraph } from "./CriticalPathGraph"; import { computeFreshnessBreakdown } from "./freshnessBreakdown"; import { FreshnessBreakdownStrip } from "./FreshnessBreakdownStrip"; @@ -85,8 +86,9 @@ export const ObjectFreshness = ({ const breakdown = computeFreshnessBreakdown(criticalPath?.directInputs ?? []); // Sources ingest from external systems and have no Materialize-internal - // upstream chain to walk — hide the critical path section for them. - const hasUpstreamChain = item.objectType !== "source"; + // upstream chain to walk. Their "critical path" is the cluster they ingest + // on, so they get its replica metrics instead. + const isSource = item.objectType === "source"; return ( @@ -122,7 +124,31 @@ export const ObjectFreshness = ({ isLocked={isLocked} /> - {hasUpstreamChain && ( + {isSource && ( + <> + + + + Source cluster metrics + + Resources of the cluster this source ingests on, at the point + selected on the freshness graph above. + + + + + + )} + + {!isSource && ( <> diff --git a/console/src/platform/maintained-objects/ObjectMetadata.tsx b/console/src/platform/maintained-objects/ObjectMetadata.tsx index b4b77b474964e..1a1a651fabea2 100644 --- a/console/src/platform/maintained-objects/ObjectMetadata.tsx +++ b/console/src/platform/maintained-objects/ObjectMetadata.tsx @@ -27,7 +27,13 @@ export const ObjectMetadata = ({ item }: ObjectMetadataProps) => { return ( - {isSource && } + {isSource && ( + + )} { +/** + * Renders only while the source has something diagnostic to say: a status + * error, or an in-progress snapshot. Steady-state lifecycle facts live on the + * source details page instead. + */ +export const SourceDiagnostics = ({ + sourceId, + sourceType, + sourceStatus, +}: SourceDiagnosticsProps) => { const { colors } = useTheme(); const { data: stats } = useObjectSourceStatistics(sourceId); - if (!stats) return null; + const error = sourceStatus?.error; + // `snapshot_committed` is authoritative. Judging by the staged/known ratio + // would report a live source as stuck at 99% forever. + const snapshotting = sourceStatus?.snapshotCommitted === false; + if (!error && !snapshotting) return null; - const { - messagesReceived, - snapshotRecordsKnown: snapshotKnown, - snapshotRecordsStaged: snapshotStaged, - rehydrationLatency, - } = stats; + const snapshotKnown = stats?.snapshotRecordsKnown ?? 0; + const snapshotStaged = stats?.snapshotRecordsStaged ?? 0; const snapshotPercent = snapshotKnown > 0 - ? Math.round((snapshotStaged / snapshotKnown) * 100) + ? Math.min(100, Math.round((snapshotStaged / snapshotKnown) * 100)) : null; - const snapshotComplete = snapshotPercent === null || snapshotPercent === 100; - const rehydrationMs = rehydrationLatency - ? sumPostgresIntervalMs(rehydrationLatency) - : null; return ( { Source diagnostics - - - {rehydrationMs !== null - ? `${(rehydrationMs / 1000).toFixed(1)}s` - : "Still rehydrating..."} - - - {messagesReceived.toLocaleString()} - - {snapshotComplete ? ( - - Complete + {error && } + + {snapshotting && ( + + + {snapshotPercent === null + ? "In progress" + : `${snapshotPercent}% (${snapshotStaged.toLocaleString()} / ${snapshotKnown.toLocaleString()} records)`} - ) : ( - <> - - {`${snapshotPercent}% (${snapshotStaged.toLocaleString()} / ${snapshotKnown.toLocaleString()} records)`} - + {snapshotPercent !== null && ( { transition="width 0.3s ease" /> - - )} - + )} + + {snapshotEstimateNote(sourceType)} + + + )} ); diff --git a/console/src/platform/maintained-objects/criticalPathRenderable.test.ts b/console/src/platform/maintained-objects/criticalPathRenderable.test.ts index b95dbd086364d..a0d6ad23e57d1 100644 --- a/console/src/platform/maintained-objects/criticalPathRenderable.test.ts +++ b/console/src/platform/maintained-objects/criticalPathRenderable.test.ts @@ -47,6 +47,7 @@ const buildProbe = ( hydratedReplicas: 1, totalReplicas: 1, lag: { value: parse("00:00:10"), ms: 10_000 }, + sourceStatus: null, ...overrides, }); diff --git a/console/src/platform/maintained-objects/filterPanels.tsx b/console/src/platform/maintained-objects/filterPanels.tsx index 9846a370cd35d..670e18320831a 100644 --- a/console/src/platform/maintained-objects/filterPanels.tsx +++ b/console/src/platform/maintained-objects/filterPanels.tsx @@ -30,11 +30,7 @@ import { import { MaterializeTheme } from "~/theme"; import { DurationUnit, fromSeconds, toSeconds } from "~/utils/format"; -import { - HYDRATION_BUCKETS, - HYDRATION_LABELS, - HydrationBucket, -} from "./filters"; +import { STATUS_FILTER_BUCKETS, STATUS_FILTER_LABELS } from "./filters"; import { MaintainedObjectListItem } from "./queries"; type AnyColumn = Column; @@ -176,11 +172,11 @@ export const ObjectTypeFilterPanel = ({ column }: { column: AnyColumn }) => ( /> ); -export const HydrationFilterPanel = ({ column }: { column: AnyColumn }) => ( - +export const StatusFilterPanel = ({ column }: { column: AnyColumn }) => ( + column={column} - items={HYDRATION_BUCKETS} - getLabel={(b) => HYDRATION_LABELS[b]} + items={STATUS_FILTER_BUCKETS} + getLabel={(b) => STATUS_FILTER_LABELS[b]} /> ); diff --git a/console/src/platform/maintained-objects/filters.ts b/console/src/platform/maintained-objects/filters.ts index 9ec64a32584ff..8b9fabc6526e9 100644 --- a/console/src/platform/maintained-objects/filters.ts +++ b/console/src/platform/maintained-objects/filters.ts @@ -13,8 +13,12 @@ import { MAINTAINED_OBJECT_TYPES, MaintainedObjectType, } from "~/api/materialize/maintained-objects/constants"; +import { snapshotting } from "~/platform/connectors/utils"; -import { MaintainedObjectListItem } from "./queries"; +import { + MaintainedObjectListItem, + MaintainedObjectSourceStatus, +} from "./queries"; /** * Row-threshold options for the freshness filter, keyed by seconds. Selecting @@ -38,7 +42,7 @@ export const HYDRATION_BUCKETS = [ export type HydrationBucket = (typeof HYDRATION_BUCKETS)[number]; export const HYDRATION_LABELS: Record = { - hydrated: "Running", + hydrated: "Hydrated", hydrating: "Hydrating", not_hydrated: "Not Hydrated", }; @@ -79,9 +83,73 @@ export const objectTypeFilterFn = arrayMatchFilter( (r) => r.objectType, ); -export const hydrationFilterFn = arrayMatchFilter((r) => - bucketForHydration(r.hydratedReplicas, r.totalReplicas), -); +/** + * Status values offered for sources. `snapshotting` is derived rather than + * reported: `mz_source_statuses` says `running` throughout the initial + * snapshot, so both the pill and this filter fold in `snapshot_committed`. + */ +export const SOURCE_STATUS_BUCKETS = [ + "snapshotting", + "running", + "starting", + "created", + "paused", + "stalled", + "failed", +] as const; +export type SourceStatusBucket = (typeof SOURCE_STATUS_BUCKETS)[number]; + +export const SOURCE_STATUS_LABELS: Record = { + snapshotting: "Snapshotting", + running: "Running", + starting: "Starting", + created: "Created", + paused: "Paused", + stalled: "Stalled", + failed: "Failed", +}; + +/** Derives a source's displayed status, matching `ConnectorStatusPill`. */ +export const bucketForSourceStatus = ( + sourceStatus: MaintainedObjectSourceStatus, + sourceType: string | null, +): string => + snapshotting({ + status: sourceStatus.status, + type: sourceType ?? "", + snapshotCommitted: sourceStatus.snapshotCommitted, + }) + ? "snapshotting" + : sourceStatus.status; + +/** Options the Status filter offers, sources first. */ +export const STATUS_FILTER_BUCKETS = [ + ...SOURCE_STATUS_BUCKETS, + ...HYDRATION_BUCKETS, +] as const; + +export const STATUS_FILTER_LABELS: Record = { + ...SOURCE_STATUS_LABELS, + ...HYDRATION_LABELS, +}; + +/** + * The bucket the Status cell displays: ingestion status for sources, replica + * hydration for everything else. The filter matches on this so it can never + * select a value the cell doesn't show. + */ +export const statusBucketForRow = ( + row: MaintainedObjectListItem, +): string | undefined => { + if (row.objectType === "source") { + return row.sourceStatus + ? bucketForSourceStatus(row.sourceStatus, row.sourceType) + : undefined; + } + return bucketForHydration(row.hydratedReplicas, row.totalReplicas); +}; + +export const statusFilterFn = arrayMatchFilter(statusBucketForRow); export const freshnessFilterFn = ( row: Row, @@ -144,18 +212,20 @@ export const FILTER_URL_SPECS: readonly FilterUrlSpec[] = [ typeof v === "number" ? { key: "freshness", value: v } : undefined, }, { - columnId: "hydration", + // Matches the Status column's id. The filter, its URL param and its chip + // all have to agree on this key or the filter silently stops round-tripping. + columnId: "status", fromUrl: (p) => nonEmpty( p - .getAll("hydration[]") - .filter((h): h is HydrationBucket => - (HYDRATION_BUCKETS as readonly string[]).includes(h), + .getAll("status[]") + .filter((s) => + (STATUS_FILTER_BUCKETS as readonly string[]).includes(s), ), ), toUrl: (v) => { - const arr = v as HydrationBucket[]; - return arr?.length ? { key: "hydration[]", value: arr } : undefined; + const arr = v as string[]; + return arr?.length ? { key: "status[]", value: arr } : undefined; }, }, ]; diff --git a/console/src/platform/maintained-objects/queries.ts b/console/src/platform/maintained-objects/queries.ts index 9bdd12f73c7b1..ca50e4f3d0068 100644 --- a/console/src/platform/maintained-objects/queries.ts +++ b/console/src/platform/maintained-objects/queries.ts @@ -63,6 +63,22 @@ export interface MaintainedObjectLag { ms: number; } +/** + * Ingestion state from the hydration-aggregate feed, which joins + * `mz_source_statuses` and the snapshot flag. This is what the Status column + * shows for a source, in place of replica hydration, which for a source only + * tracks whether `rehydration_latency` is set and so reads false throughout the + * initial snapshot. + */ +export interface MaintainedObjectSourceStatus { + /** One of `created`, `starting`, `running`, `paused`, `stalled`, `failed`. */ + status: string; + /** Populated when the source is `stalled` or `failed`. */ + error: string | null; + /** Null while no replica has reported statistics yet. */ + snapshotCommitted: boolean | null; +} + export interface MaintainedObjectListItem { id: string; name: string; @@ -78,6 +94,8 @@ export interface MaintainedObjectListItem { totalReplicas: number; /** Null until the lag snapshot arrives. */ lag: MaintainedObjectLag | null; + /** Null for non-sources, and until the hydration snapshot arrives. */ + sourceStatus: MaintainedObjectSourceStatus | null; } const MAINTAINED_OBJECT_TYPE_SET: ReadonlySet = new Set( @@ -133,9 +151,9 @@ export interface UseMaintainedObjectsListResult { /** * Composes maintained objects from three live sources: `useAllObjects()` for - * object metadata, plus SUBSCRIBEs for pMAX lag and hydration aggregate. The - * table renders as soon as object metadata is ready; lag and hydration cells - * fill in as their snapshots arrive. + * object metadata, plus SUBSCRIBEs for pMAX lag and the hydration aggregate, + * which also carries source status. The table renders as soon as object + * metadata is ready; lag and status cells fill in as their snapshots arrive. */ export const useMaintainedObjectsList = ({ lookbackMinutes, @@ -179,6 +197,13 @@ export const useMaintainedObjectsList = ({ lag: lagInterval ? { value: lagInterval, ms: sumPostgresIntervalMs(lagInterval) } : null, + sourceStatus: hydrationRow?.sourceStatus + ? { + status: hydrationRow.sourceStatus, + error: hydrationRow.sourceError, + snapshotCommitted: hydrationRow.snapshotCommitted, + } + : null, }); } return [...objectsById.values()].sort((a, b) => diff --git a/console/src/platform/sources/SourceOverview/SnapshotProgress.tsx b/console/src/platform/sources/SourceOverview/SnapshotProgress.tsx index a2fc7f2e19dd0..119db6fa27111 100644 --- a/console/src/platform/sources/SourceOverview/SnapshotProgress.tsx +++ b/console/src/platform/sources/SourceOverview/SnapshotProgress.tsx @@ -10,7 +10,11 @@ import { Box, HStack, Text, useTheme, VStack } from "@chakra-ui/react"; import React from "react"; -import { ConnectorStatusInfo, snapshotting } from "~/platform/connectors/utils"; +import { + ConnectorStatusInfo, + snapshotEstimateNote, + snapshotting, +} from "~/platform/connectors/utils"; import { MaterializeTheme } from "~/theme"; import { pluralize } from "~/util"; @@ -36,23 +40,6 @@ const mapSourceTypeToProgressUnit = (type: string, count: number) => { } }; -// The known count for database sources comes from table statistics (Postgres -// `reltuples` and equivalents), which are estimates that drift until an ANALYZE. -// Kafka derives it from offset watermarks, which overcount compacted or -// transactional topics, so the note stays type-specific rather than claiming -// table statistics everywhere. -const mapSourceTypeToEstimateNote = (type: string) => { - switch (type) { - case "postgres": - case "mysql": - case "sql-server": - return "Snapshot progress and row counts are estimates, based on table statistics."; - case "kafka": - return "Snapshot progress and message counts are estimates."; - default: - return "Snapshot progress and record counts are estimates."; - } -}; export const SnapshotProgress = ({ snapshotRecordsKnown, snapshotRecordsStaged, @@ -112,7 +99,7 @@ export const SnapshotProgress = ({ - {mapSourceTypeToEstimateNote(source.type)} + {snapshotEstimateNote(source.type)} diff --git a/doc/developer/design/20260210_incremental_occ_read_then_write.md b/doc/developer/design/20260210_incremental_occ_read_then_write.md index 8f60ea1cde37f..2d99887ab60db 100644 --- a/doc/developer/design/20260210_incremental_occ_read_then_write.md +++ b/doc/developer/design/20260210_incremental_occ_read_then_write.md @@ -235,6 +235,11 @@ The correctness argument has two parts: (1) the OCC loop produces the right diffs, and (2) the timestamped write mechanism ensures they are applied at the right timestamp. +User statements require every relation leaf to be a user table owned by the +table-write transaction domain. Source-backed tables, including webhook tables, +receive updates outside that domain, so the target table's oracle cannot make +their updates atomic with the resulting write. + ### The subscribe produces the right diffs The subscribe starts at the oracle read timestamp and emits the current state diff --git a/doc/developer/design/20260817_durable_object_hydration_history.md b/doc/developer/design/20260817_durable_object_hydration_history.md new file mode 100644 index 0000000000000..31271fabe9c5d --- /dev/null +++ b/doc/developer/design/20260817_durable_object_hydration_history.md @@ -0,0 +1,348 @@ +# Durable Object Hydration History + +## Context + +Materialize exposes current hydration state, but that state disappears when a +dataflow or a replica restarts. A user can tell whether an object is hydrated +now, but not how long the last hydration took, or whether today's is unusually +slow. + +This records completed compute-object hydration episodes in a durable table. It +is the first stage of broader hydration history. Replica-wide episodes, resource +peaks, failed episodes, and storage objects need signals that do not exist yet. + +## Goals + +- Record successful hydration of indexes and materialized views per replica. +- Survive environmentd and replica restarts. +- Stay idempotent across concurrent environmentd processes. +- Leave the values of existing hydration relations untouched. +- Bound storage with configurable retention. + +## Non-Goals + +Each of these is excluded because the signal to do it correctly is missing, not +because it is unwanted: + +- **Failed or canceled episodes.** A replica cannot report its own crash, and the + current-state log retracts an object without recording why. +- **Replica-wide episodes.** Correct boundaries need the object set present at the + transition from hydrated to hydrating, and explicit transitions between those + states. +- **Resource peaks.** The metrics history holds samples, not high-water values. + Presenting samples as peaks would mislead. +- **Sources and sinks.** Storage publishes no equivalent lifecycle timestamps. +- **Failed replicas.** A replica that never finishes hydrating an object records + nothing for it, since only completions are stamped. + +## Design + +Compute already stamps three replica-side timestamps per export and worker in +`mz_introspection.mz_compute_hydration_times_per_worker`: `installed_at` when the +dataflow is installed, `started_at` when it is unsuspended, and `hydrated_at` when +the output frontier passes the as-of. See +`doc/developer/design/20260817_compute_hydration_timestamps.md`. This design adds +the history table and the sweep that writes those timestamps down. + +A coordinator task visits one user replica per interval, in a rotation, skipping +replicas with introspection disabled, whose logs are installed but never populated. +It installs an internal subscribe on that replica which aggregates every worker's +completed rows, anti-joins them against the history table, and writes the missing +ones through the timestamped OCC read-then-write path. + +Collection has no explicit batch bound. It returns at most one row per +not-yet-recorded dataflow, and the OCC path rejects a result that exceeds +`max_result_size` or `max_query_result_size`. At their 1 GiB defaults that ceiling +only matters at millions of dataflows per replica. + +Two dyncfgs control it. `hydration_history_collection_interval` sets the sweep +cadence and disables collection at zero, which is the production default. +`hydration_history_retention_period` bounds how long rows live, defaulting to 30 +days. + +An *episode* throughout this document is one such row: one dataflow's hydration on +one replica, from installation to hydrated, recorded once. + +## History Table + +```text +mz_internal.mz_object_hydration_history + object_id text not null + cluster_id text not null + replica_id text not null + installed_at timestamptz not null + started_at timestamptz null + hydrated_at timestamptz null + status text not null +``` + +An episode is identified by `(object_id, replica_id, installed_at)`. The +installation time is replica-stamped and stable across an environmentd restart. The +nullability of each column matches the compute log the values come from, so +`installed_at` is not null there and here, while the two later stamps are nullable. + +`object_id` is named the way the other hydration relations name it, because to a +user these are objects that exist somewhere rather than exports of a dataflow. The +value is the id compute reports for the object's dataflow, and one catalog item can +own several dataflows at once, so consumers reach a catalog object through +`mz_internal.mz_object_global_ids` rather than joining `mz_objects` directly. + +Only terminal `hydrated` rows are written today, so every row has a finish time. +The nullable columns and `status` reserve a compatible shape for canceled or +unfinished episodes once those become observable. Retention keys off `hydrated_at`, +so an unfinished episode needs a second age basis before it can be recorded at all. + +The table lives in `mz_internal` to mark it unstable while it settles. + +No index initially, and not a retained-metrics object either, both because of size. +The collector does anti-join by `(object_id, replica_id, installed_at)`, but its +subscribe runs on the selected user replica. An index arranged on the catalog +server cannot serve that dataflow, so it would pin the whole table without removing +the collector's recurring import and arrangement cost. That cost grows with retained +history and is accepted by this sampling design. Adding an index later is a +migration we can do when a query that can use it needs one. The retained-metrics +flag would independently pin a 30 day compaction window, so it stays off too. + +## Why concurrent writers converge + +The subscribe reads the history table it is about to write. That is what makes the +write idempotent across processes, and it is the reason the read expression looks +redundant. + +Two environmentd processes computing the same missing row at frontier `T` both +submit a timestamped write for `T`. One commits and advances the table upper. The +other is told the timestamp passed, waits for its own subscribe to progress, sees +the committed row appear, and its anti-join retracts the candidate. There is +nothing left to write. + +The losing writer never retries stale diffs at the next eligible timestamp. It +only retries with state it has observed to be valid at the frontier it is writing +at. That same property is what makes a trailing replica record nothing, which the +last section covers. + +## Aggregating a replica's workers + +Collection aggregates every worker's row for an export and records nothing until all +of them have hydrated, taking `max(hydrated_at)` and `min` of the two start stamps. + +One worker would be enough for an index, whose output frontier is the arrangement +upper and therefore moves only as timely's dataflow-wide progress allows. It is not +enough for a materialized view. Its output frontier also depends on the sink write +frontier, and only the sink's single active worker, `hash(sink_id) % workers`, +tracks the shard upper. Every other worker stamps at compute completion, before the +snapshot is durable, and the sink buffers that snapshot until computation finishes, +so the gap is the whole write. Reading one worker would mean `hydrated_at` said +"durable" for some objects and "computed" for others, decided by a hash. The +aggregate gives one meaning for every object, and it is what +`mz_compute_hydration_times` already does, so the durable history and the live +signal agree. + +Completeness uses no separately configured worker count. The replica owns this +introspection collection and recreates it as a unit on restart, so a snapshot cannot +combine rows from before and after a restart. Installation adds one row per +`(export_id, worker_id)` with a null `hydrated_at`, and +`count(*) = count(hydrated_at)` means every row visible at the read timestamp has +finished. An object still hydrating is skipped and picked up by a later sweep. The +cutoff and the anti-join apply to the aggregate's output, since as `WHERE` clauses +either one would drop visible unfinished rows and make that check trivially true. + +Each process's logging clock also determines the logical timestamps of its updates. +A process whose clock is ahead can therefore place its row beyond the read timestamp +while the other rows are complete below it. The collector deliberately accepts this +sampling race rather than depending on replica configuration for an expected worker +count. In that case the durable finish can precede the latest worker's finish. This +is separate from restart behavior, which discards the replica collection as a unit. + +The missing worker cannot later change the episode key. Its logging clock stamps +both the Differential update and `installed_at`, so appearing after the read means +its installation stamp is later than the visible minimum. A later sweep therefore +keeps the same `min(installed_at)` and the anti-join matches the row already written. +The worker can raise `max(hydrated_at)`, but the durable row is not repaired after +its episode key has been recorded. + +Compute is adding an append-only lifecycle log for the same stages, currently +proposed in #38403: one row per export, worker and event, with a reason and the +dataflow's as-of. It leaves `mz_compute_hydration_times_per_worker` alone, so +everything above keeps its meaning and nothing recorded here is relabelled. + +That log is the signal this design says is missing, and moving onto it later buys +three things at once: stages that mean the same thing on every worker, a reason +that distinguishes a replacement waiting for cutover from an index that will never +write, and the as-of, without which an interval says nothing about how much work +was done, since a replacement with a far behind as-of does far more of it in the +same wall-clock time. Recording should keep gating on hydration when that happens. +A replacement runs read-only and does not write until cutover, so gating on a write +stage would never record the hydration a deployment most wants to measure, and +would wait forever on a rollback. + +Two consequences. An interval can span workers in different processes, each +anchoring its logging clock at its own `SystemTime`, so skew inflates a duration. +Nothing rejects a row for looking inconsistent, deliberately, because a guard on +cross-worker stamps rejects complete episodes permanently once the log values +settle. And `started_at` equal to `installed_at` means no start was observed rather +than an immediate start, since the log substitutes the installation time. An +import-free dataflow is never suspended, so that is the common case. + +## What is and is not recorded + +Collection samples current state rather than consuming events, so it writes down +whatever the replica collection holds when that replica's turn comes around. The +log retracts an export's row when the dataflow goes away, so a short-lived object, +or one dropped before its turn, leaves no trace. Nothing wrong is recorded, it is +simply absent. + +Making these durable requires compute to emit hydration transitions into an +append-only collection that survives until an observer acknowledges them. Until +then, best effort is the honest description of a sampler. + +## Retention + +Retention is another OCC mutation: it subscribes to rows older than the cutoff and +writes their retractions at the observed frontier. Collection applies the same +cutoff, so a still-live log row cannot resurrect an episode retention just +retracted. + +Retention deletes one bounded batch per sweep. The fixed cutoff makes the eligible +set finite, and collection refuses to insert rows behind that cutoff, so later +sweeps continue draining the same backlog without starving collection. The batch +bound is not a nicety. The OCC path refuses a selection larger than +`max_result_size` before submitting any write, so one unbounded delete over a large +backlog would fail identically forever and never shrink the table. The bound has to +sit inside a derived table, because a top-level `LIMIT` lands in the plan's +`RowSetFinishing`, which this OCC stage cannot apply. + +`mz_hydration_history_retention_batch_full_total` increments when the batch deletes +all 1,000 rows. Repeated increments mean retention may not be keeping up. An operator +can lower `hydration_history_collection_interval` to schedule sweeps more often, +then compare appended and deleted row rates to confirm the backlog is shrinking. + +Retention runs on the catalog server, so it keeps working when there are no user +replicas at all, and it runs even when that sweep's collection failed. A +crash-looping replica must not be able to stop the table from shrinking. The +dependency does not run the other way: a catalog server without a replica skips +retention and leaves collection running. + +Disabling collection also suspends retention. The alternative is an always-on +subscribe in the default configuration, where the table is empty and there is +nothing to retain. Rows already collected are therefore kept while collection is +off. + +## Durability is best effort + +Builtin tables are reset at bootstrap and re-sharded on a forced schema migration. +This table is exempt from both, because a sampled history cannot be rebuilt from +anything else. Those two exemptions are the whole promise. + +We do not promise to carry the contents across every future version. A shard +replacement clears the table, and that is an acceptable trade: what has value here +is the distribution the table accumulates, and starting over costs one retention +period, while freezing the schema to protect it costs every later improvement. +Schema *evolution* keeps the rows, so an additive column costs nothing. + +Giving the exemption up is meant to be deliberate. An assert fires if a replacement +step ever names this table, and the comment there says to remove the assert and the +exemption together and to note in the release notes that the history restarts. + +## Scheduling and isolation + +Sweeps never overlap and each visits one replica, which bounds compute load and +keeps the collector from contending with itself in the serialized timestamped-write +path. Fires align to interval boundaries and each sleep is capped, so lowering a +long interval takes effect within the cap rather than after the old interval +elapses. The grid is offset by an amount seeded from the full environment id, since +the interval is one fleet-wide setting and an unshifted grid would have every +environment sweep at the same instant. Using the full id also separates regions and +ordinals belonging to one organization. + +One replica per interval means an environment with `N` eligible replicas revisits +each one approximately every `N * interval`. Freshness therefore degrades linearly +with replica count. Lowering the interval improves freshness at the cost of more +replica dataflow installs. + +**Background mutations take no OCC write permit.** The permits are one semaphore +shared by every read-then-write in the process, not one per table. A session's wait +is bounded by its statement timeout, a sweep's is not, and a sweep's subscribe has +to hydrate a dataflow on a user replica first, so holding a permit would let +background sampling stall user DML on any table for as long as that takes. Being +single-flight, skipping the permit adds at most one concurrent read-then-write. + +**The sweep is aborted when the coordinator is dropped.** It holds no `Client`, so +nothing else stops it outliving the coordinator, and the teardown that follows drops +the timestamp oracle's worker task. A sweep still running would then read a +timestamp from a dead oracle and panic, so the coordinator owns the task handle. + +**Each mutation's timeout is deliberately generous.** Subscribe installation, +replica-side progress, OCC conflict retries, and the external commit can all wait +indefinitely. The bound keeps one unavailable replica or service from starving +retention and every later replica in the single-flight rotation. + +Replica drop, cluster drop, dependency replacement, replica failure, and timeout all +skip the attempt, and a later sweep recomputes from current state. Read-only +generations do nothing, which the next section covers. + +## Upgrades record late, not never + +A read-only generation writes nothing, and a 0dt upgrade hydrates the incoming +replicas while it is still read-only. That looks like it drops exactly the window +worth measuring, since that is when most objects hydrate at once. + +It does not, because collection samples current state rather than events. The +stamps live in the replica's own log, the replicas keep running across promotion, +and the first sweep after promotion writes those episodes with the replica's +original `installed_at` and `hydrated_at`. The history is written late, the +timestamps are not. + +What is genuinely lost is an episode whose dataflow goes away before that first +post-promotion sweep, which is the general sampling limit rather than anything +specific to upgrades. + +## Known limitation: a trailing replica records nothing + +The write timestamp comes from the timeline's oracle, and the subscribe's frontier +certifies that the loop has a complete view below it. The write happens once that +frontier reaches the target. One of the subscribe's inputs is a replica-local +introspection log whose frontier the *replica* advances from its own clock, rounded +up to its introspection interval. + +For a non-empty selection, a replica whose clock trails environmentd by more than +one introspection interval repeatedly loses the same race. Its frontier eventually +certifies the current target, but by then keepalives or other writes have advanced +the oracle past it. The committer refuses the stale target, the OCC loop adopts a +new one, and the replica has to catch up again. The sweep's timeout normally ends +that loop. A lower `max_occ_retries` can end it first with a contention error. + +Nothing wrong is recorded in either case. That replica records nothing at all, and +the sweep stretches while it retries, which delays others in the rotation. Empty +selections exit without entering the conflict loop. The condition resolves itself +when the clocks converge. + +Accepted for now: skew between processes is normally milliseconds while the +tolerance is a whole introspection interval, and the failure mode is retrying rather +than wrong data. Because the symptom is otherwise hard to attribute, both a timeout +and retry-budget exhaustion log that the replica's introspection frontier may be +trailing. + +## Rollout + +Collection is disabled by default, and the interval is runtime configurable, so +enabling it needs no restart. The plan is to enable it in CI when this merges, then +in staging, then in production, a week apart, so each step has a week of real +traffic behind it before the next. + +CI enables it at a 60 second interval through the mzcompose configuration, which is +what exercises the hydration, restart, retention, and catalog tests. It stays off in +the sqllogictest runner defaults, against the usual preference for enabling new +paths in tests: the collector installs subscribes and writes a builtin table, while +those runs assert on catalog contents and plans. + +## Future Work + +- Record installation and start before completion, then finalize canceled and + failed episodes by joining replica lifecycle events. +- Define a replica episode state machine from the object set present at the + hydrated-to-hydrating transition. +- Publish resettable per-process high-water values for RAM, swap, and scratch disk, + and define replica aggregation without pretending sampled maxima are simultaneous + peaks. +- Give storage objects equivalent lifecycle timestamps. +- Build replica history and progress views on those signals. diff --git a/doc/user/content/concepts/snapshotting.md b/doc/user/content/concepts/snapshotting.md index 73522e7c58c4a..02cd807b58ad0 100644 --- a/doc/user/content/concepts/snapshotting.md +++ b/doc/user/content/concepts/snapshotting.md @@ -18,6 +18,10 @@ menu: {{% include-headless "/headless/ingestion/snapshotting-duration" %}} +### Parallelism + +{{% include-headless "/headless/ingestion/snapshotting-parallelism" %}} + ## Queries during snapshotting {{% include-headless "/headless/ingestion/snapshotting-queries" %}} @@ -27,7 +31,9 @@ menu: Snapshotting has the following upstream impacts: - **Read load.** Snapshotting puts read, CPU, and network load on the upstream - system, proportional to the data volume. + system. The total load is proportional to the volume of data being + snapshotted, while the source cluster's [parallelism](#parallelism) affects + the peak load: more workers compress the reads into a shorter window. - **Change-log retention for CDC database sources.** When ingesting data from CDC database sources (PostgreSQL, MySQL, SQL Server), the upstream system must @@ -41,3 +47,4 @@ Snapshotting has the following upstream impacts: - [Ingest data](/ingest-data/) - [Sources](/concepts/sources/) +- [Troubleshooting data ingestion](/ingest-data/troubleshooting/) diff --git a/doc/user/content/headless/ingestion/snapshotting-parallelism.md b/doc/user/content/headless/ingestion/snapshotting-parallelism.md new file mode 100644 index 0000000000000..e23bbafee03cd --- /dev/null +++ b/doc/user/content/headless/ingestion/snapshotting-parallelism.md @@ -0,0 +1,34 @@ +--- +headless: true +--- + +Materialize can parallelize snapshotting across the workers of the cluster +hosting the source. + +- **PostgreSQL sources** are parallelized by table, i.e., different tables + are read concurrently by different workers. On PostgreSQL 14 and later, + Materialize additionally attempts to partition each table's read across + workers. Tables that cannot be partitioned fall back to a single worker. + +- **MySQL sources** are parallelized by table, i.e., different tables are + read concurrently by different workers. For tables that meet certain + requirements, Materialize can additionally partition the table's read + across workers {{< private-preview-inline />}}. See [MySQL snapshot + parallelism](/ingest-data/mysql/snapshot-parallelism/). + +- **Kafka sources** are parallelized by topic partition, with partitions + distributed across workers, so parallelism is bounded by the topic's + partition count. + +- **SQL Server sources** are not parallelized: a single worker reads all + tables. + +The degree of snapshot parallelism depends on the number of workers. A +cluster's [size](/sql/create-cluster/#available-sizes) determines its number +of workers, so a larger cluster can shorten the snapshot, to the extent the +work parallelizes and the upstream database keeps up. The volume read from +the upstream database is unchanged, it is compressed into a shorter window +of more concurrent queries and connections. To determine whether +snapshotting is overloading the upstream database, and for ways to mitigate +the load, see [Is the upstream database +overloaded?](/ingest-data/troubleshooting/#is-the-upstream-database-overloaded) diff --git a/doc/user/content/ingest-data/_index.md b/doc/user/content/ingest-data/_index.md index 3a16c1e89a072..528371b2a0bbc 100644 --- a/doc/user/content/ingest-data/_index.md +++ b/doc/user/content/ingest-data/_index.md @@ -60,6 +60,10 @@ we recommend: the steady-state resource needs of your upsert source(s). See [Best practices: Upsert sources](#upsert-sources). +### Parallelism + +{{% include-headless "/headless/ingestion/snapshotting-parallelism" %}} + ### Monitoring progress While snapshotting is taking place, you can monitor the progress of the diff --git a/doc/user/content/ingest-data/mysql/_index.md b/doc/user/content/ingest-data/mysql/_index.md index 54cedf30a29c9..eaf67640d7be4 100644 --- a/doc/user/content/ingest-data/mysql/_index.md +++ b/doc/user/content/ingest-data/mysql/_index.md @@ -33,6 +33,11 @@ gives you the following benefits: read-replica to build views on top of your MySQL data that are efficiently maintained and always up-to-date. +When a source is created, Materialize parallelizes the initial snapshot +across the cluster's workers and can split the read of large tables that meet +certain requirements {{< private-preview-inline />}}. See [Snapshot +parallelism](/ingest-data/mysql/snapshot-parallelism/). + ## Supported versions and services {{< note >}} diff --git a/doc/user/content/ingest-data/mysql/snapshot-parallelism.md b/doc/user/content/ingest-data/mysql/snapshot-parallelism.md new file mode 100644 index 0000000000000..023e5ab3dc932 --- /dev/null +++ b/doc/user/content/ingest-data/mysql/snapshot-parallelism.md @@ -0,0 +1,108 @@ +--- +title: "Snapshot parallelism" +description: "How Materialize splits the snapshot of a single MySQL table across the workers of a cluster." +menu: + main: + parent: "mysql" + name: "Snapshot parallelism" + identifier: "mysql-snapshot-parallelism" + weight: 70 +--- + +{{< private-preview />}} + +When you create a [MySQL source](/sql/create-source/mysql-v2/), Materialize +performs an initial, snapshot-based sync of the selected tables before it +starts ingesting change events from the binlog. For large tables, this +snapshot dominates the time until the source becomes healthy. + +How snapshot work is spread across the workers of a cluster, and what that +means for the upstream database, is covered in +[Snapshotting](/concepts/snapshotting/#parallelism). Materialize can split +the read of a **single table** across all the workers of the cluster, so +that even a source dominated by one very large table benefits from a larger +cluster. This page covers what is specific to MySQL: which tables are +eligible for splitting, and how their reads are partitioned. + +## Which tables are split + +Materialize splits the snapshot of an individual table across workers when +all of the following conditions are met: + +- The table has a **single-column primary key**. Composite primary keys are + not supported. +- The primary key column is of type **`CHAR` or `VARCHAR`**, with a declared + length of **at most 768 characters**. Other types, including numeric keys, + are not supported. +- The primary key column uses the **`utf8mb4` character set** with the + **`utf8mb4_bin` collation**. +- The table is **large enough to be worth splitting**. Small tables are read + by a single worker, where splitting would add overhead without benefit. + +How evenly the split lands also depends on the distribution of the key +values. See [How a table is partitioned](#how-a-table-is-partitioned). + +If a table does not meet these requirements, or if the [boundary +sampling](#how-a-table-is-partitioned) fails, its snapshot is not split: a +single worker reads the table in full. Different tables are still read +concurrently by different workers. + +## How a table is partitioned + +Materialize partitions an [eligible](#which-tables-are-split) table using the +leading characters of its primary key values. Before reading the table, +Materialize probes the primary key index to discover key prefixes and uses +the MySQL optimizer's row estimates to gauge how many rows fall under each +prefix. It extends the prefixes as needed to find boundaries that divide the +table into roughly even ranges. The probes are inexpensive point lookups, +capped in proportion to the table's estimated size, so the sampling phase +stays negligible next to the snapshot itself. + +Each worker then reads only its assigned range, within the same consistent +snapshot of the upstream database, so the result is identical to a +single-worker snapshot, only faster. + +Because partitioning is based on key prefixes and optimizer estimates, how +evenly the work divides depends on the shape of your keys: + +- **Evenly distributed keys partition well.** Keys whose leading characters + spread rows uniformly, such as UUIDs, hashes, or other randomized + identifiers, produce well-balanced ranges. + +- **Skewed keys partition less evenly.** If a large share of the table's rows + sort under a few common prefixes, some ranges end up with more rows than + others, and the workers assigned to them finish later. + +- **The probe budget can run out.** If finding even boundaries would require + examining very many distinct prefixes, Materialize stops probing and uses + the coarser boundaries found so far, which can also leave ranges uneven. + +Uneven partitioning is never incorrect. It only reduces the speedup, since +the snapshot finishes when the busiest worker finishes. + +## MySQL-specific upstream considerations + +- **Connection count.** While the snapshot is being set up, Materialize + briefly holds up to two connections per worker, plus one. Once reading is + underway, this settles to one connection per worker reading a range, plus + one coordination connection. After the snapshot completes, the source drops + back to a single replication connection. If your MySQL server or connection + pooler enforces a low + [`max_connections`](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_max_connections) + limit, account for this burst when sizing it. + +- **Statistics freshness.** Range boundaries are placed using the MySQL + optimizer's row estimates. Stale statistics don't affect correctness, but + can skew how evenly work divides across workers. Running + [`ANALYZE TABLE`](https://dev.mysql.com/doc/refman/8.0/en/analyze-table.html) + on very large tables before creating the source can improve balance. + +For general guidance on read load, IOPS, and other upstream impact, which is +not specific to MySQL, see [Is the upstream database +overloaded?](/ingest-data/troubleshooting/#is-the-upstream-database-overloaded) + +## Observability + +To observe the progress of an ongoing snapshot, see [Monitoring the +snapshotting +progress](/ingest-data/monitoring-data-ingestion/#monitoring-the-snapshotting-progress). diff --git a/doc/user/content/ingest-data/postgres/_index.md b/doc/user/content/ingest-data/postgres/_index.md index 15f961c52e4dc..9525a8344cf7d 100644 --- a/doc/user/content/ingest-data/postgres/_index.md +++ b/doc/user/content/ingest-data/postgres/_index.md @@ -37,6 +37,11 @@ Materialize gives you the following benefits: Materialize as a read-replica to build views on top of your PostgreSQL data that are efficiently maintained and always up-to-date. +When a source is created, Materialize parallelizes the initial snapshot +across the cluster's workers and, on PostgreSQL 14 and later, splits each +table's read across workers. See [Snapshot +parallelism](/concepts/snapshotting/#parallelism). + ## Supported versions and services The PostgreSQL source requires **PostgreSQL 11+** and is compatible with most diff --git a/doc/user/content/ingest-data/postgres/partitioned-tables.md b/doc/user/content/ingest-data/postgres/partitioned-tables.md index 4c925eba72286..a3ef863084f36 100644 --- a/doc/user/content/ingest-data/postgres/partitioned-tables.md +++ b/doc/user/content/ingest-data/postgres/partitioned-tables.md @@ -40,8 +40,8 @@ that uses this option, and doing so can produce incorrect results. The configuration is accepted rather than rejected: the source is created, the initial snapshot is correct, and inserts, updates, and deletes appear to -replicate normally. Problems surface only once partitions are added, attached, -or detached. Use one of the approaches on this page instead. +replicate normally. Problems surface only once partitions are attached, +detached, or truncated. Use one of the approaches on this page instead. {{< /warning >}} diff --git a/doc/user/content/ingest-data/troubleshooting.md b/doc/user/content/ingest-data/troubleshooting.md index 03d39904a242a..6dbd2d9c0ca56 100644 --- a/doc/user/content/ingest-data/troubleshooting.md +++ b/doc/user/content/ingest-data/troubleshooting.md @@ -99,6 +99,41 @@ also be necessary to support increased memory usage during the process. For more information, see [Use a larger cluster for upsert source snapshotting](/ingest-data/#use-a-larger-cluster-for-upsert-source-snapshotting). +## Is the upstream database overloaded? + +Snapshotting can put significant load on the upstream database (see [Impact +on upstream system](/concepts/snapshotting/#impact-on-upstream-system)). + +Check the upstream database when a snapshot progresses more slowly than +expected, when applications sharing the database slow down while +it runs, or when the source reports upstream connection errors or timeouts. +The relevant metrics are in your cloud provider's monitoring console, or in +OS tools like `iostat` and the database's activity views for self-hosted +databases. Look for: + +- **Read IOPS or throughput** flat at a provisioned cap. +- **CPU** pinned at the instance's limit for the duration of the snapshot. +- **Network throughput** at the instance type's cap. +- **Connections** near the database's limit. For PostgreSQL and MySQL + sources, snapshotting opens connections in proportion to the source + cluster's workers. + +Also watch disk usage on the upstream database during a long-running +snapshot: CDC database sources must retain their change log until Materialize +consumes it (see [Impact on upstream +system](/concepts/snapshotting/#impact-on-upstream-system)). + +If the database is overloaded, you can upsize the source database or cancel +the snapshot by dropping the source, and retry: + +- on a smaller source cluster to spread the load over a longer window. +- with more IOPS, throughput, or instance capacity provisioned for the + database. +- during off-peak hours when the database is less busy, as recommended in the + [ingestion best practices](/ingest-data/#scheduling). +- with a smaller [volume of data to + sync](/ingest-data/#limit-the-volume-of-data). + ## Adding a new subsource to an existing source blocks replication. Should I just create a new source instead? It depends. Materialize provides transactional guarantees for subsource of the diff --git a/doc/user/content/reference/system-catalog/mz_internal.md b/doc/user/content/reference/system-catalog/mz_internal.md index 8067b94a54fc0..8d6f3347be034 100644 --- a/doc/user/content/reference/system-catalog/mz_internal.md +++ b/doc/user/content/reference/system-catalog/mz_internal.md @@ -704,6 +704,33 @@ The `mz_object_history` view enriches the [`mz_catalog.mz_objects`](/reference/s | `created_at` | [`timestamp with time zone`] | Wall-clock timestamp of when the object was created. `NULL` for built in system objects. | | `dropped_at` | [`timestamp with time zone`] | Wall-clock timestamp of when the object was dropped. `NULL` for built in system objects or if the object hasn't been dropped. | +## `mz_object_hydration_history` + +The `mz_object_hydration_history` table records completed hydration of indexes and +materialized views, with one row for each time a dataflow hydrated on a replica. +By default, rows are retained for 30 days while collection is enabled. Disabling +collection also suspends retention, so existing rows remain until collection is +enabled again. `object_id`, `cluster_id`, and `replica_id` may name objects that no +longer exist. + +Recording is best effort. Only successful hydration is recorded, an episode can be +missed if the object or its replica goes away before the episode is recorded, and a +schema change to this table in a future release may clear its contents. On a +multi-process replica, timestamps come from process-local logging clocks and include +their clock skew. A process whose clock is ahead can be absent at the sampled +logical timestamp, so the recorded finish can precede the latest process's finish. + + +| Field | Type | Meaning | +| -------------- | ---------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `object_id` | [`text`] | The ID of the object's dataflow, as reported by the replica. Join `mz_internal.mz_object_global_ids` to reach the index or materialized view while that mapping exists. Dropping the dataflow retracts the mapping, so historical IDs may no longer resolve. | +| `cluster_id` | [`text`] | The ID of the object's cluster. | +| `replica_id` | [`text`] | The ID of the cluster replica. May name a replica that no longer exists. | +| `installed_at` | [`timestamp with time zone`] | When the object's dataflow was installed on the replica. | +| `started_at` | [`timestamp with time zone`] | When hydration work began, or `NULL` if the replica reported none. A replica that observed no start reports the installation time instead, so a zero interval between the two does not mean the dataflow started immediately. | +| `hydrated_at` | [`timestamp with time zone`] | When hydration finished. | +| `status` | [`text`] | The terminal status. Currently always `hydrated`. | + ## `mz_object_transitive_dependencies` The `mz_object_transitive_dependencies` view describes the transitive dependency structure between diff --git a/doc/user/content/releases/v26.39.md b/doc/user/content/releases/v26.39.md index e8f03d3c121d6..79999b6a4c083 100644 --- a/doc/user/content/releases/v26.39.md +++ b/doc/user/content/releases/v26.39.md @@ -1,9 +1,8 @@ --- title: Materialize v26.39 date: 2026-08-26 -released: false +released: true patch: 0 -rc: 1 publish_helm_chart: true build: render: never diff --git a/doc/user/data/metrics.yml b/doc/user/data/metrics.yml index d269dca4d9ca7..034f404364004 100644 --- a/doc/user/data/metrics.yml +++ b/doc/user/data/metrics.yml @@ -97,7 +97,7 @@ metrics: source: src/adapter/src/metrics.rs visibility: internal - name: mz_active_internal_subscribes - help: The number of active internal subscribes, which serve frontend-sequenced read-then-write. + help: The number of active internal subscribes used by read-then-write operations and background maintenance. labels: - session_type source: src/adapter/src/metrics.rs @@ -955,6 +955,37 @@ metrics: help: The time it takes to advance the catalog shard upper for a txns-shard write (group commits and table register/forget). source: src/adapter/src/metrics.rs visibility: internal +- name: mz_hydration_history_mutations_total + help: Total hydration-history collection and retention mutations since process start. + labels: + - operation + - outcome + source: src/adapter/src/metrics.rs + visibility: internal +- name: mz_hydration_history_retention_batch_full_total + help: Total hydration-history sweeps whose retention batch was full. Repeated increments mean retention may not be keeping up with its schedule. + source: src/adapter/src/metrics.rs + visibility: internal +- name: mz_hydration_history_rows_affected_total + help: Total rows changed by hydration-history maintenance since process start. + labels: + - action + source: src/adapter/src/metrics.rs + visibility: internal +- name: mz_hydration_history_sweep_duration_seconds_bucket + help: Wall time of a complete hydration-history collection and retention sweep. + labels: + - le + source: src/adapter/src/metrics.rs + visibility: internal +- name: mz_hydration_history_sweep_duration_seconds_count + help: Wall time of a complete hydration-history collection and retention sweep. + source: src/adapter/src/metrics.rs + visibility: internal +- name: mz_hydration_history_sweep_duration_seconds_sum + help: Wall time of a complete hydration-history collection and retention sweep. + source: src/adapter/src/metrics.rs + visibility: internal - name: mz_index_peek_cursor_setup_seconds_bucket help: Time setting up cursor and literal constraints. labels: diff --git a/misc/python/materialize/feature_benchmark/scenarios/benchmark_main.py b/misc/python/materialize/feature_benchmark/scenarios/benchmark_main.py index 8c3c743adb8a7..7ee0fcb0fa0b4 100644 --- a/misc/python/materialize/feature_benchmark/scenarios/benchmark_main.py +++ b/misc/python/materialize/feature_benchmark/scenarios/benchmark_main.py @@ -2012,18 +2012,6 @@ class MySqlInitialLoad(MySqlCdc): FIXED_SCALE = True # TODO: Remove when database-issues#7556 is fixed - # The parallel snapshot path has a wider memory envelope during the initial - # load, so allow a bit more headroom than the base default before flagging a - # regression. - RELATIVE_THRESHOLD: dict[MeasurementType, float] = { - # Primary key splitting supports only string primary keys, so this - # BIGINT-keyed snapshot reads serially where older versions split it across - # workers. The increased wallclock tolerance covers that lost parallelism. - MeasurementType.WALLCLOCK: 0.25, - MeasurementType.MEMORY_MZ: 0.30, - MeasurementType.MEMORY_CLUSTERD: 0.50, - } - def shared(self) -> Action: return TdAction(f""" $ mysql-connect name=mysql url=mysql://root@mysql password=${{arg.mysql-root-password}} @@ -2071,12 +2059,6 @@ class MySqlInitialLoadMultiWorkerSampled(MySqlCdc): """Measure an 8-worker snapshot across 20 tables with a single-column non-integer primary key (a CHAR(26) ULID-like key).""" - RELATIVE_THRESHOLD: dict[MeasurementType, float] = { - MeasurementType.WALLCLOCK: 0.10, - MeasurementType.MEMORY_MZ: 0.60, - MeasurementType.MEMORY_CLUSTERD: 0.60, - } - # Cap scale so each per-table load fits one mysql.time_zone self-join # (~3M rows), so a single INSERT per table suffices. MAX_SCALE = 6 @@ -2170,16 +2152,6 @@ class MySqlInitialLoadMultiWorkerSingleTable(MySqlCdc): single-column non-integer primary key (a CHAR(26) ULID-like key). The single-table companion to MySqlInitialLoadMultiWorkerSampled.""" - RELATIVE_THRESHOLD: dict[MeasurementType, float] = { - # In MySQL, probing primary keys (for approximately even partitions) is - # slower on small datasets than using OFFSET (for exactly even - # partitions). Measured locally at 2-14% slower by wallclock at - # SCALE=6 (1M rows). - MeasurementType.WALLCLOCK: 0.25, - MeasurementType.MEMORY_MZ: 0.60, - MeasurementType.MEMORY_CLUSTERD: 0.60, - } - # Cap scale so the load fits one mysql.time_zone self-join (~3M rows), so a # single INSERT suffices. MAX_SCALE = 6 diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py index 9ce47a13dc0a5..0244c6f41f21d 100644 --- a/misc/python/materialize/mzcompose/__init__.py +++ b/misc/python/materialize/mzcompose/__init__.py @@ -147,6 +147,9 @@ def get_minimal_system_parameters( if version < MzVersion.parse_mz("v26.25.0-dev"): config["enable_multi_replica_sources"] = "true" + if version >= MzVersion.parse_mz("v26.40.0-dev"): + config["hydration_history_collection_interval"] = "60s" + if sanitizer_enabled(): config["with_0dt_deployment_max_wait"] = "18000s" @@ -495,6 +498,17 @@ def get_variable_system_parameters( VariableSystemParameter( "arrangement_size_history_retention_period", "7d", ["1min", "1h", "7d"] ), + *( + [ + VariableSystemParameter( + "hydration_history_retention_period", + "30d", + ["1min", "1h", "30d"], + ) + ] + if version >= MzVersion.parse_mz("v26.40.0-dev") + else [] + ), VariableSystemParameter( "persist_validate_part_bounds_on_read", "false", ["true", "false"] ), diff --git a/misc/python/materialize/parallel_workload/action.py b/misc/python/materialize/parallel_workload/action.py index 3e2751d164899..405cce9e78092 100644 --- a/misc/python/materialize/parallel_workload/action.py +++ b/misc/python/materialize/parallel_workload/action.py @@ -3009,6 +3009,16 @@ def __init__( "'1h'", "'7d'", ] + self.flags_with_values["hydration_history_collection_interval"] = [ + "'0s'", + "'1s'", + "'1min'", + ] + self.flags_with_values["hydration_history_retention_period"] = [ + "'1min'", + "'1h'", + "'30d'", + ] # Keep these generous: a tight timeout would abort the oracle's own # queries (they are retried, but it adds noise). "0s" leaves it unset. self.flags_with_values["pg_timestamp_oracle_statement_timeout"] = [ diff --git a/misc/python/materialize/sqlancer.py b/misc/python/materialize/sqlancer.py index de02534aa3d09..655eb1d6a3a95 100644 --- a/misc/python/materialize/sqlancer.py +++ b/misc/python/materialize/sqlancer.py @@ -108,7 +108,7 @@ r"null character not permitted", r"expressions must appear in select list", r"expressions must match initial", - r"invalid selection: operation may only", + r"invalid selection: every relation leaf", r"array size exceeds the maximum allowed", r"does not allow subqueries", r"must use value within", diff --git a/misc/python/materialize/sqlsmith.py b/misc/python/materialize/sqlsmith.py index b8f0ad1398a7d..f8f913b7678cb 100644 --- a/misc/python/materialize/sqlsmith.py +++ b/misc/python/materialize/sqlsmith.py @@ -83,7 +83,7 @@ "result exceeds max size of", # Seems expected with huge queries "expected expression, but found reserved keyword", # Should fix, but only happens rarely with subqueries "Expected right parenthesis, found left parenthesis", # Should fix, but only happens rarely with cast+coalesce - "invalid selection: operation may only (transitively) refer to non-source, non-system tables", # Seems expected when using catalog tables + "invalid selection: every relation leaf must be a writable user table", # Expected when using catalog tables "Unsupported temporal predicate", # Expected, see https://github.com/MaterializeInc/database-issues/issues/5288 "Unsupported temporal operation: NotEq", "Unsupported binary temporal operation: NotEq", diff --git a/misc/python/materialize/version_ancestor_overrides.py b/misc/python/materialize/version_ancestor_overrides.py index a13becfa03843..9ad23d3c797a4 100644 --- a/misc/python/materialize/version_ancestor_overrides.py +++ b/misc/python/materialize/version_ancestor_overrides.py @@ -28,8 +28,16 @@ def get_ancestor_overrides_for_performance_regressions( min_ancestor_mz_version_per_commit = dict() - if scenario_class_name == "MySqlInitialLoadMultiWorkerSampled": - # PR#38094 (storage: Fast approximate snapshot partitioning) increased wallclock by ~30% + if scenario_class_name in ( + "MySqlInitialLoad", + "MySqlInitialLoadMultiWorkerSampled", + "MySqlInitialLoadMultiWorkerSingleTable", + ): + # PR#38094 (storage: Fast approximate snapshot partitioning) increased + # snapshot wallclock: integer primary keys are no longer partitioned, + # so MySqlInitialLoad reads serially, and approximate key probing is + # slower than the exact OFFSET sampler on small datasets (~30% on + # MySqlInitialLoadMultiWorkerSampled). min_ancestor_mz_version_per_commit[ "de8eac0d1683ffd788dfba6edaa88aef7f0f1740" ] = MzVersion.parse_mz("v26.39.0") diff --git a/src/adapter-types/src/dyncfgs.rs b/src/adapter-types/src/dyncfgs.rs index 259972ef21973..1f50dafbd0aad 100644 --- a/src/adapter-types/src/dyncfgs.rs +++ b/src/adapter-types/src/dyncfgs.rs @@ -421,6 +421,22 @@ pub const ARRANGEMENT_SIZE_HISTORY_RETENTION_PERIOD: Config = Config:: ParameterScope::Environment, ); +/// How often to sweep replicas for completed object hydration episodes. +pub const HYDRATION_HISTORY_COLLECTION_INTERVAL: Config = Config::new( + "hydration_history_collection_interval", + Duration::ZERO, + "How often to record completed object hydration episodes. A zero duration disables collection.", + ParameterScope::Environment, +); + +/// How long to retain completed object hydration episodes. +pub const HYDRATION_HISTORY_RETENTION_PERIOD: Config = Config::new( + "hydration_history_retention_period", + Duration::from_hours(30 * 24), + "How long to retain rows in mz_internal.mz_object_hydration_history.", + ParameterScope::Environment, +); + /// How frequently the catalog `*_info` metrics (`mz_object_info`, /// `mz_cluster_info`, …) are reconciled with the catalog. A zero duration /// disables reconciliation. @@ -557,6 +573,8 @@ pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet { .add(&CONSOLE_OIDC_SCOPES) .add(&ARRANGEMENT_SIZE_HISTORY_COLLECTION_INTERVAL) .add(&ARRANGEMENT_SIZE_HISTORY_RETENTION_PERIOD) + .add(&HYDRATION_HISTORY_COLLECTION_INTERVAL) + .add(&HYDRATION_HISTORY_RETENTION_PERIOD) .add(&CATALOG_INFO_METRICS_RECONCILE_INTERVAL) .add(&PG_TIMESTAMP_ORACLE_STATEMENT_TIMEOUT) .add(&FRONTEND_READ_THEN_WRITE) diff --git a/src/adapter/src/active_compute_sink.rs b/src/adapter/src/active_compute_sink.rs index 358c76ee0f3f3..556b87544d8a9 100644 --- a/src/adapter/src/active_compute_sink.rs +++ b/src/adapter/src/active_compute_sink.rs @@ -51,10 +51,10 @@ impl ActiveComputeSink { } /// Reports the ID of the connection which created the sink. - pub fn connection_id(&self) -> &ConnectionId { + pub fn connection_id(&self) -> Option<&ConnectionId> { match &self { - ActiveComputeSink::Subscribe(subscribe) => &subscribe.conn_id, - ActiveComputeSink::CopyTo(copy_to) => ©_to.conn_id, + ActiveComputeSink::Subscribe(subscribe) => subscribe.connection_id(), + ActiveComputeSink::CopyTo(copy_to) => Some(©_to.conn_id), } } @@ -147,13 +147,26 @@ impl SubscribeBacklogAccounting { } } +/// Ownership and cleanup scope of an active subscribe. +#[derive(Debug)] +pub enum ActiveSubscribeOwner { + /// The subscribe belongs to a SQL session. + Session { + conn_id: ConnectionId, + session_uuid: Uuid, + }, + /// The subscribe belongs to a coordinator background task. + /// + /// Always `internal`, since there is no session to attribute a + /// `mz_subscriptions` row to. + Background, +} + /// A description of an active subscribe from coord's perspective #[derive(Debug)] pub struct ActiveSubscribe { - /// The ID of the connection which created the subscribe. - pub conn_id: ConnectionId, - /// The UUID of the session which created the subscribe. - pub session_uuid: Uuid, + /// The owner responsible for retiring the subscribe. + pub owner: ActiveSubscribeOwner, /// The ID of the cluster on which the subscribe is running. pub cluster_id: ClusterId, /// The IDs of the objects on which the subscribe depends. @@ -189,6 +202,25 @@ pub struct ActiveSubscribe { } impl ActiveSubscribe { + /// The session uuid for this subscribe's `mz_subscriptions` row, or `None` + /// if it does not appear there. + pub fn introspection_session_uuid(&self) -> Option { + match &self.owner { + ActiveSubscribeOwner::Session { session_uuid, .. } if !self.internal => { + Some(*session_uuid) + } + _ => None, + } + } + + /// Returns the owning connection, if this is a session subscribe. + pub fn connection_id(&self) -> Option<&ConnectionId> { + match &self.owner { + ActiveSubscribeOwner::Session { conn_id, .. } => Some(conn_id), + ActiveSubscribeOwner::Background => None, + } + } + /// Initializes the subscription. /// /// This method must be called exactly once, after constructing an diff --git a/src/adapter/src/catalog/builtin_table_updates.rs b/src/adapter/src/catalog/builtin_table_updates.rs index ffa9154e3387d..bb10328aabb56 100644 --- a/src/adapter/src/catalog/builtin_table_updates.rs +++ b/src/adapter/src/catalog/builtin_table_updates.rs @@ -51,6 +51,7 @@ use mz_sql::names::SchemaSpecifier; use mz_sql_parser::ast::display::AstDisplay; use mz_storage_client::client::TableData; use smallvec::smallvec; +use uuid::Uuid; // DO NOT add any more imports from `crate` outside of `crate::catalog`. use crate::active_compute_sink::ActiveSubscribe; @@ -811,12 +812,13 @@ impl CatalogState { &self, id: GlobalId, subscribe: &ActiveSubscribe, + session_uuid: Uuid, diff: Diff, ) -> BuiltinTableUpdate<&'static BuiltinTable> { let mut row = Row::default(); let mut packer = row.packer(); packer.push(Datum::String(&id.to_string())); - packer.push(Datum::Uuid(subscribe.session_uuid)); + packer.push(Datum::Uuid(session_uuid)); packer.push(Datum::String(&subscribe.cluster_id.to_string())); let start_dt = mz_ore::now::to_datetime(subscribe.start_time); diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index a35ff177a7326..394a479d54202 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -38,6 +38,7 @@ use mz_build_info::{BuildInfo, DUMMY_BUILD_INFO}; use mz_catalog::builtin::{ BUILTIN_LOOKUP, Builtin, Fingerprint, MZ_CATALOG_RAW, MZ_CATALOG_RAW_DESCRIPTION, MZ_CLUSTER_REPLICA_FRONTIERS_DESCRIPTION, MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY_DESCRIPTION, + MZ_OBJECT_HYDRATION_HISTORY, MZ_OBJECT_HYDRATION_HISTORY_DESCRIPTION, MZ_STORAGE_USAGE_BY_SHARD, MZ_STORAGE_USAGE_BY_SHARD_DESCRIPTION, RUNTIME_ALTERABLE_FINGERPRINT_SENTINEL, }; @@ -662,6 +663,34 @@ struct Migration { config: BuiltinItemMigrationConfig, } +/// Whether `builtin` participates in a forced migration using `mechanism`. +fn participates_in_forced_migration( + builtin: &Builtin, + mechanism: Mechanism, +) -> bool { + use Builtin::*; + match builtin { + // A forced replacement allocates a fresh shard, which discards the + // table's contents. Exclude the tables whose contents are the point: + // storage usage is retained for billing, and hydration history cannot + // be rebuilt from any other source. + // + // Hydration history takes part in a forced `Evolution`, which keeps the + // rows. It has to: dev upgrades force one for every object, and a table + // left out of the plan never gets its new schema registered, so + // `update_fingerprints` panics at open as soon as the desc changes. See + // the tripwire in `validate_migration_steps` for how to give up the + // replacement exemption deliberately. + Table(table) => { + **table != *MZ_STORAGE_USAGE_BY_SHARD + && (mechanism != Mechanism::Replacement || **table != *MZ_OBJECT_HYDRATION_HISTORY) + } + MaterializedView(..) => true, + Source(source) => **source != *MZ_CATALOG_RAW, + Log(..) | View(..) | Type(..) | Func(..) | Index(..) | Connection(..) => false, + } +} + impl Migration { async fn run(self, steps: &[MigrationStep]) -> anyhow::Result { info!( @@ -758,6 +787,23 @@ impl Migration { "mz_object_arrangement_size_history cannot be migrated or else the table will be truncated" ); + // Unlike the two tables above, there is no correctness hazard here, a + // truncation would only lose history. This is a tripwire so that the + // loss is chosen rather than stumbled into: if a schema change to + // this table is worth clearing it for, remove this assert along with + // the exemption in `plan_forced_migration`, and say in the release + // notes that the history restarts. + // + // Only `Replacement` loses the rows. An `Evolution` keeps the shard + // and its contents, so it is the path a schema change to this table + // should take and we let it through. + if step.mechanism == Mechanism::Replacement { + assert_ne!( + &*MZ_OBJECT_HYDRATION_HISTORY_DESCRIPTION, object, + "replacing mz_object_hydration_history clears it, see the comment above" + ); + } + // `mz_catalog_raw` cannot be migrated because it contains the durable catalog and it // wouldn't be very durable if we allowed it to be truncated. assert_ne!( @@ -829,17 +875,7 @@ impl Migration { // added in this version; the leader will allocate their shards during bootstrap, and // there is nothing to evolve or replace. .filter(|(_, info)| info.shard_id.is_some()) - .filter(|(_, info)| { - use Builtin::*; - match info.builtin { - // Filter out the 'mz_storage_usage_by_shard' table since we need to retain - // that info for billing purposes. - Table(table) => **table != *MZ_STORAGE_USAGE_BY_SHARD, - MaterializedView(..) => true, - Source(source) => **source != *MZ_CATALOG_RAW, - Log(..) | View(..) | Type(..) | Func(..) | Index(..) | Connection(..) => false, - } - }) + .filter(|(_, info)| participates_in_forced_migration(info.builtin, mechanism)) .map(|(object, _)| object.clone()) .collect(); diff --git a/src/adapter/src/catalog/open/builtin_schema_migration_tests.rs b/src/adapter/src/catalog/open/builtin_schema_migration_tests.rs index 3356a60dfd0ed..2d168684d5f3f 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration_tests.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration_tests.rs @@ -27,6 +27,20 @@ use tracing::info; use super::*; +#[mz_ore::test] +fn hydration_history_forced_migration_policy() { + let hydration_history = Builtin::Table(&*MZ_OBJECT_HYDRATION_HISTORY); + + assert!(participates_in_forced_migration( + &hydration_history, + Mechanism::Evolution + )); + assert!(!participates_in_forced_migration( + &hydration_history, + Mechanism::Replacement + )); +} + #[test] // allow(test-attribute) #[cfg_attr(miri, ignore)] // too slow fn test_builtin_schema_migration() { diff --git a/src/adapter/src/catalog/state.rs b/src/adapter/src/catalog/state.rs index 72b9276fea3fb..38901e8f754da 100644 --- a/src/adapter/src/catalog/state.rs +++ b/src/adapter/src/catalog/state.rs @@ -3127,7 +3127,9 @@ mod tests { #[mz_ore::test(tokio::test)] #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `TLS_client_method` async fn validate_read_then_write_deep_chain_no_stack_overflow() { - use crate::coord::read_then_write::validate_read_then_write_dependencies; + use crate::coord::read_then_write::{ + DependencyPolicy, validate_read_then_write_dependencies, + }; Catalog::with_debug(|mut catalog| async move { // Deep enough that the previous recursive implementation overflowed @@ -3142,6 +3144,7 @@ mod tests { &catalog, [CatalogItemId::User(BASE)], usize::MAX, + DependencyPolicy::UserDml, ) .expect("deep chain of user views is valid for read-then-write"); @@ -3156,7 +3159,9 @@ mod tests { #[mz_ore::test(tokio::test)] #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `TLS_client_method` async fn validate_read_then_write_dependency_limit() { - use crate::coord::read_then_write::validate_read_then_write_dependencies; + use crate::coord::read_then_write::{ + DependencyPolicy, validate_read_then_write_dependencies, + }; use crate::error::AdapterError; Catalog::with_debug(|mut catalog| async move { @@ -3168,14 +3173,20 @@ mod tests { const OBJECTS: usize = DEPTH + 1; // Exactly at the limit is allowed. - validate_read_then_write_dependencies(&catalog, [CatalogItemId::User(BASE)], OBJECTS) - .expect("chain at the limit is valid"); + validate_read_then_write_dependencies( + &catalog, + [CatalogItemId::User(BASE)], + OBJECTS, + DependencyPolicy::UserDml, + ) + .expect("chain at the limit is valid"); // One below the limit is rejected with a clean error. let err = validate_read_then_write_dependencies( &catalog, [CatalogItemId::User(BASE)], OBJECTS - 1, + DependencyPolicy::UserDml, ) .expect_err("chain over the limit is rejected"); assert!(matches!( diff --git a/src/adapter/src/client.rs b/src/adapter/src/client.rs index 565241b44c0cc..997c73dfff2ab 100644 --- a/src/adapter/src/client.rs +++ b/src/adapter/src/client.rs @@ -63,6 +63,7 @@ use crate::command::{ SASLChallengeResponse, SASLVerifyProofResponse, SuperuserAttribute, }; use crate::config::{ScopedParameters, ScopedParametersScope, SystemParameterFrontend}; +use crate::coord::read_then_write::DependencyPolicy; use crate::coord::{Coordinator, ExecuteContextGuard}; use crate::error::AdapterError; use crate::frontend_read_then_write::{ @@ -72,7 +73,7 @@ use crate::frontend_read_then_write::{ use crate::metrics::Metrics; use crate::optimize::dataflows::{EvalTime, ExprPrepOneShot}; use crate::optimize::{self, Optimize, OptimizerError}; -use crate::peek_client::{ExecutionLogging, TakeOver}; +use crate::peek_client::{CoordinatorClient, ExecutionLogging, TakeOver}; use crate::session::{ EndTransactionAction, PreparedStatement, Session, SessionConfig, StateRevision, TransactionId, TransactionStatus, @@ -313,7 +314,7 @@ impl Client { } = response; let peek_client = PeekClient::new( - self.clone(), + CoordinatorClient::Session(self.clone()), &catalog, storage_collections, transient_id_gen, @@ -1542,7 +1543,7 @@ impl SessionClient { }); tokio::pin!(register); tokio::select! { - rx = &mut register => rx, + rx = &mut register => rx?, _ = &mut cancel_future => { inner_client.try_send(Command::PrivilegedCancelRequest { conn_id: conn_id.clone(), @@ -1973,7 +1974,7 @@ impl SessionClient { "calls to mz_now in write statements", )); } - validate_selection_dependencies(&catalog, &depends_on)?; + validate_selection_dependencies(&catalog, &depends_on, DependencyPolicy::UserDml)?; return Err(prohibited_in_transaction(&stmt)); } } diff --git a/src/adapter/src/command.rs b/src/adapter/src/command.rs index d343becaf3010..cd260d7f379ef 100644 --- a/src/adapter/src/command.rs +++ b/src/adapter/src/command.rs @@ -45,6 +45,7 @@ use mz_timestamp_oracle::TimestampOracle; use tokio::sync::{Semaphore, mpsc, oneshot, watch}; use uuid::Uuid; +use crate::active_compute_sink::ActiveSubscribeOwner; use crate::catalog::Catalog; use crate::config::{ScopedParameters, ScopedParametersScope, SystemParameterFrontend}; use crate::coord::appends::{BuiltinTableAppendNotify, WriteResult}; @@ -419,8 +420,7 @@ pub enum Command { as_of: mz_repr::Timestamp, arity: usize, sink_id: GlobalId, - conn_id: ConnectionId, - session_uuid: Uuid, + owner: ActiveSubscribeOwner, start_time: mz_ore::now::EpochMillis, read_holds: ReadHolds, tx: oneshot::Sender, AdapterError>>, @@ -437,14 +437,10 @@ pub enum Command { /// including read-only, a changed target and cancellation, is reported the /// same way in both modes. AttemptWrite { - /// Connection originating the write. Used so the coordinator can - /// cancel this pending write if the connection is cancelled before - /// the write commits. - conn_id: ConnectionId, + attempt: WriteAttemptKind, target_id: CatalogItemId, target_global_id: GlobalId, diffs: Vec<(Row, Diff)>, - write_ts: Option, tx: oneshot::Sender, }, @@ -455,6 +451,24 @@ pub enum Command { }, } +/// Who a read-then-write commits on behalf of, and how its timestamp is chosen. +/// +/// Group commit picking the timestamp requires a connection to answer through, +/// so that combination is only reachable from a session. +#[derive(Debug)] +pub enum WriteAttemptKind { + /// A session's write, cancelled with `conn_id` if the connection goes away + /// before it commits. A `write_ts` of `None` lets group commit pick the + /// timestamp, which then cannot be reported as passed. + Session { + conn_id: ConnectionId, + write_ts: Option, + }, + /// Coordinator background work. There is no connection to cancel with, so + /// the caller names the timestamp and handles `TimestampPassed` itself. + Background { write_ts: mz_repr::Timestamp }, +} + impl Command { pub fn session(&self) -> Option<&Session> { match self { diff --git a/src/adapter/src/coord.rs b/src/adapter/src/coord.rs index 0018adb4471d7..721ba8975bcd4 100644 --- a/src/adapter/src/coord.rs +++ b/src/adapter/src/coord.rs @@ -99,7 +99,8 @@ use mz_adapter_types::dyncfgs::{ use mz_auth::password::Password; use mz_build_info::BuildInfo; use mz_catalog::builtin::{ - BUILTINS, BUILTINS_STATIC, MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY, MZ_STORAGE_USAGE_BY_SHARD, + BUILTINS, BUILTINS_STATIC, MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY, MZ_OBJECT_HYDRATION_HISTORY, + MZ_STORAGE_USAGE_BY_SHARD, }; use mz_catalog::config::{AwsPrincipalContext, BuiltinItemMigrationConfig, ClusterReplicaSizeMap}; use mz_catalog::durable::OpenableDurableCatalogState; @@ -130,7 +131,7 @@ use mz_ore::channel::trigger::Trigger; use mz_ore::future::TimeoutError; use mz_ore::metrics::MetricsRegistry; use mz_ore::now::{EpochMillis, NowFn}; -use mz_ore::task::{JoinHandle, spawn}; +use mz_ore::task::{AbortOnDropHandle, JoinHandle, spawn}; use mz_ore::thread::JoinHandleExt; use mz_ore::tracing::{OpenTelemetryContext, TracingHandle}; use mz_ore::url::SensitiveUrl; @@ -150,7 +151,7 @@ use mz_secrets::cache::CachingSecretsReader; use mz_secrets::{SecretsController, SecretsReader}; use mz_sql::ast::{Raw, Statement}; use mz_sql::catalog::{CatalogCluster, EnvironmentId}; -use mz_sql::names::{QualifiedItemName, ResolvedIds, SchemaSpecifier}; +use mz_sql::names::{QualifiedItemName, ResolvedIds}; use mz_sql::optimizer_metrics::OptimizerMetrics; use mz_sql::plan::{ self, AlterSinkPlan, ConnectionDetails, CreateConnectionPlan, HirRelationExpr, @@ -237,6 +238,7 @@ mod caught_up; mod command_handler; mod ddl; pub(crate) mod group_sync; +mod hydration_history; mod indexes; mod info_metrics; mod introspection; @@ -398,6 +400,8 @@ pub enum Message { ArrangementSizesSnapshot, ArrangementSizesWrite(Vec), ArrangementSizesPrune(Vec), + HydrationHistorySchedule, + HydrationHistoryRun, /// Performs any cleanup and logging actions necessary for /// finalizing a statement execution. RetireExecute { @@ -553,6 +557,8 @@ impl Message { Message::ArrangementSizesSnapshot => "arrangement_sizes_snapshot", Message::ArrangementSizesWrite(_) => "arrangement_sizes_write", Message::ArrangementSizesPrune(_) => "arrangement_sizes_prune", + Message::HydrationHistorySchedule => "hydration_history_schedule", + Message::HydrationHistoryRun => "hydration_history_run", Message::RetireExecute { .. } => "retire_execute", Message::ExecuteSingleStatementTransaction { .. } => { "execute_single_statement_transaction" @@ -2107,6 +2113,10 @@ pub struct Coordinator { connection_cancel_watches: BTreeMap, watch::Receiver)>, /// Active introspection subscribes. introspection_subscribes: BTreeMap, + /// The last replica visited by the sequential hydration-history sweep. + hydration_history_replica_cursor: Option, + /// Hydration-history sweep owned by the coordinator while one is in flight. + hydration_history_sweep: Option>, /// Locks that grant access to a specific object, populated lazily as objects are written to. write_locks: BTreeMap>>, @@ -3169,29 +3179,19 @@ impl Coordinator { debug!("coordinator init: resetting system tables"); let read_ts = self.get_local_read_ts().await; - // Filter out tables whose contents must survive restarts: - // 'mz_storage_usage_by_shard' for billing, and - // 'mz_object_arrangement_size_history', which accumulates history that - // is pruned by its own retention period instead. - let mz_storage_usage_by_shard_schema: SchemaSpecifier = self - .catalog() - .resolve_system_schema(MZ_STORAGE_USAGE_BY_SHARD.schema) - .into(); - let arrangement_size_history_schema: SchemaSpecifier = self - .catalog() - .resolve_system_schema(MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY.schema) - .into(); - let is_retained_across_restarts = |meta: &TableMetadata| -> bool { - (meta.name.item == MZ_STORAGE_USAGE_BY_SHARD.name - && meta.name.qualifiers.schema_spec == mz_storage_usage_by_shard_schema) - || (meta.name.item == MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY.name - && meta.name.qualifiers.schema_spec == arrangement_size_history_schema) - }; + let retained_across_restarts = BTreeSet::from([ + self.catalog() + .resolve_builtin_table(&MZ_STORAGE_USAGE_BY_SHARD), + self.catalog() + .resolve_builtin_table(&MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY), + self.catalog() + .resolve_builtin_table(&MZ_OBJECT_HYDRATION_HISTORY), + ]); let mut retraction_tasks = Vec::new(); let system_tables: Vec<_> = table_metas .iter() - .filter(|meta| meta.id.is_system() && !is_retained_across_restarts(meta)) + .filter(|meta| meta.id.is_system() && !retained_across_restarts.contains(&meta.id)) .collect(); for system_table in system_tables { @@ -4105,6 +4105,7 @@ impl Coordinator { self.schedule_storage_usage_collection().await; self.schedule_arrangement_sizes_collection().await; + self.schedule_hydration_history_collection(); self.spawn_privatelink_vpc_endpoints_watch_task(); self.spawn_statement_logging_task(); self.spawn_catalog_info_metrics_task(); @@ -5338,6 +5339,8 @@ pub fn serve( active_copies: BTreeMap::new(), connection_cancel_watches: BTreeMap::new(), introspection_subscribes: BTreeMap::new(), + hydration_history_replica_cursor: None, + hydration_history_sweep: None, write_locks: BTreeMap::new(), deferred_write_ops: BTreeMap::new(), pending_writes: Vec::new(), diff --git a/src/adapter/src/coord/appends.rs b/src/adapter/src/coord/appends.rs index 051f2e0af4ba3..ddddf67892c49 100644 --- a/src/adapter/src/coord/appends.rs +++ b/src/adapter/src/coord/appends.rs @@ -194,10 +194,9 @@ pub enum WriteResult { /// Delivers an internal write result, including on task shutdown. /// /// The `Drop` impl is load-bearing. The session task waiting on the other end -/// `expect`s a reply, so a `oneshot::Sender` that is dropped silently would -/// panic that session on coordinator shutdown or on a dead group-committer -/// task. Reporting [`WriteResult::Indeterminate`] instead lets the session -/// report an error. +/// needs a definitive response so that it can release its OCC permit and +/// subscribe. Reporting [`WriteResult::Indeterminate`] lets it unwind when the +/// coordinator or group committer shuts down. #[derive(Debug)] pub struct InternalWriteResponder { tx: Option>, @@ -1719,6 +1718,13 @@ mod tests { use super::*; use crate::catalog::Catalog; + #[mz_ore::test(tokio::test)] + async fn internal_write_responder_reports_indeterminate_on_drop() { + let (tx, rx) = oneshot::channel(); + drop(InternalWriteResponder::new(tx)); + assert!(matches!(rx.await, Ok(WriteResult::Indeterminate))); + } + #[derive(Debug, Default)] struct MemTimestampOracle { read_write_ts: Mutex<(Timestamp, Timestamp)>, diff --git a/src/adapter/src/coord/command_handler.rs b/src/adapter/src/coord/command_handler.rs index 4b334b3a8f31f..0d29050f5c447 100644 --- a/src/adapter/src/coord/command_handler.rs +++ b/src/adapter/src/coord/command_handler.rs @@ -638,44 +638,25 @@ impl Coordinator { as_of, arity, sink_id, - conn_id, - session_uuid, + owner, start_time, read_holds, tx, } => { self.handle_create_internal_subscribe( - *df_desc, - cluster_id, - replica_id, - depends_on, - as_of, - arity, - sink_id, - conn_id, - session_uuid, - start_time, - read_holds, - tx, + *df_desc, cluster_id, replica_id, depends_on, as_of, arity, sink_id, owner, + start_time, read_holds, tx, ) .await; } Command::AttemptWrite { - conn_id, + attempt, target_id, target_global_id, diffs, - write_ts, tx, } => { - self.handle_attempt_write( - conn_id, - target_id, - target_global_id, - diffs, - write_ts, - tx, - ); + self.handle_attempt_write(attempt, target_id, target_global_id, diffs, tx); } Command::DropInternalSubscribe { sink_id } => { self.drop_internal_subscribe(sink_id).await; diff --git a/src/adapter/src/coord/hydration_history.rs b/src/adapter/src/coord/hydration_history.rs new file mode 100644 index 0000000000000..b7362dff9be09 --- /dev/null +++ b/src/adapter/src/coord/hydration_history.rs @@ -0,0 +1,720 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//! Durable history collection for completed compute-object hydration episodes. +//! +//! One sweep visits a single user replica, installs a replica-targeted +//! subscribe that diffs that replica's live hydration timestamps against the +//! durable history table, and appends what is missing through the timestamped +//! OCC write path. Including the history table in the read expression is what +//! makes the write idempotent across concurrent `environmentd` processes: two +//! collectors that compute the same row race for one write timestamp, and the +//! loser observes the winner's append through its own subscribe and finds +//! nothing left to write. +//! +//! One replica is sampled per interval, so an environment with `N` eligible +//! replicas revisits each one approximately every `N * interval`. Lowering the +//! interval improves freshness at the cost of more replica dataflow installs. +//! +//! Collection is sampling, not an event log. An episode whose live row is +//! retracted before its replica's turn in the sweep (a dropped object, or a +//! replica that restarts first) is not recorded, and cannot be, because +//! the only evidence is gone. See the design doc for why that is accepted here. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use itertools::Itertools; +use mz_adapter_types::dyncfgs::{ + FRONTEND_READ_THEN_WRITE, HYDRATION_HISTORY_COLLECTION_INTERVAL, + HYDRATION_HISTORY_RETENTION_PERIOD, +}; +use mz_catalog::builtin::{MZ_CATALOG_SERVER_CLUSTER, MZ_OBJECT_HYDRATION_HISTORY}; +use mz_cluster_client::ReplicaId; +use mz_controller::clusters::{ClusterStatus, ReplicaLocation}; +use mz_controller_types::ClusterId; +use mz_ore::cast::CastFrom; +use mz_ore::collections::CollectionExt; +use mz_ore::now::EpochMillis; +use mz_ore::task; +use mz_repr::CatalogItemId; +use mz_sql::plan::{MutationKind, Params, Plan, ReadThenWritePlan}; +use sha2::{Digest, Sha256}; +use tracing::warn; + +use crate::catalog::Catalog; +use crate::command::ExecuteResponse; +use crate::coord::{Coordinator, Message}; +use crate::metrics::Metrics; +use crate::peek_client::CoordinatorClient; +use crate::session::Session; +use crate::{AdapterError, PeekClient}; + +/// Longest a scheduler sleep may run before it rechecks the configuration. +/// +/// Sleeping the whole interval would leave a dyncfg change ineffective until the +/// old interval elapsed, so lowering the interval at runtime (which tests do) +/// would not take effect for up to the previous interval. +const SCHEDULE_RECHECK_CAP: Duration = Duration::from_secs(5); + +/// How often a disabled collector rechecks whether it was enabled. +/// +/// This is the cadence of every environment in the default configuration, so it +/// is much coarser than the enabled one: nothing is waiting on it. +const DISABLED_RECHECK_INTERVAL: Duration = Duration::from_secs(60); + +/// Bound on one replica-targeted mutation. +/// +/// Subscribe installation, replica-side progress, OCC conflict retries, and the +/// external commit can all wait indefinitely. Exceeding the bound skips the +/// step. The next sweep recomputes from current state. +const MUTATION_TIMEOUT: Duration = Duration::from_secs(300); + +/// Rows retracted per retention step. +/// +/// Retention has to be bounded: the OCC path refuses a selection larger than +/// `max_result_size` before submitting any write, so an unbounded delete over a +/// large backlog would fail identically on every sweep and never shrink the +/// table. Retention repeats bounded batches across sweeps until it drains the +/// fixed-cutoff backlog. +const RETENTION_BATCH_SIZE: usize = 1000; + +/// Milliseconds until the next fire on this environment's own grid. +/// +/// The grid has period `interval_ms` and is shifted by `offset`. When this +/// environment's point in the current period has already passed, the next one is a +/// full period later. +fn next_fire_delay(now: EpochMillis, interval_ms: EpochMillis, offset: EpochMillis) -> Duration { + debug_assert!(interval_ms > 0); + let this_period = (now - (now % interval_ms)).saturating_add(offset); + let next = if this_period > now { + this_period + } else { + this_period.saturating_add(interval_ms) + }; + Duration::from_millis(next.saturating_sub(now)) +} + +/// Stable offset within `interval_ms` for one environment id. +fn environment_schedule_offset(environment_id: &str, interval_ms: EpochMillis) -> EpochMillis { + debug_assert!(interval_ms > 0); + let digest = Sha256::digest(environment_id); + let hash = u64::from_le_bytes(digest[..8].try_into().expect("SHA-256 digest has 32 bytes")); + hash % interval_ms +} + +impl Coordinator { + /// Schedules the next hydration history sweep. + /// + /// Fires are aligned to interval boundaries so that they stay evenly spaced + /// across restarts, offset per environment so that a fleet-wide interval does + /// not make every environment sweep at the same instant, and each sleep is + /// capped so a configuration change is picked up promptly. Sweeps never + /// overlap: the next one is only scheduled once the previous one has finished + /// or failed. + /// + /// NOTE: Alignment reads the wall clock, so a test that freezes `NowFn` and + /// configures an interval longer than the recheck cap never reaches a + /// boundary and never fires. + pub(super) fn schedule_hydration_history_collection(&self) { + let interval = + HYDRATION_HISTORY_COLLECTION_INTERVAL.get(self.catalog().system_config().dyncfgs()); + + // A zero interval disables collection. Keep polling so that enabling it + // takes effect without an `environmentd` restart. + let (delay, fire) = if interval.is_zero() { + (DISABLED_RECHECK_INTERVAL, false) + } else { + // An absurd interval saturates rather than panicking. The setting is + // durable, so a panic here would recur on every restart. + let interval_ms = EpochMillis::try_from(interval.as_millis()) + .unwrap_or(EpochMillis::MAX) + .max(1); + let remaining = next_fire_delay( + self.now(), + interval_ms, + self.hydration_history_schedule_offset(interval_ms), + ); + if remaining <= SCHEDULE_RECHECK_CAP { + (remaining, true) + } else { + (SCHEDULE_RECHECK_CAP, false) + } + }; + + let internal_cmd_tx = self.internal_cmd_tx.clone(); + task::spawn(|| "hydration_history_schedule", async move { + tokio::time::sleep(delay).await; + let message = if fire { + Message::HydrationHistoryRun + } else { + Message::HydrationHistorySchedule + }; + // Best effort: the coordinator may be shutting down. + let _ = internal_cmd_tx.send(message); + }); + } + + /// A stable offset into the collection interval for this environment. + /// + /// Seeded from the full environment id, so it survives restarts but differs + /// between environments in the same organization. Without it every + /// environment would sweep on the same absolute grid, turning each boundary + /// into a fleet-wide burst of dataflow installs, oracle round trips and persist + /// writes. + fn hydration_history_schedule_offset(&self, interval_ms: EpochMillis) -> EpochMillis { + let environment_id = self.catalog().state().config().environment_id.to_string(); + environment_schedule_offset(&environment_id, interval_ms) + } + + /// Runs one sweep: collect from the next replica, then apply retention. + pub(super) fn run_hydration_history_collection(&mut self) { + let (collection_interval, retention) = { + let dyncfgs = self.catalog().system_config().dyncfgs(); + ( + HYDRATION_HISTORY_COLLECTION_INTERVAL.get(dyncfgs), + HYDRATION_HISTORY_RETENTION_PERIOD.get(dyncfgs), + ) + }; + // Builtin tables are not writable in read-only mode, and a disabled + // collector must do no background work at all. Retention is part of the + // sweep, so disabling collection also suspends it. That is deliberate: + // the table can only be non-empty if collection ran at some point, and + // the alternative is an always-on subscribe in the default (disabled) + // production configuration. + if collection_interval.is_zero() || self.controller.read_only() { + self.schedule_hydration_history_collection(); + return; + } + + let replicas = self + .catalog() + .user_cluster_replicas() + .filter(|replica| replica.config.compute.logging.enabled()) + .filter(|replica| match &replica.config.location { + ReplicaLocation::Managed(_) => { + self.cluster_replica_statuses + .get_cluster_replica_status(replica.cluster_id, replica.replica_id) + == ClusterStatus::Online + } + // Unmanaged replicas have no orchestrator status and are only + // used by tests. Their bounded mutation determines readiness. + ReplicaLocation::Unmanaged(_) => true, + }) + .map(|replica| (replica.cluster_id, replica.replica_id)) + .sorted_by_key(|(_, replica_id)| *replica_id) + .collect_vec(); + + let catalog = self.owned_catalog(); + // Retention runs on the catalog server, so that it keeps working when + // there are no user replicas to collect from at all. Without a replica + // there it is skipped, while collection still runs. + let catalog_server = catalog.resolve_builtin_cluster(&MZ_CATALOG_SERVER_CLUSTER); + let catalog_server_target = catalog_server + .replicas() + .next() + .map(|replica| (catalog_server.id, replica.replica_id)); + + let replica = next_replica(&replicas, self.hydration_history_replica_cursor); + if let Some((_, replica_id)) = replica { + self.hydration_history_replica_cursor = Some(replica_id); + } + let mut sweep = self.new_sweep(catalog, retention); + let internal_cmd_tx = self.internal_cmd_tx.clone(); + + let handle = task::spawn(|| "hydration_history_sweep", async move { + let started = Instant::now(); + if let Some((cluster_id, replica_id)) = replica { + sweep.collect(cluster_id, replica_id).await; + } + + // Retention runs even when collection failed above. A replica that + // is crash-looping or slow must not be able to stop the table from + // shrinking back to its retention bound. + if let Some((cluster_id, replica_id)) = catalog_server_target { + sweep.retain(cluster_id, replica_id).await; + } + + sweep + .metrics + .hydration_history_sweep_duration_seconds + .observe(started.elapsed().as_secs_f64()); + let _ = internal_cmd_tx.send(Message::HydrationHistorySchedule); + }); + + // Keep the sweep coordinator-owned so dropping the coordinator requests + // its abort. Fallible coordinator calls make concurrent shutdown safe. + self.hydration_history_sweep = Some(handle.abort_on_drop()); + } + + /// Assembles the sweep context, including the client it writes through. + fn new_sweep(&self, catalog: Arc, retention: Duration) -> Sweep { + let retention_ms = u64::try_from(retention.as_millis()).unwrap_or(u64::MAX); + let build_version = catalog.state().config().build_info.human_version(None); + // Background read-then-write always uses the frontend OCC path. This + // shared constructor field only controls session fallback, so the flag + // does not gate history collection. + let client = PeekClient::new( + CoordinatorClient::Background { + tx: self.internal_cmd_tx.clone(), + metrics: self.metrics.clone(), + }, + &catalog, + Arc::clone(&self.controller.storage_collections), + Arc::clone(&self.transient_id_gen), + self.optimizer_metrics.clone(), + self.persist_client.clone(), + self.statement_logging.create_frontend(build_version), + Arc::clone(&self.occ_write_semaphore), + FRONTEND_READ_THEN_WRITE.get(self.catalog().system_config().dyncfgs()), + self.group_commit_tx.clone(), + self.controller.read_only(), + ); + Sweep { + client, + history_id: catalog.resolve_builtin_table(&MZ_OBJECT_HYDRATION_HISTORY), + catalog, + metrics: self.metrics.clone(), + wall_time: self.now_datetime(), + cutoff: mz_ore::now::to_datetime(self.now().saturating_sub(retention_ms)).to_rfc3339(), + } + } +} + +/// Picks the replica after `cursor`, wrapping around at the end. +/// +/// `replicas` must be sorted ascending by replica id. Unsorted input still +/// returns a replica but degenerates the rotation, revisiting some replicas and +/// starving others. +fn next_replica( + replicas: &[(ClusterId, ReplicaId)], + cursor: Option, +) -> Option<(ClusterId, ReplicaId)> { + replicas + .iter() + .find(|(_, replica_id)| cursor.is_none_or(|cursor| *replica_id > cursor)) + .or_else(|| replicas.first()) + .copied() +} + +/// The rows this replica has completed that the history table is missing. +/// +/// Aggregates every worker's row for an export, and records nothing until all of +/// them have hydrated. One worker is not enough, because a materialized view's +/// persist sink has a single active worker, `hash(sink_id) % workers`. Only that +/// worker's reported output frontier is gated on the shard upper, so only its +/// `hydrated_at` covers the initial snapshot write. Every other worker clears its +/// sink write frontier and stamps at compute completion, which for a materialized +/// view is before the data is durable. Taking `max` over a complete set of workers +/// is therefore the only way to get a finish that means the same thing for every +/// object, and it is the rule `mz_compute_hydration_times` already applies. +/// +/// Completeness needs no configured worker count. The log carries a row per +/// `(export_id, worker_id)` from installation with a null `hydrated_at`, so +/// `count(*) = count(hydrated_at)` says every row visible at the OCC read +/// timestamp has finished. Per-process logging clocks also determine Differential +/// update timestamps, so a worker whose clock is ahead can be absent at that +/// timestamp. A visible unfinished object is skipped and picked up by a later +/// sweep. +/// +/// A worker missing at the read timestamp cannot later change the episode key. +/// Its logging clock stamps both the Differential update and `installed_at`, so +/// late visibility means its installation stamp is later than the visible +/// minimum. The anti-join therefore keeps matching the recorded row. A later +/// `hydrated_at` can raise the aggregate's maximum, but history is not repaired +/// after the episode key has been recorded. +/// +/// The collector deliberately accepts this sampling race rather than depending on +/// `ReplicaLocation::workers()`. A durable finish can therefore precede the latest +/// worker's finish. A whole-replica restart resets the collection as a unit. +/// +/// The interval spans workers, so it carries whatever skew there is between the +/// process clocks that stamped its ends. Each process anchors its logging clock at +/// its own `SystemTime`. That inflates a duration, and nothing here rejects a row +/// for being inconsistent, which is deliberate: an ordering guard on cross-worker +/// stamps rejects complete episodes permanently, since the log values never change. +/// +/// Collection has no explicit batch bound. It returns at most one row per +/// not-yet-recorded dataflow, and the OCC path rejects a result that exceeds +/// `max_result_size` or `max_query_result_size`. At their 1 GiB defaults that +/// ceiling only matters at millions of dataflows per replica. +fn collect_sql(cluster_id: ClusterId, replica_id: ReplicaId, cutoff: &str) -> String { + // Interpolating into SQL is safe here: the ids are catalog-internal and the + // cutoff is an RFC 3339 timestamp we formatted ourselves. Nothing in this + // query comes from a user. + // + // NOTE: The cutoff and the anti-join sit outside the aggregate deliberately. As + // a `WHERE` clause either one drops not-yet-hydrated rows, which would make + // `count(*) = count(hydrated_at)` trivially true and hand back a compute-only + // finish for a materialized view whose active worker is still writing. + // + // NOTE: `hydrated_at` is the terminal stamp for a history episode. Nothing + // waits for the history row before proceeding. If the log gains a separate + // `written_at` stamp, only `hydrated_at` belongs in this completeness check. + // A materialized view being replaced can hydrate while it runs read-only, and + // may never write if the replacement is rolled back. + format!( + "SELECT + e.object_id, + '{cluster_id}'::text AS cluster_id, + '{replica_id}'::text AS replica_id, + e.installed_at, + e.started_at, + e.hydrated_at, + 'hydrated'::text AS status + FROM ( + SELECT + t.export_id AS object_id, + min(t.installed_at) AS installed_at, + min(t.started_at) AS started_at, + max(t.hydrated_at) AS hydrated_at + FROM mz_introspection.mz_compute_hydration_times_per_worker AS t + JOIN mz_internal.mz_object_global_ids AS ids ON ids.global_id = t.export_id + JOIN mz_catalog.mz_objects AS o ON o.id = ids.id + WHERE t.export_id LIKE 'u%' + AND o.type IN ('index', 'materialized-view') + GROUP BY t.export_id + HAVING count(*) = count(t.hydrated_at) + ) AS e + WHERE e.hydrated_at >= TIMESTAMPTZ '{cutoff}' + AND NOT EXISTS ( + SELECT 1 + FROM mz_internal.mz_object_hydration_history AS h + WHERE h.object_id = e.object_id + AND h.replica_id = '{replica_id}'::text + AND h.installed_at = e.installed_at + )" + ) +} + +/// A bounded batch of history rows that have aged out. +/// +/// Only rows with a `hydrated_at` age out. Every row written today has one, and +/// a row without one would be immortal here, so an unfinished-episode +/// representation needs a second age basis before it can be recorded. +fn retention_sql(cutoff: &str) -> String { + // The LIMIT has to sit inside a subquery. A top-level LIMIT lands in the + // plan's `RowSetFinishing`, which this OCC stage cannot apply. Inside a + // derived table it lowers into the relation expression instead. + format!( + "SELECT * FROM ( + SELECT + object_id, cluster_id, replica_id, installed_at, started_at, + hydrated_at, status + FROM mz_internal.mz_object_hydration_history + WHERE hydrated_at < TIMESTAMPTZ '{cutoff}' + ORDER BY hydrated_at + LIMIT {RETENTION_BATCH_SIZE} + )" + ) +} + +/// What one sweep needs to run its mutations against the history table. +struct Sweep { + client: PeekClient, + catalog: Arc, + history_id: CatalogItemId, + metrics: Metrics, + wall_time: chrono::DateTime, + /// Rows finishing before this have aged out. Both steps apply it, so a live + /// log row cannot resurrect an episode retention just retracted. + cutoff: String, +} + +impl Sweep { + /// Appends this replica's completed episodes that the table is missing. + async fn collect(&mut self, cluster_id: ClusterId, replica_id: ReplicaId) { + let sql = collect_sql(cluster_id, replica_id, &self.cutoff); + let _ = self + .run( + "collection", + cluster_id, + replica_id, + MutationKind::Insert, + &sql, + ) + .await; + } + + /// Retracts one bounded batch of aged-out rows. + async fn retain(&mut self, cluster_id: ClusterId, replica_id: ReplicaId) { + let sql = retention_sql(&self.cutoff); + let Some(deleted) = self + .run( + "retention", + cluster_id, + replica_id, + MutationKind::Delete, + &sql, + ) + .await + else { + return; + }; + if deleted == RETENTION_BATCH_SIZE { + self.metrics.hydration_history_retention_batch_full.inc(); + } + } + + /// Runs one mutation, leaving transient failures for the next sweep to retry. + /// + /// Replica loss, dependency replacement, and write races are logged rather + /// than propagated because the next sweep recomputes from current state. + /// + /// NOTE: A timed-out mutation can still commit. The write is submitted + /// before we wait for its answer, and a background write carries no + /// connection to cancel it with, so a `timeout` outcome says we stopped + /// waiting, not that nothing landed. Rows such a write commits afterwards + /// are never counted, which makes `rows_affected` a lower bound. + async fn run( + &mut self, + step: &'static str, + cluster_id: ClusterId, + replica_id: ReplicaId, + kind: MutationKind, + sql: &str, + ) -> Option { + let mutation = async { + let plan = plan_mutation(&self.catalog, self.history_id, kind, sql)?; + let mut session = Session::dummy(); + session.start_transaction_single_stmt(self.wall_time); + let response = self + .client + .background_read_then_write( + &mut session, + plan, + cluster_id, + replica_id, + &self.catalog, + ) + .await?; + Ok::<_, AdapterError>(response) + }; + match tokio::time::timeout(MUTATION_TIMEOUT, mutation).await { + Ok(Ok(response)) => { + let (rows, action) = match (kind, response) { + (MutationKind::Insert, ExecuteResponse::Inserted(rows)) => (rows, "appended"), + (MutationKind::Update, ExecuteResponse::Updated(rows)) => (rows, "updated"), + (MutationKind::Delete, ExecuteResponse::Deleted(rows)) => (rows, "deleted"), + (_, response) => { + self.observe_mutation(step, "error"); + mz_ore::soft_panic_or_log!( + "hydration history {step} returned an unexpected response: {response:?}" + ); + return None; + } + }; + self.metrics + .hydration_history_rows_affected + .with_label_values(&[action]) + .inc_by(u64::cast_from(rows)); + let outcome = if rows == 0 { "noop" } else { "success" }; + self.observe_mutation(step, outcome); + Some(rows) + } + Ok(Err(error)) => { + self.observe_mutation(step, "error"); + if step == "collection" && matches!(&error, AdapterError::ReadThenWriteContention) { + warn!( + %step, %cluster_id, %replica_id, %error, + "hydration history step failed, the replica's introspection frontier \ + may be trailing the write frontier" + ); + } else { + warn!(%step, %cluster_id, %replica_id, %error, "hydration history step failed"); + } + None + } + // A trailing replica can repeatedly certify a target only after the + // oracle has advanced past it. Each refused write raises the target, + // and the conflict loop can continue until this timeout fires. + Err(_) if step == "collection" => { + self.observe_mutation(step, "timeout"); + warn!( + %step, %cluster_id, %replica_id, + "hydration history step timed out, \ + the replica's introspection frontier may be trailing the write frontier" + ); + None + } + Err(_) => { + self.observe_mutation(step, "timeout"); + warn!(%step, %cluster_id, %replica_id, "hydration history step timed out"); + None + } + } + } + + fn observe_mutation(&self, operation: &str, outcome: &str) { + self.metrics + .hydration_history_mutations + .with_label_values(&[operation, outcome]) + .inc(); + } +} + +/// Plans `sql` as the read side of a mutation against `target_id`. +/// +/// The statement is planned as a `SELECT` whose columns are already in the +/// target table's order, so the mutation needs no assignments or projection. +/// +/// The selection's column types are checked against the target table here. A +/// user `INSERT ... SELECT` gets that from the planner, but a hand-built plan +/// bypasses it, and a wrong type would be written into the shard verbatim and +/// break every later read of a table that is deliberately never truncated. +fn plan_mutation( + catalog: &Arc, + target_id: CatalogItemId, + kind: MutationKind, + sql: &str, +) -> Result { + let session_catalog = catalog.for_system_session(); + let parsed = mz_sql::parse::parse(sql) + .map_err(AdapterError::from)? + .into_element(); + let (stmt, resolved_ids) = mz_sql::names::resolve(&session_catalog, parsed.ast)?; + let (plan, _) = mz_sql::plan::plan( + None, + &session_catalog, + stmt, + &Params::empty(), + &resolved_ids, + )?; + let Plan::Select(select) = plan else { + return Err(AdapterError::Internal( + "hydration history query did not plan as SELECT".into(), + )); + }; + + let target_desc = catalog + .get_entry(&target_id) + .relation_desc_latest() + .expect("hydration history target is a table"); + let selection_types = select.source.typ(&[], &BTreeMap::new()).column_types; + let target_types = &target_desc.typ().column_types; + let matches = selection_types.len() == target_types.len() + && selection_types + .iter() + .zip_eq(target_types) + // Nullability may be tighter than the column allows, only the + // scalar types have to agree. + .all(|(selected, target)| selected.scalar_type == target.scalar_type); + if !matches { + return Err(AdapterError::Internal(format!( + "hydration history query does not match the target table: \ + selection {selection_types:?}, table {target_types:?}" + ))); + } + + Ok(ReadThenWritePlan { + id: target_id, + selection: select.source, + finishing: select.finishing, + assignments: BTreeMap::new(), + kind, + returning: Vec::new(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[mz_ore::test] + fn replica_sweep_advances_and_wraps() { + let cluster = ClusterId::user(1).expect("valid cluster ID"); + let replicas = [(cluster, ReplicaId::User(1)), (cluster, ReplicaId::User(3))]; + + assert_eq!(next_replica(&replicas, None), Some(replicas[0])); + assert_eq!( + next_replica(&replicas, Some(ReplicaId::User(1))), + Some(replicas[1]) + ); + assert_eq!( + next_replica(&replicas, Some(ReplicaId::User(3))), + Some(replicas[0]) + ); + assert_eq!(next_replica(&[], None), None); + } + + /// Every environment shares one interval, so the grid has to be shifted per + /// environment or the whole fleet sweeps at the same instant. + #[mz_ore::test] + fn fire_delay_is_offset_within_the_interval() { + let interval = 60_000; + + // Before this environment's point in the period, we wait for it. + assert_eq!( + next_fire_delay(1_000, interval, 5_000), + Duration::from_millis(4_000) + ); + // On it, we take the next period rather than firing twice. + assert_eq!( + next_fire_delay(5_000, interval, 5_000), + Duration::from_millis(interval) + ); + // After it, the next period's point. + assert_eq!( + next_fire_delay(6_000, interval, 5_000), + Duration::from_millis(59_000) + ); + // A zero offset is plain alignment, and never returns a zero delay. + assert_eq!( + next_fire_delay(59_999, interval, 0), + Duration::from_millis(1) + ); + assert_eq!( + next_fire_delay(60_000, interval, 0), + Duration::from_millis(interval) + ); + + // Region and ordinal are part of the seed, not just the organization. + let one = environment_schedule_offset( + "aws-us-east-1-00000000-0000-0000-0000-000000000000-0", + interval, + ); + let two = environment_schedule_offset( + "aws-us-west-1-00000000-0000-0000-0000-000000000000-1", + interval, + ); + assert_eq!(one, 30_189); + assert_eq!(two, 38_252); + assert_ne!(one, two); + } + + /// A materialized view's finish is only durable on the sink's active worker, so + /// the query has to see every worker and take the latest stamp. Pinning a single + /// worker, or letting the cutoff or the anti-join filter rows before the + /// completeness check, silently reintroduces a finish that precedes the write. + #[mz_ore::test] + fn collect_requires_every_worker() { + let cutoff = "1970-01-01T00:00:00+00:00"; + let sql = collect_sql( + ClusterId::user(1).expect("valid cluster ID"), + ReplicaId::User(2), + cutoff, + ); + assert!( + sql.contains("HAVING count(*) = count(t.hydrated_at)"), + "{sql}" + ); + assert!(sql.contains("max(t.hydrated_at)"), "{sql}"); + assert!(!sql.contains("worker_id"), "{sql}"); + + // Both of these have to apply to the aggregate's output, not to the rows + // feeding it. + let aggregate_end = sql.find(") AS e").expect("aggregate subquery"); + assert!(sql.find(cutoff).expect("cutoff") > aggregate_end, "{sql}"); + assert!( + sql.find("NOT EXISTS").expect("anti-join") > aggregate_end, + "{sql}" + ); + } +} diff --git a/src/adapter/src/coord/message_handler.rs b/src/adapter/src/coord/message_handler.rs index a8bc6342976c7..bb8fc01546ec3 100644 --- a/src/adapter/src/coord/message_handler.rs +++ b/src/adapter/src/coord/message_handler.rs @@ -185,6 +185,12 @@ impl Coordinator { Message::ArrangementSizesPrune(expired) => { self.arrangement_sizes_prune(expired).boxed_local().await; } + Message::HydrationHistorySchedule => { + self.schedule_hydration_history_collection(); + } + Message::HydrationHistoryRun => { + self.run_hydration_history_collection(); + } Message::RetireExecute { otel_ctx, data, diff --git a/src/adapter/src/coord/read_then_write.rs b/src/adapter/src/coord/read_then_write.rs index d4f05b0a2b954..fafd38d19a292 100644 --- a/src/adapter/src/coord/read_then_write.rs +++ b/src/adapter/src/coord/read_then_write.rs @@ -26,14 +26,16 @@ use tokio::sync::mpsc; use tracing::Span; use crate::PeekResponseUnary; -use crate::active_compute_sink::{ActiveComputeSink, ActiveSubscribe}; +use crate::active_compute_sink::{ActiveComputeSink, ActiveSubscribe, ActiveSubscribeOwner}; use crate::catalog::Catalog; +use crate::command::WriteAttemptKind; use crate::coord::Coordinator; use crate::coord::appends::{ InternalWriteResponder, PendingWriteTxn, TableWriteCmd, TimestampedWriteRequest, UserWriteResponder, WriteResult, WriteTarget, }; use crate::error::AdapterError; +use mz_ore::soft_panic_or_log; /// Adds `id` to the worklist the first time it is seen, enforcing the /// dependency bound. @@ -67,8 +69,8 @@ impl Coordinator { /// Takes ownership of `read_holds` and drops them only once the dataflow is /// shipped, so the `since` cannot advance past `as_of` in between. /// - /// Answers through `response_tx`, with an error if the connection went away - /// or if a dependency was dropped since the plan was optimized. + /// Answers through `response_tx`, with an error if the owning connection + /// went away or if a dependency was dropped since the plan was optimized. #[allow(clippy::too_many_arguments)] pub(crate) async fn handle_create_internal_subscribe( &mut self, @@ -79,25 +81,29 @@ impl Coordinator { as_of: Timestamp, arity: usize, sink_id: GlobalId, - conn_id: mz_adapter_types::connection::ConnectionId, - session_uuid: uuid::Uuid, + owner: ActiveSubscribeOwner, start_time: mz_ore::now::EpochMillis, read_holds: crate::ReadHolds, response_tx: tokio::sync::oneshot::Sender< Result, AdapterError>, >, ) { - // Client disconnected while waiting for the semaphore. - if !self.active_conns.contains_key(&conn_id) { - let _ = response_tx.send(Err(AdapterError::Canceled)); - return; + match &owner { + // The client may have disconnected while we waited for the semaphore. + ActiveSubscribeOwner::Session { conn_id, .. } => { + if !self.active_conns.contains_key(conn_id) { + let _ = response_tx.send(Err(AdapterError::Canceled)); + return; + } + } + // Background work has no connection to lose. + ActiveSubscribeOwner::Background => {} } let (tx, rx) = mpsc::unbounded_channel(); let active_subscribe = ActiveSubscribe { - conn_id: conn_id.clone(), - session_uuid, + owner, channel: tx, backlog_accounting: std::sync::Arc::new(std::sync::Mutex::new( crate::active_compute_sink::SubscribeBacklogAccounting::default(), @@ -164,17 +170,21 @@ impl Coordinator { /// commit, whose staging re-checks the target generation. pub(crate) fn handle_attempt_write( &mut self, - conn_id: mz_adapter_types::connection::ConnectionId, + attempt: WriteAttemptKind, target_id: mz_repr::CatalogItemId, target_global_id: GlobalId, diffs: Vec<(Row, Diff)>, - write_ts: Option, result_tx: tokio::sync::oneshot::Sender, ) { let result = InternalWriteResponder::new(result_tx); - if !self.active_conns.contains_key(&conn_id) { - result.send(WriteResult::Canceled); - return; + match &attempt { + WriteAttemptKind::Session { conn_id, .. } => { + if !self.active_conns.contains_key(conn_id) { + result.send(WriteResult::Canceled); + return; + } + } + WriteAttemptKind::Background { .. } => {} } if self.controller.read_only() { result.send(WriteResult::ReadOnly); @@ -191,7 +201,11 @@ impl Coordinator { } let table_data = TableData::Rows(diffs); - match write_ts { + let timestamped = match &attempt { + WriteAttemptKind::Session { write_ts, .. } => *write_ts, + WriteAttemptKind::Background { write_ts } => Some(*write_ts), + }; + match timestamped { Some(target_timestamp) => { let request = TimestampedWriteRequest { appends: vec![(target_global_id, vec![table_data])], @@ -207,7 +221,15 @@ impl Coordinator { tracing::warn!("group committer task gone, dropping timestamped write"); } } + // Only a session reaches this: `WriteAttemptKind::Background` always + // names a timestamp, and group commit needs a connection to answer + // through. None => { + let WriteAttemptKind::Session { conn_id, .. } = attempt else { + soft_panic_or_log!("background write reached the blind write path"); + result.send(WriteResult::Indeterminate); + return; + }; let writes = BTreeMap::from([(target_id, smallvec![table_data])]); self.pending_writes.push(PendingWriteTxn::User { span: Span::current(), @@ -235,16 +257,20 @@ impl Coordinator { } } -/// Validates that all dependencies are valid for read-then-write operations. -/// -/// Ensures all objects the selection transitively depends on (seeded by `ids`) are valid for -/// `ReadThenWrite` operations: +/// Which dependency rules a read-then-write is held to. +#[derive(Clone, Copy, Debug)] +pub(crate) enum DependencyPolicy { + /// A user statement whose relation leaves must be writable user tables. + UserDml, + /// Coordinator-authored work, which may read system objects across time domains. + SystemReads, +} + +/// Validates all transitive dependencies of a read-then-write selection. /// -/// - They do not refer to any objects whose notion of time moves differently than that of -/// user tables. This limitation is meant to ensure no writes occur between this read and the -/// subsequent write. -/// - They do not use mz_now(), whose time produced during read will differ from the write -/// timestamp. +/// User DML requires every relation leaf to be a writable user table. System +/// reads accept supported system objects across time domains. Both reject +/// `mz_now()`. /// /// The first invalid or temporal dependency encountered short-circuits with the corresponding /// error. Traversal is bounded at `max_rw_dependencies` distinct objects, returning @@ -253,6 +279,7 @@ pub(crate) fn validate_read_then_write_dependencies( catalog: &Catalog, ids: impl IntoIterator, max_rw_dependencies: usize, + policy: DependencyPolicy, ) -> Result<(), AdapterError> { use CatalogItemType::*; use mz_catalog::memory::objects; @@ -266,74 +293,82 @@ pub(crate) fn validate_read_then_write_dependencies( enqueue(&mut seen, &mut stack, id, max_rw_dependencies)?; } while let Some(id) = stack.pop() { - let mut ids_to_check = Vec::new(); - let valid = match catalog.try_get_entry(&id) { - Some(entry) => { - if let CatalogItem::View(objects::View { - locally_optimized_expr: optimized_expr, - .. - }) - | CatalogItem::MaterializedView(objects::MaterializedView { - locally_optimized_expr: optimized_expr, - .. - }) = entry.item() - { - if optimized_expr.contains_temporal() { - return Err(AdapterError::Unsupported( - "calls to mz_now in write statements", - )); - } - } - match entry.item().typ() { - typ @ (Func | View | MaterializedView) => { - ids_to_check.extend(entry.uses()); - let valid_id = id.is_user() || matches!(typ, Func); - valid_id - } - Source | Secret | Connection => false, - // Cannot select from sinks or indexes. - Sink | MetricSink | Index => unreachable!(), - Table => { - if !id.is_user() { - // We can't read from non-user tables - false - } else { - // We can't read from tables that are source-exports - entry.source_export_details().is_none() - } - } - Type => true, - } + let Some(entry) = catalog.try_get_entry(&id) else { + return Err(AdapterError::InvalidTableMutationSelection { + object_name: id.to_string(), + object_type: "unknown".to_string(), + }); + }; + + if let CatalogItem::View(objects::View { + locally_optimized_expr: optimized_expr, + .. + }) + | CatalogItem::MaterializedView(objects::MaterializedView { + locally_optimized_expr: optimized_expr, + .. + }) = entry.item() + { + if optimized_expr.contains_temporal() { + return Err(AdapterError::Unsupported( + "calls to mz_now in write statements", + )); + } + } + + let item_type = entry.item().typ(); + let ids_to_check = entry.item().query_dependencies(); + let is_writable_table = matches!( + entry.item(), + CatalogItem::Table(objects::Table { + data_source: objects::TableDataSource::TableWrites { .. }, + .. + }) + ); + let valid = match policy { + DependencyPolicy::UserDml => match item_type { + typ @ (Func | View | MaterializedView) => id.is_user() || matches!(typ, Func), + Source | Secret | Connection => false, + // Cannot select from sinks or indexes. + Sink | MetricSink | Index => unreachable!(), + Table => id.is_user() && is_writable_table, + Type => true, + }, + DependencyPolicy::SystemReads => { + id.is_system() + && matches!( + item_type, + Func | View | MaterializedView | Source | Table | Type + ) } - None => false, }; if !valid { - let (object_name, object_type) = match catalog.try_get_entry(&id) { - Some(entry) => { - let object_name = catalog.resolve_full_name(entry.name(), None).to_string(); - let object_type = match entry.item().typ() { - // We only need the disallowed types here; the allowed types are handled above. - Source => "source", - Secret => "secret", - Connection => "connection", - Table => { - if !id.is_user() { - "system table" - } else { - "source-export table" - } - } - View => "system view", - MaterializedView => "system materialized view", - _ => "invalid dependency", - }; - (object_name, object_type.to_string()) + let object_name = catalog.resolve_full_name(entry.name(), None).to_string(); + let object_type = match item_type { + // We only need the disallowed types here; the allowed types are handled above. + Source => "source", + Secret => "secret", + Connection => "connection", + Table => { + if !id.is_user() { + "system table" + } else if is_writable_table { + "user table" + } else if entry.source_export_details().is_some() { + "source-export table" + } else { + "source-backed table" + } } - None => (id.to_string(), "unknown".to_string()), + View if id.is_user() => "user view", + View => "system view", + MaterializedView if id.is_user() => "user materialized view", + MaterializedView => "system materialized view", + _ => "invalid dependency", }; return Err(AdapterError::InvalidTableMutationSelection { object_name, - object_type, + object_type: object_type.to_string(), }); } for dep in ids_to_check { diff --git a/src/adapter/src/coord/sequencer/inner.rs b/src/adapter/src/coord/sequencer/inner.rs index af7b3b2a5501b..7f1d4dcd8b025 100644 --- a/src/adapter/src/coord/sequencer/inner.rs +++ b/src/adapter/src/coord/sequencer/inner.rs @@ -113,7 +113,7 @@ use crate::command::{ExecuteResponse, Response}; use crate::coord::appends::{ BuiltinTableAppendNotify, DeferredOp, DeferredPlan, PendingWriteTxn, UserWriteResponder, }; -use crate::coord::read_then_write::validate_read_then_write_dependencies; +use crate::coord::read_then_write::{DependencyPolicy, validate_read_then_write_dependencies}; use crate::coord::sequencer::emit_optimizer_notices; use crate::coord::{ AlterConnectionValidationReady, AlterMaterializedViewReadyContext, AlterSinkReadyContext, @@ -2878,6 +2878,7 @@ impl Coordinator { self.catalog(), dependency_ids, max_rw_dependencies, + DependencyPolicy::UserDml, ) { ctx.retire(Err(err)); return; diff --git a/src/adapter/src/coord/sequencer/inner/subscribe.rs b/src/adapter/src/coord/sequencer/inner/subscribe.rs index 4e55f9b3347a2..7d3bb33043160 100644 --- a/src/adapter/src/coord/sequencer/inner/subscribe.rs +++ b/src/adapter/src/coord/sequencer/inner/subscribe.rs @@ -31,7 +31,9 @@ use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{Instrument, Span}; use uuid::Uuid; -use crate::active_compute_sink::{ActiveComputeSink, ActiveSubscribe, SubscribeBacklogAccounting}; +use crate::active_compute_sink::{ + ActiveComputeSink, ActiveSubscribe, ActiveSubscribeOwner, SubscribeBacklogAccounting, +}; use crate::command::ExecuteResponse; use crate::coord::appends::BuiltinTableAppendNotify; use crate::coord::peek::PeekResponseUnary; @@ -557,8 +559,10 @@ impl Coordinator { let max_buffered_bytes = SUBSCRIBE_MAX_BUFFERED_BYTES.get(self.catalog().system_config().dyncfgs()); let active_subscribe = ActiveSubscribe { - conn_id: conn_id.clone(), - session_uuid, + owner: ActiveSubscribeOwner::Session { + conn_id: conn_id.clone(), + session_uuid, + }, channel: tx, backlog_accounting: Arc::clone(&backlog_accounting), max_buffered_bytes, diff --git a/src/adapter/src/coord/sql.rs b/src/adapter/src/coord/sql.rs index ebf9804e7d6c8..354586e8ca2dd 100644 --- a/src/adapter/src/coord/sql.rs +++ b/src/adapter/src/coord/sql.rs @@ -269,49 +269,58 @@ impl Coordinator { id: GlobalId, active_sink: ActiveComputeSink, ) -> BuiltinTableAppendNotify { - let user = self.active_conns()[active_sink.connection_id()].user(); - let session_type = metrics::session_type_label_value(user); - - self.active_conns - .get_mut(active_sink.connection_id()) - .expect("must exist for active sessions") - .drop_sinks - .insert(id); + let session_type = match active_sink.connection_id() { + Some(conn_id) => { + let session_type = + metrics::session_type_label_value(self.active_conns()[conn_id].user()); + self.active_conns + .get_mut(conn_id) + .expect("must exist for active sessions") + .drop_sinks + .insert(id); + session_type + } + None => "system", + }; let ret_fut: BuiltinTableAppendNotify = match &active_sink { ActiveComputeSink::Subscribe(active_subscribe) => { - if active_subscribe.internal { + match active_subscribe.introspection_session_uuid() { // An internal subscribe writes no `mz_subscriptions` row, so // it stays out of the public `mz_active_subscribes` gauge // too. Counting it there would report subscribes that // introspection deliberately shows nothing of. It gets its // own internal gauge instead, since it is still a dataflow // holding cluster resources. - self.metrics - .active_internal_subscribes - .with_label_values(&[session_type]) - .inc(); + None => { + self.metrics + .active_internal_subscribes + .with_label_values(&[session_type]) + .inc(); - Box::pin(std::future::ready(())) - } else { - let update = self.catalog().state().pack_subscribe_update( - id, - active_subscribe, - Diff::ONE, - ); - let update = self.catalog().state().resolve_builtin_table_update(update); + Box::pin(std::future::ready(())) + } + Some(session_uuid) => { + let update = self.catalog().state().pack_subscribe_update( + id, + active_subscribe, + session_uuid, + Diff::ONE, + ); + let update = self.catalog().state().resolve_builtin_table_update(update); - self.metrics - .active_subscribes - .with_label_values(&[session_type]) - .inc(); + self.metrics + .active_subscribes + .with_label_values(&[session_type]) + .inc(); - // Defer the introspection-row write to a group commit instead of - // committing it inline. An inline `execute` would block the coordinator - // loop on a timestamp-oracle round trip and stall every other session. - // `implement_subscribe` waits for this write before returning the - // `SUBSCRIBE` response to the subscribing session. - self.builtin_table_update().defer(vec![update]) + // Defer the introspection-row write to a group commit instead of + // committing it inline. An inline `execute` would block the coordinator + // loop on a timestamp-oracle round trip and stall every other session. + // `implement_subscribe` waits for this write before returning the + // `SUBSCRIBE` response to the subscribing session. + self.builtin_table_update().defer(vec![update]) + } } } ActiveComputeSink::CopyTo(_) => { @@ -345,46 +354,56 @@ impl Coordinator { id: GlobalId, ) -> Option<(ActiveComputeSink, BuiltinTableAppendNotify)> { if let Some(sink) = self.active_compute_sinks.remove(&id) { - let user = self.active_conns()[sink.connection_id()].user(); - let session_type = metrics::session_type_label_value(user); - - self.active_conns - .get_mut(sink.connection_id()) - .expect("must exist for active compute sink") - .drop_sinks - .remove(&id); + let session_type = match sink.connection_id() { + Some(conn_id) => { + let session_type = + metrics::session_type_label_value(self.active_conns()[conn_id].user()); + self.active_conns + .get_mut(conn_id) + .expect("must exist for active compute sink") + .drop_sinks + .remove(&id); + session_type + } + None => "system", + }; let write_notify: BuiltinTableAppendNotify = match &sink { ActiveComputeSink::Subscribe(active_subscribe) => { - if active_subscribe.internal { + match active_subscribe.introspection_session_uuid() { // No introspection row to retract, see // `add_active_compute_sink`. The internal gauge is // decremented here to stay symmetric with it. - self.metrics - .active_internal_subscribes - .with_label_values(&[session_type]) - .dec(); + None => { + self.metrics + .active_internal_subscribes + .with_label_values(&[session_type]) + .dec(); - Box::pin(std::future::ready(())) - } else { - let update = self.catalog().state().pack_subscribe_update( - id, - active_subscribe, - Diff::MINUS_ONE, - ); - let update = self.catalog().state().resolve_builtin_table_update(update); + Box::pin(std::future::ready(())) + } + Some(session_uuid) => { + let update = self.catalog().state().pack_subscribe_update( + id, + active_subscribe, + session_uuid, + Diff::MINUS_ONE, + ); + let update = + self.catalog().state().resolve_builtin_table_update(update); - self.metrics - .active_subscribes - .with_label_values(&[session_type]) - .dec(); + self.metrics + .active_subscribes + .with_label_values(&[session_type]) + .dec(); - // Defer the retraction to a group commit, for the same reason we - // defer the insert (see `add_active_compute_sink`): committing inline - // would block the coordinator loop. Callers that expose the - // retirement wait on the notify off the coordinator loop - // before responding. - self.builtin_table_update().defer(vec![update]) + // Defer the retraction to a group commit, for the same reason we + // defer the insert (see `add_active_compute_sink`): committing inline + // would block the coordinator loop. Callers that expose the + // retirement wait on the notify off the coordinator loop + // before responding. + self.builtin_table_update().defer(vec![update]) + } } } ActiveComputeSink::CopyTo(_) => { diff --git a/src/adapter/src/error.rs b/src/adapter/src/error.rs index a7cafa8cc51ec..6f000e8b120ff 100644 --- a/src/adapter/src/error.rs +++ b/src/adapter/src/error.rs @@ -16,7 +16,6 @@ use dec::TryFromDecimalError; use itertools::Itertools; use mz_catalog::builtin::MZ_CATALOG_SERVER_CLUSTER; use mz_compute_client::controller::error as compute_error; -use mz_compute_client::controller::error::InstanceMissing; use mz_compute_types::ComputeInstanceId; use mz_controller_types::ClusterId; @@ -106,9 +105,9 @@ pub enum AdapterError { }, /// The selection value for a table mutation operation refers to an invalid object. InvalidTableMutationSelection { - /// The full name of the problematic object (e.g. a source or source-export table). + /// The full name of the problematic object, such as a source or source-backed table. object_name: String, - /// Human-readable type of the object (e.g. "source", "source-export table"). + /// Human-readable type of the object, such as "source-backed table". object_type: String, }, /// A read-then-write statement's read set has more transitive dependencies @@ -581,8 +580,7 @@ impl AdapterError { object_type, } => Some(format!( "{object_type} '{}' may not be used in this operation; \ - the selection may refer to views and materialized views, but transitive \ - dependencies must not include sources or source-export tables", + every relation leaf in the selection must be a writable user table", object_name.quoted() )), AdapterError::ReadThenWriteDependencyLimitExceeded { @@ -1089,13 +1087,6 @@ impl AdapterError { // For example, maybe we get an `InstanceMissing` if the user specifies a non-existing cluster, // in which case `ConcurrentDependencyDrop` would not be appropriate. - pub fn concurrent_dependency_drop_from_instance_missing(e: InstanceMissing) -> Self { - AdapterError::ConcurrentDependencyDrop { - dependency_kind: "cluster", - dependency_id: e.0.to_string(), - } - } - pub fn concurrent_dependency_drop_from_collection_missing(e: CollectionMissing) -> Self { AdapterError::ConcurrentDependencyDrop { dependency_kind: "collection", @@ -1278,7 +1269,7 @@ impl fmt::Display for AdapterError { AdapterError::InvalidTableMutationSelection { .. } => { write!( f, - "invalid selection: operation may only (transitively) refer to non-source, non-system tables" + "invalid selection: every relation leaf must be a writable user table" ) } AdapterError::ReadThenWriteDependencyLimitExceeded { diff --git a/src/adapter/src/frontend_peek.rs b/src/adapter/src/frontend_peek.rs index fc05dab424522..37af6a1ef4895 100644 --- a/src/adapter/src/frontend_peek.rs +++ b/src/adapter/src/frontend_peek.rs @@ -353,7 +353,7 @@ impl PeekClient { connection_id: *connection_id, tx, }) - .await + .await? } SideEffectingFunc::PgCancelBackend { connection_id: None, @@ -378,7 +378,7 @@ impl PeekClient { conn_id: session.conn_id().clone(), tx, }) - .await?; + .await??; // We held the target's `ConnectionId` handle from the RBAC // check until the Coordinator executed the function, which @@ -563,7 +563,7 @@ impl PeekClient { real_time_recency_timeout: *vars.real_time_recency_timeout(), tx, }) - .await? + .await?? } else { None }; @@ -611,7 +611,7 @@ impl PeekClient { conn_id: session.conn_id().clone(), tx, }) - .await; + .await?; if let Some(txn_read_holds) = txn_read_holds_opt { let allowed_id_bundle = txn_read_holds.id_bundle(); @@ -695,7 +695,7 @@ impl PeekClient { read_holds: read_holds.clone(), tx, }) - .await; + .await?; } (determination, read_holds) @@ -1334,7 +1334,7 @@ impl PeekClient { watch_set, tx, }) - .await?; + .await??; // On success the peek is registered in `pending_peeks`, // which now owns end-of-execution logging. On error the // coordinator logs nothing (see @@ -1356,7 +1356,7 @@ impl PeekClient { determination, tx, }) - .await; + .await?; session.add_notice(AdapterNotice::QueryTimestamp { explanation }); } @@ -1405,7 +1405,7 @@ impl PeekClient { statement_logging_id: logging.id(), tx, }) - .await?; + .await??; // On success the `Subscribing` response carries the // coordinator-side logging guard and the protocol layer logs // the end when the subscribe terminates. On error the @@ -1456,7 +1456,7 @@ impl PeekClient { sink_id, tx, }) - .await?; + .await??; // Preflight succeeded, now execute the actual COPY TO dataflow let watch_set = logging.id().map(|logging_id| { @@ -1482,7 +1482,7 @@ impl PeekClient { watch_set, tx, }) - .await?; + .await??; Ok(Some(response)) } diff --git a/src/adapter/src/frontend_read_then_write.rs b/src/adapter/src/frontend_read_then_write.rs index 78d04e3894efa..63e6012530979 100644 --- a/src/adapter/src/frontend_read_then_write.rs +++ b/src/adapter/src/frontend_read_then_write.rs @@ -133,6 +133,7 @@ use std::time::Duration; use bytesize::ByteSize; use differential_dataflow::consolidation; use mz_catalog::memory::error::ErrorKind; +use mz_catalog::memory::objects::CatalogItem; use mz_cluster_client::ReplicaId; use mz_compute_types::ComputeInstanceId; use mz_expr::Eval; @@ -154,15 +155,17 @@ use timely::progress::Antichain; use tokio::sync::mpsc; use uuid::Uuid; +use crate::active_compute_sink::ActiveSubscribeOwner; use crate::catalog::Catalog; -use crate::command::{Command, ExecuteResponse}; +use crate::command::{Command, ExecuteResponse, WriteAttemptKind}; use crate::coord::appends::WriteResult; -use crate::coord::read_then_write::validate_read_then_write_dependencies; +use crate::coord::read_then_write::{DependencyPolicy, validate_read_then_write_dependencies}; use crate::coord::timestamp_selection::TimestampProvider; use crate::coord::{Coordinator, TargetCluster}; use crate::error::AdapterError; use crate::optimize::Optimize; use crate::optimize::dataflows::{ComputeInstanceSnapshot, EvalTime, ExprPrep, ExprPrepOneShot}; +use crate::peek_client::CoordinatorClient; use crate::session::{Session, TransactionOps, WriteOp}; use crate::statement_logging::{StatementLifecycleEvent, StatementLoggingId}; use crate::{PeekClient, PeekResponseUnary, TimelineContext, optimize}; @@ -245,6 +248,73 @@ impl FrontendWriteAttemptState { } } +/// Which kind of caller is driving a read-then-write. +/// +/// Dependency rules, replica selection, and write cancellation follow from the +/// caller kind. +#[derive(Clone, Copy)] +enum RtwCaller { + /// A user statement. Cancelled with its connection, and restricted to + /// reading user tables. + Session, + /// Coordinator-owned maintenance, pinned to one replica. + /// + /// The caller must build the statement itself rather than accept one from a + /// user, and must tolerate reading a log relation that is sealed empty, + /// which is how a replica with introspection disabled presents one. + Background { replica_id: ReplicaId }, +} + +impl RtwCaller { + fn is_background(&self) -> bool { + matches!(self, RtwCaller::Background { .. }) + } + + /// Which relations the selection may read. + fn dependency_policy(&self) -> DependencyPolicy { + match self { + RtwCaller::Session => DependencyPolicy::UserDml, + RtwCaller::Background { .. } => DependencyPolicy::SystemReads, + } + } + + /// The replica a background caller pins its subscribe to, overriding the + /// session's replica selection. + fn replica_override(&self) -> Option { + match self { + RtwCaller::Background { replica_id } => Some(*replica_id), + RtwCaller::Session => None, + } + } + + /// Who owns the subscribe, which decides whether it is cancelled with a + /// connection and whether it counts against one. + fn subscribe_owner( + &self, + conn_id: &mz_adapter_types::connection::ConnectionId, + session_uuid: Uuid, + ) -> ActiveSubscribeOwner { + match self { + RtwCaller::Session => ActiveSubscribeOwner::Session { + conn_id: conn_id.clone(), + session_uuid, + }, + RtwCaller::Background { .. } => ActiveSubscribeOwner::Background, + } + } + + /// The connection a pending write is cancelled with, if any. + fn write_conn_id( + &self, + conn_id: &mz_adapter_types::connection::ConnectionId, + ) -> Option { + match self { + RtwCaller::Session => Some(conn_id.clone()), + RtwCaller::Background { .. } => None, + } + } +} + /// What the OCC loop produced. enum OccOutcome { /// The write is durable at `write_ts`. @@ -356,6 +426,7 @@ fn end_own_transaction(session: &mut Session, stages_rows: bool) { pub(crate) fn validate_selection_dependencies( catalog: &Catalog, depends_on: &BTreeSet, + policy: DependencyPolicy, ) -> Result<(), AdapterError> { let dependency_ids = depends_on .iter() @@ -370,7 +441,7 @@ pub(crate) fn validate_selection_dependencies( .collect::, _>>()?; let max_rw_dependencies = mz_adapter_types::dyncfgs::READ_THEN_WRITE_MAX_DEPENDENCIES .get(catalog.system_config().dyncfgs()); - validate_read_then_write_dependencies(catalog, dependency_ids, max_rw_dependencies) + validate_read_then_write_dependencies(catalog, dependency_ids, max_rw_dependencies, policy) } /// Validates a read-then-write and resolves the context the rest of the @@ -387,11 +458,15 @@ pub(crate) fn validate_selection_dependencies( /// serves planning, validation and optimization, so items the plan names /// cannot disappear from it, and the missing-entry branches below are /// failsafes rather than a live concurrent-DDL path. +/// +/// `dependency_policy` decides which relations the selection may read. Both +/// policies reject `mz_now()` anywhere in the transitive dependencies. fn validate_read_then_write( catalog: &Arc, session: &Session, plan: &plan::ReadThenWritePlan, target_cluster: TargetCluster, + dependency_policy: DependencyPolicy, ) -> Result { if contains_mz_now(plan) { return Err(AdapterError::Unsupported( @@ -403,7 +478,7 @@ fn validate_read_then_write( // timeline validation below. let depends_on = plan.selection.depends_on(); - validate_selection_dependencies(catalog, &depends_on)?; + validate_selection_dependencies(catalog, &depends_on, dependency_policy)?; let cluster = catalog.resolve_target_cluster(target_cluster, session)?; let cluster_id = cluster.id; @@ -602,7 +677,7 @@ struct SubscribeHandle { rx: mpsc::UnboundedReceiver, sink_id: GlobalId, /// Wrapped in `Option` so we can move it out in `Drop`. - client: Option, + client: Option, } impl SubscribeHandle { @@ -638,6 +713,68 @@ impl PeekClient { /// `catalog`, which stays in force through optimization and write-target /// generation capture. pub(crate) async fn frontend_read_then_write( + &mut self, + session: &mut Session, + plan: plan::ReadThenWritePlan, + target_cluster: TargetCluster, + catalog: &Arc, + statement_logging_id: Option, + attempt_state: Arc, + ) -> Result { + self.read_then_write( + session, + plan, + target_cluster, + catalog, + statement_logging_id, + attempt_state, + RtwCaller::Session, + ) + .await + } + + /// Executes a coordinator-owned read-then-write against system relations, + /// pinned to `replica_id`. + /// + /// See `RtwCaller::Background` for what the caller takes on by using this. + pub(crate) async fn background_read_then_write( + &mut self, + session: &mut Session, + plan: plan::ReadThenWritePlan, + cluster_id: ComputeInstanceId, + replica_id: ReplicaId, + catalog: &Arc, + ) -> Result { + let is_system_table = matches!(plan.id, CatalogItemId::System(_)) + && catalog + .try_get_entry(&plan.id) + .is_some_and(|entry| matches!(entry.item(), CatalogItem::Table(_))); + if !is_system_table { + soft_panic_or_log!( + "background read-then-write target {} is not a system table", + plan.id + ); + return Err(AdapterError::Internal( + "background read-then-write target is not a system table".into(), + )); + } + + self.read_then_write( + session, + plan, + TargetCluster::Transaction(cluster_id), + catalog, + None, + // Nothing cancels a background write, so this state only ever + // records that a write was submitted. + Arc::new(FrontendWriteAttemptState::new()), + RtwCaller::Background { replica_id }, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn read_then_write( &mut self, session: &mut Session, mut plan: plan::ReadThenWritePlan, @@ -645,7 +782,18 @@ impl PeekClient { catalog: &Arc, statement_logging_id: Option, attempt_state: Arc, + caller: RtwCaller, ) -> Result { + // The OCC dataflow emits raw diffs and does not apply top-level + // finishing. Silently dropping a LIMIT, OFFSET, projection or ordering + // can change the rows written, so this stage requires trivial finishing. + if !plan.finishing.is_trivial(plan.selection.arity()) { + soft_panic_or_log!("frontend read-then-write received nontrivial row-set finishing"); + return Err(AdapterError::Internal( + "frontend read-then-write requires trivial row-set finishing".into(), + )); + } + // A transaction that has taken a timestamped read, was opened READ // ONLY, or is committed to some other kind of operation cannot take a // write. Check up front, mirroring `sequence_insert`: the marker op @@ -661,15 +809,24 @@ impl PeekClient { return Err(AdapterError::ReadOnlyTransaction); } - let validation_result = validate_read_then_write(catalog, session, &plan, target_cluster)?; + let validation_result = validate_read_then_write( + catalog, + session, + &plan, + target_cluster, + caller.dependency_policy(), + )?; let ValidationResult { cluster_id, - replica_id, + mut replica_id, timeline, depends_on, table_desc, } = validation_result; + if let Some(pinned) = caller.replica_override() { + replica_id = Some(pinned); + } // A write that reads no persisted state may join a surrounding // transaction. Its rows do not come from a snapshot, so staging them @@ -745,10 +902,25 @@ impl PeekClient { // // The semaphore is owned by the coordinator and outlives every // session task, so `acquire_owned` cannot return `Err` in practice. - let permit = Arc::clone(&self.occ_write_semaphore) - .acquire_owned() - .await - .expect("semaphore is never closed during coordinator lifetime"); + // + // Background maintenance skips the queue entirely. It is single-flight + // by construction, one sweep at a time and one mutation at a time, so it + // adds at most one concurrent read-then-write. Taking a permit instead + // would let it hold one for as long as a subscribe on a loaded user + // replica takes to hydrate, and with `max_concurrent_occ_writes` set low + // that stalls user DML behind a background sampler. The bound above does + // not apply to it either: it has no statement timeout, only its own much + // longer one. + let permit = if caller.is_background() { + None + } else { + Some( + Arc::clone(&self.occ_write_semaphore) + .acquire_owned() + .await + .expect("semaphore is never closed during coordinator lifetime"), + ) + }; // Determine timestamp and acquire read holds. let oracle_read_ts = self.oracle_read_ts(&timeline).await?; @@ -767,7 +939,7 @@ impl PeekClient { real_time_recency_timeout, tx, }) - .await? + .await?? } else { None }; @@ -823,7 +995,7 @@ impl PeekClient { determination, tx, }) - .await; + .await?; session.add_notice(crate::AdapterNotice::QueryTimestamp { explanation }); } @@ -870,8 +1042,7 @@ impl PeekClient { as_of, arity, sink_id, - conn_id.clone(), - session_uuid, + caller.subscribe_owner(&conn_id, session_uuid), start_time, read_holds, ) @@ -889,7 +1060,7 @@ impl PeekClient { row_set_finishing_seconds, max_occ_retries, table_desc, - conn_id.clone(), + caller.write_conn_id(&conn_id), statement_logging_id, as_of, write_oracle, @@ -910,7 +1081,7 @@ impl PeekClient { // // The zero-row linearization wait below is the one exception, and hands // the permit back before it parks. - let mut permit = Some(permit); + let mut permit = permit; let response = match result { Ok(OccOutcome::Committed { response, write_ts }) => { // A committed write timestamp for a statement we meant to @@ -992,6 +1163,12 @@ impl PeekClient { .map(|()| response) } Ok(OccOutcome::Blind { response, diffs }) => { + if caller.is_background() { + return Err(AdapterError::Internal( + "background read-then-write unexpectedly had no persisted dependency" + .into(), + )); + } // The subscribe closed on its own even though the selection // reads persisted state, so the input is sealed and the diffs // are frontier-independent after all. Staging them would be @@ -1209,14 +1386,16 @@ impl PeekClient { attempt_state.mark_write_submitted(); let result = self .call_coordinator(|tx| Command::AttemptWrite { - conn_id, + attempt: WriteAttemptKind::Session { + conn_id, + write_ts: None, + }, target_id, target_global_id, diffs, - write_ts: None, tx, }) - .await; + .await?; // Every outcome here terminates the attempt, so `write_submitted` // stays set per its contract. @@ -1253,8 +1432,7 @@ impl PeekClient { as_of: Timestamp, arity: usize, sink_id: GlobalId, - conn_id: mz_adapter_types::connection::ConnectionId, - session_uuid: Uuid, + owner: ActiveSubscribeOwner, start_time: mz_ore::now::EpochMillis, read_holds: crate::ReadHolds, ) -> Result { @@ -1267,13 +1445,12 @@ impl PeekClient { as_of, arity, sink_id, - conn_id, - session_uuid, + owner, start_time, read_holds, tx, }) - .await?; + .await??; Ok(SubscribeHandle { rx, @@ -1317,7 +1494,7 @@ impl PeekClient { row_set_finishing_seconds: Histogram, max_occ_retries: usize, table_desc: RelationDesc, - conn_id: mz_adapter_types::connection::ConnectionId, + write_conn_id: Option, statement_logging_id: Option, as_of: Timestamp, write_oracle: Option + Send + Sync>>, @@ -1554,16 +1731,25 @@ impl PeekClient { // in profiles. Every attempt clones every row, and we retry up to // `max_occ_retries` times. attempt_state.mark_write_submitted(); - let result = self + let result = match self .call_coordinator(|tx| Command::AttemptWrite { - conn_id: conn_id.clone(), + attempt: match write_conn_id.clone() { + Some(conn_id) => WriteAttemptKind::Session { + conn_id, + write_ts: Some(target), + }, + None => WriteAttemptKind::Background { write_ts: target }, + }, target_id, target_global_id, diffs: state.payload.clone(), - write_ts: Some(target), tx, }) - .await; + .await + { + Ok(result) => result, + Err(error) => break Err(error), + }; match classify_write_result(result, target_id, attempt_state) { WriteOutcome::Committed(timestamp) => { diff --git a/src/adapter/src/metrics.rs b/src/adapter/src/metrics.rs index 028858c9e59f1..5cc8492f32960 100644 --- a/src/adapter/src/metrics.rs +++ b/src/adapter/src/metrics.rs @@ -31,6 +31,10 @@ pub struct Metrics { pub storage_usage_collection_time_seconds: Histogram, pub arrangement_sizes_collection_time_seconds: Histogram, pub arrangement_sizes_rows_written: IntCounter, + pub hydration_history_mutations: IntCounterVec, + pub hydration_history_retention_batch_full: IntCounter, + pub hydration_history_rows_affected: IntCounterVec, + pub hydration_history_sweep_duration_seconds: Histogram, pub subscribe_outputs: IntCounterVec, pub canceled_peeks: IntCounter, pub linearize_message_seconds: HistogramVec, @@ -88,7 +92,7 @@ impl Metrics { )), active_internal_subscribes: registry.register(metric!( name: "mz_active_internal_subscribes", - help: "The number of active internal subscribes, which serve frontend-sequenced read-then-write.", + help: "The number of active internal subscribes used by read-then-write operations and background maintenance.", var_labels: ["session_type"], )), active_copy_tos: registry.register(metric!( @@ -139,6 +143,25 @@ impl Metrics { name: "mz_arrangement_sizes_rows_written_total", help: "Total rows appended to mz_object_arrangement_size_history since process start.", )), + hydration_history_mutations: registry.register(metric!( + name: "mz_hydration_history_mutations_total", + help: "Total hydration-history collection and retention mutations since process start.", + var_labels: ["operation", "outcome"], + )), + hydration_history_retention_batch_full: registry.register(metric!( + name: "mz_hydration_history_retention_batch_full_total", + help: "Total hydration-history sweeps whose retention batch was full. Repeated increments mean retention may not be keeping up with its schedule.", + )), + hydration_history_rows_affected: registry.register(metric!( + name: "mz_hydration_history_rows_affected_total", + help: "Total rows changed by hydration-history maintenance since process start.", + var_labels: ["action"], + )), + hydration_history_sweep_duration_seconds: registry.register(metric!( + name: "mz_hydration_history_sweep_duration_seconds", + help: "Wall time of a complete hydration-history collection and retention sweep.", + buckets: histogram_seconds_buckets(0.128, 1024.0), + )), subscribe_outputs: registry.register(metric!( name: "mz_subscribe_outputs", help: "The total number of different subscribe outputs used", diff --git a/src/adapter/src/peek_client.rs b/src/adapter/src/peek_client.rs index 206374f6db53d..b9b77940783ce 100644 --- a/src/adapter/src/peek_client.rs +++ b/src/adapter/src/peek_client.rs @@ -42,7 +42,8 @@ use crate::catalog::Catalog; use crate::command::{CatalogSnapshot, Command, ExecuteResponse}; use crate::coord::appends::GroupCommitNotifier; use crate::coord::peek::FastPathPlan; -use crate::coord::{Coordinator, ExecuteContextExtra, ExecuteContextGuard}; +use crate::coord::{Coordinator, ExecuteContextExtra, ExecuteContextGuard, Message}; +use crate::metrics::Metrics; use crate::session::{LifecycleTimestamps, Session}; use crate::statement_logging::{ FrontendStatementLoggingEvent, PreparedStatementEvent, PreparedStatementLoggingInfo, @@ -57,7 +58,7 @@ pub type StorageCollectionsHandle = /// Clients needed for peek sequencing in the Adapter Frontend. #[derive(Debug)] pub struct PeekClient { - coordinator_client: Client, + coordinator_client: CoordinatorClient, /// Cache of the latest catalog snapshot. Serves /// [`PeekClient::catalog_snapshot`] without a Coordinator round-trip /// while the catalog's transient revision is unchanged. @@ -91,13 +92,65 @@ pub struct PeekClient { pub read_only: bool, } +/// A command sender that does not make background work keep the coordinator alive. +#[derive(Debug, Clone)] +pub(crate) enum CoordinatorClient { + Session(Client), + Background { + tx: tokio::sync::mpsc::UnboundedSender, + metrics: Metrics, + }, +} + +impl CoordinatorClient { + /// Sends a command, dropping it if the coordinator is gone. + /// + /// A session cannot outrun the coordinator: it holds a `Client`, and the + /// coordinator's loop only exits once every client is dropped, so a failed + /// send there is a real bug. Background work holds no client on purpose, so + /// losing the race with shutdown is normal and must not crash the process. + /// Dropping the command closes its response channel, which the caller handles + /// as an error. + pub(crate) fn send(&self, command: Command) { + if self.try_send(command) { + return; + } + match self { + CoordinatorClient::Session(_) => panic!("coordinator unexpectedly gone"), + CoordinatorClient::Background { .. } => { + tracing::debug!("dropping background command, coordinator is gone") + } + } + } + + pub(crate) fn try_send(&self, command: Command) -> bool { + match self { + CoordinatorClient::Session(client) => client.try_send(command), + CoordinatorClient::Background { tx, .. } => tx + .send(Message::Command( + mz_ore::tracing::OpenTelemetryContext::obtain(), + command, + )) + .is_ok(), + } + } + + pub(crate) fn metrics(&self) -> &Metrics { + match self { + CoordinatorClient::Session(client) => client.metrics(), + CoordinatorClient::Background { metrics, .. } => metrics, + } + } +} + impl PeekClient { /// Creates a PeekClient. /// /// `catalog` seeds the catalog snapshot cache, so that the session's /// first statements don't need a `Command::CatalogSnapshot` round-trip. - pub fn new( - coordinator_client: Client, + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + coordinator_client: CoordinatorClient, catalog: &Arc, storage_collections: StorageCollectionsHandle, transient_id_gen: Arc, @@ -129,14 +182,15 @@ impl PeekClient { pub async fn ensure_compute_instance_client( &mut self, compute_instance: ComputeInstanceId, - ) -> Result { + ) -> Result { if !self.compute_instances.contains_key(&compute_instance) { let client = self .call_coordinator(|tx| Command::GetComputeInstanceClient { instance_id: compute_instance, tx, }) - .await?; + .await + .map_err(|_| CollectionLookupError::InstanceShutDown)??; self.compute_instances.insert(compute_instance, client); } Ok(self @@ -156,7 +210,7 @@ impl PeekClient { timeline: timeline.clone(), tx, }) - .await?; + .await??; self.oracles.insert(timeline.clone(), oracle); } Ok(self.oracles.get_mut(&timeline).expect("ensured above")) @@ -196,9 +250,13 @@ impl PeekClient { // The cache is empty, stale, or its allocation is gone: do the // round-trip. let start = std::time::Instant::now(); + // Session clients keep the coordinator loop alive. A dropped response + // here therefore indicates an internal lifecycle bug, unlike a + // background compute lookup racing coordinator shutdown. let CatalogSnapshot { catalog } = self .call_coordinator(|tx| Command::CatalogSnapshot { tx }) - .await; + .await + .expect("coordinator unexpectedly dropped catalog snapshot response"); let metrics = self.coordinator_client.metrics(); metrics .catalog_snapshot_seconds @@ -212,18 +270,18 @@ impl PeekClient { catalog } - pub(crate) async fn call_coordinator(&self, f: F) -> T + /// Calls the coordinator and returns an error if it drops the response. + pub(crate) async fn call_coordinator(&self, f: F) -> Result where F: FnOnce(oneshot::Sender) -> Command, { let (tx, rx) = oneshot::channel(); self.coordinator_client.send(f(tx)); - rx.await - .expect("if the coordinator is still alive, it shouldn't have dropped our call") + Ok(rx.await?) } /// The client for sending commands to the coordinator. - pub(crate) fn coordinator_client(&self) -> &crate::Client { + pub(crate) fn coordinator_client(&self) -> &CoordinatorClient { &self.coordinator_client } @@ -435,7 +493,12 @@ impl PeekClient { let client = self .ensure_compute_instance_client(compute_instance) .await - .map_err(AdapterError::concurrent_dependency_drop_from_instance_missing)?; + .map_err(|error| { + AdapterError::concurrent_dependency_drop_from_collection_lookup_error( + error, + compute_instance, + ) + })?; // Register coordinator tracking of this peek. This has to complete before issuing the peek. // @@ -450,7 +513,7 @@ impl PeekClient { watch_set, tx, }) - .await?; + .await??; // The peek is registered: the coordinator's `pending_peeks` entry now // owns end-of-execution logging. It logs the end on peek completion, @@ -491,14 +554,15 @@ impl PeekClient { // ask it to unregister the peek and retire it with this error. If // a concurrent teardown already retired the peek, the end is // already logged and the unregistration is a no-op. - self.call_coordinator(|tx| Command::UnregisterFrontendPeek { - uuid, - reason: statement_logging::StatementEndedExecutionReason::Errored { - error: err.to_string(), - }, - tx, - }) - .await; + let _ = self + .call_coordinator(|tx| Command::UnregisterFrontendPeek { + uuid, + reason: statement_logging::StatementEndedExecutionReason::Errored { + error: err.to_string(), + }, + tx, + }) + .await; return Err(err); } @@ -645,7 +709,7 @@ impl PeekClient { struct StatementLoggingGuard { /// `None` if the statement was not sampled for logging. id: Option, - coordinator_client: Client, + coordinator_client: CoordinatorClient, now: mz_ore::now::NowFn, } diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs index 14623facb0d8e..0a4c6d914c209 100644 --- a/src/catalog/src/builtin.rs +++ b/src/catalog/src/builtin.rs @@ -837,6 +837,14 @@ pub static MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY_DESCRIPTION: LazyLock = + LazyLock::new(|| SystemObjectDescription { + schema_name: MZ_OBJECT_HYDRATION_HISTORY.schema.to_string(), + object_type: CatalogItemType::Table, + object_name: MZ_OBJECT_HYDRATION_HISTORY.name.to_string(), + }); + /// Identifies [`MZ_CLUSTER_REPLICA_FRONTIERS`] for the schema-migration guard in /// `builtin_schema_migration.rs`, which forbids migrating this source because the 0dt /// caught-up gate reads the leader's shard for it to learn the live frontiers. @@ -1477,6 +1485,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::View(&MZ_INDEX_ADVICE), Builtin::View(&MZ_MCP_DATA_PRODUCTS), Builtin::View(&MZ_MCP_DATA_PRODUCT_DETAILS), + Builtin::Table(&MZ_OBJECT_HYDRATION_HISTORY), ]; builtin_items.extend(notice::builtins()); diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs index d0546454025f0..463df5d4168cd 100644 --- a/src/catalog/src/builtin/mz_internal.rs +++ b/src/catalog/src/builtin/mz_internal.rs @@ -5142,6 +5142,106 @@ pub static MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY_TS_IND: LazyLock = is_retained_metrics_object: true, }); +/// Completed hydration episodes, one row per object, replica, and installation. +/// +/// Exempt from the bootstrap reset and from forced shard replacement, since the +/// contents cannot be rebuilt from anything else. Schema evolution keeps them and +/// applies normally. Clearing them for a schema change is still allowed, see the +/// tripwire in `validate_migration_steps`. +pub static MZ_OBJECT_HYDRATION_HISTORY: LazyLock = LazyLock::new(|| BuiltinTable { + name: "mz_object_hydration_history", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::TABLE_MZ_OBJECT_HYDRATION_HISTORY_OID, + desc: RelationDesc::builder() + .with_column("object_id", SqlScalarType::String.nullable(false)) + .with_column("cluster_id", SqlScalarType::String.nullable(false)) + .with_column("replica_id", SqlScalarType::String.nullable(false)) + .with_column( + "installed_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column( + "started_at", + SqlScalarType::TimestampTz { precision: None }.nullable(true), + ) + .with_column( + "hydrated_at", + SqlScalarType::TimestampTz { precision: None }.nullable(true), + ) + .with_column("status", SqlScalarType::String.nullable(false)) + .finish(), + column_comments: BTreeMap::from_iter([ + ( + "object_id", + "The ID of the object's dataflow, as reported by the replica. Join `mz_internal.mz_object_global_ids` to reach the index or materialized view while that mapping exists. Dropping the dataflow retracts the mapping, so historical IDs may no longer resolve.", + ), + ("cluster_id", "The ID of the object's cluster."), + ( + "replica_id", + "The ID of the cluster replica. May name a replica that no longer exists.", + ), + ( + "installed_at", + "When the object's dataflow was installed on the replica.", + ), + ( + "started_at", + "When hydration work began, or `NULL` if the replica reported none. A replica that observed no start reports the installation time instead, so a zero interval between the two does not mean the dataflow started immediately.", + ), + ("hydrated_at", "When hydration finished."), + ( + "status", + "The terminal status. Currently always `hydrated`.", + ), + ]), + // Not a retained-metrics object: that would pin a 30 day compaction window, + // and our history lives in the rows, which the retention sweep retracts on + // its own schedule. Nothing reads this table at an old timestamp. + is_retained_metrics_object: false, + access: vec![PUBLIC_SELECT], + ontology: Some(Ontology { + entity_name: "object_hydration_event", + description: "Completed hydration of an index or materialized view on a replica", + // NOTE: These references outlive what they point at. A row deliberately + // survives the object and the replica it describes, so resolving one + // against the catalog can come up empty. + links: &const { + [ + OntologyLink { + name: "hydration_of_dataflow", + target: "object_global_id", + properties: LinkProperties::fk_typed( + "object_id", + "global_id", + Cardinality::ManyToOne, + mz_repr::SemanticType::GlobalId, + ), + }, + OntologyLink { + name: "hydrated_on_cluster", + target: "cluster", + properties: LinkProperties::fk("cluster_id", "id", Cardinality::ManyToOne), + }, + OntologyLink { + name: "hydrated_on_replica", + target: "replica", + properties: LinkProperties::fk_typed( + "replica_id", + "id", + Cardinality::ManyToOne, + mz_repr::SemanticType::ReplicaId, + ), + }, + ] + }, + column_semantic_types: &[ + ("object_id", SemanticType::GlobalId), + ("cluster_id", SemanticType::ClusterId), + ("replica_id", SemanticType::ReplicaId), + ], + }), +}); + pub static MZ_COMPUTE_HYDRATION_STATUSES: LazyLock = LazyLock::new(|| BuiltinView { name: "mz_compute_hydration_statuses", schema: MZ_INTERNAL_SCHEMA, diff --git a/src/catalog/src/memory/objects.rs b/src/catalog/src/memory/objects.rs index 39a84ca35c32a..e6fd520c3a8a2 100644 --- a/src/catalog/src/memory/objects.rs +++ b/src/catalog/src/memory/objects.rs @@ -2033,6 +2033,33 @@ impl CatalogItem { uses } + /// Returns direct dependencies to traverse when finding this item's query leaves. + /// + /// Functions and views return all of their uses. Materialized views do the + /// same except for the replacement target, which is lifecycle metadata. All + /// other items are query leaves or cannot be selected from. + pub fn query_dependencies(&self) -> BTreeSet { + match self { + CatalogItem::Func(_) | CatalogItem::View(_) => self.uses(), + CatalogItem::MaterializedView(mv) => { + let mut dependencies = self.uses(); + if let Some(target) = mv.replacement_target { + dependencies.remove(&target); + } + dependencies + } + CatalogItem::Index(_) + | CatalogItem::Sink(_) + | CatalogItem::Source(_) + | CatalogItem::Log(_) + | CatalogItem::Table(_) + | CatalogItem::Type(_) + | CatalogItem::Secret(_) + | CatalogItem::Connection(_) + | CatalogItem::MetricSink(_) => BTreeSet::new(), + } + } + /// Returns the connection ID that this item belongs to, if this item is /// temporary. pub fn conn_id(&self) -> Option<&ConnectionId> { diff --git a/src/mysql-util/src/probe.rs b/src/mysql-util/src/probe.rs index a4b0bfed8d0ad..0df710547dd95 100644 --- a/src/mysql-util/src/probe.rs +++ b/src/mysql-util/src/probe.rs @@ -25,6 +25,10 @@ pub const MAX_KEY_LENGTH: u32 = 768; /// Probes a string primary key column. Only supports `utf8mb4_bin` against CHAR/VARCHAR /// columns up to 768 characters. Enforcement is deferred to the caller. There may be /// other collations we can support, but we should do more validation. +/// +/// NOTE: The partitioning walk's local bookkeeping scales quadratically with the +/// number of probes. Local benchmarking of 5k probes over 5k entries was fast +/// (under a second of CPU), so exceed that budget at your own risk. pub struct KeyProber<'a, 't> { tx: &'a mut Transaction<'t>, /// Quoted `` `schema`.`table` `` for SQL interpolation. diff --git a/src/pgrepr-consts/src/oid.rs b/src/pgrepr-consts/src/oid.rs index 7177523ef7d1e..c2f7d677821a6 100644 --- a/src/pgrepr-consts/src/oid.rs +++ b/src/pgrepr-consts/src/oid.rs @@ -828,3 +828,4 @@ pub const VIEW_MZ_BUILTIN_TABLES_OID: u32 = 17118; pub const VIEW_MZ_BUILTIN_VIEWS_OID: u32 = 17119; pub const MV_MZ_METRIC_SINKS_OID: u32 = 17120; pub const INDEX_MZ_METRIC_SINKS_IND_OID: u32 = 17121; +pub const TABLE_MZ_OBJECT_HYDRATION_HISTORY_OID: u32 = 17122; diff --git a/src/repr/src/row.rs b/src/repr/src/row.rs index 52bfb57665988..c6dc97f6c7e8d 100644 --- a/src/repr/src/row.rs +++ b/src/repr/src/row.rs @@ -361,7 +361,6 @@ impl Ord for Row { /// `SourceData`, and `row.proto` is covered by the buf breaking lint. #[derive( Clone, - Debug, Default, Eq, PartialEq, @@ -387,6 +386,12 @@ impl Deref for StableRow { } } +impl Debug for StableRow { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + mod stable_row_proto { use mz_proto::RustType; use prost::Message; diff --git a/src/storage-types/src/dyncfgs.rs b/src/storage-types/src/dyncfgs.rs index d7bbdc26c77f4..3df7d647302e8 100644 --- a/src/storage-types/src/dyncfgs.rs +++ b/src/storage-types/src/dyncfgs.rs @@ -255,7 +255,8 @@ pub static MYSQL_SOURCE_SNAPSHOT_PARTITION_PROBED_PREFIXES_PER_BILLION_ROWS: Con "mysql_source_snapshot_partition_probed_prefixes_per_billion_rows", 1_000, "Cap on MySQL snapshot PK-prefix partitioning probed prefixes per table, per billion \ - estimated rows; when exhausted, splitting stops early with coarser partition boundaries.", + estimated rows; when exhausted, splitting stops early with coarser partition boundaries. \ + The per-table budget is additionally hard-capped at 5000.", ParameterScope::Replica, ); diff --git a/src/storage/src/source/mysql/snapshot.rs b/src/storage/src/source/mysql/snapshot.rs index fbe6967c40afd..b27176a5ba9b8 100644 --- a/src/storage/src/source/mysql/snapshot.rs +++ b/src/storage/src/source/mysql/snapshot.rs @@ -236,6 +236,7 @@ fn worker_pk_range( const SUPPORTED_PK_COLLATION: &str = "utf8mb4_bin"; const SUPPORTED_PK_CHARSET: &str = "utf8mb4"; const MIN_PROBED_PREFIXES: u64 = 64; +const MAX_PROBED_PREFIXES: u64 = 5_000; /// Partitioning configuration settings bundled to avoid accidental mixing. struct PartitionSettings { @@ -277,7 +278,7 @@ async fn compute_sampled_splits( // that can take hours to snapshot. let max_probed_prefixes = (row_count.saturating_mul(settings.probed_prefixes_per_billion_rows) / 1_000_000_000) - .max(MIN_PROBED_PREFIXES); + .clamp(MIN_PROBED_PREFIXES, MAX_PROBED_PREFIXES); let params = mz_mysql_util::PartitionParams { num_workers: worker_count, estimated_row_count: row_count, diff --git a/test/launchdarkly-flag-consistency/mzcompose.py b/test/launchdarkly-flag-consistency/mzcompose.py index 51cfe8432f2d0..da92bb3bf07ee 100644 --- a/test/launchdarkly-flag-consistency/mzcompose.py +++ b/test/launchdarkly-flag-consistency/mzcompose.py @@ -290,6 +290,8 @@ enable_will_distinct_propagation enable_with_ordinality_legacy_fallback grpc_client_connect_timeout + hydration_history_collection_interval + hydration_history_retention_period kafka_buffered_event_resize_threshold_elements kafka_default_aws_privatelink_endpoint_identification_algorithm kafka_poll_max_wait diff --git a/test/restart/mzcompose.py b/test/restart/mzcompose.py index 7742f66b4ea6a..c8b2dd56d3a80 100644 --- a/test/restart/mzcompose.py +++ b/test/restart/mzcompose.py @@ -1436,6 +1436,162 @@ def wait_for(sql: str, expected: list[tuple], what: str) -> None: pass +def workflow_hydration_history_survives_restart(c: Composition) -> None: + """`mz_object_hydration_history` rows outlive the process that wrote them. + + Killing the service also restarts clusterd, so the replica hydrates again + and legitimately records a *second* episode with a fresh `installed_at`. + What must hold is that the pre-restart episode is still there afterwards, + unchanged, and that repeated sweeps do not duplicate it. Asserting a total + row count of one would instead assert that rehydration goes unrecorded. + """ + + def episodes(name: str = "hydration_history_i") -> list[list]: + return c.sql_query(f""" + SELECT h.installed_at::text, h.started_at::text, + h.hydrated_at::text, h.status + FROM mz_internal.mz_object_hydration_history AS h + JOIN mz_internal.mz_object_global_ids AS g ON g.global_id = h.object_id + JOIN mz_catalog.mz_objects AS o ON o.id = g.id + WHERE o.name = '{name}' + ORDER BY h.installed_at""") + + c.down(destroy_volumes=True) + with c.override( + Materialized( + additional_system_parameter_defaults={ + "hydration_history_collection_interval": "1s", + # Pin retention: CI randomizes it, and a short period would + # prune the episode this test restarts around. + "hydration_history_retention_period": "30d", + }, + sanity_restart=False, + ) + ): + c.up("materialized") + c.sql(dedent("""\ + CREATE CLUSTER hydration_history SIZE 'scale=1,workers=2'; + CREATE TABLE hydration_history_t (a int); + INSERT INTO hydration_history_t SELECT generate_series(1, 100000); + CREATE INDEX hydration_history_i + IN CLUSTER hydration_history ON hydration_history_t (a); + CREATE MATERIALIZED VIEW hydration_history_mv_a + IN CLUSTER hydration_history AS SELECT a + 1 AS a FROM hydration_history_t; + CREATE MATERIALIZED VIEW hydration_history_mv_b + IN CLUSTER hydration_history AS SELECT a + 2 AS a FROM hydration_history_t; + """)) + + deadline = time.time() + 120 + before = [] + while time.time() < deadline: + before = episodes() + if before: + break + time.sleep(0.5) + assert ( + len(before) == 1 + ), f"expected exactly one episode, got {before} (empty means it timed out)" + + # Discover which MV's persist-sink worker is off worker 0 instead of + # predicting it from user-ID allocation and hashing. Enough input data + # separates compute completion from the active worker's durable write. + deadline = time.time() + 120 + candidates = [] + worker_rows = [] + with c.sql_cursor(reuse_connection=True) as cursor: + try: + cursor.execute("SET cluster = hydration_history") + cursor.execute("SET cluster_replica = r1") + while time.time() < deadline: + cursor.execute(""" + SELECT + mv.name, + max(h.hydrated_at)::text, + (max(h.hydrated_at) + FILTER (WHERE h.worker_id = 0))::text + FROM mz_introspection.mz_compute_hydration_times_per_worker AS h + JOIN mz_internal.mz_object_global_ids AS g + ON g.global_id = h.export_id + JOIN mz_catalog.mz_materialized_views AS mv ON mv.id = g.id + WHERE mv.name IN ( + 'hydration_history_mv_a', + 'hydration_history_mv_b' + ) + GROUP BY mv.name + HAVING count(*) = 2 + AND count(*) = count(h.hydrated_at) + ORDER BY mv.name + """) + worker_rows = cursor.fetchall() + candidates = [ + row + for row in worker_rows + if row[1] is not None and row[2] != row[1] + ] + if candidates: + break + time.sleep(0.5) + finally: + cursor.execute("RESET cluster_replica") + cursor.execute("RESET cluster") + assert candidates, ( + "test fixture did not produce an MV with its persist-sink worker " + f"off worker 0: {worker_rows}" + ) + mv_name, latest_worker_finish, worker_zero_finish = candidates[0] + + # The selected MV's history episode must use the all-worker maximum, + # not worker 0's earlier compute-only finish. + deadline = time.time() + 120 + mv_episodes = [] + while time.time() < deadline: + mv_episodes = episodes(mv_name) + if mv_episodes: + break + time.sleep(0.5) + assert ( + len(mv_episodes) == 1 + ), f"expected one materialized view episode for {mv_name}, got {mv_episodes}" + assert mv_episodes[0][2] == latest_worker_finish, ( + f"durable finish {mv_episodes[0][2]} did not match " + f"latest worker finish {latest_worker_finish}. " + f"worker 0 finished at {worker_zero_finish}" + ) + + c.kill("materialized") + c.up("materialized") + + # The pre-restart episode must remain byte-identical, and restarting the + # replica must produce exactly one episode with a fresh installation. + deadline = time.time() + 120 + after = [] + fresh = [] + while time.time() < deadline: + after = episodes() + fresh = [episode for episode in after if episode[0] != before[0][0]] + if before[0] in after and len(fresh) == 1: + break + time.sleep(0.5) + assert ( + before[0] in after + ), f"restart lost the pre-restart episode: had {before}, now {after}" + assert ( + len(after) == 2 and len(fresh) == 1 + ), f"expected one preserved and one fresh episode, got {after}" + + # Let several sweeps run. The pre-restart episode must not be duplicated, + # and the post-restart episode must settle at one row too. + time.sleep(10) + settled = episodes() + assert ( + before[0] in settled + ), f"pre-restart episode disappeared: had {before}, now {settled}" + assert len(settled) == len( + set(tuple(row) for row in settled) + ), f"sweeps duplicated a hydration episode: {settled}" + assert len(settled) == 2, f"expected two settled episodes, got {settled}" + + def workflow_default(c: Composition) -> None: def process(name: str) -> None: if name == "default": diff --git a/test/sqllogictest/autogenerated/mz_internal.slt b/test/sqllogictest/autogenerated/mz_internal.slt index a7cda93eb235b..47c55bb3d6813 100644 --- a/test/sqllogictest/autogenerated/mz_internal.slt +++ b/test/sqllogictest/autogenerated/mz_internal.slt @@ -388,6 +388,17 @@ object_type text The␠type␠of␠the␠object:␠one␠of␠`table`,␠`sour created_at timestamp␠with␠time␠zone Wall-clock␠timestamp␠of␠when␠the␠object␠was␠created.␠`NULL`␠for␠built␠in␠system␠objects. dropped_at timestamp␠with␠time␠zone Wall-clock␠timestamp␠of␠when␠the␠object␠was␠dropped.␠`NULL`␠for␠built␠in␠system␠objects␠or␠if␠the␠object␠hasn't␠been␠dropped. +query TTT +SELECT name, type, comment FROM objects WHERE schema = 'mz_internal' AND object = 'mz_object_hydration_history' ORDER BY position +---- +object_id text The␠ID␠of␠the␠object's␠dataflow,␠as␠reported␠by␠the␠replica.␠Join␠`mz_internal.mz_object_global_ids`␠to␠reach␠the␠index␠or␠materialized␠view␠while␠that␠mapping␠exists.␠Dropping␠the␠dataflow␠retracts␠the␠mapping,␠so␠historical␠IDs␠may␠no␠longer␠resolve. +cluster_id text The␠ID␠of␠the␠object's␠cluster. +replica_id text The␠ID␠of␠the␠cluster␠replica.␠May␠name␠a␠replica␠that␠no␠longer␠exists. +installed_at timestamp␠with␠time␠zone When␠the␠object's␠dataflow␠was␠installed␠on␠the␠replica. +started_at timestamp␠with␠time␠zone When␠hydration␠work␠began,␠or␠`NULL`␠if␠the␠replica␠reported␠none.␠A␠replica␠that␠observed␠no␠start␠reports␠the␠installation␠time␠instead,␠so␠a␠zero␠interval␠between␠the␠two␠does␠not␠mean␠the␠dataflow␠started␠immediately. +hydrated_at timestamp␠with␠time␠zone When␠hydration␠finished. +status text The␠terminal␠status.␠Currently␠always␠`hydrated`. + query TTT SELECT name, type, comment FROM objects WHERE schema = 'mz_internal' AND object = 'mz_object_transitive_dependencies' ORDER BY position ---- @@ -840,6 +851,7 @@ mz_object_fully_qualified_names mz_object_global_ids mz_object_graph_edges mz_object_history +mz_object_hydration_history mz_object_lifetimes mz_object_oid_alias mz_object_transitive_dependencies diff --git a/test/sqllogictest/catalog_server_explain.slt b/test/sqllogictest/catalog_server_explain.slt index 7d5531c356c0e..769ed420cb0ab 100644 --- a/test/sqllogictest/catalog_server_explain.slt +++ b/test/sqllogictest/catalog_server_explain.slt @@ -5472,7 +5472,7 @@ mz_catalog.mz_tables: Project: #3, #0, #4, #1, #5, #2, #6, #6, #6 Map: "s1", null →Arrange (#1{schema_name}, #2{name}) - →Constant (29 rows) + →Constant (30 rows) →Arrange (#0{schema_name}) (#0{schema_name}, #1{name}) →Fused with Child Map/Filter/Project Project: #5, #4, #6 @@ -8246,7 +8246,7 @@ query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_tables"; ---- Explained Query (fast path): - →Constant (29 rows) + →Constant (30 rows) Target cluster: mz_catalog_server @@ -10238,7 +10238,7 @@ query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_ontology_entity_types"; ---- Explained Query (fast path): - →Constant (135 rows) + →Constant (136 rows) Target cluster: mz_catalog_server @@ -10248,7 +10248,7 @@ query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_ontology_link_types"; ---- Explained Query (fast path): - →Constant (177 rows) + →Constant (180 rows) Target cluster: mz_catalog_server @@ -10262,7 +10262,7 @@ Explained Query: cte l0 = →Differential Join %1:mz_schemas[#0{id}] » %2:mz_objects[#2{schema_id}] » %0[#0{schema_name}, #1{table_name}] » %3:mz_columns[#0{id}] →Arrange (#0{schema_name}, #1{table_name}) - →Constant (135 rows) + →Constant (136 rows) →Arrange (#0{id}) →Fused with Child Map/Filter/Project Project: #1, #3 @@ -10315,7 +10315,7 @@ Explained Query: →Differential Join %0:l4[#0{entity_name}, #1{name}] » %1[#0{entity_name}, #1{column_name}] →Arranged l4 →Arrange (#0{entity_name}, #1{column_name}) - →Constant (278 rows) + →Constant (281 rows) →Return →Union →Map/Filter/Project diff --git a/test/sqllogictest/distinct_arrangements.slt b/test/sqllogictest/distinct_arrangements.slt index 522db9cc25a15..05d66f9d9a694 100644 --- a/test/sqllogictest/distinct_arrangements.slt +++ b/test/sqllogictest/distinct_arrangements.slt @@ -978,8 +978,8 @@ Arrange errors ArrangeAccumulable [val: empty] ArrangeAccumulable [val: empty] ArrangeAccumulable [val: empty] -ArrangeBy[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(StableRow(Row{[Int64(2)]})), ReprColumnType { scalar_type: Int64, nullable: false }))]] -ArrangeBy[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(StableRow(Row{[Int64(2)]})), ReprColumnType { scalar_type: Int64, nullable: false }))]]-errors +ArrangeBy[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(Row{[Int64(2)]}), ReprColumnType { scalar_type: Int64, nullable: false }))]] +ArrangeBy[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(Row{[Int64(2)]}), ReprColumnType { scalar_type: Int64, nullable: false }))]]-errors ArrangeBy[[CallUnary(CastInt32ToInt64(CastInt32ToInt64), Column(0, "id"))]] ArrangeBy[[Column(0)]] ArrangeBy[[Column(0, "data")]] @@ -1017,7 +1017,7 @@ Distinct errors Distinct errors Distinct errors Distinct errors -Distinct errors[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(StableRow(Row{[Int64(2)]})), ReprColumnType { scalar_type: Int64, nullable: false }))]] +Distinct errors[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(Row{[Int64(2)]}), ReprColumnType { scalar_type: Int64, nullable: false }))]] Distinct errors[[Column(0)]] Distinct errors[[Column(0)]] Distinct errors[[]] diff --git a/test/sqllogictest/information_schema_tables.slt b/test/sqllogictest/information_schema_tables.slt index 922d078954dcd..2081606984765 100644 --- a/test/sqllogictest/information_schema_tables.slt +++ b/test/sqllogictest/information_schema_tables.slt @@ -513,6 +513,10 @@ mz_object_history VIEW materialize mz_internal +mz_object_hydration_history +BASE TABLE +materialize +mz_internal mz_object_lifetimes VIEW materialize diff --git a/test/sqllogictest/insert-into-select-source-error.slt b/test/sqllogictest/insert-into-select-source-error.slt index 573b6de6b446b..a2916909c534c 100644 --- a/test/sqllogictest/insert-into-select-source-error.slt +++ b/test/sqllogictest/insert-into-select-source-error.slt @@ -7,9 +7,9 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -# Regression test: INSERT INTO ... SELECT ... cannot transitively refer to sources. -# The selection may refer to views/materialized views, but the transitive -# dependencies must not include sources or source-export tables. +# Regression test: read-then-write selections must resolve to writable user tables. +# The selection may refer to views/materialized views, but its relation leaves +# cannot be sources or source-backed tables. # # Simple (employee, salary, type) example to reproduce the error. @@ -35,16 +35,16 @@ simple INSERT INTO salary_exceptions (employee, salary, type) SELECT 'Alice', salary, type FROM emp_view LIMIT 1 ---- -db error: ERROR: invalid selection: operation may only (transitively) refer to non-source, non-system tables -DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; the selection may refer to views and materialized views, but transitive dependencies must not include sources or source-export tables +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; every relation leaf in the selection must be a writable user table # Same error when selecting directly from source-export table simple INSERT INTO salary_exceptions (employee, salary, type) SELECT 'Bob', counter, 'part_time' FROM employees_from_src LIMIT 1 ---- -db error: ERROR: invalid selection: operation may only (transitively) refer to non-source, non-system tables -DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; the selection may refer to views and materialized views, but transitive dependencies must not include sources or source-export tables +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; every relation leaf in the selection must be a writable user table # Same error when selecting directly from the source. Use a derived table to # alias the single column from emp_src (SELECT * works; column name does not). @@ -52,22 +52,48 @@ simple INSERT INTO salary_exceptions (employee, salary, type) SELECT 'Dave', f1::int, 'contractor' FROM (SELECT * FROM emp_src LIMIT 1) AS t(f1) ---- -db error: ERROR: invalid selection: operation may only (transitively) refer to non-source, non-system tables -DETAIL: source '"materialize.public.emp_src"' may not be used in this operation; the selection may refer to views and materialized views, but transitive dependencies must not include sources or source-export tables +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source '"materialize.public.emp_src"' may not be used in this operation; every relation leaf in the selection must be a writable user table # Same error: UPDATE with WHERE subquery that transitively refers to source simple UPDATE salary_exceptions SET type = 'updated' WHERE salary IN (SELECT salary FROM emp_view) ---- -db error: ERROR: invalid selection: operation may only (transitively) refer to non-source, non-system tables -DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; the selection may refer to views and materialized views, but transitive dependencies must not include sources or source-export tables +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; every relation leaf in the selection must be a writable user table # Same error: DELETE with WHERE subquery that transitively refers to source simple DELETE FROM salary_exceptions WHERE salary IN (SELECT salary FROM emp_view) ---- -db error: ERROR: invalid selection: operation may only (transitively) refer to non-source, non-system tables -DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; the selection may refer to views and materialized views, but transitive dependencies must not include sources or source-export tables +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; every relation leaf in the selection must be a writable user table + +# A webhook table is source-backed even though it appears in the catalog as a table. +statement ok +CREATE TABLE webhook_events FROM WEBHOOK BODY FORMAT TEXT + +simple +INSERT INTO salary_exceptions (employee, salary, type) +SELECT body, 0, 'webhook' FROM webhook_events +---- +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source-backed table '"materialize.public.webhook_events"' may not be used in this operation; every relation leaf in the selection must be a writable user table + +# A secret referenced by webhook validation is DDL metadata, not a read dependency. +statement ok +CREATE SECRET webhook_secret AS 'secret' + +statement ok +CREATE TABLE authenticated_webhook_events FROM WEBHOOK BODY FORMAT TEXT + CHECK (WITH (BODY, SECRET webhook_secret) body = webhook_secret) + +simple +INSERT INTO salary_exceptions (employee, salary, type) +SELECT body, 0, 'webhook' FROM authenticated_webhook_events +---- +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source-backed table '"materialize.public.authenticated_webhook_events"' may not be used in this operation; every relation leaf in the selection must be a writable user table # Set up a plain old table and do the same thing, it should now work # Valid: INSERT INTO ... SELECT ... from plain table only diff --git a/test/sqllogictest/mz_catalog_server_index_accounting.slt b/test/sqllogictest/mz_catalog_server_index_accounting.slt index 75363db62c437..1cc1103fdf6bc 100644 --- a/test/sqllogictest/mz_catalog_server_index_accounting.slt +++ b/test/sqllogictest/mz_catalog_server_index_accounting.slt @@ -84,7 +84,7 @@ mz_message_batch_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_ba mz_message_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_metric_sinks_ind CREATE␠INDEX␠"mz_metric_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s545␠AS␠"mz_internal"."mz_metric_sinks"]␠("id") -mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s841␠AS␠"mz_internal"."mz_notices"]␠("id") +mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s842␠AS␠"mz_internal"."mz_notices"]␠("id") mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s757␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s757␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s755␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") diff --git a/test/sqllogictest/oid.slt b/test/sqllogictest/oid.slt index 5e09c3f0ff0ad..f7a72291710a0 100644 --- a/test/sqllogictest/oid.slt +++ b/test/sqllogictest/oid.slt @@ -1251,3 +1251,4 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17119 mz_builtin_views 17120 mz_metric_sinks 17121 mz_metric_sinks_ind +17122 mz_object_hydration_history diff --git a/test/testdrive/catalog.td b/test/testdrive/catalog.td index 4098fe0a89e88..3e4ad1606a6a3 100644 --- a/test/testdrive/catalog.td +++ b/test/testdrive/catalog.td @@ -610,6 +610,7 @@ mz_materialized_view_refresh_strategies "" mz_object_arrangement_size_history "" mz_object_dependencies "" mz_object_global_ids "" +mz_object_hydration_history "" mz_optimizer_notices "" mz_replacements "" mz_sessions "" @@ -834,7 +835,7 @@ test_table "" # `SHOW TABLES` and `mz_tables` should agree. > SELECT COUNT(*) FROM mz_tables WHERE id LIKE 's%' -29 +30 # There is one entry in mz_indexes for each field_number/expression of the index. > SELECT COUNT(id) FROM mz_indexes WHERE id LIKE 's%' diff --git a/test/testdrive/hydration-status.td b/test/testdrive/hydration-status.td index 9a05529abf3ca..2757c76d3dfb1 100644 --- a/test/testdrive/hydration-status.td +++ b/test/testdrive/hydration-status.td @@ -23,6 +23,15 @@ # introspection relations are asynchronously updated, so DDL commands are not # immediately reflected there. +# Durable hydration history is collected by a background sweep, which visits one +# replica per cycle. Shorten its cadence so episodes land within this test's +# retry budget. +$ postgres-execute connection=mz_system +ALTER SYSTEM SET hydration_history_collection_interval = '1s'; +# Pin retention too: CI randomizes it, and a short period would prune the +# episodes asserted below before the assertions run. +ALTER SYSTEM SET hydration_history_retention_period = '30d'; + > CREATE CLUSTER test REPLICAS (hydrated_test_1 (SIZE 'scale=1,workers=1')) > SET cluster = test @@ -66,6 +75,35 @@ idx hydrated_test_1 true true mv hydrated_test_1 true true mv_const hydrated_test_1 true false +# Durable history for the objects that hydrated on this replica. Collection +# samples current state, so an object that disappears before its replica's turn is +# not recorded at all. +# +# `mv_const` is deliberately not asserted on either way. It does hydrate and its +# log row does briefly carry a complete episode, but the controller then drops the +# collection and `ExportDropped` retracts the row for good. Whether a sweep lands +# in that window is a race, so pinning either outcome makes this file flaky. It is +# an instance of the general limit, that an episode retracted before its replica's +# turn cannot be recorded, rather than a special case. `idx` and `mv` keep their +# dataflows installed, so they are stable. +# +# The recorded interval must be ordered. The stamps are aggregated over a replica's +# workers, so on a multi-process replica the ends can come from different process +# clocks and the interval carries their skew. These replicas are single-process, so +# the ordering is exact here. The coalesce covers a NULL start, which the replica no +# longer produces but the column still permits. +> SELECT o.name, r.name, h.status, + h.installed_at <= h.hydrated_at, + coalesce(h.installed_at <= h.started_at AND h.started_at <= h.hydrated_at, true) + FROM mz_internal.mz_object_hydration_history AS h + JOIN mz_internal.mz_object_global_ids AS g ON g.global_id = h.object_id + JOIN mz_catalog.mz_objects AS o ON o.id = g.id + JOIN mz_catalog.mz_cluster_replicas AS r ON r.id = h.replica_id + WHERE o.name IN ('idx', 'mv') + ORDER BY o.name, r.name; +idx hydrated_test_1 hydrated true true +mv hydrated_test_1 hydrated true true + > SELECT o.name, r.name, h.hydrated FROM mz_internal.mz_hydration_statuses h JOIN mz_cluster_replicas r ON (r.id = h.replica_id) @@ -109,6 +147,18 @@ mv hydrated_test_2 true true mv_const hydrated_test_1 true false mv_const hydrated_test_2 true false +> SELECT o.name, r.name, h.status + FROM mz_internal.mz_object_hydration_history AS h + JOIN mz_internal.mz_object_global_ids AS g ON g.global_id = h.object_id + JOIN mz_catalog.mz_objects AS o ON o.id = g.id + JOIN mz_catalog.mz_cluster_replicas AS r ON r.id = h.replica_id + WHERE o.name IN ('idx', 'mv') + ORDER BY o.name, r.name; +idx hydrated_test_1 hydrated +idx hydrated_test_2 hydrated +mv hydrated_test_1 hydrated +mv hydrated_test_2 hydrated + > SELECT o.name, r.name, h.hydrated FROM mz_internal.mz_hydration_statuses h JOIN mz_cluster_replicas r ON (r.id = h.replica_id) @@ -140,6 +190,15 @@ mv hydrated_test_2 true > DROP CLUSTER REPLICA test.hydrated_test_1 +# History outlives the replica it describes, so dropping a replica retracts +# nothing. Repeated sweeps must also not duplicate an already recorded episode. +> SELECT count(*) + FROM mz_internal.mz_object_hydration_history AS h + JOIN mz_internal.mz_object_global_ids AS g ON g.global_id = h.object_id + JOIN mz_catalog.mz_objects AS o ON o.id = g.id + WHERE o.name IN ('idx', 'mv'); +4 + > SELECT o.name, r.name, h.hydrated, h.hydration_time IS NOT NULL FROM mz_internal.mz_compute_hydration_statuses h JOIN mz_cluster_replicas r ON (r.id = h.replica_id) @@ -341,6 +400,15 @@ snk hydrated_test_4 true > DROP SOURCE src CASCADE > DROP MATERIALIZED VIEW mv +# Retention runs in the same sweep on the catalog server. It retracts aged-out rows +# through the same OCC path, in bounded batches. A zero retention period expires +# everything. +$ postgres-execute connection=mz_system +ALTER SYSTEM SET hydration_history_retention_period = '0s'; + +> SELECT count(*) FROM mz_internal.mz_object_hydration_history; +0 + > SELECT o.name, r.name, h.hydrated FROM mz_internal.mz_hydration_statuses h JOIN mz_cluster_replicas r ON (r.id = h.replica_id) diff --git a/test/workload-replay/objects.txt b/test/workload-replay/objects.txt index 7c14325d961cd..ca116ead25529 100644 --- a/test/workload-replay/objects.txt +++ b/test/workload-replay/objects.txt @@ -615,6 +615,7 @@ mz_object_fully_qualified_names mz_object_global_ids mz_object_history mz_object_history_ind +mz_object_hydration_history mz_object_lifetimes mz_object_lifetimes_ind mz_object_oid_alias diff --git a/test/workload-replay/system_catalog_identifiers.txt b/test/workload-replay/system_catalog_identifiers.txt index bd7b055ae5042..32959e9a05343 100644 --- a/test/workload-replay/system_catalog_identifiers.txt +++ b/test/workload-replay/system_catalog_identifiers.txt @@ -924,6 +924,7 @@ mz_object_fully_qualified_names mz_object_global_ids mz_object_history mz_object_history_ind +mz_object_hydration_history mz_object_lifetimes mz_object_lifetimes_ind mz_object_oid_alias