From c3ad2b4dd8249f986a0d4b298cde820435807bbf Mon Sep 17 00:00:00 2001 From: Peter Larsen Date: Wed, 26 Aug 2026 13:09:46 -0400 Subject: [PATCH 01/11] Hard-cap snapshot partition prefix probes at 5000 per table (#38496) ### Motivation There's a little O(depth*prefixes) algorithm going on. I think it can kind of be viewed as O(probes^2). Anyways, I did a little testing locally that confirmed it would be fast even up to 5k iterations over 5k prefixes and told folks I'd implement a cap. I just realized I never actually set that hard 5k cap. ### Description Implement a hard cap on probes to address some light concerns we had with the way we select prefixes. Co-authored-by: Claude Fable 5 --- src/mysql-util/src/probe.rs | 4 ++++ src/storage-types/src/dyncfgs.rs | 3 ++- src/storage/src/source/mysql/snapshot.rs | 3 ++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/mysql-util/src/probe.rs b/src/mysql-util/src/probe.rs index a4b0bfed8d0ad..0df710547dd95 100644 --- a/src/mysql-util/src/probe.rs +++ b/src/mysql-util/src/probe.rs @@ -25,6 +25,10 @@ pub const MAX_KEY_LENGTH: u32 = 768; /// Probes a string primary key column. Only supports `utf8mb4_bin` against CHAR/VARCHAR /// columns up to 768 characters. Enforcement is deferred to the caller. There may be /// other collations we can support, but we should do more validation. +/// +/// NOTE: The partitioning walk's local bookkeeping scales quadratically with the +/// number of probes. Local benchmarking of 5k probes over 5k entries was fast +/// (under a second of CPU), so exceed that budget at your own risk. pub struct KeyProber<'a, 't> { tx: &'a mut Transaction<'t>, /// Quoted `` `schema`.`table` `` for SQL interpolation. diff --git a/src/storage-types/src/dyncfgs.rs b/src/storage-types/src/dyncfgs.rs index d7bbdc26c77f4..3df7d647302e8 100644 --- a/src/storage-types/src/dyncfgs.rs +++ b/src/storage-types/src/dyncfgs.rs @@ -255,7 +255,8 @@ pub static MYSQL_SOURCE_SNAPSHOT_PARTITION_PROBED_PREFIXES_PER_BILLION_ROWS: Con "mysql_source_snapshot_partition_probed_prefixes_per_billion_rows", 1_000, "Cap on MySQL snapshot PK-prefix partitioning probed prefixes per table, per billion \ - estimated rows; when exhausted, splitting stops early with coarser partition boundaries.", + estimated rows; when exhausted, splitting stops early with coarser partition boundaries. \ + The per-table budget is additionally hard-capped at 5000.", ParameterScope::Replica, ); diff --git a/src/storage/src/source/mysql/snapshot.rs b/src/storage/src/source/mysql/snapshot.rs index fbe6967c40afd..b27176a5ba9b8 100644 --- a/src/storage/src/source/mysql/snapshot.rs +++ b/src/storage/src/source/mysql/snapshot.rs @@ -236,6 +236,7 @@ fn worker_pk_range( const SUPPORTED_PK_COLLATION: &str = "utf8mb4_bin"; const SUPPORTED_PK_CHARSET: &str = "utf8mb4"; const MIN_PROBED_PREFIXES: u64 = 64; +const MAX_PROBED_PREFIXES: u64 = 5_000; /// Partitioning configuration settings bundled to avoid accidental mixing. struct PartitionSettings { @@ -277,7 +278,7 @@ async fn compute_sampled_splits( // that can take hours to snapshot. let max_probed_prefixes = (row_count.saturating_mul(settings.probed_prefixes_per_billion_rows) / 1_000_000_000) - .max(MIN_PROBED_PREFIXES); + .clamp(MIN_PROBED_PREFIXES, MAX_PROBED_PREFIXES); let params = mz_mysql_util::PartitionParams { num_workers: worker_count, estimated_row_count: row_count, From ec639cba038d02d999eaa81ade48c34ce9da312d Mon Sep 17 00:00:00 2001 From: Michael Greenberg Date: Wed, 26 Aug 2026 14:40:02 -0400 Subject: [PATCH 02/11] sql: Transparent `Debug` impl for `StableRow` to avoid noise in debug output (#38494) ### Motivation Closes SQL-646. ### Description The `StableRow` newtype struct derived `Debug`, which added the struct name as chatter. This PR removes it, per @antiguru's review. ### Verification Tests pass. --- src/repr/src/row.rs | 7 ++++++- test/sqllogictest/distinct_arrangements.slt | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/repr/src/row.rs b/src/repr/src/row.rs index 52bfb57665988..c6dc97f6c7e8d 100644 --- a/src/repr/src/row.rs +++ b/src/repr/src/row.rs @@ -361,7 +361,6 @@ impl Ord for Row { /// `SourceData`, and `row.proto` is covered by the buf breaking lint. #[derive( Clone, - Debug, Default, Eq, PartialEq, @@ -387,6 +386,12 @@ impl Deref for StableRow { } } +impl Debug for StableRow { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + mod stable_row_proto { use mz_proto::RustType; use prost::Message; diff --git a/test/sqllogictest/distinct_arrangements.slt b/test/sqllogictest/distinct_arrangements.slt index 522db9cc25a15..05d66f9d9a694 100644 --- a/test/sqllogictest/distinct_arrangements.slt +++ b/test/sqllogictest/distinct_arrangements.slt @@ -978,8 +978,8 @@ Arrange errors ArrangeAccumulable [val: empty] ArrangeAccumulable [val: empty] ArrangeAccumulable [val: empty] -ArrangeBy[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(StableRow(Row{[Int64(2)]})), ReprColumnType { scalar_type: Int64, nullable: false }))]] -ArrangeBy[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(StableRow(Row{[Int64(2)]})), ReprColumnType { scalar_type: Int64, nullable: false }))]]-errors +ArrangeBy[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(Row{[Int64(2)]}), ReprColumnType { scalar_type: Int64, nullable: false }))]] +ArrangeBy[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(Row{[Int64(2)]}), ReprColumnType { scalar_type: Int64, nullable: false }))]]-errors ArrangeBy[[CallUnary(CastInt32ToInt64(CastInt32ToInt64), Column(0, "id"))]] ArrangeBy[[Column(0)]] ArrangeBy[[Column(0, "data")]] @@ -1017,7 +1017,7 @@ Distinct errors Distinct errors Distinct errors Distinct errors -Distinct errors[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(StableRow(Row{[Int64(2)]})), ReprColumnType { scalar_type: Int64, nullable: false }))]] +Distinct errors[[CallBinary(ModInt64(ModInt64), Column(0, "data"), Literal(Ok(Row{[Int64(2)]}), ReprColumnType { scalar_type: Int64, nullable: false }))]] Distinct errors[[Column(0)]] Distinct errors[[Column(0)]] Distinct errors[[]] From 73d9db937e3c79789bb7a96d44244d83df2cba31 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Wed, 26 Aug 2026 21:42:28 +0200 Subject: [PATCH 03/11] doc: design for durable object hydration history (#38344) ### Motivation Materialize exposes current hydration state, but it disappears when a dataflow or replica restarts. A user can tell whether an object is hydrated now, not how long the last hydration took or whether today's is unusually slow. This is the design for recording completed compute-object hydration episodes in a durable table. It lands first so the implementation PRs stacked on it can be reviewed against it. ### Description The document focuses on the parts that are not obvious from the implementation: * Why the collector's subscribe reads the table it writes, which makes collection idempotent across concurrent environmentd processes. * Why the collector aggregates every worker row visible at the OCC read timestamp, requires each visible row to have a non-null `hydrated_at`, and takes the latest finish. Replica restart resets the introspection collection as a unit. Per-process logging clocks can place a worker row beyond the sampled timestamp, and the design explicitly accepts that race rather than depending on configured worker counts. * Why the cutoff and anti-join apply after aggregation. Applying either before it would drop incomplete worker rows and make the completeness check trivially pass. * Why the sweep schedule is stable across restarts but offset from the full environment id, avoiding a fleet-wide burst on every interval boundary, including between regions and ordinals of one organization. * Why durability is best effort, how successive bounded retention batches drain a fixed-cutoff backlog, and how schema evolution preserves rows while shard replacement deliberately restarts the history. * Why the replica-targeted anti-join cannot use a catalog-server index, and why its recurring history import and arrangement cost is accepted. * What sampling a retracting current-state collection cannot record, and what the append-only lifecycle log proposed in #38403 would add later. Scope is limited to successful hydration of indexes and materialized views. Failed episodes, replica-wide episodes, resource peaks, and storage objects remain out of scope because the required signals are not available on `main` today. ### Verification Documentation only. Closes: [SQL-632](https://linear.app/materializeinc/issue/SQL-632) --- ...260817_durable_object_hydration_history.md | 321 ++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 doc/developer/design/20260817_durable_object_hydration_history.md diff --git a/doc/developer/design/20260817_durable_object_hydration_history.md b/doc/developer/design/20260817_durable_object_hydration_history.md new file mode 100644 index 0000000000000..53ed9b7d7de97 --- /dev/null +++ b/doc/developer/design/20260817_durable_object_hydration_history.md @@ -0,0 +1,321 @@ +# Durable Object Hydration History + +## Context + +Materialize exposes current hydration state, but that state disappears when a +dataflow or a replica restarts. A user can tell whether an object is hydrated +now, but not how long the last hydration took, or whether today's is unusually +slow. + +This records completed compute-object hydration episodes in a durable table. It +is the first stage of broader hydration history. Replica-wide episodes, resource +peaks, failed episodes, and storage objects need signals that do not exist yet. + +## Goals + +- Record successful hydration of indexes and materialized views per replica. +- Survive environmentd and replica restarts. +- Stay idempotent across concurrent environmentd processes. +- Leave the values of existing hydration relations untouched. +- Bound storage with configurable retention. + +## Non-Goals + +Each of these is excluded because the signal to do it correctly is missing, not +because it is unwanted: + +- **Failed or canceled episodes.** A replica cannot report its own crash, and the + current-state log retracts an object without recording why. +- **Replica-wide episodes.** Correct boundaries need the object set present at the + transition from hydrated to hydrating, and explicit transitions between those + states. +- **Resource peaks.** The metrics history holds samples, not high-water values. + Presenting samples as peaks would mislead. +- **Sources and sinks.** Storage publishes no equivalent lifecycle timestamps. +- **Failed replicas.** A replica that never finishes hydrating an object records + nothing for it, since only completions are stamped. + +## Design + +Compute already stamps three replica-side timestamps per export and worker in +`mz_introspection.mz_compute_hydration_times_per_worker`: `installed_at` when the +dataflow is installed, `started_at` when it is unsuspended, and `hydrated_at` when +the output frontier passes the as-of. See +`doc/developer/design/20260817_compute_hydration_timestamps.md`. This design adds +the history table and the sweep that writes those timestamps down. + +A coordinator task visits one user replica per interval, in a rotation, skipping +replicas with introspection disabled, whose logs are installed but never populated. +It installs an internal subscribe on that replica which aggregates every worker's +completed rows, anti-joins them against the history table, and writes the missing +ones through the timestamped OCC read-then-write path. + +Two dyncfgs control it. `hydration_history_collection_interval` sets the sweep +cadence and disables collection at zero, which is the production default. +`hydration_history_retention_period` bounds how long rows live, defaulting to 30 +days. + +An *episode* throughout this document is one such row: one dataflow's hydration on +one replica, from installation to hydrated, recorded once. + +## History Table + +```text +mz_internal.mz_object_hydration_history + object_id text not null + cluster_id text not null + replica_id text not null + installed_at timestamptz not null + started_at timestamptz null + hydrated_at timestamptz null + status text not null +``` + +An episode is identified by `(object_id, replica_id, installed_at)`. The +installation time is replica-stamped and stable across an environmentd restart. The +nullability of each column matches the compute log the values come from, so +`installed_at` is not null there and here, while the two later stamps are nullable. + +`object_id` is named the way the other hydration relations name it, because to a +user these are objects that exist somewhere rather than exports of a dataflow. The +value is the id compute reports for the object's dataflow, and one catalog item can +own several dataflows at once, so consumers reach a catalog object through +`mz_internal.mz_object_global_ids` rather than joining `mz_objects` directly. + +Only terminal `hydrated` rows are written today, so every row has a finish time. +The nullable columns and `status` reserve a compatible shape for canceled or +unfinished episodes once those become observable. Retention keys off `hydrated_at`, +so an unfinished episode needs a second age basis before it can be recorded at all. + +The table lives in `mz_internal` to mark it unstable while it settles. + +No index initially, and not a retained-metrics object either, both because of size. +The collector does anti-join by `(object_id, replica_id, installed_at)`, but its +subscribe runs on the selected user replica. An index arranged on the catalog +server cannot serve that dataflow, so it would pin the whole table without removing +the collector's recurring import and arrangement cost. That cost grows with retained +history and is accepted by this sampling design. Adding an index later is a +migration we can do when a query that can use it needs one. The retained-metrics +flag would independently pin a 30 day compaction window, so it stays off too. + +## Why concurrent writers converge + +The subscribe reads the history table it is about to write. That is what makes the +write idempotent across processes, and it is the reason the read expression looks +redundant. + +Two environmentd processes computing the same missing row at frontier `T` both +submit a timestamped write for `T`. One commits and advances the table upper. The +other is told the timestamp passed, waits for its own subscribe to progress, sees +the committed row appear, and its anti-join retracts the candidate. There is +nothing left to write. + +The losing writer never retries stale diffs at the next eligible timestamp. It +only retries with state it has observed to be valid at the frontier it is writing +at. That same property is what makes a trailing replica record nothing, which the +last section covers. + +## Aggregating a replica's workers + +Collection aggregates every worker's row for an export and records nothing until all +of them have hydrated, taking `max(hydrated_at)` and `min` of the two start stamps. + +One worker would be enough for an index, whose output frontier is the arrangement +upper and therefore moves only as timely's dataflow-wide progress allows. It is not +enough for a materialized view. Its output frontier also depends on the sink write +frontier, and only the sink's single active worker, `hash(sink_id) % workers`, +tracks the shard upper. Every other worker stamps at compute completion, before the +snapshot is durable, and the sink buffers that snapshot until computation finishes, +so the gap is the whole write. Reading one worker would mean `hydrated_at` said +"durable" for some objects and "computed" for others, decided by a hash. The +aggregate gives one meaning for every object, and it is what +`mz_compute_hydration_times` already does, so the durable history and the live +signal agree. + +Completeness uses no separately configured worker count. The replica owns this +introspection collection and recreates it as a unit on restart, so a snapshot cannot +combine rows from before and after a restart. Installation adds one row per +`(export_id, worker_id)` with a null `hydrated_at`, and +`count(*) = count(hydrated_at)` means every row visible at the read timestamp has +finished. An object still hydrating is skipped and picked up by a later sweep. The +cutoff and the anti-join apply to the aggregate's output, since as `WHERE` clauses +either one would drop visible unfinished rows and make that check trivially true. + +Each process's logging clock also determines the logical timestamps of its updates. +A process whose clock is ahead can therefore place its row beyond the read timestamp +while the other rows are complete below it. The collector deliberately accepts this +sampling race rather than depending on replica configuration for an expected worker +count. In that case the durable finish can precede the latest worker's finish. This +is separate from restart behavior, which discards the replica collection as a unit. + +Compute is adding an append-only lifecycle log for the same stages, currently +proposed in #38403: one row per export, worker and event, with a reason and the +dataflow's as-of. It leaves `mz_compute_hydration_times_per_worker` alone, so +everything above keeps its meaning and nothing recorded here is relabelled. + +That log is the signal this design says is missing, and moving onto it later buys +three things at once: stages that mean the same thing on every worker, a reason +that distinguishes a replacement waiting for cutover from an index that will never +write, and the as-of, without which an interval says nothing about how much work +was done, since a replacement with a far behind as-of does far more of it in the +same wall-clock time. Recording should keep gating on hydration when that happens. +A replacement runs read-only and does not write until cutover, so gating on a write +stage would never record the hydration a deployment most wants to measure, and +would wait forever on a rollback. + +Two consequences. An interval can span workers in different processes, each +anchoring its logging clock at its own `SystemTime`, so skew inflates a duration. +Nothing rejects a row for looking inconsistent, deliberately, because a guard on +cross-worker stamps rejects complete episodes permanently once the log values +settle. And `started_at` equal to `installed_at` means no start was observed rather +than an immediate start, since the log substitutes the installation time. An +import-free dataflow is never suspended, so that is the common case. + +## What is and is not recorded + +Collection samples current state rather than consuming events, so it writes down +whatever the replica collection holds when that replica's turn comes around. The +log retracts an export's row when the dataflow goes away, so a short-lived object, +or one dropped before its turn, leaves no trace. Nothing wrong is recorded, it is +simply absent. + +Making these durable requires compute to emit hydration transitions into an +append-only collection that survives until an observer acknowledges them. Until +then, best effort is the honest description of a sampler. + +## Retention + +Retention is another OCC mutation: it subscribes to rows older than the cutoff and +writes their retractions at the observed frontier. Collection applies the same +cutoff, so a still-live log row cannot resurrect an episode retention just +retracted. + +Retention deletes successive bounded batches until one is not full. The fixed +cutoff makes the eligible set finite, and collection refuses to insert rows behind +that cutoff, so a successful sweep drains the backlog even when more than one batch +expires at once. The bound is not a nicety. The OCC path refuses a selection larger +than `max_result_size` before submitting any write, so one unbounded delete over a +large backlog would fail identically forever and never shrink the table. The bound +has to sit inside a derived table, because a top-level `LIMIT` lands in the plan's +`RowSetFinishing`, which the OCC path deliberately discards, and the delete would be +silently unbounded again. + +Retention runs on the catalog server, so it keeps working when there are no user +replicas at all, and it runs even when that sweep's collection failed. A +crash-looping replica must not be able to stop the table from shrinking. The +dependency does not run the other way: a catalog server without a replica skips +retention and leaves collection running. + +Disabling collection also suspends retention. The alternative is an always-on +subscribe in the default configuration, where the table is empty and there is +nothing to retain. Rows already collected are therefore kept while collection is +off. + +## Durability is best effort + +Builtin tables are reset at bootstrap and re-sharded on a forced schema migration. +This table is exempt from both, because a sampled history cannot be rebuilt from +anything else. Those two exemptions are the whole promise. + +We do not promise to carry the contents across every future version. A shard +replacement clears the table, and that is an acceptable trade: what has value here +is the distribution the table accumulates, and starting over costs one retention +period, while freezing the schema to protect it costs every later improvement. +Schema *evolution* keeps the rows, so an additive column costs nothing. + +Giving the exemption up is meant to be deliberate. An assert fires if a replacement +step ever names this table, and the comment there says to remove the assert and the +exemption together and to note in the release notes that the history restarts. + +## Scheduling and isolation + +Sweeps never overlap and each visits one replica, which bounds compute load and +keeps the collector from contending with itself in the serialized timestamped-write +path. Fires align to interval boundaries and each sleep is capped, so lowering a +long interval takes effect within the cap rather than after the old interval +elapses. The grid is offset by an amount seeded from the full environment id, since +the interval is one fleet-wide setting and an unshifted grid would have every +environment sweep at the same instant. Using the full id also separates regions and +ordinals belonging to one organization. + +**Background mutations take no OCC write permit.** The permits are one semaphore +shared by every read-then-write in the process, not one per table. A session's wait +is bounded by its statement timeout, a sweep's is not, and a sweep's subscribe has +to hydrate a dataflow on a user replica first, so holding a permit would let +background sampling stall user DML on any table for as long as that takes. Being +single-flight, skipping the permit adds at most one concurrent read-then-write. + +**The sweep is aborted when the coordinator is dropped.** It holds no `Client`, so +nothing else stops it outliving the coordinator, and the teardown that follows drops +the timestamp oracle's worker task. A sweep still running would then read a +timestamp from a dead oracle and panic, so the coordinator owns the task handle. + +**Each mutation's timeout is deliberately generous.** Even a mutation with nothing +to write waits for its read to linearize, which can take a full +`default_timestamp_interval`, a parameter with no upper bound. A tighter bound would +let a large timestamp interval starve retention permanently. + +Replica drop, cluster drop, dependency replacement, replica failure, and timeout all +skip the attempt, and a later sweep recomputes from current state. Read-only +generations do nothing, which the next section covers. + +## Upgrades record late, not never + +A read-only generation writes nothing, and a 0dt upgrade hydrates the incoming +replicas while it is still read-only. That looks like it drops exactly the window +worth measuring, since that is when most objects hydrate at once. + +It does not, because collection samples current state rather than events. The +stamps live in the replica's own log, the replicas keep running across promotion, +and the first sweep after promotion writes those episodes with the replica's +original `installed_at` and `hydrated_at`. The history is written late, the +timestamps are not. + +What is genuinely lost is an episode whose dataflow goes away before that first +post-promotion sweep, which is the general sampling limit rather than anything +specific to upgrades. + +## Known limitation: a trailing replica records nothing + +The write timestamp comes from the timeline's oracle, and the subscribe's frontier +certifies that the loop has a complete view below it. The write happens once that +frontier reaches the target. One of the subscribe's inputs is a replica-local +introspection log whose frontier the *replica* advances from its own clock, rounded +up to its introspection interval. + +So a replica whose clock trails environmentd by more than one introspection +interval produces a frontier that keeps sitting below the target the oracle hands +out. The mutation waits, and the sweep's own timeout ends it. The consequence is +bounded. Nothing wrong is recorded, that replica records nothing at all, and the +sweep stretches while it waits, which delays others in the rotation. It resolves +itself when the clocks converge. + +Accepted for now: skew between processes is normally milliseconds while the +tolerance is a whole introspection interval, and the failure mode is waiting rather +than wrong data. Because the symptom is otherwise hard to attribute, a step that +times out logs that the replica's introspection frontier may be trailing. + +## Rollout + +Collection is disabled by default, and the interval is runtime configurable, so +enabling it needs no restart. The plan is to enable it in CI when this merges, then +in staging, then in production, a week apart, so each step has a week of real +traffic behind it before the next. + +CI enables it at a 60 second interval through the mzcompose configuration, which is +what exercises the hydration, restart, retention, and catalog tests. It stays off in +the sqllogictest runner defaults, against the usual preference for enabling new +paths in tests: the collector installs subscribes and writes a builtin table, while +those runs assert on catalog contents and plans. + +## Future Work + +- Record installation and start before completion, then finalize canceled and + failed episodes by joining replica lifecycle events. +- Define a replica episode state machine from the object set present at the + hydrated-to-hydrating transition. +- Publish resettable per-process high-water values for RAM, swap, and scratch disk, + and define replica aggregation without pretending sampled maxima are simultaneous + peaks. +- Give storage objects equivalent lifecycle timestamps. +- Build replica history and progress views on those signals. From 3a822d8457ca7afc7a1e0511bd708b6dda6390ce Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Wed, 26 Aug 2026 21:42:28 +0200 Subject: [PATCH 04/11] catalog: add mz_object_hydration_history (#38346) Stacked on #38344. ### Motivation Adds the durable surface that hydration episodes are recorded into. The collector arrives in the next PR, so this rung can be reviewed as a catalog change. Design doc: [`20260817_durable_object_hydration_history.md`](https://github.com/MaterializeInc/materialize/blob/aljoscha/hydration-01-design/doc/developer/design/20260817_durable_object_hydration_history.md) ### Description Adds `mz_internal.mz_object_hydration_history`, with no index. The table starts in `mz_internal` to mark it unstable while its shape and semantics settle. An episode is identified operationally by `(object_id, replica_id, installed_at)`. `object_id` holds the dataflow id reported by the replica, with `SemanticType::GlobalId`, so consumers resolve it through `mz_internal.mz_object_global_ids` rather than joining `mz_objects` directly. `installed_at` is replica-stamped and stable across environmentd restarts. The identity is not declared as a relation key. A key is an optimizer contract, while this best-effort sampler enforces uniqueness through its anti-join. A duplicate must remain visible rather than letting the optimizer silently elide it. There is no index initially. The collector anti-joins by history identity, but its subscribe runs on the selected user replica and cannot use an index arranged on the catalog server. Such an index would pin the whole table without removing the collector's recurring import and arrangement cost. Adding one later also changes the `mz_indexes` fingerprint, so it requires a `MigrationStep::replacement` for `mz_indexes` pinned to the then-current development version. Durability is best effort. The table is exempt from the bootstrap reset and from forced **replacement**, because either would discard sampled history that cannot be rebuilt. Schema evolution is allowed and keeps the shard and rows. A replacement step naming this table trips an assert in `validate_migration_steps`; giving the exemption up is meant to be deliberate and release-noted. A focused policy test pins evolution participation and replacement exclusion. The bootstrap-reset filter is converted from name/schema comparisons to a set of resolved ids, which is clearer now that several history tables are exempt. The ontology link for `replica_id` uses `SemanticType::ReplicaId`, matching both the column and its target. User documentation states that 30 days is the retention default, disabling collection pauses retention, and dropped dataflow IDs may no longer resolve through `mz_object_global_ids`. ### Verification Catalog goldens cover builtin counts, OIDs, index accounting, `information_schema`, and the autogenerated `mz_internal` relation spec. The catalog migration tests exercise the migration path. This release will add the `mz_internal.mz_object_hydration_history` table. Ref: [SQL-644](https://linear.app/materializeinc/issue/SQL-644) --- .../reference/system-catalog/mz_internal.md | 27 +++++ .../catalog/open/builtin_schema_migration.rs | 58 ++++++++-- .../open/builtin_schema_migration_tests.rs | 14 +++ src/adapter/src/coord.rs | 33 +++--- src/catalog/src/builtin.rs | 9 ++ src/catalog/src/builtin/mz_internal.rs | 100 ++++++++++++++++++ src/pgrepr-consts/src/oid.rs | 1 + .../autogenerated/mz_internal.slt | 12 +++ test/sqllogictest/catalog_server_explain.slt | 12 +-- .../information_schema_tables.slt | 4 + .../mz_catalog_server_index_accounting.slt | 2 +- test/sqllogictest/oid.slt | 1 + test/testdrive/catalog.td | 3 +- test/workload-replay/objects.txt | 1 + .../system_catalog_identifiers.txt | 1 + 15 files changed, 238 insertions(+), 40 deletions(-) diff --git a/doc/user/content/reference/system-catalog/mz_internal.md b/doc/user/content/reference/system-catalog/mz_internal.md index 8067b94a54fc0..8d6f3347be034 100644 --- a/doc/user/content/reference/system-catalog/mz_internal.md +++ b/doc/user/content/reference/system-catalog/mz_internal.md @@ -704,6 +704,33 @@ The `mz_object_history` view enriches the [`mz_catalog.mz_objects`](/reference/s | `created_at` | [`timestamp with time zone`] | Wall-clock timestamp of when the object was created. `NULL` for built in system objects. | | `dropped_at` | [`timestamp with time zone`] | Wall-clock timestamp of when the object was dropped. `NULL` for built in system objects or if the object hasn't been dropped. | +## `mz_object_hydration_history` + +The `mz_object_hydration_history` table records completed hydration of indexes and +materialized views, with one row for each time a dataflow hydrated on a replica. +By default, rows are retained for 30 days while collection is enabled. Disabling +collection also suspends retention, so existing rows remain until collection is +enabled again. `object_id`, `cluster_id`, and `replica_id` may name objects that no +longer exist. + +Recording is best effort. Only successful hydration is recorded, an episode can be +missed if the object or its replica goes away before the episode is recorded, and a +schema change to this table in a future release may clear its contents. On a +multi-process replica, timestamps come from process-local logging clocks and include +their clock skew. A process whose clock is ahead can be absent at the sampled +logical timestamp, so the recorded finish can precede the latest process's finish. + + +| Field | Type | Meaning | +| -------------- | ---------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `object_id` | [`text`] | The ID of the object's dataflow, as reported by the replica. Join `mz_internal.mz_object_global_ids` to reach the index or materialized view while that mapping exists. Dropping the dataflow retracts the mapping, so historical IDs may no longer resolve. | +| `cluster_id` | [`text`] | The ID of the object's cluster. | +| `replica_id` | [`text`] | The ID of the cluster replica. May name a replica that no longer exists. | +| `installed_at` | [`timestamp with time zone`] | When the object's dataflow was installed on the replica. | +| `started_at` | [`timestamp with time zone`] | When hydration work began, or `NULL` if the replica reported none. A replica that observed no start reports the installation time instead, so a zero interval between the two does not mean the dataflow started immediately. | +| `hydrated_at` | [`timestamp with time zone`] | When hydration finished. | +| `status` | [`text`] | The terminal status. Currently always `hydrated`. | + ## `mz_object_transitive_dependencies` The `mz_object_transitive_dependencies` view describes the transitive dependency structure between diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index a35ff177a7326..394a479d54202 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -38,6 +38,7 @@ use mz_build_info::{BuildInfo, DUMMY_BUILD_INFO}; use mz_catalog::builtin::{ BUILTIN_LOOKUP, Builtin, Fingerprint, MZ_CATALOG_RAW, MZ_CATALOG_RAW_DESCRIPTION, MZ_CLUSTER_REPLICA_FRONTIERS_DESCRIPTION, MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY_DESCRIPTION, + MZ_OBJECT_HYDRATION_HISTORY, MZ_OBJECT_HYDRATION_HISTORY_DESCRIPTION, MZ_STORAGE_USAGE_BY_SHARD, MZ_STORAGE_USAGE_BY_SHARD_DESCRIPTION, RUNTIME_ALTERABLE_FINGERPRINT_SENTINEL, }; @@ -662,6 +663,34 @@ struct Migration { config: BuiltinItemMigrationConfig, } +/// Whether `builtin` participates in a forced migration using `mechanism`. +fn participates_in_forced_migration( + builtin: &Builtin, + mechanism: Mechanism, +) -> bool { + use Builtin::*; + match builtin { + // A forced replacement allocates a fresh shard, which discards the + // table's contents. Exclude the tables whose contents are the point: + // storage usage is retained for billing, and hydration history cannot + // be rebuilt from any other source. + // + // Hydration history takes part in a forced `Evolution`, which keeps the + // rows. It has to: dev upgrades force one for every object, and a table + // left out of the plan never gets its new schema registered, so + // `update_fingerprints` panics at open as soon as the desc changes. See + // the tripwire in `validate_migration_steps` for how to give up the + // replacement exemption deliberately. + Table(table) => { + **table != *MZ_STORAGE_USAGE_BY_SHARD + && (mechanism != Mechanism::Replacement || **table != *MZ_OBJECT_HYDRATION_HISTORY) + } + MaterializedView(..) => true, + Source(source) => **source != *MZ_CATALOG_RAW, + Log(..) | View(..) | Type(..) | Func(..) | Index(..) | Connection(..) => false, + } +} + impl Migration { async fn run(self, steps: &[MigrationStep]) -> anyhow::Result { info!( @@ -758,6 +787,23 @@ impl Migration { "mz_object_arrangement_size_history cannot be migrated or else the table will be truncated" ); + // Unlike the two tables above, there is no correctness hazard here, a + // truncation would only lose history. This is a tripwire so that the + // loss is chosen rather than stumbled into: if a schema change to + // this table is worth clearing it for, remove this assert along with + // the exemption in `plan_forced_migration`, and say in the release + // notes that the history restarts. + // + // Only `Replacement` loses the rows. An `Evolution` keeps the shard + // and its contents, so it is the path a schema change to this table + // should take and we let it through. + if step.mechanism == Mechanism::Replacement { + assert_ne!( + &*MZ_OBJECT_HYDRATION_HISTORY_DESCRIPTION, object, + "replacing mz_object_hydration_history clears it, see the comment above" + ); + } + // `mz_catalog_raw` cannot be migrated because it contains the durable catalog and it // wouldn't be very durable if we allowed it to be truncated. assert_ne!( @@ -829,17 +875,7 @@ impl Migration { // added in this version; the leader will allocate their shards during bootstrap, and // there is nothing to evolve or replace. .filter(|(_, info)| info.shard_id.is_some()) - .filter(|(_, info)| { - use Builtin::*; - match info.builtin { - // Filter out the 'mz_storage_usage_by_shard' table since we need to retain - // that info for billing purposes. - Table(table) => **table != *MZ_STORAGE_USAGE_BY_SHARD, - MaterializedView(..) => true, - Source(source) => **source != *MZ_CATALOG_RAW, - Log(..) | View(..) | Type(..) | Func(..) | Index(..) | Connection(..) => false, - } - }) + .filter(|(_, info)| participates_in_forced_migration(info.builtin, mechanism)) .map(|(object, _)| object.clone()) .collect(); diff --git a/src/adapter/src/catalog/open/builtin_schema_migration_tests.rs b/src/adapter/src/catalog/open/builtin_schema_migration_tests.rs index 3356a60dfd0ed..2d168684d5f3f 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration_tests.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration_tests.rs @@ -27,6 +27,20 @@ use tracing::info; use super::*; +#[mz_ore::test] +fn hydration_history_forced_migration_policy() { + let hydration_history = Builtin::Table(&*MZ_OBJECT_HYDRATION_HISTORY); + + assert!(participates_in_forced_migration( + &hydration_history, + Mechanism::Evolution + )); + assert!(!participates_in_forced_migration( + &hydration_history, + Mechanism::Replacement + )); +} + #[test] // allow(test-attribute) #[cfg_attr(miri, ignore)] // too slow fn test_builtin_schema_migration() { diff --git a/src/adapter/src/coord.rs b/src/adapter/src/coord.rs index 0018adb4471d7..03420a429e248 100644 --- a/src/adapter/src/coord.rs +++ b/src/adapter/src/coord.rs @@ -99,7 +99,8 @@ use mz_adapter_types::dyncfgs::{ use mz_auth::password::Password; use mz_build_info::BuildInfo; use mz_catalog::builtin::{ - BUILTINS, BUILTINS_STATIC, MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY, MZ_STORAGE_USAGE_BY_SHARD, + BUILTINS, BUILTINS_STATIC, MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY, MZ_OBJECT_HYDRATION_HISTORY, + MZ_STORAGE_USAGE_BY_SHARD, }; use mz_catalog::config::{AwsPrincipalContext, BuiltinItemMigrationConfig, ClusterReplicaSizeMap}; use mz_catalog::durable::OpenableDurableCatalogState; @@ -150,7 +151,7 @@ use mz_secrets::cache::CachingSecretsReader; use mz_secrets::{SecretsController, SecretsReader}; use mz_sql::ast::{Raw, Statement}; use mz_sql::catalog::{CatalogCluster, EnvironmentId}; -use mz_sql::names::{QualifiedItemName, ResolvedIds, SchemaSpecifier}; +use mz_sql::names::{QualifiedItemName, ResolvedIds}; use mz_sql::optimizer_metrics::OptimizerMetrics; use mz_sql::plan::{ self, AlterSinkPlan, ConnectionDetails, CreateConnectionPlan, HirRelationExpr, @@ -3169,29 +3170,19 @@ impl Coordinator { debug!("coordinator init: resetting system tables"); let read_ts = self.get_local_read_ts().await; - // Filter out tables whose contents must survive restarts: - // 'mz_storage_usage_by_shard' for billing, and - // 'mz_object_arrangement_size_history', which accumulates history that - // is pruned by its own retention period instead. - let mz_storage_usage_by_shard_schema: SchemaSpecifier = self - .catalog() - .resolve_system_schema(MZ_STORAGE_USAGE_BY_SHARD.schema) - .into(); - let arrangement_size_history_schema: SchemaSpecifier = self - .catalog() - .resolve_system_schema(MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY.schema) - .into(); - let is_retained_across_restarts = |meta: &TableMetadata| -> bool { - (meta.name.item == MZ_STORAGE_USAGE_BY_SHARD.name - && meta.name.qualifiers.schema_spec == mz_storage_usage_by_shard_schema) - || (meta.name.item == MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY.name - && meta.name.qualifiers.schema_spec == arrangement_size_history_schema) - }; + let retained_across_restarts = BTreeSet::from([ + self.catalog() + .resolve_builtin_table(&MZ_STORAGE_USAGE_BY_SHARD), + self.catalog() + .resolve_builtin_table(&MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY), + self.catalog() + .resolve_builtin_table(&MZ_OBJECT_HYDRATION_HISTORY), + ]); let mut retraction_tasks = Vec::new(); let system_tables: Vec<_> = table_metas .iter() - .filter(|meta| meta.id.is_system() && !is_retained_across_restarts(meta)) + .filter(|meta| meta.id.is_system() && !retained_across_restarts.contains(&meta.id)) .collect(); for system_table in system_tables { diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs index 14623facb0d8e..0a4c6d914c209 100644 --- a/src/catalog/src/builtin.rs +++ b/src/catalog/src/builtin.rs @@ -837,6 +837,14 @@ pub static MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY_DESCRIPTION: LazyLock = + LazyLock::new(|| SystemObjectDescription { + schema_name: MZ_OBJECT_HYDRATION_HISTORY.schema.to_string(), + object_type: CatalogItemType::Table, + object_name: MZ_OBJECT_HYDRATION_HISTORY.name.to_string(), + }); + /// Identifies [`MZ_CLUSTER_REPLICA_FRONTIERS`] for the schema-migration guard in /// `builtin_schema_migration.rs`, which forbids migrating this source because the 0dt /// caught-up gate reads the leader's shard for it to learn the live frontiers. @@ -1477,6 +1485,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::View(&MZ_INDEX_ADVICE), Builtin::View(&MZ_MCP_DATA_PRODUCTS), Builtin::View(&MZ_MCP_DATA_PRODUCT_DETAILS), + Builtin::Table(&MZ_OBJECT_HYDRATION_HISTORY), ]; builtin_items.extend(notice::builtins()); diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs index d0546454025f0..463df5d4168cd 100644 --- a/src/catalog/src/builtin/mz_internal.rs +++ b/src/catalog/src/builtin/mz_internal.rs @@ -5142,6 +5142,106 @@ pub static MZ_OBJECT_ARRANGEMENT_SIZE_HISTORY_TS_IND: LazyLock = is_retained_metrics_object: true, }); +/// Completed hydration episodes, one row per object, replica, and installation. +/// +/// Exempt from the bootstrap reset and from forced shard replacement, since the +/// contents cannot be rebuilt from anything else. Schema evolution keeps them and +/// applies normally. Clearing them for a schema change is still allowed, see the +/// tripwire in `validate_migration_steps`. +pub static MZ_OBJECT_HYDRATION_HISTORY: LazyLock = LazyLock::new(|| BuiltinTable { + name: "mz_object_hydration_history", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::TABLE_MZ_OBJECT_HYDRATION_HISTORY_OID, + desc: RelationDesc::builder() + .with_column("object_id", SqlScalarType::String.nullable(false)) + .with_column("cluster_id", SqlScalarType::String.nullable(false)) + .with_column("replica_id", SqlScalarType::String.nullable(false)) + .with_column( + "installed_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column( + "started_at", + SqlScalarType::TimestampTz { precision: None }.nullable(true), + ) + .with_column( + "hydrated_at", + SqlScalarType::TimestampTz { precision: None }.nullable(true), + ) + .with_column("status", SqlScalarType::String.nullable(false)) + .finish(), + column_comments: BTreeMap::from_iter([ + ( + "object_id", + "The ID of the object's dataflow, as reported by the replica. Join `mz_internal.mz_object_global_ids` to reach the index or materialized view while that mapping exists. Dropping the dataflow retracts the mapping, so historical IDs may no longer resolve.", + ), + ("cluster_id", "The ID of the object's cluster."), + ( + "replica_id", + "The ID of the cluster replica. May name a replica that no longer exists.", + ), + ( + "installed_at", + "When the object's dataflow was installed on the replica.", + ), + ( + "started_at", + "When hydration work began, or `NULL` if the replica reported none. A replica that observed no start reports the installation time instead, so a zero interval between the two does not mean the dataflow started immediately.", + ), + ("hydrated_at", "When hydration finished."), + ( + "status", + "The terminal status. Currently always `hydrated`.", + ), + ]), + // Not a retained-metrics object: that would pin a 30 day compaction window, + // and our history lives in the rows, which the retention sweep retracts on + // its own schedule. Nothing reads this table at an old timestamp. + is_retained_metrics_object: false, + access: vec![PUBLIC_SELECT], + ontology: Some(Ontology { + entity_name: "object_hydration_event", + description: "Completed hydration of an index or materialized view on a replica", + // NOTE: These references outlive what they point at. A row deliberately + // survives the object and the replica it describes, so resolving one + // against the catalog can come up empty. + links: &const { + [ + OntologyLink { + name: "hydration_of_dataflow", + target: "object_global_id", + properties: LinkProperties::fk_typed( + "object_id", + "global_id", + Cardinality::ManyToOne, + mz_repr::SemanticType::GlobalId, + ), + }, + OntologyLink { + name: "hydrated_on_cluster", + target: "cluster", + properties: LinkProperties::fk("cluster_id", "id", Cardinality::ManyToOne), + }, + OntologyLink { + name: "hydrated_on_replica", + target: "replica", + properties: LinkProperties::fk_typed( + "replica_id", + "id", + Cardinality::ManyToOne, + mz_repr::SemanticType::ReplicaId, + ), + }, + ] + }, + column_semantic_types: &[ + ("object_id", SemanticType::GlobalId), + ("cluster_id", SemanticType::ClusterId), + ("replica_id", SemanticType::ReplicaId), + ], + }), +}); + pub static MZ_COMPUTE_HYDRATION_STATUSES: LazyLock = LazyLock::new(|| BuiltinView { name: "mz_compute_hydration_statuses", schema: MZ_INTERNAL_SCHEMA, diff --git a/src/pgrepr-consts/src/oid.rs b/src/pgrepr-consts/src/oid.rs index 7177523ef7d1e..c2f7d677821a6 100644 --- a/src/pgrepr-consts/src/oid.rs +++ b/src/pgrepr-consts/src/oid.rs @@ -828,3 +828,4 @@ pub const VIEW_MZ_BUILTIN_TABLES_OID: u32 = 17118; pub const VIEW_MZ_BUILTIN_VIEWS_OID: u32 = 17119; pub const MV_MZ_METRIC_SINKS_OID: u32 = 17120; pub const INDEX_MZ_METRIC_SINKS_IND_OID: u32 = 17121; +pub const TABLE_MZ_OBJECT_HYDRATION_HISTORY_OID: u32 = 17122; diff --git a/test/sqllogictest/autogenerated/mz_internal.slt b/test/sqllogictest/autogenerated/mz_internal.slt index a7cda93eb235b..47c55bb3d6813 100644 --- a/test/sqllogictest/autogenerated/mz_internal.slt +++ b/test/sqllogictest/autogenerated/mz_internal.slt @@ -388,6 +388,17 @@ object_type text The␠type␠of␠the␠object:␠one␠of␠`table`,␠`sour created_at timestamp␠with␠time␠zone Wall-clock␠timestamp␠of␠when␠the␠object␠was␠created.␠`NULL`␠for␠built␠in␠system␠objects. dropped_at timestamp␠with␠time␠zone Wall-clock␠timestamp␠of␠when␠the␠object␠was␠dropped.␠`NULL`␠for␠built␠in␠system␠objects␠or␠if␠the␠object␠hasn't␠been␠dropped. +query TTT +SELECT name, type, comment FROM objects WHERE schema = 'mz_internal' AND object = 'mz_object_hydration_history' ORDER BY position +---- +object_id text The␠ID␠of␠the␠object's␠dataflow,␠as␠reported␠by␠the␠replica.␠Join␠`mz_internal.mz_object_global_ids`␠to␠reach␠the␠index␠or␠materialized␠view␠while␠that␠mapping␠exists.␠Dropping␠the␠dataflow␠retracts␠the␠mapping,␠so␠historical␠IDs␠may␠no␠longer␠resolve. +cluster_id text The␠ID␠of␠the␠object's␠cluster. +replica_id text The␠ID␠of␠the␠cluster␠replica.␠May␠name␠a␠replica␠that␠no␠longer␠exists. +installed_at timestamp␠with␠time␠zone When␠the␠object's␠dataflow␠was␠installed␠on␠the␠replica. +started_at timestamp␠with␠time␠zone When␠hydration␠work␠began,␠or␠`NULL`␠if␠the␠replica␠reported␠none.␠A␠replica␠that␠observed␠no␠start␠reports␠the␠installation␠time␠instead,␠so␠a␠zero␠interval␠between␠the␠two␠does␠not␠mean␠the␠dataflow␠started␠immediately. +hydrated_at timestamp␠with␠time␠zone When␠hydration␠finished. +status text The␠terminal␠status.␠Currently␠always␠`hydrated`. + query TTT SELECT name, type, comment FROM objects WHERE schema = 'mz_internal' AND object = 'mz_object_transitive_dependencies' ORDER BY position ---- @@ -840,6 +851,7 @@ mz_object_fully_qualified_names mz_object_global_ids mz_object_graph_edges mz_object_history +mz_object_hydration_history mz_object_lifetimes mz_object_oid_alias mz_object_transitive_dependencies diff --git a/test/sqllogictest/catalog_server_explain.slt b/test/sqllogictest/catalog_server_explain.slt index 7d5531c356c0e..769ed420cb0ab 100644 --- a/test/sqllogictest/catalog_server_explain.slt +++ b/test/sqllogictest/catalog_server_explain.slt @@ -5472,7 +5472,7 @@ mz_catalog.mz_tables: Project: #3, #0, #4, #1, #5, #2, #6, #6, #6 Map: "s1", null →Arrange (#1{schema_name}, #2{name}) - →Constant (29 rows) + →Constant (30 rows) →Arrange (#0{schema_name}) (#0{schema_name}, #1{name}) →Fused with Child Map/Filter/Project Project: #5, #4, #6 @@ -8246,7 +8246,7 @@ query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_tables"; ---- Explained Query (fast path): - →Constant (29 rows) + →Constant (30 rows) Target cluster: mz_catalog_server @@ -10238,7 +10238,7 @@ query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_ontology_entity_types"; ---- Explained Query (fast path): - →Constant (135 rows) + →Constant (136 rows) Target cluster: mz_catalog_server @@ -10248,7 +10248,7 @@ query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_ontology_link_types"; ---- Explained Query (fast path): - →Constant (177 rows) + →Constant (180 rows) Target cluster: mz_catalog_server @@ -10262,7 +10262,7 @@ Explained Query: cte l0 = →Differential Join %1:mz_schemas[#0{id}] » %2:mz_objects[#2{schema_id}] » %0[#0{schema_name}, #1{table_name}] » %3:mz_columns[#0{id}] →Arrange (#0{schema_name}, #1{table_name}) - →Constant (135 rows) + →Constant (136 rows) →Arrange (#0{id}) →Fused with Child Map/Filter/Project Project: #1, #3 @@ -10315,7 +10315,7 @@ Explained Query: →Differential Join %0:l4[#0{entity_name}, #1{name}] » %1[#0{entity_name}, #1{column_name}] →Arranged l4 →Arrange (#0{entity_name}, #1{column_name}) - →Constant (278 rows) + →Constant (281 rows) →Return →Union →Map/Filter/Project diff --git a/test/sqllogictest/information_schema_tables.slt b/test/sqllogictest/information_schema_tables.slt index 922d078954dcd..2081606984765 100644 --- a/test/sqllogictest/information_schema_tables.slt +++ b/test/sqllogictest/information_schema_tables.slt @@ -513,6 +513,10 @@ mz_object_history VIEW materialize mz_internal +mz_object_hydration_history +BASE TABLE +materialize +mz_internal mz_object_lifetimes VIEW materialize diff --git a/test/sqllogictest/mz_catalog_server_index_accounting.slt b/test/sqllogictest/mz_catalog_server_index_accounting.slt index 75363db62c437..1cc1103fdf6bc 100644 --- a/test/sqllogictest/mz_catalog_server_index_accounting.slt +++ b/test/sqllogictest/mz_catalog_server_index_accounting.slt @@ -84,7 +84,7 @@ mz_message_batch_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_ba mz_message_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_metric_sinks_ind CREATE␠INDEX␠"mz_metric_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s545␠AS␠"mz_internal"."mz_metric_sinks"]␠("id") -mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s841␠AS␠"mz_internal"."mz_notices"]␠("id") +mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s842␠AS␠"mz_internal"."mz_notices"]␠("id") mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s757␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s757␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s755␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") diff --git a/test/sqllogictest/oid.slt b/test/sqllogictest/oid.slt index 5e09c3f0ff0ad..f7a72291710a0 100644 --- a/test/sqllogictest/oid.slt +++ b/test/sqllogictest/oid.slt @@ -1251,3 +1251,4 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17119 mz_builtin_views 17120 mz_metric_sinks 17121 mz_metric_sinks_ind +17122 mz_object_hydration_history diff --git a/test/testdrive/catalog.td b/test/testdrive/catalog.td index 4098fe0a89e88..3e4ad1606a6a3 100644 --- a/test/testdrive/catalog.td +++ b/test/testdrive/catalog.td @@ -610,6 +610,7 @@ mz_materialized_view_refresh_strategies "" mz_object_arrangement_size_history "" mz_object_dependencies "" mz_object_global_ids "" +mz_object_hydration_history "" mz_optimizer_notices "" mz_replacements "" mz_sessions "" @@ -834,7 +835,7 @@ test_table "" # `SHOW TABLES` and `mz_tables` should agree. > SELECT COUNT(*) FROM mz_tables WHERE id LIKE 's%' -29 +30 # There is one entry in mz_indexes for each field_number/expression of the index. > SELECT COUNT(id) FROM mz_indexes WHERE id LIKE 's%' diff --git a/test/workload-replay/objects.txt b/test/workload-replay/objects.txt index 7c14325d961cd..ca116ead25529 100644 --- a/test/workload-replay/objects.txt +++ b/test/workload-replay/objects.txt @@ -615,6 +615,7 @@ mz_object_fully_qualified_names mz_object_global_ids mz_object_history mz_object_history_ind +mz_object_hydration_history mz_object_lifetimes mz_object_lifetimes_ind mz_object_oid_alias diff --git a/test/workload-replay/system_catalog_identifiers.txt b/test/workload-replay/system_catalog_identifiers.txt index bd7b055ae5042..32959e9a05343 100644 --- a/test/workload-replay/system_catalog_identifiers.txt +++ b/test/workload-replay/system_catalog_identifiers.txt @@ -924,6 +924,7 @@ mz_object_fully_qualified_names mz_object_global_ids mz_object_history mz_object_history_ind +mz_object_hydration_history mz_object_lifetimes mz_object_lifetimes_ind mz_object_oid_alias From a1bcaebfe6b2c65338382cf9e0e23d75ebd59203 Mon Sep 17 00:00:00 2001 From: Aljoscha Krettek Date: Wed, 26 Aug 2026 21:42:29 +0200 Subject: [PATCH 05/11] adapter: collect durable object hydration history (#38347) Stacked on #38346. ### Motivation Populates `mz_internal.mz_object_hydration_history`, which the preceding rung adds but leaves empty. Design doc: [`20260817_durable_object_hydration_history.md`](https://github.com/MaterializeInc/materialize/blob/aljoscha/hydration-01-design/doc/developer/design/20260817_durable_object_hydration_history.md) ### Description A single-flight sweep visits one user replica per interval, skipping replicas with introspection disabled and managed replicas whose rolled-up status is not `Online`. It installs an internal subscribe, aggregates the replica's worker rows per dataflow, anti-joins against history, and writes missing rows through timestamped OCC read-then-write. Retention retracts successive bounded batches on the catalog server even when collection from the user replica fails. The main correctness boundaries are: * **Idempotence across environmentd processes.** The subscribe reads its own target table. Concurrent collectors may compute the same candidate, but the oracle selects a write timestamp, the subscribe frontier certifies completeness below it, and a loser recomputes after observing the winner's row. * **Replica-scoped worker aggregation.** Installation adds one row per worker with a nullable `hydrated_at`, and the replica owns and resets the collection as a unit. `HAVING count(*) = count(hydrated_at)` waits for every row visible at the OCC read timestamp, and `max(hydrated_at)` normally includes the persist sink's elected worker, whose stamp covers the snapshot write. Per-process logging clocks can place an ahead worker beyond the sampled timestamp. The collector explicitly accepts that race instead of depending on configured worker counts. * **One row per dataflow.** The table stores the dataflow id under `object_id`; `mz_object_global_ids` is used only to filter to user indexes and materialized views. This preserves separate episodes when one catalog item has several live dataflows. * **Fleet-safe scheduling.** Fires use a stable SHA-256 seed over the full environment id, so a fleet-wide dyncfg does not synchronize environments, including regions and ordinals in one organization, on the same absolute boundary. * **Bounded retention catch-up.** The `LIMIT` is inside a derived table because top-level `RowSetFinishing` is not part of the OCC write selection. Each sweep retracts one bounded batch, so collection gets another turn while later sweeps drain the finite fixed-cutoff backlog without exceeding `max_result_size`. * **Background isolation.** Background mutations do not take the process-wide OCC semaphore, because their replica-targeted subscribe may take arbitrarily long to hydrate. The sweep is single-flight, so this adds at most one concurrent read-then-write. Its task is coordinator-owned and abort-on-drop, while fallible coordinator calls handle concurrent shutdown. The entry point requires an actual system-table target and system-only dependencies, separating it from session DML. * **Operational visibility.** Internal metrics report collection and retention outcomes, affected rows, complete sweep duration, and full retention batches without cluster, replica, or object labels. Repeated full batches indicate that retention may not be keeping up with the configured sweep interval. The read-then-write path gains explicit background-caller contracts for replica targeting, system-table writes, system-read dependency validation, ownership, cancellation, exact-timestamp writes, and trivial top-level row-set finishing. Background subscribes are coordinator-owned, write no `mz_subscriptions` row, and count against an internal gauge. Dropped coordinator responses unwind as errors, including a background compute-client lookup racing shutdown. Known limitations: a process whose logging clock is ahead can place its hydration row beyond the OCC read timestamp, so the durable finish can precede the latest worker's finish. Separately, the oracle chooses the write timestamp and the replica-targeted subscribe frontier certifies the read. If the replica's clock trails environmentd by more than an introspection interval, the frontier can certify each target only after the oracle has advanced past it. The resulting conflict loop ends at the sweep timeout or retry budget. It records nothing and logs a diagnostic naming the likely frontier. ### Verification * `test/testdrive/hydration-status.td` covers timestamp ordering, multiple replicas, idempotence, replica removal, and retention. * `test/restart/mzcompose.py` verifies one pre-restart episode survives unchanged, one fresh episode is collected, neither is duplicated, and a materialized view on a two-worker replica records the all-worker maximum rather than worker 0. * Unit tests cover replica rotation, all-worker completeness, filter placement, full-environment scheduling offsets, and forced migration policy. Collection defaults off in production and is runtime configurable. The rollout plan is CI on merge, then staging, then production, one week apart. This release will record completed hydration of indexes and materialized views in `mz_internal.mz_object_hydration_history` when collection is enabled. Closes: [SQL-644](https://linear.app/materializeinc/issue/SQL-644) --- ...0260210_incremental_occ_read_then_write.md | 5 + ...260817_durable_object_hydration_history.md | 71 +- doc/user/data/metrics.yml | 33 +- misc/python/materialize/mzcompose/__init__.py | 14 + .../materialize/parallel_workload/action.py | 10 + misc/python/materialize/sqlancer.py | 2 +- misc/python/materialize/sqlsmith.py | 2 +- src/adapter-types/src/dyncfgs.rs | 18 + src/adapter/src/active_compute_sink.rs | 46 +- .../src/catalog/builtin_table_updates.rs | 4 +- src/adapter/src/catalog/state.rs | 19 +- src/adapter/src/client.rs | 9 +- src/adapter/src/command.rs | 28 +- src/adapter/src/coord.rs | 14 +- src/adapter/src/coord/appends.rs | 14 +- src/adapter/src/coord/command_handler.rs | 29 +- src/adapter/src/coord/hydration_history.rs | 720 ++++++++++++++++++ src/adapter/src/coord/message_handler.rs | 6 + src/adapter/src/coord/read_then_write.rs | 209 ++--- src/adapter/src/coord/sequencer/inner.rs | 3 +- .../src/coord/sequencer/inner/subscribe.rs | 10 +- src/adapter/src/coord/sql.rs | 143 ++-- src/adapter/src/error.rs | 17 +- src/adapter/src/frontend_peek.rs | 20 +- src/adapter/src/frontend_read_then_write.rs | 246 +++++- src/adapter/src/metrics.rs | 25 +- src/adapter/src/peek_client.rs | 110 ++- src/catalog/src/memory/objects.rs | 27 + .../mzcompose.py | 2 + test/restart/mzcompose.py | 156 ++++ .../insert-into-select-source-error.slt | 52 +- test/testdrive/hydration-status.td | 68 ++ 32 files changed, 1812 insertions(+), 320 deletions(-) create mode 100644 src/adapter/src/coord/hydration_history.rs diff --git a/doc/developer/design/20260210_incremental_occ_read_then_write.md b/doc/developer/design/20260210_incremental_occ_read_then_write.md index 8f60ea1cde37f..2d99887ab60db 100644 --- a/doc/developer/design/20260210_incremental_occ_read_then_write.md +++ b/doc/developer/design/20260210_incremental_occ_read_then_write.md @@ -235,6 +235,11 @@ The correctness argument has two parts: (1) the OCC loop produces the right diffs, and (2) the timestamped write mechanism ensures they are applied at the right timestamp. +User statements require every relation leaf to be a user table owned by the +table-write transaction domain. Source-backed tables, including webhook tables, +receive updates outside that domain, so the target table's oracle cannot make +their updates atomic with the resulting write. + ### The subscribe produces the right diffs The subscribe starts at the oracle read timestamp and emits the current state diff --git a/doc/developer/design/20260817_durable_object_hydration_history.md b/doc/developer/design/20260817_durable_object_hydration_history.md index 53ed9b7d7de97..31271fabe9c5d 100644 --- a/doc/developer/design/20260817_durable_object_hydration_history.md +++ b/doc/developer/design/20260817_durable_object_hydration_history.md @@ -50,6 +50,11 @@ It installs an internal subscribe on that replica which aggregates every worker' completed rows, anti-joins them against the history table, and writes the missing ones through the timestamped OCC read-then-write path. +Collection has no explicit batch bound. It returns at most one row per +not-yet-recorded dataflow, and the OCC path rejects a result that exceeds +`max_result_size` or `max_query_result_size`. At their 1 GiB defaults that ceiling +only matters at millions of dataflows per replica. + Two dyncfgs control it. `hydration_history_collection_interval` sets the sweep cadence and disables collection at zero, which is the production default. `hydration_history_retention_period` bounds how long rows live, defaulting to 30 @@ -148,6 +153,13 @@ sampling race rather than depending on replica configuration for an expected wor count. In that case the durable finish can precede the latest worker's finish. This is separate from restart behavior, which discards the replica collection as a unit. +The missing worker cannot later change the episode key. Its logging clock stamps +both the Differential update and `installed_at`, so appearing after the read means +its installation stamp is later than the visible minimum. A later sweep therefore +keeps the same `min(installed_at)` and the anti-join matches the row already written. +The worker can raise `max(hydrated_at)`, but the durable row is not repaired after +its episode key has been recorded. + Compute is adding an append-only lifecycle log for the same stages, currently proposed in #38403: one row per export, worker and event, with a reason and the dataflow's as-of. It leaves `mz_compute_hydration_times_per_worker` alone, so @@ -190,15 +202,19 @@ writes their retractions at the observed frontier. Collection applies the same cutoff, so a still-live log row cannot resurrect an episode retention just retracted. -Retention deletes successive bounded batches until one is not full. The fixed -cutoff makes the eligible set finite, and collection refuses to insert rows behind -that cutoff, so a successful sweep drains the backlog even when more than one batch -expires at once. The bound is not a nicety. The OCC path refuses a selection larger -than `max_result_size` before submitting any write, so one unbounded delete over a -large backlog would fail identically forever and never shrink the table. The bound -has to sit inside a derived table, because a top-level `LIMIT` lands in the plan's -`RowSetFinishing`, which the OCC path deliberately discards, and the delete would be -silently unbounded again. +Retention deletes one bounded batch per sweep. The fixed cutoff makes the eligible +set finite, and collection refuses to insert rows behind that cutoff, so later +sweeps continue draining the same backlog without starving collection. The batch +bound is not a nicety. The OCC path refuses a selection larger than +`max_result_size` before submitting any write, so one unbounded delete over a large +backlog would fail identically forever and never shrink the table. The bound has to +sit inside a derived table, because a top-level `LIMIT` lands in the plan's +`RowSetFinishing`, which this OCC stage cannot apply. + +`mz_hydration_history_retention_batch_full_total` increments when the batch deletes +all 1,000 rows. Repeated increments mean retention may not be keeping up. An operator +can lower `hydration_history_collection_interval` to schedule sweeps more often, +then compare appended and deleted row rates to confirm the backlog is shrinking. Retention runs on the catalog server, so it keeps working when there are no user replicas at all, and it runs even when that sweep's collection failed. A @@ -238,6 +254,11 @@ the interval is one fleet-wide setting and an unshifted grid would have every environment sweep at the same instant. Using the full id also separates regions and ordinals belonging to one organization. +One replica per interval means an environment with `N` eligible replicas revisits +each one approximately every `N * interval`. Freshness therefore degrades linearly +with replica count. Lowering the interval improves freshness at the cost of more +replica dataflow installs. + **Background mutations take no OCC write permit.** The permits are one semaphore shared by every read-then-write in the process, not one per table. A session's wait is bounded by its statement timeout, a sweep's is not, and a sweep's subscribe has @@ -250,10 +271,10 @@ nothing else stops it outliving the coordinator, and the teardown that follows d the timestamp oracle's worker task. A sweep still running would then read a timestamp from a dead oracle and panic, so the coordinator owns the task handle. -**Each mutation's timeout is deliberately generous.** Even a mutation with nothing -to write waits for its read to linearize, which can take a full -`default_timestamp_interval`, a parameter with no upper bound. A tighter bound would -let a large timestamp interval starve retention permanently. +**Each mutation's timeout is deliberately generous.** Subscribe installation, +replica-side progress, OCC conflict retries, and the external commit can all wait +indefinitely. The bound keeps one unavailable replica or service from starving +retention and every later replica in the single-flight rotation. Replica drop, cluster drop, dependency replacement, replica failure, and timeout all skip the attempt, and a later sweep recomputes from current state. Read-only @@ -283,17 +304,23 @@ frontier reaches the target. One of the subscribe's inputs is a replica-local introspection log whose frontier the *replica* advances from its own clock, rounded up to its introspection interval. -So a replica whose clock trails environmentd by more than one introspection -interval produces a frontier that keeps sitting below the target the oracle hands -out. The mutation waits, and the sweep's own timeout ends it. The consequence is -bounded. Nothing wrong is recorded, that replica records nothing at all, and the -sweep stretches while it waits, which delays others in the rotation. It resolves -itself when the clocks converge. +For a non-empty selection, a replica whose clock trails environmentd by more than +one introspection interval repeatedly loses the same race. Its frontier eventually +certifies the current target, but by then keepalives or other writes have advanced +the oracle past it. The committer refuses the stale target, the OCC loop adopts a +new one, and the replica has to catch up again. The sweep's timeout normally ends +that loop. A lower `max_occ_retries` can end it first with a contention error. + +Nothing wrong is recorded in either case. That replica records nothing at all, and +the sweep stretches while it retries, which delays others in the rotation. Empty +selections exit without entering the conflict loop. The condition resolves itself +when the clocks converge. Accepted for now: skew between processes is normally milliseconds while the -tolerance is a whole introspection interval, and the failure mode is waiting rather -than wrong data. Because the symptom is otherwise hard to attribute, a step that -times out logs that the replica's introspection frontier may be trailing. +tolerance is a whole introspection interval, and the failure mode is retrying rather +than wrong data. Because the symptom is otherwise hard to attribute, both a timeout +and retry-budget exhaustion log that the replica's introspection frontier may be +trailing. ## Rollout diff --git a/doc/user/data/metrics.yml b/doc/user/data/metrics.yml index d269dca4d9ca7..034f404364004 100644 --- a/doc/user/data/metrics.yml +++ b/doc/user/data/metrics.yml @@ -97,7 +97,7 @@ metrics: source: src/adapter/src/metrics.rs visibility: internal - name: mz_active_internal_subscribes - help: The number of active internal subscribes, which serve frontend-sequenced read-then-write. + help: The number of active internal subscribes used by read-then-write operations and background maintenance. labels: - session_type source: src/adapter/src/metrics.rs @@ -955,6 +955,37 @@ metrics: help: The time it takes to advance the catalog shard upper for a txns-shard write (group commits and table register/forget). source: src/adapter/src/metrics.rs visibility: internal +- name: mz_hydration_history_mutations_total + help: Total hydration-history collection and retention mutations since process start. + labels: + - operation + - outcome + source: src/adapter/src/metrics.rs + visibility: internal +- name: mz_hydration_history_retention_batch_full_total + help: Total hydration-history sweeps whose retention batch was full. Repeated increments mean retention may not be keeping up with its schedule. + source: src/adapter/src/metrics.rs + visibility: internal +- name: mz_hydration_history_rows_affected_total + help: Total rows changed by hydration-history maintenance since process start. + labels: + - action + source: src/adapter/src/metrics.rs + visibility: internal +- name: mz_hydration_history_sweep_duration_seconds_bucket + help: Wall time of a complete hydration-history collection and retention sweep. + labels: + - le + source: src/adapter/src/metrics.rs + visibility: internal +- name: mz_hydration_history_sweep_duration_seconds_count + help: Wall time of a complete hydration-history collection and retention sweep. + source: src/adapter/src/metrics.rs + visibility: internal +- name: mz_hydration_history_sweep_duration_seconds_sum + help: Wall time of a complete hydration-history collection and retention sweep. + source: src/adapter/src/metrics.rs + visibility: internal - name: mz_index_peek_cursor_setup_seconds_bucket help: Time setting up cursor and literal constraints. labels: diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py index 9ce47a13dc0a5..0244c6f41f21d 100644 --- a/misc/python/materialize/mzcompose/__init__.py +++ b/misc/python/materialize/mzcompose/__init__.py @@ -147,6 +147,9 @@ def get_minimal_system_parameters( if version < MzVersion.parse_mz("v26.25.0-dev"): config["enable_multi_replica_sources"] = "true" + if version >= MzVersion.parse_mz("v26.40.0-dev"): + config["hydration_history_collection_interval"] = "60s" + if sanitizer_enabled(): config["with_0dt_deployment_max_wait"] = "18000s" @@ -495,6 +498,17 @@ def get_variable_system_parameters( VariableSystemParameter( "arrangement_size_history_retention_period", "7d", ["1min", "1h", "7d"] ), + *( + [ + VariableSystemParameter( + "hydration_history_retention_period", + "30d", + ["1min", "1h", "30d"], + ) + ] + if version >= MzVersion.parse_mz("v26.40.0-dev") + else [] + ), VariableSystemParameter( "persist_validate_part_bounds_on_read", "false", ["true", "false"] ), diff --git a/misc/python/materialize/parallel_workload/action.py b/misc/python/materialize/parallel_workload/action.py index 3e2751d164899..405cce9e78092 100644 --- a/misc/python/materialize/parallel_workload/action.py +++ b/misc/python/materialize/parallel_workload/action.py @@ -3009,6 +3009,16 @@ def __init__( "'1h'", "'7d'", ] + self.flags_with_values["hydration_history_collection_interval"] = [ + "'0s'", + "'1s'", + "'1min'", + ] + self.flags_with_values["hydration_history_retention_period"] = [ + "'1min'", + "'1h'", + "'30d'", + ] # Keep these generous: a tight timeout would abort the oracle's own # queries (they are retried, but it adds noise). "0s" leaves it unset. self.flags_with_values["pg_timestamp_oracle_statement_timeout"] = [ diff --git a/misc/python/materialize/sqlancer.py b/misc/python/materialize/sqlancer.py index de02534aa3d09..655eb1d6a3a95 100644 --- a/misc/python/materialize/sqlancer.py +++ b/misc/python/materialize/sqlancer.py @@ -108,7 +108,7 @@ r"null character not permitted", r"expressions must appear in select list", r"expressions must match initial", - r"invalid selection: operation may only", + r"invalid selection: every relation leaf", r"array size exceeds the maximum allowed", r"does not allow subqueries", r"must use value within", diff --git a/misc/python/materialize/sqlsmith.py b/misc/python/materialize/sqlsmith.py index b8f0ad1398a7d..f8f913b7678cb 100644 --- a/misc/python/materialize/sqlsmith.py +++ b/misc/python/materialize/sqlsmith.py @@ -83,7 +83,7 @@ "result exceeds max size of", # Seems expected with huge queries "expected expression, but found reserved keyword", # Should fix, but only happens rarely with subqueries "Expected right parenthesis, found left parenthesis", # Should fix, but only happens rarely with cast+coalesce - "invalid selection: operation may only (transitively) refer to non-source, non-system tables", # Seems expected when using catalog tables + "invalid selection: every relation leaf must be a writable user table", # Expected when using catalog tables "Unsupported temporal predicate", # Expected, see https://github.com/MaterializeInc/database-issues/issues/5288 "Unsupported temporal operation: NotEq", "Unsupported binary temporal operation: NotEq", diff --git a/src/adapter-types/src/dyncfgs.rs b/src/adapter-types/src/dyncfgs.rs index 259972ef21973..1f50dafbd0aad 100644 --- a/src/adapter-types/src/dyncfgs.rs +++ b/src/adapter-types/src/dyncfgs.rs @@ -421,6 +421,22 @@ pub const ARRANGEMENT_SIZE_HISTORY_RETENTION_PERIOD: Config = Config:: ParameterScope::Environment, ); +/// How often to sweep replicas for completed object hydration episodes. +pub const HYDRATION_HISTORY_COLLECTION_INTERVAL: Config = Config::new( + "hydration_history_collection_interval", + Duration::ZERO, + "How often to record completed object hydration episodes. A zero duration disables collection.", + ParameterScope::Environment, +); + +/// How long to retain completed object hydration episodes. +pub const HYDRATION_HISTORY_RETENTION_PERIOD: Config = Config::new( + "hydration_history_retention_period", + Duration::from_hours(30 * 24), + "How long to retain rows in mz_internal.mz_object_hydration_history.", + ParameterScope::Environment, +); + /// How frequently the catalog `*_info` metrics (`mz_object_info`, /// `mz_cluster_info`, …) are reconciled with the catalog. A zero duration /// disables reconciliation. @@ -557,6 +573,8 @@ pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet { .add(&CONSOLE_OIDC_SCOPES) .add(&ARRANGEMENT_SIZE_HISTORY_COLLECTION_INTERVAL) .add(&ARRANGEMENT_SIZE_HISTORY_RETENTION_PERIOD) + .add(&HYDRATION_HISTORY_COLLECTION_INTERVAL) + .add(&HYDRATION_HISTORY_RETENTION_PERIOD) .add(&CATALOG_INFO_METRICS_RECONCILE_INTERVAL) .add(&PG_TIMESTAMP_ORACLE_STATEMENT_TIMEOUT) .add(&FRONTEND_READ_THEN_WRITE) diff --git a/src/adapter/src/active_compute_sink.rs b/src/adapter/src/active_compute_sink.rs index 358c76ee0f3f3..556b87544d8a9 100644 --- a/src/adapter/src/active_compute_sink.rs +++ b/src/adapter/src/active_compute_sink.rs @@ -51,10 +51,10 @@ impl ActiveComputeSink { } /// Reports the ID of the connection which created the sink. - pub fn connection_id(&self) -> &ConnectionId { + pub fn connection_id(&self) -> Option<&ConnectionId> { match &self { - ActiveComputeSink::Subscribe(subscribe) => &subscribe.conn_id, - ActiveComputeSink::CopyTo(copy_to) => ©_to.conn_id, + ActiveComputeSink::Subscribe(subscribe) => subscribe.connection_id(), + ActiveComputeSink::CopyTo(copy_to) => Some(©_to.conn_id), } } @@ -147,13 +147,26 @@ impl SubscribeBacklogAccounting { } } +/// Ownership and cleanup scope of an active subscribe. +#[derive(Debug)] +pub enum ActiveSubscribeOwner { + /// The subscribe belongs to a SQL session. + Session { + conn_id: ConnectionId, + session_uuid: Uuid, + }, + /// The subscribe belongs to a coordinator background task. + /// + /// Always `internal`, since there is no session to attribute a + /// `mz_subscriptions` row to. + Background, +} + /// A description of an active subscribe from coord's perspective #[derive(Debug)] pub struct ActiveSubscribe { - /// The ID of the connection which created the subscribe. - pub conn_id: ConnectionId, - /// The UUID of the session which created the subscribe. - pub session_uuid: Uuid, + /// The owner responsible for retiring the subscribe. + pub owner: ActiveSubscribeOwner, /// The ID of the cluster on which the subscribe is running. pub cluster_id: ClusterId, /// The IDs of the objects on which the subscribe depends. @@ -189,6 +202,25 @@ pub struct ActiveSubscribe { } impl ActiveSubscribe { + /// The session uuid for this subscribe's `mz_subscriptions` row, or `None` + /// if it does not appear there. + pub fn introspection_session_uuid(&self) -> Option { + match &self.owner { + ActiveSubscribeOwner::Session { session_uuid, .. } if !self.internal => { + Some(*session_uuid) + } + _ => None, + } + } + + /// Returns the owning connection, if this is a session subscribe. + pub fn connection_id(&self) -> Option<&ConnectionId> { + match &self.owner { + ActiveSubscribeOwner::Session { conn_id, .. } => Some(conn_id), + ActiveSubscribeOwner::Background => None, + } + } + /// Initializes the subscription. /// /// This method must be called exactly once, after constructing an diff --git a/src/adapter/src/catalog/builtin_table_updates.rs b/src/adapter/src/catalog/builtin_table_updates.rs index ffa9154e3387d..bb10328aabb56 100644 --- a/src/adapter/src/catalog/builtin_table_updates.rs +++ b/src/adapter/src/catalog/builtin_table_updates.rs @@ -51,6 +51,7 @@ use mz_sql::names::SchemaSpecifier; use mz_sql_parser::ast::display::AstDisplay; use mz_storage_client::client::TableData; use smallvec::smallvec; +use uuid::Uuid; // DO NOT add any more imports from `crate` outside of `crate::catalog`. use crate::active_compute_sink::ActiveSubscribe; @@ -811,12 +812,13 @@ impl CatalogState { &self, id: GlobalId, subscribe: &ActiveSubscribe, + session_uuid: Uuid, diff: Diff, ) -> BuiltinTableUpdate<&'static BuiltinTable> { let mut row = Row::default(); let mut packer = row.packer(); packer.push(Datum::String(&id.to_string())); - packer.push(Datum::Uuid(subscribe.session_uuid)); + packer.push(Datum::Uuid(session_uuid)); packer.push(Datum::String(&subscribe.cluster_id.to_string())); let start_dt = mz_ore::now::to_datetime(subscribe.start_time); diff --git a/src/adapter/src/catalog/state.rs b/src/adapter/src/catalog/state.rs index 72b9276fea3fb..38901e8f754da 100644 --- a/src/adapter/src/catalog/state.rs +++ b/src/adapter/src/catalog/state.rs @@ -3127,7 +3127,9 @@ mod tests { #[mz_ore::test(tokio::test)] #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `TLS_client_method` async fn validate_read_then_write_deep_chain_no_stack_overflow() { - use crate::coord::read_then_write::validate_read_then_write_dependencies; + use crate::coord::read_then_write::{ + DependencyPolicy, validate_read_then_write_dependencies, + }; Catalog::with_debug(|mut catalog| async move { // Deep enough that the previous recursive implementation overflowed @@ -3142,6 +3144,7 @@ mod tests { &catalog, [CatalogItemId::User(BASE)], usize::MAX, + DependencyPolicy::UserDml, ) .expect("deep chain of user views is valid for read-then-write"); @@ -3156,7 +3159,9 @@ mod tests { #[mz_ore::test(tokio::test)] #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `TLS_client_method` async fn validate_read_then_write_dependency_limit() { - use crate::coord::read_then_write::validate_read_then_write_dependencies; + use crate::coord::read_then_write::{ + DependencyPolicy, validate_read_then_write_dependencies, + }; use crate::error::AdapterError; Catalog::with_debug(|mut catalog| async move { @@ -3168,14 +3173,20 @@ mod tests { const OBJECTS: usize = DEPTH + 1; // Exactly at the limit is allowed. - validate_read_then_write_dependencies(&catalog, [CatalogItemId::User(BASE)], OBJECTS) - .expect("chain at the limit is valid"); + validate_read_then_write_dependencies( + &catalog, + [CatalogItemId::User(BASE)], + OBJECTS, + DependencyPolicy::UserDml, + ) + .expect("chain at the limit is valid"); // One below the limit is rejected with a clean error. let err = validate_read_then_write_dependencies( &catalog, [CatalogItemId::User(BASE)], OBJECTS - 1, + DependencyPolicy::UserDml, ) .expect_err("chain over the limit is rejected"); assert!(matches!( diff --git a/src/adapter/src/client.rs b/src/adapter/src/client.rs index 565241b44c0cc..997c73dfff2ab 100644 --- a/src/adapter/src/client.rs +++ b/src/adapter/src/client.rs @@ -63,6 +63,7 @@ use crate::command::{ SASLChallengeResponse, SASLVerifyProofResponse, SuperuserAttribute, }; use crate::config::{ScopedParameters, ScopedParametersScope, SystemParameterFrontend}; +use crate::coord::read_then_write::DependencyPolicy; use crate::coord::{Coordinator, ExecuteContextGuard}; use crate::error::AdapterError; use crate::frontend_read_then_write::{ @@ -72,7 +73,7 @@ use crate::frontend_read_then_write::{ use crate::metrics::Metrics; use crate::optimize::dataflows::{EvalTime, ExprPrepOneShot}; use crate::optimize::{self, Optimize, OptimizerError}; -use crate::peek_client::{ExecutionLogging, TakeOver}; +use crate::peek_client::{CoordinatorClient, ExecutionLogging, TakeOver}; use crate::session::{ EndTransactionAction, PreparedStatement, Session, SessionConfig, StateRevision, TransactionId, TransactionStatus, @@ -313,7 +314,7 @@ impl Client { } = response; let peek_client = PeekClient::new( - self.clone(), + CoordinatorClient::Session(self.clone()), &catalog, storage_collections, transient_id_gen, @@ -1542,7 +1543,7 @@ impl SessionClient { }); tokio::pin!(register); tokio::select! { - rx = &mut register => rx, + rx = &mut register => rx?, _ = &mut cancel_future => { inner_client.try_send(Command::PrivilegedCancelRequest { conn_id: conn_id.clone(), @@ -1973,7 +1974,7 @@ impl SessionClient { "calls to mz_now in write statements", )); } - validate_selection_dependencies(&catalog, &depends_on)?; + validate_selection_dependencies(&catalog, &depends_on, DependencyPolicy::UserDml)?; return Err(prohibited_in_transaction(&stmt)); } } diff --git a/src/adapter/src/command.rs b/src/adapter/src/command.rs index d343becaf3010..cd260d7f379ef 100644 --- a/src/adapter/src/command.rs +++ b/src/adapter/src/command.rs @@ -45,6 +45,7 @@ use mz_timestamp_oracle::TimestampOracle; use tokio::sync::{Semaphore, mpsc, oneshot, watch}; use uuid::Uuid; +use crate::active_compute_sink::ActiveSubscribeOwner; use crate::catalog::Catalog; use crate::config::{ScopedParameters, ScopedParametersScope, SystemParameterFrontend}; use crate::coord::appends::{BuiltinTableAppendNotify, WriteResult}; @@ -419,8 +420,7 @@ pub enum Command { as_of: mz_repr::Timestamp, arity: usize, sink_id: GlobalId, - conn_id: ConnectionId, - session_uuid: Uuid, + owner: ActiveSubscribeOwner, start_time: mz_ore::now::EpochMillis, read_holds: ReadHolds, tx: oneshot::Sender, AdapterError>>, @@ -437,14 +437,10 @@ pub enum Command { /// including read-only, a changed target and cancellation, is reported the /// same way in both modes. AttemptWrite { - /// Connection originating the write. Used so the coordinator can - /// cancel this pending write if the connection is cancelled before - /// the write commits. - conn_id: ConnectionId, + attempt: WriteAttemptKind, target_id: CatalogItemId, target_global_id: GlobalId, diffs: Vec<(Row, Diff)>, - write_ts: Option, tx: oneshot::Sender, }, @@ -455,6 +451,24 @@ pub enum Command { }, } +/// Who a read-then-write commits on behalf of, and how its timestamp is chosen. +/// +/// Group commit picking the timestamp requires a connection to answer through, +/// so that combination is only reachable from a session. +#[derive(Debug)] +pub enum WriteAttemptKind { + /// A session's write, cancelled with `conn_id` if the connection goes away + /// before it commits. A `write_ts` of `None` lets group commit pick the + /// timestamp, which then cannot be reported as passed. + Session { + conn_id: ConnectionId, + write_ts: Option, + }, + /// Coordinator background work. There is no connection to cancel with, so + /// the caller names the timestamp and handles `TimestampPassed` itself. + Background { write_ts: mz_repr::Timestamp }, +} + impl Command { pub fn session(&self) -> Option<&Session> { match self { diff --git a/src/adapter/src/coord.rs b/src/adapter/src/coord.rs index 03420a429e248..721ba8975bcd4 100644 --- a/src/adapter/src/coord.rs +++ b/src/adapter/src/coord.rs @@ -131,7 +131,7 @@ use mz_ore::channel::trigger::Trigger; use mz_ore::future::TimeoutError; use mz_ore::metrics::MetricsRegistry; use mz_ore::now::{EpochMillis, NowFn}; -use mz_ore::task::{JoinHandle, spawn}; +use mz_ore::task::{AbortOnDropHandle, JoinHandle, spawn}; use mz_ore::thread::JoinHandleExt; use mz_ore::tracing::{OpenTelemetryContext, TracingHandle}; use mz_ore::url::SensitiveUrl; @@ -238,6 +238,7 @@ mod caught_up; mod command_handler; mod ddl; pub(crate) mod group_sync; +mod hydration_history; mod indexes; mod info_metrics; mod introspection; @@ -399,6 +400,8 @@ pub enum Message { ArrangementSizesSnapshot, ArrangementSizesWrite(Vec), ArrangementSizesPrune(Vec), + HydrationHistorySchedule, + HydrationHistoryRun, /// Performs any cleanup and logging actions necessary for /// finalizing a statement execution. RetireExecute { @@ -554,6 +557,8 @@ impl Message { Message::ArrangementSizesSnapshot => "arrangement_sizes_snapshot", Message::ArrangementSizesWrite(_) => "arrangement_sizes_write", Message::ArrangementSizesPrune(_) => "arrangement_sizes_prune", + Message::HydrationHistorySchedule => "hydration_history_schedule", + Message::HydrationHistoryRun => "hydration_history_run", Message::RetireExecute { .. } => "retire_execute", Message::ExecuteSingleStatementTransaction { .. } => { "execute_single_statement_transaction" @@ -2108,6 +2113,10 @@ pub struct Coordinator { connection_cancel_watches: BTreeMap, watch::Receiver)>, /// Active introspection subscribes. introspection_subscribes: BTreeMap, + /// The last replica visited by the sequential hydration-history sweep. + hydration_history_replica_cursor: Option, + /// Hydration-history sweep owned by the coordinator while one is in flight. + hydration_history_sweep: Option>, /// Locks that grant access to a specific object, populated lazily as objects are written to. write_locks: BTreeMap>>, @@ -4096,6 +4105,7 @@ impl Coordinator { self.schedule_storage_usage_collection().await; self.schedule_arrangement_sizes_collection().await; + self.schedule_hydration_history_collection(); self.spawn_privatelink_vpc_endpoints_watch_task(); self.spawn_statement_logging_task(); self.spawn_catalog_info_metrics_task(); @@ -5329,6 +5339,8 @@ pub fn serve( active_copies: BTreeMap::new(), connection_cancel_watches: BTreeMap::new(), introspection_subscribes: BTreeMap::new(), + hydration_history_replica_cursor: None, + hydration_history_sweep: None, write_locks: BTreeMap::new(), deferred_write_ops: BTreeMap::new(), pending_writes: Vec::new(), diff --git a/src/adapter/src/coord/appends.rs b/src/adapter/src/coord/appends.rs index 051f2e0af4ba3..ddddf67892c49 100644 --- a/src/adapter/src/coord/appends.rs +++ b/src/adapter/src/coord/appends.rs @@ -194,10 +194,9 @@ pub enum WriteResult { /// Delivers an internal write result, including on task shutdown. /// /// The `Drop` impl is load-bearing. The session task waiting on the other end -/// `expect`s a reply, so a `oneshot::Sender` that is dropped silently would -/// panic that session on coordinator shutdown or on a dead group-committer -/// task. Reporting [`WriteResult::Indeterminate`] instead lets the session -/// report an error. +/// needs a definitive response so that it can release its OCC permit and +/// subscribe. Reporting [`WriteResult::Indeterminate`] lets it unwind when the +/// coordinator or group committer shuts down. #[derive(Debug)] pub struct InternalWriteResponder { tx: Option>, @@ -1719,6 +1718,13 @@ mod tests { use super::*; use crate::catalog::Catalog; + #[mz_ore::test(tokio::test)] + async fn internal_write_responder_reports_indeterminate_on_drop() { + let (tx, rx) = oneshot::channel(); + drop(InternalWriteResponder::new(tx)); + assert!(matches!(rx.await, Ok(WriteResult::Indeterminate))); + } + #[derive(Debug, Default)] struct MemTimestampOracle { read_write_ts: Mutex<(Timestamp, Timestamp)>, diff --git a/src/adapter/src/coord/command_handler.rs b/src/adapter/src/coord/command_handler.rs index 4b334b3a8f31f..0d29050f5c447 100644 --- a/src/adapter/src/coord/command_handler.rs +++ b/src/adapter/src/coord/command_handler.rs @@ -638,44 +638,25 @@ impl Coordinator { as_of, arity, sink_id, - conn_id, - session_uuid, + owner, start_time, read_holds, tx, } => { self.handle_create_internal_subscribe( - *df_desc, - cluster_id, - replica_id, - depends_on, - as_of, - arity, - sink_id, - conn_id, - session_uuid, - start_time, - read_holds, - tx, + *df_desc, cluster_id, replica_id, depends_on, as_of, arity, sink_id, owner, + start_time, read_holds, tx, ) .await; } Command::AttemptWrite { - conn_id, + attempt, target_id, target_global_id, diffs, - write_ts, tx, } => { - self.handle_attempt_write( - conn_id, - target_id, - target_global_id, - diffs, - write_ts, - tx, - ); + self.handle_attempt_write(attempt, target_id, target_global_id, diffs, tx); } Command::DropInternalSubscribe { sink_id } => { self.drop_internal_subscribe(sink_id).await; diff --git a/src/adapter/src/coord/hydration_history.rs b/src/adapter/src/coord/hydration_history.rs new file mode 100644 index 0000000000000..b7362dff9be09 --- /dev/null +++ b/src/adapter/src/coord/hydration_history.rs @@ -0,0 +1,720 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//! Durable history collection for completed compute-object hydration episodes. +//! +//! One sweep visits a single user replica, installs a replica-targeted +//! subscribe that diffs that replica's live hydration timestamps against the +//! durable history table, and appends what is missing through the timestamped +//! OCC write path. Including the history table in the read expression is what +//! makes the write idempotent across concurrent `environmentd` processes: two +//! collectors that compute the same row race for one write timestamp, and the +//! loser observes the winner's append through its own subscribe and finds +//! nothing left to write. +//! +//! One replica is sampled per interval, so an environment with `N` eligible +//! replicas revisits each one approximately every `N * interval`. Lowering the +//! interval improves freshness at the cost of more replica dataflow installs. +//! +//! Collection is sampling, not an event log. An episode whose live row is +//! retracted before its replica's turn in the sweep (a dropped object, or a +//! replica that restarts first) is not recorded, and cannot be, because +//! the only evidence is gone. See the design doc for why that is accepted here. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use itertools::Itertools; +use mz_adapter_types::dyncfgs::{ + FRONTEND_READ_THEN_WRITE, HYDRATION_HISTORY_COLLECTION_INTERVAL, + HYDRATION_HISTORY_RETENTION_PERIOD, +}; +use mz_catalog::builtin::{MZ_CATALOG_SERVER_CLUSTER, MZ_OBJECT_HYDRATION_HISTORY}; +use mz_cluster_client::ReplicaId; +use mz_controller::clusters::{ClusterStatus, ReplicaLocation}; +use mz_controller_types::ClusterId; +use mz_ore::cast::CastFrom; +use mz_ore::collections::CollectionExt; +use mz_ore::now::EpochMillis; +use mz_ore::task; +use mz_repr::CatalogItemId; +use mz_sql::plan::{MutationKind, Params, Plan, ReadThenWritePlan}; +use sha2::{Digest, Sha256}; +use tracing::warn; + +use crate::catalog::Catalog; +use crate::command::ExecuteResponse; +use crate::coord::{Coordinator, Message}; +use crate::metrics::Metrics; +use crate::peek_client::CoordinatorClient; +use crate::session::Session; +use crate::{AdapterError, PeekClient}; + +/// Longest a scheduler sleep may run before it rechecks the configuration. +/// +/// Sleeping the whole interval would leave a dyncfg change ineffective until the +/// old interval elapsed, so lowering the interval at runtime (which tests do) +/// would not take effect for up to the previous interval. +const SCHEDULE_RECHECK_CAP: Duration = Duration::from_secs(5); + +/// How often a disabled collector rechecks whether it was enabled. +/// +/// This is the cadence of every environment in the default configuration, so it +/// is much coarser than the enabled one: nothing is waiting on it. +const DISABLED_RECHECK_INTERVAL: Duration = Duration::from_secs(60); + +/// Bound on one replica-targeted mutation. +/// +/// Subscribe installation, replica-side progress, OCC conflict retries, and the +/// external commit can all wait indefinitely. Exceeding the bound skips the +/// step. The next sweep recomputes from current state. +const MUTATION_TIMEOUT: Duration = Duration::from_secs(300); + +/// Rows retracted per retention step. +/// +/// Retention has to be bounded: the OCC path refuses a selection larger than +/// `max_result_size` before submitting any write, so an unbounded delete over a +/// large backlog would fail identically on every sweep and never shrink the +/// table. Retention repeats bounded batches across sweeps until it drains the +/// fixed-cutoff backlog. +const RETENTION_BATCH_SIZE: usize = 1000; + +/// Milliseconds until the next fire on this environment's own grid. +/// +/// The grid has period `interval_ms` and is shifted by `offset`. When this +/// environment's point in the current period has already passed, the next one is a +/// full period later. +fn next_fire_delay(now: EpochMillis, interval_ms: EpochMillis, offset: EpochMillis) -> Duration { + debug_assert!(interval_ms > 0); + let this_period = (now - (now % interval_ms)).saturating_add(offset); + let next = if this_period > now { + this_period + } else { + this_period.saturating_add(interval_ms) + }; + Duration::from_millis(next.saturating_sub(now)) +} + +/// Stable offset within `interval_ms` for one environment id. +fn environment_schedule_offset(environment_id: &str, interval_ms: EpochMillis) -> EpochMillis { + debug_assert!(interval_ms > 0); + let digest = Sha256::digest(environment_id); + let hash = u64::from_le_bytes(digest[..8].try_into().expect("SHA-256 digest has 32 bytes")); + hash % interval_ms +} + +impl Coordinator { + /// Schedules the next hydration history sweep. + /// + /// Fires are aligned to interval boundaries so that they stay evenly spaced + /// across restarts, offset per environment so that a fleet-wide interval does + /// not make every environment sweep at the same instant, and each sleep is + /// capped so a configuration change is picked up promptly. Sweeps never + /// overlap: the next one is only scheduled once the previous one has finished + /// or failed. + /// + /// NOTE: Alignment reads the wall clock, so a test that freezes `NowFn` and + /// configures an interval longer than the recheck cap never reaches a + /// boundary and never fires. + pub(super) fn schedule_hydration_history_collection(&self) { + let interval = + HYDRATION_HISTORY_COLLECTION_INTERVAL.get(self.catalog().system_config().dyncfgs()); + + // A zero interval disables collection. Keep polling so that enabling it + // takes effect without an `environmentd` restart. + let (delay, fire) = if interval.is_zero() { + (DISABLED_RECHECK_INTERVAL, false) + } else { + // An absurd interval saturates rather than panicking. The setting is + // durable, so a panic here would recur on every restart. + let interval_ms = EpochMillis::try_from(interval.as_millis()) + .unwrap_or(EpochMillis::MAX) + .max(1); + let remaining = next_fire_delay( + self.now(), + interval_ms, + self.hydration_history_schedule_offset(interval_ms), + ); + if remaining <= SCHEDULE_RECHECK_CAP { + (remaining, true) + } else { + (SCHEDULE_RECHECK_CAP, false) + } + }; + + let internal_cmd_tx = self.internal_cmd_tx.clone(); + task::spawn(|| "hydration_history_schedule", async move { + tokio::time::sleep(delay).await; + let message = if fire { + Message::HydrationHistoryRun + } else { + Message::HydrationHistorySchedule + }; + // Best effort: the coordinator may be shutting down. + let _ = internal_cmd_tx.send(message); + }); + } + + /// A stable offset into the collection interval for this environment. + /// + /// Seeded from the full environment id, so it survives restarts but differs + /// between environments in the same organization. Without it every + /// environment would sweep on the same absolute grid, turning each boundary + /// into a fleet-wide burst of dataflow installs, oracle round trips and persist + /// writes. + fn hydration_history_schedule_offset(&self, interval_ms: EpochMillis) -> EpochMillis { + let environment_id = self.catalog().state().config().environment_id.to_string(); + environment_schedule_offset(&environment_id, interval_ms) + } + + /// Runs one sweep: collect from the next replica, then apply retention. + pub(super) fn run_hydration_history_collection(&mut self) { + let (collection_interval, retention) = { + let dyncfgs = self.catalog().system_config().dyncfgs(); + ( + HYDRATION_HISTORY_COLLECTION_INTERVAL.get(dyncfgs), + HYDRATION_HISTORY_RETENTION_PERIOD.get(dyncfgs), + ) + }; + // Builtin tables are not writable in read-only mode, and a disabled + // collector must do no background work at all. Retention is part of the + // sweep, so disabling collection also suspends it. That is deliberate: + // the table can only be non-empty if collection ran at some point, and + // the alternative is an always-on subscribe in the default (disabled) + // production configuration. + if collection_interval.is_zero() || self.controller.read_only() { + self.schedule_hydration_history_collection(); + return; + } + + let replicas = self + .catalog() + .user_cluster_replicas() + .filter(|replica| replica.config.compute.logging.enabled()) + .filter(|replica| match &replica.config.location { + ReplicaLocation::Managed(_) => { + self.cluster_replica_statuses + .get_cluster_replica_status(replica.cluster_id, replica.replica_id) + == ClusterStatus::Online + } + // Unmanaged replicas have no orchestrator status and are only + // used by tests. Their bounded mutation determines readiness. + ReplicaLocation::Unmanaged(_) => true, + }) + .map(|replica| (replica.cluster_id, replica.replica_id)) + .sorted_by_key(|(_, replica_id)| *replica_id) + .collect_vec(); + + let catalog = self.owned_catalog(); + // Retention runs on the catalog server, so that it keeps working when + // there are no user replicas to collect from at all. Without a replica + // there it is skipped, while collection still runs. + let catalog_server = catalog.resolve_builtin_cluster(&MZ_CATALOG_SERVER_CLUSTER); + let catalog_server_target = catalog_server + .replicas() + .next() + .map(|replica| (catalog_server.id, replica.replica_id)); + + let replica = next_replica(&replicas, self.hydration_history_replica_cursor); + if let Some((_, replica_id)) = replica { + self.hydration_history_replica_cursor = Some(replica_id); + } + let mut sweep = self.new_sweep(catalog, retention); + let internal_cmd_tx = self.internal_cmd_tx.clone(); + + let handle = task::spawn(|| "hydration_history_sweep", async move { + let started = Instant::now(); + if let Some((cluster_id, replica_id)) = replica { + sweep.collect(cluster_id, replica_id).await; + } + + // Retention runs even when collection failed above. A replica that + // is crash-looping or slow must not be able to stop the table from + // shrinking back to its retention bound. + if let Some((cluster_id, replica_id)) = catalog_server_target { + sweep.retain(cluster_id, replica_id).await; + } + + sweep + .metrics + .hydration_history_sweep_duration_seconds + .observe(started.elapsed().as_secs_f64()); + let _ = internal_cmd_tx.send(Message::HydrationHistorySchedule); + }); + + // Keep the sweep coordinator-owned so dropping the coordinator requests + // its abort. Fallible coordinator calls make concurrent shutdown safe. + self.hydration_history_sweep = Some(handle.abort_on_drop()); + } + + /// Assembles the sweep context, including the client it writes through. + fn new_sweep(&self, catalog: Arc, retention: Duration) -> Sweep { + let retention_ms = u64::try_from(retention.as_millis()).unwrap_or(u64::MAX); + let build_version = catalog.state().config().build_info.human_version(None); + // Background read-then-write always uses the frontend OCC path. This + // shared constructor field only controls session fallback, so the flag + // does not gate history collection. + let client = PeekClient::new( + CoordinatorClient::Background { + tx: self.internal_cmd_tx.clone(), + metrics: self.metrics.clone(), + }, + &catalog, + Arc::clone(&self.controller.storage_collections), + Arc::clone(&self.transient_id_gen), + self.optimizer_metrics.clone(), + self.persist_client.clone(), + self.statement_logging.create_frontend(build_version), + Arc::clone(&self.occ_write_semaphore), + FRONTEND_READ_THEN_WRITE.get(self.catalog().system_config().dyncfgs()), + self.group_commit_tx.clone(), + self.controller.read_only(), + ); + Sweep { + client, + history_id: catalog.resolve_builtin_table(&MZ_OBJECT_HYDRATION_HISTORY), + catalog, + metrics: self.metrics.clone(), + wall_time: self.now_datetime(), + cutoff: mz_ore::now::to_datetime(self.now().saturating_sub(retention_ms)).to_rfc3339(), + } + } +} + +/// Picks the replica after `cursor`, wrapping around at the end. +/// +/// `replicas` must be sorted ascending by replica id. Unsorted input still +/// returns a replica but degenerates the rotation, revisiting some replicas and +/// starving others. +fn next_replica( + replicas: &[(ClusterId, ReplicaId)], + cursor: Option, +) -> Option<(ClusterId, ReplicaId)> { + replicas + .iter() + .find(|(_, replica_id)| cursor.is_none_or(|cursor| *replica_id > cursor)) + .or_else(|| replicas.first()) + .copied() +} + +/// The rows this replica has completed that the history table is missing. +/// +/// Aggregates every worker's row for an export, and records nothing until all of +/// them have hydrated. One worker is not enough, because a materialized view's +/// persist sink has a single active worker, `hash(sink_id) % workers`. Only that +/// worker's reported output frontier is gated on the shard upper, so only its +/// `hydrated_at` covers the initial snapshot write. Every other worker clears its +/// sink write frontier and stamps at compute completion, which for a materialized +/// view is before the data is durable. Taking `max` over a complete set of workers +/// is therefore the only way to get a finish that means the same thing for every +/// object, and it is the rule `mz_compute_hydration_times` already applies. +/// +/// Completeness needs no configured worker count. The log carries a row per +/// `(export_id, worker_id)` from installation with a null `hydrated_at`, so +/// `count(*) = count(hydrated_at)` says every row visible at the OCC read +/// timestamp has finished. Per-process logging clocks also determine Differential +/// update timestamps, so a worker whose clock is ahead can be absent at that +/// timestamp. A visible unfinished object is skipped and picked up by a later +/// sweep. +/// +/// A worker missing at the read timestamp cannot later change the episode key. +/// Its logging clock stamps both the Differential update and `installed_at`, so +/// late visibility means its installation stamp is later than the visible +/// minimum. The anti-join therefore keeps matching the recorded row. A later +/// `hydrated_at` can raise the aggregate's maximum, but history is not repaired +/// after the episode key has been recorded. +/// +/// The collector deliberately accepts this sampling race rather than depending on +/// `ReplicaLocation::workers()`. A durable finish can therefore precede the latest +/// worker's finish. A whole-replica restart resets the collection as a unit. +/// +/// The interval spans workers, so it carries whatever skew there is between the +/// process clocks that stamped its ends. Each process anchors its logging clock at +/// its own `SystemTime`. That inflates a duration, and nothing here rejects a row +/// for being inconsistent, which is deliberate: an ordering guard on cross-worker +/// stamps rejects complete episodes permanently, since the log values never change. +/// +/// Collection has no explicit batch bound. It returns at most one row per +/// not-yet-recorded dataflow, and the OCC path rejects a result that exceeds +/// `max_result_size` or `max_query_result_size`. At their 1 GiB defaults that +/// ceiling only matters at millions of dataflows per replica. +fn collect_sql(cluster_id: ClusterId, replica_id: ReplicaId, cutoff: &str) -> String { + // Interpolating into SQL is safe here: the ids are catalog-internal and the + // cutoff is an RFC 3339 timestamp we formatted ourselves. Nothing in this + // query comes from a user. + // + // NOTE: The cutoff and the anti-join sit outside the aggregate deliberately. As + // a `WHERE` clause either one drops not-yet-hydrated rows, which would make + // `count(*) = count(hydrated_at)` trivially true and hand back a compute-only + // finish for a materialized view whose active worker is still writing. + // + // NOTE: `hydrated_at` is the terminal stamp for a history episode. Nothing + // waits for the history row before proceeding. If the log gains a separate + // `written_at` stamp, only `hydrated_at` belongs in this completeness check. + // A materialized view being replaced can hydrate while it runs read-only, and + // may never write if the replacement is rolled back. + format!( + "SELECT + e.object_id, + '{cluster_id}'::text AS cluster_id, + '{replica_id}'::text AS replica_id, + e.installed_at, + e.started_at, + e.hydrated_at, + 'hydrated'::text AS status + FROM ( + SELECT + t.export_id AS object_id, + min(t.installed_at) AS installed_at, + min(t.started_at) AS started_at, + max(t.hydrated_at) AS hydrated_at + FROM mz_introspection.mz_compute_hydration_times_per_worker AS t + JOIN mz_internal.mz_object_global_ids AS ids ON ids.global_id = t.export_id + JOIN mz_catalog.mz_objects AS o ON o.id = ids.id + WHERE t.export_id LIKE 'u%' + AND o.type IN ('index', 'materialized-view') + GROUP BY t.export_id + HAVING count(*) = count(t.hydrated_at) + ) AS e + WHERE e.hydrated_at >= TIMESTAMPTZ '{cutoff}' + AND NOT EXISTS ( + SELECT 1 + FROM mz_internal.mz_object_hydration_history AS h + WHERE h.object_id = e.object_id + AND h.replica_id = '{replica_id}'::text + AND h.installed_at = e.installed_at + )" + ) +} + +/// A bounded batch of history rows that have aged out. +/// +/// Only rows with a `hydrated_at` age out. Every row written today has one, and +/// a row without one would be immortal here, so an unfinished-episode +/// representation needs a second age basis before it can be recorded. +fn retention_sql(cutoff: &str) -> String { + // The LIMIT has to sit inside a subquery. A top-level LIMIT lands in the + // plan's `RowSetFinishing`, which this OCC stage cannot apply. Inside a + // derived table it lowers into the relation expression instead. + format!( + "SELECT * FROM ( + SELECT + object_id, cluster_id, replica_id, installed_at, started_at, + hydrated_at, status + FROM mz_internal.mz_object_hydration_history + WHERE hydrated_at < TIMESTAMPTZ '{cutoff}' + ORDER BY hydrated_at + LIMIT {RETENTION_BATCH_SIZE} + )" + ) +} + +/// What one sweep needs to run its mutations against the history table. +struct Sweep { + client: PeekClient, + catalog: Arc, + history_id: CatalogItemId, + metrics: Metrics, + wall_time: chrono::DateTime, + /// Rows finishing before this have aged out. Both steps apply it, so a live + /// log row cannot resurrect an episode retention just retracted. + cutoff: String, +} + +impl Sweep { + /// Appends this replica's completed episodes that the table is missing. + async fn collect(&mut self, cluster_id: ClusterId, replica_id: ReplicaId) { + let sql = collect_sql(cluster_id, replica_id, &self.cutoff); + let _ = self + .run( + "collection", + cluster_id, + replica_id, + MutationKind::Insert, + &sql, + ) + .await; + } + + /// Retracts one bounded batch of aged-out rows. + async fn retain(&mut self, cluster_id: ClusterId, replica_id: ReplicaId) { + let sql = retention_sql(&self.cutoff); + let Some(deleted) = self + .run( + "retention", + cluster_id, + replica_id, + MutationKind::Delete, + &sql, + ) + .await + else { + return; + }; + if deleted == RETENTION_BATCH_SIZE { + self.metrics.hydration_history_retention_batch_full.inc(); + } + } + + /// Runs one mutation, leaving transient failures for the next sweep to retry. + /// + /// Replica loss, dependency replacement, and write races are logged rather + /// than propagated because the next sweep recomputes from current state. + /// + /// NOTE: A timed-out mutation can still commit. The write is submitted + /// before we wait for its answer, and a background write carries no + /// connection to cancel it with, so a `timeout` outcome says we stopped + /// waiting, not that nothing landed. Rows such a write commits afterwards + /// are never counted, which makes `rows_affected` a lower bound. + async fn run( + &mut self, + step: &'static str, + cluster_id: ClusterId, + replica_id: ReplicaId, + kind: MutationKind, + sql: &str, + ) -> Option { + let mutation = async { + let plan = plan_mutation(&self.catalog, self.history_id, kind, sql)?; + let mut session = Session::dummy(); + session.start_transaction_single_stmt(self.wall_time); + let response = self + .client + .background_read_then_write( + &mut session, + plan, + cluster_id, + replica_id, + &self.catalog, + ) + .await?; + Ok::<_, AdapterError>(response) + }; + match tokio::time::timeout(MUTATION_TIMEOUT, mutation).await { + Ok(Ok(response)) => { + let (rows, action) = match (kind, response) { + (MutationKind::Insert, ExecuteResponse::Inserted(rows)) => (rows, "appended"), + (MutationKind::Update, ExecuteResponse::Updated(rows)) => (rows, "updated"), + (MutationKind::Delete, ExecuteResponse::Deleted(rows)) => (rows, "deleted"), + (_, response) => { + self.observe_mutation(step, "error"); + mz_ore::soft_panic_or_log!( + "hydration history {step} returned an unexpected response: {response:?}" + ); + return None; + } + }; + self.metrics + .hydration_history_rows_affected + .with_label_values(&[action]) + .inc_by(u64::cast_from(rows)); + let outcome = if rows == 0 { "noop" } else { "success" }; + self.observe_mutation(step, outcome); + Some(rows) + } + Ok(Err(error)) => { + self.observe_mutation(step, "error"); + if step == "collection" && matches!(&error, AdapterError::ReadThenWriteContention) { + warn!( + %step, %cluster_id, %replica_id, %error, + "hydration history step failed, the replica's introspection frontier \ + may be trailing the write frontier" + ); + } else { + warn!(%step, %cluster_id, %replica_id, %error, "hydration history step failed"); + } + None + } + // A trailing replica can repeatedly certify a target only after the + // oracle has advanced past it. Each refused write raises the target, + // and the conflict loop can continue until this timeout fires. + Err(_) if step == "collection" => { + self.observe_mutation(step, "timeout"); + warn!( + %step, %cluster_id, %replica_id, + "hydration history step timed out, \ + the replica's introspection frontier may be trailing the write frontier" + ); + None + } + Err(_) => { + self.observe_mutation(step, "timeout"); + warn!(%step, %cluster_id, %replica_id, "hydration history step timed out"); + None + } + } + } + + fn observe_mutation(&self, operation: &str, outcome: &str) { + self.metrics + .hydration_history_mutations + .with_label_values(&[operation, outcome]) + .inc(); + } +} + +/// Plans `sql` as the read side of a mutation against `target_id`. +/// +/// The statement is planned as a `SELECT` whose columns are already in the +/// target table's order, so the mutation needs no assignments or projection. +/// +/// The selection's column types are checked against the target table here. A +/// user `INSERT ... SELECT` gets that from the planner, but a hand-built plan +/// bypasses it, and a wrong type would be written into the shard verbatim and +/// break every later read of a table that is deliberately never truncated. +fn plan_mutation( + catalog: &Arc, + target_id: CatalogItemId, + kind: MutationKind, + sql: &str, +) -> Result { + let session_catalog = catalog.for_system_session(); + let parsed = mz_sql::parse::parse(sql) + .map_err(AdapterError::from)? + .into_element(); + let (stmt, resolved_ids) = mz_sql::names::resolve(&session_catalog, parsed.ast)?; + let (plan, _) = mz_sql::plan::plan( + None, + &session_catalog, + stmt, + &Params::empty(), + &resolved_ids, + )?; + let Plan::Select(select) = plan else { + return Err(AdapterError::Internal( + "hydration history query did not plan as SELECT".into(), + )); + }; + + let target_desc = catalog + .get_entry(&target_id) + .relation_desc_latest() + .expect("hydration history target is a table"); + let selection_types = select.source.typ(&[], &BTreeMap::new()).column_types; + let target_types = &target_desc.typ().column_types; + let matches = selection_types.len() == target_types.len() + && selection_types + .iter() + .zip_eq(target_types) + // Nullability may be tighter than the column allows, only the + // scalar types have to agree. + .all(|(selected, target)| selected.scalar_type == target.scalar_type); + if !matches { + return Err(AdapterError::Internal(format!( + "hydration history query does not match the target table: \ + selection {selection_types:?}, table {target_types:?}" + ))); + } + + Ok(ReadThenWritePlan { + id: target_id, + selection: select.source, + finishing: select.finishing, + assignments: BTreeMap::new(), + kind, + returning: Vec::new(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[mz_ore::test] + fn replica_sweep_advances_and_wraps() { + let cluster = ClusterId::user(1).expect("valid cluster ID"); + let replicas = [(cluster, ReplicaId::User(1)), (cluster, ReplicaId::User(3))]; + + assert_eq!(next_replica(&replicas, None), Some(replicas[0])); + assert_eq!( + next_replica(&replicas, Some(ReplicaId::User(1))), + Some(replicas[1]) + ); + assert_eq!( + next_replica(&replicas, Some(ReplicaId::User(3))), + Some(replicas[0]) + ); + assert_eq!(next_replica(&[], None), None); + } + + /// Every environment shares one interval, so the grid has to be shifted per + /// environment or the whole fleet sweeps at the same instant. + #[mz_ore::test] + fn fire_delay_is_offset_within_the_interval() { + let interval = 60_000; + + // Before this environment's point in the period, we wait for it. + assert_eq!( + next_fire_delay(1_000, interval, 5_000), + Duration::from_millis(4_000) + ); + // On it, we take the next period rather than firing twice. + assert_eq!( + next_fire_delay(5_000, interval, 5_000), + Duration::from_millis(interval) + ); + // After it, the next period's point. + assert_eq!( + next_fire_delay(6_000, interval, 5_000), + Duration::from_millis(59_000) + ); + // A zero offset is plain alignment, and never returns a zero delay. + assert_eq!( + next_fire_delay(59_999, interval, 0), + Duration::from_millis(1) + ); + assert_eq!( + next_fire_delay(60_000, interval, 0), + Duration::from_millis(interval) + ); + + // Region and ordinal are part of the seed, not just the organization. + let one = environment_schedule_offset( + "aws-us-east-1-00000000-0000-0000-0000-000000000000-0", + interval, + ); + let two = environment_schedule_offset( + "aws-us-west-1-00000000-0000-0000-0000-000000000000-1", + interval, + ); + assert_eq!(one, 30_189); + assert_eq!(two, 38_252); + assert_ne!(one, two); + } + + /// A materialized view's finish is only durable on the sink's active worker, so + /// the query has to see every worker and take the latest stamp. Pinning a single + /// worker, or letting the cutoff or the anti-join filter rows before the + /// completeness check, silently reintroduces a finish that precedes the write. + #[mz_ore::test] + fn collect_requires_every_worker() { + let cutoff = "1970-01-01T00:00:00+00:00"; + let sql = collect_sql( + ClusterId::user(1).expect("valid cluster ID"), + ReplicaId::User(2), + cutoff, + ); + assert!( + sql.contains("HAVING count(*) = count(t.hydrated_at)"), + "{sql}" + ); + assert!(sql.contains("max(t.hydrated_at)"), "{sql}"); + assert!(!sql.contains("worker_id"), "{sql}"); + + // Both of these have to apply to the aggregate's output, not to the rows + // feeding it. + let aggregate_end = sql.find(") AS e").expect("aggregate subquery"); + assert!(sql.find(cutoff).expect("cutoff") > aggregate_end, "{sql}"); + assert!( + sql.find("NOT EXISTS").expect("anti-join") > aggregate_end, + "{sql}" + ); + } +} diff --git a/src/adapter/src/coord/message_handler.rs b/src/adapter/src/coord/message_handler.rs index a8bc6342976c7..bb8fc01546ec3 100644 --- a/src/adapter/src/coord/message_handler.rs +++ b/src/adapter/src/coord/message_handler.rs @@ -185,6 +185,12 @@ impl Coordinator { Message::ArrangementSizesPrune(expired) => { self.arrangement_sizes_prune(expired).boxed_local().await; } + Message::HydrationHistorySchedule => { + self.schedule_hydration_history_collection(); + } + Message::HydrationHistoryRun => { + self.run_hydration_history_collection(); + } Message::RetireExecute { otel_ctx, data, diff --git a/src/adapter/src/coord/read_then_write.rs b/src/adapter/src/coord/read_then_write.rs index d4f05b0a2b954..fafd38d19a292 100644 --- a/src/adapter/src/coord/read_then_write.rs +++ b/src/adapter/src/coord/read_then_write.rs @@ -26,14 +26,16 @@ use tokio::sync::mpsc; use tracing::Span; use crate::PeekResponseUnary; -use crate::active_compute_sink::{ActiveComputeSink, ActiveSubscribe}; +use crate::active_compute_sink::{ActiveComputeSink, ActiveSubscribe, ActiveSubscribeOwner}; use crate::catalog::Catalog; +use crate::command::WriteAttemptKind; use crate::coord::Coordinator; use crate::coord::appends::{ InternalWriteResponder, PendingWriteTxn, TableWriteCmd, TimestampedWriteRequest, UserWriteResponder, WriteResult, WriteTarget, }; use crate::error::AdapterError; +use mz_ore::soft_panic_or_log; /// Adds `id` to the worklist the first time it is seen, enforcing the /// dependency bound. @@ -67,8 +69,8 @@ impl Coordinator { /// Takes ownership of `read_holds` and drops them only once the dataflow is /// shipped, so the `since` cannot advance past `as_of` in between. /// - /// Answers through `response_tx`, with an error if the connection went away - /// or if a dependency was dropped since the plan was optimized. + /// Answers through `response_tx`, with an error if the owning connection + /// went away or if a dependency was dropped since the plan was optimized. #[allow(clippy::too_many_arguments)] pub(crate) async fn handle_create_internal_subscribe( &mut self, @@ -79,25 +81,29 @@ impl Coordinator { as_of: Timestamp, arity: usize, sink_id: GlobalId, - conn_id: mz_adapter_types::connection::ConnectionId, - session_uuid: uuid::Uuid, + owner: ActiveSubscribeOwner, start_time: mz_ore::now::EpochMillis, read_holds: crate::ReadHolds, response_tx: tokio::sync::oneshot::Sender< Result, AdapterError>, >, ) { - // Client disconnected while waiting for the semaphore. - if !self.active_conns.contains_key(&conn_id) { - let _ = response_tx.send(Err(AdapterError::Canceled)); - return; + match &owner { + // The client may have disconnected while we waited for the semaphore. + ActiveSubscribeOwner::Session { conn_id, .. } => { + if !self.active_conns.contains_key(conn_id) { + let _ = response_tx.send(Err(AdapterError::Canceled)); + return; + } + } + // Background work has no connection to lose. + ActiveSubscribeOwner::Background => {} } let (tx, rx) = mpsc::unbounded_channel(); let active_subscribe = ActiveSubscribe { - conn_id: conn_id.clone(), - session_uuid, + owner, channel: tx, backlog_accounting: std::sync::Arc::new(std::sync::Mutex::new( crate::active_compute_sink::SubscribeBacklogAccounting::default(), @@ -164,17 +170,21 @@ impl Coordinator { /// commit, whose staging re-checks the target generation. pub(crate) fn handle_attempt_write( &mut self, - conn_id: mz_adapter_types::connection::ConnectionId, + attempt: WriteAttemptKind, target_id: mz_repr::CatalogItemId, target_global_id: GlobalId, diffs: Vec<(Row, Diff)>, - write_ts: Option, result_tx: tokio::sync::oneshot::Sender, ) { let result = InternalWriteResponder::new(result_tx); - if !self.active_conns.contains_key(&conn_id) { - result.send(WriteResult::Canceled); - return; + match &attempt { + WriteAttemptKind::Session { conn_id, .. } => { + if !self.active_conns.contains_key(conn_id) { + result.send(WriteResult::Canceled); + return; + } + } + WriteAttemptKind::Background { .. } => {} } if self.controller.read_only() { result.send(WriteResult::ReadOnly); @@ -191,7 +201,11 @@ impl Coordinator { } let table_data = TableData::Rows(diffs); - match write_ts { + let timestamped = match &attempt { + WriteAttemptKind::Session { write_ts, .. } => *write_ts, + WriteAttemptKind::Background { write_ts } => Some(*write_ts), + }; + match timestamped { Some(target_timestamp) => { let request = TimestampedWriteRequest { appends: vec![(target_global_id, vec![table_data])], @@ -207,7 +221,15 @@ impl Coordinator { tracing::warn!("group committer task gone, dropping timestamped write"); } } + // Only a session reaches this: `WriteAttemptKind::Background` always + // names a timestamp, and group commit needs a connection to answer + // through. None => { + let WriteAttemptKind::Session { conn_id, .. } = attempt else { + soft_panic_or_log!("background write reached the blind write path"); + result.send(WriteResult::Indeterminate); + return; + }; let writes = BTreeMap::from([(target_id, smallvec![table_data])]); self.pending_writes.push(PendingWriteTxn::User { span: Span::current(), @@ -235,16 +257,20 @@ impl Coordinator { } } -/// Validates that all dependencies are valid for read-then-write operations. -/// -/// Ensures all objects the selection transitively depends on (seeded by `ids`) are valid for -/// `ReadThenWrite` operations: +/// Which dependency rules a read-then-write is held to. +#[derive(Clone, Copy, Debug)] +pub(crate) enum DependencyPolicy { + /// A user statement whose relation leaves must be writable user tables. + UserDml, + /// Coordinator-authored work, which may read system objects across time domains. + SystemReads, +} + +/// Validates all transitive dependencies of a read-then-write selection. /// -/// - They do not refer to any objects whose notion of time moves differently than that of -/// user tables. This limitation is meant to ensure no writes occur between this read and the -/// subsequent write. -/// - They do not use mz_now(), whose time produced during read will differ from the write -/// timestamp. +/// User DML requires every relation leaf to be a writable user table. System +/// reads accept supported system objects across time domains. Both reject +/// `mz_now()`. /// /// The first invalid or temporal dependency encountered short-circuits with the corresponding /// error. Traversal is bounded at `max_rw_dependencies` distinct objects, returning @@ -253,6 +279,7 @@ pub(crate) fn validate_read_then_write_dependencies( catalog: &Catalog, ids: impl IntoIterator, max_rw_dependencies: usize, + policy: DependencyPolicy, ) -> Result<(), AdapterError> { use CatalogItemType::*; use mz_catalog::memory::objects; @@ -266,74 +293,82 @@ pub(crate) fn validate_read_then_write_dependencies( enqueue(&mut seen, &mut stack, id, max_rw_dependencies)?; } while let Some(id) = stack.pop() { - let mut ids_to_check = Vec::new(); - let valid = match catalog.try_get_entry(&id) { - Some(entry) => { - if let CatalogItem::View(objects::View { - locally_optimized_expr: optimized_expr, - .. - }) - | CatalogItem::MaterializedView(objects::MaterializedView { - locally_optimized_expr: optimized_expr, - .. - }) = entry.item() - { - if optimized_expr.contains_temporal() { - return Err(AdapterError::Unsupported( - "calls to mz_now in write statements", - )); - } - } - match entry.item().typ() { - typ @ (Func | View | MaterializedView) => { - ids_to_check.extend(entry.uses()); - let valid_id = id.is_user() || matches!(typ, Func); - valid_id - } - Source | Secret | Connection => false, - // Cannot select from sinks or indexes. - Sink | MetricSink | Index => unreachable!(), - Table => { - if !id.is_user() { - // We can't read from non-user tables - false - } else { - // We can't read from tables that are source-exports - entry.source_export_details().is_none() - } - } - Type => true, - } + let Some(entry) = catalog.try_get_entry(&id) else { + return Err(AdapterError::InvalidTableMutationSelection { + object_name: id.to_string(), + object_type: "unknown".to_string(), + }); + }; + + if let CatalogItem::View(objects::View { + locally_optimized_expr: optimized_expr, + .. + }) + | CatalogItem::MaterializedView(objects::MaterializedView { + locally_optimized_expr: optimized_expr, + .. + }) = entry.item() + { + if optimized_expr.contains_temporal() { + return Err(AdapterError::Unsupported( + "calls to mz_now in write statements", + )); + } + } + + let item_type = entry.item().typ(); + let ids_to_check = entry.item().query_dependencies(); + let is_writable_table = matches!( + entry.item(), + CatalogItem::Table(objects::Table { + data_source: objects::TableDataSource::TableWrites { .. }, + .. + }) + ); + let valid = match policy { + DependencyPolicy::UserDml => match item_type { + typ @ (Func | View | MaterializedView) => id.is_user() || matches!(typ, Func), + Source | Secret | Connection => false, + // Cannot select from sinks or indexes. + Sink | MetricSink | Index => unreachable!(), + Table => id.is_user() && is_writable_table, + Type => true, + }, + DependencyPolicy::SystemReads => { + id.is_system() + && matches!( + item_type, + Func | View | MaterializedView | Source | Table | Type + ) } - None => false, }; if !valid { - let (object_name, object_type) = match catalog.try_get_entry(&id) { - Some(entry) => { - let object_name = catalog.resolve_full_name(entry.name(), None).to_string(); - let object_type = match entry.item().typ() { - // We only need the disallowed types here; the allowed types are handled above. - Source => "source", - Secret => "secret", - Connection => "connection", - Table => { - if !id.is_user() { - "system table" - } else { - "source-export table" - } - } - View => "system view", - MaterializedView => "system materialized view", - _ => "invalid dependency", - }; - (object_name, object_type.to_string()) + let object_name = catalog.resolve_full_name(entry.name(), None).to_string(); + let object_type = match item_type { + // We only need the disallowed types here; the allowed types are handled above. + Source => "source", + Secret => "secret", + Connection => "connection", + Table => { + if !id.is_user() { + "system table" + } else if is_writable_table { + "user table" + } else if entry.source_export_details().is_some() { + "source-export table" + } else { + "source-backed table" + } } - None => (id.to_string(), "unknown".to_string()), + View if id.is_user() => "user view", + View => "system view", + MaterializedView if id.is_user() => "user materialized view", + MaterializedView => "system materialized view", + _ => "invalid dependency", }; return Err(AdapterError::InvalidTableMutationSelection { object_name, - object_type, + object_type: object_type.to_string(), }); } for dep in ids_to_check { diff --git a/src/adapter/src/coord/sequencer/inner.rs b/src/adapter/src/coord/sequencer/inner.rs index af7b3b2a5501b..7f1d4dcd8b025 100644 --- a/src/adapter/src/coord/sequencer/inner.rs +++ b/src/adapter/src/coord/sequencer/inner.rs @@ -113,7 +113,7 @@ use crate::command::{ExecuteResponse, Response}; use crate::coord::appends::{ BuiltinTableAppendNotify, DeferredOp, DeferredPlan, PendingWriteTxn, UserWriteResponder, }; -use crate::coord::read_then_write::validate_read_then_write_dependencies; +use crate::coord::read_then_write::{DependencyPolicy, validate_read_then_write_dependencies}; use crate::coord::sequencer::emit_optimizer_notices; use crate::coord::{ AlterConnectionValidationReady, AlterMaterializedViewReadyContext, AlterSinkReadyContext, @@ -2878,6 +2878,7 @@ impl Coordinator { self.catalog(), dependency_ids, max_rw_dependencies, + DependencyPolicy::UserDml, ) { ctx.retire(Err(err)); return; diff --git a/src/adapter/src/coord/sequencer/inner/subscribe.rs b/src/adapter/src/coord/sequencer/inner/subscribe.rs index 4e55f9b3347a2..7d3bb33043160 100644 --- a/src/adapter/src/coord/sequencer/inner/subscribe.rs +++ b/src/adapter/src/coord/sequencer/inner/subscribe.rs @@ -31,7 +31,9 @@ use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{Instrument, Span}; use uuid::Uuid; -use crate::active_compute_sink::{ActiveComputeSink, ActiveSubscribe, SubscribeBacklogAccounting}; +use crate::active_compute_sink::{ + ActiveComputeSink, ActiveSubscribe, ActiveSubscribeOwner, SubscribeBacklogAccounting, +}; use crate::command::ExecuteResponse; use crate::coord::appends::BuiltinTableAppendNotify; use crate::coord::peek::PeekResponseUnary; @@ -557,8 +559,10 @@ impl Coordinator { let max_buffered_bytes = SUBSCRIBE_MAX_BUFFERED_BYTES.get(self.catalog().system_config().dyncfgs()); let active_subscribe = ActiveSubscribe { - conn_id: conn_id.clone(), - session_uuid, + owner: ActiveSubscribeOwner::Session { + conn_id: conn_id.clone(), + session_uuid, + }, channel: tx, backlog_accounting: Arc::clone(&backlog_accounting), max_buffered_bytes, diff --git a/src/adapter/src/coord/sql.rs b/src/adapter/src/coord/sql.rs index ebf9804e7d6c8..354586e8ca2dd 100644 --- a/src/adapter/src/coord/sql.rs +++ b/src/adapter/src/coord/sql.rs @@ -269,49 +269,58 @@ impl Coordinator { id: GlobalId, active_sink: ActiveComputeSink, ) -> BuiltinTableAppendNotify { - let user = self.active_conns()[active_sink.connection_id()].user(); - let session_type = metrics::session_type_label_value(user); - - self.active_conns - .get_mut(active_sink.connection_id()) - .expect("must exist for active sessions") - .drop_sinks - .insert(id); + let session_type = match active_sink.connection_id() { + Some(conn_id) => { + let session_type = + metrics::session_type_label_value(self.active_conns()[conn_id].user()); + self.active_conns + .get_mut(conn_id) + .expect("must exist for active sessions") + .drop_sinks + .insert(id); + session_type + } + None => "system", + }; let ret_fut: BuiltinTableAppendNotify = match &active_sink { ActiveComputeSink::Subscribe(active_subscribe) => { - if active_subscribe.internal { + match active_subscribe.introspection_session_uuid() { // An internal subscribe writes no `mz_subscriptions` row, so // it stays out of the public `mz_active_subscribes` gauge // too. Counting it there would report subscribes that // introspection deliberately shows nothing of. It gets its // own internal gauge instead, since it is still a dataflow // holding cluster resources. - self.metrics - .active_internal_subscribes - .with_label_values(&[session_type]) - .inc(); + None => { + self.metrics + .active_internal_subscribes + .with_label_values(&[session_type]) + .inc(); - Box::pin(std::future::ready(())) - } else { - let update = self.catalog().state().pack_subscribe_update( - id, - active_subscribe, - Diff::ONE, - ); - let update = self.catalog().state().resolve_builtin_table_update(update); + Box::pin(std::future::ready(())) + } + Some(session_uuid) => { + let update = self.catalog().state().pack_subscribe_update( + id, + active_subscribe, + session_uuid, + Diff::ONE, + ); + let update = self.catalog().state().resolve_builtin_table_update(update); - self.metrics - .active_subscribes - .with_label_values(&[session_type]) - .inc(); + self.metrics + .active_subscribes + .with_label_values(&[session_type]) + .inc(); - // Defer the introspection-row write to a group commit instead of - // committing it inline. An inline `execute` would block the coordinator - // loop on a timestamp-oracle round trip and stall every other session. - // `implement_subscribe` waits for this write before returning the - // `SUBSCRIBE` response to the subscribing session. - self.builtin_table_update().defer(vec![update]) + // Defer the introspection-row write to a group commit instead of + // committing it inline. An inline `execute` would block the coordinator + // loop on a timestamp-oracle round trip and stall every other session. + // `implement_subscribe` waits for this write before returning the + // `SUBSCRIBE` response to the subscribing session. + self.builtin_table_update().defer(vec![update]) + } } } ActiveComputeSink::CopyTo(_) => { @@ -345,46 +354,56 @@ impl Coordinator { id: GlobalId, ) -> Option<(ActiveComputeSink, BuiltinTableAppendNotify)> { if let Some(sink) = self.active_compute_sinks.remove(&id) { - let user = self.active_conns()[sink.connection_id()].user(); - let session_type = metrics::session_type_label_value(user); - - self.active_conns - .get_mut(sink.connection_id()) - .expect("must exist for active compute sink") - .drop_sinks - .remove(&id); + let session_type = match sink.connection_id() { + Some(conn_id) => { + let session_type = + metrics::session_type_label_value(self.active_conns()[conn_id].user()); + self.active_conns + .get_mut(conn_id) + .expect("must exist for active compute sink") + .drop_sinks + .remove(&id); + session_type + } + None => "system", + }; let write_notify: BuiltinTableAppendNotify = match &sink { ActiveComputeSink::Subscribe(active_subscribe) => { - if active_subscribe.internal { + match active_subscribe.introspection_session_uuid() { // No introspection row to retract, see // `add_active_compute_sink`. The internal gauge is // decremented here to stay symmetric with it. - self.metrics - .active_internal_subscribes - .with_label_values(&[session_type]) - .dec(); + None => { + self.metrics + .active_internal_subscribes + .with_label_values(&[session_type]) + .dec(); - Box::pin(std::future::ready(())) - } else { - let update = self.catalog().state().pack_subscribe_update( - id, - active_subscribe, - Diff::MINUS_ONE, - ); - let update = self.catalog().state().resolve_builtin_table_update(update); + Box::pin(std::future::ready(())) + } + Some(session_uuid) => { + let update = self.catalog().state().pack_subscribe_update( + id, + active_subscribe, + session_uuid, + Diff::MINUS_ONE, + ); + let update = + self.catalog().state().resolve_builtin_table_update(update); - self.metrics - .active_subscribes - .with_label_values(&[session_type]) - .dec(); + self.metrics + .active_subscribes + .with_label_values(&[session_type]) + .dec(); - // Defer the retraction to a group commit, for the same reason we - // defer the insert (see `add_active_compute_sink`): committing inline - // would block the coordinator loop. Callers that expose the - // retirement wait on the notify off the coordinator loop - // before responding. - self.builtin_table_update().defer(vec![update]) + // Defer the retraction to a group commit, for the same reason we + // defer the insert (see `add_active_compute_sink`): committing inline + // would block the coordinator loop. Callers that expose the + // retirement wait on the notify off the coordinator loop + // before responding. + self.builtin_table_update().defer(vec![update]) + } } } ActiveComputeSink::CopyTo(_) => { diff --git a/src/adapter/src/error.rs b/src/adapter/src/error.rs index a7cafa8cc51ec..6f000e8b120ff 100644 --- a/src/adapter/src/error.rs +++ b/src/adapter/src/error.rs @@ -16,7 +16,6 @@ use dec::TryFromDecimalError; use itertools::Itertools; use mz_catalog::builtin::MZ_CATALOG_SERVER_CLUSTER; use mz_compute_client::controller::error as compute_error; -use mz_compute_client::controller::error::InstanceMissing; use mz_compute_types::ComputeInstanceId; use mz_controller_types::ClusterId; @@ -106,9 +105,9 @@ pub enum AdapterError { }, /// The selection value for a table mutation operation refers to an invalid object. InvalidTableMutationSelection { - /// The full name of the problematic object (e.g. a source or source-export table). + /// The full name of the problematic object, such as a source or source-backed table. object_name: String, - /// Human-readable type of the object (e.g. "source", "source-export table"). + /// Human-readable type of the object, such as "source-backed table". object_type: String, }, /// A read-then-write statement's read set has more transitive dependencies @@ -581,8 +580,7 @@ impl AdapterError { object_type, } => Some(format!( "{object_type} '{}' may not be used in this operation; \ - the selection may refer to views and materialized views, but transitive \ - dependencies must not include sources or source-export tables", + every relation leaf in the selection must be a writable user table", object_name.quoted() )), AdapterError::ReadThenWriteDependencyLimitExceeded { @@ -1089,13 +1087,6 @@ impl AdapterError { // For example, maybe we get an `InstanceMissing` if the user specifies a non-existing cluster, // in which case `ConcurrentDependencyDrop` would not be appropriate. - pub fn concurrent_dependency_drop_from_instance_missing(e: InstanceMissing) -> Self { - AdapterError::ConcurrentDependencyDrop { - dependency_kind: "cluster", - dependency_id: e.0.to_string(), - } - } - pub fn concurrent_dependency_drop_from_collection_missing(e: CollectionMissing) -> Self { AdapterError::ConcurrentDependencyDrop { dependency_kind: "collection", @@ -1278,7 +1269,7 @@ impl fmt::Display for AdapterError { AdapterError::InvalidTableMutationSelection { .. } => { write!( f, - "invalid selection: operation may only (transitively) refer to non-source, non-system tables" + "invalid selection: every relation leaf must be a writable user table" ) } AdapterError::ReadThenWriteDependencyLimitExceeded { diff --git a/src/adapter/src/frontend_peek.rs b/src/adapter/src/frontend_peek.rs index fc05dab424522..37af6a1ef4895 100644 --- a/src/adapter/src/frontend_peek.rs +++ b/src/adapter/src/frontend_peek.rs @@ -353,7 +353,7 @@ impl PeekClient { connection_id: *connection_id, tx, }) - .await + .await? } SideEffectingFunc::PgCancelBackend { connection_id: None, @@ -378,7 +378,7 @@ impl PeekClient { conn_id: session.conn_id().clone(), tx, }) - .await?; + .await??; // We held the target's `ConnectionId` handle from the RBAC // check until the Coordinator executed the function, which @@ -563,7 +563,7 @@ impl PeekClient { real_time_recency_timeout: *vars.real_time_recency_timeout(), tx, }) - .await? + .await?? } else { None }; @@ -611,7 +611,7 @@ impl PeekClient { conn_id: session.conn_id().clone(), tx, }) - .await; + .await?; if let Some(txn_read_holds) = txn_read_holds_opt { let allowed_id_bundle = txn_read_holds.id_bundle(); @@ -695,7 +695,7 @@ impl PeekClient { read_holds: read_holds.clone(), tx, }) - .await; + .await?; } (determination, read_holds) @@ -1334,7 +1334,7 @@ impl PeekClient { watch_set, tx, }) - .await?; + .await??; // On success the peek is registered in `pending_peeks`, // which now owns end-of-execution logging. On error the // coordinator logs nothing (see @@ -1356,7 +1356,7 @@ impl PeekClient { determination, tx, }) - .await; + .await?; session.add_notice(AdapterNotice::QueryTimestamp { explanation }); } @@ -1405,7 +1405,7 @@ impl PeekClient { statement_logging_id: logging.id(), tx, }) - .await?; + .await??; // On success the `Subscribing` response carries the // coordinator-side logging guard and the protocol layer logs // the end when the subscribe terminates. On error the @@ -1456,7 +1456,7 @@ impl PeekClient { sink_id, tx, }) - .await?; + .await??; // Preflight succeeded, now execute the actual COPY TO dataflow let watch_set = logging.id().map(|logging_id| { @@ -1482,7 +1482,7 @@ impl PeekClient { watch_set, tx, }) - .await?; + .await??; Ok(Some(response)) } diff --git a/src/adapter/src/frontend_read_then_write.rs b/src/adapter/src/frontend_read_then_write.rs index 78d04e3894efa..63e6012530979 100644 --- a/src/adapter/src/frontend_read_then_write.rs +++ b/src/adapter/src/frontend_read_then_write.rs @@ -133,6 +133,7 @@ use std::time::Duration; use bytesize::ByteSize; use differential_dataflow::consolidation; use mz_catalog::memory::error::ErrorKind; +use mz_catalog::memory::objects::CatalogItem; use mz_cluster_client::ReplicaId; use mz_compute_types::ComputeInstanceId; use mz_expr::Eval; @@ -154,15 +155,17 @@ use timely::progress::Antichain; use tokio::sync::mpsc; use uuid::Uuid; +use crate::active_compute_sink::ActiveSubscribeOwner; use crate::catalog::Catalog; -use crate::command::{Command, ExecuteResponse}; +use crate::command::{Command, ExecuteResponse, WriteAttemptKind}; use crate::coord::appends::WriteResult; -use crate::coord::read_then_write::validate_read_then_write_dependencies; +use crate::coord::read_then_write::{DependencyPolicy, validate_read_then_write_dependencies}; use crate::coord::timestamp_selection::TimestampProvider; use crate::coord::{Coordinator, TargetCluster}; use crate::error::AdapterError; use crate::optimize::Optimize; use crate::optimize::dataflows::{ComputeInstanceSnapshot, EvalTime, ExprPrep, ExprPrepOneShot}; +use crate::peek_client::CoordinatorClient; use crate::session::{Session, TransactionOps, WriteOp}; use crate::statement_logging::{StatementLifecycleEvent, StatementLoggingId}; use crate::{PeekClient, PeekResponseUnary, TimelineContext, optimize}; @@ -245,6 +248,73 @@ impl FrontendWriteAttemptState { } } +/// Which kind of caller is driving a read-then-write. +/// +/// Dependency rules, replica selection, and write cancellation follow from the +/// caller kind. +#[derive(Clone, Copy)] +enum RtwCaller { + /// A user statement. Cancelled with its connection, and restricted to + /// reading user tables. + Session, + /// Coordinator-owned maintenance, pinned to one replica. + /// + /// The caller must build the statement itself rather than accept one from a + /// user, and must tolerate reading a log relation that is sealed empty, + /// which is how a replica with introspection disabled presents one. + Background { replica_id: ReplicaId }, +} + +impl RtwCaller { + fn is_background(&self) -> bool { + matches!(self, RtwCaller::Background { .. }) + } + + /// Which relations the selection may read. + fn dependency_policy(&self) -> DependencyPolicy { + match self { + RtwCaller::Session => DependencyPolicy::UserDml, + RtwCaller::Background { .. } => DependencyPolicy::SystemReads, + } + } + + /// The replica a background caller pins its subscribe to, overriding the + /// session's replica selection. + fn replica_override(&self) -> Option { + match self { + RtwCaller::Background { replica_id } => Some(*replica_id), + RtwCaller::Session => None, + } + } + + /// Who owns the subscribe, which decides whether it is cancelled with a + /// connection and whether it counts against one. + fn subscribe_owner( + &self, + conn_id: &mz_adapter_types::connection::ConnectionId, + session_uuid: Uuid, + ) -> ActiveSubscribeOwner { + match self { + RtwCaller::Session => ActiveSubscribeOwner::Session { + conn_id: conn_id.clone(), + session_uuid, + }, + RtwCaller::Background { .. } => ActiveSubscribeOwner::Background, + } + } + + /// The connection a pending write is cancelled with, if any. + fn write_conn_id( + &self, + conn_id: &mz_adapter_types::connection::ConnectionId, + ) -> Option { + match self { + RtwCaller::Session => Some(conn_id.clone()), + RtwCaller::Background { .. } => None, + } + } +} + /// What the OCC loop produced. enum OccOutcome { /// The write is durable at `write_ts`. @@ -356,6 +426,7 @@ fn end_own_transaction(session: &mut Session, stages_rows: bool) { pub(crate) fn validate_selection_dependencies( catalog: &Catalog, depends_on: &BTreeSet, + policy: DependencyPolicy, ) -> Result<(), AdapterError> { let dependency_ids = depends_on .iter() @@ -370,7 +441,7 @@ pub(crate) fn validate_selection_dependencies( .collect::, _>>()?; let max_rw_dependencies = mz_adapter_types::dyncfgs::READ_THEN_WRITE_MAX_DEPENDENCIES .get(catalog.system_config().dyncfgs()); - validate_read_then_write_dependencies(catalog, dependency_ids, max_rw_dependencies) + validate_read_then_write_dependencies(catalog, dependency_ids, max_rw_dependencies, policy) } /// Validates a read-then-write and resolves the context the rest of the @@ -387,11 +458,15 @@ pub(crate) fn validate_selection_dependencies( /// serves planning, validation and optimization, so items the plan names /// cannot disappear from it, and the missing-entry branches below are /// failsafes rather than a live concurrent-DDL path. +/// +/// `dependency_policy` decides which relations the selection may read. Both +/// policies reject `mz_now()` anywhere in the transitive dependencies. fn validate_read_then_write( catalog: &Arc, session: &Session, plan: &plan::ReadThenWritePlan, target_cluster: TargetCluster, + dependency_policy: DependencyPolicy, ) -> Result { if contains_mz_now(plan) { return Err(AdapterError::Unsupported( @@ -403,7 +478,7 @@ fn validate_read_then_write( // timeline validation below. let depends_on = plan.selection.depends_on(); - validate_selection_dependencies(catalog, &depends_on)?; + validate_selection_dependencies(catalog, &depends_on, dependency_policy)?; let cluster = catalog.resolve_target_cluster(target_cluster, session)?; let cluster_id = cluster.id; @@ -602,7 +677,7 @@ struct SubscribeHandle { rx: mpsc::UnboundedReceiver, sink_id: GlobalId, /// Wrapped in `Option` so we can move it out in `Drop`. - client: Option, + client: Option, } impl SubscribeHandle { @@ -638,6 +713,68 @@ impl PeekClient { /// `catalog`, which stays in force through optimization and write-target /// generation capture. pub(crate) async fn frontend_read_then_write( + &mut self, + session: &mut Session, + plan: plan::ReadThenWritePlan, + target_cluster: TargetCluster, + catalog: &Arc, + statement_logging_id: Option, + attempt_state: Arc, + ) -> Result { + self.read_then_write( + session, + plan, + target_cluster, + catalog, + statement_logging_id, + attempt_state, + RtwCaller::Session, + ) + .await + } + + /// Executes a coordinator-owned read-then-write against system relations, + /// pinned to `replica_id`. + /// + /// See `RtwCaller::Background` for what the caller takes on by using this. + pub(crate) async fn background_read_then_write( + &mut self, + session: &mut Session, + plan: plan::ReadThenWritePlan, + cluster_id: ComputeInstanceId, + replica_id: ReplicaId, + catalog: &Arc, + ) -> Result { + let is_system_table = matches!(plan.id, CatalogItemId::System(_)) + && catalog + .try_get_entry(&plan.id) + .is_some_and(|entry| matches!(entry.item(), CatalogItem::Table(_))); + if !is_system_table { + soft_panic_or_log!( + "background read-then-write target {} is not a system table", + plan.id + ); + return Err(AdapterError::Internal( + "background read-then-write target is not a system table".into(), + )); + } + + self.read_then_write( + session, + plan, + TargetCluster::Transaction(cluster_id), + catalog, + None, + // Nothing cancels a background write, so this state only ever + // records that a write was submitted. + Arc::new(FrontendWriteAttemptState::new()), + RtwCaller::Background { replica_id }, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn read_then_write( &mut self, session: &mut Session, mut plan: plan::ReadThenWritePlan, @@ -645,7 +782,18 @@ impl PeekClient { catalog: &Arc, statement_logging_id: Option, attempt_state: Arc, + caller: RtwCaller, ) -> Result { + // The OCC dataflow emits raw diffs and does not apply top-level + // finishing. Silently dropping a LIMIT, OFFSET, projection or ordering + // can change the rows written, so this stage requires trivial finishing. + if !plan.finishing.is_trivial(plan.selection.arity()) { + soft_panic_or_log!("frontend read-then-write received nontrivial row-set finishing"); + return Err(AdapterError::Internal( + "frontend read-then-write requires trivial row-set finishing".into(), + )); + } + // A transaction that has taken a timestamped read, was opened READ // ONLY, or is committed to some other kind of operation cannot take a // write. Check up front, mirroring `sequence_insert`: the marker op @@ -661,15 +809,24 @@ impl PeekClient { return Err(AdapterError::ReadOnlyTransaction); } - let validation_result = validate_read_then_write(catalog, session, &plan, target_cluster)?; + let validation_result = validate_read_then_write( + catalog, + session, + &plan, + target_cluster, + caller.dependency_policy(), + )?; let ValidationResult { cluster_id, - replica_id, + mut replica_id, timeline, depends_on, table_desc, } = validation_result; + if let Some(pinned) = caller.replica_override() { + replica_id = Some(pinned); + } // A write that reads no persisted state may join a surrounding // transaction. Its rows do not come from a snapshot, so staging them @@ -745,10 +902,25 @@ impl PeekClient { // // The semaphore is owned by the coordinator and outlives every // session task, so `acquire_owned` cannot return `Err` in practice. - let permit = Arc::clone(&self.occ_write_semaphore) - .acquire_owned() - .await - .expect("semaphore is never closed during coordinator lifetime"); + // + // Background maintenance skips the queue entirely. It is single-flight + // by construction, one sweep at a time and one mutation at a time, so it + // adds at most one concurrent read-then-write. Taking a permit instead + // would let it hold one for as long as a subscribe on a loaded user + // replica takes to hydrate, and with `max_concurrent_occ_writes` set low + // that stalls user DML behind a background sampler. The bound above does + // not apply to it either: it has no statement timeout, only its own much + // longer one. + let permit = if caller.is_background() { + None + } else { + Some( + Arc::clone(&self.occ_write_semaphore) + .acquire_owned() + .await + .expect("semaphore is never closed during coordinator lifetime"), + ) + }; // Determine timestamp and acquire read holds. let oracle_read_ts = self.oracle_read_ts(&timeline).await?; @@ -767,7 +939,7 @@ impl PeekClient { real_time_recency_timeout, tx, }) - .await? + .await?? } else { None }; @@ -823,7 +995,7 @@ impl PeekClient { determination, tx, }) - .await; + .await?; session.add_notice(crate::AdapterNotice::QueryTimestamp { explanation }); } @@ -870,8 +1042,7 @@ impl PeekClient { as_of, arity, sink_id, - conn_id.clone(), - session_uuid, + caller.subscribe_owner(&conn_id, session_uuid), start_time, read_holds, ) @@ -889,7 +1060,7 @@ impl PeekClient { row_set_finishing_seconds, max_occ_retries, table_desc, - conn_id.clone(), + caller.write_conn_id(&conn_id), statement_logging_id, as_of, write_oracle, @@ -910,7 +1081,7 @@ impl PeekClient { // // The zero-row linearization wait below is the one exception, and hands // the permit back before it parks. - let mut permit = Some(permit); + let mut permit = permit; let response = match result { Ok(OccOutcome::Committed { response, write_ts }) => { // A committed write timestamp for a statement we meant to @@ -992,6 +1163,12 @@ impl PeekClient { .map(|()| response) } Ok(OccOutcome::Blind { response, diffs }) => { + if caller.is_background() { + return Err(AdapterError::Internal( + "background read-then-write unexpectedly had no persisted dependency" + .into(), + )); + } // The subscribe closed on its own even though the selection // reads persisted state, so the input is sealed and the diffs // are frontier-independent after all. Staging them would be @@ -1209,14 +1386,16 @@ impl PeekClient { attempt_state.mark_write_submitted(); let result = self .call_coordinator(|tx| Command::AttemptWrite { - conn_id, + attempt: WriteAttemptKind::Session { + conn_id, + write_ts: None, + }, target_id, target_global_id, diffs, - write_ts: None, tx, }) - .await; + .await?; // Every outcome here terminates the attempt, so `write_submitted` // stays set per its contract. @@ -1253,8 +1432,7 @@ impl PeekClient { as_of: Timestamp, arity: usize, sink_id: GlobalId, - conn_id: mz_adapter_types::connection::ConnectionId, - session_uuid: Uuid, + owner: ActiveSubscribeOwner, start_time: mz_ore::now::EpochMillis, read_holds: crate::ReadHolds, ) -> Result { @@ -1267,13 +1445,12 @@ impl PeekClient { as_of, arity, sink_id, - conn_id, - session_uuid, + owner, start_time, read_holds, tx, }) - .await?; + .await??; Ok(SubscribeHandle { rx, @@ -1317,7 +1494,7 @@ impl PeekClient { row_set_finishing_seconds: Histogram, max_occ_retries: usize, table_desc: RelationDesc, - conn_id: mz_adapter_types::connection::ConnectionId, + write_conn_id: Option, statement_logging_id: Option, as_of: Timestamp, write_oracle: Option + Send + Sync>>, @@ -1554,16 +1731,25 @@ impl PeekClient { // in profiles. Every attempt clones every row, and we retry up to // `max_occ_retries` times. attempt_state.mark_write_submitted(); - let result = self + let result = match self .call_coordinator(|tx| Command::AttemptWrite { - conn_id: conn_id.clone(), + attempt: match write_conn_id.clone() { + Some(conn_id) => WriteAttemptKind::Session { + conn_id, + write_ts: Some(target), + }, + None => WriteAttemptKind::Background { write_ts: target }, + }, target_id, target_global_id, diffs: state.payload.clone(), - write_ts: Some(target), tx, }) - .await; + .await + { + Ok(result) => result, + Err(error) => break Err(error), + }; match classify_write_result(result, target_id, attempt_state) { WriteOutcome::Committed(timestamp) => { diff --git a/src/adapter/src/metrics.rs b/src/adapter/src/metrics.rs index 028858c9e59f1..5cc8492f32960 100644 --- a/src/adapter/src/metrics.rs +++ b/src/adapter/src/metrics.rs @@ -31,6 +31,10 @@ pub struct Metrics { pub storage_usage_collection_time_seconds: Histogram, pub arrangement_sizes_collection_time_seconds: Histogram, pub arrangement_sizes_rows_written: IntCounter, + pub hydration_history_mutations: IntCounterVec, + pub hydration_history_retention_batch_full: IntCounter, + pub hydration_history_rows_affected: IntCounterVec, + pub hydration_history_sweep_duration_seconds: Histogram, pub subscribe_outputs: IntCounterVec, pub canceled_peeks: IntCounter, pub linearize_message_seconds: HistogramVec, @@ -88,7 +92,7 @@ impl Metrics { )), active_internal_subscribes: registry.register(metric!( name: "mz_active_internal_subscribes", - help: "The number of active internal subscribes, which serve frontend-sequenced read-then-write.", + help: "The number of active internal subscribes used by read-then-write operations and background maintenance.", var_labels: ["session_type"], )), active_copy_tos: registry.register(metric!( @@ -139,6 +143,25 @@ impl Metrics { name: "mz_arrangement_sizes_rows_written_total", help: "Total rows appended to mz_object_arrangement_size_history since process start.", )), + hydration_history_mutations: registry.register(metric!( + name: "mz_hydration_history_mutations_total", + help: "Total hydration-history collection and retention mutations since process start.", + var_labels: ["operation", "outcome"], + )), + hydration_history_retention_batch_full: registry.register(metric!( + name: "mz_hydration_history_retention_batch_full_total", + help: "Total hydration-history sweeps whose retention batch was full. Repeated increments mean retention may not be keeping up with its schedule.", + )), + hydration_history_rows_affected: registry.register(metric!( + name: "mz_hydration_history_rows_affected_total", + help: "Total rows changed by hydration-history maintenance since process start.", + var_labels: ["action"], + )), + hydration_history_sweep_duration_seconds: registry.register(metric!( + name: "mz_hydration_history_sweep_duration_seconds", + help: "Wall time of a complete hydration-history collection and retention sweep.", + buckets: histogram_seconds_buckets(0.128, 1024.0), + )), subscribe_outputs: registry.register(metric!( name: "mz_subscribe_outputs", help: "The total number of different subscribe outputs used", diff --git a/src/adapter/src/peek_client.rs b/src/adapter/src/peek_client.rs index 206374f6db53d..b9b77940783ce 100644 --- a/src/adapter/src/peek_client.rs +++ b/src/adapter/src/peek_client.rs @@ -42,7 +42,8 @@ use crate::catalog::Catalog; use crate::command::{CatalogSnapshot, Command, ExecuteResponse}; use crate::coord::appends::GroupCommitNotifier; use crate::coord::peek::FastPathPlan; -use crate::coord::{Coordinator, ExecuteContextExtra, ExecuteContextGuard}; +use crate::coord::{Coordinator, ExecuteContextExtra, ExecuteContextGuard, Message}; +use crate::metrics::Metrics; use crate::session::{LifecycleTimestamps, Session}; use crate::statement_logging::{ FrontendStatementLoggingEvent, PreparedStatementEvent, PreparedStatementLoggingInfo, @@ -57,7 +58,7 @@ pub type StorageCollectionsHandle = /// Clients needed for peek sequencing in the Adapter Frontend. #[derive(Debug)] pub struct PeekClient { - coordinator_client: Client, + coordinator_client: CoordinatorClient, /// Cache of the latest catalog snapshot. Serves /// [`PeekClient::catalog_snapshot`] without a Coordinator round-trip /// while the catalog's transient revision is unchanged. @@ -91,13 +92,65 @@ pub struct PeekClient { pub read_only: bool, } +/// A command sender that does not make background work keep the coordinator alive. +#[derive(Debug, Clone)] +pub(crate) enum CoordinatorClient { + Session(Client), + Background { + tx: tokio::sync::mpsc::UnboundedSender, + metrics: Metrics, + }, +} + +impl CoordinatorClient { + /// Sends a command, dropping it if the coordinator is gone. + /// + /// A session cannot outrun the coordinator: it holds a `Client`, and the + /// coordinator's loop only exits once every client is dropped, so a failed + /// send there is a real bug. Background work holds no client on purpose, so + /// losing the race with shutdown is normal and must not crash the process. + /// Dropping the command closes its response channel, which the caller handles + /// as an error. + pub(crate) fn send(&self, command: Command) { + if self.try_send(command) { + return; + } + match self { + CoordinatorClient::Session(_) => panic!("coordinator unexpectedly gone"), + CoordinatorClient::Background { .. } => { + tracing::debug!("dropping background command, coordinator is gone") + } + } + } + + pub(crate) fn try_send(&self, command: Command) -> bool { + match self { + CoordinatorClient::Session(client) => client.try_send(command), + CoordinatorClient::Background { tx, .. } => tx + .send(Message::Command( + mz_ore::tracing::OpenTelemetryContext::obtain(), + command, + )) + .is_ok(), + } + } + + pub(crate) fn metrics(&self) -> &Metrics { + match self { + CoordinatorClient::Session(client) => client.metrics(), + CoordinatorClient::Background { metrics, .. } => metrics, + } + } +} + impl PeekClient { /// Creates a PeekClient. /// /// `catalog` seeds the catalog snapshot cache, so that the session's /// first statements don't need a `Command::CatalogSnapshot` round-trip. - pub fn new( - coordinator_client: Client, + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + coordinator_client: CoordinatorClient, catalog: &Arc, storage_collections: StorageCollectionsHandle, transient_id_gen: Arc, @@ -129,14 +182,15 @@ impl PeekClient { pub async fn ensure_compute_instance_client( &mut self, compute_instance: ComputeInstanceId, - ) -> Result { + ) -> Result { if !self.compute_instances.contains_key(&compute_instance) { let client = self .call_coordinator(|tx| Command::GetComputeInstanceClient { instance_id: compute_instance, tx, }) - .await?; + .await + .map_err(|_| CollectionLookupError::InstanceShutDown)??; self.compute_instances.insert(compute_instance, client); } Ok(self @@ -156,7 +210,7 @@ impl PeekClient { timeline: timeline.clone(), tx, }) - .await?; + .await??; self.oracles.insert(timeline.clone(), oracle); } Ok(self.oracles.get_mut(&timeline).expect("ensured above")) @@ -196,9 +250,13 @@ impl PeekClient { // The cache is empty, stale, or its allocation is gone: do the // round-trip. let start = std::time::Instant::now(); + // Session clients keep the coordinator loop alive. A dropped response + // here therefore indicates an internal lifecycle bug, unlike a + // background compute lookup racing coordinator shutdown. let CatalogSnapshot { catalog } = self .call_coordinator(|tx| Command::CatalogSnapshot { tx }) - .await; + .await + .expect("coordinator unexpectedly dropped catalog snapshot response"); let metrics = self.coordinator_client.metrics(); metrics .catalog_snapshot_seconds @@ -212,18 +270,18 @@ impl PeekClient { catalog } - pub(crate) async fn call_coordinator(&self, f: F) -> T + /// Calls the coordinator and returns an error if it drops the response. + pub(crate) async fn call_coordinator(&self, f: F) -> Result where F: FnOnce(oneshot::Sender) -> Command, { let (tx, rx) = oneshot::channel(); self.coordinator_client.send(f(tx)); - rx.await - .expect("if the coordinator is still alive, it shouldn't have dropped our call") + Ok(rx.await?) } /// The client for sending commands to the coordinator. - pub(crate) fn coordinator_client(&self) -> &crate::Client { + pub(crate) fn coordinator_client(&self) -> &CoordinatorClient { &self.coordinator_client } @@ -435,7 +493,12 @@ impl PeekClient { let client = self .ensure_compute_instance_client(compute_instance) .await - .map_err(AdapterError::concurrent_dependency_drop_from_instance_missing)?; + .map_err(|error| { + AdapterError::concurrent_dependency_drop_from_collection_lookup_error( + error, + compute_instance, + ) + })?; // Register coordinator tracking of this peek. This has to complete before issuing the peek. // @@ -450,7 +513,7 @@ impl PeekClient { watch_set, tx, }) - .await?; + .await??; // The peek is registered: the coordinator's `pending_peeks` entry now // owns end-of-execution logging. It logs the end on peek completion, @@ -491,14 +554,15 @@ impl PeekClient { // ask it to unregister the peek and retire it with this error. If // a concurrent teardown already retired the peek, the end is // already logged and the unregistration is a no-op. - self.call_coordinator(|tx| Command::UnregisterFrontendPeek { - uuid, - reason: statement_logging::StatementEndedExecutionReason::Errored { - error: err.to_string(), - }, - tx, - }) - .await; + let _ = self + .call_coordinator(|tx| Command::UnregisterFrontendPeek { + uuid, + reason: statement_logging::StatementEndedExecutionReason::Errored { + error: err.to_string(), + }, + tx, + }) + .await; return Err(err); } @@ -645,7 +709,7 @@ impl PeekClient { struct StatementLoggingGuard { /// `None` if the statement was not sampled for logging. id: Option, - coordinator_client: Client, + coordinator_client: CoordinatorClient, now: mz_ore::now::NowFn, } diff --git a/src/catalog/src/memory/objects.rs b/src/catalog/src/memory/objects.rs index 39a84ca35c32a..e6fd520c3a8a2 100644 --- a/src/catalog/src/memory/objects.rs +++ b/src/catalog/src/memory/objects.rs @@ -2033,6 +2033,33 @@ impl CatalogItem { uses } + /// Returns direct dependencies to traverse when finding this item's query leaves. + /// + /// Functions and views return all of their uses. Materialized views do the + /// same except for the replacement target, which is lifecycle metadata. All + /// other items are query leaves or cannot be selected from. + pub fn query_dependencies(&self) -> BTreeSet { + match self { + CatalogItem::Func(_) | CatalogItem::View(_) => self.uses(), + CatalogItem::MaterializedView(mv) => { + let mut dependencies = self.uses(); + if let Some(target) = mv.replacement_target { + dependencies.remove(&target); + } + dependencies + } + CatalogItem::Index(_) + | CatalogItem::Sink(_) + | CatalogItem::Source(_) + | CatalogItem::Log(_) + | CatalogItem::Table(_) + | CatalogItem::Type(_) + | CatalogItem::Secret(_) + | CatalogItem::Connection(_) + | CatalogItem::MetricSink(_) => BTreeSet::new(), + } + } + /// Returns the connection ID that this item belongs to, if this item is /// temporary. pub fn conn_id(&self) -> Option<&ConnectionId> { diff --git a/test/launchdarkly-flag-consistency/mzcompose.py b/test/launchdarkly-flag-consistency/mzcompose.py index 51cfe8432f2d0..da92bb3bf07ee 100644 --- a/test/launchdarkly-flag-consistency/mzcompose.py +++ b/test/launchdarkly-flag-consistency/mzcompose.py @@ -290,6 +290,8 @@ enable_will_distinct_propagation enable_with_ordinality_legacy_fallback grpc_client_connect_timeout + hydration_history_collection_interval + hydration_history_retention_period kafka_buffered_event_resize_threshold_elements kafka_default_aws_privatelink_endpoint_identification_algorithm kafka_poll_max_wait diff --git a/test/restart/mzcompose.py b/test/restart/mzcompose.py index 7742f66b4ea6a..c8b2dd56d3a80 100644 --- a/test/restart/mzcompose.py +++ b/test/restart/mzcompose.py @@ -1436,6 +1436,162 @@ def wait_for(sql: str, expected: list[tuple], what: str) -> None: pass +def workflow_hydration_history_survives_restart(c: Composition) -> None: + """`mz_object_hydration_history` rows outlive the process that wrote them. + + Killing the service also restarts clusterd, so the replica hydrates again + and legitimately records a *second* episode with a fresh `installed_at`. + What must hold is that the pre-restart episode is still there afterwards, + unchanged, and that repeated sweeps do not duplicate it. Asserting a total + row count of one would instead assert that rehydration goes unrecorded. + """ + + def episodes(name: str = "hydration_history_i") -> list[list]: + return c.sql_query(f""" + SELECT h.installed_at::text, h.started_at::text, + h.hydrated_at::text, h.status + FROM mz_internal.mz_object_hydration_history AS h + JOIN mz_internal.mz_object_global_ids AS g ON g.global_id = h.object_id + JOIN mz_catalog.mz_objects AS o ON o.id = g.id + WHERE o.name = '{name}' + ORDER BY h.installed_at""") + + c.down(destroy_volumes=True) + with c.override( + Materialized( + additional_system_parameter_defaults={ + "hydration_history_collection_interval": "1s", + # Pin retention: CI randomizes it, and a short period would + # prune the episode this test restarts around. + "hydration_history_retention_period": "30d", + }, + sanity_restart=False, + ) + ): + c.up("materialized") + c.sql(dedent("""\ + CREATE CLUSTER hydration_history SIZE 'scale=1,workers=2'; + CREATE TABLE hydration_history_t (a int); + INSERT INTO hydration_history_t SELECT generate_series(1, 100000); + CREATE INDEX hydration_history_i + IN CLUSTER hydration_history ON hydration_history_t (a); + CREATE MATERIALIZED VIEW hydration_history_mv_a + IN CLUSTER hydration_history AS SELECT a + 1 AS a FROM hydration_history_t; + CREATE MATERIALIZED VIEW hydration_history_mv_b + IN CLUSTER hydration_history AS SELECT a + 2 AS a FROM hydration_history_t; + """)) + + deadline = time.time() + 120 + before = [] + while time.time() < deadline: + before = episodes() + if before: + break + time.sleep(0.5) + assert ( + len(before) == 1 + ), f"expected exactly one episode, got {before} (empty means it timed out)" + + # Discover which MV's persist-sink worker is off worker 0 instead of + # predicting it from user-ID allocation and hashing. Enough input data + # separates compute completion from the active worker's durable write. + deadline = time.time() + 120 + candidates = [] + worker_rows = [] + with c.sql_cursor(reuse_connection=True) as cursor: + try: + cursor.execute("SET cluster = hydration_history") + cursor.execute("SET cluster_replica = r1") + while time.time() < deadline: + cursor.execute(""" + SELECT + mv.name, + max(h.hydrated_at)::text, + (max(h.hydrated_at) + FILTER (WHERE h.worker_id = 0))::text + FROM mz_introspection.mz_compute_hydration_times_per_worker AS h + JOIN mz_internal.mz_object_global_ids AS g + ON g.global_id = h.export_id + JOIN mz_catalog.mz_materialized_views AS mv ON mv.id = g.id + WHERE mv.name IN ( + 'hydration_history_mv_a', + 'hydration_history_mv_b' + ) + GROUP BY mv.name + HAVING count(*) = 2 + AND count(*) = count(h.hydrated_at) + ORDER BY mv.name + """) + worker_rows = cursor.fetchall() + candidates = [ + row + for row in worker_rows + if row[1] is not None and row[2] != row[1] + ] + if candidates: + break + time.sleep(0.5) + finally: + cursor.execute("RESET cluster_replica") + cursor.execute("RESET cluster") + assert candidates, ( + "test fixture did not produce an MV with its persist-sink worker " + f"off worker 0: {worker_rows}" + ) + mv_name, latest_worker_finish, worker_zero_finish = candidates[0] + + # The selected MV's history episode must use the all-worker maximum, + # not worker 0's earlier compute-only finish. + deadline = time.time() + 120 + mv_episodes = [] + while time.time() < deadline: + mv_episodes = episodes(mv_name) + if mv_episodes: + break + time.sleep(0.5) + assert ( + len(mv_episodes) == 1 + ), f"expected one materialized view episode for {mv_name}, got {mv_episodes}" + assert mv_episodes[0][2] == latest_worker_finish, ( + f"durable finish {mv_episodes[0][2]} did not match " + f"latest worker finish {latest_worker_finish}. " + f"worker 0 finished at {worker_zero_finish}" + ) + + c.kill("materialized") + c.up("materialized") + + # The pre-restart episode must remain byte-identical, and restarting the + # replica must produce exactly one episode with a fresh installation. + deadline = time.time() + 120 + after = [] + fresh = [] + while time.time() < deadline: + after = episodes() + fresh = [episode for episode in after if episode[0] != before[0][0]] + if before[0] in after and len(fresh) == 1: + break + time.sleep(0.5) + assert ( + before[0] in after + ), f"restart lost the pre-restart episode: had {before}, now {after}" + assert ( + len(after) == 2 and len(fresh) == 1 + ), f"expected one preserved and one fresh episode, got {after}" + + # Let several sweeps run. The pre-restart episode must not be duplicated, + # and the post-restart episode must settle at one row too. + time.sleep(10) + settled = episodes() + assert ( + before[0] in settled + ), f"pre-restart episode disappeared: had {before}, now {settled}" + assert len(settled) == len( + set(tuple(row) for row in settled) + ), f"sweeps duplicated a hydration episode: {settled}" + assert len(settled) == 2, f"expected two settled episodes, got {settled}" + + def workflow_default(c: Composition) -> None: def process(name: str) -> None: if name == "default": diff --git a/test/sqllogictest/insert-into-select-source-error.slt b/test/sqllogictest/insert-into-select-source-error.slt index 573b6de6b446b..a2916909c534c 100644 --- a/test/sqllogictest/insert-into-select-source-error.slt +++ b/test/sqllogictest/insert-into-select-source-error.slt @@ -7,9 +7,9 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -# Regression test: INSERT INTO ... SELECT ... cannot transitively refer to sources. -# The selection may refer to views/materialized views, but the transitive -# dependencies must not include sources or source-export tables. +# Regression test: read-then-write selections must resolve to writable user tables. +# The selection may refer to views/materialized views, but its relation leaves +# cannot be sources or source-backed tables. # # Simple (employee, salary, type) example to reproduce the error. @@ -35,16 +35,16 @@ simple INSERT INTO salary_exceptions (employee, salary, type) SELECT 'Alice', salary, type FROM emp_view LIMIT 1 ---- -db error: ERROR: invalid selection: operation may only (transitively) refer to non-source, non-system tables -DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; the selection may refer to views and materialized views, but transitive dependencies must not include sources or source-export tables +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; every relation leaf in the selection must be a writable user table # Same error when selecting directly from source-export table simple INSERT INTO salary_exceptions (employee, salary, type) SELECT 'Bob', counter, 'part_time' FROM employees_from_src LIMIT 1 ---- -db error: ERROR: invalid selection: operation may only (transitively) refer to non-source, non-system tables -DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; the selection may refer to views and materialized views, but transitive dependencies must not include sources or source-export tables +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; every relation leaf in the selection must be a writable user table # Same error when selecting directly from the source. Use a derived table to # alias the single column from emp_src (SELECT * works; column name does not). @@ -52,22 +52,48 @@ simple INSERT INTO salary_exceptions (employee, salary, type) SELECT 'Dave', f1::int, 'contractor' FROM (SELECT * FROM emp_src LIMIT 1) AS t(f1) ---- -db error: ERROR: invalid selection: operation may only (transitively) refer to non-source, non-system tables -DETAIL: source '"materialize.public.emp_src"' may not be used in this operation; the selection may refer to views and materialized views, but transitive dependencies must not include sources or source-export tables +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source '"materialize.public.emp_src"' may not be used in this operation; every relation leaf in the selection must be a writable user table # Same error: UPDATE with WHERE subquery that transitively refers to source simple UPDATE salary_exceptions SET type = 'updated' WHERE salary IN (SELECT salary FROM emp_view) ---- -db error: ERROR: invalid selection: operation may only (transitively) refer to non-source, non-system tables -DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; the selection may refer to views and materialized views, but transitive dependencies must not include sources or source-export tables +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; every relation leaf in the selection must be a writable user table # Same error: DELETE with WHERE subquery that transitively refers to source simple DELETE FROM salary_exceptions WHERE salary IN (SELECT salary FROM emp_view) ---- -db error: ERROR: invalid selection: operation may only (transitively) refer to non-source, non-system tables -DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; the selection may refer to views and materialized views, but transitive dependencies must not include sources or source-export tables +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source-export table '"materialize.public.employees_from_src"' may not be used in this operation; every relation leaf in the selection must be a writable user table + +# A webhook table is source-backed even though it appears in the catalog as a table. +statement ok +CREATE TABLE webhook_events FROM WEBHOOK BODY FORMAT TEXT + +simple +INSERT INTO salary_exceptions (employee, salary, type) +SELECT body, 0, 'webhook' FROM webhook_events +---- +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source-backed table '"materialize.public.webhook_events"' may not be used in this operation; every relation leaf in the selection must be a writable user table + +# A secret referenced by webhook validation is DDL metadata, not a read dependency. +statement ok +CREATE SECRET webhook_secret AS 'secret' + +statement ok +CREATE TABLE authenticated_webhook_events FROM WEBHOOK BODY FORMAT TEXT + CHECK (WITH (BODY, SECRET webhook_secret) body = webhook_secret) + +simple +INSERT INTO salary_exceptions (employee, salary, type) +SELECT body, 0, 'webhook' FROM authenticated_webhook_events +---- +db error: ERROR: invalid selection: every relation leaf must be a writable user table +DETAIL: source-backed table '"materialize.public.authenticated_webhook_events"' may not be used in this operation; every relation leaf in the selection must be a writable user table # Set up a plain old table and do the same thing, it should now work # Valid: INSERT INTO ... SELECT ... from plain table only diff --git a/test/testdrive/hydration-status.td b/test/testdrive/hydration-status.td index 9a05529abf3ca..2757c76d3dfb1 100644 --- a/test/testdrive/hydration-status.td +++ b/test/testdrive/hydration-status.td @@ -23,6 +23,15 @@ # introspection relations are asynchronously updated, so DDL commands are not # immediately reflected there. +# Durable hydration history is collected by a background sweep, which visits one +# replica per cycle. Shorten its cadence so episodes land within this test's +# retry budget. +$ postgres-execute connection=mz_system +ALTER SYSTEM SET hydration_history_collection_interval = '1s'; +# Pin retention too: CI randomizes it, and a short period would prune the +# episodes asserted below before the assertions run. +ALTER SYSTEM SET hydration_history_retention_period = '30d'; + > CREATE CLUSTER test REPLICAS (hydrated_test_1 (SIZE 'scale=1,workers=1')) > SET cluster = test @@ -66,6 +75,35 @@ idx hydrated_test_1 true true mv hydrated_test_1 true true mv_const hydrated_test_1 true false +# Durable history for the objects that hydrated on this replica. Collection +# samples current state, so an object that disappears before its replica's turn is +# not recorded at all. +# +# `mv_const` is deliberately not asserted on either way. It does hydrate and its +# log row does briefly carry a complete episode, but the controller then drops the +# collection and `ExportDropped` retracts the row for good. Whether a sweep lands +# in that window is a race, so pinning either outcome makes this file flaky. It is +# an instance of the general limit, that an episode retracted before its replica's +# turn cannot be recorded, rather than a special case. `idx` and `mv` keep their +# dataflows installed, so they are stable. +# +# The recorded interval must be ordered. The stamps are aggregated over a replica's +# workers, so on a multi-process replica the ends can come from different process +# clocks and the interval carries their skew. These replicas are single-process, so +# the ordering is exact here. The coalesce covers a NULL start, which the replica no +# longer produces but the column still permits. +> SELECT o.name, r.name, h.status, + h.installed_at <= h.hydrated_at, + coalesce(h.installed_at <= h.started_at AND h.started_at <= h.hydrated_at, true) + FROM mz_internal.mz_object_hydration_history AS h + JOIN mz_internal.mz_object_global_ids AS g ON g.global_id = h.object_id + JOIN mz_catalog.mz_objects AS o ON o.id = g.id + JOIN mz_catalog.mz_cluster_replicas AS r ON r.id = h.replica_id + WHERE o.name IN ('idx', 'mv') + ORDER BY o.name, r.name; +idx hydrated_test_1 hydrated true true +mv hydrated_test_1 hydrated true true + > SELECT o.name, r.name, h.hydrated FROM mz_internal.mz_hydration_statuses h JOIN mz_cluster_replicas r ON (r.id = h.replica_id) @@ -109,6 +147,18 @@ mv hydrated_test_2 true true mv_const hydrated_test_1 true false mv_const hydrated_test_2 true false +> SELECT o.name, r.name, h.status + FROM mz_internal.mz_object_hydration_history AS h + JOIN mz_internal.mz_object_global_ids AS g ON g.global_id = h.object_id + JOIN mz_catalog.mz_objects AS o ON o.id = g.id + JOIN mz_catalog.mz_cluster_replicas AS r ON r.id = h.replica_id + WHERE o.name IN ('idx', 'mv') + ORDER BY o.name, r.name; +idx hydrated_test_1 hydrated +idx hydrated_test_2 hydrated +mv hydrated_test_1 hydrated +mv hydrated_test_2 hydrated + > SELECT o.name, r.name, h.hydrated FROM mz_internal.mz_hydration_statuses h JOIN mz_cluster_replicas r ON (r.id = h.replica_id) @@ -140,6 +190,15 @@ mv hydrated_test_2 true > DROP CLUSTER REPLICA test.hydrated_test_1 +# History outlives the replica it describes, so dropping a replica retracts +# nothing. Repeated sweeps must also not duplicate an already recorded episode. +> SELECT count(*) + FROM mz_internal.mz_object_hydration_history AS h + JOIN mz_internal.mz_object_global_ids AS g ON g.global_id = h.object_id + JOIN mz_catalog.mz_objects AS o ON o.id = g.id + WHERE o.name IN ('idx', 'mv'); +4 + > SELECT o.name, r.name, h.hydrated, h.hydration_time IS NOT NULL FROM mz_internal.mz_compute_hydration_statuses h JOIN mz_cluster_replicas r ON (r.id = h.replica_id) @@ -341,6 +400,15 @@ snk hydrated_test_4 true > DROP SOURCE src CASCADE > DROP MATERIALIZED VIEW mv +# Retention runs in the same sweep on the catalog server. It retracts aged-out rows +# through the same OCC path, in bounded batches. A zero retention period expires +# everything. +$ postgres-execute connection=mz_system +ALTER SYSTEM SET hydration_history_retention_period = '0s'; + +> SELECT count(*) FROM mz_internal.mz_object_hydration_history; +0 + > SELECT o.name, r.name, h.hydrated FROM mz_internal.mz_hydration_statuses h JOIN mz_cluster_replicas r ON (r.id = h.replica_id) From ad062d00d65ca61ebc86b2581c88e6922473cc26 Mon Sep 17 00:00:00 2001 From: Qindeel Ishtiaq Date: Wed, 26 Aug 2026 15:49:05 -0400 Subject: [PATCH 06/11] console: show real source status on the Objects page (#38379) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Sources on the maintained-objects (Objects) page were described in replica-hydration terms, which misrepresents them three ways: * The status column showed `Not Hydrated` / `Hydrating` / `Running` buckets derived from `mz_hydration_statuses`. For a source, `hydrated` only tracks whether `rehydration_latency` is set, so a healthy source could read `Not Hydrated` (Slack: #console, Aug 19 thread on what "Not Hydrated" means). * A source could show `99%` snapshot progress next to `1s` freshness because completion was judged by the staged/known ratio, whose denominator is an estimate that can sit below 100% forever. * Webhook sources have no hydration rows at all, so they showed no status. Separately, the source detail panel gave no way to see the ingest cluster's resources while diagnosing a lagging source. ## Changes One commit per concern, reviewable in order: 1. **Source ingestion status on the Objects page.** The status column, filter, and detail-panel badge show the source's ingestion status (`snapshotting`, `running`, `paused`, `stalled`, ...) via `ConnectorStatusPill`, matching the Sources page. Non-source objects keep the hydration buckets. The filter derives from the same function the cell renders, so it can never offer a value the cell doesn't show. Status rides on the existing hydration-aggregate subscribe: the aggregate now FULL JOINs `mz_source_statuses` (so webhook sources get a row) and carries `bool_and(snapshot_committed)` from `mz_source_statistics`. 2. **Snapshot display keys off `snapshot_committed`.** The diagnostics card uses the authoritative boolean instead of the staged/known ratio, and now renders only when it has something diagnostic to say: a status error, or an in-progress snapshot (with a per-source-type note that counts are estimates, shared with the Sources page via `snapshotEstimateNote`). Steady-state lifecycle facts (rehydration time, messages received) already live on the source details page, so the card no longer duplicates them. 3. **Source cluster metrics on the Freshness tab.** The detail panel shows the ingest cluster's replica memory/CPU at the point selected on the freshness graph, including replicas dropped in the window (reuses the critical-path cluster metrics via a new shared `ClusterReplicaMetrics` component; falls back to each replica's latest bucket when the anchor bucket predates a replica swap). 4. **Feed hardening.** Subsources and progress collections are excluded from the aggregate (the UI hides them, and they multiply the feed's cardinality several times over), and the `snapshot_committed` aggregate is restricted to statistics rows from live replicas (keeping null-`replica_id` rows, which is how webhook sources report), so a dropped replica's stale row cannot pin a committed source back to snapshotting. ## Tests Extended `hydrationAggregate.test.sql.ts` for the FULL JOIN, `bool_and(snapshot_committed)`, subsource/progress exclusion, and live-replica semantics, and `MaintainedObjects.test.tsx` for the source status column and filter. Closes [CNS-97](https://linear.app/materializeinc/issue/CNS-97) Closes [CNS-82](https://linear.app/materializeinc/issue/CNS-82) Closes [CNS-85](https://linear.app/materializeinc/issue/CNS-85) 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- .../hydrationAggregate.test.sql.ts | 253 ++++++++++++++---- .../maintained-objects/hydrationAggregate.ts | 80 +++++- console/src/platform/connectors/utils.ts | 22 ++ .../CriticalPathClusterMetrics.tsx | 65 +++-- .../maintained-objects/FilterChips.tsx | 10 +- .../MaintainedObjects.test.tsx | 62 ++++- .../maintained-objects/MaintainedObjects.tsx | 81 ++++-- .../maintained-objects/ObjectDetailsCard.tsx | 21 +- .../maintained-objects/ObjectFreshness.tsx | 32 ++- .../maintained-objects/ObjectMetadata.tsx | 8 +- .../maintained-objects/SourceDiagnostics.tsx | 87 +++--- .../criticalPathRenderable.test.ts | 1 + .../maintained-objects/filterPanels.tsx | 14 +- .../platform/maintained-objects/filters.ts | 92 ++++++- .../platform/maintained-objects/queries.ts | 31 ++- .../SourceOverview/SnapshotProgress.tsx | 25 +- 16 files changed, 686 insertions(+), 198 deletions(-) diff --git a/console/src/api/materialize/maintained-objects/hydrationAggregate.test.sql.ts b/console/src/api/materialize/maintained-objects/hydrationAggregate.test.sql.ts index 5e91e4b963a67..2c0655847ea55 100644 --- a/console/src/api/materialize/maintained-objects/hydrationAggregate.test.sql.ts +++ b/console/src/api/materialize/maintained-objects/hydrationAggregate.test.sql.ts @@ -16,23 +16,76 @@ import { testdrive } from "~/test/sql/mzcompose"; import { SEARCH_PATH } from "../executeSql"; import { buildHydrationAggregateQuery } from "./hydrationAggregate"; +const TEST_SCHEMA = "test_hydration_aggregate"; + +/** Shadows the four relations the query reads. Empty value lists are elided. + * `replicas` defaults to the ids the fixtures use, so statistics rows count + * as coming from live replicas unless a test overrides it. */ +const seed = ({ + hydration = "", + statuses = "", + statistics = "", + replicas = "('r1'), ('r2'), ('r3')", + sources = "", +}: { + hydration?: string; + statuses?: string; + statistics?: string; + replicas?: string; + sources?: string; +}) => ` + > DROP SCHEMA IF EXISTS ${TEST_SCHEMA} CASCADE; + > CREATE SCHEMA ${TEST_SCHEMA}; + > CREATE TABLE ${TEST_SCHEMA}.mz_hydration_statuses ( + object_id TEXT NOT NULL, + replica_id TEXT, + hydrated BOOLEAN + ); + > CREATE TABLE ${TEST_SCHEMA}.mz_sources ( + id TEXT NOT NULL, + type TEXT NOT NULL + ); + > CREATE TABLE ${TEST_SCHEMA}.mz_source_statuses ( + id TEXT NOT NULL, + status TEXT NOT NULL, + error TEXT, + type TEXT NOT NULL + ); + > CREATE TABLE ${TEST_SCHEMA}.mz_source_statistics ( + id TEXT NOT NULL, + replica_id TEXT, + snapshot_committed BOOLEAN + ); + > CREATE TABLE ${TEST_SCHEMA}.mz_cluster_replicas ( + id TEXT NOT NULL + ); + ${hydration ? `> INSERT INTO ${TEST_SCHEMA}.mz_hydration_statuses VALUES ${hydration};` : ""} + ${statuses ? `> INSERT INTO ${TEST_SCHEMA}.mz_source_statuses VALUES ${statuses};` : ""} + ${statistics ? `> INSERT INTO ${TEST_SCHEMA}.mz_source_statistics VALUES ${statistics};` : ""} + ${replicas ? `> INSERT INTO ${TEST_SCHEMA}.mz_cluster_replicas VALUES ${replicas};` : ""} + ${sources ? `> INSERT INTO ${TEST_SCHEMA}.mz_sources VALUES ${sources};` : ""} +`; + +const run = async () => { + const compiled = buildHydrationAggregateQuery().compile(); + const result = await executeSqlHttp(compiled, { + sessionVariables: { + cluster: QUICKSTART_CLUSTER, + search_path: `${TEST_SCHEMA}, ${SEARCH_PATH}`, + }, + }); + return result.rows.sort((a, b) => a.object_id.localeCompare(b.object_id)); +}; + describe("buildHydrationAggregateQuery", () => { it("aggregates hydrated and total replicas per object", async () => { - const testSchema = "test_hydration_aggregate"; - // u1: single replica, hydrated → 1/1 // u2: 3 replicas, all hydrated → 3/3 // u3: 3 replicas, 1 hydrated → 1/3 // u4: 2 replicas, none hydrated → 0/2 - await testdrive(` - > DROP SCHEMA IF EXISTS ${testSchema} CASCADE; - > CREATE SCHEMA ${testSchema}; - > CREATE TABLE ${testSchema}.mz_hydration_statuses ( - object_id TEXT NOT NULL, - replica_id TEXT, - hydrated BOOLEAN - ); - > INSERT INTO ${testSchema}.mz_hydration_statuses VALUES + await testdrive( + seed({ + hydration: ` ('u1', 'r1', true), ('u2', 'r1', true), ('u2', 'r2', true), @@ -41,20 +94,11 @@ describe("buildHydrationAggregateQuery", () => { ('u3', 'r2', false), ('u3', 'r3', false), ('u4', 'r1', false), - ('u4', 'r2', false); - `); - - const compiled = buildHydrationAggregateQuery().compile(); - const result = await executeSqlHttp(compiled, { - sessionVariables: { - cluster: QUICKSTART_CLUSTER, - search_path: `${testSchema}, ${SEARCH_PATH}`, - }, - }); - - const rows = result.rows.sort((a, b) => - a.object_id.localeCompare(b.object_id), + ('u4', 'r2', false)`, + }), ); + + const rows = await run(); expect(rows).toHaveLength(4); expect(rows[0]).toMatchObject({ object_id: "u1" }); @@ -72,38 +116,149 @@ describe("buildHydrationAggregateQuery", () => { expect(rows[3]).toMatchObject({ object_id: "u4" }); expect(Number(rows[3].hydratedReplicas)).toBe(0); expect(Number(rows[3].totalReplicas)).toBe(2); + + // No source rows seeded, so the source columns are null throughout. + expect(rows[0].sourceStatus).toBeNull(); + expect(rows[0].snapshotCommitted).toBeNull(); }); it("treats null hydrated values as not-hydrated for the FILTER clause", async () => { - const testSchema = "test_hydration_aggregate"; - // A replica that hasn't reported a status yet shows up as a null row; // the FILTER (WHERE hydrated) should not count it as hydrated, but the // total count still includes it. - await testdrive(` - > DROP SCHEMA IF EXISTS ${testSchema} CASCADE; - > CREATE SCHEMA ${testSchema}; - > CREATE TABLE ${testSchema}.mz_hydration_statuses ( - object_id TEXT NOT NULL, - replica_id TEXT, - hydrated BOOLEAN - ); - > INSERT INTO ${testSchema}.mz_hydration_statuses VALUES - ('u1', 'r1', true), - ('u1', 'r2', NULL); - `); - - const compiled = buildHydrationAggregateQuery().compile(); - const result = await executeSqlHttp(compiled, { - sessionVariables: { - cluster: QUICKSTART_CLUSTER, - search_path: `${testSchema}, ${SEARCH_PATH}`, - }, + await testdrive( + seed({ hydration: `('u1', 'r1', true), ('u1', 'r2', NULL)` }), + ); + + const rows = await run(); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ object_id: "u1" }); + expect(Number(rows[0].hydratedReplicas)).toBe(1); + expect(Number(rows[0].totalReplicas)).toBe(2); + }); + + it("joins source status, error and the snapshot flag onto source rows", async () => { + await testdrive( + seed({ + hydration: `('u1', 'r1', true), ('u2', 'r1', false)`, + statuses: ` + ('u1', 'running', NULL, 'kafka'), + ('u2', 'stalled', 'broker unreachable', 'kafka')`, + statistics: `('u1', 'r1', true), ('u2', 'r1', false)`, + }), + ); + + const rows = await run(); + expect(rows).toHaveLength(2); + expect(rows[0]).toMatchObject({ + object_id: "u1", + sourceStatus: "running", + sourceError: null, + snapshotCommitted: true, }); + expect(rows[1]).toMatchObject({ + object_id: "u2", + sourceStatus: "stalled", + sourceError: "broker unreachable", + snapshotCommitted: false, + }); + }); + + it("only reports the snapshot as committed once every replica agrees", async () => { + // A replica that is still snapshotting holds the whole source at false, so + // the UI keeps showing Snapshotting rather than flipping to Running early. + await testdrive( + seed({ + hydration: `('u1', 'r1', true)`, + statuses: `('u1', 'running', NULL, 'kafka')`, + statistics: `('u1', 'r1', true), ('u1', 'r2', false)`, + }), + ); + + const rows = await run(); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + object_id: "u1", + snapshotCommitted: false, + }); + }); + + it("keeps sources that have a status but no hydration rows", async () => { + // Webhook sources are omitted from mz_hydration_statuses, so the FULL + // JOIN is what keeps their status visible. + await testdrive( + seed({ + hydration: `('u1', 'r1', true)`, + statuses: `('u1', 'running', NULL, 'kafka'), ('u7', 'running', NULL, 'webhook')`, + }), + ); + + const rows = await run(); + expect(rows).toHaveLength(2); + expect(rows[1]).toMatchObject({ + object_id: "u7", + sourceStatus: "running", + snapshotCommitted: null, + }); + expect(rows[1].hydratedReplicas).toBeNull(); + expect(rows[1].totalReplicas).toBeNull(); + }); - expect(result.rows).toHaveLength(1); - expect(result.rows[0]).toMatchObject({ object_id: "u1" }); - expect(Number(result.rows[0].hydratedReplicas)).toBe(1); - expect(Number(result.rows[0].totalReplicas)).toBe(2); + it("excludes subsources and progress collections from the feed", async () => { + // The UI hides them, and each source carries one progress collection and + // often several subsources, so keeping them out shrinks the subscribe. + // Subsources also have hydration rows of their own (they run on the + // parent's cluster), so the exclusion must hold on both join sides. + await testdrive( + seed({ + hydration: `('u1', 'r1', true), ('u2', 'r1', true)`, + statuses: ` + ('u1', 'running', NULL, 'postgres'), + ('u2', 'running', NULL, 'subsource'), + ('u3', 'running', NULL, 'progress')`, + sources: `('u1', 'postgres'), ('u2', 'subsource'), ('u3', 'progress')`, + }), + ); + + const rows = await run(); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ object_id: "u1", sourceStatus: "running" }); + }); + + it("ignores statistics rows from dropped replicas", async () => { + // Statistics rows can outlive their replica; a dropped replica's stale + // false must not pin a committed source back to snapshotting. + await testdrive( + seed({ + statuses: `('u1', 'running', NULL, 'kafka')`, + statistics: `('u1', 'r1', true), ('u1', 'r_dropped', false)`, + replicas: `('r1')`, + }), + ); + + const rows = await run(); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + object_id: "u1", + snapshotCommitted: true, + }); + }); + + it("keeps statistics rows that report no replica", async () => { + // Webhook sources report statistics with a null replica_id. + await testdrive( + seed({ + statuses: `('u1', 'running', NULL, 'webhook')`, + statistics: `('u1', NULL, true)`, + replicas: `('r1')`, + }), + ); + + const rows = await run(); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + object_id: "u1", + snapshotCommitted: true, + }); }); }); diff --git a/console/src/api/materialize/maintained-objects/hydrationAggregate.ts b/console/src/api/materialize/maintained-objects/hydrationAggregate.ts index 5856929f96d30..9e4c26c461241 100644 --- a/console/src/api/materialize/maintained-objects/hydrationAggregate.ts +++ b/console/src/api/materialize/maintained-objects/hydrationAggregate.ts @@ -13,23 +13,85 @@ import { queryBuilder } from "~/api/materialize"; export interface HydrationAggregateRow { object_id: string; - hydratedReplicas: bigint; - totalReplicas: bigint; + hydratedReplicas: bigint | null; + totalReplicas: bigint | null; + sourceStatus: string | null; + sourceError: string | null; + snapshotCommitted: boolean | null; } /** - * One row per object: how many of its replicas report `hydrated = true`, out - * of the total replicas recorded in `mz_hydration_statuses` + * One row per object: how many of its replicas report `hydrated = true` out of + * the total replicas recorded in `mz_hydration_statuses`, plus, for sources, + * their ingestion status, error, and snapshot flag. + * + * Source status rides on this feed rather than its own SUBSCRIBE: it is the + * same one-row-per-object grain, and `mz_hydration_statuses` already reads + * `mz_source_statistics`, so this adds no new frontier dependency. The join + * with `mz_source_statuses` is FULL so webhook sources, which have a status but + * no hydration rows, still get a row. + * + * `snapshot_committed` is aggregated with `bool_and`, so a source only reads as + * committed once every replica reporting statistics agrees. A replica that has + * not reported yet contributes no row, so a scale-up cannot flip a committed + * source back to snapshotting. Statistics rows can outlive their replica, so + * the aggregate is restricted to live replicas (keeping rows with no + * `replica_id`, which is how webhook sources report); a dropped replica's + * stale row must not pin the flag. + * + * Subsources and progress collections are excluded: the UI hides them, and + * they multiply the feed's cardinality several times over. */ export function buildHydrationAggregateQuery() { - return queryBuilder - .selectFrom("mz_hydration_statuses") + // Subsources have hydration rows of their own (they run on their parent's + // cluster), so both sides of the FULL JOIN need the exclusion. + const hydration = queryBuilder + .selectFrom("mz_hydration_statuses as hs") + .leftJoin("mz_sources as s", "s.id", "hs.object_id") + .where((eb) => + eb.or([ + eb("s.type", "is", null), + eb("s.type", "not in", ["subsource", "progress"]), + ]), + ) .select((eb) => [ - "object_id", - sql`count(*) FILTER (WHERE ${eb.ref("hydrated")})`.as( + "hs.object_id", + sql`count(*) FILTER (WHERE ${eb.ref("hs.hydrated")})`.as( "hydratedReplicas", ), sql`count(*)`.as("totalReplicas"), ]) - .groupBy("object_id"); + .groupBy("hs.object_id"); + + const statuses = queryBuilder + .selectFrom("mz_source_statuses") + .where("type", "not in", ["subsource", "progress"]) + .select(["id", "status", "error"]); + + const snapshotCommittedBySource = queryBuilder + .selectFrom("mz_source_statistics as st") + .leftJoin("mz_cluster_replicas as r", "r.id", "st.replica_id") + .where((eb) => + eb.or([eb("r.id", "is not", null), eb("st.replica_id", "is", null)]), + ) + .select((eb) => [ + "st.id", + sql`bool_and(${eb.ref("st.snapshot_committed")})`.as( + "snapshotCommitted", + ), + ]) + .groupBy("st.id"); + + return queryBuilder + .selectFrom(hydration.as("h")) + .fullJoin(statuses.as("ss"), "ss.id", "h.object_id") + .leftJoin(snapshotCommittedBySource.as("stats"), "stats.id", "ss.id") + .select([ + sql`coalesce(h.object_id, ss.id)`.as("object_id"), + "h.hydratedReplicas", + "h.totalReplicas", + "ss.status as sourceStatus", + "ss.error as sourceError", + "stats.snapshotCommitted", + ]); } diff --git a/console/src/platform/connectors/utils.ts b/console/src/platform/connectors/utils.ts index 957393bce79c7..94c2845f7023c 100644 --- a/console/src/platform/connectors/utils.ts +++ b/console/src/platform/connectors/utils.ts @@ -40,6 +40,28 @@ export const snapshotting = (connector: ConnectorStatusInfo) => { ); }; +/** + * Explains why a snapshot's record count is approximate, per source type. + * + * Postgres and MySQL read the count from table statistics (`reltuples` and + * equivalents), which drift until an ANALYZE. Kafka derives it from offset + * watermarks, which overcount compacted or transactional topics. Either way the + * staged/known ratio can plateau below 100% after the snapshot is done, so + * `snapshot_committed` decides completion and this note explains the number. + */ +export const snapshotEstimateNote = (type: string | null) => { + switch (type) { + case "postgres": + case "mysql": + case "sql-server": + return "Snapshot progress and row counts are estimates, based on table statistics."; + case "kafka": + return "Snapshot progress and message counts are estimates."; + default: + return "Snapshot progress and record counts are estimates."; + } +}; + /** * Derives a simple `healthy` | `unhealthy` status from the underlying connector status. */ diff --git a/console/src/platform/maintained-objects/CriticalPathClusterMetrics.tsx b/console/src/platform/maintained-objects/CriticalPathClusterMetrics.tsx index 6245791a9fa3e..ad9b24d093e84 100644 --- a/console/src/platform/maintained-objects/CriticalPathClusterMetrics.tsx +++ b/console/src/platform/maintained-objects/CriticalPathClusterMetrics.tsx @@ -317,22 +317,22 @@ const DroppedReplicasFooter = ({ ); }; -export interface CriticalPathClusterMetricsProps { - node: RenderableNode; - pageObjectId: string; +export interface ClusterReplicaMetricsProps { + cluster: Cluster | null; timestamp: Date | null; bucketSizeMs: number; lookbackMs: number; } -export const CriticalPathClusterMetrics = ({ - node, - pageObjectId, +/** Per-replica memory/CPU utilization for one cluster at a point in time, + * with any replicas dropped inside the window called out. */ +export const ClusterReplicaMetrics = ({ + cluster, timestamp, bucketSizeMs, lookbackMs, -}: CriticalPathClusterMetricsProps) => { - const clusterId = node.cluster?.id ?? null; +}: ClusterReplicaMetricsProps) => { + const clusterId = cluster?.id ?? null; const { data: bucketsByReplicaId, isLoading, @@ -348,9 +348,19 @@ export const CriticalPathClusterMetrics = ({ Math.floor((timestamp ?? new Date()).getTime() / bucketSizeMs) * bucketSizeMs; const buckets = Object.values(bucketsByReplicaId ?? {}); - const replicas = buckets.flatMap((bs) => + const atAnchor = buckets.flatMap((bs) => bs.filter((b) => b.bucketStart.getTime() === targetBucketStartMs), ); + // Right after a replica swap the current bucket can predate the new + // replica's first metrics; fall back to each replica's latest bucket so a + // live cluster is not reported as inactive. Only for a now-ish anchor: for + // a historical anchor an empty bucket means the cluster really had no + // replicas then, and present-time metrics must not impersonate it. + const anchorIsRecent = Date.now() - targetBucketStartMs < 2 * bucketSizeMs; + const replicas = + atAnchor.length > 0 || !anchorIsRecent + ? atAnchor + : buckets.flatMap((bs) => bs.slice(-1)); const replicasInWindow: ReplicaInWindow[] = buckets .flatMap((bs) => bs.slice(-1)) .map(({ replicaId, name, size, bucketEnd }) => ({ @@ -362,13 +372,8 @@ export const CriticalPathClusterMetrics = ({ return ( - ); }; + +export interface CriticalPathClusterMetricsProps { + node: RenderableNode; + pageObjectId: string; + timestamp: Date | null; + bucketSizeMs: number; + lookbackMs: number; +} + +export const CriticalPathClusterMetrics = ({ + node, + pageObjectId, + timestamp, + bucketSizeMs, + lookbackMs, +}: CriticalPathClusterMetricsProps) => ( + + + + +); diff --git a/console/src/platform/maintained-objects/FilterChips.tsx b/console/src/platform/maintained-objects/FilterChips.tsx index 498c518199050..66f3d7d7c2374 100644 --- a/console/src/platform/maintained-objects/FilterChips.tsx +++ b/console/src/platform/maintained-objects/FilterChips.tsx @@ -14,7 +14,7 @@ import React from "react"; import { MaintainedObjectType } from "~/api/materialize/maintained-objects/constants"; import { fromSeconds } from "~/utils/format"; -import { HYDRATION_LABELS, HydrationBucket } from "./filters"; +import { STATUS_FILTER_LABELS } from "./filters"; import { MaintainedObjectListItem } from "./queries"; interface Chip { @@ -80,12 +80,12 @@ const CHIP_SOURCES: ChipSource[] = [ }, }, { - columnId: "hydration", + columnId: "status", build: (column) => - multiSelectChips( + multiSelectChips( column, - "Hydration", - (b) => HYDRATION_LABELS[b], + "Status", + (b) => STATUS_FILTER_LABELS[b], ), }, ]; diff --git a/console/src/platform/maintained-objects/MaintainedObjects.test.tsx b/console/src/platform/maintained-objects/MaintainedObjects.test.tsx index d7793768dbbe2..44f0434d39408 100644 --- a/console/src/platform/maintained-objects/MaintainedObjects.test.tsx +++ b/console/src/platform/maintained-objects/MaintainedObjects.test.tsx @@ -7,7 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { screen, waitFor } from "@testing-library/react"; +import { screen, waitFor, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import React from "react"; @@ -29,6 +29,7 @@ const buildObject = ( hydratedReplicas: 1, totalReplicas: 1, lag: null, + sourceStatus: null, ...overrides, }); @@ -54,9 +55,11 @@ const eventsSrc = buildObject({ id: "u103", name: "events_src", objectType: "source", + sourceType: "kafka", cluster: { id: "u200", name: "ingest_cluster" }, hydratedReplicas: 0, totalReplicas: 1, + sourceStatus: { status: "running", error: null, snapshotCommitted: true }, }); const buildProps = ( @@ -108,6 +111,63 @@ describe("MaintainedObjects", () => { expect(screen.getByText("Type: index")).toBeVisible(); }); + it("shows a source's ingestion status rather than replica hydration", async () => { + // A source mid-snapshot has no rehydration latency, so hydration reports 0 + // of 1 replicas hydrated. That must not surface as "Not Hydrated". + const snapshottingSrc = buildObject({ + id: "u104", + name: "snapshotting_src", + objectType: "source", + sourceType: "postgres", + hydratedReplicas: 0, + totalReplicas: 1, + sourceStatus: { + status: "running", + error: null, + snapshotCommitted: false, + }, + }); + await renderMaintainedObjects({ objects: [snapshottingSrc] }); + + const row = await screen.findByRole("row", { name: /snapshotting_src/ }); + expect(within(row).getByText("Snapshotting")).toBeVisible(); + expect(within(row).queryByText("Not Hydrated")).not.toBeInTheDocument(); + }); + + it("surfaces a stalled source's status", async () => { + const stalledSrc = buildObject({ + id: "u105", + name: "stalled_src", + objectType: "source", + sourceType: "kafka", + hydratedReplicas: 1, + totalReplicas: 1, + sourceStatus: { + status: "stalled", + error: "broker unreachable", + snapshotCommitted: true, + }, + }); + await renderMaintainedObjects({ objects: [stalledSrc] }); + + const row = await screen.findByRole("row", { name: /stalled_src/ }); + expect(within(row).getByText("Stalled")).toBeVisible(); + }); + + it("filters rows by status and reflects it in a chip", async () => { + const user = userEvent.setup(); + await renderMaintainedObjects(); + + await user.click( + await screen.findByRole("button", { name: "Filter status" }), + ); + await user.click(await screen.findByLabelText("Running")); + + expect(screen.getByText("events_src")).toBeVisible(); + expect(screen.queryByText("inventory_idx")).not.toBeInTheDocument(); + expect(screen.getByText("Status: Running")).toBeVisible(); + }); + it("filters rows by search term", async () => { const user = userEvent.setup(); await renderMaintainedObjects(); diff --git a/console/src/platform/maintained-objects/MaintainedObjects.tsx b/console/src/platform/maintained-objects/MaintainedObjects.tsx index 73acf8c33f0ff..7d1527e64c1f7 100644 --- a/console/src/platform/maintained-objects/MaintainedObjects.tsx +++ b/console/src/platform/maintained-objects/MaintainedObjects.tsx @@ -35,7 +35,7 @@ import { Link as RouterLink, useLocation, useNavigate } from "react-router-dom"; import { createNamespace } from "~/api/materialize"; import { OUTDATED_THRESHOLD_SECONDS } from "~/api/materialize/cluster/materializationLag"; -import StatusPill from "~/components/StatusPill"; +import StatusPill, { ConnectorStatusPill } from "~/components/StatusPill"; import { sortingFunctions } from "~/components/Table/tableColumnBuilders"; import { TablePagination } from "~/components/Table/TablePagination"; import { TableSearch } from "~/components/Table/TableSearch"; @@ -68,8 +68,8 @@ import { FilterChips } from "./FilterChips"; import { ClusterFilterPanel, FreshnessFilterPanel, - HydrationFilterPanel, ObjectTypeFilterPanel, + StatusFilterPanel, } from "./filterPanels"; import { bucketForHydration, @@ -77,10 +77,11 @@ import { FILTER_URL_SPECS, freshnessFilterFn, HYDRATION_LABELS, - hydrationFilterFn, initialColumnFiltersFromUrl, objectTypeFilterFn, STATUS_COLOR_SCHEMES, + statusBucketForRow, + statusFilterFn, } from "./filters"; import { MaintainedObjectListItem } from "./queries"; @@ -259,16 +260,47 @@ const FreshnessCell = ({ ); }; -const StatusCell = ({ +const pendingPill = ; + +/** + * Sources report their own ingestion status, so they don't go through replica + * hydration. For a source, `hydrated` only tracks whether `rehydration_latency` + * is set, which stays false for the whole initial snapshot and would render a + * healthy source as `Not Hydrated`. + */ +const SourceStatusCell = ({ row, ready, }: { row: MaintainedObjectListItem; ready: boolean; }) => { + // Status rides on the hydration-aggregate feed, so it can trail the row. + if (!row.sourceStatus) return ready ? "-" : pendingPill; + return ( + + ); +}; + +const StatusCell = ({ + row, + hydrationReady, +}: { + row: MaintainedObjectListItem; + hydrationReady: boolean; +}) => { + if (row.objectType === "source") { + return ; + } const { hydratedReplicas, totalReplicas } = row; if (totalReplicas === 0) { - return ready ? "-" : ; + return hydrationReady ? "-" : pendingPill; } const bucket = bucketForHydration(hydratedReplicas, totalReplicas); if (!bucket) return "-"; @@ -320,27 +352,26 @@ const columns = [ renderFilter: (column) => , }, }), - columnHelper.accessor( - (row) => - row.totalReplicas === 0 ? null : row.hydratedReplicas / row.totalReplicas, - { - id: "status", - header: "Status", - sortingFn: sortingFunctions.nullsLast, - filterFn: hydrationFilterFn, - cell: (info) => { - const meta = info.table.options.meta as MaintainedObjectsTableMeta; - return ( - - ); - }, - meta: { - tooltip: - "Object status derived from replica hydration across all clusters.", - renderFilter: (column) => , - }, + columnHelper.accessor((row) => statusBucketForRow(row) ?? null, { + id: "status", + header: "Status", + sortingFn: sortingFunctions.nullsLast, + filterFn: statusFilterFn, + cell: (info) => { + const meta = info.table.options.meta as MaintainedObjectsTableMeta; + return ( + + ); }, - ), + meta: { + tooltip: + "Ingestion status for sources, replica hydration across all clusters for everything else.", + renderFilter: (column) => , + }, + }), columnHelper.accessor((row) => row.cluster?.name ?? null, { id: "clusterName", header: "Cluster", diff --git a/console/src/platform/maintained-objects/ObjectDetailsCard.tsx b/console/src/platform/maintained-objects/ObjectDetailsCard.tsx index 878bf525f5517..cce9edf9534d9 100644 --- a/console/src/platform/maintained-objects/ObjectDetailsCard.tsx +++ b/console/src/platform/maintained-objects/ObjectDetailsCard.tsx @@ -21,7 +21,7 @@ import { Link as RouterLink } from "react-router-dom"; import { createNamespace } from "~/api/materialize"; import { OUTDATED_THRESHOLD_SECONDS } from "~/api/materialize/cluster/materializationLag"; -import StatusPill from "~/components/StatusPill"; +import StatusPill, { ConnectorStatusPill } from "~/components/StatusPill"; import TextLink from "~/components/TextLink"; import { DetailItem } from "~/platform/connectors/AsideBox"; import { absoluteClusterPath } from "~/platform/routeHelpers"; @@ -91,7 +91,7 @@ export const ObjectDetailsCard = ({ {item.name} - + @@ -143,7 +143,22 @@ export const ObjectDetailsCard = ({ ); }; -const HydrationBadge = ({ item }: { item: MaintainedObjectListItem }) => { +/** Mirrors the list's Status column: ingestion status for sources, replica + * hydration for everything else. */ +const StatusBadge = ({ item }: { item: MaintainedObjectListItem }) => { + if (item.objectType === "source") { + if (!item.sourceStatus) return null; + return ( + + ); + } const bucket = bucketForHydration(item.hydratedReplicas, item.totalReplicas); if (!bucket) return null; return ( diff --git a/console/src/platform/maintained-objects/ObjectFreshness.tsx b/console/src/platform/maintained-objects/ObjectFreshness.tsx index c8faab82587c9..2e96263ffd83a 100644 --- a/console/src/platform/maintained-objects/ObjectFreshness.tsx +++ b/console/src/platform/maintained-objects/ObjectFreshness.tsx @@ -24,6 +24,7 @@ import { MaterializeTheme } from "~/theme"; import { formatDate } from "~/utils/dateFormat"; import { LOOKBACK_OPTIONS } from "./constants"; +import { ClusterReplicaMetrics } from "./CriticalPathClusterMetrics"; import { CriticalPathGraph } from "./CriticalPathGraph"; import { computeFreshnessBreakdown } from "./freshnessBreakdown"; import { FreshnessBreakdownStrip } from "./FreshnessBreakdownStrip"; @@ -85,8 +86,9 @@ export const ObjectFreshness = ({ const breakdown = computeFreshnessBreakdown(criticalPath?.directInputs ?? []); // Sources ingest from external systems and have no Materialize-internal - // upstream chain to walk — hide the critical path section for them. - const hasUpstreamChain = item.objectType !== "source"; + // upstream chain to walk. Their "critical path" is the cluster they ingest + // on, so they get its replica metrics instead. + const isSource = item.objectType === "source"; return ( @@ -122,7 +124,31 @@ export const ObjectFreshness = ({ isLocked={isLocked} /> - {hasUpstreamChain && ( + {isSource && ( + <> + + + + Source cluster metrics + + Resources of the cluster this source ingests on, at the point + selected on the freshness graph above. + + + + + + )} + + {!isSource && ( <> diff --git a/console/src/platform/maintained-objects/ObjectMetadata.tsx b/console/src/platform/maintained-objects/ObjectMetadata.tsx index b4b77b474964e..1a1a651fabea2 100644 --- a/console/src/platform/maintained-objects/ObjectMetadata.tsx +++ b/console/src/platform/maintained-objects/ObjectMetadata.tsx @@ -27,7 +27,13 @@ export const ObjectMetadata = ({ item }: ObjectMetadataProps) => { return ( - {isSource && } + {isSource && ( + + )} { +/** + * Renders only while the source has something diagnostic to say: a status + * error, or an in-progress snapshot. Steady-state lifecycle facts live on the + * source details page instead. + */ +export const SourceDiagnostics = ({ + sourceId, + sourceType, + sourceStatus, +}: SourceDiagnosticsProps) => { const { colors } = useTheme(); const { data: stats } = useObjectSourceStatistics(sourceId); - if (!stats) return null; + const error = sourceStatus?.error; + // `snapshot_committed` is authoritative. Judging by the staged/known ratio + // would report a live source as stuck at 99% forever. + const snapshotting = sourceStatus?.snapshotCommitted === false; + if (!error && !snapshotting) return null; - const { - messagesReceived, - snapshotRecordsKnown: snapshotKnown, - snapshotRecordsStaged: snapshotStaged, - rehydrationLatency, - } = stats; + const snapshotKnown = stats?.snapshotRecordsKnown ?? 0; + const snapshotStaged = stats?.snapshotRecordsStaged ?? 0; const snapshotPercent = snapshotKnown > 0 - ? Math.round((snapshotStaged / snapshotKnown) * 100) + ? Math.min(100, Math.round((snapshotStaged / snapshotKnown) * 100)) : null; - const snapshotComplete = snapshotPercent === null || snapshotPercent === 100; - const rehydrationMs = rehydrationLatency - ? sumPostgresIntervalMs(rehydrationLatency) - : null; return ( { Source diagnostics - - - {rehydrationMs !== null - ? `${(rehydrationMs / 1000).toFixed(1)}s` - : "Still rehydrating..."} - - - {messagesReceived.toLocaleString()} - - {snapshotComplete ? ( - - Complete + {error && } + + {snapshotting && ( + + + {snapshotPercent === null + ? "In progress" + : `${snapshotPercent}% (${snapshotStaged.toLocaleString()} / ${snapshotKnown.toLocaleString()} records)`} - ) : ( - <> - - {`${snapshotPercent}% (${snapshotStaged.toLocaleString()} / ${snapshotKnown.toLocaleString()} records)`} - + {snapshotPercent !== null && ( { transition="width 0.3s ease" /> - - )} - + )} + + {snapshotEstimateNote(sourceType)} + + + )} ); diff --git a/console/src/platform/maintained-objects/criticalPathRenderable.test.ts b/console/src/platform/maintained-objects/criticalPathRenderable.test.ts index b95dbd086364d..a0d6ad23e57d1 100644 --- a/console/src/platform/maintained-objects/criticalPathRenderable.test.ts +++ b/console/src/platform/maintained-objects/criticalPathRenderable.test.ts @@ -47,6 +47,7 @@ const buildProbe = ( hydratedReplicas: 1, totalReplicas: 1, lag: { value: parse("00:00:10"), ms: 10_000 }, + sourceStatus: null, ...overrides, }); diff --git a/console/src/platform/maintained-objects/filterPanels.tsx b/console/src/platform/maintained-objects/filterPanels.tsx index 9846a370cd35d..670e18320831a 100644 --- a/console/src/platform/maintained-objects/filterPanels.tsx +++ b/console/src/platform/maintained-objects/filterPanels.tsx @@ -30,11 +30,7 @@ import { import { MaterializeTheme } from "~/theme"; import { DurationUnit, fromSeconds, toSeconds } from "~/utils/format"; -import { - HYDRATION_BUCKETS, - HYDRATION_LABELS, - HydrationBucket, -} from "./filters"; +import { STATUS_FILTER_BUCKETS, STATUS_FILTER_LABELS } from "./filters"; import { MaintainedObjectListItem } from "./queries"; type AnyColumn = Column; @@ -176,11 +172,11 @@ export const ObjectTypeFilterPanel = ({ column }: { column: AnyColumn }) => ( /> ); -export const HydrationFilterPanel = ({ column }: { column: AnyColumn }) => ( - +export const StatusFilterPanel = ({ column }: { column: AnyColumn }) => ( + column={column} - items={HYDRATION_BUCKETS} - getLabel={(b) => HYDRATION_LABELS[b]} + items={STATUS_FILTER_BUCKETS} + getLabel={(b) => STATUS_FILTER_LABELS[b]} /> ); diff --git a/console/src/platform/maintained-objects/filters.ts b/console/src/platform/maintained-objects/filters.ts index 9ec64a32584ff..8b9fabc6526e9 100644 --- a/console/src/platform/maintained-objects/filters.ts +++ b/console/src/platform/maintained-objects/filters.ts @@ -13,8 +13,12 @@ import { MAINTAINED_OBJECT_TYPES, MaintainedObjectType, } from "~/api/materialize/maintained-objects/constants"; +import { snapshotting } from "~/platform/connectors/utils"; -import { MaintainedObjectListItem } from "./queries"; +import { + MaintainedObjectListItem, + MaintainedObjectSourceStatus, +} from "./queries"; /** * Row-threshold options for the freshness filter, keyed by seconds. Selecting @@ -38,7 +42,7 @@ export const HYDRATION_BUCKETS = [ export type HydrationBucket = (typeof HYDRATION_BUCKETS)[number]; export const HYDRATION_LABELS: Record = { - hydrated: "Running", + hydrated: "Hydrated", hydrating: "Hydrating", not_hydrated: "Not Hydrated", }; @@ -79,9 +83,73 @@ export const objectTypeFilterFn = arrayMatchFilter( (r) => r.objectType, ); -export const hydrationFilterFn = arrayMatchFilter((r) => - bucketForHydration(r.hydratedReplicas, r.totalReplicas), -); +/** + * Status values offered for sources. `snapshotting` is derived rather than + * reported: `mz_source_statuses` says `running` throughout the initial + * snapshot, so both the pill and this filter fold in `snapshot_committed`. + */ +export const SOURCE_STATUS_BUCKETS = [ + "snapshotting", + "running", + "starting", + "created", + "paused", + "stalled", + "failed", +] as const; +export type SourceStatusBucket = (typeof SOURCE_STATUS_BUCKETS)[number]; + +export const SOURCE_STATUS_LABELS: Record = { + snapshotting: "Snapshotting", + running: "Running", + starting: "Starting", + created: "Created", + paused: "Paused", + stalled: "Stalled", + failed: "Failed", +}; + +/** Derives a source's displayed status, matching `ConnectorStatusPill`. */ +export const bucketForSourceStatus = ( + sourceStatus: MaintainedObjectSourceStatus, + sourceType: string | null, +): string => + snapshotting({ + status: sourceStatus.status, + type: sourceType ?? "", + snapshotCommitted: sourceStatus.snapshotCommitted, + }) + ? "snapshotting" + : sourceStatus.status; + +/** Options the Status filter offers, sources first. */ +export const STATUS_FILTER_BUCKETS = [ + ...SOURCE_STATUS_BUCKETS, + ...HYDRATION_BUCKETS, +] as const; + +export const STATUS_FILTER_LABELS: Record = { + ...SOURCE_STATUS_LABELS, + ...HYDRATION_LABELS, +}; + +/** + * The bucket the Status cell displays: ingestion status for sources, replica + * hydration for everything else. The filter matches on this so it can never + * select a value the cell doesn't show. + */ +export const statusBucketForRow = ( + row: MaintainedObjectListItem, +): string | undefined => { + if (row.objectType === "source") { + return row.sourceStatus + ? bucketForSourceStatus(row.sourceStatus, row.sourceType) + : undefined; + } + return bucketForHydration(row.hydratedReplicas, row.totalReplicas); +}; + +export const statusFilterFn = arrayMatchFilter(statusBucketForRow); export const freshnessFilterFn = ( row: Row, @@ -144,18 +212,20 @@ export const FILTER_URL_SPECS: readonly FilterUrlSpec[] = [ typeof v === "number" ? { key: "freshness", value: v } : undefined, }, { - columnId: "hydration", + // Matches the Status column's id. The filter, its URL param and its chip + // all have to agree on this key or the filter silently stops round-tripping. + columnId: "status", fromUrl: (p) => nonEmpty( p - .getAll("hydration[]") - .filter((h): h is HydrationBucket => - (HYDRATION_BUCKETS as readonly string[]).includes(h), + .getAll("status[]") + .filter((s) => + (STATUS_FILTER_BUCKETS as readonly string[]).includes(s), ), ), toUrl: (v) => { - const arr = v as HydrationBucket[]; - return arr?.length ? { key: "hydration[]", value: arr } : undefined; + const arr = v as string[]; + return arr?.length ? { key: "status[]", value: arr } : undefined; }, }, ]; diff --git a/console/src/platform/maintained-objects/queries.ts b/console/src/platform/maintained-objects/queries.ts index 9bdd12f73c7b1..ca50e4f3d0068 100644 --- a/console/src/platform/maintained-objects/queries.ts +++ b/console/src/platform/maintained-objects/queries.ts @@ -63,6 +63,22 @@ export interface MaintainedObjectLag { ms: number; } +/** + * Ingestion state from the hydration-aggregate feed, which joins + * `mz_source_statuses` and the snapshot flag. This is what the Status column + * shows for a source, in place of replica hydration, which for a source only + * tracks whether `rehydration_latency` is set and so reads false throughout the + * initial snapshot. + */ +export interface MaintainedObjectSourceStatus { + /** One of `created`, `starting`, `running`, `paused`, `stalled`, `failed`. */ + status: string; + /** Populated when the source is `stalled` or `failed`. */ + error: string | null; + /** Null while no replica has reported statistics yet. */ + snapshotCommitted: boolean | null; +} + export interface MaintainedObjectListItem { id: string; name: string; @@ -78,6 +94,8 @@ export interface MaintainedObjectListItem { totalReplicas: number; /** Null until the lag snapshot arrives. */ lag: MaintainedObjectLag | null; + /** Null for non-sources, and until the hydration snapshot arrives. */ + sourceStatus: MaintainedObjectSourceStatus | null; } const MAINTAINED_OBJECT_TYPE_SET: ReadonlySet = new Set( @@ -133,9 +151,9 @@ export interface UseMaintainedObjectsListResult { /** * Composes maintained objects from three live sources: `useAllObjects()` for - * object metadata, plus SUBSCRIBEs for pMAX lag and hydration aggregate. The - * table renders as soon as object metadata is ready; lag and hydration cells - * fill in as their snapshots arrive. + * object metadata, plus SUBSCRIBEs for pMAX lag and the hydration aggregate, + * which also carries source status. The table renders as soon as object + * metadata is ready; lag and status cells fill in as their snapshots arrive. */ export const useMaintainedObjectsList = ({ lookbackMinutes, @@ -179,6 +197,13 @@ export const useMaintainedObjectsList = ({ lag: lagInterval ? { value: lagInterval, ms: sumPostgresIntervalMs(lagInterval) } : null, + sourceStatus: hydrationRow?.sourceStatus + ? { + status: hydrationRow.sourceStatus, + error: hydrationRow.sourceError, + snapshotCommitted: hydrationRow.snapshotCommitted, + } + : null, }); } return [...objectsById.values()].sort((a, b) => diff --git a/console/src/platform/sources/SourceOverview/SnapshotProgress.tsx b/console/src/platform/sources/SourceOverview/SnapshotProgress.tsx index a2fc7f2e19dd0..119db6fa27111 100644 --- a/console/src/platform/sources/SourceOverview/SnapshotProgress.tsx +++ b/console/src/platform/sources/SourceOverview/SnapshotProgress.tsx @@ -10,7 +10,11 @@ import { Box, HStack, Text, useTheme, VStack } from "@chakra-ui/react"; import React from "react"; -import { ConnectorStatusInfo, snapshotting } from "~/platform/connectors/utils"; +import { + ConnectorStatusInfo, + snapshotEstimateNote, + snapshotting, +} from "~/platform/connectors/utils"; import { MaterializeTheme } from "~/theme"; import { pluralize } from "~/util"; @@ -36,23 +40,6 @@ const mapSourceTypeToProgressUnit = (type: string, count: number) => { } }; -// The known count for database sources comes from table statistics (Postgres -// `reltuples` and equivalents), which are estimates that drift until an ANALYZE. -// Kafka derives it from offset watermarks, which overcount compacted or -// transactional topics, so the note stays type-specific rather than claiming -// table statistics everywhere. -const mapSourceTypeToEstimateNote = (type: string) => { - switch (type) { - case "postgres": - case "mysql": - case "sql-server": - return "Snapshot progress and row counts are estimates, based on table statistics."; - case "kafka": - return "Snapshot progress and message counts are estimates."; - default: - return "Snapshot progress and record counts are estimates."; - } -}; export const SnapshotProgress = ({ snapshotRecordsKnown, snapshotRecordsStaged, @@ -112,7 +99,7 @@ export const SnapshotProgress = ({ - {mapSourceTypeToEstimateNote(source.type)} + {snapshotEstimateNote(source.type)} From 40ede80515ca124d4e5b5ab9ef59da958babaca7 Mon Sep 17 00:00:00 2001 From: Pranshu Maheshwari Date: Wed, 26 Aug 2026 16:36:17 -0400 Subject: [PATCH 07/11] docs: correct the partition operations that break publish_via_partition_root (#38432) ### Motivation Review feedback on #38199: the `publish_via_partition_root = true` warning on the PostgreSQL partitioned tables page listed *adding* a partition as one of the operations that surfaces problems. That isn't right. Partitions created with `CREATE TABLE ... PARTITION OF` are always empty, so they move no data into the root table and cause no divergence. Truncating a partition, which does remove root-table data, was missing from the list. ### Description Replaces "added, attached, or detached" with "attached, detached, or truncated" in the warning. All three of those are DDL that adds or removes root-table data without emitting CDC events, which is precisely the failure mode the warning describes. ### Verification Docs-only prose change; no code or test surface. The paragraph rewraps to stay within the file's line width. --- _Generated by [Claude Code](https://claude.ai/code/session_01HBUT5iCf2hkNY2hEfruLTp)_ Co-authored-by: Claude --- doc/user/content/ingest-data/postgres/partitioned-tables.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/user/content/ingest-data/postgres/partitioned-tables.md b/doc/user/content/ingest-data/postgres/partitioned-tables.md index 4c925eba72286..a3ef863084f36 100644 --- a/doc/user/content/ingest-data/postgres/partitioned-tables.md +++ b/doc/user/content/ingest-data/postgres/partitioned-tables.md @@ -40,8 +40,8 @@ that uses this option, and doing so can produce incorrect results. The configuration is accepted rather than rejected: the source is created, the initial snapshot is correct, and inserts, updates, and deletes appear to -replicate normally. Problems surface only once partitions are added, attached, -or detached. Use one of the approaches on this page instead. +replicate normally. Problems surface only once partitions are attached, +detached, or truncated. Use one of the approaches on this page instead. {{< /warning >}} From e574b4c3792bc8ed4638475afc5ab9e1337e2e9d Mon Sep 17 00:00:00 2001 From: Peter Larsen Date: Wed, 26 Aug 2026 18:28:55 -0400 Subject: [PATCH 08/11] benchmark: replace MySQL snapshot threshold bumps with ancestor overrides (#38492) ### Motivation Fixes SS-439 The MySQL snapshot feature-benchmark scenarios have raised `RELATIVE_THRESHOLD` values to absorb the wallclock changes from #38094. We don't want to leave the higher thresholds around indefinitely because they won't catch regressions. ### Changes Remove bumped thresholds and prevent comparison against older versions at the tighter threshold. Co-authored-by: Claude Fable 5 --- .../scenarios/benchmark_main.py | 28 ------------------- .../materialize/version_ancestor_overrides.py | 12 ++++++-- 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/misc/python/materialize/feature_benchmark/scenarios/benchmark_main.py b/misc/python/materialize/feature_benchmark/scenarios/benchmark_main.py index 8c3c743adb8a7..7ee0fcb0fa0b4 100644 --- a/misc/python/materialize/feature_benchmark/scenarios/benchmark_main.py +++ b/misc/python/materialize/feature_benchmark/scenarios/benchmark_main.py @@ -2012,18 +2012,6 @@ class MySqlInitialLoad(MySqlCdc): FIXED_SCALE = True # TODO: Remove when database-issues#7556 is fixed - # The parallel snapshot path has a wider memory envelope during the initial - # load, so allow a bit more headroom than the base default before flagging a - # regression. - RELATIVE_THRESHOLD: dict[MeasurementType, float] = { - # Primary key splitting supports only string primary keys, so this - # BIGINT-keyed snapshot reads serially where older versions split it across - # workers. The increased wallclock tolerance covers that lost parallelism. - MeasurementType.WALLCLOCK: 0.25, - MeasurementType.MEMORY_MZ: 0.30, - MeasurementType.MEMORY_CLUSTERD: 0.50, - } - def shared(self) -> Action: return TdAction(f""" $ mysql-connect name=mysql url=mysql://root@mysql password=${{arg.mysql-root-password}} @@ -2071,12 +2059,6 @@ class MySqlInitialLoadMultiWorkerSampled(MySqlCdc): """Measure an 8-worker snapshot across 20 tables with a single-column non-integer primary key (a CHAR(26) ULID-like key).""" - RELATIVE_THRESHOLD: dict[MeasurementType, float] = { - MeasurementType.WALLCLOCK: 0.10, - MeasurementType.MEMORY_MZ: 0.60, - MeasurementType.MEMORY_CLUSTERD: 0.60, - } - # Cap scale so each per-table load fits one mysql.time_zone self-join # (~3M rows), so a single INSERT per table suffices. MAX_SCALE = 6 @@ -2170,16 +2152,6 @@ class MySqlInitialLoadMultiWorkerSingleTable(MySqlCdc): single-column non-integer primary key (a CHAR(26) ULID-like key). The single-table companion to MySqlInitialLoadMultiWorkerSampled.""" - RELATIVE_THRESHOLD: dict[MeasurementType, float] = { - # In MySQL, probing primary keys (for approximately even partitions) is - # slower on small datasets than using OFFSET (for exactly even - # partitions). Measured locally at 2-14% slower by wallclock at - # SCALE=6 (1M rows). - MeasurementType.WALLCLOCK: 0.25, - MeasurementType.MEMORY_MZ: 0.60, - MeasurementType.MEMORY_CLUSTERD: 0.60, - } - # Cap scale so the load fits one mysql.time_zone self-join (~3M rows), so a # single INSERT suffices. MAX_SCALE = 6 diff --git a/misc/python/materialize/version_ancestor_overrides.py b/misc/python/materialize/version_ancestor_overrides.py index a13becfa03843..9ad23d3c797a4 100644 --- a/misc/python/materialize/version_ancestor_overrides.py +++ b/misc/python/materialize/version_ancestor_overrides.py @@ -28,8 +28,16 @@ def get_ancestor_overrides_for_performance_regressions( min_ancestor_mz_version_per_commit = dict() - if scenario_class_name == "MySqlInitialLoadMultiWorkerSampled": - # PR#38094 (storage: Fast approximate snapshot partitioning) increased wallclock by ~30% + if scenario_class_name in ( + "MySqlInitialLoad", + "MySqlInitialLoadMultiWorkerSampled", + "MySqlInitialLoadMultiWorkerSingleTable", + ): + # PR#38094 (storage: Fast approximate snapshot partitioning) increased + # snapshot wallclock: integer primary keys are no longer partitioned, + # so MySqlInitialLoad reads serially, and approximate key probing is + # slower than the exact OFFSET sampler on small datasets (~30% on + # MySqlInitialLoadMultiWorkerSampled). min_ancestor_mz_version_per_commit[ "de8eac0d1683ffd788dfba6edaa88aef7f0f1740" ] = MzVersion.parse_mz("v26.39.0") From deca6105f411cb3cbad75db1fadfdeea11ceb8ab Mon Sep 17 00:00:00 2001 From: Peter Larsen Date: Wed, 26 Aug 2026 18:59:11 -0400 Subject: [PATCH 09/11] docs: draft MySQL snapshot parallelism page (#38091) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of [SS-358](https://linear.app/materializeinc/issue/SS-358). Adding docs for the new MySQL parallel snapshotting feature, some more context on the load we place on upstream databases, and some high-level debugging tips for triaging obvious database issues. The table-splitting feature ships with `mysql_source_snapshot_parallelism` off by default, so it is documented as private preview: a banner on the MySQL snapshot parallelism page and inline preview markers where the splitting is mentioned elsewhere. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_016AgSuGEvoVT9TFHAgq6FCi --------- Co-authored-by: Claude Fable 5 --- doc/user/content/concepts/snapshotting.md | 9 +- .../ingestion/snapshotting-parallelism.md | 34 ++++++ doc/user/content/ingest-data/_index.md | 4 + doc/user/content/ingest-data/mysql/_index.md | 5 + .../ingest-data/mysql/snapshot-parallelism.md | 108 ++++++++++++++++++ .../content/ingest-data/postgres/_index.md | 5 + .../content/ingest-data/troubleshooting.md | 35 ++++++ 7 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 doc/user/content/headless/ingestion/snapshotting-parallelism.md create mode 100644 doc/user/content/ingest-data/mysql/snapshot-parallelism.md diff --git a/doc/user/content/concepts/snapshotting.md b/doc/user/content/concepts/snapshotting.md index 73522e7c58c4a..02cd807b58ad0 100644 --- a/doc/user/content/concepts/snapshotting.md +++ b/doc/user/content/concepts/snapshotting.md @@ -18,6 +18,10 @@ menu: {{% include-headless "/headless/ingestion/snapshotting-duration" %}} +### Parallelism + +{{% include-headless "/headless/ingestion/snapshotting-parallelism" %}} + ## Queries during snapshotting {{% include-headless "/headless/ingestion/snapshotting-queries" %}} @@ -27,7 +31,9 @@ menu: Snapshotting has the following upstream impacts: - **Read load.** Snapshotting puts read, CPU, and network load on the upstream - system, proportional to the data volume. + system. The total load is proportional to the volume of data being + snapshotted, while the source cluster's [parallelism](#parallelism) affects + the peak load: more workers compress the reads into a shorter window. - **Change-log retention for CDC database sources.** When ingesting data from CDC database sources (PostgreSQL, MySQL, SQL Server), the upstream system must @@ -41,3 +47,4 @@ Snapshotting has the following upstream impacts: - [Ingest data](/ingest-data/) - [Sources](/concepts/sources/) +- [Troubleshooting data ingestion](/ingest-data/troubleshooting/) diff --git a/doc/user/content/headless/ingestion/snapshotting-parallelism.md b/doc/user/content/headless/ingestion/snapshotting-parallelism.md new file mode 100644 index 0000000000000..e23bbafee03cd --- /dev/null +++ b/doc/user/content/headless/ingestion/snapshotting-parallelism.md @@ -0,0 +1,34 @@ +--- +headless: true +--- + +Materialize can parallelize snapshotting across the workers of the cluster +hosting the source. + +- **PostgreSQL sources** are parallelized by table, i.e., different tables + are read concurrently by different workers. On PostgreSQL 14 and later, + Materialize additionally attempts to partition each table's read across + workers. Tables that cannot be partitioned fall back to a single worker. + +- **MySQL sources** are parallelized by table, i.e., different tables are + read concurrently by different workers. For tables that meet certain + requirements, Materialize can additionally partition the table's read + across workers {{< private-preview-inline />}}. See [MySQL snapshot + parallelism](/ingest-data/mysql/snapshot-parallelism/). + +- **Kafka sources** are parallelized by topic partition, with partitions + distributed across workers, so parallelism is bounded by the topic's + partition count. + +- **SQL Server sources** are not parallelized: a single worker reads all + tables. + +The degree of snapshot parallelism depends on the number of workers. A +cluster's [size](/sql/create-cluster/#available-sizes) determines its number +of workers, so a larger cluster can shorten the snapshot, to the extent the +work parallelizes and the upstream database keeps up. The volume read from +the upstream database is unchanged, it is compressed into a shorter window +of more concurrent queries and connections. To determine whether +snapshotting is overloading the upstream database, and for ways to mitigate +the load, see [Is the upstream database +overloaded?](/ingest-data/troubleshooting/#is-the-upstream-database-overloaded) diff --git a/doc/user/content/ingest-data/_index.md b/doc/user/content/ingest-data/_index.md index 3a16c1e89a072..528371b2a0bbc 100644 --- a/doc/user/content/ingest-data/_index.md +++ b/doc/user/content/ingest-data/_index.md @@ -60,6 +60,10 @@ we recommend: the steady-state resource needs of your upsert source(s). See [Best practices: Upsert sources](#upsert-sources). +### Parallelism + +{{% include-headless "/headless/ingestion/snapshotting-parallelism" %}} + ### Monitoring progress While snapshotting is taking place, you can monitor the progress of the diff --git a/doc/user/content/ingest-data/mysql/_index.md b/doc/user/content/ingest-data/mysql/_index.md index 54cedf30a29c9..eaf67640d7be4 100644 --- a/doc/user/content/ingest-data/mysql/_index.md +++ b/doc/user/content/ingest-data/mysql/_index.md @@ -33,6 +33,11 @@ gives you the following benefits: read-replica to build views on top of your MySQL data that are efficiently maintained and always up-to-date. +When a source is created, Materialize parallelizes the initial snapshot +across the cluster's workers and can split the read of large tables that meet +certain requirements {{< private-preview-inline />}}. See [Snapshot +parallelism](/ingest-data/mysql/snapshot-parallelism/). + ## Supported versions and services {{< note >}} diff --git a/doc/user/content/ingest-data/mysql/snapshot-parallelism.md b/doc/user/content/ingest-data/mysql/snapshot-parallelism.md new file mode 100644 index 0000000000000..023e5ab3dc932 --- /dev/null +++ b/doc/user/content/ingest-data/mysql/snapshot-parallelism.md @@ -0,0 +1,108 @@ +--- +title: "Snapshot parallelism" +description: "How Materialize splits the snapshot of a single MySQL table across the workers of a cluster." +menu: + main: + parent: "mysql" + name: "Snapshot parallelism" + identifier: "mysql-snapshot-parallelism" + weight: 70 +--- + +{{< private-preview />}} + +When you create a [MySQL source](/sql/create-source/mysql-v2/), Materialize +performs an initial, snapshot-based sync of the selected tables before it +starts ingesting change events from the binlog. For large tables, this +snapshot dominates the time until the source becomes healthy. + +How snapshot work is spread across the workers of a cluster, and what that +means for the upstream database, is covered in +[Snapshotting](/concepts/snapshotting/#parallelism). Materialize can split +the read of a **single table** across all the workers of the cluster, so +that even a source dominated by one very large table benefits from a larger +cluster. This page covers what is specific to MySQL: which tables are +eligible for splitting, and how their reads are partitioned. + +## Which tables are split + +Materialize splits the snapshot of an individual table across workers when +all of the following conditions are met: + +- The table has a **single-column primary key**. Composite primary keys are + not supported. +- The primary key column is of type **`CHAR` or `VARCHAR`**, with a declared + length of **at most 768 characters**. Other types, including numeric keys, + are not supported. +- The primary key column uses the **`utf8mb4` character set** with the + **`utf8mb4_bin` collation**. +- The table is **large enough to be worth splitting**. Small tables are read + by a single worker, where splitting would add overhead without benefit. + +How evenly the split lands also depends on the distribution of the key +values. See [How a table is partitioned](#how-a-table-is-partitioned). + +If a table does not meet these requirements, or if the [boundary +sampling](#how-a-table-is-partitioned) fails, its snapshot is not split: a +single worker reads the table in full. Different tables are still read +concurrently by different workers. + +## How a table is partitioned + +Materialize partitions an [eligible](#which-tables-are-split) table using the +leading characters of its primary key values. Before reading the table, +Materialize probes the primary key index to discover key prefixes and uses +the MySQL optimizer's row estimates to gauge how many rows fall under each +prefix. It extends the prefixes as needed to find boundaries that divide the +table into roughly even ranges. The probes are inexpensive point lookups, +capped in proportion to the table's estimated size, so the sampling phase +stays negligible next to the snapshot itself. + +Each worker then reads only its assigned range, within the same consistent +snapshot of the upstream database, so the result is identical to a +single-worker snapshot, only faster. + +Because partitioning is based on key prefixes and optimizer estimates, how +evenly the work divides depends on the shape of your keys: + +- **Evenly distributed keys partition well.** Keys whose leading characters + spread rows uniformly, such as UUIDs, hashes, or other randomized + identifiers, produce well-balanced ranges. + +- **Skewed keys partition less evenly.** If a large share of the table's rows + sort under a few common prefixes, some ranges end up with more rows than + others, and the workers assigned to them finish later. + +- **The probe budget can run out.** If finding even boundaries would require + examining very many distinct prefixes, Materialize stops probing and uses + the coarser boundaries found so far, which can also leave ranges uneven. + +Uneven partitioning is never incorrect. It only reduces the speedup, since +the snapshot finishes when the busiest worker finishes. + +## MySQL-specific upstream considerations + +- **Connection count.** While the snapshot is being set up, Materialize + briefly holds up to two connections per worker, plus one. Once reading is + underway, this settles to one connection per worker reading a range, plus + one coordination connection. After the snapshot completes, the source drops + back to a single replication connection. If your MySQL server or connection + pooler enforces a low + [`max_connections`](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_max_connections) + limit, account for this burst when sizing it. + +- **Statistics freshness.** Range boundaries are placed using the MySQL + optimizer's row estimates. Stale statistics don't affect correctness, but + can skew how evenly work divides across workers. Running + [`ANALYZE TABLE`](https://dev.mysql.com/doc/refman/8.0/en/analyze-table.html) + on very large tables before creating the source can improve balance. + +For general guidance on read load, IOPS, and other upstream impact, which is +not specific to MySQL, see [Is the upstream database +overloaded?](/ingest-data/troubleshooting/#is-the-upstream-database-overloaded) + +## Observability + +To observe the progress of an ongoing snapshot, see [Monitoring the +snapshotting +progress](/ingest-data/monitoring-data-ingestion/#monitoring-the-snapshotting-progress). diff --git a/doc/user/content/ingest-data/postgres/_index.md b/doc/user/content/ingest-data/postgres/_index.md index 15f961c52e4dc..9525a8344cf7d 100644 --- a/doc/user/content/ingest-data/postgres/_index.md +++ b/doc/user/content/ingest-data/postgres/_index.md @@ -37,6 +37,11 @@ Materialize gives you the following benefits: Materialize as a read-replica to build views on top of your PostgreSQL data that are efficiently maintained and always up-to-date. +When a source is created, Materialize parallelizes the initial snapshot +across the cluster's workers and, on PostgreSQL 14 and later, splits each +table's read across workers. See [Snapshot +parallelism](/concepts/snapshotting/#parallelism). + ## Supported versions and services The PostgreSQL source requires **PostgreSQL 11+** and is compatible with most diff --git a/doc/user/content/ingest-data/troubleshooting.md b/doc/user/content/ingest-data/troubleshooting.md index 03d39904a242a..6dbd2d9c0ca56 100644 --- a/doc/user/content/ingest-data/troubleshooting.md +++ b/doc/user/content/ingest-data/troubleshooting.md @@ -99,6 +99,41 @@ also be necessary to support increased memory usage during the process. For more information, see [Use a larger cluster for upsert source snapshotting](/ingest-data/#use-a-larger-cluster-for-upsert-source-snapshotting). +## Is the upstream database overloaded? + +Snapshotting can put significant load on the upstream database (see [Impact +on upstream system](/concepts/snapshotting/#impact-on-upstream-system)). + +Check the upstream database when a snapshot progresses more slowly than +expected, when applications sharing the database slow down while +it runs, or when the source reports upstream connection errors or timeouts. +The relevant metrics are in your cloud provider's monitoring console, or in +OS tools like `iostat` and the database's activity views for self-hosted +databases. Look for: + +- **Read IOPS or throughput** flat at a provisioned cap. +- **CPU** pinned at the instance's limit for the duration of the snapshot. +- **Network throughput** at the instance type's cap. +- **Connections** near the database's limit. For PostgreSQL and MySQL + sources, snapshotting opens connections in proportion to the source + cluster's workers. + +Also watch disk usage on the upstream database during a long-running +snapshot: CDC database sources must retain their change log until Materialize +consumes it (see [Impact on upstream +system](/concepts/snapshotting/#impact-on-upstream-system)). + +If the database is overloaded, you can upsize the source database or cancel +the snapshot by dropping the source, and retry: + +- on a smaller source cluster to spread the load over a longer window. +- with more IOPS, throughput, or instance capacity provisioned for the + database. +- during off-peak hours when the database is less busy, as recommended in the + [ingestion best practices](/ingest-data/#scheduling). +- with a smaller [volume of data to + sync](/ingest-data/#limit-the-volume-of-data). + ## Adding a new subsource to an existing source blocks replication. Should I just create a new source instead? It depends. Materialize provides transactional guarantees for subsource of the From ac13ded121cb023a79a90781b1e8bc514ec27e97 Mon Sep 17 00:00:00 2001 From: Materialize Date: Thu, 27 Aug 2026 00:00:00 +0000 Subject: [PATCH 10/11] LICENSE: update change date --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index e42009d58edca..936bb0dc3db90 100644 --- a/LICENSE +++ b/LICENSE @@ -13,7 +13,7 @@ Business Source License 1.1 Licensor: Materialize, Inc. -Licensed Work: Materialize Version 20260826 +Licensed Work: Materialize Version 20260827 The Licensed Work is © 2026 Materialize, Inc. Additional Use Grant: Within a single installation of the Licensed Work, @@ -39,7 +39,7 @@ Additional Use Grant: Within a single installation of the Licensed Work, whose definitions are controlled by such third parties. -Change Date: August 26, 2030 +Change Date: August 27, 2030 Change License: Apache License, Version 2.0 From 149bcf3cba625affc49e145615a33f18f3e4d847 Mon Sep 17 00:00:00 2001 From: Materialize Bot Date: Thu, 27 Aug 2026 02:39:01 +0000 Subject: [PATCH 11/11] release: mark v26.39 as released --- doc/user/content/releases/v26.39.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/user/content/releases/v26.39.md b/doc/user/content/releases/v26.39.md index e8f03d3c121d6..79999b6a4c083 100644 --- a/doc/user/content/releases/v26.39.md +++ b/doc/user/content/releases/v26.39.md @@ -1,9 +1,8 @@ --- title: Materialize v26.39 date: 2026-08-26 -released: false +released: true patch: 0 -rc: 1 publish_helm_chart: true build: render: never