From 541fb018a255306ab7eb1a6670e9eb3efa7420ab Mon Sep 17 00:00:00 2001 From: Qindeel Ishtiaq Date: Wed, 19 Aug 2026 13:11:01 -0400 Subject: [PATCH 01/10] catalog: add mz_object_graph_edges builtin view and index (#38224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `mz_internal.mz_object_graph_edges`, a maintained view (indexed on `mz_catalog_server`) that exposes object dependency edges. Each row `(object_id, dependency_id)` means `object_id` depends on `dependency_id`. The view unions the dataflow-layer dependencies between maintained objects (indexes, materialized views, sinks, sources, tables) with the source-to-subsource and source-to-table edges that stitch a source to the children carrying its data. ## Console surfaces The console reads these edges to walk object dependencies from a single maintained arrangement, instead of recomputing a per-view union of the dependency relations on every request. Surfaces that consume the DAG: - The object **workflow graph**, the dependency graph on an object's detail page. - **Critical-path freshness analysis** for maintained objects, which walks upstream to find the slowest input. - **Impact and dependents** views, the blast radius of changing or dropping an object. ## Operational impact Adds one maintained arrangement to `mz_catalog_server` (~200kB of memory). ## Tests - Existence and schema goldens: `information_schema_tables.slt`, `oid.slt`, `mz_catalog_server_index_accounting.slt`, `autogenerated/mz_internal.slt`, `catalog.td`, `indexes.td`. - A functional edge test added to `dataflow-dependencies.td` covering all three union branches: the maintained dependency chain (materialized view, index, and sink), the source-to-subsource stitching, and the source-to-table stitching. ## Release notes This release will add the `mz_internal.mz_object_graph_edges` system catalog view, which lists the dependency edges between maintained objects and the sources of their subsources and tables. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 --- .../reference/system-catalog/mz_internal.md | 16 +++ .../catalog/open/builtin_schema_migration.rs | 9 ++ src/catalog/src/builtin.rs | 2 + src/catalog/src/builtin/mz_internal.rs | 69 +++++++++++ src/pgrepr-consts/src/oid.rs | 2 + .../autogenerated/mz_internal.slt | 7 ++ test/sqllogictest/catalog_server_explain.slt | 113 +++++++++++++++++- test/sqllogictest/cluster.slt | 4 +- test/sqllogictest/cockroach/srfs.slt | 1 + .../information_schema_tables.slt | 4 + .../mz_catalog_server_index_accounting.slt | 31 +++-- test/sqllogictest/oid.slt | 2 + test/testdrive/catalog.td | 3 +- test/testdrive/dataflow-dependencies.td | 58 +++++++++ test/testdrive/indexes.td | 1 + 15 files changed, 305 insertions(+), 17 deletions(-) diff --git a/doc/user/content/reference/system-catalog/mz_internal.md b/doc/user/content/reference/system-catalog/mz_internal.md index e8d4a88ce2fc4..e16c77b2d1bf0 100644 --- a/doc/user/content/reference/system-catalog/mz_internal.md +++ b/doc/user/content/reference/system-catalog/mz_internal.md @@ -646,6 +646,22 @@ all database objects in the system. | `object_id` | [`text`] | The ID of the dependent object. Corresponds to [`mz_objects.id`](../mz_catalog/#mz_objects). | | `referenced_object_id` | [`text`] | The ID of the referenced object. Corresponds to [`mz_objects.id`](../mz_catalog/#mz_objects). | +## `mz_object_graph_edges` + +The `mz_object_graph_edges` view describes the dependency edges between maintained +objects, as rendered by object dependency graphs. + +It unions the dataflow-layer dependencies from +[`mz_materialization_dependencies`](#mz_materialization_dependencies), restricted to +indexes, materialized views, sinks, sources, and tables, with the source-to-subsource +and source-to-table edges that stand in for sources whose subsources carry the data. + + +| Field | Type | Meaning | +| --------------- | -------- | -------------------------------------------------------------------------------------- | +| `object_id` | [`text`] | The ID of the dependent object. Corresponds to [`mz_objects.id`](../mz_catalog/#mz_objects). | +| `dependency_id` | [`text`] | The ID of the object it depends on. Corresponds to [`mz_objects.id`](../mz_catalog/#mz_objects). | + ## `mz_object_fully_qualified_names` The `mz_object_fully_qualified_names` view enriches the [`mz_catalog.mz_objects`](/reference/system-catalog/mz_catalog/#mz_objects) view with namespace information. diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index 5adf8fe395df3..823fe46dea9a6 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -390,6 +390,15 @@ static MIGRATIONS: LazyLock> = LazyLock::new(|| { MZ_CATALOG_SCHEMA, "mz_audit_events", ), + // Required because we added the `mz_object_graph_edges_ind` builtin index. + // make_mz_indexes inlines the builtin-index set as VALUES, so any add or + // remove changes its SQL fingerprint and requires an explicit replacement. + MigrationStep::replacement( + "26.39.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_indexes", + ), ] }); diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs index a29b875cf999f..0a1ec9ce80086 100644 --- a/src/catalog/src/builtin.rs +++ b/src/catalog/src/builtin.rs @@ -1372,6 +1372,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::Source(&MZ_MATERIALIZED_VIEW_REFRESHES), Builtin::Source(&MZ_COMPUTE_DEPENDENCIES), Builtin::View(&MZ_MATERIALIZATION_DEPENDENCIES), + Builtin::View(&MZ_OBJECT_GRAPH_EDGES), Builtin::View(&MZ_MATERIALIZATION_LAG), Builtin::View(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW), Builtin::View(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H), @@ -1437,6 +1438,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::Index(&MZ_OBJECT_DEPENDENCIES_IND), Builtin::Index(&MZ_COMPUTE_DEPENDENCIES_IND), Builtin::Index(&MZ_OBJECT_TRANSITIVE_DEPENDENCIES_IND), + Builtin::Index(&MZ_OBJECT_GRAPH_EDGES_IND), Builtin::Index(&MZ_FRONTIERS_IND), Builtin::Index(&MZ_WALLCLOCK_GLOBAL_LAG_RECENT_HISTORY_IND), Builtin::Index(&MZ_KAFKA_SOURCES_IND), diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs index f8adc09251b81..eeedca513823a 100644 --- a/src/catalog/src/builtin/mz_internal.rs +++ b/src/catalog/src/builtin/mz_internal.rs @@ -3898,6 +3898,66 @@ pub static MZ_OBJECTS_ID_NAMESPACE_TYPES: LazyLock = LazyLock::new( ontology: None, }); +/// Object dependency edges. Each row `(object_id, dependency_id)` means +/// `object_id` depends on `dependency_id`. +/// +/// Unions the dataflow dependencies between maintained objects (index, +/// materialized view, sink, source, table) with the source-to-subsource and +/// source-to-table edges that connect a source to the children carrying its +/// data. Indexed on `mz_catalog_server` so the console surfaces that walk the +/// dependency graph read one maintained arrangement instead of recomputing the +/// union per request: the object workflow graph, critical-path freshness +/// analysis, and impact/dependents views. +pub static MZ_OBJECT_GRAPH_EDGES: LazyLock = LazyLock::new(|| BuiltinView { + name: "mz_object_graph_edges", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::VIEW_MZ_OBJECT_GRAPH_EDGES_OID, + desc: RelationDesc::builder() + .with_column("object_id", SqlScalarType::String.nullable(false)) + .with_column("dependency_id", SqlScalarType::String.nullable(false)) + .with_key(vec![0, 1]) + .finish(), + column_comments: BTreeMap::from_iter([ + ( + "object_id", + "The ID of the dependent object. Corresponds to `mz_objects.id`.", + ), + ( + "dependency_id", + "The ID of the object it depends on. Corresponds to `mz_objects.id`.", + ), + ]), + sql: " +SELECT md.object_id, md.dependency_id +FROM mz_internal.mz_materialization_dependencies md +JOIN mz_catalog.mz_objects po ON po.id = md.dependency_id + AND po.type IN ('index', 'materialized-view', 'sink', 'source', 'table') +JOIN mz_catalog.mz_objects co ON co.id = md.object_id + AND co.type IN ('index', 'materialized-view', 'sink', 'source', 'table') +UNION +-- Subsource -> parent-source edges: a subsource depends on the (user) source it +-- belongs to, an edge mz_materialization_dependencies doesn't carry. +SELECT od.object_id, od.referenced_object_id +FROM mz_internal.mz_object_dependencies od +JOIN mz_catalog.mz_sources ps ON ps.id = od.referenced_object_id +JOIN mz_catalog.mz_sources cs ON cs.id = od.object_id +-- Progress collections are deliberately left out: their dependency edge points +-- source -> progress, and they only exist for old-syntax sources, which the +-- source-table migration is removing. +WHERE ps.id LIKE 'u%' AND cs.type = 'subsource' +UNION +-- Select the (non-null) source id from the join rather than the nullable +-- mz_tables.source_id, so dependency_id is non-null across all branches. +SELECT t.id, ps.id +FROM mz_catalog.mz_tables t +JOIN mz_catalog.mz_sources ps ON ps.id = t.source_id", + access: vec![PUBLIC_SELECT], + // No ontology entity: these edges are already in the ontology via the + // DependsOn links of mz_object_dependencies and + // mz_materialization_dependencies. An entity here would duplicate them. + ontology: None, +}); + pub static MZ_OBJECT_OID_ALIAS: LazyLock = LazyLock::new(|| BuiltinView { name: "mz_object_oid_alias", schema: MZ_INTERNAL_SCHEMA, @@ -8760,6 +8820,15 @@ ON mz_internal.mz_object_transitive_dependencies (object_id)", is_retained_metrics_object: false, }; +pub const MZ_OBJECT_GRAPH_EDGES_IND: BuiltinIndex = BuiltinIndex { + name: "mz_object_graph_edges_ind", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::INDEX_MZ_OBJECT_GRAPH_EDGES_IND_OID, + sql: "IN CLUSTER mz_catalog_server +ON mz_internal.mz_object_graph_edges (object_id)", + is_retained_metrics_object: false, +}; + pub const MZ_FRONTIERS_IND: BuiltinIndex = BuiltinIndex { name: "mz_frontiers_ind", schema: MZ_INTERNAL_SCHEMA, diff --git a/src/pgrepr-consts/src/oid.rs b/src/pgrepr-consts/src/oid.rs index 7a4716922990f..4db963663c2d2 100644 --- a/src/pgrepr-consts/src/oid.rs +++ b/src/pgrepr-consts/src/oid.rs @@ -822,3 +822,5 @@ pub const FUNC_PARSE_CONNECTION_DETAILS_OID: u32 = 17112; pub const FUNC_MZ_AWS_ACCOUNT_ID_OID: u32 = 17113; pub const FUNC_MZ_AWS_EXTERNAL_ID_PREFIX_OID: u32 = 17114; pub const FUNC_MZ_AWS_CONNECTION_ROLE_ARN_OID: u32 = 17115; +pub const VIEW_MZ_OBJECT_GRAPH_EDGES_OID: u32 = 17116; +pub const INDEX_MZ_OBJECT_GRAPH_EDGES_IND_OID: u32 = 17117; diff --git a/test/sqllogictest/autogenerated/mz_internal.slt b/test/sqllogictest/autogenerated/mz_internal.slt index c09f62672307c..8716803cef900 100644 --- a/test/sqllogictest/autogenerated/mz_internal.slt +++ b/test/sqllogictest/autogenerated/mz_internal.slt @@ -352,6 +352,12 @@ SELECT name, type, comment FROM objects WHERE schema = 'mz_internal' AND object object_id text The␠ID␠of␠the␠dependent␠object.␠Corresponds␠to␠`mz_objects.id`. referenced_object_id text The␠ID␠of␠the␠referenced␠object.␠Corresponds␠to␠`mz_objects.id`. +query TTT +SELECT name, type, comment FROM objects WHERE schema = 'mz_internal' AND object = 'mz_object_graph_edges' ORDER BY position +---- +object_id text The␠ID␠of␠the␠dependent␠object.␠Corresponds␠to␠`mz_objects.id`. +dependency_id text The␠ID␠of␠the␠object␠it␠depends␠on.␠Corresponds␠to␠`mz_objects.id`. + query TTT SELECT name, type, comment FROM objects WHERE schema = 'mz_internal' AND object = 'mz_object_fully_qualified_names' ORDER BY position ---- @@ -829,6 +835,7 @@ mz_object_arrangement_sizes mz_object_dependencies mz_object_fully_qualified_names mz_object_global_ids +mz_object_graph_edges mz_object_history mz_object_lifetimes mz_object_oid_alias diff --git a/test/sqllogictest/catalog_server_explain.slt b/test/sqllogictest/catalog_server_explain.slt index ee7a28553dc06..648f9305963cf 100644 --- a/test/sqllogictest/catalog_server_explain.slt +++ b/test/sqllogictest/catalog_server_explain.slt @@ -1558,6 +1558,103 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN INDEX "mz_internal"."mz_object_graph_edges_ind"; +---- +mz_internal.mz_object_graph_edges_ind: + →Arrange (#0{object_id}) + →Arranged mz_internal.mz_object_graph_edges + +mz_internal.mz_object_graph_edges: + →With + cte l0 = + →Arrange (#0{id}) + →Fused with Child Map/Filter/Project + Project: #1 + Filter: ((#4{type} = "sink") OR (#4{type} = "index") OR (#4{type} = "table") OR (#4{type} = "source") OR (#4{type} = "materialized-view")) + →Arranged mz_catalog.mz_objects + Key: (#2{schema_id}) + cte l1 = + →Arranged mz_catalog.mz_sources + →Return + →Distinct GroupAggregate + →Union + →Delta Join [%0[#0{object_id}] » %1:l0[#0{id}] » %2:l0[#0{id}]] [%1:l0[#0{id}] » %0[#1{dependency_id}] » %2:l0[#0{id}]] [%2:l0[#0{id}] » %0[#0{object_id}] » %1:l0[#0{id}]] + →Arrange (#0{object_id}) (#1{dependency_id}) + →Union + →Unarranged Raw Stream + →Fused with Parent Map/Filter/Project + Project: #1, #0 + →Arranged mz_internal.mz_compute_dependencies + →Delta Join [%0:mz_object_dependencies[#0{object_id}] » %1:mz_sinks[#0{id}] » %2[#0{id}]] [%1:mz_sinks[#0{id}] » %0:mz_object_dependencies[#0{object_id}] » %2[#0{id}]] [%2[#0{id}] » %0:mz_object_dependencies[#1{referenced_object_id}] » %1:mz_sinks[#0{id}]] + →Arrange (#1{referenced_object_id}) + →Arranged mz_internal.mz_object_dependencies + →Arranged mz_catalog.mz_sinks + →Arrange (#0{id}) + →Union + →Fused with Child Map/Filter/Project + Project: #1 + →Arranged mz_catalog.mz_tables + Key: (#2{schema_id}) + →Fused with Child Map/Filter/Project + Project: #0 + →Arranged mz_catalog.mz_sources + Key: (#0{id}) + →Fused with Child Map/Filter/Project + Project: #1 + →Arranged mz_catalog.mz_views + Key: (#2{schema_id}) + →Fused with Child Map/Filter/Project + Project: #0 + →Arranged mz_catalog.mz_materialized_views + Key: (#0{id}) + →Arranged l0 + →Arranged l0 + →Delta Join [%0:mz_object_dependencies[#0{object_id}] » %2:l1[#0{id}] » %1:l1[#0{id}]] [%1:l1[#0{id}] » %0:mz_object_dependencies[#1{referenced_object_id}] » %2:l1[#0{id}]] [%2:l1[#0{id}] » %0:mz_object_dependencies[#0{object_id}] » %1:l1[#0{id}]] + path %0: + after %2: + Project: #0, #1 + Filter: (#5{type} = "subsource") + path %1: + after %0: + Project: #1, #0 + Filter: like["u%"](#0{referenced_object_id}) + after %2: + Project: #0, #1 + Filter: (#5{type} = "subsource") + path %2: + after %0: + Filter: like["u%"](#1{referenced_object_id}) + →Arrange (#0{object_id}) (#1{referenced_object_id}) + →Fused with Child Map/Filter/Project + Filter: like["u%"](#1{referenced_object_id}) + →Arranged mz_internal.mz_object_dependencies + Key: (#0{object_id}) + →Arranged l1 + →Arranged l1 + →Differential Join %1:l1[#0{id}] » %0:mz_tables[#1{source_id}] + →Arrange (#1{source_id}) + →Fused with Child Map/Filter/Project + Project: #1, #8 + Filter: (#8{source_id}) IS NOT NULL + →Arranged mz_catalog.mz_tables + Key: (#2{schema_id}) + →Arranged l1 + +Used Indexes: + - mz_catalog.mz_sources_ind (*** full scan ***, differential join, delta join lookup) + - mz_catalog.mz_sinks_ind (delta join lookup) + - mz_catalog.mz_materialized_views_ind (*** full scan ***) + - mz_internal.mz_object_dependencies_ind (*** full scan ***, delta join 1st input (full scan)) + - mz_internal.mz_compute_dependencies_ind (*** full scan ***) + - mz_catalog.mz_tables_ind (*** full scan ***) + - mz_catalog.mz_objects_ind (*** full scan ***) + - mz_catalog.mz_views_ind (*** full scan ***) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN INDEX "mz_internal"."mz_object_history_ind"; ---- @@ -4811,7 +4908,7 @@ mz_catalog.mz_indexes: Project: #2, #0, #1, #3, #4, #7, #5, #6 Map: "s1" →Arrange (#1{name}) (#2{on_schema}, #3{on_name}) - →Constant (80 rows) + →Constant (81 rows) →Arrange (empty key) (#1{name}) →Fused with Child Map/Filter/Project Project: #4, #3 @@ -9903,6 +10000,20 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN SELECT * FROM "mz_internal"."mz_object_graph_edges"; +---- +Explained Query (fast path): + →Map/Filter/Project + →Indexed mz_internal.mz_object_graph_edges (using mz_internal.mz_object_graph_edges_ind) + +Used Indexes: + - mz_internal.mz_object_graph_edges_ind (*** full scan ***) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_object_history"; ---- diff --git a/test/sqllogictest/cluster.slt b/test/sqllogictest/cluster.slt index 5766418e810a0..10cb6649b6d62 100644 --- a/test/sqllogictest/cluster.slt +++ b/test/sqllogictest/cluster.slt @@ -420,7 +420,7 @@ CREATE CLUSTER test REPLICAS (foo (SIZE 'scale=1,workers=1')); query I SELECT COUNT(name) FROM mz_indexes; ---- -304 +305 statement ok DROP CLUSTER test CASCADE @@ -428,7 +428,7 @@ DROP CLUSTER test CASCADE query T SELECT COUNT(name) FROM mz_indexes; ---- -272 +273 simple conn=mz_system,user=mz_system ALTER CLUSTER quickstart OWNER TO materialize diff --git a/test/sqllogictest/cockroach/srfs.slt b/test/sqllogictest/cockroach/srfs.slt index b8a169c3836e1..10592810e8f52 100644 --- a/test/sqllogictest/cockroach/srfs.slt +++ b/test/sqllogictest/cockroach/srfs.slt @@ -1472,6 +1472,7 @@ mz_object_arrangement_size_history 2 mz_object_arrangement_size_history 4 mz_object_arrangement_sizes 1 mz_object_dependencies 1 +mz_object_graph_edges 1 mz_object_history 1 mz_object_lifetimes 1 mz_object_transitive_dependencies 1 diff --git a/test/sqllogictest/information_schema_tables.slt b/test/sqllogictest/information_schema_tables.slt index 3f1d1a0dca450..f949fb8e19f98 100644 --- a/test/sqllogictest/information_schema_tables.slt +++ b/test/sqllogictest/information_schema_tables.slt @@ -493,6 +493,10 @@ mz_object_global_ids BASE TABLE materialize mz_internal +mz_object_graph_edges +VIEW +materialize +mz_internal mz_object_history VIEW materialize diff --git a/test/sqllogictest/mz_catalog_server_index_accounting.slt b/test/sqllogictest/mz_catalog_server_index_accounting.slt index 95138d7704663..6db27834a2053 100644 --- a/test/sqllogictest/mz_catalog_server_index_accounting.slt +++ b/test/sqllogictest/mz_catalog_server_index_accounting.slt @@ -38,10 +38,10 @@ mz_arrangement_heap_size_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_he mz_arrangement_records_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_records_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_records_raw"␠("operator_id",␠"worker_id") mz_arrangement_sharing_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_sharing_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_sharing_raw"␠("operator_id",␠"worker_id") mz_cluster_auto_scaling_strategies_ind CREATE␠INDEX␠"mz_cluster_auto_scaling_strategies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s518␠AS␠"mz_internal"."mz_cluster_auto_scaling_strategies"]␠("cluster_id") -mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s765␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") +mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s766␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") mz_cluster_prometheus_metrics_s2_primary_idx CREATE␠INDEX␠"mz_cluster_prometheus_metrics_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_cluster_prometheus_metrics"␠("process_id",␠"metric_name",␠"labels") mz_cluster_reconfigurations_ind CREATE␠INDEX␠"mz_cluster_reconfigurations_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s517␠AS␠"mz_internal"."mz_cluster_reconfigurations"]␠("cluster_id") -mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s759␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") +mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s760␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") mz_cluster_replica_history_ind CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s615␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at") mz_cluster_replica_metrics_history_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s523␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id") mz_cluster_replica_metrics_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s524␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id") @@ -59,23 +59,23 @@ mz_compute_dependencies_ind CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN mz_compute_error_counts_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_error_counts_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_error_counts_raw"␠("export_id",␠"worker_id") mz_compute_exports_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_exports_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_exports_per_worker"␠("export_id",␠"worker_id") mz_compute_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_frontiers_per_worker"␠("export_id",␠"worker_id") -mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s750␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") +mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s751␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") mz_compute_hydration_times_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_hydration_times_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_hydration_times_per_worker"␠("export_id",␠"worker_id") mz_compute_import_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_import_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_import_frontiers_per_worker"␠("export_id",␠"import_id",␠"worker_id") mz_compute_lir_mapping_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_lir_mapping_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_lir_mapping_per_worker"␠("global_id",␠"lir_id",␠"worker_id") mz_compute_operator_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_durations_histogram_raw"␠("id",␠"worker_id",␠"duration_ns") mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_hydration_statuses_per_worker"␠("export_id",␠"lir_id",␠"worker_id") mz_connections_ind CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s520␠AS␠"mz_catalog"."mz_connections"]␠("schema_id") -mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s746␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") -mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s745␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") -mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s744␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") +mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s747␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") +mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s746␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") +mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s745␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") mz_databases_ind CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s483␠AS␠"mz_catalog"."mz_databases"]␠("name") mz_dataflow_addresses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_addresses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_addresses_per_worker"␠("id",␠"worker_id") mz_dataflow_channels_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_channels_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_channels_per_worker"␠("id",␠"worker_id") mz_dataflow_operator_reachability_raw_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operator_reachability_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operator_reachability_raw"␠("id",␠"worker_id",␠"source",␠"port",␠"update_type",␠"time") mz_dataflow_operators_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operators_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operators_per_worker"␠("id",␠"worker_id") mz_frontiers_ind CREATE␠INDEX␠"mz_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s732␠AS␠"mz_internal"."mz_frontiers"]␠("object_id") -mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s761␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") +mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s762␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") mz_indexes_ind CREATE␠INDEX␠"mz_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s486␠AS␠"mz_catalog"."mz_indexes"]␠("id") mz_kafka_sources_ind CREATE␠INDEX␠"mz_kafka_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s480␠AS␠"mz_catalog"."mz_kafka_sources"]␠("id") mz_materialized_views_ind CREATE␠INDEX␠"mz_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s544␠AS␠"mz_catalog"."mz_materialized_views"]␠("id") @@ -83,11 +83,12 @@ mz_message_batch_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_messag mz_message_batch_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_batch_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") -mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s835␠AS␠"mz_internal"."mz_notices"]␠("id") -mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") -mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") -mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s751␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") +mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s837␠AS␠"mz_internal"."mz_notices"]␠("id") +mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s754␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") +mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s754␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") +mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s752␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") mz_object_dependencies_ind CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s481␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id") +mz_object_graph_edges_ind CREATE␠INDEX␠"mz_object_graph_edges_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s743␠AS␠"mz_internal"."mz_object_graph_edges"]␠("object_id") mz_object_history_ind CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s555␠AS␠"mz_internal"."mz_object_history"]␠("id") mz_object_lifetimes_ind CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s556␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id") mz_object_transitive_dependencies_ind CREATE␠INDEX␠"mz_object_transitive_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s572␠AS␠"mz_internal"."mz_object_transitive_dependencies"]␠("object_id") @@ -95,14 +96,14 @@ mz_objects_ind CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[ mz_peek_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_peek_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_peek_durations_histogram_raw"␠("worker_id",␠"type",␠"duration_ns") mz_recent_activity_log_thinned_ind CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s716␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash") mz_recent_sql_text_ind CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s712␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash") -mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s828␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") +mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s830␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") mz_roles_ind CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s504␠AS␠"mz_catalog"."mz_roles"]␠("id") mz_scheduling_elapsed_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_elapsed_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_elapsed_raw"␠("id",␠"worker_id") mz_scheduling_parks_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_parks_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_parks_histogram_raw"␠("worker_id",␠"slept_for_ns",␠"requested_ns") mz_schemas_ind CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s484␠AS␠"mz_catalog"."mz_schemas"]␠("database_id") mz_secrets_ind CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s519␠AS␠"mz_catalog"."mz_secrets"]␠("name") mz_show_all_objects_ind CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s600␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id") -mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s763␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") +mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s764␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") mz_show_clusters_ind CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s602␠AS␠"mz_internal"."mz_show_clusters"]␠("name") mz_show_columns_ind CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s601␠AS␠"mz_internal"."mz_show_columns"]␠("id") mz_show_connections_ind CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s610␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id") @@ -489,6 +490,8 @@ mz_map_types key_id mz_map_types key_modifiers mz_map_types value_id mz_map_types value_modifiers +mz_materialization_dependencies dependency_id +mz_materialization_dependencies object_id mz_materialized_views cluster_id mz_materialized_views create_sql mz_materialized_views definition @@ -532,6 +535,8 @@ mz_object_arrangement_sizes replica_id mz_object_arrangement_sizes size mz_object_dependencies object_id mz_object_dependencies referenced_object_id +mz_object_graph_edges dependency_id +mz_object_graph_edges object_id mz_object_history cluster_id mz_object_history created_at mz_object_history dropped_at diff --git a/test/sqllogictest/oid.slt b/test/sqllogictest/oid.slt index a563054773926..c32d7555d73f5 100644 --- a/test/sqllogictest/oid.slt +++ b/test/sqllogictest/oid.slt @@ -1245,3 +1245,5 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17113 mz_aws_account_id 17114 mz_aws_external_id_prefix 17115 mz_aws_connection_role_arn +17116 mz_object_graph_edges +17117 mz_object_graph_edges_ind diff --git a/test/testdrive/catalog.td b/test/testdrive/catalog.td index 336e31057b87a..6cc485a3a3bbc 100644 --- a/test/testdrive/catalog.td +++ b/test/testdrive/catalog.td @@ -646,6 +646,7 @@ mz_mcp_data_products "" mz_notices "" mz_notices_redacted "" mz_object_fully_qualified_names "" +mz_object_graph_edges "" mz_object_history "" mz_object_lifetimes "" mz_object_oid_alias "" @@ -834,7 +835,7 @@ test_table "" # There is one entry in mz_indexes for each field_number/expression of the index. > SELECT COUNT(id) FROM mz_indexes WHERE id LIKE 's%' -272 +273 # Create a second schema with the same table name as above > CREATE SCHEMA tester2 diff --git a/test/testdrive/dataflow-dependencies.td b/test/testdrive/dataflow-dependencies.td index f8be9eb9b8e65..d5d715d3902b9 100644 --- a/test/testdrive/dataflow-dependencies.td +++ b/test/testdrive/dataflow-dependencies.td @@ -199,3 +199,61 @@ snk_cleanup t1 LEFT JOIN mz_objects object ON dep.object_id = object.id LEFT JOIN mz_objects import ON dep.dependency_id = import.id WHERE object_id LIKE 'u%' + +# Test `mz_internal.mz_object_graph_edges`, the console workflow-graph edge set: +# the dependencies between maintained objects, plus the edges that stitch a +# source to the subsources/tables carrying its data. Like the relations above it +# is updated asynchronously, so these rely on testdrive's retry feature. + +> CREATE TABLE base (a int) +> CREATE VIEW unmaintained AS SELECT * FROM base +> CREATE MATERIALIZED VIEW mvg AS SELECT * FROM unmaintained +> CREATE DEFAULT INDEX ON mvg +> CREATE SINK snk_graph + IN CLUSTER ${arg.single-replica-cluster} + FROM mvg + INTO KAFKA CONNECTION kafka_conn (TOPIC 'testdrive-sink-graph-${testdrive.seed}') + FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_conn + ENVELOPE DEBEZIUM + +# The intermediate view is not a maintained object, so it carries no edge: the +# chain is index -> mvg -> base, and sink -> mvg alongside it. +> SELECT object.name, dependency.name + FROM mz_internal.mz_object_graph_edges e + JOIN mz_objects object ON e.object_id = object.id + JOIN mz_objects dependency ON e.dependency_id = dependency.id + WHERE object.name IN ('mvg', 'mvg_primary_idx', 'snk_graph') + ORDER BY object.name, dependency.name +mvg base +mvg_primary_idx mvg +snk_graph mvg + +> CREATE SOURCE src_sub IN CLUSTER ${arg.single-replica-cluster} FROM LOAD GENERATOR AUCTION FOR ALL TABLES +> CREATE MATERIALIZED VIEW mv_bids AS SELECT count(*) FROM bids + +# `FOR ALL TABLES` creates subsources, which are stitched to their parent source +# (bids -> src_sub), an edge that mz_materialization_dependencies does not carry, +# alongside the maintained dependency mv_bids -> bids. +> SELECT object.name, dependency.name + FROM mz_internal.mz_object_graph_edges e + JOIN mz_objects object ON e.object_id = object.id + JOIN mz_objects dependency ON e.dependency_id = dependency.id + WHERE object.name IN ('bids', 'mv_bids') + ORDER BY object.name, dependency.name +bids src_sub +mv_bids bids + +> CREATE SOURCE src_tbl IN CLUSTER ${arg.single-replica-cluster} FROM LOAD GENERATOR AUCTION (UP TO 100) +> CREATE TABLE bids_tbl FROM SOURCE src_tbl (REFERENCE bids) + +# A table fed by a source is stitched to it through `mz_tables.source_id`. +> SELECT object.name, dependency.name + FROM mz_internal.mz_object_graph_edges e + JOIN mz_objects object ON e.object_id = object.id + JOIN mz_objects dependency ON e.dependency_id = dependency.id + WHERE object.name = 'bids_tbl' +bids_tbl src_tbl + +> DROP SOURCE src_sub CASCADE +> DROP SOURCE src_tbl CASCADE +> DROP TABLE base CASCADE diff --git a/test/testdrive/indexes.td b/test/testdrive/indexes.td index 8c9dc0dc73b3c..d54a1752031be 100644 --- a/test/testdrive/indexes.td +++ b/test/testdrive/indexes.td @@ -345,6 +345,7 @@ mz_object_arrangement_size_history_object_ind mz_object_arrangemen mz_object_arrangement_size_history_ts_ind mz_object_arrangement_size_history mz_catalog_server {collection_timestamp} "" mz_object_arrangement_sizes_ind mz_object_arrangement_sizes mz_catalog_server {replica_id} "" mz_object_dependencies_ind mz_object_dependencies mz_catalog_server {object_id} "" +mz_object_graph_edges_ind mz_object_graph_edges mz_catalog_server {object_id} "" mz_object_history_ind mz_object_history mz_catalog_server {id} "" mz_object_lifetimes_ind mz_object_lifetimes mz_catalog_server {id} "" mz_object_transitive_dependencies_ind mz_object_transitive_dependencies mz_catalog_server {object_id} "" From 34f652d85350df51c0a1ab107af306d5189e4b90 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Wed, 19 Aug 2026 19:15:47 +0200 Subject: [PATCH 02/10] compute: run the MV sink batch read-back under block_in_place (#38358) Follow-up to #38118, which moved the sync MV sink's correction maintenance to a blocking thread but left the read-back that follows it on the async task, with a comment claiming the batch builder's part flushes bound how long that occupies a Tokio worker. They do not. `BatchBuilder::add` only awaits once a part fills, and the part writes are spawned, so the builder awaits nothing until more than `persist_batch_builder_max_outstanding_parts` (2) writes are in flight. Feeding it therefore pulls on the order of two parts' worth of updates, `persist_blob_target_size` (128 MiB) each, before the first real yield. Every pull clones a row out of a correction chunk and, with the pager enabled, can page that chunk back in, so the worker ends up blocked in the kernel for the same reason the maintenance pass was. `emitted` holds every update below `upper`, so the read-back sweeps a buffer as large as the pass that already runs on a blocking thread. Move it there too: the blocking closure keeps ownership of the buffers, consolidates, and streams the consolidated updates back over a bounded channel in chunks, and the async task feeds those chunks to the persist batch builder. The row clones and page-ins land on a preemptible thread, and the handoff adds at most a few thousand buffered updates. `writer` no longer needs an exclusive borrow, because the builder is created with `WriteHandle::builder` rather than `WriteHandle::batch`. Based on QA LLM review: https://github.com/MaterializeInc/qa-llm-review/blob/master/commit-bugs/done/analysis-commit-0f7e5b187226c54a5feefd91feb929af6642c8b8.md ### Release notes This release will not include user-visible changes. Co-authored-by: Claude Opus 5 (1M context) --- src/compute/src/sink/materialized_view_v2.rs | 21 +++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/compute/src/sink/materialized_view_v2.rs b/src/compute/src/sink/materialized_view_v2.rs index 841a46a862c62..02c9a7f89d183 100644 --- a/src/compute/src/sink/materialized_view_v2.rs +++ b/src/compute/src/sink/materialized_view_v2.rs @@ -68,6 +68,7 @@ use timely::dataflow::operators::vec::Broadcast; use timely::dataflow::operators::{Capability, CapabilitySet}; use timely::progress::Antichain; use timely::progress::frontier::AntichainRef; +use tokio::runtime::Handle; use tokio::sync::{mpsc, watch}; use tracing::trace; @@ -868,9 +869,8 @@ mod write { ) .await; - // Reading the consolidated updates back only clones them into the batch builder, - // which awaits at every part flush, so it stays on the async task. Chain ok and - // err correction iterators directly, avoiding an intermediate Vec allocation. + // Chain ok and err correction iterators directly, avoiding an intermediate Vec + // allocation. let oks = corrections .ok .consolidated_updates_before(&desc.upper) @@ -888,10 +888,17 @@ mod write { return corrections; } - let batch = writer - .batch(updates, desc.lower, desc.upper) - .await - .expect("valid usage"); + // Feeding the builder pulls the updates out of the correction buffers on the + // calling thread: every pull clones a row and can page a chunk back in, while the + // builder only awaits once a part fills, i.e. after `blob_target_size` times + // `batch_builder_max_outstanding_parts` worth of updates. That is the same stall + // the consolidation above avoids, so it gets the same treatment. + // `block_in_place` rather than `spawn_blocking`, because the updates borrow the + // buffers and the builder needs an async context. + let batch = tokio::task::block_in_place(|| { + Handle::current().block_on(writer.batch(updates, desc.lower, desc.upper)) + }) + .expect("valid usage"); let proto_batch = batch.into_transmittable_batch(); if let Err(err) = resp_tx.send(WriteResponse { batch: Some(proto_batch), From 8933b16354b76d16b747c18c29a4182d39a65955 Mon Sep 17 00:00:00 2001 From: Justin Bradfield Date: Wed, 19 Aug 2026 12:26:58 -0500 Subject: [PATCH 03/10] CLO-117: reject AWS PrivateLink SERVICE NAME that is not a service name (#38185) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes [CLO-117](https://linear.app/materializeinc/issue/CLO-117/invalid-service-name-used-during-connection-reports-availability-zone). ### Motivation A user created an AWS PrivateLink connection with an ELB DNS hostname in `SERVICE NAME` instead of a VPC endpoint *service* name: ```sql CREATE CONNECTION c TO AWS PRIVATELINK ( SERVICE NAME 'my-db-lb-0123456789abcdef.elb.eu-central-1.amazonaws.com', AVAILABILITY ZONES () ); -- CREATE CONNECTION ``` `CREATE CONNECTION` accepted it, and the mistake only surfaced later as ``` Error: The Endpoint cannot be created due to missing availability zones ``` which points at the wrong option. The environment controller cannot parse that value as a cross-region service name, so it treats the endpoint as same-region and reports the empty `AVAILABILITY ZONES` list as the failure. ### Changes * `AwsPrivatelinkConnection::check_service_name` rejects values that cannot be AWS VPC endpoint service names. It is deliberately permissive: it requires only the `com.amazonaws.` prefix, so both customer-owned (`com.amazonaws.vpce..vpce-svc-`) and AWS-managed (`com.amazonaws..`) services pass. * A new `PlanError::InvalidPrivatelinkServiceName` carries a hint that gives the expected shape and says the value is not the DNS name of the target. * `CREATE CONNECTION` and `ALTER CONNECTION` now fail up front with that error. * `AwsPrivatelinkConnection::validate` performs the same check before reading endpoint status, so `VALIDATE CONNECTION` on a connection that already stores an offending value reports the true cause instead of the availability zone message. This is the exact symptom in the report, and it also removes the need for a new `VpcEndpointState` variant in the cloud repo. * Docs: the `SERVICE NAME` syntax element now says what the value is and is not. ### Why the sequencer and not the planner The obvious home for this is `plan_connection`, next to the existing `AVAILABILITY ZONES` validation. That is not safe. Catalog items are re-planned on boot via `CatalogState::deserialize_item`, and a failure there panics (`src/adapter/src/catalog/apply.rs`). Any environment that already stores an offending service name would crash-loop on upgrade, which is precisely the population this change is about. `with_enable_for_item_parsing` does not help either, since it force-*enables* flags during item parsing, so a flag-gated planner rule would still fire on boot. The sequencer runs only for statements a client issues, so the rule cannot stop an existing item from loading. For `CREATE` the check precedes id allocation, secret creation, resource-limit validation, and the catalog transaction, so nothing leaks before the rejection. Two accepted behavior changes for environments that already store an offending value: * `CREATE CONNECTION IF NOT EXISTS` with the offending name now errors instead of being a no-op. This matches how existing option validation already behaves (an invalid `AVAILABILITY ZONES` entry errors at plan time regardless of `IF NOT EXISTS`). * `ALTER CONNECTION` re-plans the full `create_sql`, so it reports the stored offending name even when the statement does not touch `SERVICE NAME`. Setting a valid `SERVICE NAME` in the same `ALTER` clears it. ### Tips for reviewer The permissiveness of the rule is the main thing to sanity check. It is prefix-only on purpose. Worth confirming with the cloud team that no live connection uses a service name outside `com.amazonaws.*` before this ships, since such a connection would start failing `VALIDATE CONNECTION` and `ALTER`. ### Checklist - [x] This PR has adequate test coverage / QA involvement has been duly considered. ([trigger-ci for additional test/nightly runs](https://trigger-ci.dev.materialize.com/)) - [ ] This PR has an associated up-to-date [design doc](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/design/README.md), is a design doc ([template](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/design/00000000_template.md)), or is sufficiently small to not require a design. - [ ] If this PR evolves [an existing `$T ⇔ Proto$T` mapping](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/command-and-response-binary-encoding.md) (possibly in a backwards-incompatible way), then it is tagged with a `T-proto` label. - [ ] If this PR will require changes to cloud orchestration or tests, there is a companion cloud PR to account for those changes that is tagged with the release-blocker label ([example](https://github.com/MaterializeInc/cloud/pull/5021)). - [x] This PR includes the following [user-facing behavior changes](https://github.com/MaterializeInc/materialize/blob/main/doc/developer/guide-changes.md#what-changes-require-a-release-note): - `CREATE CONNECTION ... TO AWS PRIVATELINK` and `ALTER CONNECTION` now reject a `SERVICE NAME` that is not an AWS VPC endpoint service name, and `VALIDATE CONNECTION` reports that as the cause instead of a missing availability zone. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 --- doc/user/data/examples/create_connection.yml | 3 +- src/adapter/src/coord/sequencer/inner.rs | 25 ++++- src/sql/src/plan/error.rs | 4 + src/storage-types/src/connections.rs | 98 +++++++++++++++++++- test/testdrive/connection-create-drop.td | 17 ++++ 5 files changed, 141 insertions(+), 6 deletions(-) diff --git a/doc/user/data/examples/create_connection.yml b/doc/user/data/examples/create_connection.yml index 57486bb95b9cd..fadab6d0e63dd 100644 --- a/doc/user/data/examples/create_connection.yml +++ b/doc/user/data/examples/create_connection.yml @@ -656,7 +656,8 @@ description: | *Value:* `text`. Required. - The name of the AWS PrivateLink service. + The name of the AWS VPC endpoint service, which always starts with + `com.amazonaws.`. - name: "`AVAILABILITY ZONES`" description: | *Value:* `text[]`. Required. diff --git a/src/adapter/src/coord/sequencer/inner.rs b/src/adapter/src/coord/sequencer/inner.rs index 3a8ccfd116224..af7b3b2a5501b 100644 --- a/src/adapter/src/coord/sequencer/inner.rs +++ b/src/adapter/src/coord/sequencer/inner.rs @@ -67,7 +67,7 @@ use mz_sql::names::{ SchemaSpecifier, SystemObjectId, }; use mz_sql::plan::{ - AlterMaterializedViewApplyReplacementPlan, ConnectionDetails, NetworkPolicyRule, + AlterMaterializedViewApplyReplacementPlan, ConnectionDetails, NetworkPolicyRule, PlanError, StatementContext, }; use mz_sql::pure::{PurifiedSourceExport, generate_subsource_statements}; @@ -95,6 +95,7 @@ use mz_sql_parser::ast::{ use mz_ssh_util::keys::SshKeyPairSet; use mz_storage_client::controller::ExportDescription; use mz_storage_types::AlterCompatible; +use mz_storage_types::connections::AwsPrivatelinkConnection; use mz_storage_types::connections::inline::IntoInlineConnection; use mz_storage_types::controller::StorageError; use mz_storage_types::sources::SourceConnection; @@ -183,6 +184,18 @@ where } } +/// Rejects connection options whose values cannot work, independent of any +/// external system. +fn check_connection_details(details: &ConnectionDetails) -> Result<(), PlanError> { + match details { + ConnectionDetails::AwsPrivatelink(privatelink) => { + AwsPrivatelinkConnection::check_service_name(&privatelink.service_name) + .map_err(PlanError::InvalidPrivatelinkServiceName) + } + _ => Ok(()), + } +} + struct DropOps { ops: Vec, dropped_active_db: bool, @@ -720,6 +733,10 @@ impl Coordinator { plan: plan::CreateConnectionPlan, resolved_ids: ResolvedIds, ) { + if let Err(err) = check_connection_details(&plan.connection.details) { + return ctx.retire(Err(AdapterError::PlanError(err))); + } + let (connection_id, connection_gid) = match self.allocate_user_id().await { Ok(item_id) => item_id, Err(err) => return ctx.retire(Err(err)), @@ -3777,6 +3794,12 @@ impl Coordinator { } }; + // `conn` is the whole re-planned connection, so this also rejects a + // stored value the statement did not touch. + if let Err(err) = check_connection_details(&conn.details) { + return ctx.retire(Err(AdapterError::InvalidAlter("CONNECTION", err))); + } + // Inspect guarded secrets whether or not validation was requested, // before the altered connection is installed in the catalog. if let Err(err) = self diff --git a/src/sql/src/plan/error.rs b/src/sql/src/plan/error.rs index 4644b2e497a50..ec848436bf9dc 100644 --- a/src/sql/src/plan/error.rs +++ b/src/sql/src/plan/error.rs @@ -31,6 +31,7 @@ use mz_sql_parser::ast::display::AstDisplay; use mz_sql_parser::ast::{IdentError, UnresolvedItemName}; use mz_sql_parser::parser::{ParserError, ParserStatementError}; use mz_sql_server_util::SqlServerError; +use mz_storage_types::connections::InvalidAwsPrivatelinkServiceName; use mz_storage_types::sources::ExternalReferenceResolutionError; use crate::catalog::{ @@ -237,6 +238,7 @@ pub enum PlanError { name: String, supported_azs: BTreeSet, }, + InvalidPrivatelinkServiceName(InvalidAwsPrivatelinkServiceName), DuplicatePrivatelinkAvailabilityZone { duplicate_azs: BTreeSet, }, @@ -457,6 +459,7 @@ impl PlanError { let supported_azs_str = supported_azs.iter().join("\n "); Some(format!("Did you supply an availability zone name instead of an ID? Known availability zone IDs:\n {}", supported_azs_str)) } + Self::InvalidPrivatelinkServiceName(err) => Some(err.hint()), Self::DuplicatePrivatelinkAvailabilityZone { duplicate_azs, ..} => { let duplicate_azs = duplicate_azs.iter().join("\n "); Some(format!("Duplicated availability zones:\n {}", duplicate_azs)) @@ -762,6 +765,7 @@ impl fmt::Display for PlanError { }) }, Self::InvalidPrivatelinkAvailabilityZone { name, ..} => write!(f, "invalid AWS PrivateLink availability zone {}", name.quoted()), + Self::InvalidPrivatelinkServiceName(err) => err.fmt(f), Self::DuplicatePrivatelinkAvailabilityZone {..} => write!(f, "connection cannot contain duplicate availability zones"), Self::InvalidSchemaName => write!(f, "no valid schema selected"), Self::ItemAlreadyExists { name, item_type } => write!(f, "{item_type} {} already exists", name.quoted()), diff --git a/src/storage-types/src/connections.rs b/src/storage-types/src/connections.rs index 7f76c8ff2b5f4..b5977fcaba52b 100644 --- a/src/storage-types/src/connections.rs +++ b/src/storage-types/src/connections.rs @@ -11,6 +11,7 @@ use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; use std::net::SocketAddr; use std::sync::Arc; use std::time::SystemTime; @@ -50,6 +51,7 @@ use mz_ore::error::ErrorExt; use mz_ore::future::{InTask, OreFutureExt}; use mz_ore::netio::resolve_address; use mz_ore::num::NonNeg; +use mz_ore::str::StrExt; use mz_repr::{CatalogItemId, GlobalId}; use mz_secrets::SecretsReader; use mz_sql_parser::ast::ConnectionRulePattern; @@ -543,6 +545,8 @@ pub enum ConnectionValidationError { Aws(#[from] AwsConnectionValidationError), #[error(transparent)] Gcp(#[from] gcp::GcpConnectionValidationError), + #[error(transparent)] + AwsPrivatelinkServiceName(#[from] InvalidAwsPrivatelinkServiceName), #[error("{}", .0.display_with_causes())] Other(#[from] anyhow::Error), } @@ -556,6 +560,7 @@ impl ConnectionValidationError { ConnectionValidationError::SqlServer(e) => e.detail(), ConnectionValidationError::Aws(e) => e.detail(), ConnectionValidationError::Gcp(e) => e.detail(), + ConnectionValidationError::AwsPrivatelinkServiceName(_) => None, ConnectionValidationError::Other(_) => None, } } @@ -568,6 +573,7 @@ impl ConnectionValidationError { ConnectionValidationError::SqlServer(e) => e.hint(), ConnectionValidationError::Aws(e) => e.hint(), ConnectionValidationError::Gcp(e) => e.hint(), + ConnectionValidationError::AwsPrivatelinkServiceName(e) => Some(e.hint()), ConnectionValidationError::Other(_) => None, } } @@ -1014,6 +1020,51 @@ impl AlterCompatible for AwsPrivatelinkConnection { } } +/// A `SERVICE NAME` that cannot name an AWS VPC endpoint service. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct InvalidAwsPrivatelinkServiceName { + pub name: String, +} + +impl fmt::Display for InvalidAwsPrivatelinkServiceName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "invalid AWS PrivateLink service name {}", + self.name.quoted() + ) + } +} + +impl std::error::Error for InvalidAwsPrivatelinkServiceName {} + +impl InvalidAwsPrivatelinkServiceName { + /// Explains how to find the right value. + pub fn hint(&self) -> String { + "SERVICE NAME must name an AWS VPC endpoint service, for example \ + `com.amazonaws.vpce.us-east-1.vpce-svc-0e123abc123198abc`. Endpoint service names are \ + listed in the AWS console under VPC > Endpoint services." + .into() + } +} + +impl AwsPrivatelinkConnection { + /// Checks that `service_name` could name an AWS VPC endpoint service. + /// + /// Every endpoint service name starts with `com.amazonaws.`, whether the + /// service is customer-owned (`com.amazonaws.vpce..vpce-svc-`) + /// or AWS-managed (`com.amazonaws..`). Only the prefix is + /// checked, so a name AWS would accept is never rejected here. + pub fn check_service_name(service_name: &str) -> Result<(), InvalidAwsPrivatelinkServiceName> { + if service_name.starts_with("com.amazonaws.") { + return Ok(()); + } + Err(InvalidAwsPrivatelinkServiceName { + name: service_name.to_string(), + }) + } +} + #[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub struct KafkaTlsConfig { pub identity: Option, @@ -3213,12 +3264,16 @@ impl AwsPrivatelinkConnection { &self, id: CatalogItemId, storage_configuration: &StorageConfiguration, - ) -> Result<(), anyhow::Error> { + ) -> Result<(), ConnectionValidationError> { + // An endpoint for an unusable service name reports a misleading + // condition (missing availability zones), so check the name first. + Self::check_service_name(&self.service_name)?; + let Some(ref cloud_resource_reader) = storage_configuration .connection_context .cloud_resource_reader else { - return Err(anyhow!("AWS PrivateLink connections are unsupported")); + return Err(anyhow!("AWS PrivateLink connections are unsupported").into()); }; // No need to optionally run this in a task, as we are just validating from envd. @@ -3231,8 +3286,8 @@ impl AwsPrivatelinkConnection { match availability { Some(condition) if condition.status == "True" => Ok(()), - Some(condition) => Err(anyhow!("{}", condition.message)), - None => Err(anyhow!("Endpoint availability is unknown")), + Some(condition) => Err(anyhow!("{}", condition.message).into()), + None => Err(anyhow!("Endpoint availability is unknown").into()), } } @@ -3240,3 +3295,38 @@ impl AwsPrivatelinkConnection { false } } + +#[cfg(test)] +mod tests { + use super::*; + + #[mz_ore::test] + fn test_check_service_name() { + // Customer-owned and AWS-managed endpoint services are both accepted, + // as is anything else that could be an endpoint service name. + for name in [ + "com.amazonaws.vpce.us-east-1.vpce-svc-0e123abc123198abc", + "com.amazonaws.vpce.test.vpce-svc-e2e-test", + "com.amazonaws.us-east-1.s3", + "com.amazonaws.anything", + ] { + assert_eq!( + AwsPrivatelinkConnection::check_service_name(name), + Ok(()), + "expected {name} to be accepted" + ); + } + + for name in [ + "", + "com.amazonaws", + "vpce-svc-0e123abc123198abc", + "my-db-lb-0123456789abcdef.elb.eu-central-1.amazonaws.com", + "db.internal.example.org", + ] { + let err = AwsPrivatelinkConnection::check_service_name(name) + .expect_err("service name should be rejected"); + assert_eq!(err.name, name); + } + } +} diff --git a/test/testdrive/connection-create-drop.td b/test/testdrive/connection-create-drop.td index 0cbe43171f76d..3a2f7dbffe97f 100644 --- a/test/testdrive/connection-create-drop.td +++ b/test/testdrive/connection-create-drop.td @@ -532,6 +532,9 @@ contains: BROKERS must contain at least one static broker address > DROP CONNECTION kafka_matching_ok +! ALTER CONNECTION pl_for_matching SET (SERVICE NAME 'my-db-lb-0123456789abcdef.elb.eu-central-1.amazonaws.com') +contains: invalid AWS PrivateLink service name "my-db-lb-0123456789abcdef.elb.eu-central-1.amazonaws.com" + $ postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr} ALTER SYSTEM RESET enable_kafka_broker_matching_rules; ALTER SYSTEM RESET max_aws_privatelink_connections; @@ -549,3 +552,17 @@ contains: unknown catalog item 'foo' AVAILABILITY ZONES ('use1-az1', 'use1-az2') ) contains: AWS PrivateLink + +# SERVICE NAME must name a VPC endpoint service, not the DNS name of the target. +! CREATE CONNECTION privatelink_elb_hostname TO AWS PRIVATELINK ( + SERVICE NAME 'my-db-lb-0123456789abcdef.elb.eu-central-1.amazonaws.com', + AVAILABILITY ZONES () + ) +contains: invalid AWS PrivateLink service name "my-db-lb-0123456789abcdef.elb.eu-central-1.amazonaws.com" +hint: SERVICE NAME must name an AWS VPC endpoint service + +! CREATE CONNECTION privatelink_not_a_service_name TO AWS PRIVATELINK ( + SERVICE NAME 'vpce-svc-0e123abc123198abc', + AVAILABILITY ZONES () + ) +contains: invalid AWS PrivateLink service name "vpce-svc-0e123abc123198abc" From 3935183473e90b21aecaf13c36ad19c7576cb199 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Wed, 19 Aug 2026 20:35:05 +0200 Subject: [PATCH 04/10] tests: Unflake test_termination_races (#38357) Closes: [SQL-643](https://linear.app/materializeinc/issue/SQL-643) Related to https://github.com/MaterializeInc/materialize/pull/37988 --- src/environmentd/tests/server.rs | 54 ++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/src/environmentd/tests/server.rs b/src/environmentd/tests/server.rs index 4c1c8bb2d1ecc..14e3d0eac3c6d 100644 --- a/src/environmentd/tests/server.rs +++ b/src/environmentd/tests/server.rs @@ -1835,27 +1835,41 @@ async fn test_termination_races() { .unwrap(); let adapter_client = server.inner.adapter_client(); - let conn_id = adapter_client.new_conn_id().unwrap(); - let session = adapter_client.new_session( - SessionConfig { - conn_id, - uuid: Uuid::new_v4(), - user: "materialize".to_string(), - client_ip: None, - external_metadata_rx: None, - helm_chart_version: None, - authenticator_kind: AuthenticatorKind::None, - groups: None, - }, - Authenticated, - ); - // Box the future so that dropping it below drops the future itself, not - // just a reference to it. - let mut fut = Box::pin(adapter_client.startup(session)); - // The first poll sends the Startup command to the Coordinator and installs - // the cleanup guard around the response channel. - assert!(futures::poll!(&mut fut).is_pending()); + // The cleanup guard only runs on a response the startup future never observed, so + // the future has to be left pending on its first poll. That poll both sends the + // Startup command and polls the response channel, and the Coordinator runs on its + // own thread, so it can answer in between and resolve the future right away. Retry + // until the poll lands while the response is still in flight. A lost attempt + // leaves nothing behind, a failed startup registers no state in the Coordinator. + let mut fut = None; + for _ in 0..100 { + let conn_id = adapter_client.new_conn_id().unwrap(); + let session = adapter_client.new_session( + SessionConfig { + conn_id, + uuid: Uuid::new_v4(), + user: "materialize".to_string(), + client_ip: None, + external_metadata_rx: None, + helm_chart_version: None, + authenticator_kind: AuthenticatorKind::None, + groups: None, + }, + Authenticated, + ); + // Box the future so that dropping it below drops the future itself, not + // just a reference to it. + let mut candidate = Box::pin(adapter_client.startup(session)); + // The first poll sends the Startup command to the Coordinator and installs + // the cleanup guard around the response channel. + if futures::poll!(&mut candidate).is_pending() { + fut = Some(candidate); + break; + } + } + let fut = fut.expect("startup always answered before its first poll finished"); + // Wait for the Coordinator to process the Startup command and place the // startup error in the response channel. Commands on the client channel // are processed in order, so a completed round trip implies the Startup From b0d4c751f694f14f7456bc2efe16d657a446e3cb Mon Sep 17 00:00:00 2001 From: mtabebe Date: Wed, 19 Aug 2026 15:30:12 -0400 Subject: [PATCH 05/10] environmentd: bound HTTP and SUBSCRIBE output buffering to prevent OOM (SQL-428, SQL-423) (#37905) Problem: A single query can drive environmentd to OOM, even when the client read as fast as it could, because two paths buffered unbounded output in the shared process. - **HTTP `SELECT`.** The peek path already streams results out of the replica in bounded batches, but the HTTP SQL layer re-buffered every batch into a `serde_json::Value` tree roughly 10-15x the wire size. A large `SELECT` over `/api/ws` or `/api/sql` grew memory without limit. The `/api/sql` size guard was a stale `TODO` and was never enforced. - **SUBSCRIBE / COPY-TO.** The replica -> controller -> coordinator response path is unbounded at every hop, and the subscribe producer runs on the non-blockable coordinator main loop, so it cannot apply backpressure to a slow client. One slow or stopped SUBSCRIBE consumer could make the coordinator buffer the entire subscribe output in memory. Solution: **HTTP SQL layer ([SQL-428](https://linear.app/materializeinc/issue/SQL-428)).** - WebSocket `/api/ws` streams rows straight through. `SELECT` now flows through a new `StatementResult::Rows` variant whose sender emits `Rows(desc)` plus per-row `Row` messages, flushing between stash batches so a slow client applies backpressure and only one batch is resident. `max_result_size` is enforced incrementally. Large results over WebSocket now get the same bounded memory profile as pgwire `SELECT`. - Plain JSON `/api/sql` is a single buffered document. Rows are accumulated as pre-serialized `RawValue`s instead of a `Value` tree, and `max_result_size` is enforced on the actual serialized byte count. An oversized result fails fast with `ResultSize`. - MCP and prometheus consumers of `SqlResult::Rows` parse each `RawValue` on demand. **SUBSCRIBE / COPY-TO buffer ([SQL-423](https://linear.app/materializeinc/issue/SQL-423)).** Bound the terminal buffer and retire the sink when it overflows. `ActiveSubscribe` carries a shared byte counter and a per-sink budget. Each send adds the batch's byte size and a receiver wrapper subtracts it as the client writer drains, so the counter tracks the currently buffered depth. After each response the coordinator retires the subscribe if the depth exceeds the budget, surfacing `AdapterError::SubscribeFellBehind` with a hint to use a non-buffering client or `COPY (SUBSCRIBE ...) TO STDOUT`. The budget is a new dyncfg `subscribe_max_buffered_bytes` (default 128 MiB). `COPY (SUBSCRIBE ...) TO STDOUT` shares the channel and is covered too. Testing: New integration tests in `src/environmentd/tests/server.rs`: - `test_ws_select_streams_rows`: a `SELECT` over `/api/ws` emits one `Row` message per row before `CommandComplete`, confirming the non-buffering path. - `test_http_sql_result_size_limit`: with `max_result_size` lowered to 1 MB, an oversized `/api/sql` result fails with a result-size error while a small one succeeds. - `test_subscribe_buffer_bound`: with `subscribe_max_buffered_bytes` set to 1 KiB, a COPY-wrapped SUBSCRIBE whose client never reads surfaces the "fell behind" error instead of growing without bound. --- Cargo.lock | 1 + Cargo.toml | 2 +- misc/python/materialize/mzcompose/__init__.py | 1 + .../materialize/parallel_workload/action.py | 1 + src/adapter-types/src/dyncfgs.rs | 20 ++ src/adapter/Cargo.toml | 1 + src/adapter/src/active_compute_sink.rs | 158 ++++++++- src/adapter/src/command.rs | 1 + src/adapter/src/coord/message_handler.rs | 31 +- src/adapter/src/coord/read_then_write.rs | 10 + .../src/coord/sequencer/inner/subscribe.rs | 28 +- src/adapter/src/error.rs | 31 ++ src/adapter/src/session.rs | 12 +- src/environmentd/src/http/mcp.rs | 39 ++- src/environmentd/src/http/sql.rs | 330 ++++++++++++++---- src/environmentd/tests/server.rs | 293 ++++++++++++++++ src/environmentd/tests/testdata/http/ws | 2 +- .../tests/testdata/mcp/agent_query_tool | 4 +- src/environmentd/tests/testdata/mcp/developer | 8 +- .../tests/testdata/mcp/developer_query_tool | 14 +- .../testdata/mcp/developer_response_limit | 4 +- src/pgwire/src/protocol.rs | 5 +- 22 files changed, 880 insertions(+), 116 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bdbd43576308d..3e1b0dbcd3a39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6175,6 +6175,7 @@ dependencies = [ "timely", "tokio", "tokio-postgres", + "tokio-stream", "tracing", "tracing-core", "tracing-opentelemetry", diff --git a/Cargo.toml b/Cargo.toml index 2752a257f242e..c04ed90d17184 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -494,7 +494,7 @@ sentry-tracing = "0.49.1" serde = { version = "1.0.219", features = ["derive"] } serde-aux = "4.7.0" serde_bytes = "0.11.19" -serde_json = "1.0.150" +serde_json = { version = "1.0.150", features = ["raw_value"] } serde_plain = "1.0.2" serde_repr = "0.1.20" serde_urlencoded = "0.7.1" diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py index 6eea32f67d41a..d984ad2ef86d4 100644 --- a/misc/python/materialize/mzcompose/__init__.py +++ b/misc/python/materialize/mzcompose/__init__.py @@ -781,6 +781,7 @@ def get_default_system_parameters( "user_id_pool_batch_size", "webhook_max_request_size_bytes", "webhook_validation_memory_budget_bytes", + "subscribe_max_buffered_bytes", "cluster_controller_tick_interval", "default_cluster_reconfiguration_timeout", "read_then_write_max_dependencies", diff --git a/misc/python/materialize/parallel_workload/action.py b/misc/python/materialize/parallel_workload/action.py index 5c5957fbd3777..3963129aba924 100644 --- a/misc/python/materialize/parallel_workload/action.py +++ b/misc/python/materialize/parallel_workload/action.py @@ -3132,6 +3132,7 @@ def __init__( "compute_server_maintenance_interval", "compute_dataflow_max_inflight_bytes", "compute_dataflow_max_inflight_bytes_cc", + "subscribe_max_buffered_bytes", "compute_flat_map_fuel", "consolidating_vec_growth_dampener", "compute_hydration_concurrency", diff --git a/src/adapter-types/src/dyncfgs.rs b/src/adapter-types/src/dyncfgs.rs index 7b0da8964ee6e..59798c1d15000 100644 --- a/src/adapter-types/src/dyncfgs.rs +++ b/src/adapter-types/src/dyncfgs.rs @@ -297,6 +297,25 @@ pub const WEBHOOK_VALIDATION_MEMORY_BUDGET_BYTES: Config = Config::new( "The maximum bytes of temporary storage a webhook CHECK expression may allocate while validating one request.", ); +/// Budget for the backlog a `SUBSCRIBE` (or `COPY (SUBSCRIBE ...) TO STDOUT`) +/// may accumulate in environmentd while waiting for a slow client to read. +/// +/// The subscribe producer runs on the non-blockable coordinator loop, so it +/// cannot apply backpressure to a slow client. Instead the coordinator retires +/// the subscribe once its buffered backlog exceeds this budget, bounding the +/// memory a slow client can make the shared process hold. +/// +/// The backlog excludes the message the client is currently draining, so this +/// bounds the accumulation of messages, not the size of any single one. +/// `max_result_size` is what bounds an individual message. A client that keeps +/// up holds at most one message at a time, so it stays at a zero backlog and a +/// large snapshot batch is delivered rather than retired. +pub const SUBSCRIBE_MAX_BUFFERED_BYTES: Config = Config::new( + "subscribe_max_buffered_bytes", + 128 * 1024 * 1024, + "Maximum bytes a SUBSCRIBE may buffer in environmentd for a slow client before it is retired with an error.", +); + /// Number of user IDs to pre-allocate in a batch. Pre-allocating IDs avoids /// a persist write + oracle call per DDL statement. pub const USER_ID_POOL_BATCH_SIZE: Config = Config::new( @@ -464,6 +483,7 @@ pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet { .add(&MCP_REQUEST_TIMEOUT) .add(&WEBHOOK_MAX_REQUEST_SIZE_BYTES) .add(&WEBHOOK_VALIDATION_MEMORY_BUDGET_BYTES) + .add(&SUBSCRIBE_MAX_BUFFERED_BYTES) .add(&USER_ID_POOL_BATCH_SIZE) .add(&GROUP_COMMIT_MAX_ATTEMPTS) .add(&CONSOLE_OIDC_CLIENT_ID) diff --git a/src/adapter/Cargo.toml b/src/adapter/Cargo.toml index ffa3da657fe53..cccc24a57cfdb 100644 --- a/src/adapter/Cargo.toml +++ b/src/adapter/Cargo.toml @@ -93,6 +93,7 @@ static_assertions.workspace = true timely.workspace = true tokio.workspace = true tokio-postgres.workspace = true +tokio-stream.workspace = true tracing.workspace = true tracing-core.workspace = true tracing-opentelemetry.workspace = true diff --git a/src/adapter/src/active_compute_sink.rs b/src/adapter/src/active_compute_sink.rs index 4520868c4617e..358c76ee0f3f3 100644 --- a/src/adapter/src/active_compute_sink.rs +++ b/src/adapter/src/active_compute_sink.rs @@ -10,8 +10,9 @@ //! Coordinator bookkeeping for active compute sinks. use std::cmp::Ordering; -use std::collections::BTreeSet; +use std::collections::{BTreeSet, VecDeque}; use std::num::NonZeroUsize; +use std::sync::{Arc, Mutex}; use mz_adapter_types::connection::ConnectionId; use mz_compute_client::protocol::response::SubscribeBatch; @@ -88,6 +89,62 @@ pub enum ActiveComputeSinkRetireReason { /// The compute sink was forcibly terminated because an object it depended on /// was dropped. DependencyDropped(DroppedDependency), + /// The compute sink was retired because its coordinator-side buffer exceeded + /// its budget while the client was not reading fast enough. Carries the + /// buffered and budget byte counts for the terminal error. + BufferExceeded { + buffered_bytes: usize, + max_buffered_bytes: usize, + }, +} + +/// Overhead charged to every queued message on top of its payload. +/// +/// A frontier-only advance carries no rows, so its payload is zero bytes. It +/// still costs real memory: a node in the unbounded channel and an entry in +/// `footprints`. Charging payload alone would leave the backlog flat while +/// those two grow without bound, so the retire check would never trip on a +/// stalled client that only receives progress messages. The fixed charge makes +/// the budget bound message *count* as well as payload bytes. +const SUBSCRIBE_MESSAGE_OVERHEAD_BYTES: usize = 1024; + +/// Footprints of the subscribe messages queued to the client but not yet +/// drained, oldest first. The producer pushes one per message sent, the +/// client-writer task pops as it drains. FIFO delivery keeps the queue aligned +/// with the channel. +/// +/// The message at the front is the one the client is currently draining. It is +/// always tolerated, however large, so a client working through one big batch is +/// not retired. Only what is queued behind it counts as backlog. +#[derive(Debug, Default)] +pub struct SubscribeBacklogAccounting { + /// Per-message footprints, in send order. + footprints: VecDeque, + /// Sum of `footprints`. + total: usize, +} + +impl SubscribeBacklogAccounting { + /// Records a queued message of the given footprint. + pub fn push(&mut self, footprint: usize) { + self.footprints.push_back(footprint); + self.total = self.total.saturating_add(footprint); + } + + /// Records the oldest queued message being drained by the client writer. + pub fn pop(&mut self) { + if let Some(footprint) = self.footprints.pop_front() { + self.total = self.total.saturating_sub(footprint); + } + } + + /// Bytes queued behind the message the client is currently draining. That + /// message is tolerated whatever its size, so only this counts against the + /// budget. + pub fn backlog_size(&self) -> usize { + self.total + .saturating_sub(self.footprints.front().copied().unwrap_or(0)) + } } /// A description of an active subscribe from coord's perspective @@ -105,6 +162,17 @@ pub struct ActiveSubscribe { // The responses have the form `PeekResponseUnary` but should perhaps // become `SubscribeResponse`. pub channel: mpsc::UnboundedSender, + /// Footprints of the messages queued in `channel` but not yet drained by the + /// client writer. Shared with the receiver side, which pops as it drains. + /// + /// The producer runs on the non-blockable coordinator loop and cannot block + /// on a slow client, so instead of applying backpressure the coordinator + /// watches `backlog_size` against `max_buffered_bytes` and retires the + /// subscribe once the backlog exceeds it. + pub backlog_accounting: Arc>, + /// Budget for the buffered backlog. A snapshot of `subscribe_max_buffered_bytes` + /// taken when the subscribe was created. + pub max_buffered_bytes: usize, /// Whether progress information should be emitted. pub emit_progress: bool, /// The logical timestamp at which the subscribe began execution. @@ -154,8 +222,9 @@ impl ActiveSubscribe { } } + let bytes = row_buf.byte_len(); let row_iter = Box::new(row_buf.into_row_iter()); - self.send(PeekResponseUnary::Rows(row_iter)); + self.send(PeekResponseUnary::Rows(row_iter), bytes); } } @@ -178,7 +247,7 @@ impl ActiveSubscribe { mz_ore::iter::consolidate_update_iter(merged) } Err(s) => { - self.send(PeekResponseUnary::Error(s)); + self.send(PeekResponseUnary::Error(s), 0); return true; } }; @@ -376,8 +445,9 @@ impl ActiveSubscribe { }; let rows = output_builder.build(); + let bytes = rows.byte_len(); let rows = Box::new(rows.into_row_iter()); - self.send(PeekResponseUnary::Rows(rows)); + self.send(PeekResponseUnary::Rows(rows), bytes); // Emit progress message if requested. Don't emit progress for the first // batch if the upper is exactly `as_of` (we're guaranteed it is not @@ -402,15 +472,34 @@ impl ActiveSubscribe { ActiveComputeSinkRetireReason::DependencyDropped(d) => { PeekResponseUnary::DependencyDropped(d) } + ActiveComputeSinkRetireReason::BufferExceeded { + buffered_bytes, + max_buffered_bytes, + } => PeekResponseUnary::Error( + AdapterError::SubscribeFellBehind { + buffered_bytes, + max_buffered_bytes, + } + .to_string(), + ), }; - self.send(message); + self.send(message, 0); } /// Sends a message to the client if the subscribe has not already completed /// and if the client has not already gone away. - fn send(&self, response: PeekResponseUnary) { - // TODO(benesch): the lack of backpressure here can result in - // unbounded memory usage. + /// + /// `bytes` is the message's payload size. Its footprint (payload plus a fixed + /// per-message overhead) is recorded in `backlog_accounting` here and + /// released by the receiver side when the message is drained. Overflow of + /// the budget is detected by the coordinator after `process_response` + /// returns, not here, because this method cannot retire the sink. + fn send(&self, response: PeekResponseUnary, bytes: usize) { + let footprint = bytes.saturating_add(SUBSCRIBE_MESSAGE_OVERHEAD_BYTES); + self.backlog_accounting + .lock() + .expect("subscribe backlog accounting poisoned") + .push(footprint); let _ = self.channel.send(response); } } @@ -456,6 +545,13 @@ impl ActiveCopyTo { ActiveComputeSinkRetireReason::DependencyDropped(dep) => { Err(dep.to_concurrent_dependency_drop()) } + ActiveComputeSinkRetireReason::BufferExceeded { + buffered_bytes, + max_buffered_bytes, + } => Err(AdapterError::SubscribeFellBehind { + buffered_bytes, + max_buffered_bytes, + }), }; let _ = self.tx.send(message); } @@ -473,3 +569,49 @@ pub(crate) struct ActiveCopyFrom { /// Context of the SQL session that ran the statement. pub ctx: ExecuteContext, } + +#[cfg(test)] +mod tests { + use crate::active_compute_sink::SubscribeBacklogAccounting; + + /// The backlog excludes the message being drained, and zero-payload + /// messages (footprint = overhead only) still accumulate against it. + #[mz_ore::test] + fn test_subscribe_backlog_accounting() { + let mut acc = SubscribeBacklogAccounting::default(); + assert_eq!(acc.backlog_size(), 0); + + // A single large message is fully tolerated: nothing is queued behind it. + acc.push(10_000); + assert_eq!(acc.backlog_size(), 0); + + // Near-empty messages (only per-message overhead) still build backlog, so + // a flood of frontier-only advances cannot grow without bound. + acc.push(1_024); + acc.push(1_024); + assert_eq!(acc.backlog_size(), 2_048); + + // Draining the oldest message advances the tolerated front. + acc.pop(); + assert_eq!(acc.backlog_size(), 1_024); + + acc.pop(); + acc.pop(); + assert_eq!(acc.backlog_size(), 0); + } + + /// A client that drains each message before the next is sent never + /// accumulates backlog, however many messages flow and however large they + /// are. This is the property that keeps a well-behaved subscribe from ever + /// being retired, so it is asserted after every step rather than at the end. + #[mz_ore::test] + fn test_subscribe_backlog_keeping_up_client() { + let mut acc = SubscribeBacklogAccounting::default(); + for i in 0..1_000 { + acc.push(1_024 + i * 4_096); + assert_eq!(acc.backlog_size(), 0, "message {i} built backlog"); + acc.pop(); + assert_eq!(acc.backlog_size(), 0, "message {i} left backlog behind"); + } + } +} diff --git a/src/adapter/src/command.rs b/src/adapter/src/command.rs index 4c49fbf82bfc2..d343becaf3010 100644 --- a/src/adapter/src/command.rs +++ b/src/adapter/src/command.rs @@ -781,6 +781,7 @@ pub enum ExecuteResponse { /// Updates to the requested source or view will be streamed to the /// contained receiver. Subscribing { + #[derivative(Debug = "ignore")] rx: RowBatchStream, ctx_extra: ExecuteContextGuard, instance_id: ComputeInstanceId, diff --git a/src/adapter/src/coord/message_handler.rs b/src/adapter/src/coord/message_handler.rs index 8f80962510230..a8bc6342976c7 100644 --- a/src/adapter/src/coord/message_handler.rs +++ b/src/adapter/src/coord/message_handler.rs @@ -701,10 +701,37 @@ impl Coordinator { self.active_compute_sinks.get_mut(&sink_id) { let finished = active_subscribe.process_response(response); - if finished { + // Read the backlog into a `Copy` local so the mutable borrow + // of `active_subscribe` (and the accounting lock) ends before + // we call `self.retire_compute_sinks`. The producer runs on + // this loop, so it cannot block on a slow client. Instead we + // bound the backlog here and retire the subscribe once it + // exceeds the budget. + // + // The backlog excludes the message the client is currently + // draining, so a client working through a single large batch + // (e.g. the initial snapshot) is never retired for it. + let buffered_bytes = active_subscribe + .backlog_accounting + .lock() + .expect("subscribe backlog accounting poisoned") + .backlog_size(); + let max_buffered_bytes = active_subscribe.max_buffered_bytes; + + let reason = if finished { + Some(ActiveComputeSinkRetireReason::Finished) + } else if buffered_bytes > max_buffered_bytes { + Some(ActiveComputeSinkRetireReason::BufferExceeded { + buffered_bytes, + max_buffered_bytes, + }) + } else { + None + }; + if let Some(reason) = reason { let retire_notify = self .retire_compute_sinks(btreemap! { - sink_id => ActiveComputeSinkRetireReason::Finished, + sink_id => reason, }) .await; // `retire_compute_sinks` waits before sending the terminal diff --git a/src/adapter/src/coord/read_then_write.rs b/src/adapter/src/coord/read_then_write.rs index 747734d40b62f..d4f05b0a2b954 100644 --- a/src/adapter/src/coord/read_then_write.rs +++ b/src/adapter/src/coord/read_then_write.rs @@ -99,6 +99,16 @@ impl Coordinator { conn_id: conn_id.clone(), session_uuid, channel: tx, + backlog_accounting: std::sync::Arc::new(std::sync::Mutex::new( + crate::active_compute_sink::SubscribeBacklogAccounting::default(), + )), + // This internal subscribe is drained by the coordinator for OCC + // read-then-write, not by a slow external client, so the slow-client + // backlog budget must not apply. A large read-then-write read set + // (e.g. an UPDATE that rewrites every row of a big table) + // legitimately exceeds the budget, so bounding it here would + // spuriously retire the statement with `SubscribeFellBehind`. + max_buffered_bytes: usize::MAX, emit_progress: true, // We need progress updates for OCC as_of, arity, diff --git a/src/adapter/src/coord/sequencer/inner/subscribe.rs b/src/adapter/src/coord/sequencer/inner/subscribe.rs index 331398a83013e..4e55f9b3347a2 100644 --- a/src/adapter/src/coord/sequencer/inner/subscribe.rs +++ b/src/adapter/src/coord/sequencer/inner/subscribe.rs @@ -7,8 +7,10 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use futures::StreamExt; use maplit::btreemap; use mz_adapter_types::connection::ConnectionId; +use mz_adapter_types::dyncfgs::SUBSCRIBE_MAX_BUFFERED_BYTES; use mz_cluster_client::ReplicaId; use mz_compute_types::ComputeInstanceId; use mz_compute_types::dataflows::DataflowDescription; @@ -22,14 +24,17 @@ use mz_repr::optimize::{OptimizerFeatures, OverrideFrom}; use mz_sql::plan::{self, QueryWhen, SubscribeFrom}; use mz_sql::session::metadata::SessionMetadata; use std::collections::BTreeSet; +use std::sync::{Arc, Mutex}; use timely::progress::Antichain; use tokio::sync::mpsc; +use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{Instrument, Span}; use uuid::Uuid; -use crate::active_compute_sink::{ActiveComputeSink, ActiveSubscribe}; +use crate::active_compute_sink::{ActiveComputeSink, ActiveSubscribe, SubscribeBacklogAccounting}; use crate::command::ExecuteResponse; use crate::coord::appends::BuiltinTableAppendNotify; +use crate::coord::peek::PeekResponseUnary; use crate::coord::sequencer::inner::{return_if_err, spawn_linearized_read_ts}; use crate::coord::sequencer::{check_log_reads, emit_optimizer_notices}; use crate::coord::{ @@ -547,11 +552,16 @@ impl Coordinator { ) -> Result<(ExecuteResponse, BuiltinTableAppendNotify), AdapterError> { let sink_id = df_desc.sink_id(); - let (tx, rx) = mpsc::unbounded_channel(); + let (tx, rx) = mpsc::unbounded_channel::(); + let backlog_accounting = Arc::new(Mutex::new(SubscribeBacklogAccounting::default())); + let max_buffered_bytes = + SUBSCRIBE_MAX_BUFFERED_BYTES.get(self.catalog().system_config().dyncfgs()); let active_subscribe = ActiveSubscribe { conn_id: conn_id.clone(), session_uuid, channel: tx, + backlog_accounting: Arc::clone(&backlog_accounting), + max_buffered_bytes, emit_progress: plan.emit_progress, as_of: df_desc .as_of @@ -602,8 +612,20 @@ impl Coordinator { // Explicitly drop read holds, just to make it obvious what's happening. drop(read_holds); + // Wrap the receiver so draining a message releases its footprint from the + // shared accounting. FIFO delivery keeps the queue aligned with the + // channel, so popping the oldest footprint matches the message just + // drained. This keeps the accounting equal to the currently buffered + // depth, which the coordinator watches to bound this subscribe. + let rx = UnboundedReceiverStream::new(rx).map(move |response| { + backlog_accounting + .lock() + .expect("subscribe backlog accounting poisoned") + .pop(); + response + }); let resp = ExecuteResponse::Subscribing { - rx, + rx: Box::new(rx), ctx_extra: std::mem::take(ctx_extra), instance_id: cluster_id, }; diff --git a/src/adapter/src/error.rs b/src/adapter/src/error.rs index d84be0969198c..106fa7da919e9 100644 --- a/src/adapter/src/error.rs +++ b/src/adapter/src/error.rs @@ -21,6 +21,7 @@ use mz_compute_client::controller::error::InstanceMissing; use mz_compute_types::ComputeInstanceId; use mz_controller_types::ClusterId; use mz_expr::EvalError; +use mz_ore::cast::CastFrom; use mz_ore::error::ErrorExt; use mz_ore::stack::RecursionLimitError; use mz_ore::str::StrExt; @@ -180,6 +181,16 @@ pub enum AdapterError { }, /// Result size of a query is too large. ResultSize(String), + /// A `SUBSCRIBE` (or `COPY (SUBSCRIBE ...) TO STDOUT`) buffered more bytes in + /// environmentd than its budget allows because the client was not reading + /// fast enough. The subscribe is retired rather than let one slow client + /// grow the shared process's memory without bound. + SubscribeFellBehind { + /// Bytes buffered when the budget was exceeded. + buffered_bytes: usize, + /// The `subscribe_max_buffered_bytes` budget that was exceeded. + max_buffered_bytes: usize, + }, /// The specified feature is not permitted in safe mode. SafeModeViolation(String), /// The current transaction had the wrong set of write locks. @@ -743,6 +754,11 @@ impl AdapterError { ), AdapterError::Catalog(c) => c.hint(), AdapterError::Eval(e) => e.hint(), + AdapterError::SubscribeFellBehind { .. } => Some( + "The client is not reading results fast enough. Use a client that reads output \ + without buffering, or raise the subscribe_max_buffered_bytes system variable." + .to_string(), + ), AdapterError::AlterClusterUnmanagedWhileReconfiguring => Some( "Cancel the reconfiguration by altering the cluster back to its current \ configuration, or wait for it to settle, then convert." @@ -936,6 +952,7 @@ impl AdapterError { AdapterError::RelationOutsideTimeDomain { .. } => SqlState::INVALID_TRANSACTION_STATE, AdapterError::ResourceExhaustion { .. } => SqlState::INSUFFICIENT_RESOURCES, AdapterError::ResultSize(_) => SqlState::OUT_OF_MEMORY, + AdapterError::SubscribeFellBehind { .. } => SqlState::OUT_OF_MEMORY, AdapterError::SafeModeViolation(_) => SqlState::INTERNAL_ERROR, AdapterError::SubscribeOnlyTransaction => SqlState::INVALID_TRANSACTION_STATE, AdapterError::Optimizer(e) => match e { @@ -1345,6 +1362,20 @@ impl fmt::Display for AdapterError { ) } AdapterError::ResultSize(e) => write!(f, "{e}"), + AdapterError::SubscribeFellBehind { + buffered_bytes, + max_buffered_bytes, + } => { + use bytesize::ByteSize; + let buffered = u64::cast_from(*buffered_bytes); + let max = u64::cast_from(*max_buffered_bytes); + write!( + f, + "SUBSCRIBE fell behind: the client did not read results fast enough, so its backlog reached {}, exceeding the {} budget", + ByteSize::b(buffered), + ByteSize::b(max), + ) + } AdapterError::SafeModeViolation(feature) => { write!(f, "cannot create {} in safe mode", feature) } diff --git a/src/adapter/src/session.rs b/src/adapter/src/session.rs index cf6d6c18b8c92..905f543259f49 100644 --- a/src/adapter/src/session.rs +++ b/src/adapter/src/session.rs @@ -48,7 +48,7 @@ use mz_storage_client::client::TableData; use mz_storage_types::sources::Timeline; use qcell::{QCell, QCellOwner}; use timely::progress::Timestamp as _; -use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; +use tokio::sync::mpsc::{self, UnboundedSender}; use tokio::sync::watch; use uuid::Uuid; @@ -1069,8 +1069,14 @@ impl InProgressRows { } } -/// A channel of batched rows. -pub type RowBatchStream = UnboundedReceiver; +/// A stream of batched rows delivered to a client writer. +/// +/// This is a boxed stream, not a bare channel receiver, so the coordinator can +/// insert an adapter between itself and the client writer. The SUBSCRIBE path +/// wraps the receiver in a byte-accounting `map` whose closure type cannot be +/// named, so it has to be boxed. Consumers box the receiver into a +/// `RecordFirstRowStream` regardless, so this adds no allocation. +pub type RowBatchStream = Box + Unpin + Send + Sync>; /// Part of statement lifecycle. These are timestamps that come from the Adapter frontend /// (`mz-pgwire`) part of the lifecycle. diff --git a/src/environmentd/src/http/mcp.rs b/src/environmentd/src/http/mcp.rs index 4c8b69538c53b..e91207586626e 100644 --- a/src/environmentd/src/http/mcp.rs +++ b/src/environmentd/src/http/mcp.rs @@ -1031,7 +1031,7 @@ fn call_status(result: &Result) -> McpCallStatus { async fn execute_sql( client: &mut AuthedClient, query: &str, -) -> Result>, McpRequestError> { +) -> Result>, McpRequestError> { let mut response = SqlResponse::new(); execute_request( @@ -1054,7 +1054,7 @@ async fn execute_sql( /// row-returning statement is an error rather than a dropped result. fn select_single_rows( results: Vec, -) -> Result>, McpRequestError> { +) -> Result>, McpRequestError> { let mut rows = None; for result in results { match result { @@ -1085,9 +1085,11 @@ fn select_single_rows( /// endpoint handles `max_result_size` in sql.rs — fail cleanly rather /// than silently truncating. fn format_rows_response( - rows: Vec>, + rows: Vec>, max_size: usize, ) -> Result { + // Each row is already-serialized compact JSON. `RawValue` re-serializes + // verbatim, so the outer array is pretty-printed while rows stay compact. let text = serde_json::to_string_pretty(&rows).map_err(|e| McpRequestError::Internal(anyhow!(e)))?; @@ -1215,17 +1217,20 @@ async fn read_data_product( if lookup_rows.is_empty() { return Err(McpRequestError::DataProductNotFound(name.to_string())); } - let catalog_cluster: Option<&str> = lookup_rows + // Rows are stored as pre-serialized JSON arrays, so parse the single-cell + // lookup row to read `dp.cluster`. + let catalog_cluster: Option = lookup_rows .first() - .and_then(|row| row.first()) - .and_then(|v| v.as_str()); + .and_then(|row| serde_json::from_str::>(row.get()).ok()) + .and_then(|row| row.into_iter().next()) + .and_then(|v| v.as_str().map(|s| s.to_string())); // An override wins. Otherwise route to the catalog cluster when the role // can use it (non-null); when it is null the role lacks USAGE on the // object's cluster, so leave the cluster unset and read on the session's // default (serving) cluster. That still works: materialized views serve // from persist and views recompute, just without index benefit. - let target_cluster: Option<&str> = cluster_override.or(catalog_cluster); + let target_cluster: Option<&str> = cluster_override.or(catalog_cluster.as_deref()); // No row cap is applied here: the response is bounded by the size cap // enforced in format_rows_response (MCP_MAX_RESPONSE_SIZE), and by @@ -1529,10 +1534,18 @@ mod tests { use super::*; use crate::http::sql::{Description, SqlError}; + /// Serializes each row to a compact JSON array, matching how the HTTP SQL + /// layer stores rows. + fn raw_rows(rows: Vec>) -> Vec> { + rows.iter() + .map(|r| serde_json::value::to_raw_value(r).unwrap()) + .collect() + } + fn rows_result(rows: Vec>) -> SqlResult { SqlResult::Rows { tag: String::new(), - rows, + rows: raw_rows(rows), desc: Description { columns: vec![] }, notices: vec![], } @@ -1570,7 +1583,9 @@ mod tests { rows_result(rows.clone()), ok_result(), ]; - assert_eq!(select_single_rows(results).unwrap(), rows); + let got = select_single_rows(results).unwrap(); + let got: Vec<&str> = got.iter().map(|r| r.get()).collect(); + assert_eq!(got, vec!["[1]"]); } /// A response with no row-returning statement is an error. @@ -2248,7 +2263,7 @@ mod tests { #[mz_ore::test] fn test_format_rows_response_within_limit() { let rows = vec![vec![json!("a"), json!(1)], vec![json!("b"), json!(2)]]; - let result = format_rows_response(rows, 1_000_000).unwrap(); + let result = format_rows_response(raw_rows(rows), 1_000_000).unwrap(); let McpResult::ToolContent(content) = result else { panic!("Expected ToolContent"); }; @@ -2262,7 +2277,7 @@ mod tests { let rows: Vec> = (0..100) .map(|i| vec![json!(format!("row_{}", i)), json!(i)]) .collect(); - let err = format_rows_response(rows, 500).unwrap_err(); + let err = format_rows_response(raw_rows(rows), 500).unwrap_err(); let msg = err.to_string(); assert!( msg.contains("exceeds the 500 byte limit"), @@ -2277,7 +2292,7 @@ mod tests { #[mz_ore::test] fn test_format_rows_response_empty_rows() { let rows: Vec> = vec![]; - let result = format_rows_response(rows, 1000).unwrap(); + let result = format_rows_response(raw_rows(rows), 1000).unwrap(); let McpResult::ToolContent(content) = result else { panic!("Expected ToolContent"); }; diff --git a/src/environmentd/src/http/sql.rs b/src/environmentd/src/http/sql.rs index 360eeb5da1e62..d47c8db4e3561 100644 --- a/src/environmentd/src/http/sql.rs +++ b/src/environmentd/src/http/sql.rs @@ -51,7 +51,6 @@ use prometheus::core::{AtomicF64, GenericGaugeVec}; use serde::{Deserialize, Serialize}; use tokio::{select, time}; use tokio_postgres::error::SqlState; -use tokio_stream::wrappers::UnboundedReceiverStream; use tower_sessions::Session as TowerSession; use tracing::{debug, info}; use tungstenite::protocol::frame::coding::CloseCode; @@ -180,6 +179,12 @@ async fn execute_promsql_query( }); for row in rows { + // Rows are stored as pre-serialized JSON arrays. Parse each one back to + // `Value`s here. Promsql results are tiny, so the parse cost is + // negligible. + let row: Vec = + serde_json::from_str(row.get()).expect("row is a valid JSON array"); + // Non-value columns become Prometheus label values. A SQL `NULL` // arrives as JSON `null` and yields `None` from `as_str()`; fall back // to an empty label rather than panicking. The query author is @@ -188,7 +193,7 @@ async fn execute_promsql_query( let mut label_values = desc .columns .iter() - .zip_eq(row) + .zip_eq(&row) .filter(|(col, _)| col.name != query.value_column_name) .map(|(_, val)| val.as_str().unwrap_or("")) .collect::>(); @@ -196,7 +201,7 @@ async fn execute_promsql_query( let value = desc .columns .iter() - .zip_eq(row) + .zip_eq(&row) .find(|(col, _)| col.name == query.value_column_name) .map(|(_, val)| val.as_str().unwrap_or("0").parse::().unwrap_or(0.0)) .unwrap_or(0.0); @@ -560,6 +565,15 @@ impl SqlResponse { pub(in crate::http) enum StatementResult { SqlResult(SqlResult), + /// A peek (`SELECT`) result whose rows are streamed out of the peek response + /// stash. The stream is consumed lazily by the sender so a large result is + /// not buffered whole. Contrast `SqlResult::Rows`, which holds + /// already-collected rows and exists only for the buffered JSON transport. + Rows { + desc: RelationDesc, + rows_stream: RecordFirstRowStream, + max_result_size: usize, + }, Subscribe { desc: RelationDesc, tag: String, @@ -582,8 +596,15 @@ pub enum SqlResult { Rows { /// The command complete tag. tag: String, - /// The result rows. - rows: Vec>, + /// The result rows, each already serialized to a compact JSON array. + /// + /// We accumulate pre-serialized rows rather than a `serde_json::Value` + /// tree so the buffered footprint stays close to the wire size. A + /// `Value` tree is ~10-15x larger (each cell a heap `Value`, in a + /// per-row `Vec`, in the outer `Vec`), which is what let a large + /// `SELECT` over the JSON endpoint OOM the process. `RawValue` + /// re-serializes verbatim, so the wire format is unchanged. + rows: Vec>, /// Information about each column. desc: Description, // Any notices generated during execution of the query. @@ -610,9 +631,16 @@ pub enum SqlResult { } impl SqlResult { - /// Convert adapter Row results into the web row result format. Error if the row format does not - /// match the expected descriptor. - // TODO(aljoscha): Bail when max_result_size is exceeded. + /// Convert adapter Row results into the buffered web row result format. Error + /// if the row format does not match the expected descriptor, or if the + /// result exceeds `max_query_result_size`. + /// + /// This buffers the whole result, so it is used only by the JSON transport, + /// whose response is a single document. The WebSocket transport streams rows + /// through `StatementResult::Rows` and never calls this. The size guard + /// counts `Row::byte_len`, matching the WebSocket transport and pgwire, and + /// rows are stored as pre-serialized compact `RawValue`s to avoid the + /// amplified `Value`-tree buffering that could OOM. async fn rows( sender: &mut S, client: &mut SessionClient, @@ -623,11 +651,11 @@ impl SqlResult { where S: ResultSender, { - let mut rows: Vec> = vec![]; + let mut rows: Vec> = vec![]; let mut datum_vec = mz_repr::DatumVec::new(); let types = &desc.typ().column_types; - let mut query_result_size = 0; + let mut query_result_size: usize = 0; loop { let peek_response = tokio::select! { @@ -665,7 +693,10 @@ impl SqlResult { } while let Some(row) = sql_rows.next() { - query_result_size += row.byte_len(); + // Enforce `max_result_size` on `Row::byte_len`, the same quantity + // pgwire and the WebSocket transport use, so the cap means one + // thing across every transport. + query_result_size = query_result_size.saturating_add(row.byte_len()); if query_result_size > max_query_result_size { use bytesize::ByteSize; return Ok(SqlResult::err( @@ -676,18 +707,21 @@ impl SqlResult { )), )); } - let datums = datum_vec.borrow_with(row); - rows.push( - datums - .iter() - .enumerate() - .map(|(i, d)| { - TypedDatum::new(*d, &types[i]) - .json(&JsonNumberPolicy::ConvertNumberToString) - }) - .collect(), - ); + let json_row: Vec = datums + .iter() + .enumerate() + .map(|(i, d)| { + TypedDatum::new(*d, &types[i]) + .json(&JsonNumberPolicy::ConvertNumberToString) + }) + .collect(); + // Keep only the compact serialized JSON text. The transient + // `Value` tree above is dropped per row, so at most one row's tree + // is resident, avoiding the amplified buffering that could OOM. + let raw = serde_json::value::to_raw_value(&json_row) + .expect("row of JSON values always serializes"); + rows.push(raw); } } @@ -873,7 +907,7 @@ impl ResultSender for SqlResponse { // needs to be retired for statement logging purposes. async fn add_result( &mut self, - _client: &mut SessionClient, + client: &mut SessionClient, res: StatementResult, ) -> ( Result, Error>, @@ -886,6 +920,25 @@ impl ResultSender for SqlResponse { let res = if is_err { Err(()) } else { Ok(()) }; (res, None) } + StatementResult::Rows { + desc, + rows_stream, + max_result_size, + } => { + // The JSON transport is a single buffered document, so the rows + // must be collected before the response is serialized. + // `SqlResult::rows` bounds that buffer against `max_result_size`. + let res = match SqlResult::rows(self, client, rows_stream, max_result_size, &desc) + .await + { + Ok(res) => res, + Err(e) => return (Err(e), None), + }; + let is_err = matches!(res, SqlResult::Err { .. }); + self.results.push(res); + let res = if is_err { Err(()) } else { Ok(()) }; + (res, None) + } StatementResult::Subscribe { ctx_extra, .. } => { let message = "SUBSCRIBE only supported over websocket"; self.results.push(SqlResult::Err { @@ -936,6 +989,7 @@ impl ResultSender for WebSocket { StatementResult::SqlResult(SqlResult::Err { .. }) => (false, false), StatementResult::SqlResult(SqlResult::Ok { .. }) => (false, false), StatementResult::SqlResult(SqlResult::Rows { .. }) => (true, false), + StatementResult::Rows { .. } => (true, false), StatementResult::Subscribe { .. } => (true, true), }; if let Err(e) = send_ws_response( @@ -951,26 +1005,25 @@ impl ResultSender for WebSocket { } let (is_err, msgs, stmt_logging) = match res { - StatementResult::SqlResult(SqlResult::Rows { - tag, - rows, - desc, - notices, - }) => { - // Stream rows directly to avoid buffering all rows as - // WebSocketResponse, which inflates memory due to enum sizing. - if let Err(e) = send_ws_response(self, WebSocketResponse::Rows(desc)).await { - return (Err(e), None); - } - for row in rows { - if let Err(e) = send_ws_response(self, WebSocketResponse::Row(row)).await { - return (Err(e), None); - } - } - let mut msgs = vec![WebSocketResponse::CommandComplete(tag)]; - msgs.extend(notices.into_iter().map(WebSocketResponse::Notice)); - (false, msgs, None) + StatementResult::SqlResult(SqlResult::Rows { .. }) => { + // `SqlResult::Rows` holds already-collected rows for the buffered + // JSON transport only. The WebSocket transport streams peek + // results through `StatementResult::Rows` and never materializes + // a `SqlResult::Rows`. + unreachable!("WebSocket streams peek rows via StatementResult::Rows") } + StatementResult::Rows { + ref desc, + mut rows_stream, + max_result_size, + } => match stream_ws_peek_rows(self, client, desc, &mut rows_stream, max_result_size) + .await + { + Ok(result) => result, + // A write failure means the remote broke the connection, which we + // treat as a cancellation to match pgwire. + Err(e) => return (Err(e), None), + }, StatementResult::SqlResult(SqlResult::Ok { ok, parameters, @@ -1036,7 +1089,7 @@ impl ResultSender for WebSocket { rows_returned += rows.count(); while let Some(row) = rows.next() { - result_size += row.byte_len(); + result_size = result_size.saturating_add(row.byte_len()); let datums = datum_vec.borrow_with(row); let types = &desc.typ().column_types; if let Err(e) = send_ws_response( @@ -1162,6 +1215,161 @@ where } } +/// Streams a peek (`SELECT`) result to a WebSocket client one stash batch at a +/// time, flushing between batches so a slow client applies real backpressure +/// and only one batch is resident. This gives a large `SELECT` over WebSocket +/// the same bounded memory profile as pgwire `SELECT`, and mirrors the +/// `Subscribe` arm of `WebSocket::add_result`. +/// +/// On success returns the `(is_err, msgs, stmt_logging)` triple that +/// `add_result` folds into its response. An `Err` means a write to the socket +/// failed, so the server should disconnect. `add_result` turns that into a +/// cancellation to match pgwire. +/// +/// The `Rows` descriptor is sent lazily, right before the first batch of rows +/// or an empty successful result, but never before an error. So a query that +/// fails before producing any rows emits only an `Error`. +async fn stream_ws_peek_rows( + ws: &mut WebSocket, + client: &mut SessionClient, + desc: &RelationDesc, + rows_stream: &mut RecordFirstRowStream, + max_result_size: usize, +) -> Result< + ( + bool, + Vec, + Option<(StatementEndedExecutionReason, ExecuteContextGuard)>, + ), + Error, +> { + let mut datum_vec = mz_repr::DatumVec::new(); + let mut result_size: usize = 0; + let mut rows_returned: usize = 0; + let mut sent_rows_desc = false; + loop { + // Bind before matching so the `Option` (which has a + // significant `Drop`) is not a temporary living for the whole match. + let res = await_rows(ws, client, rows_stream.recv()).await?; + match res { + Some(PeekResponseUnary::Rows(mut rows)) => { + if let Err(err) = verify_datum_desc(desc, &mut rows) { + return Ok(ws_peek_result( + client, + true, + vec![WebSocketResponse::Error(err.into())], + )); + } + // The header waits until a batch has passed `verify_datum_desc`, + // so a query that fails before producing any rows emits only an + // `Error`. Sending it before the loop, as the `Subscribe` arm + // does, would put a `Rows` header in front of that `Error`. + if !sent_rows_desc { + send_ws_response(ws, WebSocketResponse::Rows(desc.into())).await?; + sent_rows_desc = true; + } + let types = &desc.typ().column_types; + while let Some(row) = rows.next() { + result_size = result_size.saturating_add(row.byte_len()); + if result_size > max_result_size { + use bytesize::ByteSize; + return Ok(ws_peek_result( + client, + true, + vec![WebSocketResponse::Error( + AdapterError::ResultSize(format!( + "result exceeds max size of {}", + ByteSize::b(u64::cast_from(max_result_size)) + )) + .into(), + )], + )); + } + let datums = datum_vec.borrow_with(row); + send_ws_response( + ws, + WebSocketResponse::Row( + datums + .iter() + .enumerate() + .map(|(i, d)| { + TypedDatum::new(*d, &types[i]) + .json(&JsonNumberPolicy::ConvertNumberToString) + }) + .collect(), + ), + ) + .await?; + rows_returned += 1; + } + } + Some(PeekResponseUnary::Error(error)) => { + return Ok(ws_peek_result( + client, + true, + vec![WebSocketResponse::Error( + Error::Unstructured(anyhow!(error)).into(), + )], + )); + } + Some(PeekResponseUnary::DependencyDropped(dep)) => { + return Ok(ws_peek_result( + client, + true, + vec![WebSocketResponse::Error( + dep.to_concurrent_dependency_drop().into(), + )], + )); + } + Some(PeekResponseUnary::Canceled) => { + return Ok(ws_peek_result( + client, + true, + vec![WebSocketResponse::Error(AdapterError::Canceled.into())], + )); + } + None => { + // An empty successful result still owes the client a `Rows` + // descriptor before `CommandComplete`. + if !sent_rows_desc { + send_ws_response(ws, WebSocketResponse::Rows(desc.into())).await?; + } + return Ok(ws_peek_result( + client, + false, + vec![WebSocketResponse::CommandComplete(format!( + "SELECT {rows_returned}" + ))], + )); + } + } + } +} + +/// Packages a terminal result of `stream_ws_peek_rows`, appending any notices +/// still buffered on the session after `msgs`. +/// +/// The streaming loop forwards notices live through `await_rows`' `recv_notice` +/// select, but the terminal `recv() -> None` can race a still-buffered notice +/// and end the loop first. Draining here restores the buffered path's guarantee +/// that all notices reach the client, after `CommandComplete` or `Error`. +fn ws_peek_result( + client: &mut SessionClient, + is_err: bool, + mut msgs: Vec, +) -> ( + bool, + Vec, + Option<(StatementEndedExecutionReason, ExecuteContextGuard)>, +) { + msgs.extend( + make_notices(client) + .into_iter() + .map(WebSocketResponse::Notice), + ); + (is_err, msgs, None) +} + async fn send_and_retire( res: StatementResult, client: &mut SessionClient, @@ -1614,7 +1822,7 @@ async fn execute_stmt( instance_id, strategy, } => { - let max_query_result_size = + let max_result_size = usize::cast_from(client.get_system_vars().await.max_result_size()); let rows_stream = RecordFirstRowStream::new( @@ -1625,33 +1833,25 @@ async fn execute_stmt( Some(strategy), ); - SqlResult::rows( - sender, - client, + StatementResult::Rows { + desc: desc.relation_desc.expect("RelationDesc must exist"), rows_stream, - max_query_result_size, - &desc.relation_desc.expect("RelationDesc must exist"), - ) - .await? - .into() + max_result_size, + } } ExecuteResponse::SendingRowsImmediate { rows } => { - let max_query_result_size = + let max_result_size = usize::cast_from(client.get_system_vars().await.max_result_size()); let rows = futures::stream::once(futures::future::ready(PeekResponseUnary::Rows(rows))); let rows_stream = RecordFirstRowStream::new(Box::new(rows), execute_started, client, None, None); - SqlResult::rows( - sender, - client, + StatementResult::Rows { + desc: desc.relation_desc.expect("RelationDesc must exist"), rows_stream, - max_query_result_size, - &desc.relation_desc.expect("RelationDesc must exist"), - ) - .await? - .into() + max_result_size, + } } ExecuteResponse::Subscribing { rx, @@ -1660,13 +1860,7 @@ async fn execute_stmt( } => StatementResult::Subscribe { tag: "SUBSCRIBE".into(), desc: desc.relation_desc.unwrap(), - rx: RecordFirstRowStream::new( - Box::new(UnboundedReceiverStream::new(rx)), - execute_started, - client, - Some(instance_id), - None, - ), + rx: RecordFirstRowStream::new(rx, execute_started, client, Some(instance_id), None), ctx_extra, }, res @ (ExecuteResponse::Fetch { .. } diff --git a/src/environmentd/tests/server.rs b/src/environmentd/tests/server.rs index 14e3d0eac3c6d..acfd4cb088da8 100644 --- a/src/environmentd/tests/server.rs +++ b/src/environmentd/tests/server.rs @@ -1297,6 +1297,299 @@ fn test_ws_notifies_for_bad_options() { }; } +/// A `SELECT` over `/api/ws` emits a `Rows` descriptor, then one `Row` message +/// per row, then `CommandComplete`. This pins the wire shape, which is the same +/// shape a buffered implementation would produce. What proves the rows are +/// emitted incrementally is `test_ws_select_streams_rows_then_size_error`. +#[mz_ore::test] +#[cfg_attr(miri, ignore)] // too slow +fn test_ws_select_row_message_shape() { + let server = test_util::TestHarness::default().start_blocking(); + let ws_url = server.ws_addr(); + let (mut ws, _resp) = tungstenite::connect(ws_url).unwrap(); + test_util::auth_with_ws(&mut ws, BTreeMap::default()).unwrap(); + + let query = "SELECT * FROM generate_series(1, 1000)"; + let json = serde_json::json!({ "query": query }); + ws.send(Message::Text(json.to_string().into())).unwrap(); + + let mut read_msg = || -> WebSocketResponse { + let msg = ws.read().unwrap(); + let msg = msg.into_text().expect("response should be text"); + serde_json::from_str(&msg).unwrap() + }; + + match read_msg() { + WebSocketResponse::CommandStarting(_) => {} + other => panic!("expected CommandStarting, got {other:?}"), + } + match read_msg() { + WebSocketResponse::Rows(desc) => assert_eq!(desc.columns.len(), 1), + other => panic!("expected Rows description, got {other:?}"), + } + + // Every row arrives as its own `Row` message before `CommandComplete`. + let mut rows_seen = 0; + loop { + match read_msg() { + WebSocketResponse::Row(_) => rows_seen += 1, + WebSocketResponse::Notice(_) => continue, + WebSocketResponse::CommandComplete(tag) => { + assert_eq!(tag, "SELECT 1000"); + break; + } + other => panic!("unexpected message before CommandComplete: {other:?}"), + } + } + assert_eq!(rows_seen, 1000); +} + +/// Regression test for SQL-428: a `SELECT` over `/api/ws` emits rows as it +/// streams them, so a result that outgrows `max_result_size` mid-flight arrives +/// as some `Row` messages followed by an `Error`. An implementation that +/// buffered the result would size it before emitting anything and send the +/// `Error` alone, so this is the assertion that pins streaming. +/// +/// NOTE: clients must therefore tolerate rows that precede an error, which is +/// the contract pgwire has always had. +#[mz_ore::test] +#[cfg_attr(miri, ignore)] // too slow +#[allow(clippy::disallowed_methods)] +fn test_ws_select_streams_rows_then_size_error() { + let server = test_util::TestHarness::default().start_blocking(); + + // `max_result_size` is a system parameter, so lower it over the internal + // (system) port. A normal session cannot. + let mut system = server.connect_internal(postgres::NoTls).unwrap(); + system + .batch_execute("ALTER SYSTEM SET max_result_size = '1MB'") + .unwrap(); + + let (mut ws, _resp) = tungstenite::connect(server.ws_addr()).unwrap(); + test_util::auth_with_ws(&mut ws, BTreeMap::default()).unwrap(); + + // Each row is ~10 KB, so the cap trips around row 100 of 500. + const ROWS: usize = 500; + let query = format!("SELECT repeat('x', 10000) FROM generate_series(1, {ROWS})"); + let json = serde_json::json!({ "query": query }); + ws.send(Message::Text(json.to_string().into())).unwrap(); + + let mut read_msg = || -> WebSocketResponse { + let msg = ws.read().unwrap(); + let msg = msg.into_text().expect("response should be text"); + serde_json::from_str(&msg).unwrap() + }; + + match read_msg() { + WebSocketResponse::CommandStarting(_) => {} + other => panic!("expected CommandStarting, got {other:?}"), + } + match read_msg() { + WebSocketResponse::Rows(desc) => assert_eq!(desc.columns.len(), 1), + other => panic!("expected Rows description, got {other:?}"), + } + + let mut rows_seen = 0; + let error = loop { + match read_msg() { + WebSocketResponse::Row(_) => rows_seen += 1, + WebSocketResponse::Notice(_) => continue, + WebSocketResponse::Error(err) => break err, + other => panic!("unexpected message before the size error: {other:?}"), + } + }; + + assert!( + error.message.contains("result exceeds max size"), + "expected a result-size error, got: {error:?}" + ); + assert!( + rows_seen > 0 && rows_seen < ROWS, + "expected a truncated stream of rows before the error, got {rows_seen}" + ); +} + +/// Regression test for SQL-428: the JSON `/api/sql` endpoint enforces +/// `max_result_size` on `Row::byte_len`, the same quantity the WebSocket +/// transport and pgwire use. A result that exceeds the cap fails cleanly with a +/// result-size error rather than being buffered whole, while a result under the +/// cap succeeds. +#[mz_ore::test] +#[cfg_attr(miri, ignore)] // too slow +#[allow(clippy::disallowed_methods)] +fn test_http_sql_result_size_limit() { + let server = test_util::TestHarness::default().start_blocking(); + + // Lower the cap to its minimum. `max_result_size` is a system parameter, so + // set it via the internal (system) port. A normal session cannot. + let mut system = server.connect_internal(postgres::NoTls).unwrap(); + system + .batch_execute("ALTER SYSTEM SET max_result_size = '1MB'") + .unwrap(); + + let http_url = Url::parse(&format!("http://{}/api/sql", server.http_local_addr())).unwrap(); + let post = |query: &str| -> serde_json::Value { + let json = serde_json::json!({ "query": query }); + let res = Client::new() + .post(http_url.clone()) + .json(&json) + .send() + .unwrap(); + res.json().unwrap() + }; + + // A result whose rows exceed the cap fails cleanly. + let body = post("SELECT * FROM generate_series(1, 500000)"); + let results = body.get("results").unwrap().as_array().unwrap(); + let err = results[0] + .get("error") + .and_then(|e| e.get("message")) + .and_then(|m| m.as_str()); + assert!( + err.map_or(false, |m| m.contains("result exceeds max size")), + "expected a result-size error, got: {body}" + ); + + // A result that fits under the cap succeeds. + let body = post("SELECT * FROM generate_series(1, 3)"); + let results = body.get("results").unwrap().as_array().unwrap(); + assert!( + results[0].get("error").is_none(), + "expected success, got: {body}" + ); + let rows = results[0].get("rows").unwrap().as_array().unwrap(); + assert_eq!(rows.len(), 3); +} + +/// Regression test for SQL-423: a SUBSCRIBE whose client stops reading is +/// retired once its coordinator-side buffer exceeds +/// `subscribe_max_buffered_bytes`, rather than growing environmentd's memory +/// without bound. The client sees a clean "fell behind" error. +#[mz_ore::test] +#[cfg_attr(miri, ignore)] // too slow +#[allow(clippy::disallowed_methods)] +fn test_subscribe_buffer_bound() { + let server = test_util::TestHarness::default() + .with_system_parameter_default( + "subscribe_max_buffered_bytes".to_string(), + "1024".to_string(), + ) + .start_blocking(); + + let mut writer = server.connect(postgres::NoTls).unwrap(); + writer.batch_execute("CREATE TABLE t (a bigint)").unwrap(); + + // Open a SUBSCRIBE over COPY but never read from it, so the coordinator-side + // buffer fills once the socket backs up. + let mut reader_client = server.connect(postgres::NoTls).unwrap(); + let mut copy = reader_client + .copy_out("COPY (SUBSCRIBE t) TO STDOUT") + .unwrap(); + + // Drive far more than the 1 KiB budget through the subscribe while the + // reader is not draining. + for _ in 0..200 { + writer + .batch_execute("INSERT INTO t SELECT generate_series(1, 1000)") + .unwrap(); + } + + // Draining now surfaces the terminal "fell behind" error rather than an + // unbounded backlog. The COPY read fails with the underlying DB error, whose + // message lives in the error's source chain (the top-level display is just + // "db error"). + let mut buf = Vec::new(); + let io_err = std::io::Read::read_to_end(&mut copy, &mut buf).unwrap_err(); + let mut chain = io_err.to_string(); + let mut source = std::error::Error::source(&io_err); + while let Some(err) = source { + chain.push_str(" | "); + chain.push_str(&err.to_string()); + source = err.source(); + } + assert!( + chain.contains("fell behind"), + "expected a fell-behind error, got: {chain}" + ); +} + +/// Regression test for SQL-423: a client that drains each batch as it arrives is +/// never retired, however many batches flow past the budget in total. A false +/// `SubscribeFellBehind` on a well-behaved client is the failure mode this +/// budget must not introduce. +/// +/// The budget here is 64 KiB and each batch costs at least the 1 KiB per-message +/// overhead, so a client that stopped draining would be retired within ~64 of +/// the 200 batches. +#[mz_ore::test] +#[cfg_attr(miri, ignore)] // too slow +#[allow(clippy::disallowed_methods)] +fn test_subscribe_keeping_up_client_not_retired() { + let server = test_util::TestHarness::default() + .with_system_parameter_default( + "subscribe_max_buffered_bytes".to_string(), + "65536".to_string(), + ) + .start_blocking(); + + let mut writer = server.connect(postgres::NoTls).unwrap(); + writer.batch_execute("CREATE TABLE t (a bigint)").unwrap(); + + let mut reader = server.connect(postgres::NoTls).unwrap(); + reader + .batch_execute("BEGIN; DECLARE c CURSOR FOR SUBSCRIBE t") + .unwrap(); + + // `FETCH` defaults to `WaitOnce`, so each round trip blocks until the batch + // for the insert it follows has arrived. That makes the drain deterministic + // rather than timing-dependent. + const BATCHES: usize = 200; + let mut rows_seen = 0; + for i in 0..BATCHES { + writer + .execute("INSERT INTO t VALUES ($1)", &[&i64::try_from(i).unwrap()]) + .unwrap(); + rows_seen += reader + .query("FETCH ALL c", &[]) + .expect("a client that keeps up must not be retired") + .len(); + } + assert_eq!(rows_seen, BATCHES); +} + +/// Regression test for SQL-423: a SUBSCRIBE whose snapshot is a single batch +/// larger than `subscribe_max_buffered_bytes` completes when the client reads it +/// promptly. The budget bounds accumulated backlog, not the size of any single +/// batch, and it is independent of `max_result_size`, which is what bounds an +/// individual batch. +#[mz_ore::test] +#[cfg_attr(miri, ignore)] // too slow +#[allow(clippy::disallowed_methods)] +fn test_subscribe_single_large_batch_not_retired() { + let server = test_util::TestHarness::default() + .with_system_parameter_default( + "subscribe_max_buffered_bytes".to_string(), + "1024".to_string(), + ) + .start_blocking(); + + let mut client = server.connect(postgres::NoTls).unwrap(); + + // Subscribe to a finite collection whose snapshot is a single batch far + // larger than the 1 KiB budget. The subscribe finishes once the snapshot is + // delivered, and a client reading it promptly (via `query`) keeps the + // backlog at that single batch, so it must stream to completion rather than + // be retired with a "fell behind" error. + let rows: usize = 10_000; + let returned = client + .query( + &format!("SUBSCRIBE TO (SELECT * FROM generate_series(1, {rows}))"), + &[], + ) + .expect("single large snapshot batch should stream, not be retired"); + assert_eq!(returned.len(), rows); +} + #[derive(Debug, Deserialize)] struct HttpResponse { results: Vec, diff --git a/src/environmentd/tests/testdata/http/ws b/src/environmentd/tests/testdata/http/ws index 17dbeff38719b..23e7a86a396e7 100644 --- a/src/environmentd/tests/testdata/http/ws +++ b/src/environmentd/tests/testdata/http/ws @@ -413,7 +413,7 @@ ws-text fixid=true {"query": "SELECT 1 / 0 FROM mz_sources LIMIT 1"} ---- {"type":"Notice","payload":{"message":"{\n \"plans\": {\n \"raw\": {\n \"text\": \"Finish limit=1 output=[#0]\\n Project (#15)\\n Map ((1 / 0))\\n Get mz_catalog.mz_sources\\n\\nTarget cluster: mz_catalog_server\\n\",\n \"json\": {\n \"Project\": {\n \"input\": {\n \"Map\": {\n \"input\": {\n \"Get\": {\n \"id\": {\n \"Global\": {\n \n }\n },\n \"typ\": {\n \"column_types\": [\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"Oid\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": {\n \"Array\": \"MzAclItem\"\n },\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n }\n ],\n \"keys\": [\n [\n 0\n ],\n [\n 1\n ]\n ]\n }\n }\n },\n \"scalars\": [\n {\n \"CallBinary\": {\n \"func\": {\n \"DivInt32\": null\n },\n \"expr1\": {\n \"Literal\": [\n {\n \"data\": [\n 45,\n 1\n ]\n },\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n },\n null\n ]\n },\n \"expr2\": {\n \"Literal\": [\n {\n \"data\": [\n 44\n ]\n },\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n },\n null\n ]\n },\n \"name\": null\n }\n }\n ]\n }\n },\n \"outputs\": [\n 15\n ]\n }\n }\n },\n \"optimized\": {\n \"global\": {\n \"text\": \":\\n Finish limit=1 output=[#0]\\n ArrangeBy keys=[[#0]]\\n ReadGlobalFromSameDataflow \\n\\n:\\n Project (#15)\\n Map (error(\\\"division by zero\\\"))\\n ReadIndex on=mz_sources mz_sources_ind=[*** full scan ***]\\n\\nTarget cluster: mz_catalog_server\\n\",\n \"json\": {\n \"plans\": [\n {\n \"id\": \"\",\n \"plan\": {\n \"ArrangeBy\": {\n \"input\": {\n \"Get\": {\n \"id\": {\n \"Global\": {\n \n }\n },\n \"typ\": {\n \"column_types\": [\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n }\n ],\n \"keys\": []\n },\n \"access_strategy\": \"SameDataflow\"\n }\n },\n \"keys\": [\n [\n {\n \"Column\": [\n 0,\n null\n ]\n }\n ]\n ]\n }\n }\n },\n {\n \"id\": \"\",\n \"plan\": {\n \"Project\": {\n \"input\": {\n \"Map\": {\n \"input\": {\n \"Get\": {\n \"id\": {\n \"Global\": {\n \n }\n },\n \"typ\": {\n \"column_types\": [\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"UInt32\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": {\n \"Array\": \"MzAclItem\"\n },\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n }\n ],\n \"keys\": [\n [\n 0\n ],\n [\n 1\n ]\n ]\n },\n \"access_strategy\": {\n \"Index\": [\n [\n {\n \n },\n \"FullScan\"\n ]\n ]\n }\n }\n },\n \"scalars\": [\n {\n \"Literal\": [\n {\n \"Err\": \"DivisionByZero\"\n },\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n }\n ]\n }\n ]\n }\n },\n \"outputs\": [\n 15\n ]\n }\n }\n }\n ],\n \"sources\": []\n }\n },\n \"fast_path\": {\n \"text\": \"Explained Query (fast path):\\n Finish limit=1 output=[#0]\\n →Map/Filter/Project\\n Project: #15\\n Map: error(\\\"division by zero\\\")\\n →Indexed mz_catalog.mz_sources (using mz_catalog.mz_sources_ind)\\n\\nTarget cluster: mz_catalog_server\\n\",\n \"json\": {\n \"plans\": [\n {\n \"id\": \"Explained Query (fast path)\",\n \"plan\": {\n \"PeekExisting\": [\n {\n \n },\n {\n \n },\n null,\n {\n \"mfp\": {\n \"expressions\": [\n {\n \"Literal\": [\n {\n \"Err\": \"DivisionByZero\"\n },\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n }\n ]\n }\n ],\n \"predicates\": [],\n \"projection\": [\n 15\n ],\n \"input_arity\": 15\n }\n }\n ]\n }\n }\n ],\n \"sources\": []\n }\n }\n }\n },\n \"insights\": {\n \"imports\": {\n \"\": {\n \"name\": {\n \"schema\": \"mz_catalog\",\n \"item\": \"mz_sources_ind\"\n },\n \"type\": \"compute\"\n }\n },\n \"fast_path_clusters\": {},\n \"fast_path_limit\": null,\n \"persist_count\": []\n },\n \"cluster\": {\n \"name\": \"mz_catalog_server\",\n \"id\": {\n \n }\n },\n \"redacted_sql\": \"SELECT '' / '' FROM [ AS mz_catalog.mz_sources] LIMIT ''\"\n}","code":"MZ001","severity":"notice"}} -{"type":"CommandStarting","payload":{"has_rows":false,"is_streaming":false}} +{"type":"CommandStarting","payload":{"has_rows":true,"is_streaming":false}} {"type":"Error","payload":{"message":"division by zero","code":"XX000"}} {"type":"ReadyForQuery","payload":"I"} diff --git a/src/environmentd/tests/testdata/mcp/agent_query_tool b/src/environmentd/tests/testdata/mcp/agent_query_tool index 504f32a1b6b85..52342df008825 100644 --- a/src/environmentd/tests/testdata/mcp/agent_query_tool +++ b/src/environmentd/tests/testdata/mcp/agent_query_tool @@ -27,14 +27,14 @@ mcp-agent {"jsonrpc":"2.0","id":10,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"SELECT 1 AS val"}}} ---- 200 OK -{"jsonrpc":"2.0","id":10,"result":{"content":[{"type":"text","text":"[\n [\n \"1\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":10,"result":{"content":[{"type":"text","text":"[\n [\"1\"]\n]"}],"isError":false}} # SHOW statement should work. mcp-agent {"jsonrpc":"2.0","id":11,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"SHOW DATABASES"}}} ---- 200 OK -{"jsonrpc":"2.0","id":11,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\",\n \"\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":11,"result":{"content":[{"type":"text","text":"[\n [\"materialize\",\"\"]\n]"}],"isError":false}} # ============================================================================= # query tool - DDL rejected diff --git a/src/environmentd/tests/testdata/mcp/developer b/src/environmentd/tests/testdata/mcp/developer index dc9bc02b3a1bf..6bd5fcc707644 100644 --- a/src/environmentd/tests/testdata/mcp/developer +++ b/src/environmentd/tests/testdata/mcp/developer @@ -38,14 +38,14 @@ mcp-developer {"jsonrpc":"2.0","id":10,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT name FROM mz_databases WHERE name = 'materialize'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":10,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":10,"result":{"content":[{"type":"text","text":"[\n [\"materialize\"]\n]"}],"isError":false}} # Query with qualified schema should work. mcp-developer {"jsonrpc":"2.0","id":11,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT name FROM mz_catalog.mz_databases WHERE name = 'materialize'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":11,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":11,"result":{"content":[{"type":"text","text":"[\n [\"materialize\"]\n]"}],"isError":false}} # ============================================================================= # query_system_catalog - pg_catalog and information_schema @@ -56,14 +56,14 @@ mcp-developer {"jsonrpc":"2.0","id":12,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT typname FROM pg_catalog.pg_type WHERE typname = 'bool'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":12,"result":{"content":[{"type":"text","text":"[\n [\n \"bool\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":12,"result":{"content":[{"type":"text","text":"[\n [\"bool\"]\n]"}],"isError":false}} # information_schema should be accessible. mcp-developer {"jsonrpc":"2.0","id":13,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT table_schema FROM information_schema.tables WHERE table_name = 'mz_databases' AND table_schema = 'mz_catalog' LIMIT 1"}}} ---- 200 OK -{"jsonrpc":"2.0","id":13,"result":{"content":[{"type":"text","text":"[\n [\n \"mz_catalog\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":13,"result":{"content":[{"type":"text","text":"[\n [\"mz_catalog\"]\n]"}],"isError":false}} # ============================================================================= # query_system_catalog - non-system tables rejected diff --git a/src/environmentd/tests/testdata/mcp/developer_query_tool b/src/environmentd/tests/testdata/mcp/developer_query_tool index 724d56630c392..042c40e42bfcc 100644 --- a/src/environmentd/tests/testdata/mcp/developer_query_tool +++ b/src/environmentd/tests/testdata/mcp/developer_query_tool @@ -39,7 +39,7 @@ mcp-developer {"jsonrpc":"2.0","id":10,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"SELECT 1 AS val"}}} ---- 200 OK -{"jsonrpc":"2.0","id":10,"result":{"content":[{"type":"text","text":"[\n [\n \"1\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":10,"result":{"content":[{"type":"text","text":"[\n [\"1\"]\n]"}],"isError":false}} # SELECT against a system catalog table should work too: the developer query # tool is intentionally NOT restricted to mz_*. @@ -47,14 +47,14 @@ mcp-developer {"jsonrpc":"2.0","id":11,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"SELECT name FROM mz_databases WHERE name = 'materialize'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":11,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":11,"result":{"content":[{"type":"text","text":"[\n [\"materialize\"]\n]"}],"isError":false}} # SHOW statement should work. mcp-developer {"jsonrpc":"2.0","id":12,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"SHOW DATABASES"}}} ---- 200 OK -{"jsonrpc":"2.0","id":12,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\",\n \"\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":12,"result":{"content":[{"type":"text","text":"[\n [\"materialize\",\"\"]\n]"}],"isError":false}} # EXPLAIN should work — this is the motivating use case for the dev query tool # (EXPLAIN ANALYZE needs a cluster and so cannot use query_system_catalog). @@ -62,7 +62,7 @@ mcp-developer {"jsonrpc":"2.0","id":13,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"EXPLAIN SELECT 1"}}} ---- 200 OK -{"jsonrpc":"2.0","id":13,"result":{"content":[{"type":"text","text":"[\n [\n \"Explained Query (fast path):\\n →Constant (1 rows)\\n\\nTarget cluster: mz_catalog_server\\n\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":13,"result":{"content":[{"type":"text","text":"[\n [\"Explained Query (fast path):\\n →Constant (1 rows)\\n\\nTarget cluster: mz_catalog_server\\n\"]\n]"}],"isError":false}} # ============================================================================= # query - EXPLAIN ANALYZE @@ -92,14 +92,14 @@ mcp-developer {"jsonrpc":"2.0","id":14,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"EXPLAIN ANALYZE MEMORY FOR INDEX t_idx AS SQL"}}} ---- 200 OK -{"jsonrpc":"2.0","id":14,"result":{"content":[{"type":"text","text":"[\n [\n \"WITH\\n summary_memory AS\\n (\\n SELECT\\n mlm.global_id AS global_id,\\n mlm.lir_id AS lir_id,\\n sum(mas.size) AS total_memory,\\n sum(mas.records) AS total_records,\\n CASE\\n WHEN count(DISTINCT mas.worker_id) <> 0\\n THEN sum(mas.size) / count(DISTINCT mas.worker_id)\\n ELSE NULL\\n END\\n AS avg_memory,\\n CASE\\n WHEN count(DISTINCT mas.worker_id) <> 0\\n THEN\\n sum(mas.records) / count(DISTINCT mas.worker_id)\\n ELSE NULL\\n END\\n AS avg_records\\n FROM\\n mz_introspection.mz_lir_mapping AS mlm\\n CROSS JOIN generate_series((mlm.operator_id_start)::int8, (mlm.operator_id_end - 1)::int8) AS valid_id\\n JOIN\\n mz_introspection.mz_arrangement_sizes_per_worker AS mas\\n ON (mas.operator_id = valid_id)\\n GROUP BY mlm.global_id, mlm.lir_id\\n )\\nSELECT\\n repeat(' ', nesting * 2) || operator AS operator,\\n pg_size_pretty(sm.total_memory) AS total_memory,\\n sm.total_records AS total_records\\nFROM\\n mz_introspection.mz_lir_mapping AS mlm\\n LEFT JOIN summary_memory AS sm USING(global_id, lir_id)\\n JOIN\\n mz_introspection.mz_mappable_objects AS mo\\n ON (mlm.global_id = mo.global_id)\\nWHERE mo.name = 'materialize.public.t_idx'\\nORDER BY mlm.lir_id DESC;\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":14,"result":{"content":[{"type":"text","text":"[\n [\"WITH\\n summary_memory AS\\n (\\n SELECT\\n mlm.global_id AS global_id,\\n mlm.lir_id AS lir_id,\\n sum(mas.size) AS total_memory,\\n sum(mas.records) AS total_records,\\n CASE\\n WHEN count(DISTINCT mas.worker_id) <> 0\\n THEN sum(mas.size) / count(DISTINCT mas.worker_id)\\n ELSE NULL\\n END\\n AS avg_memory,\\n CASE\\n WHEN count(DISTINCT mas.worker_id) <> 0\\n THEN\\n sum(mas.records) / count(DISTINCT mas.worker_id)\\n ELSE NULL\\n END\\n AS avg_records\\n FROM\\n mz_introspection.mz_lir_mapping AS mlm\\n CROSS JOIN generate_series((mlm.operator_id_start)::int8, (mlm.operator_id_end - 1)::int8) AS valid_id\\n JOIN\\n mz_introspection.mz_arrangement_sizes_per_worker AS mas\\n ON (mas.operator_id = valid_id)\\n GROUP BY mlm.global_id, mlm.lir_id\\n )\\nSELECT\\n repeat(' ', nesting * 2) || operator AS operator,\\n pg_size_pretty(sm.total_memory) AS total_memory,\\n sm.total_records AS total_records\\nFROM\\n mz_introspection.mz_lir_mapping AS mlm\\n LEFT JOIN summary_memory AS sm USING(global_id, lir_id)\\n JOIN\\n mz_introspection.mz_mappable_objects AS mo\\n ON (mlm.global_id = mo.global_id)\\nWHERE mo.name = 'materialize.public.t_idx'\\nORDER BY mlm.lir_id DESC;\"]\n]"}],"isError":false}} # EXPLAIN ANALYZE CLUSTER (Statement::ExplainAnalyzeCluster). mcp-developer {"jsonrpc":"2.0","id":15,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"EXPLAIN ANALYZE CLUSTER MEMORY AS SQL"}}} ---- 200 OK -{"jsonrpc":"2.0","id":15,"result":{"content":[{"type":"text","text":"[\n [\n \"WITH\\n per_operator_memory_totals AS\\n (\\n SELECT\\n mlm.global_id AS global_id,\\n mlm.lir_id AS lir_id,\\n sum(mas.size) AS total_memory,\\n sum(mas.records) AS total_records\\n FROM\\n mz_introspection.mz_lir_mapping AS mlm\\n CROSS JOIN generate_series((mlm.operator_id_start)::int8, (mlm.operator_id_end - 1)::int8) AS valid_id\\n JOIN\\n mz_introspection.mz_arrangement_sizes_per_worker AS mas\\n ON (mas.operator_id = valid_id)\\n GROUP BY mlm.global_id, mlm.lir_id\\n ),\\n object_memory_totals AS\\n (\\n SELECT\\n pomt.global_id AS global_id,\\n sum(pomt.total_memory) AS total_memory,\\n sum(pomt.total_records) AS total_records\\n FROM per_operator_memory_totals AS pomt\\n GROUP BY pomt.global_id\\n )\\nSELECT\\n mo.name AS object,\\n mo.global_id AS global_id,\\n pg_size_pretty(omt.total_memory) AS total_memory,\\n omt.total_records AS total_records\\nFROM\\n mz_introspection.mz_mappable_objects AS mo\\n LEFT JOIN object_memory_totals AS omt USING(global_id)\\nORDER BY\\n omt.total_memory DESC NULLS LAST,\\n total_records DESC NULLS LAST,\\n mo.name DESC;\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":15,"result":{"content":[{"type":"text","text":"[\n [\"WITH\\n per_operator_memory_totals AS\\n (\\n SELECT\\n mlm.global_id AS global_id,\\n mlm.lir_id AS lir_id,\\n sum(mas.size) AS total_memory,\\n sum(mas.records) AS total_records\\n FROM\\n mz_introspection.mz_lir_mapping AS mlm\\n CROSS JOIN generate_series((mlm.operator_id_start)::int8, (mlm.operator_id_end - 1)::int8) AS valid_id\\n JOIN\\n mz_introspection.mz_arrangement_sizes_per_worker AS mas\\n ON (mas.operator_id = valid_id)\\n GROUP BY mlm.global_id, mlm.lir_id\\n ),\\n object_memory_totals AS\\n (\\n SELECT\\n pomt.global_id AS global_id,\\n sum(pomt.total_memory) AS total_memory,\\n sum(pomt.total_records) AS total_records\\n FROM per_operator_memory_totals AS pomt\\n GROUP BY pomt.global_id\\n )\\nSELECT\\n mo.name AS object,\\n mo.global_id AS global_id,\\n pg_size_pretty(omt.total_memory) AS total_memory,\\n omt.total_records AS total_records\\nFROM\\n mz_introspection.mz_mappable_objects AS mo\\n LEFT JOIN object_memory_totals AS omt USING(global_id)\\nORDER BY\\n omt.total_memory DESC NULLS LAST,\\n total_records DESC NULLS LAST,\\n mo.name DESC;\"]\n]"}],"isError":false}} # ============================================================================= # query - writes and DDL rejected @@ -182,7 +182,7 @@ mcp-developer {"jsonrpc":"2.0","id":50,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT name FROM mz_databases WHERE name = 'materialize'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":50,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":50,"result":{"content":[{"type":"text","text":"[\n [\"materialize\"]\n]"}],"isError":false}} # ============================================================================= # query - cluster_replica validation diff --git a/src/environmentd/tests/testdata/mcp/developer_response_limit b/src/environmentd/tests/testdata/mcp/developer_response_limit index 91b87971d29f3..ab0ecaa87d60f 100644 --- a/src/environmentd/tests/testdata/mcp/developer_response_limit +++ b/src/environmentd/tests/testdata/mcp/developer_response_limit @@ -17,7 +17,7 @@ mcp-developer {"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT name FROM mz_databases WHERE name = 'materialize'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"[\n [\"materialize\"]\n]"}],"isError":false}} # Responses over the limit are rejected with an error telling the agent to # narrow its query. repeat() gives a deterministic response size. @@ -25,4 +25,4 @@ mcp-developer {"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT repeat('x', 5000) FROM mz_catalog.mz_databases WHERE name = 'materialize'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":2,"error":{"code":-32603,"message":"Query execution failed: Response size (5018 bytes) exceeds the 1024 byte limit. Use LIMIT or WHERE to narrow your query.","data":{"error_type":"ExecutionError"}}} +{"jsonrpc":"2.0","id":2,"error":{"code":-32603,"message":"Query execution failed: Response size (5010 bytes) exceeds the 1024 byte limit. Use LIMIT or WHERE to narrow your query.","data":{"error_type":"ExecutionError"}}} diff --git a/src/pgwire/src/protocol.rs b/src/pgwire/src/protocol.rs index cfe140bfac742..d98e75820ae4c 100644 --- a/src/pgwire/src/protocol.rs +++ b/src/pgwire/src/protocol.rs @@ -66,7 +66,6 @@ use tokio::io::{self, AsyncRead, AsyncWrite}; use tokio::select; use tokio::time::{self}; use tokio_metrics::TaskMetrics; -use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{Instrument, debug, debug_span, info, warn}; use uuid::Uuid; @@ -2245,7 +2244,7 @@ where row_desc, portal_name, InProgressRows::new(RecordFirstRowStream::new( - Box::new(UnboundedReceiverStream::new(rx)), + rx, execute_started, &self.adapter_client, Some(instance_id), @@ -2302,7 +2301,7 @@ where format, row_desc, RecordFirstRowStream::new( - Box::new(UnboundedReceiverStream::new(rx)), + rx, execute_started, &self.adapter_client, Some(instance_id), From d457e6c14b9f327f261b7c416bc6053c2b1368a4 Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 19 Aug 2026 21:34:54 +0200 Subject: [PATCH 06/10] compute: stop error collections multiplying their cardinality (#38196) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation Closes: [CPU-204](https://linear.app/materializeinc/issue/CPU-204) Follow-up: [CPU-209](https://linear.app/materializeinc/issue/CPU-209) An error collection carries a multiplicity almost nothing reads. A dataflow is in an error state if its error collection is non-empty, and the error surfaced to the user is chosen arbitrarily, so an error's diff conveys nothing about the query's answer. Several rendering patterns nonetheless grow that diff multiplicatively rather than additively, and because every error in a dataflow consolidates onto one of a handful of distinct error values, those diffs pile onto single records and reach `Diff` overflow on plans of quite ordinary size. The reported shape is plan-level sharing: a chain of diamond-shaped CTEs over a dataflow that has any `EvalError`. The asymmetry that makes this an error-only problem is worth stating plainly, since it explains why the ok side of the same plan is fine: the branches of a diamond compute *different* ok rows, so ok multiplicity does not compound, but they propagate the *identical* upstream error collection, so error multiplicity doubles at every reconvergence. ### Description Three multipliers. Two are fixed here, the third only for materialized views. **Delta join, N copies per join.** `render_delta_join` propagated each input's pre-existing errors once per delta path: `build_update_stream` concatenated the source's errors and every stage's `build_halfjoin` concatenated a lookup arrangement's errors, inside the per-path region. With N inputs that is N copies of every input's errors assembled over N² concat edges, and since a join's output is another join's input those factors compound through a nested plan. Input error collections are now gathered once, outside the path loop, and the two helpers return only the errors they themselves produce. No operator added, no semantic change: the same union of streams, formed once instead of N times. **Sharing within a dataflow, f copies per level.** Every reader of a binding propagates that binding's errors independently, so a binding read f times contributes its errors f times, and the factors apply again at each further level of sharing. `CollectionBundle::distinct_errs` collapses a binding's error multiplicities to one, applied at every `Let` and rec binding definition. This is the multiplier the reported incidents were made of, and the one the regression test drives to overflow. **Sharing across objects, f copies per level.** A level of sharing expressed as an indexed view rather than a CTE reads `Get(Global)`, which a binding collapse never sees, so the importing dataflow accumulates the exported diffs. Only half of this is closed here, at the persist sink, whose shard another dataflow reads back verbatim and whose contents are durable. Subscribes and one-shot copies are read once by a client and never re-imported, so they do not pay for the arrangement. The sink normalizes after the null assertions, whose errors follow the ok row's multiplicity rather than the error collection's. The index half is **not** fixed, and [CPU-209](https://linear.app/materializeinc/issue/CPU-209) records why: collapsing at the export gives the exported arrangement a second trace reader, which pins the shared spine's physical frontier and makes importing dataflows panic in `cursor_through` with `upper` straddles batch. That was measured four ways, including a variant that builds the reduce, discards its output, and exports the original trace, which panics too. So the fault is the reader's existence, not its output or the frontier probe. Cross-object multiplicity through indexes therefore stays unbounded. In proportion: object-graph path counts for the reported incidents top out at 8, against roughly 7×10^30 within one dataflow, so this half is reachable synthetically but is not what those incidents were. Two decisions the diff cannot explain: *Why a reduce and not arithmetic on the diffs.* Saturating the add, or collapsing to a sign, looks cheaper and does not work: neither is additive, so neither cancels when the errors retract, and a partially-retracted error would persist forever as a phantom. Bounding multiplicity requires reading the accumulated collection, which means a reduce over an arrangement. `LetRec` already collapses its error variable this way and for this reason. *Why every representation in a bundle is collapsed, not just one.* A bundle's raw collection and each of its arrangements carry their own independent error stream, and they are not even the same content: an arrangement's errors include the key-formation errors that the raw collection's do not. Which form a consumer reads is the consumer's choice, and a delta join reads both in one operator, so the binding definition cannot know that choice. ### Behavior and rollout `enable_compute_error_distinct` gates the per-binding collapse, and now defaults **on**. The divergence that argued for shipping dark is closed: the sink normalization is ungated, so nothing this flag decides reaches durable state and two replicas rendering under different values write the same thing. A default change also arrives with a new binary that every replica is recreated on, whereas a runtime flip is what leaves two live replicas rendering differently. Production LaunchDarkly still serves off, so the flag is allowlisted in `launchdarkly-flag-consistency` until that is reconciled. Two parts are deliberately **not** gated, so "behavior is unchanged with the flag off" would be wrong: - The delta-join rewiring is unconditional. It is equivalence-preserving on the set of errors and strictly reduces copies, and the failure mode worth guarding against is `bundle_errs` missing a form and silently dropping a dataflow's errors, which a test catches and a flag does not. - The sink normalization is unconditional, because gating it would leave the durable content of a materialized view's shard dependent on the flag. Error counts change, in two places. `mz_compute_error_counts_raw_unified` sums error diffs, so a diamond over one failing row previously reported 8 and now reports 1; unshared plans are unaffected. And anything reading a materialized view's shard now sees one error per distinct error rather than one per failing row, which is what the `idx2_div_by_zero` golden moves record. Neither number is promised: an object is tainted by having any error at all, and that is the part that is preserved. The Console does surface these counts, so the change is visible. ### Verification `test/sqllogictest/error_semantics.slt` gains a 70-level diamond CTE chain over an erroring expression, asserting the error still comes back, where an overflow is a panic CI fails on. Measured both directions: | Flag on | Flag off | | --- | --- | | PASS | `Overflow: 4611686018427387904 + 4611686018427387904` | Full file with the flag on: 153/153. With the flag off it panics in `consolidate_updates_slice_slow` over `DataflowErrorSer` and the replica dies. `union` rather than `union all` is what makes the chain reach the overflow: the distinct arranges the error collection so duplicates consolidate into one record with a doubling diff. Under `union all` the records duplicate physically instead and the query exhausts memory, 34 GB with no completion, without ever overflowing. A cross-object chain of indexed views is deliberately not in the suite, since nothing here bounds it. It belongs with [CPU-209](https://linear.app/materializeinc/issue/CPU-209). Thanks to @ggevay for measuring that the first version of that test did not reproduce the bug at all, for the `union all` to `union` diagnosis, for finding that the cross-object half was still unfixed, and for the reference-count measurement that removed a gate this PR did not need; and to @def- for the mzcompose workflow showing two replicas of one cluster correcting each other's writes to an errored materialized view's shard indefinitely, which the sink normalization closes. Operator cost, measured on `mz_catalog_server` in an empty environment: +1,014 dataflow operators (+6.4%) with the flag on, measured before the sink normalization was added. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01GbtepuqcxmAffaM2qXZ4uA --------- Co-authored-by: Claude --- misc/python/materialize/mzcompose/__init__.py | 1 + .../materialize/parallel_workload/action.py | 1 + src/compute-types/src/dyncfgs.rs | 20 ++++ src/compute/src/render.rs | 40 +++++++- src/compute/src/render/context.rs | 80 ++++++++++++++++ src/compute/src/render/join/delta_join.rs | 79 ++++++++++++---- src/compute/src/render/sinks.rs | 12 ++- .../mzcompose.py | 1 + test/sqllogictest/error_semantics.slt | 92 +++++++++++++++++++ .../top-1-monotonic.td | 32 +++++++ test/testdrive/introspection-sources.td | 6 +- test/testdrive/kafka-avro-upsert-sinks.td | 6 ++ test/testdrive/materializations.td | 12 +++ test/testdrive/monotonic.td | 8 ++ test/testdrive/top-1-monotonic.td | 32 +++++++ .../unified-compute-introspection.td | 12 +-- 16 files changed, 404 insertions(+), 30 deletions(-) diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py index d984ad2ef86d4..322c6f05cf0fa 100644 --- a/misc/python/materialize/mzcompose/__init__.py +++ b/misc/python/materialize/mzcompose/__init__.py @@ -116,6 +116,7 @@ def get_minimal_system_parameters( "enable_s3_tables_region_check": "false", "enable_statement_lifecycle_logging": "true", "enable_storage_introspection_logs": "true", + "enable_compute_error_distinct": "true", "enable_compute_temporal_bucketing": "true", "enable_variadic_left_join_lowering": "true", "enable_worker_core_affinity": "true", diff --git a/misc/python/materialize/parallel_workload/action.py b/misc/python/materialize/parallel_workload/action.py index 3963129aba924..ebe9e4a9d8ad8 100644 --- a/misc/python/materialize/parallel_workload/action.py +++ b/misc/python/materialize/parallel_workload/action.py @@ -2986,6 +2986,7 @@ def __init__( self.flags_with_values["enable_compute_temporal_bucketing"] = ( BOOLEAN_FLAG_VALUES ) + self.flags_with_values["enable_compute_error_distinct"] = BOOLEAN_FLAG_VALUES self.flags_with_values["enable_alter_table_add_column"] = BOOLEAN_FLAG_VALUES self.flags_with_values["enable_bounded_staleness_isolation"] = ( BOOLEAN_FLAG_VALUES diff --git a/src/compute-types/src/dyncfgs.rs b/src/compute-types/src/dyncfgs.rs index 19ca6b4cd02fb..bfc0802b28541 100644 --- a/src/compute-types/src/dyncfgs.rs +++ b/src/compute-types/src/dyncfgs.rs @@ -23,6 +23,25 @@ pub const ENABLE_HALF_JOIN2: Config = Config::new( "Whether compute should use `half_join2` rather than DD's `half_join` to render delta joins.", ); +/// Whether rendering should collapse error multiplicities to one where it arranges errors. +/// +/// Error semantics depend only on whether an error is present, so its multiplicity carries no +/// information a consumer reads. Left uncollapsed, a shared collection contributes its errors once +/// per plan path that reads it, and because those factors apply again at each level of sharing they +/// compound multiplicatively until the `Diff` overflows. +/// +/// Governs sharing within a dataflow only. Sharing across objects is bounded unconditionally, by +/// normalizing at every boundary another dataflow can read, so what this flag decides is never +/// durable state and two replicas rendering under different values still write the same thing. +/// Environment-scoped for now because nothing needs finer granularity, not because finer would be +/// unsafe. +pub const ENABLE_ERROR_DISTINCT: Config = Config::new( + "enable_compute_error_distinct", + true, + "Whether compute rendering should collapse error multiplicities to one where it arranges \ + errors.", +); + /// Use the column-paged merge batcher code path at arrange sites. When /// `true`, arrange operators use `Col2ValPagedBatcher` (in /// `mz_timely_util::columnar`) and `RowRowColPagedBuilder` (in @@ -548,6 +567,7 @@ pub const MV_SINK_ADVANCE_PERSIST_FRONTIERS: Config = Config::new( pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet { configs .add(&ENABLE_HALF_JOIN2) + .add(&ENABLE_ERROR_DISTINCT) .add(&ENABLE_MZ_JOIN_CORE) .add(&ENABLE_SYNC_MV_SINK) .add(&ENABLE_CORRECTION_V2) diff --git a/src/compute/src/render.rs b/src/compute/src/render.rs index 82fc821cd8604..ccfe93b902c97 100644 --- a/src/compute/src/render.rs +++ b/src/compute/src/render.rs @@ -125,7 +125,8 @@ use mz_compute_types::dataflows::{DataflowDescription, IndexDesc}; use mz_compute_types::dyncfgs::{ COMPUTE_APPLY_COLUMN_DEMANDS, COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK, COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES, ENABLE_COMPUTE_LOGICAL_BACKPRESSURE, - ENABLE_COMPUTE_TEMPORAL_BUCKETING, SUBSCRIBE_SNAPSHOT_OPTIMIZATION, TEMPORAL_BUCKETING_SUMMARY, + ENABLE_COMPUTE_TEMPORAL_BUCKETING, ENABLE_ERROR_DISTINCT, SUBSCRIBE_SNAPSHOT_OPTIMIZATION, + TEMPORAL_BUCKETING_SUMMARY, }; use mz_compute_types::plan::render_plan::{ self, BindStage, LetBind, LetFreePlan, RecBind, RenderPlan, @@ -711,6 +712,17 @@ impl<'g> Context<'g, mz_repr::Timestamp> { let key = &idx.key; match bundle.arrangement(key) { Some(ArrangementFlavor::Local(mut oks, mut errs)) => { + // NOTE: Do not give an exported arrangement a second reader that holds a trace + // handle, such as a `reduce`. Such a reader pins the shared spine's physical + // frontier at its own lagging progress, and `ArrangementManager::maintenance` can + // then no longer advance it, so batches pile up in `Spine::pending`. A cursor is + // only checked for straddling over pending batches, so an importing dataflow's + // `cursor_through` eventually panics with `upper` straddles batch. Watching + // `errs.stream` in `output_probe` does not help, and neither does discarding the + // reader's output. Stream-level readers like `as_collection` are unaffected. This is + // why error multiplicity is not collapsed here, leaving multiplicity that crosses + // an index boundary unbounded. TODO(CPU-209): bound it without a trace reader. + // Ensure that the frontier does not advance past the expiration time, if set. // Otherwise, we might write down incorrect data. if let Some(&expiration) = self.dataflow_expiration.as_option() { @@ -906,6 +918,7 @@ impl<'scope> Context<'scope, Product>> { .render_letfree_plan(object_id, value, binding) .leave_region(self.scope) }); + let bundle = self.distinct_binding_errs(bundle); self.insert_id(Id::Local(id), bundle); } @@ -939,6 +952,11 @@ impl<'scope> Context<'scope, Product>> { // here to cause that to happen. let (oks, mut err) = bundle.collection.clone().unwrap(); let oks = oks.into_vec(); + // Collapses what forward reads see. `err_v` below feeds reads rendered before this + // binding and is collapsed separately; without this, a `Get` in a later rec binding + // or in the body resolves to the bundle stored here and compounds level over level, + // which is exactly what the collapse prevents for non-recursive bindings. + let bundle = self.distinct_binding_errs(bundle); self.insert_id(Id::Local(id), bundle); let (oks_v, err_v) = variables.remove(&Id::Local(id)).unwrap(); @@ -1044,6 +1062,7 @@ impl<'scope, T: RenderTimestamp + MaybeBucketByTime> Context<'scope, T> { .render_letfree_plan(object_id, value, binding) .leave_region(self.scope) }); + let bundle = self.distinct_binding_errs(bundle); self.insert_id(Id::Local(id), bundle); } } @@ -1056,6 +1075,25 @@ impl<'scope, T: RenderTimestamp + MaybeBucketByTime> Context<'scope, T> { }) } + /// Collapses a binding's error multiplicities. + /// + /// Applied to every binding, not only the multiply-read ones. Gating on the reference count + /// would be a pure optimization, since collapsing a binding one `Get` reads is harmless, and + /// there is almost nothing to gate: `NormalizeLets` inlines single-use bindings, so the ones + /// reaching rendering are shared. See [`CollectionBundle::distinct_errs`] for why the collapse + /// is needed at all, and why a binding's definition is the place for it rather than the + /// multi-input operators where the duplicate copies happen to meet again. + fn distinct_binding_errs( + &self, + bundle: CollectionBundle<'scope, T>, + ) -> CollectionBundle<'scope, T> { + if ENABLE_ERROR_DISTINCT.get(&self.config_set) { + bundle.distinct_errs() + } else { + bundle + } + } + /// Renders a let-free plan to a differential dataflow, producing the collection of results. fn render_letfree_plan( &self, diff --git a/src/compute/src/render/context.rs b/src/compute/src/render/context.rs index 6e6b80119b601..c5f20a8691947 100644 --- a/src/compute/src/render/context.rs +++ b/src/compute/src/render/context.rs @@ -48,6 +48,7 @@ use timely::progress::{Antichain, Timestamp}; use crate::compute_state::ComputeState; use crate::extensions::arrange::{KeyCollection, MzArrange, MzArrangeCore}; +use crate::extensions::reduce::MzReduce; use crate::render::columnar::CollectionEdge; use crate::render::errors::{DataflowErrorSer, ErrorLogger}; use crate::render::{LinearJoinSpec, MaybeBucketByTime, RenderTimestamp}; @@ -435,6 +436,41 @@ impl<'scope, T: RenderTimestamp> ArrangementFlavor<'scope, T> { } } } +/// Rewrites an arranged error collection to hold each of its errors once. +/// +/// Sound because query semantics depend only on whether an error is present, and correct under +/// retraction only because it reads the accumulated collection: no pointwise function of the input +/// diffs (a saturating add, a sign) can collapse multiplicity and still cancel when the errors +/// retract. +/// +/// NOTE: One consumer does read the multiplicity. Error-count introspection reports it as the +/// number of failing rows, so `log_dataflow_errors` must see the collection before this collapses +/// it, or a dataflow reports one error however many rows failed. Collapse after the logging, never +/// before. +pub(crate) fn distinct_arranged_errs<'a, T: RenderTimestamp>( + errs: Arranged<'a, ErrAgent>, + name: &str, +) -> Arranged<'a, ErrAgent> { + errs.mz_reduce_abelian::<_, ErrBuilder<_, _>, ErrSpine<_, _>, _>( + name, + |_err, _input, output| output.push(((), Diff::ONE)), + ) +} + +/// Rewrites an error collection to hold each of its errors once. +/// +/// Costs an arrangement more than [`distinct_arranged_errs`], which reuses the arrangement its +/// input already has. +pub(crate) fn distinct_errs_collection<'a, T: RenderTimestamp>( + errs: VecCollection<'a, T, DataflowErrorSer, Diff>, +) -> VecCollection<'a, T, DataflowErrorSer, Diff> { + let errs: KeyCollection<_, _, _> = errs.into(); + let errs = errs + .mz_arrange::, ErrBatcher<_, _>, ErrBuilder<_, _>, ErrSpine<_, _>>( + "Arrange errors", + ); + distinct_arranged_errs(errs, "Distinct errors").as_collection(|err, _| err.clone()) +} /// A bundle of the various ways a collection can be represented. /// @@ -507,6 +543,50 @@ impl<'scope, T: RenderTimestamp> CollectionBundle<'scope, T> { } } + /// Collapses the multiplicity of every error this bundle carries to one. + /// + /// Belongs at the definition of a binding that more than one `Get` reads. Each reader + /// propagates the binding's errors independently, so a binding read `f` times contributes its + /// errors `f` times to the dataflow's error output, and those factors apply again at every + /// further level of sharing: a chain of diamonds multiplies rather than adds, and reaches + /// `Diff` overflow at a depth plans really do have. Collapsing at each definition holds the + /// dataflow's error multiplicity to the fan-out of a single level. + /// + /// Sound because error semantics depend only on whether an error is present, and correct under + /// retraction only because it reads the accumulated collection: no pointwise function of the + /// input diffs (a saturating add, a sign) can collapse multiplicity and still cancel when the + /// errors retract. + /// + /// Collapses every form the bundle offers, not just one. Each form carries its own error + /// stream, and those streams differ in content as well as identity: an arrangement's errors + /// include the key-formation errors that the raw collection's do not. Which form a consumer + /// reads is the consumer's choice, and a delta join reads both within one operator, so a + /// binding's definition cannot know which form to collapse. + /// + /// NOTE: Leaves imported arrangements (`ArrangementFlavor::Trace`) alone, whose error traces + /// this dataflow cannot rewrite in place. Their errors arrive bounded by the exporting + /// dataflow's last level of sharing rather than collapsed to one, since nothing collapses at an + /// export. A global read more than once within one dataflow is not collapsed either, because + /// only local bindings reach this. + pub fn distinct_errs(mut self) -> Self { + if let Some((oks, errs)) = self.collection.take() { + self.collection = Some((oks, distinct_errs_collection(errs))); + } + for (key, flavor) in std::mem::take(&mut self.arranged) { + let flavor = match flavor { + ArrangementFlavor::Local(oks, errs) => { + // Names the key, not the binding: an operator name carrying a `LocalId` would + // churn the introspection goldens every time the optimizer renumbers locals. + let name = format!("Distinct errors[{key:?}]"); + ArrangementFlavor::Local(oks, distinct_arranged_errs(errs, &name)) + } + flavor @ ArrangementFlavor::Trace(..) => flavor, + }; + self.arranged.insert(key, flavor); + } + self + } + /// Brings the collection bundle into a region. pub fn enter_region<'inner>(&self, region: Scope<'inner, T>) -> CollectionBundle<'inner, T> { CollectionBundle { diff --git a/src/compute/src/render/join/delta_join.rs b/src/compute/src/render/join/delta_join.rs index a16a60764c70d..cfac33a55887a 100644 --- a/src/compute/src/render/join/delta_join.rs +++ b/src/compute/src/render/join/delta_join.rs @@ -55,9 +55,6 @@ impl<'scope, T: RenderTimestamp> Context<'scope, T> { ) -> CollectionBundle<'scope, T> { // We create a new region to contain the dataflow paths for the delta join. let (oks, errs) = self.scope.clone().region_named("Join(Delta)", |inner| { - // Collects error streams for the ambient scope. - let mut inner_errs = Vec::new(); - // Our plan is to iterate through each input relation, and attempt // to find a plan that maximally uses existing keys (better: uses // existing arrangements, to which we have access). @@ -86,6 +83,11 @@ impl<'scope, T: RenderTimestamp> Context<'scope, T> { }) .collect::>(); + // Collects error streams for the ambient scope, seeded with the errors the inputs + // arrive with. Every path reads every input, but an input's pre-existing errors belong + // in the output once, not once per path. See [`bundle_errs`]. + let mut inner_errs: Vec<_> = inputs.iter().flat_map(bundle_errs).collect(); + for path_plan in join_plan.path_plans { // Deconstruct the stages of the path plan. let DeltaPathPlan { @@ -288,12 +290,51 @@ fn prune_bundle<'scope, T: RenderTimestamp>( } } +/// The error collections of every collection and arrangement `bundle` holds. +/// +/// A delta join reads every input from every one of its paths, but an input's pre-existing errors +/// belong in the join's error output once. Propagating them per path would multiply their +/// multiplicities by the number of paths, and because a join's output is another join's input, those +/// factors compound multiplicatively through a nested plan until the `Diff` overflows. Error +/// semantics depend only on presence, so the extra copies buy nothing. +/// +/// Expects the pruned bundle (see [`prune_bundle`]), so that it yields errors only for the +/// collections some path actually reads. +/// +/// NOTE: This bounds an input's errors to one copy per retained form, not to one copy outright. +/// Each form carries its own error collection, and an arrangement's is a distinct stream built from +/// the raw one plus that key's key-formation errors, so an input the join reads under two lookup +/// keys still contributes its errors twice. Reachable whenever a delta path set needs an +/// error-carrying input arranged by more than one key. +fn bundle_errs<'scope, T: RenderTimestamp>( + bundle: &CollectionBundle<'scope, T>, +) -> Vec> { + let mut collected = Vec::with_capacity(bundle.arranged.len() + 1); + if let Some((_oks, errs)) = &bundle.collection { + collected.push(errs.clone()); + } + for flavor in bundle.arranged.values() { + let errs = match flavor { + ArrangementFlavor::Local(_oks, errs) => errs.clone().as_collection(|k, _v| k.clone()), + ArrangementFlavor::Trace(_id, _oks, errs) => { + errs.clone().as_collection(|k, _v| k.clone()) + } + }; + collected.push(errs); + } + collected +} + /// Constructs a `half_join` against the arrangement held by a collection bundle. /// /// This wrapper demuxes over the two flavors of arrangement (dataflow-local or imported trace) /// that the bundle might hold for `lookup_key`, dispatching to the generic [`build_halfjoin_trace`] /// for each. `source_precedes_lookup` selects the tie-breaking comparison: `le` if the source /// relation precedes the lookup relation in the total order on relations, otherwise `lt`. +/// +/// The returned error collection holds only the errors this stage produces. The errors `bundle` +/// already carries are the caller's to propagate, once, rather than once per delta path that looks +/// the input up. See [`bundle_errs`]. fn build_halfjoin<'scope, T>( updates: VecCollection<'scope, T, (Row, T), Diff>, prev_key: Vec, @@ -311,7 +352,7 @@ where T: RenderTimestamp, { match bundle.arrangement(&lookup_key) { - Some(ArrangementFlavor::Local(oks, errs)) => { + Some(ArrangementFlavor::Local(oks, _errs)) => { let (oks, errs2) = if source_precedes_lookup { build_halfjoin_trace::<_, RowRowAgent<_, _>, _>( updates, @@ -333,9 +374,9 @@ where config_set, ) }; - (oks, errs2.concat(errs.as_collection(|k, _v| k.clone()))) + (oks, errs2) } - Some(ArrangementFlavor::Trace(_, oks, errs)) => { + Some(ArrangementFlavor::Trace(_, oks, _errs)) => { let (oks, errs2) = if source_precedes_lookup { build_halfjoin_trace::<_, RowRowEnter<_, _, _>, _>( updates, @@ -357,7 +398,7 @@ where config_set, ) }; - (oks, errs2.concat(errs.as_collection(|k, _v| k.clone()))) + (oks, errs2) } None => panic!("Arrangement promised by the planner is absent!"), } @@ -658,6 +699,10 @@ where /// With a `source_key`, demuxes over the two flavors of arrangement the bundle might hold for that /// key, dispatching to the generic [`build_update_stream_trace`]. Without a `source_key`, the source /// relation is consumed as a raw (unarranged) collection via [`build_update_stream_stream`]. +/// +/// The returned error collection holds only the errors the initial closure produces. The errors +/// `bundle` already carries are the caller's to propagate, once, rather than once per delta path. +/// See [`bundle_errs`]. fn build_update_stream<'scope, T>( bundle: &CollectionBundle<'scope, T>, as_of: Antichain, @@ -675,32 +720,28 @@ where // No source key means a single-time dataflow (e.g. a `SELECT`) whose plan was truncated to // this one path, letting us hydrate the source from its raw collection instead of an // arrangement. - let (oks, errs) = bundle + let (oks, _errs) = bundle .collection .clone() .expect("The unarranged collection doesn't exist."); - let (oks, errs2) = - build_update_stream_stream(oks.into_vec(), as_of, source_relation, initial_closure); - return (oks, errs2.concat(errs)); + return build_update_stream_stream(oks.into_vec(), as_of, source_relation, initial_closure); }; match bundle.arrangement(&source_key) { - Some(ArrangementFlavor::Local(oks, errs)) => { - let (oks, errs2) = build_update_stream_trace::<_, RowRowAgent<_, _>>( + Some(ArrangementFlavor::Local(oks, _errs)) => { + build_update_stream_trace::<_, RowRowAgent<_, _>>( oks, as_of, source_relation, initial_closure, - ); - (oks, errs2.concat(errs.as_collection(|k, _v| k.clone()))) + ) } - Some(ArrangementFlavor::Trace(_, oks, errs)) => { - let (oks, errs2) = build_update_stream_trace::<_, RowRowEnter<_, _, _>>( + Some(ArrangementFlavor::Trace(_, oks, _errs)) => { + build_update_stream_trace::<_, RowRowEnter<_, _, _>>( oks, as_of, source_relation, initial_closure, - ); - (oks, errs2.concat(errs.as_collection(|k, _v| k.clone()))) + ) } None => panic!("Arrangement promised by the planner is absent!"), } diff --git a/src/compute/src/render/sinks.rs b/src/compute/src/render/sinks.rs index 4dbdf88a4eec9..c96e0be462b7b 100644 --- a/src/compute/src/render/sinks.rs +++ b/src/compute/src/render/sinks.rs @@ -30,7 +30,7 @@ use timely::progress::Antichain; use crate::compute_state::SinkToken; use crate::logging::compute::LogDataflowErrors; -use crate::render::context::Context; +use crate::render::context::{Context, distinct_errs_collection}; use crate::render::errors::DataflowErrorSer; use crate::render::{RenderTimestamp, StartSignal}; @@ -130,6 +130,16 @@ impl<'g, T: RenderTimestamp> Context<'g, T> { err_collection = err_collection.concat(null_errs); } + // Normalize what the sink persists. A materialized view's error multiplicity is durable + // state another dataflow reads back verbatim, so leaving it at this dataflow's fan-out both + // multiplies across the object graph and makes the written value depend on plan shape. + // Placed after the null assertions so their errors, whose multiplicity follows the ok row's, + // are covered too. Only the persist-backed sink is re-importable: a subscribe or a one-shot + // copy is read once by a client, so neither pays for the arrangement. + if matches!(sink.connection, ComputeSinkConnection::MaterializedView(_)) { + err_collection = distinct_errs_collection(err_collection); + } + let region_name = match sink.connection { ComputeSinkConnection::Subscribe(_) => format!("SubscribeSink({:?})", sink_id), ComputeSinkConnection::MaterializedView(_) => { diff --git a/test/launchdarkly-flag-consistency/mzcompose.py b/test/launchdarkly-flag-consistency/mzcompose.py index 074467a98fc16..bd8d9cb6baac4 100644 --- a/test/launchdarkly-flag-consistency/mzcompose.py +++ b/test/launchdarkly-flag-consistency/mzcompose.py @@ -526,6 +526,7 @@ "compute_subscribe_snapshot_optimization", "enable_cast_elimination", "enable_compute_correction_v2", + "enable_compute_error_distinct", "enable_compute_temporal_bucketing", "enable_new_outer_join_lowering", "enable_upsert_v2", diff --git a/test/sqllogictest/error_semantics.slt b/test/sqllogictest/error_semantics.slt index 967ca375be35c..3462e888974f3 100644 --- a/test/sqllogictest/error_semantics.slt +++ b/test/sqllogictest/error_semantics.slt @@ -93,3 +93,95 @@ select coalesce(a, (select a/b from test)) from test; query error Evaluation error: division by zero select *, case when a = 5 then (select a/b from test) else 7 end from test; + +# A chain of diamond-shaped CTEs over an erroring expression. Each level reads the level +# below it twice, so an undeduplicated error collection doubles its multiplicity per level +# and overflows `Diff` well before the chain ends. The multiplicity of an error is not +# observable, so the only thing to assert is that the error still comes back, and does so +# without an overflow (which CI catches as a panic). +# +# `union` rather than `union all` is what makes this reach the overflow: the distinct arranges +# the error collection, so duplicates consolidate into one record carrying a doubling diff. +# Under `union all` nothing consolidates between levels, the records duplicate physically +# instead, and the query exhausts memory without ever overflowing. + +statement ok +create table diamond (a int, b int) + +statement ok +insert into diamond values (1, 0) + +query error division by zero +with + c0 as (select a, a / b as q from diamond), + c1 as (select a, q from c0 where a = 1 union select q, a from c0 where a = 2), + c2 as (select a, q from c1 where a = 1 union select q, a from c1 where a = 2), + c3 as (select a, q from c2 where a = 1 union select q, a from c2 where a = 2), + c4 as (select a, q from c3 where a = 1 union select q, a from c3 where a = 2), + c5 as (select a, q from c4 where a = 1 union select q, a from c4 where a = 2), + c6 as (select a, q from c5 where a = 1 union select q, a from c5 where a = 2), + c7 as (select a, q from c6 where a = 1 union select q, a from c6 where a = 2), + c8 as (select a, q from c7 where a = 1 union select q, a from c7 where a = 2), + c9 as (select a, q from c8 where a = 1 union select q, a from c8 where a = 2), + c10 as (select a, q from c9 where a = 1 union select q, a from c9 where a = 2), + c11 as (select a, q from c10 where a = 1 union select q, a from c10 where a = 2), + c12 as (select a, q from c11 where a = 1 union select q, a from c11 where a = 2), + c13 as (select a, q from c12 where a = 1 union select q, a from c12 where a = 2), + c14 as (select a, q from c13 where a = 1 union select q, a from c13 where a = 2), + c15 as (select a, q from c14 where a = 1 union select q, a from c14 where a = 2), + c16 as (select a, q from c15 where a = 1 union select q, a from c15 where a = 2), + c17 as (select a, q from c16 where a = 1 union select q, a from c16 where a = 2), + c18 as (select a, q from c17 where a = 1 union select q, a from c17 where a = 2), + c19 as (select a, q from c18 where a = 1 union select q, a from c18 where a = 2), + c20 as (select a, q from c19 where a = 1 union select q, a from c19 where a = 2), + c21 as (select a, q from c20 where a = 1 union select q, a from c20 where a = 2), + c22 as (select a, q from c21 where a = 1 union select q, a from c21 where a = 2), + c23 as (select a, q from c22 where a = 1 union select q, a from c22 where a = 2), + c24 as (select a, q from c23 where a = 1 union select q, a from c23 where a = 2), + c25 as (select a, q from c24 where a = 1 union select q, a from c24 where a = 2), + c26 as (select a, q from c25 where a = 1 union select q, a from c25 where a = 2), + c27 as (select a, q from c26 where a = 1 union select q, a from c26 where a = 2), + c28 as (select a, q from c27 where a = 1 union select q, a from c27 where a = 2), + c29 as (select a, q from c28 where a = 1 union select q, a from c28 where a = 2), + c30 as (select a, q from c29 where a = 1 union select q, a from c29 where a = 2), + c31 as (select a, q from c30 where a = 1 union select q, a from c30 where a = 2), + c32 as (select a, q from c31 where a = 1 union select q, a from c31 where a = 2), + c33 as (select a, q from c32 where a = 1 union select q, a from c32 where a = 2), + c34 as (select a, q from c33 where a = 1 union select q, a from c33 where a = 2), + c35 as (select a, q from c34 where a = 1 union select q, a from c34 where a = 2), + c36 as (select a, q from c35 where a = 1 union select q, a from c35 where a = 2), + c37 as (select a, q from c36 where a = 1 union select q, a from c36 where a = 2), + c38 as (select a, q from c37 where a = 1 union select q, a from c37 where a = 2), + c39 as (select a, q from c38 where a = 1 union select q, a from c38 where a = 2), + c40 as (select a, q from c39 where a = 1 union select q, a from c39 where a = 2), + c41 as (select a, q from c40 where a = 1 union select q, a from c40 where a = 2), + c42 as (select a, q from c41 where a = 1 union select q, a from c41 where a = 2), + c43 as (select a, q from c42 where a = 1 union select q, a from c42 where a = 2), + c44 as (select a, q from c43 where a = 1 union select q, a from c43 where a = 2), + c45 as (select a, q from c44 where a = 1 union select q, a from c44 where a = 2), + c46 as (select a, q from c45 where a = 1 union select q, a from c45 where a = 2), + c47 as (select a, q from c46 where a = 1 union select q, a from c46 where a = 2), + c48 as (select a, q from c47 where a = 1 union select q, a from c47 where a = 2), + c49 as (select a, q from c48 where a = 1 union select q, a from c48 where a = 2), + c50 as (select a, q from c49 where a = 1 union select q, a from c49 where a = 2), + c51 as (select a, q from c50 where a = 1 union select q, a from c50 where a = 2), + c52 as (select a, q from c51 where a = 1 union select q, a from c51 where a = 2), + c53 as (select a, q from c52 where a = 1 union select q, a from c52 where a = 2), + c54 as (select a, q from c53 where a = 1 union select q, a from c53 where a = 2), + c55 as (select a, q from c54 where a = 1 union select q, a from c54 where a = 2), + c56 as (select a, q from c55 where a = 1 union select q, a from c55 where a = 2), + c57 as (select a, q from c56 where a = 1 union select q, a from c56 where a = 2), + c58 as (select a, q from c57 where a = 1 union select q, a from c57 where a = 2), + c59 as (select a, q from c58 where a = 1 union select q, a from c58 where a = 2), + c60 as (select a, q from c59 where a = 1 union select q, a from c59 where a = 2), + c61 as (select a, q from c60 where a = 1 union select q, a from c60 where a = 2), + c62 as (select a, q from c61 where a = 1 union select q, a from c61 where a = 2), + c63 as (select a, q from c62 where a = 1 union select q, a from c62 where a = 2), + c64 as (select a, q from c63 where a = 1 union select q, a from c63 where a = 2), + c65 as (select a, q from c64 where a = 1 union select q, a from c64 where a = 2), + c66 as (select a, q from c65 where a = 1 union select q, a from c65 where a = 2), + c67 as (select a, q from c66 where a = 1 union select q, a from c66 where a = 2), + c68 as (select a, q from c67 where a = 1 union select q, a from c67 where a = 2), + c69 as (select a, q from c68 where a = 1 union select q, a from c68 where a = 2), + c70 as (select a, q from c69 where a = 1 union select q, a from c69 where a = 2) +select * from c70 diff --git a/test/testdrive-old-kafka-src-syntax/top-1-monotonic.td b/test/testdrive-old-kafka-src-syntax/top-1-monotonic.td index 203471e5eada5..c05dd0cc6c4c8 100644 --- a/test/testdrive-old-kafka-src-syntax/top-1-monotonic.td +++ b/test/testdrive-old-kafka-src-syntax/top-1-monotonic.td @@ -314,3 +314,35 @@ $ kafka-ingest format=avro topic=top1 schema=${schema} timestamp=5 "Dataflow: order_by_desc_limit" MonotonicTop1 "Dataflow: order_by_limit" "Arranged MonotonicTop1 partial [val: empty]" "Dataflow: order_by_limit" MonotonicTop1 +"Dataflow: group_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_in_top_1" "Arrange errors" +"Dataflow: group_by_in_top_1" "Distinct errors" +"Dataflow: group_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: group_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: group_by_limit" "Arrange bundle err" +"Dataflow: group_by_limit" "Distinct errors[[]]" +"Dataflow: group_by_order_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_order_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_order_by_in_top_1" "Arrange errors" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: limit_only" "Arrange bundle err" +"Dataflow: limit_only" "Distinct errors[[]]" +"Dataflow: order_by_desc_limit" "Arrange bundle err" +"Dataflow: order_by_desc_limit" "Distinct errors[[]]" +"Dataflow: order_by_limit" "Arrange bundle err" +"Dataflow: order_by_limit" "Distinct errors[[]]" +"Dataflow: group_by_in_top_1" "Arrange errors" +"Dataflow: group_by_in_top_1" "Distinct errors" +"Dataflow: group_by_limit" "Arrange errors" +"Dataflow: group_by_limit" "Distinct errors" +"Dataflow: group_by_order_by_in_top_1" "Arrange errors" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors" +"Dataflow: limit_only" "Arrange errors" +"Dataflow: limit_only" "Distinct errors" +"Dataflow: order_by_desc_limit" "Arrange errors" +"Dataflow: order_by_desc_limit" "Distinct errors" +"Dataflow: order_by_limit" "Arrange errors" +"Dataflow: order_by_limit" "Distinct errors" diff --git a/test/testdrive/introspection-sources.td b/test/testdrive/introspection-sources.td index 1300e662974f6..d836ae00ba8d4 100644 --- a/test/testdrive/introspection-sources.td +++ b/test/testdrive/introspection-sources.td @@ -489,7 +489,7 @@ mv_div_by_zero 1 JOIN mz_objects o ON (c.export_id = o.id) ORDER BY name idx1_div_by_zero 3 -idx2_div_by_zero 3 +idx2_div_by_zero 1 mv_div_by_zero 3 > DELETE FROM zeros @@ -508,7 +508,7 @@ mv_div_by_zero 3 JOIN mz_objects o ON (c.export_id = o.id) ORDER BY name idx1_div_by_zero 2 -idx2_div_by_zero 2 +idx2_div_by_zero 1 mv_div_by_zero 2 > DROP INDEX idx1_div_by_zero @@ -517,7 +517,7 @@ mv_div_by_zero 2 FROM mz_introspection.mz_compute_error_counts c JOIN mz_objects o ON (c.export_id = o.id) ORDER BY name -idx2_div_by_zero 2 +idx2_div_by_zero 1 mv_div_by_zero 2 > DROP MATERIALIZED VIEW mv_div_by_zero diff --git a/test/testdrive/kafka-avro-upsert-sinks.td b/test/testdrive/kafka-avro-upsert-sinks.td index b7d7b347a3393..c2bf9965ce7a2 100644 --- a/test/testdrive/kafka-avro-upsert-sinks.td +++ b/test/testdrive/kafka-avro-upsert-sinks.td @@ -447,3 +447,9 @@ contains:upsert key could not be validated as unique "Dataflow: materialize.public.input_with_deletions_keyed" "Reduced Fallibly MinsMaxesHierarchical" "Dataflow: materialize.public.input_with_deletions_keyed" "Reduced Fallibly MinsMaxesHierarchical" "Dataflow: materialize.public.input_with_deletions_keyed" "Reduced Fallibly MinsMaxesHierarchical" +"Dataflow: materialize.public.input_keyed" "Arrange errors" +"Dataflow: materialize.public.input_keyed" "Distinct errors" +"Dataflow: materialize.public.input_keyed_ab" "Arrange errors" +"Dataflow: materialize.public.input_keyed_ab" "Distinct errors" +"Dataflow: materialize.public.input_with_deletions_keyed" "Arrange errors" +"Dataflow: materialize.public.input_with_deletions_keyed" "Distinct errors" diff --git a/test/testdrive/materializations.td b/test/testdrive/materializations.td index 8fe17b5c20289..38280d337ba54 100644 --- a/test/testdrive/materializations.td +++ b/test/testdrive/materializations.td @@ -496,3 +496,15 @@ c d "Dataflow: materialize.public.test7" AccumulableErrorCheck "Dataflow: materialize.public.test7" "ArrangeAccumulable [val: empty]" "Dataflow: materialize.public.test7" ReduceAccumulable +"Dataflow: materialize.public.test1" "Arrange errors" +"Dataflow: materialize.public.test1" "Distinct errors" +"Dataflow: materialize.public.test2" "Arrange errors" +"Dataflow: materialize.public.test2" "Distinct errors" +"Dataflow: materialize.public.test4" "Arrange errors" +"Dataflow: materialize.public.test4" "Distinct errors" +"Dataflow: materialize.public.test6" "Arrange errors" +"Dataflow: materialize.public.test6" "Distinct errors" +"Dataflow: materialize.public.test7" "Arrange bundle err" +"Dataflow: materialize.public.test7" "Arrange errors" +"Dataflow: materialize.public.test7" "Distinct errors" +"Dataflow: materialize.public.test7" "Distinct errors[[]]" diff --git a/test/testdrive/monotonic.td b/test/testdrive/monotonic.td index 7428d3de28feb..eea687c6ea778 100644 --- a/test/testdrive/monotonic.td +++ b/test/testdrive/monotonic.td @@ -243,3 +243,11 @@ Target cluster: quickstart "Dataflow: materialize.public.i8_primary_idx" "ArrangeBy[[Column(0, \"b\")]]-errors" "Dataflow: materialize.public.i9_primary_idx" "ArrangeBy[[Column(0, \"b\")]]" "Dataflow: materialize.public.i9_primary_idx" "ArrangeBy[[Column(0, \"b\")]]-errors" +"Dataflow: materialize.public.m1" "Arrange errors" +"Dataflow: materialize.public.m1" "Distinct errors" +"Dataflow: materialize.public.monotonic_fused" "Arrange errors" +"Dataflow: materialize.public.monotonic_fused" "Distinct errors" +"Dataflow: materialize.public.monotonic_max" "Arrange errors" +"Dataflow: materialize.public.monotonic_max" "Distinct errors" +"Dataflow: materialize.public.monotonic_min" "Arrange errors" +"Dataflow: materialize.public.monotonic_min" "Distinct errors" diff --git a/test/testdrive/top-1-monotonic.td b/test/testdrive/top-1-monotonic.td index 13eecd6fdb4bc..ddd2d4376bfc9 100644 --- a/test/testdrive/top-1-monotonic.td +++ b/test/testdrive/top-1-monotonic.td @@ -311,3 +311,35 @@ $ kafka-ingest format=avro topic=top1 schema=${schema} timestamp=5 "Dataflow: group_by_order_by_in_top_1" "Arranged DistinctBy" "Dataflow: group_by_order_by_in_top_1" DistinctBy "Dataflow: group_by_order_by_in_top_1" DistinctByErrorCheck +"Dataflow: group_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_in_top_1" "Arrange errors" +"Dataflow: group_by_in_top_1" "Distinct errors" +"Dataflow: group_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: group_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: group_by_limit" "Arrange bundle err" +"Dataflow: group_by_limit" "Distinct errors[[]]" +"Dataflow: group_by_order_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_order_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_order_by_in_top_1" "Arrange errors" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: limit_only" "Arrange bundle err" +"Dataflow: limit_only" "Distinct errors[[]]" +"Dataflow: order_by_desc_limit" "Arrange bundle err" +"Dataflow: order_by_desc_limit" "Distinct errors[[]]" +"Dataflow: order_by_limit" "Arrange bundle err" +"Dataflow: order_by_limit" "Distinct errors[[]]" +"Dataflow: group_by_in_top_1" "Arrange errors" +"Dataflow: group_by_in_top_1" "Distinct errors" +"Dataflow: group_by_limit" "Arrange errors" +"Dataflow: group_by_limit" "Distinct errors" +"Dataflow: group_by_order_by_in_top_1" "Arrange errors" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors" +"Dataflow: limit_only" "Arrange errors" +"Dataflow: limit_only" "Distinct errors" +"Dataflow: order_by_desc_limit" "Arrange errors" +"Dataflow: order_by_desc_limit" "Distinct errors" +"Dataflow: order_by_limit" "Arrange errors" +"Dataflow: order_by_limit" "Distinct errors" diff --git a/test/testdrive/unified-compute-introspection.td b/test/testdrive/unified-compute-introspection.td index 9e23821a8caec..8b8ae48ba5017 100644 --- a/test/testdrive/unified-compute-introspection.td +++ b/test/testdrive/unified-compute-introspection.td @@ -52,7 +52,7 @@ mv_div_by_zero r1 1 JOIN mz_cluster_replicas r ON c.replica_id = r.id WHERE c.count != 0 idx1_div_by_zero r1 3 -idx2_div_by_zero r1 3 +idx2_div_by_zero r1 1 mv_div_by_zero r1 3 > DELETE FROM zeros @@ -73,7 +73,7 @@ mv_div_by_zero r1 3 JOIN mz_cluster_replicas r ON c.replica_id = r.id WHERE c.count != 0 idx1_div_by_zero r1 2 -idx2_div_by_zero r1 2 +idx2_div_by_zero r1 1 mv_div_by_zero r1 2 > ALTER CLUSTER test SET (REPLICATION FACTOR 2) @@ -85,8 +85,8 @@ mv_div_by_zero r1 2 WHERE c.count != 0 idx1_div_by_zero r1 2 idx1_div_by_zero r2 2 -idx2_div_by_zero r1 2 -idx2_div_by_zero r2 2 +idx2_div_by_zero r1 1 +idx2_div_by_zero r2 1 mv_div_by_zero r1 2 mv_div_by_zero r2 2 @@ -100,7 +100,7 @@ mv_div_by_zero r2 2 JOIN mz_cluster_replicas r ON c.replica_id = r.id WHERE c.count != 0 idx1_div_by_zero r1 2 -idx2_div_by_zero r1 2 +idx2_div_by_zero r1 1 mv_div_by_zero r1 2 # Test that error logging is retracted when objects are dropped. @@ -112,7 +112,7 @@ mv_div_by_zero r1 2 JOIN mz_objects o ON c.object_id = o.id JOIN mz_cluster_replicas r ON c.replica_id = r.id WHERE c.count != 0 -idx2_div_by_zero r1 2 +idx2_div_by_zero r1 1 mv_div_by_zero r1 2 > DROP MATERIALIZED VIEW mv_div_by_zero From 7e4fee33d0ef10fd64b3f2f8b83bfc1339e0014c Mon Sep 17 00:00:00 2001 From: Sang Jun Bak Date: Wed, 19 Aug 2026 16:34:12 -0400 Subject: [PATCH 07/10] SQL-500: convert mz_tables and mz_views to materialized views (#38152) This PR is meant to be merged with everything else in the stack and is split for review purposes ### Motivation sql-150 ### Description ## catalog: convert mz_tables and mz_views to materialized views - With temporary items durable in the catalog shard, `mz_tables` and `mz_views` can be derived from `mz_internal.mz_catalog_raw`. Both show every item including temporary ones, matching the previous builtin tables. Temporary rows keep the temporary schema sentinel `"0"` in `schema_id`. - Builtin tables and views are reported through two new generated constant views, `mz_internal.mz_builtin_tables` and `mz_internal.mz_builtin_views`, following the `mz_builtin_materialized_views` pattern. - The generated `mz_builtin_*` reporter views are listed in `mz_views` with a short placeholder instead of their real SQL. `mz_builtin_views` cannot textually contain itself, and the other reporters embed metadata about every builtin object, so re-embedding their SQL would produce enormous rows. - A deliberate cosmetic delta: builtin view rows' `create_sql`/`redacted_create_sql` used to be the raw in-memory text (`CREATE VIEW mz_catalog.x AS ...`, unquoted idents) and are now stable reprints (quoted idents). `definition` is unchanged either way. - `parse_catalog_create_sql` now exposes `'definition'` for views and `'source_id'` for tables created from sources. ## test: update tests for mz_tables/mz_views as materialized views Updates various tests to accommodate the new builtin objects, and updates the descriptions of other tests related to temporary objects. ## expr: test parse_catalog_create_sql `mz_tables` and `mz_views` select rows by `parse_catalog_create_sql(...)->>'type'` and read `'definition'` and `'source_id'` out of the same call, but the function had no tests. Pins: - the reported type for every statement kind an Item record can hold - the exact `mz_views.definition` rendering (which `pg_views` exposes and which moved here out of the deleted `pack_view_update`) - its idempotence under re-parsing - `source_id` presence for `CREATE TABLE FROM SOURCE` and absence for plain and webhook tables - the four error paths The error paths matter more than they used to: the MVs call this inside their `WHERE` clause, so an item the parser rejects makes the whole relation unreadable rather than breaking one row's packing. ## test: add mz_tables and mz_views lockdown slts `mz_tables` and `mz_views` became `BuiltinMaterializedView`s over `mz_internal.mz_catalog_raw`, deriving every column from durable catalog JSON, but the conversion added no test file. Every earlier conversion (`mz_indexes`, `mz_audit_events`, `mz_postgres_sources`) got one. Follows the established lockdown shape: one section per union branch, plus the temporary-item sentinel schema id, the exactly-once property across the user and builtin branches, and the `ASSERT NOT NULL` columns. Two checks are independent of the MV rather than golden values: - `oid` is compared against a `regclass` cast, which resolves through the in-memory catalog - `mz_views.definition` is compared against the definition of a view planned from it, so the rendering is verified to be a fixed point end to end `create_sql` cannot be compared against `SHOW CREATE`, which humanizes item ids and pretty-prints by design. Also pins the one known difference from the old builtin tables: the generated `mz_builtin_*` reporter views appear in `mz_views` with a placeholder definition, since `mz_builtin_views` cannot textually contain itself and the other reporters' SQL would be enormous. ### Verification - Added SLTs and jsonb tests following the other builtin table -> mv conversions --------- Co-authored-by: Claude Opus 5 (1M context) --- .../reference/system-catalog/mz_catalog.md | 5 + .../reference/system-catalog/mz_internal.md | 2 + .../checks/all_checks/builtin_version_pin.py | 4 +- .../src/catalog/builtin_table_updates.rs | 126 +-------- .../catalog/open/builtin_schema_migration.rs | 17 ++ src/catalog/src/builtin.rs | 4 +- src/catalog/src/builtin/builtin.rs | 173 +++++++++++- src/catalog/src/builtin/mz_catalog.rs | 142 +++++++++- src/expr/src/scalar/func/impls/jsonb.rs | 260 ++++++++++++++++- src/pgrepr-consts/src/oid.rs | 6 +- test/0dt/mzcompose.py | 8 +- test/cluster/mzcompose.py | 8 +- test/sqllogictest/alter.slt | 4 +- .../autogenerated/mz_internal.slt | 2 + test/sqllogictest/catalog_server_explain.slt | 128 ++++++++- .../information_schema_tables.slt | 12 +- .../mz_catalog_server_index_accounting.slt | 172 +++++------ test/sqllogictest/mz_tables.slt | 223 +++++++++++++++ test/sqllogictest/mz_views.slt | 266 ++++++++++++++++++ test/sqllogictest/object_ownership.slt | 2 +- test/sqllogictest/oid.slt | 2 + .../show_create_system_objects.slt | 4 +- test/testdrive-old-kafka-src-syntax/update.td | 2 +- test/testdrive/catalog.td | 8 +- test/testdrive/temporary.td | 14 +- test/testdrive/update.td | 2 +- 26 files changed, 1347 insertions(+), 249 deletions(-) create mode 100644 test/sqllogictest/mz_tables.slt create mode 100644 test/sqllogictest/mz_views.slt diff --git a/doc/user/content/reference/system-catalog/mz_catalog.md b/doc/user/content/reference/system-catalog/mz_catalog.md index 8a29ab72e09d4..199e4d47f6222 100644 --- a/doc/user/content/reference/system-catalog/mz_catalog.md +++ b/doc/user/content/reference/system-catalog/mz_catalog.md @@ -671,6 +671,11 @@ Field | Type | Meaning The `mz_views` table contains a row for each view in the system. +A few generated system views in `mz_internal` (the `mz_builtin_*` views, +which list every builtin object) are shown with a short placeholder in +`definition`, `create_sql`, and `redacted_create_sql` instead of their full +SQL, which would embed metadata about every builtin object. + Field | Type | Meaning ---------------|----------------------|---------- diff --git a/doc/user/content/reference/system-catalog/mz_internal.md b/doc/user/content/reference/system-catalog/mz_internal.md index e16c77b2d1bf0..65dcf69958410 100644 --- a/doc/user/content/reference/system-catalog/mz_internal.md +++ b/doc/user/content/reference/system-catalog/mz_internal.md @@ -1516,6 +1516,8 @@ The `mz_webhook_sources` table contains a row for each webhook source in the sys + + diff --git a/misc/python/materialize/checks/all_checks/builtin_version_pin.py b/misc/python/materialize/checks/all_checks/builtin_version_pin.py index 081bf28175519..407a4cd46deee 100644 --- a/misc/python/materialize/checks/all_checks/builtin_version_pin.py +++ b/misc/python/materialize/checks/all_checks/builtin_version_pin.py @@ -57,9 +57,9 @@ "mz_sinks": MzVersion.parse_mz("v26.38.0"), "mz_ssh_tunnel_connections": MzVersion.parse_mz("v26.37.0"), "mz_system_privileges": MzVersion.parse_mz("v26.30.0"), - "mz_tables": None, + "mz_tables": MzVersion.parse_mz("v26.39.0"), "mz_types": None, - "mz_views": None, + "mz_views": MzVersion.parse_mz("v26.39.0"), } diff --git a/src/adapter/src/catalog/builtin_table_updates.rs b/src/adapter/src/catalog/builtin_table_updates.rs index 20670dc4b11e7..61e404ceb6f01 100644 --- a/src/adapter/src/catalog/builtin_table_updates.rs +++ b/src/adapter/src/catalog/builtin_table_updates.rs @@ -20,14 +20,14 @@ use mz_catalog::builtin::{ MZ_HISTORY_RETENTION_STRATEGIES, MZ_INDEX_COLUMNS, MZ_LICENSE_KEYS, MZ_LIST_TYPES, MZ_MAP_TYPES, MZ_MATERIALIZED_VIEW_REFRESH_STRATEGIES, MZ_OBJECT_DEPENDENCIES, MZ_OBJECT_GLOBAL_IDS, MZ_OPERATORS, MZ_PSEUDO_TYPES, MZ_REPLACEMENTS, MZ_ROLE_AUTH, - MZ_SESSIONS, MZ_SOURCE_REFERENCES, MZ_STORAGE_USAGE_BY_SHARD, MZ_SUBSCRIPTIONS, MZ_TABLES, - MZ_TYPE_PG_METADATA, MZ_TYPES, MZ_VIEWS, MZ_WEBHOOKS_SOURCES, + MZ_SESSIONS, MZ_SOURCE_REFERENCES, MZ_STORAGE_USAGE_BY_SHARD, MZ_SUBSCRIPTIONS, + MZ_TYPE_PG_METADATA, MZ_TYPES, MZ_WEBHOOKS_SOURCES, }; use mz_catalog::durable::SourceReferences; use mz_catalog::memory::error::Error; use mz_catalog::memory::objects::{ CatalogEntry, CatalogItem, DataSourceDesc, Func, Index, MaterializedView, Table, - TableDataSource, Type, View, + TableDataSource, Type, }; use mz_expr::MirScalarExpr; use mz_license_keys::ValidatedLicenseKey; @@ -162,13 +162,6 @@ impl CatalogState { let privileges = privileges_row.unpack_first(); let mut updates = match entry.item() { CatalogItem::Index(index) => self.pack_index_update(id, index, diff), - CatalogItem::Table(table) => { - // Source-table metadata (mz_postgres/mysql/sql_server/kafka_source_tables) - // is now derived from the persisted create_sql by materialized - // views over mz_catalog_raw, so ingestion-export tables need no - // special packing here. - self.pack_table_update(id, oid, schema_id, name, owner_id, privileges, diff, table) - } CatalogItem::Source(source) => { match &source.data_source { DataSourceDesc::Webhook { .. } => { @@ -185,9 +178,6 @@ impl CatalogState { | DataSourceDesc::Catalog => vec![], } } - CatalogItem::View(view) => { - self.pack_view_update(id, oid, schema_id, name, owner_id, privileges, view, diff) - } CatalogItem::MaterializedView(mview) => { self.pack_materialized_view_update(id, mview, diff) } @@ -208,7 +198,11 @@ impl CatalogState { // of it, so once metric sinks are user-creatable this gap would let a reader // egress another role's rows with no catalog relation the owner could see it in. // SQL-572 has to land before user-facing metric sink DDL does. - CatalogItem::Log(_) | CatalogItem::Secret(_) | CatalogItem::MetricSink(_) => vec![], + CatalogItem::Table(_) + | CatalogItem::View(_) + | CatalogItem::Log(_) + | CatalogItem::Secret(_) + | CatalogItem::MetricSink(_) => vec![], // Connection details (mz_kafka_connections, mz_ssh_tunnel_connections, // mz_aws_connections, mz_aws_privatelink_connections) are now derived // from the persisted create_sql by materialized views over @@ -337,110 +331,6 @@ impl CatalogState { ) } - fn pack_table_update( - &self, - id: CatalogItemId, - oid: u32, - schema_id: &SchemaSpecifier, - name: &str, - owner_id: &RoleId, - privileges: Datum, - diff: Diff, - table: &Table, - ) -> Vec> { - let redacted = table.create_sql.as_ref().map(|create_sql| { - mz_sql::parse::parse(create_sql) - .unwrap_or_else(|_| panic!("create_sql cannot be invalid: {}", create_sql)) - .into_element() - .ast - .to_ast_string_redacted() - }); - let source_id = if let TableDataSource::DataSource { - desc: DataSourceDesc::IngestionExport { ingestion_id, .. }, - .. - } = &table.data_source - { - Some(ingestion_id.to_string()) - } else { - None - }; - - vec![BuiltinTableUpdate::row( - &*MZ_TABLES, - Row::pack_slice(&[ - Datum::String(&id.to_string()), - Datum::UInt32(oid), - Datum::String(&schema_id.to_string()), - Datum::String(name), - Datum::String(&owner_id.to_string()), - privileges, - if let Some(create_sql) = &table.create_sql { - Datum::String(create_sql) - } else { - Datum::Null - }, - if let Some(redacted) = &redacted { - Datum::String(redacted) - } else { - Datum::Null - }, - if let Some(source_id) = source_id.as_ref() { - Datum::String(source_id) - } else { - Datum::Null - }, - ]), - diff, - )] - } - - fn pack_view_update( - &self, - id: CatalogItemId, - oid: u32, - schema_id: &SchemaSpecifier, - name: &str, - owner_id: &RoleId, - privileges: Datum, - view: &View, - diff: Diff, - ) -> Vec> { - let create_stmt = mz_sql::parse::parse(&view.create_sql) - .unwrap_or_else(|e| { - panic!( - "create_sql cannot be invalid: `{}` --- error: `{}`", - view.create_sql, e - ) - }) - .into_element() - .ast; - let query = match &create_stmt { - Statement::CreateView(stmt) => &stmt.definition.query, - _ => unreachable!(), - }; - - let mut query_string = query.to_ast_string_stable(); - // PostgreSQL appends a semicolon in `pg_views.definition`, we - // do the same for compatibility's sake. - query_string.push(';'); - - vec![BuiltinTableUpdate::row( - &*MZ_VIEWS, - Row::pack_slice(&[ - Datum::String(&id.to_string()), - Datum::UInt32(oid), - Datum::String(&schema_id.to_string()), - Datum::String(name), - Datum::String(&query_string), - Datum::String(&owner_id.to_string()), - privileges, - Datum::String(&view.create_sql), - Datum::String(&create_stmt.to_ast_string_redacted()), - ]), - diff, - )] - } - fn pack_materialized_view_update( &self, id: CatalogItemId, diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index 823fe46dea9a6..8af544edca8b5 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -399,6 +399,23 @@ static MIGRATIONS: LazyLock> = LazyLock::new(|| { MZ_CATALOG_SCHEMA, "mz_indexes", ), + // Converting mz_tables and mz_views from builtin tables to + // materialized views over mz_catalog_raw changes their catalog + // fingerprints, so each needs an explicit replacement step. See the + // NOTE above: this version must stay at the workspace's current dev + // version until the change ships. + MigrationStep::replacement( + "26.39.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_tables", + ), + MigrationStep::replacement( + "26.39.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_views", + ), ] }); diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs index 0a1ec9ce80086..e7eeb0ccd017f 100644 --- a/src/catalog/src/builtin.rs +++ b/src/catalog/src/builtin.rs @@ -1117,7 +1117,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::Table(&MZ_COLUMNS), // mz_indexes is generated dynamically below with inlined builtin VALUES. Builtin::Table(&MZ_INDEX_COLUMNS), - Builtin::Table(&MZ_TABLES), + Builtin::MaterializedView(&MZ_TABLES), // mz_sources is generated dynamically below with inlined builtin VALUES. Builtin::Table(&MZ_SOURCE_REFERENCES), Builtin::MaterializedView(&MZ_POSTGRES_SOURCES), @@ -1126,7 +1126,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::MaterializedView(&MZ_SQL_SERVER_SOURCE_TABLES), Builtin::MaterializedView(&MZ_KAFKA_SOURCE_TABLES), Builtin::MaterializedView(&MZ_SINKS), - Builtin::Table(&MZ_VIEWS), + Builtin::MaterializedView(&MZ_VIEWS), Builtin::Table(&MZ_TYPES), Builtin::Table(&MZ_TYPE_PG_METADATA), Builtin::Table(&MZ_ARRAY_TYPES), diff --git a/src/catalog/src/builtin/builtin.rs b/src/catalog/src/builtin/builtin.rs index e62910c4ab1fe..da7b1cee76400 100644 --- a/src/catalog/src/builtin/builtin.rs +++ b/src/catalog/src/builtin/builtin.rs @@ -25,8 +25,8 @@ use mz_sql::rbac; use mz_sql::session::user::MZ_SYSTEM_ROLE_ID; use crate::builtin::{ - Builtin, BuiltinLog, BuiltinMaterializedView, BuiltinSource, BuiltinView, Cardinality, - LinkProperties, Ontology, OntologyLink, PUBLIC_SELECT, + Builtin, BuiltinLog, BuiltinMaterializedView, BuiltinSource, BuiltinTable, BuiltinView, + Cardinality, LinkProperties, Ontology, OntologyLink, PUBLIC_SELECT, }; /// Generate builtin views reporting the given builtins. @@ -47,14 +47,32 @@ pub(super) fn builtins( Builtin::MaterializedView(x) => Some(*x), _ => None, }); + let table_iter = builtin_items.iter().filter_map(|b| match b { + Builtin::Table(x) => Some(*x), + _ => None, + }); + + let sources: &'static BuiltinView = + Box::leak(Box::new(make_builtin_sources(source_iter, log_iter))); + let materialized_views: &'static BuiltinView = + Box::leak(Box::new(make_builtin_materialized_views(mv_iter))); + let tables: &'static BuiltinView = Box::leak(Box::new(make_builtin_tables(table_iter))); - let sources = make_builtin_sources(source_iter, log_iter); - let materialized_views = make_builtin_materialized_views(mv_iter); + // The generated views above, and `mz_builtin_views` itself, are listed in + // `mz_builtin_views` with placeholder SQL rather than their real + // definitions. See `make_builtin_views`. + let view_iter = builtin_items.iter().filter_map(|b| match b { + Builtin::View(x) => Some(*x), + _ => None, + }); + let views: &'static BuiltinView = Box::leak(Box::new(make_builtin_views( + view_iter, + [sources, materialized_views, tables], + ))); - [sources, materialized_views].into_iter().map(|v| { - let static_ref = Box::leak(Box::new(v)); - Builtin::View(static_ref) - }) + [sources, materialized_views, tables, views] + .into_iter() + .map(Builtin::View) } fn make_builtin_sources( @@ -170,6 +188,145 @@ FROM (VALUES {values}) AS v(oid, schema_name, name, cluster_name, definition, pr } } +fn make_builtin_tables(iter: impl Iterator) -> BuiltinView { + let owner_priv = rbac::owner_privilege(ObjectType::Table, MZ_SYSTEM_ROLE_ID); + let values = iter + .map(|table| { + let schema = escaped_string_literal(table.schema); + let name = escaped_string_literal(table.name); + let privileges = make_privileges_sql(&table.access, &owner_priv); + format!("({}::oid, {}, {}, {})", table.oid, schema, name, privileges) + }) + .join(","); + let sql = format!( + " +SELECT oid, schema_name, name, privileges +FROM (VALUES {values}) AS v(oid, schema_name, name, privileges)" + ); + + BuiltinView { + name: "mz_builtin_tables", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::VIEW_MZ_BUILTIN_TABLES_OID, + desc: RelationDesc::builder() + .with_column("oid", SqlScalarType::Oid.nullable(false)) + .with_column("schema_name", SqlScalarType::String.nullable(false)) + .with_column("name", SqlScalarType::String.nullable(false)) + .with_column( + "privileges", + SqlScalarType::Array(Box::new(SqlScalarType::MzAclItem)).nullable(false), + ) + // NOTE: The declared keys must exactly match the keys the + // optimizer derives from the generated VALUES list + // (`verify_builtin_descs` enforces this). Table names happen to + // be unique across builtin schemas today, so `name` is a key. If + // a table is ever added whose bare name collides with another + // schema's, drop the `name` key here. + .with_key(vec![0]) + .with_key(vec![2]) + .finish(), + column_comments: Default::default(), + sql: Box::leak(sql.into_boxed_str()), + access: vec![PUBLIC_SELECT], + ontology: None, + } +} + +/// Generates `mz_internal.mz_builtin_views`, listing every builtin view, +/// including itself and the `generated` views. +/// +/// Views from `iter` are listed with their real definition and create SQL. +/// The generated views are instead listed with a short placeholder query. +/// Real SQL is impossible for `mz_builtin_views` itself, its definition would +/// have to contain its own text. It is impractical for the other generated +/// views, whose SQL embeds metadata about every builtin object. +/// `mz_builtin_materialized_views` for example carries the SQL of every +/// builtin materialized view, so re-embedding its definition here would +/// produce enormous rows that make `SELECT * FROM mz_views` unusable. +/// +/// The placeholder is a valid SQL statement, because `mz_views` applies +/// `mz_internal.redact_sql` to the `create_sql` column and that function +/// errors on unparseable input, which would poison the whole materialized +/// view. The placeholder also embeds the view's qualified name so that the +/// `definition` and `create_sql` columns stay unique across rows, which the +/// declared keys rely on. +fn make_builtin_views<'a>( + iter: impl Iterator, + generated: [&BuiltinView; 3], +) -> BuiltinView { + let owner_priv = rbac::owner_privilege(ObjectType::View, MZ_SYSTEM_ROLE_ID); + + let make_row = |oid: u32, schema: &str, name: &str, access: &[MzAclItem], create_sql: &str| { + let stmt = mz_sql::parse::parse(create_sql) + .expect("valid sql") + .into_element() + .ast; + let Statement::CreateView(stmt) = stmt else { + panic!("invalid builtin view SQL"); + }; + + let definition = format!("{};", stmt.definition.query.to_ast_string_stable()); + let definition = escaped_string_literal(&definition); + let create_sql = stmt.to_ast_string_stable(); + let create_sql = escaped_string_literal(&create_sql); + + let schema = escaped_string_literal(schema); + let name = escaped_string_literal(name); + let privileges = make_privileges_sql(access, &owner_priv); + + format!( + "({}::oid, {}, {}, {}, {}, {})", + oid, schema, name, definition, privileges, create_sql + ) + }; + + let mut view = BuiltinView { + name: "mz_builtin_views", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::VIEW_MZ_BUILTIN_VIEWS_OID, + desc: RelationDesc::builder() + .with_column("oid", SqlScalarType::Oid.nullable(false)) + .with_column("schema_name", SqlScalarType::String.nullable(false)) + .with_column("name", SqlScalarType::String.nullable(false)) + .with_column("definition", SqlScalarType::String.nullable(false)) + .with_column( + "privileges", + SqlScalarType::Array(Box::new(SqlScalarType::MzAclItem)).nullable(false), + ) + .with_column("create_sql", SqlScalarType::String.nullable(false)) + // NOTE: The declared keys must exactly match the keys the + // optimizer derives from the generated VALUES list + // (`verify_builtin_descs` enforces this). + .with_key(vec![0]) + .with_key(vec![2]) + .with_key(vec![3]) + .with_key(vec![5]) + .finish(), + column_comments: Default::default(), + sql: "", + access: vec![PUBLIC_SELECT], + ontology: None, + }; + + let full_values = iter.map(|v| make_row(v.oid, v.schema, v.name, &v.access, &v.create_sql())); + let placeholder_values = generated.iter().copied().chain([&view]).map(|v| { + let create_sql = format!( + "CREATE VIEW {}.{} AS SELECT ''", + v.schema, v.name, v.schema, v.name + ); + make_row(v.oid, v.schema, v.name, &v.access, &create_sql) + }); + let values = full_values.chain(placeholder_values).join(","); + let sql = format!( + " +SELECT oid, schema_name, name, definition, privileges, create_sql +FROM (VALUES {values}) AS v(oid, schema_name, name, definition, privileges, create_sql)" + ); + + view.sql = Box::leak(sql.into_boxed_str()); + view +} + /// Convert the given list of [`MzAclItem`] to the equivalent SQL syntax. fn make_privileges_sql(privs: &[MzAclItem], owner_priv: &MzAclItem) -> String { let privs = privs.iter().chain_one(owner_priv); diff --git a/src/catalog/src/builtin/mz_catalog.rs b/src/catalog/src/builtin/mz_catalog.rs index 9dd7e103c439b..a67cd31832b1e 100644 --- a/src/catalog/src/builtin/mz_catalog.rs +++ b/src/catalog/src/builtin/mz_catalog.rs @@ -1018,10 +1018,11 @@ pub static MZ_INDEX_COLUMNS: LazyLock = LazyLock::new(|| BuiltinTa column_semantic_types: &[("index_id", SemanticType::CatalogItemId)], }), }); -pub static MZ_TABLES: LazyLock = LazyLock::new(|| BuiltinTable { +pub static MZ_TABLES: LazyLock = LazyLock::new(|| { + BuiltinMaterializedView { name: "mz_tables", schema: MZ_CATALOG_SCHEMA, - oid: oid::TABLE_MZ_TABLES_OID, + oid: oid::MV_MZ_TABLES_OID, desc: RelationDesc::builder() .with_column("id", SqlScalarType::String.nullable(false)) .with_column("oid", SqlScalarType::Oid.nullable(false)) @@ -1061,6 +1062,68 @@ pub static MZ_TABLES: LazyLock = LazyLock::new(|| BuiltinTable { "The ID of the source associated with the table, if any. Corresponds to `mz_sources.id`.", ), ]), + // Temporary tables live in a per-session temporary schema that has no + // durable schema record. Their rows keep the temporary schema sentinel + // "0" that the previous builtin-table implementation exposed. + sql: Box::leak(format!(" +IN CLUSTER mz_catalog_server +WITH ( + ASSERT NOT NULL id, + ASSERT NOT NULL oid, + ASSERT NOT NULL schema_id, + ASSERT NOT NULL name, + ASSERT NOT NULL owner_id, + ASSERT NOT NULL privileges +) AS +WITH + user_tables AS ( + SELECT + mz_internal.parse_catalog_id(data->'key'->'gid') AS id, + (data->'value'->>'oid')::oid AS oid, + CASE WHEN data->'value'->>'ephemeral_owner_session' IS NULL + THEN mz_internal.parse_catalog_id(data->'value'->'schema_id') + ELSE '0' + END AS schema_id, + data->'value'->>'name' AS name, + mz_internal.parse_catalog_id(data->'value'->'owner_id') AS owner_id, + mz_internal.parse_catalog_privileges(data->'value'->'privileges') AS privileges, + data->'value'->'definition'->'V1'->>'create_sql' AS create_sql, + mz_internal.redact_sql(data->'value'->'definition'->'V1'->>'create_sql') AS redacted_create_sql, + mz_internal.parse_catalog_create_sql(data->'value'->'definition'->'V1'->>'create_sql')->>'source_id' AS source_id + FROM mz_internal.mz_catalog_raw + WHERE + data->>'kind' = 'Item' AND + mz_internal.parse_catalog_create_sql(data->'value'->'definition'->'V1'->>'create_sql')->>'type' = 'table' + ), + builtin_mappings AS ( + SELECT + data->'key'->>'schema_name' AS schema_name, + data->'key'->>'object_name' AS name, + 's' || (data->'value'->>'catalog_id') AS id + FROM mz_internal.mz_catalog_raw + WHERE + data->>'kind' = 'GidMapping' AND + data->'key'->>'object_type' = '1' + ), + builtin_tables AS ( + SELECT + m.id, + t.oid, + s.id AS schema_id, + t.name, + '{MZ_SYSTEM_ROLE_ID}' AS owner_id, + t.privileges, + NULL::text AS create_sql, + NULL::text AS redacted_create_sql, + NULL::text AS source_id + FROM mz_internal.mz_builtin_tables t + JOIN builtin_mappings m USING (schema_name, name) + JOIN mz_schemas s ON s.name = t.schema_name + WHERE s.database_id IS NULL + ) +SELECT * FROM user_tables +UNION ALL +SELECT * FROM builtin_tables").into_boxed_str()), is_retained_metrics_object: true, access: vec![PUBLIC_SELECT], ontology: Some(Ontology { @@ -1101,6 +1164,7 @@ pub static MZ_TABLES: LazyLock = LazyLock::new(|| BuiltinTable { ] }, }), +} }); pub static MZ_CONNECTIONS: LazyLock = LazyLock::new(|| { @@ -1425,10 +1489,11 @@ WHERE }), } }); -pub static MZ_VIEWS: LazyLock = LazyLock::new(|| BuiltinTable { +pub static MZ_VIEWS: LazyLock = LazyLock::new(|| { + BuiltinMaterializedView { name: "mz_views", schema: MZ_CATALOG_SCHEMA, - oid: oid::TABLE_MZ_VIEWS_OID, + oid: oid::MV_MZ_VIEWS_OID, desc: RelationDesc::builder() .with_column("id", SqlScalarType::String.nullable(false)) .with_column("oid", SqlScalarType::Oid.nullable(false)) @@ -1465,6 +1530,74 @@ pub static MZ_VIEWS: LazyLock = LazyLock::new(|| BuiltinTable { "The redacted `CREATE` SQL statement for the view.", ), ]), + // Temporary views live in a per-session temporary schema that has no + // durable schema record. Their rows keep the temporary schema sentinel + // "0" that the previous builtin-table implementation exposed. + // + // The generated `mz_builtin_*` views appear here with placeholder + // definition and create SQL. See `make_builtin_views`. + sql: Box::leak(format!(" +IN CLUSTER mz_catalog_server +WITH ( + ASSERT NOT NULL id, + ASSERT NOT NULL oid, + ASSERT NOT NULL schema_id, + ASSERT NOT NULL name, + ASSERT NOT NULL definition, + ASSERT NOT NULL owner_id, + ASSERT NOT NULL privileges, + ASSERT NOT NULL create_sql, + ASSERT NOT NULL redacted_create_sql +) AS +WITH + user_views AS ( + SELECT + mz_internal.parse_catalog_id(data->'key'->'gid') AS id, + (data->'value'->>'oid')::oid AS oid, + CASE WHEN data->'value'->>'ephemeral_owner_session' IS NULL + THEN mz_internal.parse_catalog_id(data->'value'->'schema_id') + ELSE '0' + END AS schema_id, + data->'value'->>'name' AS name, + mz_internal.parse_catalog_create_sql(data->'value'->'definition'->'V1'->>'create_sql')->>'definition' AS definition, + mz_internal.parse_catalog_id(data->'value'->'owner_id') AS owner_id, + mz_internal.parse_catalog_privileges(data->'value'->'privileges') AS privileges, + data->'value'->'definition'->'V1'->>'create_sql' AS create_sql, + mz_internal.redact_sql(data->'value'->'definition'->'V1'->>'create_sql') AS redacted_create_sql + FROM mz_internal.mz_catalog_raw + WHERE + data->>'kind' = 'Item' AND + mz_internal.parse_catalog_create_sql(data->'value'->'definition'->'V1'->>'create_sql')->>'type' = 'view' + ), + builtin_mappings AS ( + SELECT + data->'key'->>'schema_name' AS schema_name, + data->'key'->>'object_name' AS name, + 's' || (data->'value'->>'catalog_id') AS id + FROM mz_internal.mz_catalog_raw + WHERE + data->>'kind' = 'GidMapping' AND + data->'key'->>'object_type' = '4' + ), + builtin_views AS ( + SELECT + m.id, + v.oid, + s.id AS schema_id, + v.name, + v.definition, + '{MZ_SYSTEM_ROLE_ID}' AS owner_id, + v.privileges, + v.create_sql, + mz_internal.redact_sql(v.create_sql) AS redacted_create_sql + FROM mz_internal.mz_builtin_views v + JOIN builtin_mappings m USING (schema_name, name) + JOIN mz_schemas s ON s.name = v.schema_name + WHERE s.database_id IS NULL + ) +SELECT * FROM user_views +UNION ALL +SELECT * FROM builtin_views").into_boxed_str()), is_retained_metrics_object: false, access: vec![PUBLIC_SELECT], ontology: Some(Ontology { @@ -1496,6 +1629,7 @@ pub static MZ_VIEWS: LazyLock = LazyLock::new(|| BuiltinTable { ] }, }), +} }); pub static MZ_MATERIALIZED_VIEWS: LazyLock = LazyLock::new(|| { diff --git a/src/expr/src/scalar/func/impls/jsonb.rs b/src/expr/src/scalar/func/impls/jsonb.rs index ba36ef77374e2..287027f3eedca 100644 --- a/src/expr/src/scalar/func/impls/jsonb.rs +++ b/src/expr/src/scalar/func/impls/jsonb.rs @@ -458,7 +458,15 @@ fn parse_catalog_create_sql<'a>(a: &'a str) -> Result { "connection" } - CreateView(_) => "view", + CreateView(stmt) => { + let mut definition = stmt.definition.query.to_ast_string_stable(); + // PostgreSQL appends a semicolon in `pg_views.definition`, we + // do the same for compatibility's sake. + definition.push(';'); + info.insert("definition", json!(definition)); + + "view" + } CreateMaterializedView(stmt) => { let Some(in_cluster) = stmt.in_cluster else { return Err("missing IN CLUSTER".into()); @@ -475,7 +483,13 @@ fn parse_catalog_create_sql<'a>(a: &'a str) -> Result { "materialized-view" } - CreateTable(_) | CreateTableFromSource(_) => "table", + CreateTable(_) => "table", + CreateTableFromSource(stmt) => { + let source_id = get_item_id(stmt.source)?; + info.insert("source_id", json!(source_id)); + + "table" + } CreateSource(stmt) => { let Some(in_cluster) = stmt.in_cluster else { return Err("missing IN CLUSTER".into()); @@ -2071,6 +2085,246 @@ mod tests { fn sink_arm_leaves_other_item_types_alone() { let sql = "CREATE VIEW \"materialize\".\"public\".\"v\" AS SELECT 1"; let out = super::parse_catalog_create_sql(sql).expect("ok"); - assert_eq!(as_serde(out), json!({ "type": "view" })); + assert_eq!( + as_serde(out), + json!({ "type": "view", "definition": "SELECT 1;" }) + ); + } + + // --- parse_catalog_create_sql -------------------------------------------- + + /// `type` for a `create_sql`, or the error message if parsing failed. + fn item_type(sql: &str) -> Result { + match super::parse_catalog_create_sql(sql) { + Ok(out) => match as_serde(out) { + serde_json::Value::Object(mut m) => match m.remove("type") { + Some(serde_json::Value::String(s)) => Ok(s), + other => panic!("no string `type` key: {other:?}"), + }, + other => panic!("not a JSON object: {other:?}"), + }, + Err(EvalError::InvalidCatalogJson(msg)) => Err(msg.to_string()), + Err(e) => panic!("unexpected error variant: {e:?}"), + } + } + + fn view_sql(query: &str) -> String { + format!("CREATE VIEW \"materialize\".\"public\".\"v\" AS {query}") + } + + /// `definition` for a `CREATE VIEW` whose query is `query`. + fn view_definition(query: &str) -> String { + match as_serde(super::parse_catalog_create_sql(&view_sql(query)).expect("ok")) { + serde_json::Value::Object(mut m) => match m.remove("definition") { + Some(serde_json::Value::String(s)) => s, + other => panic!("no string `definition` key: {other:?}"), + }, + other => panic!("not a JSON object: {other:?}"), + } + } + + /// `mz_tables` and `mz_views` select rows by + /// `parse_catalog_create_sql(...)->>'type'`, and the function runs over + /// every `Item` row in the catalog, so a statement kind that changes its + /// reported type silently gains or loses rows in those relations. Pin the + /// type of every kind the catalog can hold. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` + fn catalog_item_type_per_statement_kind() { + let cases = [ + ( + "CREATE TABLE \"materialize\".\"public\".\"t\" (a int4)", + "table", + ), + // A table created from a source, and a webhook table, are both + // `table`, so both land in mz_tables. + ( + "CREATE TABLE \"materialize\".\"public\".\"tbl\" \ + FROM SOURCE [u1 AS \"materialize\".\"public\".\"src\"] \ + (REFERENCE = \"topic\") FORMAT TEXT", + "table", + ), + ( + "CREATE TABLE \"materialize\".\"public\".\"wht\" FROM WEBHOOK BODY FORMAT JSON", + "table", + ), + ( + "CREATE VIEW \"materialize\".\"public\".\"v\" AS SELECT 1", + "view", + ), + ( + "CREATE MATERIALIZED VIEW \"materialize\".\"public\".\"mv\" \ + IN CLUSTER [u1] AS SELECT 1", + "materialized-view", + ), + ( + "CREATE SOURCE \"materialize\".\"public\".\"lg\" \ + IN CLUSTER [u1] FROM LOAD GENERATOR COUNTER", + "source", + ), + ( + "CREATE SOURCE \"materialize\".\"public\".\"wh\" \ + IN CLUSTER [u1] FROM WEBHOOK BODY FORMAT JSON", + "source", + ), + ( + "CREATE SUBSOURCE \"materialize\".\"public\".\"sub\" (id int4) \ + OF SOURCE [u1 AS \"materialize\".\"public\".\"src\"]", + "subsource", + ), + ( + "CREATE SUBSOURCE \"materialize\".\"public\".\"progress\" (id int4) \ + WITH (PROGRESS)", + "subsource", + ), + ( + "CREATE SINK \"materialize\".\"public\".\"snk\" IN CLUSTER [u1] \ + FROM [u1 AS \"materialize\".\"public\".\"t\"] \ + INTO KAFKA CONNECTION [u2 AS \"materialize\".\"public\".\"c\"] \ + (TOPIC 'tp') FORMAT JSON ENVELOPE DEBEZIUM", + "sink", + ), + ( + "CREATE INDEX \"i\" IN CLUSTER [u1] \ + ON [u1 AS \"materialize\".\"public\".\"t\"] (\"a\")", + "index", + ), + ( + "CREATE TYPE \"materialize\".\"public\".\"ty\" AS LIST (ELEMENT TYPE = int4)", + "type", + ), + ( + "CREATE SECRET \"materialize\".\"public\".\"s\" AS 'x'", + "secret", + ), + ( + "CREATE CONNECTION \"materialize\".\"public\".\"c\" \ + TO KAFKA (BROKER 'b', SECURITY PROTOCOL PLAINTEXT)", + "connection", + ), + ]; + for (sql, expected) in cases { + assert_eq!(item_type(sql).as_deref(), Ok(expected), "for {sql}"); + } + } + + /// `mz_views.definition` is produced here. It used to be produced by + /// `pack_view_update` in the adapter, so the exact rendering is a + /// compatibility surface: `pg_views.definition` reads it. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` + fn catalog_view_definition() { + // Identifiers and function names come back fully quoted, literals + // untouched, and PostgreSQL's trailing semicolon is appended. + assert_eq!(view_definition("SELECT 1"), "SELECT 1;"); + assert_eq!( + view_definition("WITH c AS (SELECT 1 AS a) SELECT a FROM c"), + "WITH \"c\" AS (SELECT 1 AS \"a\") SELECT \"a\" FROM \"c\";" + ); + assert_eq!( + view_definition("SELECT 1 UNION ALL SELECT 2"), + "SELECT 1 UNION ALL SELECT 2;" + ); + assert_eq!( + view_definition("SELECT (SELECT max(a) FROM [u1 AS \"materialize\".\"public\".\"t\"])"), + "SELECT (SELECT \"max\"(\"a\") FROM [u1 AS \"materialize\".\"public\".\"t\"]);" + ); + // Identifiers needing quotes, an embedded double quote, non-ASCII, an + // embedded single quote in a literal, and ORDER BY all survive. + assert_eq!( + view_definition( + "SELECT \"a b\", \"héllo\", \"q\"\"x\" \ + FROM [u1 AS \"materialize\".\"public\".\"t\"] \ + WHERE s = 'lit''eral' AND n = 42 ORDER BY 1" + ), + "SELECT \"a b\", \"héllo\", \"q\"\"x\" \ + FROM [u1 AS \"materialize\".\"public\".\"t\"] \ + WHERE \"s\" = 'lit''eral' AND \"n\" = 42 ORDER BY 1;" + ); + } + + /// The rendering must be a fixed point: `pg_views` consumers re-issue + /// `definition` as the body of a new view, so a second pass through the + /// parser has to produce the identical string. The trailing `;` is part of + /// what gets re-parsed. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` + fn catalog_view_definition_is_idempotent() { + for query in [ + "SELECT 1", + "WITH c AS (SELECT 1 AS a) SELECT a FROM c", + "SELECT 1 UNION ALL SELECT 2", + "SELECT \"a b\", \"q\"\"x\" FROM [u1 AS \"materialize\".\"public\".\"t\"] \ + WHERE s = 'lit''eral' ORDER BY 1", + ] { + let once = view_definition(query); + assert_eq!( + view_definition(&once), + once, + "not a fixed point for {query}" + ); + } + } + + /// `mz_tables.source_id` comes from this key. A table with no source must + /// omit it entirely, so the MV's `->>'source_id'` yields SQL NULL. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` + fn catalog_table_source_id() { + let from_source = as_serde( + super::parse_catalog_create_sql( + "CREATE TABLE \"materialize\".\"public\".\"tbl\" \ + FROM SOURCE [u1 AS \"materialize\".\"public\".\"src\"] \ + (REFERENCE = \"topic\") FORMAT TEXT", + ) + .expect("ok"), + ); + assert_eq!(from_source, json!({ "type": "table", "source_id": "u1" })); + + for sql in [ + "CREATE TABLE \"materialize\".\"public\".\"t\" (a int4)", + "CREATE TABLE \"materialize\".\"public\".\"wht\" FROM WEBHOOK BODY FORMAT JSON", + ] { + assert_eq!( + as_serde(super::parse_catalog_create_sql(sql).expect("ok")), + json!({ "type": "table" }), + "for {sql}" + ); + } + } + + /// Every error here is fatal to the whole of `mz_tables`/`mz_views`, not to + /// one row: the MVs call this function inside their `WHERE` clause, so an + /// item the parser rejects makes the relation unreadable for everyone. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` + fn catalog_create_sql_errors() { + assert_eq!( + item_type("this is not sql"), + Err( + "failed to parse create_sql: Expected a keyword at the beginning of a statement, \ + found identifier \"this\"" + .to_string() + ) + ); + assert_eq!( + item_type("CREATE TABLE t (a int4); CREATE TABLE u (b int4)"), + Err("expected a single statement, found 2".to_string()) + ); + // A statement that is not a CREATE of a catalog item, e.g. if a future + // change persists something else in an Item record. + assert_eq!( + item_type("SELECT 1"), + Err("not a CREATE item statement".to_string()) + ); + // Catalog `create_sql` always names items by id. An unresolved name + // means the record was written wrong. + assert_eq!( + item_type( + "CREATE TABLE \"materialize\".\"public\".\"tbl\" \ + FROM SOURCE src (REFERENCE = \"topic\")" + ), + Err("unresolved item name".to_string()) + ); } } diff --git a/src/pgrepr-consts/src/oid.rs b/src/pgrepr-consts/src/oid.rs index 4db963663c2d2..e906f223f34bd 100644 --- a/src/pgrepr-consts/src/oid.rs +++ b/src/pgrepr-consts/src/oid.rs @@ -429,12 +429,12 @@ pub const MV_MZ_SCHEMAS_OID: u32 = 16703; pub const TABLE_MZ_COLUMNS_OID: u32 = 16704; pub const MV_MZ_INDEXES_OID: u32 = 16705; pub const TABLE_MZ_INDEX_COLUMNS_OID: u32 = 16706; -pub const TABLE_MZ_TABLES_OID: u32 = 16707; +pub const MV_MZ_TABLES_OID: u32 = 16707; pub const MV_MZ_CONNECTIONS_OID: u32 = 16708; pub const MV_MZ_SSH_TUNNEL_CONNECTIONS_OID: u32 = 16709; pub const MV_MZ_SOURCES_OID: u32 = 16710; pub const MV_MZ_SINKS_OID: u32 = 16711; -pub const TABLE_MZ_VIEWS_OID: u32 = 16712; +pub const MV_MZ_VIEWS_OID: u32 = 16712; pub const MV_MZ_MATERIALIZED_VIEWS_OID: u32 = 16713; pub const TABLE_MZ_TYPES_OID: u32 = 16714; pub const TABLE_MZ_TYPE_PG_METADATA_OID: u32 = 16715; @@ -824,3 +824,5 @@ pub const FUNC_MZ_AWS_EXTERNAL_ID_PREFIX_OID: u32 = 17114; pub const FUNC_MZ_AWS_CONNECTION_ROLE_ARN_OID: u32 = 17115; pub const VIEW_MZ_OBJECT_GRAPH_EDGES_OID: u32 = 17116; pub const INDEX_MZ_OBJECT_GRAPH_EDGES_IND_OID: u32 = 17117; +pub const VIEW_MZ_BUILTIN_TABLES_OID: u32 = 17118; +pub const VIEW_MZ_BUILTIN_VIEWS_OID: u32 = 17119; diff --git a/test/0dt/mzcompose.py b/test/0dt/mzcompose.py index 2a010b4a085f2..84b520e0cf296 100644 --- a/test/0dt/mzcompose.py +++ b/test/0dt/mzcompose.py @@ -1787,7 +1787,7 @@ def get_persist_shard_id(item_id: str, service: str) -> str: ) mz_tables_gid = c.sql_query( - "SELECT id FROM mz_tables WHERE name = 'mz_tables'", + "SELECT id FROM mz_materialized_views WHERE name = 'mz_tables'", service="mz_old", )[0][0] mv_gid = c.sql_query( @@ -1817,7 +1817,7 @@ def get_persist_shard_id(item_id: str, service: str) -> str: c.await_mz_deployment_status(DeploymentStatus.IS_LEADER, "mz_new") new_mz_tables_gid = c.sql_query( - "SELECT id FROM mz_tables WHERE name = 'mz_tables'", + "SELECT id FROM mz_materialized_views WHERE name = 'mz_tables'", service="mz_new", reuse_connection=False, )[0][0] @@ -1870,7 +1870,7 @@ def get_persist_shard_id(item_id: str, service: str) -> str: ) mz_tables_gid = c.sql_query( - "SELECT id FROM mz_tables WHERE name = 'mz_tables'", + "SELECT id FROM mz_materialized_views WHERE name = 'mz_tables'", service="mz_old", )[0][0] mv_gid = c.sql_query( @@ -1901,7 +1901,7 @@ def get_persist_shard_id(item_id: str, service: str) -> str: c.await_mz_deployment_status(DeploymentStatus.IS_LEADER, "mz_new") new_mz_tables_gid = c.sql_query( - "SELECT id FROM mz_tables WHERE name = 'mz_tables'", + "SELECT id FROM mz_materialized_views WHERE name = 'mz_tables'", service="mz_new", reuse_connection=False, )[0][0] diff --git a/test/cluster/mzcompose.py b/test/cluster/mzcompose.py index c879958dd3a3a..6842f11696900 100644 --- a/test/cluster/mzcompose.py +++ b/test/cluster/mzcompose.py @@ -4268,7 +4268,7 @@ def workflow_test_subscribe_hydration_status( FROM mz_internal.mz_subscriptions s, unnest(s.referenced_object_ids) as sroi(id) JOIN mz_introspection.mz_compute_hydration_times_per_worker h ON h.export_id = s.id - JOIN mz_tables t ON (t.id = sroi.id) + JOIN mz_materialized_views t ON (t.id = sroi.id) WHERE t.name = 'mz_tables' true """)) @@ -4283,7 +4283,7 @@ def workflow_test_subscribe_hydration_status( FROM mz_internal.mz_subscriptions s, unnest(s.referenced_object_ids) as sroi(id) JOIN mz_introspection.mz_compute_hydration_times_per_worker h ON h.export_id = s.id - JOIN mz_tables t ON (t.id = sroi.id) + JOIN mz_materialized_views t ON (t.id = sroi.id) WHERE t.name = 'mz_tables' """)) @@ -6003,9 +6003,9 @@ def workflow_test_adhoc_system_indexes( WHERE i.name = 'mz_test_idx1' """) assert output[0] == ("u1", "mz_tables", "mz_catalog_server"), output - output = c.sql_query("EXPLAIN SELECT * FROM mz_tables WHERE char_length(name) = 9") + output = c.sql_query("EXPLAIN SELECT * FROM mz_tables WHERE char_length(name) = 8") assert "mz_test_idx1" in output[0][0], output - output = c.sql_query("SELECT * FROM mz_tables WHERE char_length(name) = 9") + output = c.sql_query("SELECT * FROM mz_tables WHERE char_length(name) = 8") assert len(output) > 0 # The system user should be able to create a new index on an unstable diff --git a/test/sqllogictest/alter.slt b/test/sqllogictest/alter.slt index 44f3aa6575187..1935ee5ff34e5 100644 --- a/test/sqllogictest/alter.slt +++ b/test/sqllogictest/alter.slt @@ -24,7 +24,7 @@ ALTER SYSTEM SET enable_rbac_checks TO false; ---- COMPLETE 0 -query error must be owner of TABLE mz_catalog.mz_tables +query error mz_tables is a materialized view not a table ALTER TABLE mz_tables RENAME TO foo; query error must be owner of SOURCE mz_internal.mz_storage_shards @@ -33,7 +33,7 @@ ALTER SOURCE mz_internal.mz_storage_shards RENAME TO foo; simple conn=mz_system,user=mz_system ALTER TABLE mz_tables RENAME TO foo; ---- -db error: ERROR: system item 'mz_catalog.mz_tables' cannot be modified +db error: ERROR: mz_tables is a materialized view not a table simple conn=mz_system,user=mz_system ALTER SOURCE mz_internal.mz_storage_shards RENAME TO foo; diff --git a/test/sqllogictest/autogenerated/mz_internal.slt b/test/sqllogictest/autogenerated/mz_internal.slt index 8716803cef900..66c3808a23067 100644 --- a/test/sqllogictest/autogenerated/mz_internal.slt +++ b/test/sqllogictest/autogenerated/mz_internal.slt @@ -786,6 +786,8 @@ mz_aws_privatelink_connection_status_history mz_aws_privatelink_connection_statuses mz_builtin_materialized_views mz_builtin_sources +mz_builtin_tables +mz_builtin_views mz_catalog_raw mz_cluster_auto_scaling_strategies mz_cluster_deployment_lineage diff --git a/test/sqllogictest/catalog_server_explain.slt b/test/sqllogictest/catalog_server_explain.slt index 648f9305963cf..6a2ed84d1f888 100644 --- a/test/sqllogictest/catalog_server_explain.slt +++ b/test/sqllogictest/catalog_server_explain.slt @@ -5108,7 +5108,7 @@ mz_catalog.mz_materialized_views: Project: #4, #0, #5, #1, #6, #2, #9, #8, #3, #7 Map: {=r/s1, s1=r/s1}, "s1" →Arrange (#1{schema_name}, #2{name}) - →Constant (40 rows) + →Constant (42 rows) →Arrange (empty key) (#0{schema_name}) (#0{schema_name}, #1{name}) →Fused with Child Map/Filter/Project Project: #4, #3, #5 @@ -5429,6 +5429,110 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_tables"; +---- +mz_catalog.mz_tables: + →Union + →Fused with Child Map/Filter/Project + Project: #6..=#11, #4, #12, #13 + Filter: ("Item" = #3) AND ("table" = (#5 ->> "type")) + Map: (((#2 -> "definition") -> "V1") ->> "create_sql"), parse_catalog_create_sql(#4), parse_catalog_id((#1 -> "gid")), text_to_oid((#2 ->> "oid")), case when ((#2 ->> "ephemeral_owner_session")) IS NULL then parse_catalog_id(((#0{data} -> "value") -> "schema_id")) else "0" end, (#2 ->> "name"), parse_catalog_id((#2 -> "owner_id")), parse_catalog_privileges((#2 -> "privileges")), redact_sql(#4), (#5 ->> "source_id") + →Read mz_internal.mz_catalog_raw + →Delta Join [%0[#1{schema_name}, #2{name}] » %1:mz_catalog_raw[#0{schema_name}, #1{name}] » %2:mz_schemas[#1{name}]] [%1:mz_catalog_raw[#0{schema_name}] » %0[#1{schema_name}, #2{name}] » %2:mz_schemas[#1{name}]] [%2:mz_schemas[#1{name}] » %1:mz_catalog_raw[#0{schema_name}] » %0[#1{schema_name}, #2{name}]] + path %0: + Final closure: + Project: #3, #0, #4, #1, #5, #2, #6, #6, #6 + Map: "s1", null + path %1: + Final closure: + Project: #3, #0, #4, #1, #5, #2, #6, #6, #6 + Map: "s1", null + path %2: + Final closure: + Project: #3, #0, #4, #1, #5, #2, #6, #6, #6 + Map: "s1", null + →Arrange (#1{schema_name}, #2{name}) + →Constant (29 rows) + →Arrange (#0{schema_name}) (#0{schema_name}, #1{name}) + →Fused with Child Map/Filter/Project + Project: #5, #4, #6 + Filter: ("1" = (#1 ->> "object_type")) AND ("GidMapping" = #3) AND (#4) IS NOT NULL AND (#5) IS NOT NULL + Map: (#1 ->> "object_name"), (#1 ->> "schema_name"), ("s" || (#2 ->> "catalog_id")) + →Read mz_internal.mz_catalog_raw + →Arrange (#1{name}) + →Fused with Child Map/Filter/Project + Project: #1, #3 + Filter: (#0{database_id}) IS NULL + →Arranged mz_catalog.mz_schemas + Key: (#2{database_id}) + +Source mz_internal.mz_catalog_raw + project=(#0..=#3) + map=((#0{data} -> "key"), (#0{data} -> "value"), (#0{data} ->> "kind")) + +Used Indexes: + - mz_catalog.mz_schemas_ind (*** full scan ***) + +Target cluster: mz_catalog_server + +EOF + +query T multiline +EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_views"; +---- +mz_catalog.mz_views: + →Union + →Fused with Child Map/Filter/Project + Project: #6..=#12, #4, #13 + Filter: ("Item" = #3) AND ("view" = (#5 ->> "type")) + Map: (((#2 -> "definition") -> "V1") ->> "create_sql"), parse_catalog_create_sql(#4), parse_catalog_id((#1 -> "gid")), text_to_oid((#2 ->> "oid")), case when ((#2 ->> "ephemeral_owner_session")) IS NULL then parse_catalog_id(((#0{data} -> "value") -> "schema_id")) else "0" end, (#2 ->> "name"), (#5 ->> "definition"), parse_catalog_id((#2 -> "owner_id")), parse_catalog_privileges((#2 -> "privileges")), redact_sql(#4) + →Read mz_internal.mz_catalog_raw + →Delta Join [%0[#1{schema_name}, #2{name}] » %1:mz_catalog_raw[#0{schema_name}, #1{name}] » %2:mz_schemas[#1{name}]] [%1:mz_catalog_raw[#0{schema_name}] » %0[#1{schema_name}, #2{name}] » %2:mz_schemas[#1{name}]] [%2:mz_schemas[#1{name}] » %1:mz_catalog_raw[#0{schema_name}] » %0[#1{schema_name}, #2{name}]] + path %0: + Final closure: + Project: #5, #0, #6, #1, #2, #8, #3, #4, #7 + Map: "s1" + path %1: + after %0: + Project: #3, #1, #4..=#6, #0, #2, #7 + Map: redact_sql(#6{create_sql}) + Final closure: + Project: #5, #0, #6, #1, #2, #8, #3, #4, #7 + Map: "s1" + path %2: + after %0: + Project: #4, #1, #5..=#7, #2, #3, #8 + Map: redact_sql(#7{create_sql}) + Final closure: + Project: #5, #0, #6, #1, #2, #8, #3, #4, #7 + Map: "s1" + →Arrange (#1{schema_name}, #2{name}) + →Constant (193 rows) + →Arrange (#0{schema_name}) (#0{schema_name}, #1{name}) + →Fused with Child Map/Filter/Project + Project: #5, #4, #6 + Filter: ("4" = (#1 ->> "object_type")) AND ("GidMapping" = #3) AND (#4) IS NOT NULL AND (#5) IS NOT NULL + Map: (#1 ->> "object_name"), (#1 ->> "schema_name"), ("s" || (#2 ->> "catalog_id")) + →Read mz_internal.mz_catalog_raw + →Arrange (#1{name}) + →Fused with Child Map/Filter/Project + Project: #1, #3 + Filter: (#0{database_id}) IS NULL + →Arranged mz_catalog.mz_schemas + Key: (#2{database_id}) + +Source mz_internal.mz_catalog_raw + project=(#0..=#3) + map=((#0{data} -> "key"), (#0{data} -> "value"), (#0{data} ->> "kind")) + +Used Indexes: + - mz_catalog.mz_schemas_ind (*** full scan ***) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_aws_connections"; ---- @@ -8088,7 +8192,7 @@ query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_materialized_views"; ---- Explained Query (fast path): - →Constant (40 rows) + →Constant (42 rows) Target cluster: mz_catalog_server @@ -8104,6 +8208,26 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_tables"; +---- +Explained Query (fast path): + →Constant (29 rows) + +Target cluster: mz_catalog_server + +EOF + +query T multiline +EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_views"; +---- +Explained Query (fast path): + →Constant (193 rows) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_cluster_deployment_lineage"; ---- diff --git a/test/sqllogictest/information_schema_tables.slt b/test/sqllogictest/information_schema_tables.slt index f949fb8e19f98..6aa277b21008a 100644 --- a/test/sqllogictest/information_schema_tables.slt +++ b/test/sqllogictest/information_schema_tables.slt @@ -250,7 +250,7 @@ MATERIALIZED VIEW materialize mz_catalog mz_tables -BASE TABLE +MATERIALIZED VIEW materialize mz_catalog mz_timezone_abbreviations @@ -266,7 +266,7 @@ BASE TABLE materialize mz_catalog mz_views -BASE TABLE +MATERIALIZED VIEW materialize mz_internal mz_activity_log_thinned @@ -297,6 +297,14 @@ mz_builtin_sources VIEW materialize mz_internal +mz_builtin_tables +VIEW +materialize +mz_internal +mz_builtin_views +VIEW +materialize +mz_internal mz_catalog_raw SOURCE materialize diff --git a/test/sqllogictest/mz_catalog_server_index_accounting.slt b/test/sqllogictest/mz_catalog_server_index_accounting.slt index 6db27834a2053..6e9b993db5d58 100644 --- a/test/sqllogictest/mz_catalog_server_index_accounting.slt +++ b/test/sqllogictest/mz_catalog_server_index_accounting.slt @@ -37,108 +37,108 @@ mz_arrangement_heap_capacity_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangemen mz_arrangement_heap_size_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_heap_size_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_heap_size_raw"␠("operator_id",␠"worker_id") mz_arrangement_records_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_records_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_records_raw"␠("operator_id",␠"worker_id") mz_arrangement_sharing_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_sharing_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_sharing_raw"␠("operator_id",␠"worker_id") -mz_cluster_auto_scaling_strategies_ind CREATE␠INDEX␠"mz_cluster_auto_scaling_strategies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s518␠AS␠"mz_internal"."mz_cluster_auto_scaling_strategies"]␠("cluster_id") -mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s766␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") +mz_cluster_auto_scaling_strategies_ind CREATE␠INDEX␠"mz_cluster_auto_scaling_strategies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s520␠AS␠"mz_internal"."mz_cluster_auto_scaling_strategies"]␠("cluster_id") +mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s768␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") mz_cluster_prometheus_metrics_s2_primary_idx CREATE␠INDEX␠"mz_cluster_prometheus_metrics_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_cluster_prometheus_metrics"␠("process_id",␠"metric_name",␠"labels") -mz_cluster_reconfigurations_ind CREATE␠INDEX␠"mz_cluster_reconfigurations_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s517␠AS␠"mz_internal"."mz_cluster_reconfigurations"]␠("cluster_id") -mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s760␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") -mz_cluster_replica_history_ind CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s615␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at") -mz_cluster_replica_metrics_history_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s523␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id") -mz_cluster_replica_metrics_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s524␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id") -mz_cluster_replica_name_history_ind CREATE␠INDEX␠"mz_cluster_replica_name_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s616␠AS␠"mz_internal"."mz_cluster_replica_name_history"]␠("id") -mz_cluster_replica_size_internal_ind CREATE␠INDEX␠"mz_cluster_replica_size_internal_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s513␠AS␠"mz_internal"."mz_cluster_replica_size_internal"]␠("size") -mz_cluster_replica_sizes_ind CREATE␠INDEX␠"mz_cluster_replica_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s512␠AS␠"mz_catalog"."mz_cluster_replica_sizes"]␠("size") -mz_cluster_replica_status_history_ind CREATE␠INDEX␠"mz_cluster_replica_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s525␠AS␠"mz_internal"."mz_cluster_replica_status_history"]␠("replica_id") -mz_cluster_replica_statuses_ind CREATE␠INDEX␠"mz_cluster_replica_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s526␠AS␠"mz_internal"."mz_cluster_replica_statuses"]␠("replica_id") -mz_cluster_replicas_ind CREATE␠INDEX␠"mz_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s522␠AS␠"mz_catalog"."mz_cluster_replicas"]␠("id") -mz_clusters_ind CREATE␠INDEX␠"mz_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s514␠AS␠"mz_catalog"."mz_clusters"]␠("id") -mz_columns_ind CREATE␠INDEX␠"mz_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s485␠AS␠"mz_catalog"."mz_columns"]␠("name") -mz_comments_ind CREATE␠INDEX␠"mz_comments_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s541␠AS␠"mz_internal"."mz_comments"]␠("id") +mz_cluster_reconfigurations_ind CREATE␠INDEX␠"mz_cluster_reconfigurations_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s519␠AS␠"mz_internal"."mz_cluster_reconfigurations"]␠("cluster_id") +mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s762␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") +mz_cluster_replica_history_ind CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s617␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at") +mz_cluster_replica_metrics_history_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s525␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id") +mz_cluster_replica_metrics_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s526␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id") +mz_cluster_replica_name_history_ind CREATE␠INDEX␠"mz_cluster_replica_name_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s618␠AS␠"mz_internal"."mz_cluster_replica_name_history"]␠("id") +mz_cluster_replica_size_internal_ind CREATE␠INDEX␠"mz_cluster_replica_size_internal_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s515␠AS␠"mz_internal"."mz_cluster_replica_size_internal"]␠("size") +mz_cluster_replica_sizes_ind CREATE␠INDEX␠"mz_cluster_replica_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s514␠AS␠"mz_catalog"."mz_cluster_replica_sizes"]␠("size") +mz_cluster_replica_status_history_ind CREATE␠INDEX␠"mz_cluster_replica_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s527␠AS␠"mz_internal"."mz_cluster_replica_status_history"]␠("replica_id") +mz_cluster_replica_statuses_ind CREATE␠INDEX␠"mz_cluster_replica_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s528␠AS␠"mz_internal"."mz_cluster_replica_statuses"]␠("replica_id") +mz_cluster_replicas_ind CREATE␠INDEX␠"mz_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s524␠AS␠"mz_catalog"."mz_cluster_replicas"]␠("id") +mz_clusters_ind CREATE␠INDEX␠"mz_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s516␠AS␠"mz_catalog"."mz_clusters"]␠("id") +mz_columns_ind CREATE␠INDEX␠"mz_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s487␠AS␠"mz_catalog"."mz_columns"]␠("name") +mz_comments_ind CREATE␠INDEX␠"mz_comments_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s543␠AS␠"mz_internal"."mz_comments"]␠("id") mz_compute_dataflow_global_ids_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_dataflow_global_ids_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_dataflow_global_ids_per_worker"␠("id",␠"worker_id",␠"global_id") -mz_compute_dependencies_ind CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s741␠AS␠"mz_internal"."mz_compute_dependencies"]␠("dependency_id") +mz_compute_dependencies_ind CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s743␠AS␠"mz_internal"."mz_compute_dependencies"]␠("dependency_id") mz_compute_error_counts_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_error_counts_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_error_counts_raw"␠("export_id",␠"worker_id") mz_compute_exports_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_exports_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_exports_per_worker"␠("export_id",␠"worker_id") mz_compute_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_frontiers_per_worker"␠("export_id",␠"worker_id") -mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s751␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") +mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") mz_compute_hydration_times_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_hydration_times_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_hydration_times_per_worker"␠("export_id",␠"worker_id") mz_compute_import_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_import_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_import_frontiers_per_worker"␠("export_id",␠"import_id",␠"worker_id") mz_compute_lir_mapping_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_lir_mapping_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_lir_mapping_per_worker"␠("global_id",␠"lir_id",␠"worker_id") mz_compute_operator_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_durations_histogram_raw"␠("id",␠"worker_id",␠"duration_ns") mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_hydration_statuses_per_worker"␠("export_id",␠"lir_id",␠"worker_id") -mz_connections_ind CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s520␠AS␠"mz_catalog"."mz_connections"]␠("schema_id") -mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s747␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") -mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s746␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") -mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s745␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") -mz_databases_ind CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s483␠AS␠"mz_catalog"."mz_databases"]␠("name") +mz_connections_ind CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s522␠AS␠"mz_catalog"."mz_connections"]␠("schema_id") +mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s749␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") +mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s748␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") +mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s747␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") +mz_databases_ind CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s485␠AS␠"mz_catalog"."mz_databases"]␠("name") mz_dataflow_addresses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_addresses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_addresses_per_worker"␠("id",␠"worker_id") mz_dataflow_channels_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_channels_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_channels_per_worker"␠("id",␠"worker_id") mz_dataflow_operator_reachability_raw_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operator_reachability_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operator_reachability_raw"␠("id",␠"worker_id",␠"source",␠"port",␠"update_type",␠"time") mz_dataflow_operators_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operators_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operators_per_worker"␠("id",␠"worker_id") -mz_frontiers_ind CREATE␠INDEX␠"mz_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s732␠AS␠"mz_internal"."mz_frontiers"]␠("object_id") -mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s762␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") -mz_indexes_ind CREATE␠INDEX␠"mz_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s486␠AS␠"mz_catalog"."mz_indexes"]␠("id") -mz_kafka_sources_ind CREATE␠INDEX␠"mz_kafka_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s480␠AS␠"mz_catalog"."mz_kafka_sources"]␠("id") -mz_materialized_views_ind CREATE␠INDEX␠"mz_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s544␠AS␠"mz_catalog"."mz_materialized_views"]␠("id") +mz_frontiers_ind CREATE␠INDEX␠"mz_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s734␠AS␠"mz_internal"."mz_frontiers"]␠("object_id") +mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s764␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") +mz_indexes_ind CREATE␠INDEX␠"mz_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s488␠AS␠"mz_catalog"."mz_indexes"]␠("id") +mz_kafka_sources_ind CREATE␠INDEX␠"mz_kafka_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s482␠AS␠"mz_catalog"."mz_kafka_sources"]␠("id") +mz_materialized_views_ind CREATE␠INDEX␠"mz_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s546␠AS␠"mz_catalog"."mz_materialized_views"]␠("id") mz_message_batch_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_batch_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_batch_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_batch_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") -mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s837␠AS␠"mz_internal"."mz_notices"]␠("id") -mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s754␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") -mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s754␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") -mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s752␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") -mz_object_dependencies_ind CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s481␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id") -mz_object_graph_edges_ind CREATE␠INDEX␠"mz_object_graph_edges_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s743␠AS␠"mz_internal"."mz_object_graph_edges"]␠("object_id") -mz_object_history_ind CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s555␠AS␠"mz_internal"."mz_object_history"]␠("id") -mz_object_lifetimes_ind CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s556␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id") -mz_object_transitive_dependencies_ind CREATE␠INDEX␠"mz_object_transitive_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s572␠AS␠"mz_internal"."mz_object_transitive_dependencies"]␠("object_id") -mz_objects_ind CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s552␠AS␠"mz_catalog"."mz_objects"]␠("schema_id") +mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s839␠AS␠"mz_internal"."mz_notices"]␠("id") +mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s756␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") +mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s756␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") +mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s754␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") +mz_object_dependencies_ind CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s483␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id") +mz_object_graph_edges_ind CREATE␠INDEX␠"mz_object_graph_edges_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s745␠AS␠"mz_internal"."mz_object_graph_edges"]␠("object_id") +mz_object_history_ind CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s557␠AS␠"mz_internal"."mz_object_history"]␠("id") +mz_object_lifetimes_ind CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s558␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id") +mz_object_transitive_dependencies_ind CREATE␠INDEX␠"mz_object_transitive_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s574␠AS␠"mz_internal"."mz_object_transitive_dependencies"]␠("object_id") +mz_objects_ind CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s554␠AS␠"mz_catalog"."mz_objects"]␠("schema_id") mz_peek_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_peek_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_peek_durations_histogram_raw"␠("worker_id",␠"type",␠"duration_ns") -mz_recent_activity_log_thinned_ind CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s716␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash") -mz_recent_sql_text_ind CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s712␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash") -mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s830␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") -mz_roles_ind CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s504␠AS␠"mz_catalog"."mz_roles"]␠("id") +mz_recent_activity_log_thinned_ind CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s718␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash") +mz_recent_sql_text_ind CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s714␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash") +mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s832␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") +mz_roles_ind CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s506␠AS␠"mz_catalog"."mz_roles"]␠("id") mz_scheduling_elapsed_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_elapsed_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_elapsed_raw"␠("id",␠"worker_id") mz_scheduling_parks_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_parks_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_parks_histogram_raw"␠("worker_id",␠"slept_for_ns",␠"requested_ns") -mz_schemas_ind CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s484␠AS␠"mz_catalog"."mz_schemas"]␠("database_id") -mz_secrets_ind CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s519␠AS␠"mz_catalog"."mz_secrets"]␠("name") -mz_show_all_objects_ind CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s600␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id") -mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s764␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") -mz_show_clusters_ind CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s602␠AS␠"mz_internal"."mz_show_clusters"]␠("name") -mz_show_columns_ind CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s601␠AS␠"mz_internal"."mz_show_columns"]␠("id") -mz_show_connections_ind CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s610␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id") -mz_show_databases_ind CREATE␠INDEX␠"mz_show_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s604␠AS␠"mz_internal"."mz_show_databases"]␠("name") -mz_show_indexes_ind CREATE␠INDEX␠"mz_show_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s614␠AS␠"mz_internal"."mz_show_indexes"]␠("schema_id") -mz_show_materialized_views_ind CREATE␠INDEX␠"mz_show_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s613␠AS␠"mz_internal"."mz_show_materialized_views"]␠("schema_id") -mz_show_roles_ind CREATE␠INDEX␠"mz_show_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s609␠AS␠"mz_internal"."mz_show_roles"]␠("name") -mz_show_schemas_ind CREATE␠INDEX␠"mz_show_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s605␠AS␠"mz_internal"."mz_show_schemas"]␠("database_id") -mz_show_secrets_ind CREATE␠INDEX␠"mz_show_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s603␠AS␠"mz_internal"."mz_show_secrets"]␠("schema_id") -mz_show_sinks_ind CREATE␠INDEX␠"mz_show_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s612␠AS␠"mz_internal"."mz_show_sinks"]␠("schema_id") -mz_show_sources_ind CREATE␠INDEX␠"mz_show_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s611␠AS␠"mz_internal"."mz_show_sources"]␠("schema_id") -mz_show_tables_ind CREATE␠INDEX␠"mz_show_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s606␠AS␠"mz_internal"."mz_show_tables"]␠("schema_id") -mz_show_types_ind CREATE␠INDEX␠"mz_show_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s608␠AS␠"mz_internal"."mz_show_types"]␠("schema_id") -mz_show_views_ind CREATE␠INDEX␠"mz_show_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s607␠AS␠"mz_internal"."mz_show_views"]␠("schema_id") -mz_sink_statistics_ind CREATE␠INDEX␠"mz_sink_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s729␠AS␠"mz_internal"."mz_sink_statistics"]␠("id",␠"replica_id") -mz_sink_status_history_ind CREATE␠INDEX␠"mz_sink_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s701␠AS␠"mz_internal"."mz_sink_status_history"]␠("sink_id") -mz_sink_statuses_ind CREATE␠INDEX␠"mz_sink_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s702␠AS␠"mz_internal"."mz_sink_statuses"]␠("id") -mz_sinks_ind CREATE␠INDEX␠"mz_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s496␠AS␠"mz_catalog"."mz_sinks"]␠("id") -mz_source_statistics_ind CREATE␠INDEX␠"mz_source_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s727␠AS␠"mz_internal"."mz_source_statistics"]␠("id",␠"replica_id") -mz_source_statistics_with_history_ind CREATE␠INDEX␠"mz_source_statistics_with_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s725␠AS␠"mz_internal"."mz_source_statistics_with_history"]␠("id",␠"replica_id") -mz_source_status_history_ind CREATE␠INDEX␠"mz_source_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s703␠AS␠"mz_internal"."mz_source_status_history"]␠("source_id") -mz_source_statuses_ind CREATE␠INDEX␠"mz_source_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s720␠AS␠"mz_internal"."mz_source_statuses"]␠("id") -mz_sources_ind CREATE␠INDEX␠"mz_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s489␠AS␠"mz_catalog"."mz_sources"]␠("id") -mz_tables_ind CREATE␠INDEX␠"mz_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s488␠AS␠"mz_catalog"."mz_tables"]␠("schema_id") -mz_types_ind CREATE␠INDEX␠"mz_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s498␠AS␠"mz_catalog"."mz_types"]␠("schema_id") -mz_views_ind CREATE␠INDEX␠"mz_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s497␠AS␠"mz_catalog"."mz_views"]␠("schema_id") -mz_wallclock_global_lag_recent_history_ind CREATE␠INDEX␠"mz_wallclock_global_lag_recent_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s736␠AS␠"mz_internal"."mz_wallclock_global_lag_recent_history"]␠("object_id") -mz_webhook_sources_ind CREATE␠INDEX␠"mz_webhook_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s542␠AS␠"mz_internal"."mz_webhook_sources"]␠("id") -pg_attrdef_all_databases_ind CREATE␠INDEX␠"pg_attrdef_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s641␠AS␠"mz_internal"."pg_attrdef_all_databases"]␠("oid",␠"adrelid",␠"adnum",␠"adbin",␠"adsrc") -pg_attribute_all_databases_ind CREATE␠INDEX␠"pg_attribute_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s634␠AS␠"mz_internal"."pg_attribute_all_databases"]␠("attrelid",␠"attname",␠"atttypid",␠"attlen",␠"attnum",␠"atttypmod",␠"attnotnull",␠"atthasdef",␠"attidentity",␠"attgenerated",␠"attisdropped",␠"attcollation",␠"database_name",␠"pg_type_database_name") -pg_authid_core_ind CREATE␠INDEX␠"pg_authid_core_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s651␠AS␠"mz_internal"."pg_authid_core"]␠("rolname") -pg_class_all_databases_ind CREATE␠INDEX␠"pg_class_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s622␠AS␠"mz_internal"."pg_class_all_databases"]␠("relname") -pg_description_all_databases_ind CREATE␠INDEX␠"pg_description_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s631␠AS␠"mz_internal"."pg_description_all_databases"]␠("objoid",␠"classoid",␠"objsubid",␠"description",␠"oid_database_name",␠"class_database_name") -pg_namespace_all_databases_ind CREATE␠INDEX␠"pg_namespace_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s619␠AS␠"mz_internal"."pg_namespace_all_databases"]␠("nspname") -pg_type_all_databases_ind CREATE␠INDEX␠"pg_type_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s628␠AS␠"mz_internal"."pg_type_all_databases"]␠("oid") +mz_schemas_ind CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s486␠AS␠"mz_catalog"."mz_schemas"]␠("database_id") +mz_secrets_ind CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s521␠AS␠"mz_catalog"."mz_secrets"]␠("name") +mz_show_all_objects_ind CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s602␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id") +mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s766␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") +mz_show_clusters_ind CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s604␠AS␠"mz_internal"."mz_show_clusters"]␠("name") +mz_show_columns_ind CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s603␠AS␠"mz_internal"."mz_show_columns"]␠("id") +mz_show_connections_ind CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s612␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id") +mz_show_databases_ind CREATE␠INDEX␠"mz_show_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s606␠AS␠"mz_internal"."mz_show_databases"]␠("name") +mz_show_indexes_ind CREATE␠INDEX␠"mz_show_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s616␠AS␠"mz_internal"."mz_show_indexes"]␠("schema_id") +mz_show_materialized_views_ind CREATE␠INDEX␠"mz_show_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s615␠AS␠"mz_internal"."mz_show_materialized_views"]␠("schema_id") +mz_show_roles_ind CREATE␠INDEX␠"mz_show_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s611␠AS␠"mz_internal"."mz_show_roles"]␠("name") +mz_show_schemas_ind CREATE␠INDEX␠"mz_show_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s607␠AS␠"mz_internal"."mz_show_schemas"]␠("database_id") +mz_show_secrets_ind CREATE␠INDEX␠"mz_show_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s605␠AS␠"mz_internal"."mz_show_secrets"]␠("schema_id") +mz_show_sinks_ind CREATE␠INDEX␠"mz_show_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s614␠AS␠"mz_internal"."mz_show_sinks"]␠("schema_id") +mz_show_sources_ind CREATE␠INDEX␠"mz_show_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s613␠AS␠"mz_internal"."mz_show_sources"]␠("schema_id") +mz_show_tables_ind CREATE␠INDEX␠"mz_show_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s608␠AS␠"mz_internal"."mz_show_tables"]␠("schema_id") +mz_show_types_ind CREATE␠INDEX␠"mz_show_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s610␠AS␠"mz_internal"."mz_show_types"]␠("schema_id") +mz_show_views_ind CREATE␠INDEX␠"mz_show_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s609␠AS␠"mz_internal"."mz_show_views"]␠("schema_id") +mz_sink_statistics_ind CREATE␠INDEX␠"mz_sink_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s731␠AS␠"mz_internal"."mz_sink_statistics"]␠("id",␠"replica_id") +mz_sink_status_history_ind CREATE␠INDEX␠"mz_sink_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s703␠AS␠"mz_internal"."mz_sink_status_history"]␠("sink_id") +mz_sink_statuses_ind CREATE␠INDEX␠"mz_sink_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s704␠AS␠"mz_internal"."mz_sink_statuses"]␠("id") +mz_sinks_ind CREATE␠INDEX␠"mz_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s498␠AS␠"mz_catalog"."mz_sinks"]␠("id") +mz_source_statistics_ind CREATE␠INDEX␠"mz_source_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s729␠AS␠"mz_internal"."mz_source_statistics"]␠("id",␠"replica_id") +mz_source_statistics_with_history_ind CREATE␠INDEX␠"mz_source_statistics_with_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s727␠AS␠"mz_internal"."mz_source_statistics_with_history"]␠("id",␠"replica_id") +mz_source_status_history_ind CREATE␠INDEX␠"mz_source_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s705␠AS␠"mz_internal"."mz_source_status_history"]␠("source_id") +mz_source_statuses_ind CREATE␠INDEX␠"mz_source_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s722␠AS␠"mz_internal"."mz_source_statuses"]␠("id") +mz_sources_ind CREATE␠INDEX␠"mz_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s491␠AS␠"mz_catalog"."mz_sources"]␠("id") +mz_tables_ind CREATE␠INDEX␠"mz_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s490␠AS␠"mz_catalog"."mz_tables"]␠("schema_id") +mz_types_ind CREATE␠INDEX␠"mz_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s500␠AS␠"mz_catalog"."mz_types"]␠("schema_id") +mz_views_ind CREATE␠INDEX␠"mz_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s499␠AS␠"mz_catalog"."mz_views"]␠("schema_id") +mz_wallclock_global_lag_recent_history_ind CREATE␠INDEX␠"mz_wallclock_global_lag_recent_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s738␠AS␠"mz_internal"."mz_wallclock_global_lag_recent_history"]␠("object_id") +mz_webhook_sources_ind CREATE␠INDEX␠"mz_webhook_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s544␠AS␠"mz_internal"."mz_webhook_sources"]␠("id") +pg_attrdef_all_databases_ind CREATE␠INDEX␠"pg_attrdef_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s643␠AS␠"mz_internal"."pg_attrdef_all_databases"]␠("oid",␠"adrelid",␠"adnum",␠"adbin",␠"adsrc") +pg_attribute_all_databases_ind CREATE␠INDEX␠"pg_attribute_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s636␠AS␠"mz_internal"."pg_attribute_all_databases"]␠("attrelid",␠"attname",␠"atttypid",␠"attlen",␠"attnum",␠"atttypmod",␠"attnotnull",␠"atthasdef",␠"attidentity",␠"attgenerated",␠"attisdropped",␠"attcollation",␠"database_name",␠"pg_type_database_name") +pg_authid_core_ind CREATE␠INDEX␠"pg_authid_core_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s653␠AS␠"mz_internal"."pg_authid_core"]␠("rolname") +pg_class_all_databases_ind CREATE␠INDEX␠"pg_class_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s624␠AS␠"mz_internal"."pg_class_all_databases"]␠("relname") +pg_description_all_databases_ind CREATE␠INDEX␠"pg_description_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s633␠AS␠"mz_internal"."pg_description_all_databases"]␠("objoid",␠"classoid",␠"objsubid",␠"description",␠"oid_database_name",␠"class_database_name") +pg_namespace_all_databases_ind CREATE␠INDEX␠"pg_namespace_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s621␠AS␠"mz_internal"."pg_namespace_all_databases"]␠("nspname") +pg_type_all_databases_ind CREATE␠INDEX␠"pg_type_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s630␠AS␠"mz_internal"."pg_type_all_databases"]␠("oid") # Record all transitive dependencies (tables, sources, views, mvs) of indexes on # the mz_catalog_server cluster. @@ -225,6 +225,16 @@ mz_builtin_materialized_views name mz_builtin_materialized_views oid mz_builtin_materialized_views privileges mz_builtin_materialized_views schema_name +mz_builtin_tables name +mz_builtin_tables oid +mz_builtin_tables privileges +mz_builtin_tables schema_name +mz_builtin_views create_sql +mz_builtin_views definition +mz_builtin_views name +mz_builtin_views oid +mz_builtin_views privileges +mz_builtin_views schema_name mz_catalog_raw data mz_cluster_auto_scaling_strategies cluster_id mz_cluster_auto_scaling_strategies state diff --git a/test/sqllogictest/mz_tables.slt b/test/sqllogictest/mz_tables.slt new file mode 100644 index 0000000000000..487d2ade49020 --- /dev/null +++ b/test/sqllogictest/mz_tables.slt @@ -0,0 +1,223 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Behavioural tests for mz_catalog.mz_tables. It is a BuiltinMaterializedView +# over mz_internal.mz_catalog_raw (see MZ_TABLES in +# src/catalog/src/builtin/mz_catalog.rs) with two union branches, user tables +# and builtin tables, and it derives every column from the durable catalog JSON +# through parse_catalog_id, parse_catalog_privileges, redact_sql and +# parse_catalog_create_sql (src/expr/src/scalar/func/impls/jsonb.rs). +# +# Before that it was a BuiltinTable populated by pack_table_update in the +# adapter, so the values pinned here are a compatibility surface: +# information_schema.tables, pg_tables and pg_class all read through it. +# +# Assertions filter by object name rather than counting rows, so the file also +# holds under --auto-index-selects, which creates extra views. + +mode cockroach + +# Stable object ids, so create_sql can name dependencies by id. +reset-server + +# --- user_tables branch ------------------------------------------------------- + +# `t` is the first object created after the reset, so it is u1 and create_sql +# below can name it. +statement ok +CREATE TABLE t (a int, b text NOT NULL) + +query T +SELECT id FROM mz_tables WHERE name = 't' +---- +u1 + +query T +SELECT schema_id = (SELECT id FROM mz_schemas WHERE name = 'public' AND database_id IS NOT NULL) +FROM mz_tables WHERE name = 't' +---- +true + +query T +SELECT owner_id = (SELECT id FROM mz_roles WHERE name = 'materialize') +FROM mz_tables WHERE name = 't' +---- +true + +query T multiline +SELECT create_sql FROM mz_tables WHERE name = 't' +---- +CREATE TABLE "materialize"."public"."t" ("a" [s20 AS "pg_catalog"."int4"], "b" [s46 AS "pg_catalog"."text"] NOT NULL) +EOF + +query T multiline +SELECT redacted_create_sql FROM mz_tables WHERE name = 't' +---- +CREATE TABLE materialize.public.t (a [s20 AS pg_catalog.int4], b [s46 AS pg_catalog.text] NOT NULL) +EOF + +# A table with no source omits `source_id` from the parsed create_sql, so the +# column must be NULL rather than absent or empty. +query T +SELECT source_id IS NULL FROM mz_tables WHERE name = 't' +---- +true + +# --- Cross-check against the in-memory catalog -------------------------------- + +# The oid column is read out of the durable JSON, while a regclass cast resolves +# the name through the in-memory catalog. Two independent paths that must agree. +# +# NB create_sql cannot be compared against SHOW CREATE TABLE the same way: SHOW +# humanizes item ids back to names and pretty-prints, so it is deliberately not +# the stored string. +query T +SELECT oid = 't'::regclass::oid FROM mz_tables WHERE name = 't' +---- +true + +# --- source_id, for a table created from a source ----------------------------- + +statement ok +CREATE SOURCE lg FROM LOAD GENERATOR COUNTER + +statement ok +CREATE TABLE lg_tbl FROM SOURCE lg (REFERENCE counter) + +query T +SELECT source_id = (SELECT id FROM mz_sources WHERE name = 'lg') +FROM mz_tables WHERE name = 'lg_tbl' +---- +true + +# --- Temporary tables --------------------------------------------------------- + +# Temporary items are durable catalog items tagged with their owning session, +# parented to a sentinel schema id shared by every session. The MV maps that +# sentinel to schema_id '0', which is what the old SchemaSpecifier::Temporary +# populator printed (src/sql/src/names.rs). + +statement ok +CREATE TEMP TABLE tt (x int) + +query TTT +SELECT id LIKE 'u%', schema_id, create_sql IS NOT NULL FROM mz_tables WHERE name = 'tt' +---- +true 0 true + +# Two sessions may each hold a temporary table of the same name: name +# uniqueness is scoped by the owning session. Both rows show up here, since +# mz_tables reports every item and per-session visibility lives in name +# resolution, not in this MV. +simple conn=other +CREATE TEMP TABLE tt (y text); +---- +COMPLETE 0 + +query I +SELECT count(*) FROM mz_tables WHERE name = 'tt' +---- +2 + +query T +SELECT array_agg(DISTINCT schema_id) FROM mz_tables WHERE name = 'tt' +---- +{0} + +# --- builtin_tables branch ---------------------------------------------------- + +query TTTT +SELECT id LIKE 's%', owner_id, create_sql IS NULL, source_id IS NULL +FROM mz_tables WHERE name = 'mz_columns' +---- +true s1 true true + +query T +SELECT schema_id = (SELECT id FROM mz_schemas WHERE name = 'mz_catalog' AND database_id IS NULL) +FROM mz_tables WHERE name = 'mz_columns' +---- +true + +# Exactly once: the user and builtin branches must not both claim a row. +query I +SELECT count(*) FROM mz_tables WHERE name = 'mz_columns' +---- +1 + +# Every builtin table the catalog generates must be reported. +query T +SELECT array_agg(name ORDER BY name) FROM ( + SELECT name FROM mz_internal.mz_builtin_tables + EXCEPT + SELECT name FROM mz_tables WHERE id LIKE 's%' +) +---- +NULL + +# ...and nothing else: every builtin row traces back to the reporter. +query T +SELECT array_agg(name ORDER BY name) FROM ( + SELECT name FROM mz_tables WHERE id LIKE 's%' + EXCEPT + SELECT name FROM mz_internal.mz_builtin_tables +) +---- +NULL + +# The reporter view backing this MV is itself a builtin view, reported through +# mz_views with placeholder SQL. See mz_views.slt for the full contract. +query I +SELECT count(*) FROM mz_views WHERE name = 'mz_builtin_tables' +---- +1 + +# --- Robustness over the whole catalog ---------------------------------------- + +# The MV runs parse_catalog_create_sql over every Item row in the catalog, in +# its WHERE clause, so a single item whose create_sql the parser rejects makes +# the entire relation unreadable rather than dropping one row. Create one item +# of every kind reachable here and confirm the relation still resolves. + +statement ok +CREATE VIEW v AS SELECT a FROM t + +statement ok +CREATE MATERIALIZED VIEW mv AS SELECT count(*) FROM t + +statement ok +CREATE INDEX t_idx ON t (a) + +statement ok +CREATE TYPE ty AS LIST (ELEMENT TYPE = int4) + +statement ok +CREATE SECRET sec AS 'hunter2' + +query T +SELECT count(*) > 0 FROM mz_tables +---- +true + +# --- NOT NULL invariants ------------------------------------------------------ + +# The MV declares ASSERT NOT NULL for these columns. +query I +SELECT count(*) +FROM mz_tables +WHERE id IS NULL + OR oid IS NULL + OR schema_id IS NULL + OR name IS NULL + OR owner_id IS NULL + OR privileges IS NULL +---- +0 + +statement ok +DROP TABLE t CASCADE diff --git a/test/sqllogictest/mz_views.slt b/test/sqllogictest/mz_views.slt new file mode 100644 index 0000000000000..5fd82a0c8a6b5 --- /dev/null +++ b/test/sqllogictest/mz_views.slt @@ -0,0 +1,266 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Behavioural tests for mz_catalog.mz_views. It is a BuiltinMaterializedView +# over mz_internal.mz_catalog_raw (see MZ_VIEWS in +# src/catalog/src/builtin/mz_catalog.rs) with two union branches, user views and +# builtin views, and it derives every column from the durable catalog JSON +# through parse_catalog_id, parse_catalog_privileges, redact_sql and +# parse_catalog_create_sql (src/expr/src/scalar/func/impls/jsonb.rs). +# +# Before that it was a BuiltinTable populated by pack_view_update in the +# adapter, so the values pinned here are a compatibility surface. The +# `definition` column in particular is what pg_views.definition exposes. +# +# Assertions filter by object name rather than counting rows, so the file also +# holds under --auto-index-selects, which creates extra views. + +mode cockroach + +# Stable object ids, so create_sql can name dependencies by id. +reset-server + +# --- user_views branch -------------------------------------------------------- + +# `t` is the first object created after the reset, so it is u1 and the +# definitions below can name it. +statement ok +CREATE TABLE t (a int, b text NOT NULL) + +statement ok +CREATE VIEW v AS SELECT a, b FROM t WHERE a > 42 + +query T +SELECT schema_id = (SELECT id FROM mz_schemas WHERE name = 'public' AND database_id IS NOT NULL) +FROM mz_views WHERE name = 'v' +---- +true + +query T +SELECT owner_id = (SELECT id FROM mz_roles WHERE name = 'materialize') +FROM mz_views WHERE name = 'v' +---- +true + +# definition is the view's query rendered stably, with the trailing semicolon +# PostgreSQL puts in pg_views.definition. +query T multiline +SELECT definition FROM mz_views WHERE name = 'v' +---- +SELECT "a", "b" FROM [u1 AS "materialize"."public"."t"] WHERE "a" > 42; +EOF + +query T multiline +SELECT create_sql FROM mz_views WHERE name = 'v' +---- +CREATE VIEW "materialize"."public"."v" AS SELECT "a", "b" FROM [u1 AS "materialize"."public"."t"] WHERE "a" > 42 +EOF + +# Redaction replaces literals but keeps structure and resolves quoting. +query T multiline +SELECT redacted_create_sql FROM mz_views WHERE name = 'v' +---- +CREATE VIEW materialize.public.v AS SELECT a, b FROM [u1 AS materialize.public.t] WHERE a > '' +EOF + +# --- Cross-check against the in-memory catalog -------------------------------- + +# The oid column is read out of the durable JSON, while a regclass cast resolves +# the name through the in-memory catalog. Two independent paths that must agree. +# +# NB create_sql cannot be compared against SHOW CREATE VIEW the same way: SHOW +# humanizes item ids back to names and pretty-prints, so it is deliberately not +# the stored string. +query T +SELECT oid = 'v'::regclass::oid FROM mz_views WHERE name = 'v' +---- +true + +# definition must be a fixed point: consumers re-issue it as the body of a new +# view, so planning it again and rendering it again has to reproduce it exactly. +# This goes out through the planner and back through the durable catalog, so it +# is an end-to-end check on the rendering rather than a self-comparison. +statement ok +CREATE VIEW v_roundtrip AS SELECT "a", "b" FROM [u1 AS "materialize"."public"."t"] WHERE "a" > 42; + +query T +SELECT (SELECT definition FROM mz_views WHERE name = 'v') + = (SELECT definition FROM mz_views WHERE name = 'v_roundtrip') +---- +true + +# --- Temporary views ---------------------------------------------------------- + +# Temporary items are durable catalog items tagged with their owning session, +# parented to a sentinel schema id shared by every session. The MV maps that +# sentinel to schema_id '0', which is what the old SchemaSpecifier::Temporary +# populator printed (src/sql/src/names.rs). + +statement ok +CREATE TEMP VIEW tv AS SELECT a FROM t + +query TTT +SELECT id LIKE 'u%', schema_id, create_sql IS NOT NULL FROM mz_views WHERE name = 'tv' +---- +true 0 true + +query T multiline +SELECT definition FROM mz_views WHERE name = 'tv' +---- +SELECT "a" FROM [u1 AS "materialize"."public"."t"]; +EOF + +# Two sessions may each hold a temporary view of the same name: name uniqueness +# is scoped by the owning session. Both rows show up here, since mz_views +# reports every item and per-session visibility lives in name resolution, not in +# this MV. +simple conn=other +CREATE TEMP VIEW tv AS SELECT 1 AS other; +---- +COMPLETE 0 + +query I +SELECT count(*) FROM mz_views WHERE name = 'tv' +---- +2 + +query T +SELECT array_agg(DISTINCT schema_id) FROM mz_views WHERE name = 'tv' +---- +{0} + +# --- builtin_views branch ----------------------------------------------------- + +query TTT +SELECT id LIKE 's%', owner_id, create_sql IS NOT NULL +FROM mz_views WHERE name = 'mz_objects' +---- +true s1 true + +query T +SELECT schema_id = (SELECT id FROM mz_schemas WHERE name = 'mz_catalog' AND database_id IS NULL) +FROM mz_views WHERE name = 'mz_objects' +---- +true + +# Exactly once: the user and builtin branches must not both claim a row. +query I +SELECT count(*) FROM mz_views WHERE name = 'mz_objects' +---- +1 + +# Every builtin view the catalog generates must be reported. +query T +SELECT array_agg(name ORDER BY name) FROM ( + SELECT name FROM mz_internal.mz_builtin_views + EXCEPT + SELECT name FROM mz_views WHERE id LIKE 's%' +) +---- +NULL + +# ...and nothing else: every builtin row traces back to the reporter. +query T +SELECT array_agg(name ORDER BY name) FROM ( + SELECT name FROM mz_views WHERE id LIKE 's%' + EXCEPT + SELECT name FROM mz_internal.mz_builtin_views +) +---- +NULL + +# --- Generated reporter views --------------------------------------------------- + +# The builtin branch reads mz_internal.mz_builtin_views, a generated constant +# view that lists every builtin view, including itself and the other generated +# mz_builtin_* views. The generated views are listed with a short placeholder +# instead of their real SQL: mz_builtin_views cannot textually contain itself, +# and the other reporters embed metadata about every builtin object, so +# re-embedding their SQL would produce enormous rows. + +# The placeholder rows are exactly the generated views, and nothing else is +# elided. +query T +SELECT name FROM mz_views WHERE definition LIKE '%definition elided%' AND id LIKE 's%' ORDER BY name +---- +mz_builtin_materialized_views +mz_builtin_sources +mz_builtin_tables +mz_builtin_views + +query T multiline +SELECT definition FROM mz_views WHERE name = 'mz_builtin_views' +---- +SELECT ''; +EOF + +query T multiline +SELECT create_sql FROM mz_views WHERE name = 'mz_builtin_views' +---- +CREATE VIEW "mz_internal"."mz_builtin_views" AS SELECT '' +EOF + +# The placeholder must stay parseable: redact_sql errors on unparseable input, +# and an error would poison the whole materialized view, not just this row. +query T multiline +SELECT redacted_create_sql FROM mz_views WHERE name = 'mz_builtin_views' +---- +CREATE VIEW mz_internal.mz_builtin_views AS SELECT '' +EOF + +# --- Robustness over the whole catalog ---------------------------------------- + +# The MV runs parse_catalog_create_sql over every Item row in the catalog, in +# its WHERE clause, so a single item whose create_sql the parser rejects makes +# the entire relation unreadable rather than dropping one row. Create one item +# of every kind reachable here and confirm the relation still resolves. + +statement ok +CREATE MATERIALIZED VIEW mv AS SELECT count(*) FROM t + +statement ok +CREATE INDEX t_idx ON t (a) + +statement ok +CREATE TYPE ty AS LIST (ELEMENT TYPE = int4) + +statement ok +CREATE SECRET sec AS 'hunter2' + +statement ok +CREATE SOURCE lg FROM LOAD GENERATOR COUNTER + +statement ok +CREATE TABLE lg_tbl FROM SOURCE lg (REFERENCE counter) + +query T +SELECT count(*) > 0 FROM mz_views +---- +true + +# --- NOT NULL invariants ------------------------------------------------------ + +# The MV declares ASSERT NOT NULL for every column. +query I +SELECT count(*) +FROM mz_views +WHERE id IS NULL + OR oid IS NULL + OR schema_id IS NULL + OR name IS NULL + OR definition IS NULL + OR owner_id IS NULL + OR privileges IS NULL + OR create_sql IS NULL + OR redacted_create_sql IS NULL +---- +0 + +statement ok +DROP TABLE t CASCADE diff --git a/test/sqllogictest/object_ownership.slt b/test/sqllogictest/object_ownership.slt index fb332383cf59d..531367a9b063d 100644 --- a/test/sqllogictest/object_ownership.slt +++ b/test/sqllogictest/object_ownership.slt @@ -2159,7 +2159,7 @@ db error: ERROR: cannot alter item mz_introspection.mz_dataflow_operators becaus simple conn=mz_system,user=mz_system ALTER TABLE mz_views OWNER TO mz_system ---- -db error: ERROR: cannot alter item mz_catalog.mz_views because it is required by the database system +db error: ERROR: mz_views is a materialized view not a table simple conn=mz_system,user=mz_system ALTER VIEW mz_relations OWNER TO mz_system diff --git a/test/sqllogictest/oid.slt b/test/sqllogictest/oid.slt index c32d7555d73f5..1d99040fdb6d9 100644 --- a/test/sqllogictest/oid.slt +++ b/test/sqllogictest/oid.slt @@ -1247,3 +1247,5 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17115 mz_aws_connection_role_arn 17116 mz_object_graph_edges 17117 mz_object_graph_edges_ind +17118 mz_builtin_tables +17119 mz_builtin_views diff --git a/test/sqllogictest/show_create_system_objects.slt b/test/sqllogictest/show_create_system_objects.slt index c63218213bd9e..d5d5d466ad6c8 100644 --- a/test/sqllogictest/show_create_system_objects.slt +++ b/test/sqllogictest/show_create_system_objects.slt @@ -12,5 +12,5 @@ mode cockroach query error cannot show create for system object mz_internal\.mz_source_statistics SHOW CREATE SOURCE mz_internal.mz_source_statistics -query error cannot show create for system object mz_catalog\.mz_tables -SHOW CREATE TABLE mz_tables +query error cannot show create for system object mz_catalog\.mz_columns +SHOW CREATE TABLE mz_columns diff --git a/test/testdrive-old-kafka-src-syntax/update.td b/test/testdrive-old-kafka-src-syntax/update.td index 7038ec7920bca..33339603145e0 100644 --- a/test/testdrive-old-kafka-src-syntax/update.td +++ b/test/testdrive-old-kafka-src-syntax/update.td @@ -76,7 +76,7 @@ contains:invalid input syntax for type double precision contains:cannot mutate materialized view ! UPDATE mz_tables SET a = 1 -contains:cannot mutate system table +contains:cannot mutate materialized view 'mz_catalog.mz_tables' ! UPDATE t SET a = 1 contains:unknown column a diff --git a/test/testdrive/catalog.td b/test/testdrive/catalog.td index 6cc485a3a3bbc..1ecf361507c03 100644 --- a/test/testdrive/catalog.td +++ b/test/testdrive/catalog.td @@ -522,9 +522,7 @@ mz_map_types "" mz_operators "" mz_pseudo_types "" mz_role_auth "" -mz_tables "" mz_types "" -mz_views "" > SHOW VIEWS FROM mz_catalog name comment @@ -561,6 +559,8 @@ mz_sources mz_catalog_server "" mz_ssh_tunnel_connections mz_catalog_server "" mz_system_privileges mz_catalog_server "" mz_materialized_views mz_catalog_server "" +mz_tables mz_catalog_server "" +mz_views mz_catalog_server "" # Check default sources, tables, and views in mz_catalog_unstable. @@ -625,6 +625,8 @@ name comment mz_activity_log_thinned "" mz_builtin_materialized_views "" mz_builtin_sources "" +mz_builtin_tables "" +mz_builtin_views "" mz_cluster_deployment_lineage "" mz_cluster_replica_history "" mz_cluster_replica_metrics "" @@ -831,7 +833,7 @@ test_table "" # `SHOW TABLES` and `mz_tables` should agree. > SELECT COUNT(*) FROM mz_tables WHERE id LIKE 's%' -31 +29 # There is one entry in mz_indexes for each field_number/expression of the index. > SELECT COUNT(id) FROM mz_indexes WHERE id LIKE 's%' diff --git a/test/testdrive/temporary.td b/test/testdrive/temporary.td index 4a4e7e2e615e5..70e23e45a895d 100644 --- a/test/testdrive/temporary.td +++ b/test/testdrive/temporary.td @@ -241,13 +241,13 @@ contains:unknown catalog item 'temp_v' contains:cannot create temporary item in non-temporary schema # Regression test: a persistent (non-temporary) item must not be allowed to -# depend on a TEMPORARY object. Temporary objects live only in the in-memory, -# session-scoped catalog and are never persisted, so a persisted item that -# references one is a dangling catalog entry. The coordinator applies durable -# items before temporary ones (it assumes the invariant holds, see -# apply.rs), so on the next catalog rebuild deserializing the persisted item -# fails to resolve the temporary id and panics with -# PlanError(InvalidId(...)): invalid persisted SQL. +# depend on a TEMPORARY object. A temporary object is durable, but it is owned +# by the session that created it and is reclaimed when that session ends, or at +# the next writable catalog open if the process died first. A persisted item +# referencing one therefore becomes a dangling catalog entry as soon as the +# owning session goes away, and on the next catalog rebuild deserializing it +# fails to resolve the id and panics with PlanError(InvalidId(...)): invalid +# persisted SQL. # # CREATE enforces this via ErrorKind::InvalidTemporaryDependency. ALTER SINK # ... SET FROM went through a separate catalog path that skipped the check, diff --git a/test/testdrive/update.td b/test/testdrive/update.td index 1bcb28cbcf16d..8ff37126e8a33 100644 --- a/test/testdrive/update.td +++ b/test/testdrive/update.td @@ -76,7 +76,7 @@ contains:invalid input syntax for type double precision contains:cannot mutate materialized view ! UPDATE mz_tables SET a = 1 -contains:cannot mutate system table +contains:cannot mutate materialized view 'mz_catalog.mz_tables' ! UPDATE t SET a = 1 contains:unknown column a From 4684c238d76927473e6718bde97dfd2688d77da7 Mon Sep 17 00:00:00 2001 From: Kay Kim Date: Wed, 19 Aug 2026 16:47:28 -0400 Subject: [PATCH 08/10] docs: move console docs into get started (#38362) --- doc/user/content/console/_index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/user/content/console/_index.md b/doc/user/content/console/_index.md index ec203c1eaf4af..8e2709e77afde 100644 --- a/doc/user/content/console/_index.md +++ b/doc/user/content/console/_index.md @@ -5,7 +5,8 @@ disable_list: true disable_toc: true menus: main: - weight: 5 + parent: "get-started" + weight: 24 identifier: 'console' aliases: - /self-managed/v25.2/console/ From 653daf7b5f3d8733c5a8a9c1b68adf23cba38e40 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Wed, 19 Aug 2026 22:56:18 +0200 Subject: [PATCH 09/10] tests: Unflake and fix dataflow visualizer (#38359) Keeps failing in CI, see for example https://buildkite.com/materialize/test/builds/131939#01a01a66-ff4c-43bd-a5b4-f53b0d57c626 --- .../src/http/static/js/cluster-replica.jsx | 139 ++++++++++++++ .../http/static/js/hierarchical-memory.jsx | 170 +----------------- .../src/http/static/js/memory.jsx | 107 ++--------- .../templates/hierarchical-memory.html | 1 + src/environmentd/templates/memory.html | 1 + test/dataflow-visualizer/mzcompose.py | 78 ++++++++ test/dataflow-visualizer/package.json | 1 + test/dataflow-visualizer/playwright.config.ts | 20 ++- test/dataflow-visualizer/tests/fixture.ts | 52 ++++++ .../tests/hierarchical-memory.spec.ts | 96 +++++----- test/dataflow-visualizer/tests/memory.spec.ts | 142 +++++++++------ .../tsconfig.typecheck.json | 13 +- 12 files changed, 448 insertions(+), 372 deletions(-) create mode 100644 src/environmentd/src/http/static/js/cluster-replica.jsx create mode 100644 test/dataflow-visualizer/tests/fixture.ts diff --git a/src/environmentd/src/http/static/js/cluster-replica.jsx b/src/environmentd/src/http/static/js/cluster-replica.jsx new file mode 100644 index 0000000000000..2155b2378fdd4 --- /dev/null +++ b/src/environmentd/src/http/static/js/cluster-replica.jsx @@ -0,0 +1,139 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +'use strict'; + +// NOTE: this file and the page script that consumes it are separate classic +// scripts sharing one global scope, so a top-level `const { useState } = React` +// here would collide with the same declaration in the page script. Reach +// through `React` instead. + +/** + * The cluster replica picker shared by the memory visualizer pages. + * + * Renders `props.children(clusterName, replicaName)` once a replica is settled, + * which is the one named in the URL if there is one and otherwise the one + * chosen below. The choice is mirrored back into the URL, so a page can be + * linked to a specific replica. + */ +function ClusterReplicaView(props) { + const [currentClusterName, setCurrentClusterName] = React.useState(null); + const [currentReplicaName, setCurrentReplicaName] = React.useState(null); + const [replicas, setReplicas] = React.useState(null); + const [loading, setLoading] = React.useState(true); + const [error, setError] = React.useState(false); + + // The first statement names the session's cluster, which is the starting + // point for the choice below. `mz_clusters.id` starts with `u` for the + // clusters a user created, which is what separates them from the builtins. + const queryClusterReplicas = ` + SELECT current_setting('cluster'); + + SELECT + clusters.name AS cluster_name, + replicas.name AS replica_name, + clusters.id LIKE 'u%' AS user_cluster + FROM + mz_catalog.mz_cluster_replicas replicas + LEFT JOIN mz_catalog.mz_clusters clusters ON clusters.id = replicas.cluster_id + ORDER BY cluster_name ASC, replica_name ASC + `; + + React.useEffect(() => { + const search = new URLSearchParams(location.search); + const clusterName = search.get('cluster_name'); + const replicaName = search.get('replica_name'); + if (clusterName) { + setCurrentClusterName(clusterName); + } + if (replicaName) { + setCurrentReplicaName(replicaName); + } + + query(queryClusterReplicas) + .then((data) => { + const [sessionClusterTable, replicasTable] = data.results; + const sessionCluster = sessionClusterTable.rows[0][0]; + const replicas = replicasTable.rows.map( + ([clusterName, replicaName, userCluster]) => ({ + clusterName, + replicaName, + userCluster, + }) + ); + setReplicas(replicas); + if (!replicaName && replicas.length > 0) { + // A user cluster is preferred over the session's own, because this + // page is normally reached through a proxy that authenticates as + // `mz_support`, whose default cluster is the builtin + // `mz_catalog_server`. Nobody opens the dataflow visualizer to look + // at the catalog server, and its introspection relations are an + // order of magnitude more expensive to query than a small user + // cluster's. Rows are ordered by cluster name and then replica name, + // so each fallback lands on the first replica of the first cluster + // that qualifies. + const preferred = + replicas.find( + (r) => r.userCluster && r.clusterName === sessionCluster + ) || + replicas.find((r) => r.userCluster) || + replicas[0]; + setCurrentClusterName(preferred.clusterName); + setCurrentReplicaName(preferred.replicaName); + } + setLoading(false); + }) + .catch((error) => { + setError(error); + setLoading(false); + }); + }, []); + + React.useEffect(() => { + if (!currentReplicaName) return; + const params = new URLSearchParams(location.search); + params.set('cluster_name', currentClusterName); + params.set('replica_name', currentReplicaName); + window.history.replaceState({}, '', `${location.pathname}?${params}`); + }, [currentClusterName, currentReplicaName]); + + return ( +
+ {loading ? ( +
Loading...
+ ) : error ? ( +
error: {String(error)}
+ ) : ( +
+ + + {props.children(currentClusterName, currentReplicaName)} +
+ )} +
+ ); +} diff --git a/src/environmentd/src/http/static/js/hierarchical-memory.jsx b/src/environmentd/src/http/static/js/hierarchical-memory.jsx index 9cb0119643e79..ebaecf32894eb 100644 --- a/src/environmentd/src/http/static/js/hierarchical-memory.jsx +++ b/src/environmentd/src/http/static/js/hierarchical-memory.jsx @@ -17,102 +17,11 @@ function formatNameForQuery(name) { const { useState, useEffect } = React; -function ClusterReplicaView() { - const [currentClusterName, setCurrentClusterName] = useState(null); - const [currentReplicaName, setCurrentReplicaName] = useState(null); - const [sqlResponse, setSqlResponse] = useState(null); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(false); - - const queryClusterReplicas = ` - SELECT - clusters.name AS cluster_name, replicas.name AS replica_name - FROM - mz_catalog.mz_cluster_replicas replicas - LEFT JOIN mz_catalog.mz_clusters clusters ON clusters.id = replicas.cluster_id - ORDER BY cluster_name ASC, replica_name ASC - `; - - useEffect(() => { - const search = new URLSearchParams(location.search); - const clusterName = search.get('cluster_name'); - const replicaName = search.get('replica_name'); - if (clusterName) { - setCurrentClusterName(clusterName); - } - if (replicaName) { - setCurrentReplicaName(replicaName); - } - query(queryClusterReplicas) - .then((data) => { - const results = data.results[0].rows; - setSqlResponse(results); - if (!replicaName && results.length > 0) { - if(results.some( - result => ('default' == result[0]) && ('r1' == result[1]))) { - setCurrentClusterName('default'); - setCurrentReplicaName('r1'); - } else { - setCurrentClusterName(results[0][0]); - setCurrentReplicaName(results[0][1]); - } - } - setLoading(false); - }) - .catch((error) => { - setError(error); - setLoading(false); - }); - }, []); - - useEffect(() => { - if (!currentReplicaName) return; - const params = new URLSearchParams(location.search); - params.set('cluster_name', currentClusterName); - params.set('replica_name', currentReplicaName); - window.history.replaceState({}, '', `${location.pathname}?${params}`); - }, [currentClusterName, currentReplicaName]); - - return ( -
- {loading ? ( -
Loading...
- ) : error ? ( -
error: {String(error)}
- ) : ( -
- - - -
- )} -
- ); -} - function Dataflows(props) { - const [stats, setStats] = useState(null); const [addrs, setAddrs] = useState(null); const [records, setRecords] = useState(null); const [opers, setOpers] = useState(null); const [chans, setChans] = useState(null); - const [view, setView] = useState(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(false); const [page, setPage] = useState(null); @@ -174,14 +83,6 @@ function Dataflows(props) { ) setRecords(records); - try { - const view = await getCreateView(stats.name); - setView(view); - } catch (error) { - console.debug('could not get create view:', error); - setView(null); - } - console.log("Loaded"); setLoading(false); }; @@ -337,68 +238,6 @@ function Dataflows(props) { } -async function getCreateView(dataflow_name) { - // dataflow_name is the full name of the dataflow operator. It is generally - // of the form "Dataflow: ..". We will use a - // regex to parse these out and use them to get the fully qualified view name - // which we will use with SHOW CREATE VIEW to show the SQL that created this - // dataflow. - // - // There are known problems with this method. It doesn't know anything about - // SQL parsing or escaping, assumes the dataflow operator's name is of a very - // specific shape, and assumes a CREATE VIEW statement made an index which made - // this dataflow. So we assume that problems can happen at any level here and - // will cleanly bail if anything doesn't exactly match what we want. In that - // case we will not show the SQL. This is intended to be good enough for most - // users for now. - const match = dataflow_name.match(/^Dataflow: (.*)\.(.*)\.(.*)$/); - if (!match) { - throw 'unknown dataflow name pattern'; - } - const view_name_table = await query(` - SELECT - d.name AS database, s.schema, s.view - FROM - mz_catalog.mz_databases AS d - JOIN ( - SELECT - s.database_id, s.name AS schema, v.view - FROM - mz_catalog.mz_schemas AS s - JOIN ( - SELECT - name AS view, schema_id - FROM - mz_catalog.mz_views - WHERE - id - = ( - SELECT - DISTINCT idx.on_id - FROM - mz_catalog.mz_databases AS db, - mz_catalog.mz_schemas AS sc, - mz_catalog.mz_indexes AS idx - WHERE - db.name = '${match[1]}' - AND sc.name = '${match[2]}' - AND idx.name = '${match[3]}' - ) - ) - AS v ON s.id = v.schema_id - ) - AS s ON d.id = s.database_id; - `); - if (view_name_table.rows.length !== 1) { - throw 'could not determine view'; - } - const name = view_name_table.rows[0]; - const create_table = await query( - `SHOW CREATE VIEW "${name[0]}"."${name[1]}"."${name[2]}"` - ); - return { name: create_table.rows[0][0], create: create_table.rows[0][1] }; -} - function addrStr(addr) { return addr.join(', '); } @@ -423,4 +262,11 @@ function toggle_active(e) { } const content = document.getElementById('content2'); -ReactDOM.render(, content); +ReactDOM.render( + + {(clusterName, replicaName) => ( + + )} + , + content +); diff --git a/src/environmentd/src/http/static/js/memory.jsx b/src/environmentd/src/http/static/js/memory.jsx index 2e6b822cb3000..5d3f72de23842 100644 --- a/src/environmentd/src/http/static/js/memory.jsx +++ b/src/environmentd/src/http/static/js/memory.jsx @@ -17,96 +17,6 @@ function formatNameForQuery(name) { const { useState, useEffect } = React; -function ClusterReplicaView() { - const [currentClusterName, setCurrentClusterName] = useState(null); - const [currentReplicaName, setCurrentReplicaName] = useState(null); - const [sqlResponse, setSqlResponse] = useState(null); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(false); - - const queryClusterReplicas = ` - SELECT - clusters.name AS cluster_name, replicas.name AS replica_name - FROM - mz_catalog.mz_cluster_replicas replicas - LEFT JOIN mz_catalog.mz_clusters clusters ON clusters.id = replicas.cluster_id - ORDER BY cluster_name ASC, replica_name ASC - `; - - useEffect(() => { - const search = new URLSearchParams(location.search); - const clusterName = search.get('cluster_name'); - const replicaName = search.get('replica_name'); - if (clusterName) { - setCurrentClusterName(clusterName); - } - if (replicaName) { - setCurrentReplicaName(replicaName); - } - - query(queryClusterReplicas) - .then((data) => { - const results = data.results[0].rows; - setSqlResponse(results); - if (!replicaName && results.length > 0) { - if(results.some( - result => ('default' == result[0]) && ('r1' == result[1]))) { - setCurrentClusterName('default'); - setCurrentReplicaName('r1'); - } else { - setCurrentClusterName(results[0][0]); - setCurrentReplicaName(results[0][1]); - } - } - setLoading(false); - }) - .catch((error) => { - setError(error); - setLoading(false); - }); - }, []); - - useEffect(() => { - if (!currentReplicaName) return; - const params = new URLSearchParams(location.search); - params.set('cluster_name', currentClusterName); - params.set('replica_name', currentReplicaName); - window.history.replaceState({}, '', `${location.pathname}?${params}`); - }, [currentClusterName, currentReplicaName]); - - return ( -
- {loading ? ( -
Loading...
- ) : error ? ( -
error: {String(error)}
- ) : ( -
- - - -
- )} -
- ); -} - function Views(props) { const [currentDataflow, setCurrentDataflow] = useState(null); const [includeSystemCatalog, setIncludeSystemCatalog] = useState(false); @@ -691,7 +601,9 @@ async function getCreateView(dataflow_name) { if (!match) { throw 'unknown dataflow name pattern'; } - const view_name_table = await query(` + const { + results: [view_name_table], + } = await query(` SELECT d.name AS database, s.schema, s.view FROM @@ -729,7 +641,9 @@ async function getCreateView(dataflow_name) { throw 'could not determine view'; } const name = view_name_table.rows[0]; - const create_table = await query(`SHOW CREATE VIEW "${name[0]}"."${name[1]}"."${name[2]}"`); + const { + results: [create_table], + } = await query(`SHOW CREATE VIEW "${name[0]}"."${name[1]}"."${name[2]}"`); return { name: create_table.rows[0][0], create: create_table.rows[0][1] }; } @@ -758,4 +672,11 @@ function dispNs(ns) { } const content = document.getElementById('content'); -ReactDOM.render(, content); +ReactDOM.render( + + {(clusterName, replicaName) => ( + + )} + , + content +); diff --git a/src/environmentd/templates/hierarchical-memory.html b/src/environmentd/templates/hierarchical-memory.html index 1e867a3490f5e..285c7548ad4ca 100644 --- a/src/environmentd/templates/hierarchical-memory.html +++ b/src/environmentd/templates/hierarchical-memory.html @@ -51,6 +51,7 @@ {# GraphViz in WASM #} {# Zlib compression #} + {# Shared replica picker #} {% endblock %} diff --git a/src/environmentd/templates/memory.html b/src/environmentd/templates/memory.html index ded02ab5c06b4..80dea44bad1ce 100644 --- a/src/environmentd/templates/memory.html +++ b/src/environmentd/templates/memory.html @@ -11,6 +11,7 @@ {# GraphViz in WASM #} {# Zlib compression #} + {# Shared replica picker #} {% endblock %} diff --git a/test/dataflow-visualizer/mzcompose.py b/test/dataflow-visualizer/mzcompose.py index 1f3edc78a2184..e19475f6bc8ce 100644 --- a/test/dataflow-visualizer/mzcompose.py +++ b/test/dataflow-visualizer/mzcompose.py @@ -12,12 +12,41 @@ Tests the /memory, /hierarchical-memory, and /metrics-viz endpoints on port 6878. """ +import time + from materialize import MZ_ROOT from materialize.buildkite import is_in_buildkite, upload_artifact from materialize.mzcompose.composition import Composition from materialize.mzcompose.service import Service from materialize.mzcompose.services.materialized import Materialized +# The cluster replica the tests visualize. `quickstart` holds the fixture index +# below and little else, which keeps the introspection queries behind the +# visualizer an order of magnitude cheaper than on `mz_catalog_server`, where +# they scan every builtin dataflow. +FIXTURE_CLUSTER = "quickstart" +FIXTURE_REPLICA = "r1" + +# The index whose dataflow the tests expand. Without it they would have to pick +# whatever sits at the top of the dataflow table, and that is a moving target: +# the table is ordered by record count descending, a dataflow that arranges +# nothing reports NULL rather than zero there, and NULLs sort first. Those are +# the transient `introspection-subscribe-*` dataflows and the storage +# `command_sequencer`. A transient one can be gone by the time the page queries +# its operators, and the page then reports an unknown dataflow id. +# +# The index key is load-bearing as well. An arrangement embeds the debug +# formatting of its key in its operator name, so indexing a named column yields +# `ArrangeBy[[Column(0, "id")]]`. Those double quotes have to survive into a DOT +# label, which is what the quoted-name test asserts. +# +# So is indexing a view rather than the table directly: the /memory page shows +# the SQL that created a dataflow by resolving the index back to a view, and +# that lookup finds nothing for an index on a table. +FIXTURE_TABLE = "visualizer_fixture" +FIXTURE_VIEW = "visualizer_fixture_view" +FIXTURE_INDEX = "visualizer_fixture_idx" + SERVICES = [ Materialized(), Service( @@ -29,6 +58,10 @@ ], "environment": [ "MZ_HOST=materialized", + f"FIXTURE_CLUSTER={FIXTURE_CLUSTER}", + f"FIXTURE_REPLICA={FIXTURE_REPLICA}", + f"FIXTURE_VIEW={FIXTURE_VIEW}", + f"FIXTURE_INDEX={FIXTURE_INDEX}", ], }, ), @@ -38,6 +71,7 @@ def workflow_default(c: Composition) -> None: """Run dataflow visualizer E2E tests""" c.up("materialized") + create_fixture_dataflow(c) try: c.run("playwright", "/workdir/run-tests.sh") finally: @@ -47,3 +81,47 @@ def workflow_default(c: Composition) -> None: ) if traces_path.exists() and is_in_buildkite(): upload_artifact(traces_path) + + +def create_fixture_dataflow(c: Composition) -> None: + """Install the index the browser tests visualize, and wait for its dataflow. + + The DDL runs over pgwire as `materialize`. The visualizer's own `/api/sql` + endpoint on the internal HTTP port runs as `anonymous_http_user`, which has + no CREATE privilege, so the tests cannot set this up themselves. + + Idempotent, so that re-running the workflow against a composition that is + already up does not have to start from `down -v`. + """ + c.sql(f""" + CREATE TABLE IF NOT EXISTS {FIXTURE_TABLE} (id int, other int); + CREATE VIEW IF NOT EXISTS {FIXTURE_VIEW} AS + SELECT id, other FROM {FIXTURE_TABLE}; + CREATE INDEX IF NOT EXISTS {FIXTURE_INDEX} + IN CLUSTER {FIXTURE_CLUSTER} ON {FIXTURE_VIEW} (id); + INSERT INTO {FIXTURE_TABLE} VALUES (1, 1); + """) + + # The introspection sources report the new dataflow only on their next tick, + # so without this the tests can open a page whose table does not list it yet. + deadline = time.time() + 60 + with c.sql_connection( + startup_params={"cluster": FIXTURE_CLUSTER, "cluster_replica": FIXTURE_REPLICA} + ) as conn: + cursor = conn.cursor() + while True: + cursor.execute( + b"SELECT count(*) FROM mz_introspection.mz_records_per_dataflow " + b"WHERE name LIKE %s", + (f"Dataflow: %.{FIXTURE_INDEX}",), + ) + row = cursor.fetchone() + assert row is not None + if row[0] > 0: + return + if time.time() > deadline: + raise AssertionError( + f"index {FIXTURE_INDEX} did not show up in " + f"mz_records_per_dataflow on {FIXTURE_CLUSTER}.{FIXTURE_REPLICA}" + ) + time.sleep(1) diff --git a/test/dataflow-visualizer/package.json b/test/dataflow-visualizer/package.json index 1de766f156d0e..e7fb2e2902897 100644 --- a/test/dataflow-visualizer/package.json +++ b/test/dataflow-visualizer/package.json @@ -10,6 +10,7 @@ "devDependencies": { "@playwright/test": "1.62.1", "@types/d3": "7.4.3", + "@types/node": "26.2.0", "@types/react": "16.14.69", "@types/react-dom": "16.9.25", "typescript": "7.0.2" diff --git a/test/dataflow-visualizer/playwright.config.ts b/test/dataflow-visualizer/playwright.config.ts index 32740ad9ec589..5e0290792171c 100644 --- a/test/dataflow-visualizer/playwright.config.ts +++ b/test/dataflow-visualizer/playwright.config.ts @@ -11,7 +11,12 @@ import { defineConfig } from '@playwright/test'; export default defineConfig({ testDir: './tests', - fullyParallel: false, // Run serially since we share a single Materialize instance + // Every test shares one Materialize instance, and the introspection queries + // behind the visualizer are unindexed scans. Two workers on a CI agent turn + // those into multi-second waits that trip the assertion timeouts below, so + // pin the suite to a single worker rather than only serializing each file. + fullyParallel: false, + workers: 1, retries: 1, reporter: 'list', use: { @@ -22,5 +27,16 @@ export default defineConfig({ : 'http://localhost:6878', trace: 'on-first-retry', }, - timeout: 30000, + expect: { + // The visualizer pages load in stages, each rendering its own `Loading...` + // while its introspection query is in flight. Those queries are unindexed + // scans, so a busy CI agent can take seconds over each one. Set the budget + // once here rather than per assertion. + timeout: 30000, + }, + // A test can wait out two page loads in a row (the dataflow list, then one + // dataflow's operators), so this has to exceed the sum of the assertion + // timeouts a single test can accumulate. Otherwise a slow page reports as an + // opaque test timeout instead of naming the element it was waiting for. + timeout: 90000, }); diff --git a/test/dataflow-visualizer/tests/fixture.ts b/test/dataflow-visualizer/tests/fixture.ts new file mode 100644 index 0000000000000..cf746fb3ed022 --- /dev/null +++ b/test/dataflow-visualizer/tests/fixture.ts @@ -0,0 +1,52 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file at the root of this repository. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +// The dataflow the browser tests visualize. `mzcompose.py` creates the index +// and waits for its dataflow to reach introspection before Playwright starts, +// and passes these names in the environment so both ends agree on one +// definition. See the comments there for why the tests pin a dataflow instead +// of expanding whichever row the page happens to list first. +// +// The fallbacks keep `npx playwright test` usable against a Materialize started +// by hand, as long as the same fixture exists there. +// +// The fixture lives on `quickstart`, the only user cluster in the composition, +// which is also the one an unpinned page has to select. That is not a +// coincidence: the visualizer exists to show you your own dataflows. + +export const FIXTURE_CLUSTER = process.env.FIXTURE_CLUSTER ?? 'quickstart'; +export const FIXTURE_REPLICA = process.env.FIXTURE_REPLICA ?? 'r1'; +export const FIXTURE_VIEW = + process.env.FIXTURE_VIEW ?? 'visualizer_fixture_view'; +export const FIXTURE_INDEX = + process.env.FIXTURE_INDEX ?? 'visualizer_fixture_idx'; + +// How the fixture's dataflow is named in the visualizer's dataflow table. +export const FIXTURE_DATAFLOW = `Dataflow: materialize.public.${FIXTURE_INDEX}`; + +/** A page URL that pins the cluster replica the page should show. */ +export function replicaPage( + path: string, + cluster: string, + replica: string +): string { + const params = new URLSearchParams({ + cluster_name: cluster, + replica_name: replica, + }); + return `${path}?${params}`; +} + +/** + * A page URL pinned to the fixture's replica, so a test never depends on which + * replica the page picks by default. + */ +export function fixturePage(path: string): string { + return replicaPage(path, FIXTURE_CLUSTER, FIXTURE_REPLICA); +} diff --git a/test/dataflow-visualizer/tests/hierarchical-memory.spec.ts b/test/dataflow-visualizer/tests/hierarchical-memory.spec.ts index 9a34ef7325af2..1607f1c988ba1 100644 --- a/test/dataflow-visualizer/tests/hierarchical-memory.spec.ts +++ b/test/dataflow-visualizer/tests/hierarchical-memory.spec.ts @@ -7,7 +7,22 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { test, expect } from '@playwright/test'; +import { test, expect, Page } from '@playwright/test'; +import { FIXTURE_INDEX, fixturePage } from './fixture'; + +/** + * Wait for the page to render its dataflow graphs. + * + * Scope graphs live inside collapsed `.content` divs, so assert on presence + * rather than visibility. The graphs are the page's actual output: the dropdown + * appearing only means the replica list came back, and asserting on that alone + * passes while the rest of the page still says `Loading...`. + */ +async function expectGraphs(page: Page) { + await expect + .poll(() => page.locator('#content2 svg').count()) + .toBeGreaterThan(0); +} test.describe('/hierarchical-memory page', () => { test('page loads without crashing', async ({ page }) => { @@ -28,9 +43,8 @@ test.describe('/hierarchical-memory page', () => { await page.goto('/hierarchical-memory'); // The dropdown should appear after initial data loads - // Use a longer timeout as the page queries the database const dropdown = page.locator('#cluster_replica'); - await expect(dropdown).toBeVisible({ timeout: 20000 }); + await expect(dropdown).toBeVisible(); // Should have at least one option (check count, not visibility since options are inside select) const options = dropdown.locator('option'); @@ -46,24 +60,20 @@ test.describe('/hierarchical-memory page', () => { await page.goto('/hierarchical-memory'); - // Wait for dropdown to appear (indicates initial load complete) - const dropdown = page.locator('#cluster_replica'); - await expect(dropdown).toBeVisible({ timeout: 20000 }); - - // The page queries data and renders - give it time - await page.waitForTimeout(3000); - - await expect(dropdown).toBeVisible(); + await expectGraphs(page); + await expect(page.locator('#cluster_replica')).toBeVisible(); await expect(page.locator('#content2')).not.toContainText('error:'); expect(pageErrors.map(String)).toEqual([]); }); - test('URL updates with cluster parameters after selection', async ({ page }) => { + test('URL updates with cluster parameters after selection', async ({ + page, + }) => { await page.goto('/hierarchical-memory'); // Wait for dropdown const dropdown = page.locator('#cluster_replica'); - await expect(dropdown).toBeVisible({ timeout: 20000 }); + await expect(dropdown).toBeVisible(); // URL should have been updated with cluster params await expect(page).toHaveURL(/cluster_name=/); @@ -72,45 +82,23 @@ test.describe('/hierarchical-memory page', () => { test('renders graphs for operator names containing double quotes', async ({ page, - request, }) => { // An arrangement operator embeds the debug formatting of its key in its - // name, so an index on a named column is called something like + // name, so the fixture index on a named column is called something like // `ArrangeBy[[Column(0, "id")]]`. Those double quotes have to be escaped // before they reach a DOT label, otherwise they terminate the label early // and GraphViz rejects the whole graph. - for (const query of [ - 'CREATE TABLE IF NOT EXISTS quoted_name_regression (id int, other int)', - 'CREATE INDEX IF NOT EXISTS quoted_name_regression_idx ON quoted_name_regression (id)', - ]) { - const response = await request.post('/api/sql', { data: { query } }); - // /api/sql reports SQL failures in the body, not the status code. - const body = response.ok() ? await response.json() : null; - const failure = !body - ? `HTTP ${response.status()}` - : body.results?.find((result) => result.error)?.error?.message; - // The endpoint runs as an unprivileged role, so DDL may be refused - // depending on how the environment is configured. Skip visibly rather - // than reporting a visualizer bug that isn't one. - test.skip(!!failure, `could not set up test index: ${failure}`); - } - const pageErrors: Error[] = []; page.on('pageerror', (error) => pageErrors.push(error)); - await page.goto( - '/hierarchical-memory?cluster_name=quickstart&replica_name=r1' - ); - - const content = page.locator('#content2'); + await page.goto(fixturePage('/hierarchical-memory')); - // Scope graphs live inside collapsed `.content` divs, so assert on - // presence rather than visibility. - await expect - .poll(() => content.locator('svg').count(), { timeout: 20000 }) - .toBeGreaterThan(0); + await expectGraphs(page); - // The arrangement label rendered, quotes and all. + // The fixture's dataflow rendered, and with it the arrangement label, + // quotes and all. + const content = page.locator('#content2'); + await expect(content).toContainText(FIXTURE_INDEX); await expect(content).toContainText('ArrangeBy'); await expect(content).not.toContainText('error:'); expect(pageErrors.map(String)).toEqual([]); @@ -120,7 +108,7 @@ test.describe('/hierarchical-memory page', () => { await page.goto('/hierarchical-memory'); const dropdown = page.locator('#cluster_replica'); - await expect(dropdown).toBeVisible({ timeout: 20000 }); + await expect(dropdown).toBeVisible(); const options = dropdown.locator('option'); const optionCount = await options.count(); @@ -131,18 +119,18 @@ test.describe('/hierarchical-memory page', () => { return; } - // Get initial URL - const initialUrl = page.url(); + // Pick an option other than the one the page settled on, so the URL has to + // change rather than being rewritten to the same value. + const selected = await dropdown.inputValue(); + const values = await options.evaluateAll((els) => + els.map((el) => (el as HTMLOptionElement).value) + ); + const newValue = values.find((value) => value !== selected)!; + const [clusterName, replicaName] = JSON.parse(newValue); - // Select a different option - const secondOption = options.nth(1); - const newValue = await secondOption.getAttribute('value'); - if (newValue) { - await dropdown.selectOption(newValue); + await dropdown.selectOption(newValue); - // URL should update - await page.waitForTimeout(1000); - expect(page.url()).not.toBe(initialUrl); - } + await expect(page).toHaveURL(new RegExp(`cluster_name=${clusterName}`)); + await expect(page).toHaveURL(new RegExp(`replica_name=${replicaName}`)); }); }); diff --git a/test/dataflow-visualizer/tests/memory.spec.ts b/test/dataflow-visualizer/tests/memory.spec.ts index f39e2aa57c99d..c268a4adc3e3a 100644 --- a/test/dataflow-visualizer/tests/memory.spec.ts +++ b/test/dataflow-visualizer/tests/memory.spec.ts @@ -7,7 +7,43 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { test, expect } from '@playwright/test'; +import { test, expect, Page } from '@playwright/test'; +import { + FIXTURE_CLUSTER, + FIXTURE_DATAFLOW, + FIXTURE_REPLICA, + FIXTURE_VIEW, + fixturePage, +} from './fixture'; + +/** Open a /memory page and wait for its dataflow table to render. */ +async function openMemoryPage(page: Page, url: string) { + await page.goto(url); + await expect(page.locator('text=Loading...')).toBeHidden(); +} + +/** + * Expand the fixture dataflow. + * + * Addressing the row by name matters: the table is ordered by record count + * descending, a dataflow that arranges nothing reports NULL rather than zero + * there, and NULLs sort first. Those are the transient + * `introspection-subscribe-*` dataflows and the storage `command_sequencer`. + * Expanding a transient one races its teardown, and the page then reports an + * unknown dataflow id instead of a graph. + */ +async function expandFixtureDataflow(page: Page) { + const row = page + .locator('table.dataflows tbody tr') + .filter({ hasText: FIXTURE_DATAFLOW }); + await expect(row).toBeVisible(); + await row.locator('button').click(); +} + +/** The panel one expanded dataflow renders into. */ +function vizSection(page: Page) { + return page.locator('div:has(> h3)').filter({ hasText: 'Name:' }); +} test.describe('/memory page', () => { test('page loads without errors', async ({ page }) => { @@ -18,23 +54,15 @@ test.describe('/memory page', () => { } }); - await page.goto('/memory'); + await openMemoryPage(page, '/memory'); - // Wait for loading to complete - await expect(page.locator('text=Loading...')).toBeHidden({ timeout: 10000 }); - - // Check for no console errors (excluding expected debug messages) - const unexpectedErrors = consoleErrors.filter( - (err) => !err.includes('could not get create view') - ); - expect(unexpectedErrors).toHaveLength(0); + expect(consoleErrors).toHaveLength(0); }); test('cluster replica dropdown is populated', async ({ page }) => { - await page.goto('/memory'); - - // Wait for loading to complete - await expect(page.locator('text=Loading...')).toBeHidden({ timeout: 10000 }); + // Deliberately not pinned to a replica: this covers the page picking one + // for itself when the URL names none. + await openMemoryPage(page, '/memory'); // Check that the dropdown exists and has options const dropdown = page.locator('#cluster_replica'); @@ -45,11 +73,22 @@ test.describe('/memory page', () => { expect(optionCount).toBeGreaterThan(0); }); - test('dataflow table renders', async ({ page }) => { - await page.goto('/memory'); + test('an unpinned page selects a user cluster', async ({ page }) => { + // Replicas are listed in cluster then replica name order, so a page that + // cannot identify a user cluster silently settles on the first of them, + // which is a builtin one. Those dataflows are not what a user opening this + // page came to look at, and their introspection relations are an order of + // magnitude more expensive to query. + await openMemoryPage(page, '/memory'); - // Wait for loading to complete - await expect(page.locator('text=Loading...')).toBeHidden({ timeout: 10000 }); + await expect(page.locator('#cluster_replica')).toHaveValue( + JSON.stringify([FIXTURE_CLUSTER, FIXTURE_REPLICA]) + ); + await expect(page).toHaveURL(new RegExp(`cluster_name=${FIXTURE_CLUSTER}`)); + }); + + test('dataflow table renders', async ({ page }) => { + await openMemoryPage(page, fixturePage('/memory')); // Check that the table exists with expected headers const table = page.locator('table.dataflows'); @@ -59,50 +98,44 @@ test.describe('/memory page', () => { await expect(table.locator('th:has-text("dataflow id")')).toBeVisible(); await expect(table.locator('th:has-text("index name")')).toBeVisible(); await expect(table.locator('th:has-text("records")')).toBeVisible(); - }); - test('clicking dataflow expand button shows visualization', async ({ page }) => { - await page.goto('/memory'); - - // Wait for loading to complete - await expect(page.locator('text=Loading...')).toBeHidden({ timeout: 10000 }); - - // Find the first expand button in the table - const expandButton = page.locator('table.dataflows tbody button').first(); + // The fixture index is what the tests below expand. + await expect( + table.locator('tbody tr').filter({ hasText: FIXTURE_DATAFLOW }) + ).toBeVisible(); + }); - // Skip test if no dataflows are present - if ((await expandButton.count()) === 0) { - test.skip(); - return; - } + test('clicking dataflow expand button shows visualization', async ({ + page, + }) => { + await openMemoryPage(page, fixturePage('/memory')); + await expandFixtureDataflow(page); - await expandButton.click(); + const viz = vizSection(page); + await expect(viz).toBeVisible(); + await expect(viz).toContainText(FIXTURE_DATAFLOW); + }); - // Wait for the visualization section to appear - const vizSection = page.locator('div:has(> h3)').filter({ hasText: 'Name:' }); - await expect(vizSection).toBeVisible({ timeout: 15000 }); + test('expanded dataflow shows the SQL that created it', async ({ page }) => { + // The page resolves a dataflow back to the view its index was built on and + // renders that view's SHOW CREATE VIEW. It is the only part of the page + // that reaches outside `mz_introspection`, so nothing else catches it + // breaking. + await openMemoryPage(page, fixturePage('/memory')); + await expandFixtureDataflow(page); + + const viz = vizSection(page); + await expect(viz).toContainText(`View: materialize.public.${FIXTURE_VIEW}`); + await expect(viz).toContainText('CREATE VIEW'); }); test('graphviz renders SVG when dataflow is expanded', async ({ page }) => { - await page.goto('/memory'); - - // Wait for loading to complete - await expect(page.locator('text=Loading...')).toBeHidden({ timeout: 10000 }); - - // Find the first expand button - const expandButton = page.locator('table.dataflows tbody button').first(); - - // Skip test if no dataflows are present - if ((await expandButton.count()) === 0) { - test.skip(); - return; - } - - await expandButton.click(); + await openMemoryPage(page, fixturePage('/memory')); + await expandFixtureDataflow(page); // Wait for SVG to be rendered by graphviz const svg = page.locator('svg').first(); - await expect(svg).toBeVisible({ timeout: 15000 }); + await expect(svg).toBeVisible(); // Verify it's a valid graphviz SVG (has graph elements) const graphElement = svg.locator('g.graph, g.node, g.edge').first(); @@ -110,10 +143,7 @@ test.describe('/memory page', () => { }); test('include system catalog checkbox works', async ({ page }) => { - await page.goto('/memory'); - - // Wait for loading to complete - await expect(page.locator('text=Loading...')).toBeHidden({ timeout: 10000 }); + await openMemoryPage(page, fixturePage('/memory')); const checkbox = page.locator('#include_system_catalog'); await expect(checkbox).toBeVisible(); diff --git a/test/dataflow-visualizer/tsconfig.typecheck.json b/test/dataflow-visualizer/tsconfig.typecheck.json index cfeef96aed893..6620e793d6062 100644 --- a/test/dataflow-visualizer/tsconfig.typecheck.json +++ b/test/dataflow-visualizer/tsconfig.typecheck.json @@ -9,14 +9,17 @@ "lib": ["ES2020", "DOM"], "strict": false, "moduleResolution": "bundler", - // The checked files are plain browser scripts with no imports, so nothing - // pulls `@types/*` in. List them explicitly to get `React` and `d3` as - // ambient globals. - "types": ["react", "react-dom", "d3"] + // The browser scripts have no imports, so nothing pulls `@types/*` in. + // List them explicitly to get `React` and `d3` as ambient globals, plus + // `node` for the `process.env` the Playwright specs read. + "types": ["react", "react-dom", "d3", "node"] }, "include": [ "../../src/environmentd/src/http/static/js/prometheus.js", "../../src/environmentd/src/http/static/js/metrics.jsx", - "../../src/environmentd/src/http/static/js/fetch.js" + "../../src/environmentd/src/http/static/js/fetch.js", + "../../src/environmentd/src/http/static/js/cluster-replica.jsx", + "tests/**/*.ts", + "playwright.config.ts" ] } From ddb56b0a27bffab2775625f9f2935b56e00ab241 Mon Sep 17 00:00:00 2001 From: Peter Larsen Date: Wed, 19 Aug 2026 17:47:01 -0400 Subject: [PATCH 10/10] Bump version for MySQL ingestion scenario due to RDS instance type change (#38363) ### Motivation The MySQL RDS instance backing SourceIngestionScenario is being upsized, which makes new results incomparable with prior runs. ### Description Bumping the scenario VERSION so the analytics data reflects the discontinuity. ### Verification Co-authored-by: Claude Fable 5 --- test/cluster-spec-sheet/mzcompose.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/cluster-spec-sheet/mzcompose.py b/test/cluster-spec-sheet/mzcompose.py index c02e246a7af7b..500aa80f1621c 100644 --- a/test/cluster-spec-sheet/mzcompose.py +++ b/test/cluster-spec-sheet/mzcompose.py @@ -2111,6 +2111,9 @@ def inner() -> None: class SourceIngestionScenario(ClusterScalingScenario): + # 1.1.0: MySQL RDS instance type changed + VERSION: str = "1.1.0" + def name(self) -> str: return "source_ingestion"