diff --git a/Cargo.lock b/Cargo.lock index bdbd43576308d..3e1b0dbcd3a39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6175,6 +6175,7 @@ dependencies = [ "timely", "tokio", "tokio-postgres", + "tokio-stream", "tracing", "tracing-core", "tracing-opentelemetry", diff --git a/Cargo.toml b/Cargo.toml index 2752a257f242e..c04ed90d17184 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -494,7 +494,7 @@ sentry-tracing = "0.49.1" serde = { version = "1.0.219", features = ["derive"] } serde-aux = "4.7.0" serde_bytes = "0.11.19" -serde_json = "1.0.150" +serde_json = { version = "1.0.150", features = ["raw_value"] } serde_plain = "1.0.2" serde_repr = "0.1.20" serde_urlencoded = "0.7.1" diff --git a/doc/user/content/console/_index.md b/doc/user/content/console/_index.md index ec203c1eaf4af..8e2709e77afde 100644 --- a/doc/user/content/console/_index.md +++ b/doc/user/content/console/_index.md @@ -5,7 +5,8 @@ disable_list: true disable_toc: true menus: main: - weight: 5 + parent: "get-started" + weight: 24 identifier: 'console' aliases: - /self-managed/v25.2/console/ diff --git a/doc/user/content/reference/system-catalog/mz_catalog.md b/doc/user/content/reference/system-catalog/mz_catalog.md index 8a29ab72e09d4..199e4d47f6222 100644 --- a/doc/user/content/reference/system-catalog/mz_catalog.md +++ b/doc/user/content/reference/system-catalog/mz_catalog.md @@ -671,6 +671,11 @@ Field | Type | Meaning The `mz_views` table contains a row for each view in the system. +A few generated system views in `mz_internal` (the `mz_builtin_*` views, +which list every builtin object) are shown with a short placeholder in +`definition`, `create_sql`, and `redacted_create_sql` instead of their full +SQL, which would embed metadata about every builtin object. + Field | Type | Meaning ---------------|----------------------|---------- diff --git a/doc/user/content/reference/system-catalog/mz_internal.md b/doc/user/content/reference/system-catalog/mz_internal.md index e8d4a88ce2fc4..65dcf69958410 100644 --- a/doc/user/content/reference/system-catalog/mz_internal.md +++ b/doc/user/content/reference/system-catalog/mz_internal.md @@ -646,6 +646,22 @@ all database objects in the system. | `object_id` | [`text`] | The ID of the dependent object. Corresponds to [`mz_objects.id`](../mz_catalog/#mz_objects). | | `referenced_object_id` | [`text`] | The ID of the referenced object. Corresponds to [`mz_objects.id`](../mz_catalog/#mz_objects). | +## `mz_object_graph_edges` + +The `mz_object_graph_edges` view describes the dependency edges between maintained +objects, as rendered by object dependency graphs. + +It unions the dataflow-layer dependencies from +[`mz_materialization_dependencies`](#mz_materialization_dependencies), restricted to +indexes, materialized views, sinks, sources, and tables, with the source-to-subsource +and source-to-table edges that stand in for sources whose subsources carry the data. + + +| Field | Type | Meaning | +| --------------- | -------- | -------------------------------------------------------------------------------------- | +| `object_id` | [`text`] | The ID of the dependent object. Corresponds to [`mz_objects.id`](../mz_catalog/#mz_objects). | +| `dependency_id` | [`text`] | The ID of the object it depends on. Corresponds to [`mz_objects.id`](../mz_catalog/#mz_objects). | + ## `mz_object_fully_qualified_names` The `mz_object_fully_qualified_names` view enriches the [`mz_catalog.mz_objects`](/reference/system-catalog/mz_catalog/#mz_objects) view with namespace information. @@ -1500,6 +1516,8 @@ The `mz_webhook_sources` table contains a row for each webhook source in the sys + + diff --git a/doc/user/data/examples/create_connection.yml b/doc/user/data/examples/create_connection.yml index 57486bb95b9cd..fadab6d0e63dd 100644 --- a/doc/user/data/examples/create_connection.yml +++ b/doc/user/data/examples/create_connection.yml @@ -656,7 +656,8 @@ description: | *Value:* `text`. Required. - The name of the AWS PrivateLink service. + The name of the AWS VPC endpoint service, which always starts with + `com.amazonaws.`. - name: "`AVAILABILITY ZONES`" description: | *Value:* `text[]`. Required. diff --git a/misc/python/materialize/checks/all_checks/builtin_version_pin.py b/misc/python/materialize/checks/all_checks/builtin_version_pin.py index 081bf28175519..407a4cd46deee 100644 --- a/misc/python/materialize/checks/all_checks/builtin_version_pin.py +++ b/misc/python/materialize/checks/all_checks/builtin_version_pin.py @@ -57,9 +57,9 @@ "mz_sinks": MzVersion.parse_mz("v26.38.0"), "mz_ssh_tunnel_connections": MzVersion.parse_mz("v26.37.0"), "mz_system_privileges": MzVersion.parse_mz("v26.30.0"), - "mz_tables": None, + "mz_tables": MzVersion.parse_mz("v26.39.0"), "mz_types": None, - "mz_views": None, + "mz_views": MzVersion.parse_mz("v26.39.0"), } diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py index 6eea32f67d41a..322c6f05cf0fa 100644 --- a/misc/python/materialize/mzcompose/__init__.py +++ b/misc/python/materialize/mzcompose/__init__.py @@ -116,6 +116,7 @@ def get_minimal_system_parameters( "enable_s3_tables_region_check": "false", "enable_statement_lifecycle_logging": "true", "enable_storage_introspection_logs": "true", + "enable_compute_error_distinct": "true", "enable_compute_temporal_bucketing": "true", "enable_variadic_left_join_lowering": "true", "enable_worker_core_affinity": "true", @@ -781,6 +782,7 @@ def get_default_system_parameters( "user_id_pool_batch_size", "webhook_max_request_size_bytes", "webhook_validation_memory_budget_bytes", + "subscribe_max_buffered_bytes", "cluster_controller_tick_interval", "default_cluster_reconfiguration_timeout", "read_then_write_max_dependencies", diff --git a/misc/python/materialize/parallel_workload/action.py b/misc/python/materialize/parallel_workload/action.py index 5c5957fbd3777..ebe9e4a9d8ad8 100644 --- a/misc/python/materialize/parallel_workload/action.py +++ b/misc/python/materialize/parallel_workload/action.py @@ -2986,6 +2986,7 @@ def __init__( self.flags_with_values["enable_compute_temporal_bucketing"] = ( BOOLEAN_FLAG_VALUES ) + self.flags_with_values["enable_compute_error_distinct"] = BOOLEAN_FLAG_VALUES self.flags_with_values["enable_alter_table_add_column"] = BOOLEAN_FLAG_VALUES self.flags_with_values["enable_bounded_staleness_isolation"] = ( BOOLEAN_FLAG_VALUES @@ -3132,6 +3133,7 @@ def __init__( "compute_server_maintenance_interval", "compute_dataflow_max_inflight_bytes", "compute_dataflow_max_inflight_bytes_cc", + "subscribe_max_buffered_bytes", "compute_flat_map_fuel", "consolidating_vec_growth_dampener", "compute_hydration_concurrency", diff --git a/src/adapter-types/src/dyncfgs.rs b/src/adapter-types/src/dyncfgs.rs index 7b0da8964ee6e..59798c1d15000 100644 --- a/src/adapter-types/src/dyncfgs.rs +++ b/src/adapter-types/src/dyncfgs.rs @@ -297,6 +297,25 @@ pub const WEBHOOK_VALIDATION_MEMORY_BUDGET_BYTES: Config = Config::new( "The maximum bytes of temporary storage a webhook CHECK expression may allocate while validating one request.", ); +/// Budget for the backlog a `SUBSCRIBE` (or `COPY (SUBSCRIBE ...) TO STDOUT`) +/// may accumulate in environmentd while waiting for a slow client to read. +/// +/// The subscribe producer runs on the non-blockable coordinator loop, so it +/// cannot apply backpressure to a slow client. Instead the coordinator retires +/// the subscribe once its buffered backlog exceeds this budget, bounding the +/// memory a slow client can make the shared process hold. +/// +/// The backlog excludes the message the client is currently draining, so this +/// bounds the accumulation of messages, not the size of any single one. +/// `max_result_size` is what bounds an individual message. A client that keeps +/// up holds at most one message at a time, so it stays at a zero backlog and a +/// large snapshot batch is delivered rather than retired. +pub const SUBSCRIBE_MAX_BUFFERED_BYTES: Config = Config::new( + "subscribe_max_buffered_bytes", + 128 * 1024 * 1024, + "Maximum bytes a SUBSCRIBE may buffer in environmentd for a slow client before it is retired with an error.", +); + /// Number of user IDs to pre-allocate in a batch. Pre-allocating IDs avoids /// a persist write + oracle call per DDL statement. pub const USER_ID_POOL_BATCH_SIZE: Config = Config::new( @@ -464,6 +483,7 @@ pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet { .add(&MCP_REQUEST_TIMEOUT) .add(&WEBHOOK_MAX_REQUEST_SIZE_BYTES) .add(&WEBHOOK_VALIDATION_MEMORY_BUDGET_BYTES) + .add(&SUBSCRIBE_MAX_BUFFERED_BYTES) .add(&USER_ID_POOL_BATCH_SIZE) .add(&GROUP_COMMIT_MAX_ATTEMPTS) .add(&CONSOLE_OIDC_CLIENT_ID) diff --git a/src/adapter/Cargo.toml b/src/adapter/Cargo.toml index ffa3da657fe53..cccc24a57cfdb 100644 --- a/src/adapter/Cargo.toml +++ b/src/adapter/Cargo.toml @@ -93,6 +93,7 @@ static_assertions.workspace = true timely.workspace = true tokio.workspace = true tokio-postgres.workspace = true +tokio-stream.workspace = true tracing.workspace = true tracing-core.workspace = true tracing-opentelemetry.workspace = true diff --git a/src/adapter/src/active_compute_sink.rs b/src/adapter/src/active_compute_sink.rs index 4520868c4617e..358c76ee0f3f3 100644 --- a/src/adapter/src/active_compute_sink.rs +++ b/src/adapter/src/active_compute_sink.rs @@ -10,8 +10,9 @@ //! Coordinator bookkeeping for active compute sinks. use std::cmp::Ordering; -use std::collections::BTreeSet; +use std::collections::{BTreeSet, VecDeque}; use std::num::NonZeroUsize; +use std::sync::{Arc, Mutex}; use mz_adapter_types::connection::ConnectionId; use mz_compute_client::protocol::response::SubscribeBatch; @@ -88,6 +89,62 @@ pub enum ActiveComputeSinkRetireReason { /// The compute sink was forcibly terminated because an object it depended on /// was dropped. DependencyDropped(DroppedDependency), + /// The compute sink was retired because its coordinator-side buffer exceeded + /// its budget while the client was not reading fast enough. Carries the + /// buffered and budget byte counts for the terminal error. + BufferExceeded { + buffered_bytes: usize, + max_buffered_bytes: usize, + }, +} + +/// Overhead charged to every queued message on top of its payload. +/// +/// A frontier-only advance carries no rows, so its payload is zero bytes. It +/// still costs real memory: a node in the unbounded channel and an entry in +/// `footprints`. Charging payload alone would leave the backlog flat while +/// those two grow without bound, so the retire check would never trip on a +/// stalled client that only receives progress messages. The fixed charge makes +/// the budget bound message *count* as well as payload bytes. +const SUBSCRIBE_MESSAGE_OVERHEAD_BYTES: usize = 1024; + +/// Footprints of the subscribe messages queued to the client but not yet +/// drained, oldest first. The producer pushes one per message sent, the +/// client-writer task pops as it drains. FIFO delivery keeps the queue aligned +/// with the channel. +/// +/// The message at the front is the one the client is currently draining. It is +/// always tolerated, however large, so a client working through one big batch is +/// not retired. Only what is queued behind it counts as backlog. +#[derive(Debug, Default)] +pub struct SubscribeBacklogAccounting { + /// Per-message footprints, in send order. + footprints: VecDeque, + /// Sum of `footprints`. + total: usize, +} + +impl SubscribeBacklogAccounting { + /// Records a queued message of the given footprint. + pub fn push(&mut self, footprint: usize) { + self.footprints.push_back(footprint); + self.total = self.total.saturating_add(footprint); + } + + /// Records the oldest queued message being drained by the client writer. + pub fn pop(&mut self) { + if let Some(footprint) = self.footprints.pop_front() { + self.total = self.total.saturating_sub(footprint); + } + } + + /// Bytes queued behind the message the client is currently draining. That + /// message is tolerated whatever its size, so only this counts against the + /// budget. + pub fn backlog_size(&self) -> usize { + self.total + .saturating_sub(self.footprints.front().copied().unwrap_or(0)) + } } /// A description of an active subscribe from coord's perspective @@ -105,6 +162,17 @@ pub struct ActiveSubscribe { // The responses have the form `PeekResponseUnary` but should perhaps // become `SubscribeResponse`. pub channel: mpsc::UnboundedSender, + /// Footprints of the messages queued in `channel` but not yet drained by the + /// client writer. Shared with the receiver side, which pops as it drains. + /// + /// The producer runs on the non-blockable coordinator loop and cannot block + /// on a slow client, so instead of applying backpressure the coordinator + /// watches `backlog_size` against `max_buffered_bytes` and retires the + /// subscribe once the backlog exceeds it. + pub backlog_accounting: Arc>, + /// Budget for the buffered backlog. A snapshot of `subscribe_max_buffered_bytes` + /// taken when the subscribe was created. + pub max_buffered_bytes: usize, /// Whether progress information should be emitted. pub emit_progress: bool, /// The logical timestamp at which the subscribe began execution. @@ -154,8 +222,9 @@ impl ActiveSubscribe { } } + let bytes = row_buf.byte_len(); let row_iter = Box::new(row_buf.into_row_iter()); - self.send(PeekResponseUnary::Rows(row_iter)); + self.send(PeekResponseUnary::Rows(row_iter), bytes); } } @@ -178,7 +247,7 @@ impl ActiveSubscribe { mz_ore::iter::consolidate_update_iter(merged) } Err(s) => { - self.send(PeekResponseUnary::Error(s)); + self.send(PeekResponseUnary::Error(s), 0); return true; } }; @@ -376,8 +445,9 @@ impl ActiveSubscribe { }; let rows = output_builder.build(); + let bytes = rows.byte_len(); let rows = Box::new(rows.into_row_iter()); - self.send(PeekResponseUnary::Rows(rows)); + self.send(PeekResponseUnary::Rows(rows), bytes); // Emit progress message if requested. Don't emit progress for the first // batch if the upper is exactly `as_of` (we're guaranteed it is not @@ -402,15 +472,34 @@ impl ActiveSubscribe { ActiveComputeSinkRetireReason::DependencyDropped(d) => { PeekResponseUnary::DependencyDropped(d) } + ActiveComputeSinkRetireReason::BufferExceeded { + buffered_bytes, + max_buffered_bytes, + } => PeekResponseUnary::Error( + AdapterError::SubscribeFellBehind { + buffered_bytes, + max_buffered_bytes, + } + .to_string(), + ), }; - self.send(message); + self.send(message, 0); } /// Sends a message to the client if the subscribe has not already completed /// and if the client has not already gone away. - fn send(&self, response: PeekResponseUnary) { - // TODO(benesch): the lack of backpressure here can result in - // unbounded memory usage. + /// + /// `bytes` is the message's payload size. Its footprint (payload plus a fixed + /// per-message overhead) is recorded in `backlog_accounting` here and + /// released by the receiver side when the message is drained. Overflow of + /// the budget is detected by the coordinator after `process_response` + /// returns, not here, because this method cannot retire the sink. + fn send(&self, response: PeekResponseUnary, bytes: usize) { + let footprint = bytes.saturating_add(SUBSCRIBE_MESSAGE_OVERHEAD_BYTES); + self.backlog_accounting + .lock() + .expect("subscribe backlog accounting poisoned") + .push(footprint); let _ = self.channel.send(response); } } @@ -456,6 +545,13 @@ impl ActiveCopyTo { ActiveComputeSinkRetireReason::DependencyDropped(dep) => { Err(dep.to_concurrent_dependency_drop()) } + ActiveComputeSinkRetireReason::BufferExceeded { + buffered_bytes, + max_buffered_bytes, + } => Err(AdapterError::SubscribeFellBehind { + buffered_bytes, + max_buffered_bytes, + }), }; let _ = self.tx.send(message); } @@ -473,3 +569,49 @@ pub(crate) struct ActiveCopyFrom { /// Context of the SQL session that ran the statement. pub ctx: ExecuteContext, } + +#[cfg(test)] +mod tests { + use crate::active_compute_sink::SubscribeBacklogAccounting; + + /// The backlog excludes the message being drained, and zero-payload + /// messages (footprint = overhead only) still accumulate against it. + #[mz_ore::test] + fn test_subscribe_backlog_accounting() { + let mut acc = SubscribeBacklogAccounting::default(); + assert_eq!(acc.backlog_size(), 0); + + // A single large message is fully tolerated: nothing is queued behind it. + acc.push(10_000); + assert_eq!(acc.backlog_size(), 0); + + // Near-empty messages (only per-message overhead) still build backlog, so + // a flood of frontier-only advances cannot grow without bound. + acc.push(1_024); + acc.push(1_024); + assert_eq!(acc.backlog_size(), 2_048); + + // Draining the oldest message advances the tolerated front. + acc.pop(); + assert_eq!(acc.backlog_size(), 1_024); + + acc.pop(); + acc.pop(); + assert_eq!(acc.backlog_size(), 0); + } + + /// A client that drains each message before the next is sent never + /// accumulates backlog, however many messages flow and however large they + /// are. This is the property that keeps a well-behaved subscribe from ever + /// being retired, so it is asserted after every step rather than at the end. + #[mz_ore::test] + fn test_subscribe_backlog_keeping_up_client() { + let mut acc = SubscribeBacklogAccounting::default(); + for i in 0..1_000 { + acc.push(1_024 + i * 4_096); + assert_eq!(acc.backlog_size(), 0, "message {i} built backlog"); + acc.pop(); + assert_eq!(acc.backlog_size(), 0, "message {i} left backlog behind"); + } + } +} diff --git a/src/adapter/src/catalog/builtin_table_updates.rs b/src/adapter/src/catalog/builtin_table_updates.rs index 20670dc4b11e7..61e404ceb6f01 100644 --- a/src/adapter/src/catalog/builtin_table_updates.rs +++ b/src/adapter/src/catalog/builtin_table_updates.rs @@ -20,14 +20,14 @@ use mz_catalog::builtin::{ MZ_HISTORY_RETENTION_STRATEGIES, MZ_INDEX_COLUMNS, MZ_LICENSE_KEYS, MZ_LIST_TYPES, MZ_MAP_TYPES, MZ_MATERIALIZED_VIEW_REFRESH_STRATEGIES, MZ_OBJECT_DEPENDENCIES, MZ_OBJECT_GLOBAL_IDS, MZ_OPERATORS, MZ_PSEUDO_TYPES, MZ_REPLACEMENTS, MZ_ROLE_AUTH, - MZ_SESSIONS, MZ_SOURCE_REFERENCES, MZ_STORAGE_USAGE_BY_SHARD, MZ_SUBSCRIPTIONS, MZ_TABLES, - MZ_TYPE_PG_METADATA, MZ_TYPES, MZ_VIEWS, MZ_WEBHOOKS_SOURCES, + MZ_SESSIONS, MZ_SOURCE_REFERENCES, MZ_STORAGE_USAGE_BY_SHARD, MZ_SUBSCRIPTIONS, + MZ_TYPE_PG_METADATA, MZ_TYPES, MZ_WEBHOOKS_SOURCES, }; use mz_catalog::durable::SourceReferences; use mz_catalog::memory::error::Error; use mz_catalog::memory::objects::{ CatalogEntry, CatalogItem, DataSourceDesc, Func, Index, MaterializedView, Table, - TableDataSource, Type, View, + TableDataSource, Type, }; use mz_expr::MirScalarExpr; use mz_license_keys::ValidatedLicenseKey; @@ -162,13 +162,6 @@ impl CatalogState { let privileges = privileges_row.unpack_first(); let mut updates = match entry.item() { CatalogItem::Index(index) => self.pack_index_update(id, index, diff), - CatalogItem::Table(table) => { - // Source-table metadata (mz_postgres/mysql/sql_server/kafka_source_tables) - // is now derived from the persisted create_sql by materialized - // views over mz_catalog_raw, so ingestion-export tables need no - // special packing here. - self.pack_table_update(id, oid, schema_id, name, owner_id, privileges, diff, table) - } CatalogItem::Source(source) => { match &source.data_source { DataSourceDesc::Webhook { .. } => { @@ -185,9 +178,6 @@ impl CatalogState { | DataSourceDesc::Catalog => vec![], } } - CatalogItem::View(view) => { - self.pack_view_update(id, oid, schema_id, name, owner_id, privileges, view, diff) - } CatalogItem::MaterializedView(mview) => { self.pack_materialized_view_update(id, mview, diff) } @@ -208,7 +198,11 @@ impl CatalogState { // of it, so once metric sinks are user-creatable this gap would let a reader // egress another role's rows with no catalog relation the owner could see it in. // SQL-572 has to land before user-facing metric sink DDL does. - CatalogItem::Log(_) | CatalogItem::Secret(_) | CatalogItem::MetricSink(_) => vec![], + CatalogItem::Table(_) + | CatalogItem::View(_) + | CatalogItem::Log(_) + | CatalogItem::Secret(_) + | CatalogItem::MetricSink(_) => vec![], // Connection details (mz_kafka_connections, mz_ssh_tunnel_connections, // mz_aws_connections, mz_aws_privatelink_connections) are now derived // from the persisted create_sql by materialized views over @@ -337,110 +331,6 @@ impl CatalogState { ) } - fn pack_table_update( - &self, - id: CatalogItemId, - oid: u32, - schema_id: &SchemaSpecifier, - name: &str, - owner_id: &RoleId, - privileges: Datum, - diff: Diff, - table: &Table, - ) -> Vec> { - let redacted = table.create_sql.as_ref().map(|create_sql| { - mz_sql::parse::parse(create_sql) - .unwrap_or_else(|_| panic!("create_sql cannot be invalid: {}", create_sql)) - .into_element() - .ast - .to_ast_string_redacted() - }); - let source_id = if let TableDataSource::DataSource { - desc: DataSourceDesc::IngestionExport { ingestion_id, .. }, - .. - } = &table.data_source - { - Some(ingestion_id.to_string()) - } else { - None - }; - - vec![BuiltinTableUpdate::row( - &*MZ_TABLES, - Row::pack_slice(&[ - Datum::String(&id.to_string()), - Datum::UInt32(oid), - Datum::String(&schema_id.to_string()), - Datum::String(name), - Datum::String(&owner_id.to_string()), - privileges, - if let Some(create_sql) = &table.create_sql { - Datum::String(create_sql) - } else { - Datum::Null - }, - if let Some(redacted) = &redacted { - Datum::String(redacted) - } else { - Datum::Null - }, - if let Some(source_id) = source_id.as_ref() { - Datum::String(source_id) - } else { - Datum::Null - }, - ]), - diff, - )] - } - - fn pack_view_update( - &self, - id: CatalogItemId, - oid: u32, - schema_id: &SchemaSpecifier, - name: &str, - owner_id: &RoleId, - privileges: Datum, - view: &View, - diff: Diff, - ) -> Vec> { - let create_stmt = mz_sql::parse::parse(&view.create_sql) - .unwrap_or_else(|e| { - panic!( - "create_sql cannot be invalid: `{}` --- error: `{}`", - view.create_sql, e - ) - }) - .into_element() - .ast; - let query = match &create_stmt { - Statement::CreateView(stmt) => &stmt.definition.query, - _ => unreachable!(), - }; - - let mut query_string = query.to_ast_string_stable(); - // PostgreSQL appends a semicolon in `pg_views.definition`, we - // do the same for compatibility's sake. - query_string.push(';'); - - vec![BuiltinTableUpdate::row( - &*MZ_VIEWS, - Row::pack_slice(&[ - Datum::String(&id.to_string()), - Datum::UInt32(oid), - Datum::String(&schema_id.to_string()), - Datum::String(name), - Datum::String(&query_string), - Datum::String(&owner_id.to_string()), - privileges, - Datum::String(&view.create_sql), - Datum::String(&create_stmt.to_ast_string_redacted()), - ]), - diff, - )] - } - fn pack_materialized_view_update( &self, id: CatalogItemId, diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index 5adf8fe395df3..8af544edca8b5 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -390,6 +390,32 @@ static MIGRATIONS: LazyLock> = LazyLock::new(|| { MZ_CATALOG_SCHEMA, "mz_audit_events", ), + // Required because we added the `mz_object_graph_edges_ind` builtin index. + // make_mz_indexes inlines the builtin-index set as VALUES, so any add or + // remove changes its SQL fingerprint and requires an explicit replacement. + MigrationStep::replacement( + "26.39.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_indexes", + ), + // Converting mz_tables and mz_views from builtin tables to + // materialized views over mz_catalog_raw changes their catalog + // fingerprints, so each needs an explicit replacement step. See the + // NOTE above: this version must stay at the workspace's current dev + // version until the change ships. + MigrationStep::replacement( + "26.39.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_tables", + ), + MigrationStep::replacement( + "26.39.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_views", + ), ] }); diff --git a/src/adapter/src/command.rs b/src/adapter/src/command.rs index 4c49fbf82bfc2..d343becaf3010 100644 --- a/src/adapter/src/command.rs +++ b/src/adapter/src/command.rs @@ -781,6 +781,7 @@ pub enum ExecuteResponse { /// Updates to the requested source or view will be streamed to the /// contained receiver. Subscribing { + #[derivative(Debug = "ignore")] rx: RowBatchStream, ctx_extra: ExecuteContextGuard, instance_id: ComputeInstanceId, diff --git a/src/adapter/src/coord/message_handler.rs b/src/adapter/src/coord/message_handler.rs index 8f80962510230..a8bc6342976c7 100644 --- a/src/adapter/src/coord/message_handler.rs +++ b/src/adapter/src/coord/message_handler.rs @@ -701,10 +701,37 @@ impl Coordinator { self.active_compute_sinks.get_mut(&sink_id) { let finished = active_subscribe.process_response(response); - if finished { + // Read the backlog into a `Copy` local so the mutable borrow + // of `active_subscribe` (and the accounting lock) ends before + // we call `self.retire_compute_sinks`. The producer runs on + // this loop, so it cannot block on a slow client. Instead we + // bound the backlog here and retire the subscribe once it + // exceeds the budget. + // + // The backlog excludes the message the client is currently + // draining, so a client working through a single large batch + // (e.g. the initial snapshot) is never retired for it. + let buffered_bytes = active_subscribe + .backlog_accounting + .lock() + .expect("subscribe backlog accounting poisoned") + .backlog_size(); + let max_buffered_bytes = active_subscribe.max_buffered_bytes; + + let reason = if finished { + Some(ActiveComputeSinkRetireReason::Finished) + } else if buffered_bytes > max_buffered_bytes { + Some(ActiveComputeSinkRetireReason::BufferExceeded { + buffered_bytes, + max_buffered_bytes, + }) + } else { + None + }; + if let Some(reason) = reason { let retire_notify = self .retire_compute_sinks(btreemap! { - sink_id => ActiveComputeSinkRetireReason::Finished, + sink_id => reason, }) .await; // `retire_compute_sinks` waits before sending the terminal diff --git a/src/adapter/src/coord/read_then_write.rs b/src/adapter/src/coord/read_then_write.rs index 747734d40b62f..d4f05b0a2b954 100644 --- a/src/adapter/src/coord/read_then_write.rs +++ b/src/adapter/src/coord/read_then_write.rs @@ -99,6 +99,16 @@ impl Coordinator { conn_id: conn_id.clone(), session_uuid, channel: tx, + backlog_accounting: std::sync::Arc::new(std::sync::Mutex::new( + crate::active_compute_sink::SubscribeBacklogAccounting::default(), + )), + // This internal subscribe is drained by the coordinator for OCC + // read-then-write, not by a slow external client, so the slow-client + // backlog budget must not apply. A large read-then-write read set + // (e.g. an UPDATE that rewrites every row of a big table) + // legitimately exceeds the budget, so bounding it here would + // spuriously retire the statement with `SubscribeFellBehind`. + max_buffered_bytes: usize::MAX, emit_progress: true, // We need progress updates for OCC as_of, arity, diff --git a/src/adapter/src/coord/sequencer/inner.rs b/src/adapter/src/coord/sequencer/inner.rs index 3a8ccfd116224..af7b3b2a5501b 100644 --- a/src/adapter/src/coord/sequencer/inner.rs +++ b/src/adapter/src/coord/sequencer/inner.rs @@ -67,7 +67,7 @@ use mz_sql::names::{ SchemaSpecifier, SystemObjectId, }; use mz_sql::plan::{ - AlterMaterializedViewApplyReplacementPlan, ConnectionDetails, NetworkPolicyRule, + AlterMaterializedViewApplyReplacementPlan, ConnectionDetails, NetworkPolicyRule, PlanError, StatementContext, }; use mz_sql::pure::{PurifiedSourceExport, generate_subsource_statements}; @@ -95,6 +95,7 @@ use mz_sql_parser::ast::{ use mz_ssh_util::keys::SshKeyPairSet; use mz_storage_client::controller::ExportDescription; use mz_storage_types::AlterCompatible; +use mz_storage_types::connections::AwsPrivatelinkConnection; use mz_storage_types::connections::inline::IntoInlineConnection; use mz_storage_types::controller::StorageError; use mz_storage_types::sources::SourceConnection; @@ -183,6 +184,18 @@ where } } +/// Rejects connection options whose values cannot work, independent of any +/// external system. +fn check_connection_details(details: &ConnectionDetails) -> Result<(), PlanError> { + match details { + ConnectionDetails::AwsPrivatelink(privatelink) => { + AwsPrivatelinkConnection::check_service_name(&privatelink.service_name) + .map_err(PlanError::InvalidPrivatelinkServiceName) + } + _ => Ok(()), + } +} + struct DropOps { ops: Vec, dropped_active_db: bool, @@ -720,6 +733,10 @@ impl Coordinator { plan: plan::CreateConnectionPlan, resolved_ids: ResolvedIds, ) { + if let Err(err) = check_connection_details(&plan.connection.details) { + return ctx.retire(Err(AdapterError::PlanError(err))); + } + let (connection_id, connection_gid) = match self.allocate_user_id().await { Ok(item_id) => item_id, Err(err) => return ctx.retire(Err(err)), @@ -3777,6 +3794,12 @@ impl Coordinator { } }; + // `conn` is the whole re-planned connection, so this also rejects a + // stored value the statement did not touch. + if let Err(err) = check_connection_details(&conn.details) { + return ctx.retire(Err(AdapterError::InvalidAlter("CONNECTION", err))); + } + // Inspect guarded secrets whether or not validation was requested, // before the altered connection is installed in the catalog. if let Err(err) = self diff --git a/src/adapter/src/coord/sequencer/inner/subscribe.rs b/src/adapter/src/coord/sequencer/inner/subscribe.rs index 331398a83013e..4e55f9b3347a2 100644 --- a/src/adapter/src/coord/sequencer/inner/subscribe.rs +++ b/src/adapter/src/coord/sequencer/inner/subscribe.rs @@ -7,8 +7,10 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use futures::StreamExt; use maplit::btreemap; use mz_adapter_types::connection::ConnectionId; +use mz_adapter_types::dyncfgs::SUBSCRIBE_MAX_BUFFERED_BYTES; use mz_cluster_client::ReplicaId; use mz_compute_types::ComputeInstanceId; use mz_compute_types::dataflows::DataflowDescription; @@ -22,14 +24,17 @@ use mz_repr::optimize::{OptimizerFeatures, OverrideFrom}; use mz_sql::plan::{self, QueryWhen, SubscribeFrom}; use mz_sql::session::metadata::SessionMetadata; use std::collections::BTreeSet; +use std::sync::{Arc, Mutex}; use timely::progress::Antichain; use tokio::sync::mpsc; +use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{Instrument, Span}; use uuid::Uuid; -use crate::active_compute_sink::{ActiveComputeSink, ActiveSubscribe}; +use crate::active_compute_sink::{ActiveComputeSink, ActiveSubscribe, SubscribeBacklogAccounting}; use crate::command::ExecuteResponse; use crate::coord::appends::BuiltinTableAppendNotify; +use crate::coord::peek::PeekResponseUnary; use crate::coord::sequencer::inner::{return_if_err, spawn_linearized_read_ts}; use crate::coord::sequencer::{check_log_reads, emit_optimizer_notices}; use crate::coord::{ @@ -547,11 +552,16 @@ impl Coordinator { ) -> Result<(ExecuteResponse, BuiltinTableAppendNotify), AdapterError> { let sink_id = df_desc.sink_id(); - let (tx, rx) = mpsc::unbounded_channel(); + let (tx, rx) = mpsc::unbounded_channel::(); + let backlog_accounting = Arc::new(Mutex::new(SubscribeBacklogAccounting::default())); + let max_buffered_bytes = + SUBSCRIBE_MAX_BUFFERED_BYTES.get(self.catalog().system_config().dyncfgs()); let active_subscribe = ActiveSubscribe { conn_id: conn_id.clone(), session_uuid, channel: tx, + backlog_accounting: Arc::clone(&backlog_accounting), + max_buffered_bytes, emit_progress: plan.emit_progress, as_of: df_desc .as_of @@ -602,8 +612,20 @@ impl Coordinator { // Explicitly drop read holds, just to make it obvious what's happening. drop(read_holds); + // Wrap the receiver so draining a message releases its footprint from the + // shared accounting. FIFO delivery keeps the queue aligned with the + // channel, so popping the oldest footprint matches the message just + // drained. This keeps the accounting equal to the currently buffered + // depth, which the coordinator watches to bound this subscribe. + let rx = UnboundedReceiverStream::new(rx).map(move |response| { + backlog_accounting + .lock() + .expect("subscribe backlog accounting poisoned") + .pop(); + response + }); let resp = ExecuteResponse::Subscribing { - rx, + rx: Box::new(rx), ctx_extra: std::mem::take(ctx_extra), instance_id: cluster_id, }; diff --git a/src/adapter/src/error.rs b/src/adapter/src/error.rs index d84be0969198c..106fa7da919e9 100644 --- a/src/adapter/src/error.rs +++ b/src/adapter/src/error.rs @@ -21,6 +21,7 @@ use mz_compute_client::controller::error::InstanceMissing; use mz_compute_types::ComputeInstanceId; use mz_controller_types::ClusterId; use mz_expr::EvalError; +use mz_ore::cast::CastFrom; use mz_ore::error::ErrorExt; use mz_ore::stack::RecursionLimitError; use mz_ore::str::StrExt; @@ -180,6 +181,16 @@ pub enum AdapterError { }, /// Result size of a query is too large. ResultSize(String), + /// A `SUBSCRIBE` (or `COPY (SUBSCRIBE ...) TO STDOUT`) buffered more bytes in + /// environmentd than its budget allows because the client was not reading + /// fast enough. The subscribe is retired rather than let one slow client + /// grow the shared process's memory without bound. + SubscribeFellBehind { + /// Bytes buffered when the budget was exceeded. + buffered_bytes: usize, + /// The `subscribe_max_buffered_bytes` budget that was exceeded. + max_buffered_bytes: usize, + }, /// The specified feature is not permitted in safe mode. SafeModeViolation(String), /// The current transaction had the wrong set of write locks. @@ -743,6 +754,11 @@ impl AdapterError { ), AdapterError::Catalog(c) => c.hint(), AdapterError::Eval(e) => e.hint(), + AdapterError::SubscribeFellBehind { .. } => Some( + "The client is not reading results fast enough. Use a client that reads output \ + without buffering, or raise the subscribe_max_buffered_bytes system variable." + .to_string(), + ), AdapterError::AlterClusterUnmanagedWhileReconfiguring => Some( "Cancel the reconfiguration by altering the cluster back to its current \ configuration, or wait for it to settle, then convert." @@ -936,6 +952,7 @@ impl AdapterError { AdapterError::RelationOutsideTimeDomain { .. } => SqlState::INVALID_TRANSACTION_STATE, AdapterError::ResourceExhaustion { .. } => SqlState::INSUFFICIENT_RESOURCES, AdapterError::ResultSize(_) => SqlState::OUT_OF_MEMORY, + AdapterError::SubscribeFellBehind { .. } => SqlState::OUT_OF_MEMORY, AdapterError::SafeModeViolation(_) => SqlState::INTERNAL_ERROR, AdapterError::SubscribeOnlyTransaction => SqlState::INVALID_TRANSACTION_STATE, AdapterError::Optimizer(e) => match e { @@ -1345,6 +1362,20 @@ impl fmt::Display for AdapterError { ) } AdapterError::ResultSize(e) => write!(f, "{e}"), + AdapterError::SubscribeFellBehind { + buffered_bytes, + max_buffered_bytes, + } => { + use bytesize::ByteSize; + let buffered = u64::cast_from(*buffered_bytes); + let max = u64::cast_from(*max_buffered_bytes); + write!( + f, + "SUBSCRIBE fell behind: the client did not read results fast enough, so its backlog reached {}, exceeding the {} budget", + ByteSize::b(buffered), + ByteSize::b(max), + ) + } AdapterError::SafeModeViolation(feature) => { write!(f, "cannot create {} in safe mode", feature) } diff --git a/src/adapter/src/session.rs b/src/adapter/src/session.rs index cf6d6c18b8c92..905f543259f49 100644 --- a/src/adapter/src/session.rs +++ b/src/adapter/src/session.rs @@ -48,7 +48,7 @@ use mz_storage_client::client::TableData; use mz_storage_types::sources::Timeline; use qcell::{QCell, QCellOwner}; use timely::progress::Timestamp as _; -use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; +use tokio::sync::mpsc::{self, UnboundedSender}; use tokio::sync::watch; use uuid::Uuid; @@ -1069,8 +1069,14 @@ impl InProgressRows { } } -/// A channel of batched rows. -pub type RowBatchStream = UnboundedReceiver; +/// A stream of batched rows delivered to a client writer. +/// +/// This is a boxed stream, not a bare channel receiver, so the coordinator can +/// insert an adapter between itself and the client writer. The SUBSCRIBE path +/// wraps the receiver in a byte-accounting `map` whose closure type cannot be +/// named, so it has to be boxed. Consumers box the receiver into a +/// `RecordFirstRowStream` regardless, so this adds no allocation. +pub type RowBatchStream = Box + Unpin + Send + Sync>; /// Part of statement lifecycle. These are timestamps that come from the Adapter frontend /// (`mz-pgwire`) part of the lifecycle. diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs index a29b875cf999f..e7eeb0ccd017f 100644 --- a/src/catalog/src/builtin.rs +++ b/src/catalog/src/builtin.rs @@ -1117,7 +1117,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::Table(&MZ_COLUMNS), // mz_indexes is generated dynamically below with inlined builtin VALUES. Builtin::Table(&MZ_INDEX_COLUMNS), - Builtin::Table(&MZ_TABLES), + Builtin::MaterializedView(&MZ_TABLES), // mz_sources is generated dynamically below with inlined builtin VALUES. Builtin::Table(&MZ_SOURCE_REFERENCES), Builtin::MaterializedView(&MZ_POSTGRES_SOURCES), @@ -1126,7 +1126,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::MaterializedView(&MZ_SQL_SERVER_SOURCE_TABLES), Builtin::MaterializedView(&MZ_KAFKA_SOURCE_TABLES), Builtin::MaterializedView(&MZ_SINKS), - Builtin::Table(&MZ_VIEWS), + Builtin::MaterializedView(&MZ_VIEWS), Builtin::Table(&MZ_TYPES), Builtin::Table(&MZ_TYPE_PG_METADATA), Builtin::Table(&MZ_ARRAY_TYPES), @@ -1372,6 +1372,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::Source(&MZ_MATERIALIZED_VIEW_REFRESHES), Builtin::Source(&MZ_COMPUTE_DEPENDENCIES), Builtin::View(&MZ_MATERIALIZATION_DEPENDENCIES), + Builtin::View(&MZ_OBJECT_GRAPH_EDGES), Builtin::View(&MZ_MATERIALIZATION_LAG), Builtin::View(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW), Builtin::View(&MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H), @@ -1437,6 +1438,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::Index(&MZ_OBJECT_DEPENDENCIES_IND), Builtin::Index(&MZ_COMPUTE_DEPENDENCIES_IND), Builtin::Index(&MZ_OBJECT_TRANSITIVE_DEPENDENCIES_IND), + Builtin::Index(&MZ_OBJECT_GRAPH_EDGES_IND), Builtin::Index(&MZ_FRONTIERS_IND), Builtin::Index(&MZ_WALLCLOCK_GLOBAL_LAG_RECENT_HISTORY_IND), Builtin::Index(&MZ_KAFKA_SOURCES_IND), diff --git a/src/catalog/src/builtin/builtin.rs b/src/catalog/src/builtin/builtin.rs index e62910c4ab1fe..da7b1cee76400 100644 --- a/src/catalog/src/builtin/builtin.rs +++ b/src/catalog/src/builtin/builtin.rs @@ -25,8 +25,8 @@ use mz_sql::rbac; use mz_sql::session::user::MZ_SYSTEM_ROLE_ID; use crate::builtin::{ - Builtin, BuiltinLog, BuiltinMaterializedView, BuiltinSource, BuiltinView, Cardinality, - LinkProperties, Ontology, OntologyLink, PUBLIC_SELECT, + Builtin, BuiltinLog, BuiltinMaterializedView, BuiltinSource, BuiltinTable, BuiltinView, + Cardinality, LinkProperties, Ontology, OntologyLink, PUBLIC_SELECT, }; /// Generate builtin views reporting the given builtins. @@ -47,14 +47,32 @@ pub(super) fn builtins( Builtin::MaterializedView(x) => Some(*x), _ => None, }); + let table_iter = builtin_items.iter().filter_map(|b| match b { + Builtin::Table(x) => Some(*x), + _ => None, + }); + + let sources: &'static BuiltinView = + Box::leak(Box::new(make_builtin_sources(source_iter, log_iter))); + let materialized_views: &'static BuiltinView = + Box::leak(Box::new(make_builtin_materialized_views(mv_iter))); + let tables: &'static BuiltinView = Box::leak(Box::new(make_builtin_tables(table_iter))); - let sources = make_builtin_sources(source_iter, log_iter); - let materialized_views = make_builtin_materialized_views(mv_iter); + // The generated views above, and `mz_builtin_views` itself, are listed in + // `mz_builtin_views` with placeholder SQL rather than their real + // definitions. See `make_builtin_views`. + let view_iter = builtin_items.iter().filter_map(|b| match b { + Builtin::View(x) => Some(*x), + _ => None, + }); + let views: &'static BuiltinView = Box::leak(Box::new(make_builtin_views( + view_iter, + [sources, materialized_views, tables], + ))); - [sources, materialized_views].into_iter().map(|v| { - let static_ref = Box::leak(Box::new(v)); - Builtin::View(static_ref) - }) + [sources, materialized_views, tables, views] + .into_iter() + .map(Builtin::View) } fn make_builtin_sources( @@ -170,6 +188,145 @@ FROM (VALUES {values}) AS v(oid, schema_name, name, cluster_name, definition, pr } } +fn make_builtin_tables(iter: impl Iterator) -> BuiltinView { + let owner_priv = rbac::owner_privilege(ObjectType::Table, MZ_SYSTEM_ROLE_ID); + let values = iter + .map(|table| { + let schema = escaped_string_literal(table.schema); + let name = escaped_string_literal(table.name); + let privileges = make_privileges_sql(&table.access, &owner_priv); + format!("({}::oid, {}, {}, {})", table.oid, schema, name, privileges) + }) + .join(","); + let sql = format!( + " +SELECT oid, schema_name, name, privileges +FROM (VALUES {values}) AS v(oid, schema_name, name, privileges)" + ); + + BuiltinView { + name: "mz_builtin_tables", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::VIEW_MZ_BUILTIN_TABLES_OID, + desc: RelationDesc::builder() + .with_column("oid", SqlScalarType::Oid.nullable(false)) + .with_column("schema_name", SqlScalarType::String.nullable(false)) + .with_column("name", SqlScalarType::String.nullable(false)) + .with_column( + "privileges", + SqlScalarType::Array(Box::new(SqlScalarType::MzAclItem)).nullable(false), + ) + // NOTE: The declared keys must exactly match the keys the + // optimizer derives from the generated VALUES list + // (`verify_builtin_descs` enforces this). Table names happen to + // be unique across builtin schemas today, so `name` is a key. If + // a table is ever added whose bare name collides with another + // schema's, drop the `name` key here. + .with_key(vec![0]) + .with_key(vec![2]) + .finish(), + column_comments: Default::default(), + sql: Box::leak(sql.into_boxed_str()), + access: vec![PUBLIC_SELECT], + ontology: None, + } +} + +/// Generates `mz_internal.mz_builtin_views`, listing every builtin view, +/// including itself and the `generated` views. +/// +/// Views from `iter` are listed with their real definition and create SQL. +/// The generated views are instead listed with a short placeholder query. +/// Real SQL is impossible for `mz_builtin_views` itself, its definition would +/// have to contain its own text. It is impractical for the other generated +/// views, whose SQL embeds metadata about every builtin object. +/// `mz_builtin_materialized_views` for example carries the SQL of every +/// builtin materialized view, so re-embedding its definition here would +/// produce enormous rows that make `SELECT * FROM mz_views` unusable. +/// +/// The placeholder is a valid SQL statement, because `mz_views` applies +/// `mz_internal.redact_sql` to the `create_sql` column and that function +/// errors on unparseable input, which would poison the whole materialized +/// view. The placeholder also embeds the view's qualified name so that the +/// `definition` and `create_sql` columns stay unique across rows, which the +/// declared keys rely on. +fn make_builtin_views<'a>( + iter: impl Iterator, + generated: [&BuiltinView; 3], +) -> BuiltinView { + let owner_priv = rbac::owner_privilege(ObjectType::View, MZ_SYSTEM_ROLE_ID); + + let make_row = |oid: u32, schema: &str, name: &str, access: &[MzAclItem], create_sql: &str| { + let stmt = mz_sql::parse::parse(create_sql) + .expect("valid sql") + .into_element() + .ast; + let Statement::CreateView(stmt) = stmt else { + panic!("invalid builtin view SQL"); + }; + + let definition = format!("{};", stmt.definition.query.to_ast_string_stable()); + let definition = escaped_string_literal(&definition); + let create_sql = stmt.to_ast_string_stable(); + let create_sql = escaped_string_literal(&create_sql); + + let schema = escaped_string_literal(schema); + let name = escaped_string_literal(name); + let privileges = make_privileges_sql(access, &owner_priv); + + format!( + "({}::oid, {}, {}, {}, {}, {})", + oid, schema, name, definition, privileges, create_sql + ) + }; + + let mut view = BuiltinView { + name: "mz_builtin_views", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::VIEW_MZ_BUILTIN_VIEWS_OID, + desc: RelationDesc::builder() + .with_column("oid", SqlScalarType::Oid.nullable(false)) + .with_column("schema_name", SqlScalarType::String.nullable(false)) + .with_column("name", SqlScalarType::String.nullable(false)) + .with_column("definition", SqlScalarType::String.nullable(false)) + .with_column( + "privileges", + SqlScalarType::Array(Box::new(SqlScalarType::MzAclItem)).nullable(false), + ) + .with_column("create_sql", SqlScalarType::String.nullable(false)) + // NOTE: The declared keys must exactly match the keys the + // optimizer derives from the generated VALUES list + // (`verify_builtin_descs` enforces this). + .with_key(vec![0]) + .with_key(vec![2]) + .with_key(vec![3]) + .with_key(vec![5]) + .finish(), + column_comments: Default::default(), + sql: "", + access: vec![PUBLIC_SELECT], + ontology: None, + }; + + let full_values = iter.map(|v| make_row(v.oid, v.schema, v.name, &v.access, &v.create_sql())); + let placeholder_values = generated.iter().copied().chain([&view]).map(|v| { + let create_sql = format!( + "CREATE VIEW {}.{} AS SELECT ''", + v.schema, v.name, v.schema, v.name + ); + make_row(v.oid, v.schema, v.name, &v.access, &create_sql) + }); + let values = full_values.chain(placeholder_values).join(","); + let sql = format!( + " +SELECT oid, schema_name, name, definition, privileges, create_sql +FROM (VALUES {values}) AS v(oid, schema_name, name, definition, privileges, create_sql)" + ); + + view.sql = Box::leak(sql.into_boxed_str()); + view +} + /// Convert the given list of [`MzAclItem`] to the equivalent SQL syntax. fn make_privileges_sql(privs: &[MzAclItem], owner_priv: &MzAclItem) -> String { let privs = privs.iter().chain_one(owner_priv); diff --git a/src/catalog/src/builtin/mz_catalog.rs b/src/catalog/src/builtin/mz_catalog.rs index 9dd7e103c439b..a67cd31832b1e 100644 --- a/src/catalog/src/builtin/mz_catalog.rs +++ b/src/catalog/src/builtin/mz_catalog.rs @@ -1018,10 +1018,11 @@ pub static MZ_INDEX_COLUMNS: LazyLock = LazyLock::new(|| BuiltinTa column_semantic_types: &[("index_id", SemanticType::CatalogItemId)], }), }); -pub static MZ_TABLES: LazyLock = LazyLock::new(|| BuiltinTable { +pub static MZ_TABLES: LazyLock = LazyLock::new(|| { + BuiltinMaterializedView { name: "mz_tables", schema: MZ_CATALOG_SCHEMA, - oid: oid::TABLE_MZ_TABLES_OID, + oid: oid::MV_MZ_TABLES_OID, desc: RelationDesc::builder() .with_column("id", SqlScalarType::String.nullable(false)) .with_column("oid", SqlScalarType::Oid.nullable(false)) @@ -1061,6 +1062,68 @@ pub static MZ_TABLES: LazyLock = LazyLock::new(|| BuiltinTable { "The ID of the source associated with the table, if any. Corresponds to `mz_sources.id`.", ), ]), + // Temporary tables live in a per-session temporary schema that has no + // durable schema record. Their rows keep the temporary schema sentinel + // "0" that the previous builtin-table implementation exposed. + sql: Box::leak(format!(" +IN CLUSTER mz_catalog_server +WITH ( + ASSERT NOT NULL id, + ASSERT NOT NULL oid, + ASSERT NOT NULL schema_id, + ASSERT NOT NULL name, + ASSERT NOT NULL owner_id, + ASSERT NOT NULL privileges +) AS +WITH + user_tables AS ( + SELECT + mz_internal.parse_catalog_id(data->'key'->'gid') AS id, + (data->'value'->>'oid')::oid AS oid, + CASE WHEN data->'value'->>'ephemeral_owner_session' IS NULL + THEN mz_internal.parse_catalog_id(data->'value'->'schema_id') + ELSE '0' + END AS schema_id, + data->'value'->>'name' AS name, + mz_internal.parse_catalog_id(data->'value'->'owner_id') AS owner_id, + mz_internal.parse_catalog_privileges(data->'value'->'privileges') AS privileges, + data->'value'->'definition'->'V1'->>'create_sql' AS create_sql, + mz_internal.redact_sql(data->'value'->'definition'->'V1'->>'create_sql') AS redacted_create_sql, + mz_internal.parse_catalog_create_sql(data->'value'->'definition'->'V1'->>'create_sql')->>'source_id' AS source_id + FROM mz_internal.mz_catalog_raw + WHERE + data->>'kind' = 'Item' AND + mz_internal.parse_catalog_create_sql(data->'value'->'definition'->'V1'->>'create_sql')->>'type' = 'table' + ), + builtin_mappings AS ( + SELECT + data->'key'->>'schema_name' AS schema_name, + data->'key'->>'object_name' AS name, + 's' || (data->'value'->>'catalog_id') AS id + FROM mz_internal.mz_catalog_raw + WHERE + data->>'kind' = 'GidMapping' AND + data->'key'->>'object_type' = '1' + ), + builtin_tables AS ( + SELECT + m.id, + t.oid, + s.id AS schema_id, + t.name, + '{MZ_SYSTEM_ROLE_ID}' AS owner_id, + t.privileges, + NULL::text AS create_sql, + NULL::text AS redacted_create_sql, + NULL::text AS source_id + FROM mz_internal.mz_builtin_tables t + JOIN builtin_mappings m USING (schema_name, name) + JOIN mz_schemas s ON s.name = t.schema_name + WHERE s.database_id IS NULL + ) +SELECT * FROM user_tables +UNION ALL +SELECT * FROM builtin_tables").into_boxed_str()), is_retained_metrics_object: true, access: vec![PUBLIC_SELECT], ontology: Some(Ontology { @@ -1101,6 +1164,7 @@ pub static MZ_TABLES: LazyLock = LazyLock::new(|| BuiltinTable { ] }, }), +} }); pub static MZ_CONNECTIONS: LazyLock = LazyLock::new(|| { @@ -1425,10 +1489,11 @@ WHERE }), } }); -pub static MZ_VIEWS: LazyLock = LazyLock::new(|| BuiltinTable { +pub static MZ_VIEWS: LazyLock = LazyLock::new(|| { + BuiltinMaterializedView { name: "mz_views", schema: MZ_CATALOG_SCHEMA, - oid: oid::TABLE_MZ_VIEWS_OID, + oid: oid::MV_MZ_VIEWS_OID, desc: RelationDesc::builder() .with_column("id", SqlScalarType::String.nullable(false)) .with_column("oid", SqlScalarType::Oid.nullable(false)) @@ -1465,6 +1530,74 @@ pub static MZ_VIEWS: LazyLock = LazyLock::new(|| BuiltinTable { "The redacted `CREATE` SQL statement for the view.", ), ]), + // Temporary views live in a per-session temporary schema that has no + // durable schema record. Their rows keep the temporary schema sentinel + // "0" that the previous builtin-table implementation exposed. + // + // The generated `mz_builtin_*` views appear here with placeholder + // definition and create SQL. See `make_builtin_views`. + sql: Box::leak(format!(" +IN CLUSTER mz_catalog_server +WITH ( + ASSERT NOT NULL id, + ASSERT NOT NULL oid, + ASSERT NOT NULL schema_id, + ASSERT NOT NULL name, + ASSERT NOT NULL definition, + ASSERT NOT NULL owner_id, + ASSERT NOT NULL privileges, + ASSERT NOT NULL create_sql, + ASSERT NOT NULL redacted_create_sql +) AS +WITH + user_views AS ( + SELECT + mz_internal.parse_catalog_id(data->'key'->'gid') AS id, + (data->'value'->>'oid')::oid AS oid, + CASE WHEN data->'value'->>'ephemeral_owner_session' IS NULL + THEN mz_internal.parse_catalog_id(data->'value'->'schema_id') + ELSE '0' + END AS schema_id, + data->'value'->>'name' AS name, + mz_internal.parse_catalog_create_sql(data->'value'->'definition'->'V1'->>'create_sql')->>'definition' AS definition, + mz_internal.parse_catalog_id(data->'value'->'owner_id') AS owner_id, + mz_internal.parse_catalog_privileges(data->'value'->'privileges') AS privileges, + data->'value'->'definition'->'V1'->>'create_sql' AS create_sql, + mz_internal.redact_sql(data->'value'->'definition'->'V1'->>'create_sql') AS redacted_create_sql + FROM mz_internal.mz_catalog_raw + WHERE + data->>'kind' = 'Item' AND + mz_internal.parse_catalog_create_sql(data->'value'->'definition'->'V1'->>'create_sql')->>'type' = 'view' + ), + builtin_mappings AS ( + SELECT + data->'key'->>'schema_name' AS schema_name, + data->'key'->>'object_name' AS name, + 's' || (data->'value'->>'catalog_id') AS id + FROM mz_internal.mz_catalog_raw + WHERE + data->>'kind' = 'GidMapping' AND + data->'key'->>'object_type' = '4' + ), + builtin_views AS ( + SELECT + m.id, + v.oid, + s.id AS schema_id, + v.name, + v.definition, + '{MZ_SYSTEM_ROLE_ID}' AS owner_id, + v.privileges, + v.create_sql, + mz_internal.redact_sql(v.create_sql) AS redacted_create_sql + FROM mz_internal.mz_builtin_views v + JOIN builtin_mappings m USING (schema_name, name) + JOIN mz_schemas s ON s.name = v.schema_name + WHERE s.database_id IS NULL + ) +SELECT * FROM user_views +UNION ALL +SELECT * FROM builtin_views").into_boxed_str()), is_retained_metrics_object: false, access: vec![PUBLIC_SELECT], ontology: Some(Ontology { @@ -1496,6 +1629,7 @@ pub static MZ_VIEWS: LazyLock = LazyLock::new(|| BuiltinTable { ] }, }), +} }); pub static MZ_MATERIALIZED_VIEWS: LazyLock = LazyLock::new(|| { diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs index f8adc09251b81..eeedca513823a 100644 --- a/src/catalog/src/builtin/mz_internal.rs +++ b/src/catalog/src/builtin/mz_internal.rs @@ -3898,6 +3898,66 @@ pub static MZ_OBJECTS_ID_NAMESPACE_TYPES: LazyLock = LazyLock::new( ontology: None, }); +/// Object dependency edges. Each row `(object_id, dependency_id)` means +/// `object_id` depends on `dependency_id`. +/// +/// Unions the dataflow dependencies between maintained objects (index, +/// materialized view, sink, source, table) with the source-to-subsource and +/// source-to-table edges that connect a source to the children carrying its +/// data. Indexed on `mz_catalog_server` so the console surfaces that walk the +/// dependency graph read one maintained arrangement instead of recomputing the +/// union per request: the object workflow graph, critical-path freshness +/// analysis, and impact/dependents views. +pub static MZ_OBJECT_GRAPH_EDGES: LazyLock = LazyLock::new(|| BuiltinView { + name: "mz_object_graph_edges", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::VIEW_MZ_OBJECT_GRAPH_EDGES_OID, + desc: RelationDesc::builder() + .with_column("object_id", SqlScalarType::String.nullable(false)) + .with_column("dependency_id", SqlScalarType::String.nullable(false)) + .with_key(vec![0, 1]) + .finish(), + column_comments: BTreeMap::from_iter([ + ( + "object_id", + "The ID of the dependent object. Corresponds to `mz_objects.id`.", + ), + ( + "dependency_id", + "The ID of the object it depends on. Corresponds to `mz_objects.id`.", + ), + ]), + sql: " +SELECT md.object_id, md.dependency_id +FROM mz_internal.mz_materialization_dependencies md +JOIN mz_catalog.mz_objects po ON po.id = md.dependency_id + AND po.type IN ('index', 'materialized-view', 'sink', 'source', 'table') +JOIN mz_catalog.mz_objects co ON co.id = md.object_id + AND co.type IN ('index', 'materialized-view', 'sink', 'source', 'table') +UNION +-- Subsource -> parent-source edges: a subsource depends on the (user) source it +-- belongs to, an edge mz_materialization_dependencies doesn't carry. +SELECT od.object_id, od.referenced_object_id +FROM mz_internal.mz_object_dependencies od +JOIN mz_catalog.mz_sources ps ON ps.id = od.referenced_object_id +JOIN mz_catalog.mz_sources cs ON cs.id = od.object_id +-- Progress collections are deliberately left out: their dependency edge points +-- source -> progress, and they only exist for old-syntax sources, which the +-- source-table migration is removing. +WHERE ps.id LIKE 'u%' AND cs.type = 'subsource' +UNION +-- Select the (non-null) source id from the join rather than the nullable +-- mz_tables.source_id, so dependency_id is non-null across all branches. +SELECT t.id, ps.id +FROM mz_catalog.mz_tables t +JOIN mz_catalog.mz_sources ps ON ps.id = t.source_id", + access: vec![PUBLIC_SELECT], + // No ontology entity: these edges are already in the ontology via the + // DependsOn links of mz_object_dependencies and + // mz_materialization_dependencies. An entity here would duplicate them. + ontology: None, +}); + pub static MZ_OBJECT_OID_ALIAS: LazyLock = LazyLock::new(|| BuiltinView { name: "mz_object_oid_alias", schema: MZ_INTERNAL_SCHEMA, @@ -8760,6 +8820,15 @@ ON mz_internal.mz_object_transitive_dependencies (object_id)", is_retained_metrics_object: false, }; +pub const MZ_OBJECT_GRAPH_EDGES_IND: BuiltinIndex = BuiltinIndex { + name: "mz_object_graph_edges_ind", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::INDEX_MZ_OBJECT_GRAPH_EDGES_IND_OID, + sql: "IN CLUSTER mz_catalog_server +ON mz_internal.mz_object_graph_edges (object_id)", + is_retained_metrics_object: false, +}; + pub const MZ_FRONTIERS_IND: BuiltinIndex = BuiltinIndex { name: "mz_frontiers_ind", schema: MZ_INTERNAL_SCHEMA, diff --git a/src/compute-types/src/dyncfgs.rs b/src/compute-types/src/dyncfgs.rs index 19ca6b4cd02fb..bfc0802b28541 100644 --- a/src/compute-types/src/dyncfgs.rs +++ b/src/compute-types/src/dyncfgs.rs @@ -23,6 +23,25 @@ pub const ENABLE_HALF_JOIN2: Config = Config::new( "Whether compute should use `half_join2` rather than DD's `half_join` to render delta joins.", ); +/// Whether rendering should collapse error multiplicities to one where it arranges errors. +/// +/// Error semantics depend only on whether an error is present, so its multiplicity carries no +/// information a consumer reads. Left uncollapsed, a shared collection contributes its errors once +/// per plan path that reads it, and because those factors apply again at each level of sharing they +/// compound multiplicatively until the `Diff` overflows. +/// +/// Governs sharing within a dataflow only. Sharing across objects is bounded unconditionally, by +/// normalizing at every boundary another dataflow can read, so what this flag decides is never +/// durable state and two replicas rendering under different values still write the same thing. +/// Environment-scoped for now because nothing needs finer granularity, not because finer would be +/// unsafe. +pub const ENABLE_ERROR_DISTINCT: Config = Config::new( + "enable_compute_error_distinct", + true, + "Whether compute rendering should collapse error multiplicities to one where it arranges \ + errors.", +); + /// Use the column-paged merge batcher code path at arrange sites. When /// `true`, arrange operators use `Col2ValPagedBatcher` (in /// `mz_timely_util::columnar`) and `RowRowColPagedBuilder` (in @@ -548,6 +567,7 @@ pub const MV_SINK_ADVANCE_PERSIST_FRONTIERS: Config = Config::new( pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet { configs .add(&ENABLE_HALF_JOIN2) + .add(&ENABLE_ERROR_DISTINCT) .add(&ENABLE_MZ_JOIN_CORE) .add(&ENABLE_SYNC_MV_SINK) .add(&ENABLE_CORRECTION_V2) diff --git a/src/compute/src/render.rs b/src/compute/src/render.rs index 82fc821cd8604..ccfe93b902c97 100644 --- a/src/compute/src/render.rs +++ b/src/compute/src/render.rs @@ -125,7 +125,8 @@ use mz_compute_types::dataflows::{DataflowDescription, IndexDesc}; use mz_compute_types::dyncfgs::{ COMPUTE_APPLY_COLUMN_DEMANDS, COMPUTE_LOGICAL_BACKPRESSURE_INFLIGHT_SLACK, COMPUTE_LOGICAL_BACKPRESSURE_MAX_RETAINED_CAPABILITIES, ENABLE_COMPUTE_LOGICAL_BACKPRESSURE, - ENABLE_COMPUTE_TEMPORAL_BUCKETING, SUBSCRIBE_SNAPSHOT_OPTIMIZATION, TEMPORAL_BUCKETING_SUMMARY, + ENABLE_COMPUTE_TEMPORAL_BUCKETING, ENABLE_ERROR_DISTINCT, SUBSCRIBE_SNAPSHOT_OPTIMIZATION, + TEMPORAL_BUCKETING_SUMMARY, }; use mz_compute_types::plan::render_plan::{ self, BindStage, LetBind, LetFreePlan, RecBind, RenderPlan, @@ -711,6 +712,17 @@ impl<'g> Context<'g, mz_repr::Timestamp> { let key = &idx.key; match bundle.arrangement(key) { Some(ArrangementFlavor::Local(mut oks, mut errs)) => { + // NOTE: Do not give an exported arrangement a second reader that holds a trace + // handle, such as a `reduce`. Such a reader pins the shared spine's physical + // frontier at its own lagging progress, and `ArrangementManager::maintenance` can + // then no longer advance it, so batches pile up in `Spine::pending`. A cursor is + // only checked for straddling over pending batches, so an importing dataflow's + // `cursor_through` eventually panics with `upper` straddles batch. Watching + // `errs.stream` in `output_probe` does not help, and neither does discarding the + // reader's output. Stream-level readers like `as_collection` are unaffected. This is + // why error multiplicity is not collapsed here, leaving multiplicity that crosses + // an index boundary unbounded. TODO(CPU-209): bound it without a trace reader. + // Ensure that the frontier does not advance past the expiration time, if set. // Otherwise, we might write down incorrect data. if let Some(&expiration) = self.dataflow_expiration.as_option() { @@ -906,6 +918,7 @@ impl<'scope> Context<'scope, Product>> { .render_letfree_plan(object_id, value, binding) .leave_region(self.scope) }); + let bundle = self.distinct_binding_errs(bundle); self.insert_id(Id::Local(id), bundle); } @@ -939,6 +952,11 @@ impl<'scope> Context<'scope, Product>> { // here to cause that to happen. let (oks, mut err) = bundle.collection.clone().unwrap(); let oks = oks.into_vec(); + // Collapses what forward reads see. `err_v` below feeds reads rendered before this + // binding and is collapsed separately; without this, a `Get` in a later rec binding + // or in the body resolves to the bundle stored here and compounds level over level, + // which is exactly what the collapse prevents for non-recursive bindings. + let bundle = self.distinct_binding_errs(bundle); self.insert_id(Id::Local(id), bundle); let (oks_v, err_v) = variables.remove(&Id::Local(id)).unwrap(); @@ -1044,6 +1062,7 @@ impl<'scope, T: RenderTimestamp + MaybeBucketByTime> Context<'scope, T> { .render_letfree_plan(object_id, value, binding) .leave_region(self.scope) }); + let bundle = self.distinct_binding_errs(bundle); self.insert_id(Id::Local(id), bundle); } } @@ -1056,6 +1075,25 @@ impl<'scope, T: RenderTimestamp + MaybeBucketByTime> Context<'scope, T> { }) } + /// Collapses a binding's error multiplicities. + /// + /// Applied to every binding, not only the multiply-read ones. Gating on the reference count + /// would be a pure optimization, since collapsing a binding one `Get` reads is harmless, and + /// there is almost nothing to gate: `NormalizeLets` inlines single-use bindings, so the ones + /// reaching rendering are shared. See [`CollectionBundle::distinct_errs`] for why the collapse + /// is needed at all, and why a binding's definition is the place for it rather than the + /// multi-input operators where the duplicate copies happen to meet again. + fn distinct_binding_errs( + &self, + bundle: CollectionBundle<'scope, T>, + ) -> CollectionBundle<'scope, T> { + if ENABLE_ERROR_DISTINCT.get(&self.config_set) { + bundle.distinct_errs() + } else { + bundle + } + } + /// Renders a let-free plan to a differential dataflow, producing the collection of results. fn render_letfree_plan( &self, diff --git a/src/compute/src/render/context.rs b/src/compute/src/render/context.rs index 6e6b80119b601..c5f20a8691947 100644 --- a/src/compute/src/render/context.rs +++ b/src/compute/src/render/context.rs @@ -48,6 +48,7 @@ use timely::progress::{Antichain, Timestamp}; use crate::compute_state::ComputeState; use crate::extensions::arrange::{KeyCollection, MzArrange, MzArrangeCore}; +use crate::extensions::reduce::MzReduce; use crate::render::columnar::CollectionEdge; use crate::render::errors::{DataflowErrorSer, ErrorLogger}; use crate::render::{LinearJoinSpec, MaybeBucketByTime, RenderTimestamp}; @@ -435,6 +436,41 @@ impl<'scope, T: RenderTimestamp> ArrangementFlavor<'scope, T> { } } } +/// Rewrites an arranged error collection to hold each of its errors once. +/// +/// Sound because query semantics depend only on whether an error is present, and correct under +/// retraction only because it reads the accumulated collection: no pointwise function of the input +/// diffs (a saturating add, a sign) can collapse multiplicity and still cancel when the errors +/// retract. +/// +/// NOTE: One consumer does read the multiplicity. Error-count introspection reports it as the +/// number of failing rows, so `log_dataflow_errors` must see the collection before this collapses +/// it, or a dataflow reports one error however many rows failed. Collapse after the logging, never +/// before. +pub(crate) fn distinct_arranged_errs<'a, T: RenderTimestamp>( + errs: Arranged<'a, ErrAgent>, + name: &str, +) -> Arranged<'a, ErrAgent> { + errs.mz_reduce_abelian::<_, ErrBuilder<_, _>, ErrSpine<_, _>, _>( + name, + |_err, _input, output| output.push(((), Diff::ONE)), + ) +} + +/// Rewrites an error collection to hold each of its errors once. +/// +/// Costs an arrangement more than [`distinct_arranged_errs`], which reuses the arrangement its +/// input already has. +pub(crate) fn distinct_errs_collection<'a, T: RenderTimestamp>( + errs: VecCollection<'a, T, DataflowErrorSer, Diff>, +) -> VecCollection<'a, T, DataflowErrorSer, Diff> { + let errs: KeyCollection<_, _, _> = errs.into(); + let errs = errs + .mz_arrange::, ErrBatcher<_, _>, ErrBuilder<_, _>, ErrSpine<_, _>>( + "Arrange errors", + ); + distinct_arranged_errs(errs, "Distinct errors").as_collection(|err, _| err.clone()) +} /// A bundle of the various ways a collection can be represented. /// @@ -507,6 +543,50 @@ impl<'scope, T: RenderTimestamp> CollectionBundle<'scope, T> { } } + /// Collapses the multiplicity of every error this bundle carries to one. + /// + /// Belongs at the definition of a binding that more than one `Get` reads. Each reader + /// propagates the binding's errors independently, so a binding read `f` times contributes its + /// errors `f` times to the dataflow's error output, and those factors apply again at every + /// further level of sharing: a chain of diamonds multiplies rather than adds, and reaches + /// `Diff` overflow at a depth plans really do have. Collapsing at each definition holds the + /// dataflow's error multiplicity to the fan-out of a single level. + /// + /// Sound because error semantics depend only on whether an error is present, and correct under + /// retraction only because it reads the accumulated collection: no pointwise function of the + /// input diffs (a saturating add, a sign) can collapse multiplicity and still cancel when the + /// errors retract. + /// + /// Collapses every form the bundle offers, not just one. Each form carries its own error + /// stream, and those streams differ in content as well as identity: an arrangement's errors + /// include the key-formation errors that the raw collection's do not. Which form a consumer + /// reads is the consumer's choice, and a delta join reads both within one operator, so a + /// binding's definition cannot know which form to collapse. + /// + /// NOTE: Leaves imported arrangements (`ArrangementFlavor::Trace`) alone, whose error traces + /// this dataflow cannot rewrite in place. Their errors arrive bounded by the exporting + /// dataflow's last level of sharing rather than collapsed to one, since nothing collapses at an + /// export. A global read more than once within one dataflow is not collapsed either, because + /// only local bindings reach this. + pub fn distinct_errs(mut self) -> Self { + if let Some((oks, errs)) = self.collection.take() { + self.collection = Some((oks, distinct_errs_collection(errs))); + } + for (key, flavor) in std::mem::take(&mut self.arranged) { + let flavor = match flavor { + ArrangementFlavor::Local(oks, errs) => { + // Names the key, not the binding: an operator name carrying a `LocalId` would + // churn the introspection goldens every time the optimizer renumbers locals. + let name = format!("Distinct errors[{key:?}]"); + ArrangementFlavor::Local(oks, distinct_arranged_errs(errs, &name)) + } + flavor @ ArrangementFlavor::Trace(..) => flavor, + }; + self.arranged.insert(key, flavor); + } + self + } + /// Brings the collection bundle into a region. pub fn enter_region<'inner>(&self, region: Scope<'inner, T>) -> CollectionBundle<'inner, T> { CollectionBundle { diff --git a/src/compute/src/render/join/delta_join.rs b/src/compute/src/render/join/delta_join.rs index a16a60764c70d..cfac33a55887a 100644 --- a/src/compute/src/render/join/delta_join.rs +++ b/src/compute/src/render/join/delta_join.rs @@ -55,9 +55,6 @@ impl<'scope, T: RenderTimestamp> Context<'scope, T> { ) -> CollectionBundle<'scope, T> { // We create a new region to contain the dataflow paths for the delta join. let (oks, errs) = self.scope.clone().region_named("Join(Delta)", |inner| { - // Collects error streams for the ambient scope. - let mut inner_errs = Vec::new(); - // Our plan is to iterate through each input relation, and attempt // to find a plan that maximally uses existing keys (better: uses // existing arrangements, to which we have access). @@ -86,6 +83,11 @@ impl<'scope, T: RenderTimestamp> Context<'scope, T> { }) .collect::>(); + // Collects error streams for the ambient scope, seeded with the errors the inputs + // arrive with. Every path reads every input, but an input's pre-existing errors belong + // in the output once, not once per path. See [`bundle_errs`]. + let mut inner_errs: Vec<_> = inputs.iter().flat_map(bundle_errs).collect(); + for path_plan in join_plan.path_plans { // Deconstruct the stages of the path plan. let DeltaPathPlan { @@ -288,12 +290,51 @@ fn prune_bundle<'scope, T: RenderTimestamp>( } } +/// The error collections of every collection and arrangement `bundle` holds. +/// +/// A delta join reads every input from every one of its paths, but an input's pre-existing errors +/// belong in the join's error output once. Propagating them per path would multiply their +/// multiplicities by the number of paths, and because a join's output is another join's input, those +/// factors compound multiplicatively through a nested plan until the `Diff` overflows. Error +/// semantics depend only on presence, so the extra copies buy nothing. +/// +/// Expects the pruned bundle (see [`prune_bundle`]), so that it yields errors only for the +/// collections some path actually reads. +/// +/// NOTE: This bounds an input's errors to one copy per retained form, not to one copy outright. +/// Each form carries its own error collection, and an arrangement's is a distinct stream built from +/// the raw one plus that key's key-formation errors, so an input the join reads under two lookup +/// keys still contributes its errors twice. Reachable whenever a delta path set needs an +/// error-carrying input arranged by more than one key. +fn bundle_errs<'scope, T: RenderTimestamp>( + bundle: &CollectionBundle<'scope, T>, +) -> Vec> { + let mut collected = Vec::with_capacity(bundle.arranged.len() + 1); + if let Some((_oks, errs)) = &bundle.collection { + collected.push(errs.clone()); + } + for flavor in bundle.arranged.values() { + let errs = match flavor { + ArrangementFlavor::Local(_oks, errs) => errs.clone().as_collection(|k, _v| k.clone()), + ArrangementFlavor::Trace(_id, _oks, errs) => { + errs.clone().as_collection(|k, _v| k.clone()) + } + }; + collected.push(errs); + } + collected +} + /// Constructs a `half_join` against the arrangement held by a collection bundle. /// /// This wrapper demuxes over the two flavors of arrangement (dataflow-local or imported trace) /// that the bundle might hold for `lookup_key`, dispatching to the generic [`build_halfjoin_trace`] /// for each. `source_precedes_lookup` selects the tie-breaking comparison: `le` if the source /// relation precedes the lookup relation in the total order on relations, otherwise `lt`. +/// +/// The returned error collection holds only the errors this stage produces. The errors `bundle` +/// already carries are the caller's to propagate, once, rather than once per delta path that looks +/// the input up. See [`bundle_errs`]. fn build_halfjoin<'scope, T>( updates: VecCollection<'scope, T, (Row, T), Diff>, prev_key: Vec, @@ -311,7 +352,7 @@ where T: RenderTimestamp, { match bundle.arrangement(&lookup_key) { - Some(ArrangementFlavor::Local(oks, errs)) => { + Some(ArrangementFlavor::Local(oks, _errs)) => { let (oks, errs2) = if source_precedes_lookup { build_halfjoin_trace::<_, RowRowAgent<_, _>, _>( updates, @@ -333,9 +374,9 @@ where config_set, ) }; - (oks, errs2.concat(errs.as_collection(|k, _v| k.clone()))) + (oks, errs2) } - Some(ArrangementFlavor::Trace(_, oks, errs)) => { + Some(ArrangementFlavor::Trace(_, oks, _errs)) => { let (oks, errs2) = if source_precedes_lookup { build_halfjoin_trace::<_, RowRowEnter<_, _, _>, _>( updates, @@ -357,7 +398,7 @@ where config_set, ) }; - (oks, errs2.concat(errs.as_collection(|k, _v| k.clone()))) + (oks, errs2) } None => panic!("Arrangement promised by the planner is absent!"), } @@ -658,6 +699,10 @@ where /// With a `source_key`, demuxes over the two flavors of arrangement the bundle might hold for that /// key, dispatching to the generic [`build_update_stream_trace`]. Without a `source_key`, the source /// relation is consumed as a raw (unarranged) collection via [`build_update_stream_stream`]. +/// +/// The returned error collection holds only the errors the initial closure produces. The errors +/// `bundle` already carries are the caller's to propagate, once, rather than once per delta path. +/// See [`bundle_errs`]. fn build_update_stream<'scope, T>( bundle: &CollectionBundle<'scope, T>, as_of: Antichain, @@ -675,32 +720,28 @@ where // No source key means a single-time dataflow (e.g. a `SELECT`) whose plan was truncated to // this one path, letting us hydrate the source from its raw collection instead of an // arrangement. - let (oks, errs) = bundle + let (oks, _errs) = bundle .collection .clone() .expect("The unarranged collection doesn't exist."); - let (oks, errs2) = - build_update_stream_stream(oks.into_vec(), as_of, source_relation, initial_closure); - return (oks, errs2.concat(errs)); + return build_update_stream_stream(oks.into_vec(), as_of, source_relation, initial_closure); }; match bundle.arrangement(&source_key) { - Some(ArrangementFlavor::Local(oks, errs)) => { - let (oks, errs2) = build_update_stream_trace::<_, RowRowAgent<_, _>>( + Some(ArrangementFlavor::Local(oks, _errs)) => { + build_update_stream_trace::<_, RowRowAgent<_, _>>( oks, as_of, source_relation, initial_closure, - ); - (oks, errs2.concat(errs.as_collection(|k, _v| k.clone()))) + ) } - Some(ArrangementFlavor::Trace(_, oks, errs)) => { - let (oks, errs2) = build_update_stream_trace::<_, RowRowEnter<_, _, _>>( + Some(ArrangementFlavor::Trace(_, oks, _errs)) => { + build_update_stream_trace::<_, RowRowEnter<_, _, _>>( oks, as_of, source_relation, initial_closure, - ); - (oks, errs2.concat(errs.as_collection(|k, _v| k.clone()))) + ) } None => panic!("Arrangement promised by the planner is absent!"), } diff --git a/src/compute/src/render/sinks.rs b/src/compute/src/render/sinks.rs index 4dbdf88a4eec9..c96e0be462b7b 100644 --- a/src/compute/src/render/sinks.rs +++ b/src/compute/src/render/sinks.rs @@ -30,7 +30,7 @@ use timely::progress::Antichain; use crate::compute_state::SinkToken; use crate::logging::compute::LogDataflowErrors; -use crate::render::context::Context; +use crate::render::context::{Context, distinct_errs_collection}; use crate::render::errors::DataflowErrorSer; use crate::render::{RenderTimestamp, StartSignal}; @@ -130,6 +130,16 @@ impl<'g, T: RenderTimestamp> Context<'g, T> { err_collection = err_collection.concat(null_errs); } + // Normalize what the sink persists. A materialized view's error multiplicity is durable + // state another dataflow reads back verbatim, so leaving it at this dataflow's fan-out both + // multiplies across the object graph and makes the written value depend on plan shape. + // Placed after the null assertions so their errors, whose multiplicity follows the ok row's, + // are covered too. Only the persist-backed sink is re-importable: a subscribe or a one-shot + // copy is read once by a client, so neither pays for the arrangement. + if matches!(sink.connection, ComputeSinkConnection::MaterializedView(_)) { + err_collection = distinct_errs_collection(err_collection); + } + let region_name = match sink.connection { ComputeSinkConnection::Subscribe(_) => format!("SubscribeSink({:?})", sink_id), ComputeSinkConnection::MaterializedView(_) => { diff --git a/src/compute/src/sink/materialized_view_v2.rs b/src/compute/src/sink/materialized_view_v2.rs index 841a46a862c62..02c9a7f89d183 100644 --- a/src/compute/src/sink/materialized_view_v2.rs +++ b/src/compute/src/sink/materialized_view_v2.rs @@ -68,6 +68,7 @@ use timely::dataflow::operators::vec::Broadcast; use timely::dataflow::operators::{Capability, CapabilitySet}; use timely::progress::Antichain; use timely::progress::frontier::AntichainRef; +use tokio::runtime::Handle; use tokio::sync::{mpsc, watch}; use tracing::trace; @@ -868,9 +869,8 @@ mod write { ) .await; - // Reading the consolidated updates back only clones them into the batch builder, - // which awaits at every part flush, so it stays on the async task. Chain ok and - // err correction iterators directly, avoiding an intermediate Vec allocation. + // Chain ok and err correction iterators directly, avoiding an intermediate Vec + // allocation. let oks = corrections .ok .consolidated_updates_before(&desc.upper) @@ -888,10 +888,17 @@ mod write { return corrections; } - let batch = writer - .batch(updates, desc.lower, desc.upper) - .await - .expect("valid usage"); + // Feeding the builder pulls the updates out of the correction buffers on the + // calling thread: every pull clones a row and can page a chunk back in, while the + // builder only awaits once a part fills, i.e. after `blob_target_size` times + // `batch_builder_max_outstanding_parts` worth of updates. That is the same stall + // the consolidation above avoids, so it gets the same treatment. + // `block_in_place` rather than `spawn_blocking`, because the updates borrow the + // buffers and the builder needs an async context. + let batch = tokio::task::block_in_place(|| { + Handle::current().block_on(writer.batch(updates, desc.lower, desc.upper)) + }) + .expect("valid usage"); let proto_batch = batch.into_transmittable_batch(); if let Err(err) = resp_tx.send(WriteResponse { batch: Some(proto_batch), diff --git a/src/environmentd/src/http/mcp.rs b/src/environmentd/src/http/mcp.rs index 4c8b69538c53b..e91207586626e 100644 --- a/src/environmentd/src/http/mcp.rs +++ b/src/environmentd/src/http/mcp.rs @@ -1031,7 +1031,7 @@ fn call_status(result: &Result) -> McpCallStatus { async fn execute_sql( client: &mut AuthedClient, query: &str, -) -> Result>, McpRequestError> { +) -> Result>, McpRequestError> { let mut response = SqlResponse::new(); execute_request( @@ -1054,7 +1054,7 @@ async fn execute_sql( /// row-returning statement is an error rather than a dropped result. fn select_single_rows( results: Vec, -) -> Result>, McpRequestError> { +) -> Result>, McpRequestError> { let mut rows = None; for result in results { match result { @@ -1085,9 +1085,11 @@ fn select_single_rows( /// endpoint handles `max_result_size` in sql.rs — fail cleanly rather /// than silently truncating. fn format_rows_response( - rows: Vec>, + rows: Vec>, max_size: usize, ) -> Result { + // Each row is already-serialized compact JSON. `RawValue` re-serializes + // verbatim, so the outer array is pretty-printed while rows stay compact. let text = serde_json::to_string_pretty(&rows).map_err(|e| McpRequestError::Internal(anyhow!(e)))?; @@ -1215,17 +1217,20 @@ async fn read_data_product( if lookup_rows.is_empty() { return Err(McpRequestError::DataProductNotFound(name.to_string())); } - let catalog_cluster: Option<&str> = lookup_rows + // Rows are stored as pre-serialized JSON arrays, so parse the single-cell + // lookup row to read `dp.cluster`. + let catalog_cluster: Option = lookup_rows .first() - .and_then(|row| row.first()) - .and_then(|v| v.as_str()); + .and_then(|row| serde_json::from_str::>(row.get()).ok()) + .and_then(|row| row.into_iter().next()) + .and_then(|v| v.as_str().map(|s| s.to_string())); // An override wins. Otherwise route to the catalog cluster when the role // can use it (non-null); when it is null the role lacks USAGE on the // object's cluster, so leave the cluster unset and read on the session's // default (serving) cluster. That still works: materialized views serve // from persist and views recompute, just without index benefit. - let target_cluster: Option<&str> = cluster_override.or(catalog_cluster); + let target_cluster: Option<&str> = cluster_override.or(catalog_cluster.as_deref()); // No row cap is applied here: the response is bounded by the size cap // enforced in format_rows_response (MCP_MAX_RESPONSE_SIZE), and by @@ -1529,10 +1534,18 @@ mod tests { use super::*; use crate::http::sql::{Description, SqlError}; + /// Serializes each row to a compact JSON array, matching how the HTTP SQL + /// layer stores rows. + fn raw_rows(rows: Vec>) -> Vec> { + rows.iter() + .map(|r| serde_json::value::to_raw_value(r).unwrap()) + .collect() + } + fn rows_result(rows: Vec>) -> SqlResult { SqlResult::Rows { tag: String::new(), - rows, + rows: raw_rows(rows), desc: Description { columns: vec![] }, notices: vec![], } @@ -1570,7 +1583,9 @@ mod tests { rows_result(rows.clone()), ok_result(), ]; - assert_eq!(select_single_rows(results).unwrap(), rows); + let got = select_single_rows(results).unwrap(); + let got: Vec<&str> = got.iter().map(|r| r.get()).collect(); + assert_eq!(got, vec!["[1]"]); } /// A response with no row-returning statement is an error. @@ -2248,7 +2263,7 @@ mod tests { #[mz_ore::test] fn test_format_rows_response_within_limit() { let rows = vec![vec![json!("a"), json!(1)], vec![json!("b"), json!(2)]]; - let result = format_rows_response(rows, 1_000_000).unwrap(); + let result = format_rows_response(raw_rows(rows), 1_000_000).unwrap(); let McpResult::ToolContent(content) = result else { panic!("Expected ToolContent"); }; @@ -2262,7 +2277,7 @@ mod tests { let rows: Vec> = (0..100) .map(|i| vec![json!(format!("row_{}", i)), json!(i)]) .collect(); - let err = format_rows_response(rows, 500).unwrap_err(); + let err = format_rows_response(raw_rows(rows), 500).unwrap_err(); let msg = err.to_string(); assert!( msg.contains("exceeds the 500 byte limit"), @@ -2277,7 +2292,7 @@ mod tests { #[mz_ore::test] fn test_format_rows_response_empty_rows() { let rows: Vec> = vec![]; - let result = format_rows_response(rows, 1000).unwrap(); + let result = format_rows_response(raw_rows(rows), 1000).unwrap(); let McpResult::ToolContent(content) = result else { panic!("Expected ToolContent"); }; diff --git a/src/environmentd/src/http/sql.rs b/src/environmentd/src/http/sql.rs index 360eeb5da1e62..d47c8db4e3561 100644 --- a/src/environmentd/src/http/sql.rs +++ b/src/environmentd/src/http/sql.rs @@ -51,7 +51,6 @@ use prometheus::core::{AtomicF64, GenericGaugeVec}; use serde::{Deserialize, Serialize}; use tokio::{select, time}; use tokio_postgres::error::SqlState; -use tokio_stream::wrappers::UnboundedReceiverStream; use tower_sessions::Session as TowerSession; use tracing::{debug, info}; use tungstenite::protocol::frame::coding::CloseCode; @@ -180,6 +179,12 @@ async fn execute_promsql_query( }); for row in rows { + // Rows are stored as pre-serialized JSON arrays. Parse each one back to + // `Value`s here. Promsql results are tiny, so the parse cost is + // negligible. + let row: Vec = + serde_json::from_str(row.get()).expect("row is a valid JSON array"); + // Non-value columns become Prometheus label values. A SQL `NULL` // arrives as JSON `null` and yields `None` from `as_str()`; fall back // to an empty label rather than panicking. The query author is @@ -188,7 +193,7 @@ async fn execute_promsql_query( let mut label_values = desc .columns .iter() - .zip_eq(row) + .zip_eq(&row) .filter(|(col, _)| col.name != query.value_column_name) .map(|(_, val)| val.as_str().unwrap_or("")) .collect::>(); @@ -196,7 +201,7 @@ async fn execute_promsql_query( let value = desc .columns .iter() - .zip_eq(row) + .zip_eq(&row) .find(|(col, _)| col.name == query.value_column_name) .map(|(_, val)| val.as_str().unwrap_or("0").parse::().unwrap_or(0.0)) .unwrap_or(0.0); @@ -560,6 +565,15 @@ impl SqlResponse { pub(in crate::http) enum StatementResult { SqlResult(SqlResult), + /// A peek (`SELECT`) result whose rows are streamed out of the peek response + /// stash. The stream is consumed lazily by the sender so a large result is + /// not buffered whole. Contrast `SqlResult::Rows`, which holds + /// already-collected rows and exists only for the buffered JSON transport. + Rows { + desc: RelationDesc, + rows_stream: RecordFirstRowStream, + max_result_size: usize, + }, Subscribe { desc: RelationDesc, tag: String, @@ -582,8 +596,15 @@ pub enum SqlResult { Rows { /// The command complete tag. tag: String, - /// The result rows. - rows: Vec>, + /// The result rows, each already serialized to a compact JSON array. + /// + /// We accumulate pre-serialized rows rather than a `serde_json::Value` + /// tree so the buffered footprint stays close to the wire size. A + /// `Value` tree is ~10-15x larger (each cell a heap `Value`, in a + /// per-row `Vec`, in the outer `Vec`), which is what let a large + /// `SELECT` over the JSON endpoint OOM the process. `RawValue` + /// re-serializes verbatim, so the wire format is unchanged. + rows: Vec>, /// Information about each column. desc: Description, // Any notices generated during execution of the query. @@ -610,9 +631,16 @@ pub enum SqlResult { } impl SqlResult { - /// Convert adapter Row results into the web row result format. Error if the row format does not - /// match the expected descriptor. - // TODO(aljoscha): Bail when max_result_size is exceeded. + /// Convert adapter Row results into the buffered web row result format. Error + /// if the row format does not match the expected descriptor, or if the + /// result exceeds `max_query_result_size`. + /// + /// This buffers the whole result, so it is used only by the JSON transport, + /// whose response is a single document. The WebSocket transport streams rows + /// through `StatementResult::Rows` and never calls this. The size guard + /// counts `Row::byte_len`, matching the WebSocket transport and pgwire, and + /// rows are stored as pre-serialized compact `RawValue`s to avoid the + /// amplified `Value`-tree buffering that could OOM. async fn rows( sender: &mut S, client: &mut SessionClient, @@ -623,11 +651,11 @@ impl SqlResult { where S: ResultSender, { - let mut rows: Vec> = vec![]; + let mut rows: Vec> = vec![]; let mut datum_vec = mz_repr::DatumVec::new(); let types = &desc.typ().column_types; - let mut query_result_size = 0; + let mut query_result_size: usize = 0; loop { let peek_response = tokio::select! { @@ -665,7 +693,10 @@ impl SqlResult { } while let Some(row) = sql_rows.next() { - query_result_size += row.byte_len(); + // Enforce `max_result_size` on `Row::byte_len`, the same quantity + // pgwire and the WebSocket transport use, so the cap means one + // thing across every transport. + query_result_size = query_result_size.saturating_add(row.byte_len()); if query_result_size > max_query_result_size { use bytesize::ByteSize; return Ok(SqlResult::err( @@ -676,18 +707,21 @@ impl SqlResult { )), )); } - let datums = datum_vec.borrow_with(row); - rows.push( - datums - .iter() - .enumerate() - .map(|(i, d)| { - TypedDatum::new(*d, &types[i]) - .json(&JsonNumberPolicy::ConvertNumberToString) - }) - .collect(), - ); + let json_row: Vec = datums + .iter() + .enumerate() + .map(|(i, d)| { + TypedDatum::new(*d, &types[i]) + .json(&JsonNumberPolicy::ConvertNumberToString) + }) + .collect(); + // Keep only the compact serialized JSON text. The transient + // `Value` tree above is dropped per row, so at most one row's tree + // is resident, avoiding the amplified buffering that could OOM. + let raw = serde_json::value::to_raw_value(&json_row) + .expect("row of JSON values always serializes"); + rows.push(raw); } } @@ -873,7 +907,7 @@ impl ResultSender for SqlResponse { // needs to be retired for statement logging purposes. async fn add_result( &mut self, - _client: &mut SessionClient, + client: &mut SessionClient, res: StatementResult, ) -> ( Result, Error>, @@ -886,6 +920,25 @@ impl ResultSender for SqlResponse { let res = if is_err { Err(()) } else { Ok(()) }; (res, None) } + StatementResult::Rows { + desc, + rows_stream, + max_result_size, + } => { + // The JSON transport is a single buffered document, so the rows + // must be collected before the response is serialized. + // `SqlResult::rows` bounds that buffer against `max_result_size`. + let res = match SqlResult::rows(self, client, rows_stream, max_result_size, &desc) + .await + { + Ok(res) => res, + Err(e) => return (Err(e), None), + }; + let is_err = matches!(res, SqlResult::Err { .. }); + self.results.push(res); + let res = if is_err { Err(()) } else { Ok(()) }; + (res, None) + } StatementResult::Subscribe { ctx_extra, .. } => { let message = "SUBSCRIBE only supported over websocket"; self.results.push(SqlResult::Err { @@ -936,6 +989,7 @@ impl ResultSender for WebSocket { StatementResult::SqlResult(SqlResult::Err { .. }) => (false, false), StatementResult::SqlResult(SqlResult::Ok { .. }) => (false, false), StatementResult::SqlResult(SqlResult::Rows { .. }) => (true, false), + StatementResult::Rows { .. } => (true, false), StatementResult::Subscribe { .. } => (true, true), }; if let Err(e) = send_ws_response( @@ -951,26 +1005,25 @@ impl ResultSender for WebSocket { } let (is_err, msgs, stmt_logging) = match res { - StatementResult::SqlResult(SqlResult::Rows { - tag, - rows, - desc, - notices, - }) => { - // Stream rows directly to avoid buffering all rows as - // WebSocketResponse, which inflates memory due to enum sizing. - if let Err(e) = send_ws_response(self, WebSocketResponse::Rows(desc)).await { - return (Err(e), None); - } - for row in rows { - if let Err(e) = send_ws_response(self, WebSocketResponse::Row(row)).await { - return (Err(e), None); - } - } - let mut msgs = vec![WebSocketResponse::CommandComplete(tag)]; - msgs.extend(notices.into_iter().map(WebSocketResponse::Notice)); - (false, msgs, None) + StatementResult::SqlResult(SqlResult::Rows { .. }) => { + // `SqlResult::Rows` holds already-collected rows for the buffered + // JSON transport only. The WebSocket transport streams peek + // results through `StatementResult::Rows` and never materializes + // a `SqlResult::Rows`. + unreachable!("WebSocket streams peek rows via StatementResult::Rows") } + StatementResult::Rows { + ref desc, + mut rows_stream, + max_result_size, + } => match stream_ws_peek_rows(self, client, desc, &mut rows_stream, max_result_size) + .await + { + Ok(result) => result, + // A write failure means the remote broke the connection, which we + // treat as a cancellation to match pgwire. + Err(e) => return (Err(e), None), + }, StatementResult::SqlResult(SqlResult::Ok { ok, parameters, @@ -1036,7 +1089,7 @@ impl ResultSender for WebSocket { rows_returned += rows.count(); while let Some(row) = rows.next() { - result_size += row.byte_len(); + result_size = result_size.saturating_add(row.byte_len()); let datums = datum_vec.borrow_with(row); let types = &desc.typ().column_types; if let Err(e) = send_ws_response( @@ -1162,6 +1215,161 @@ where } } +/// Streams a peek (`SELECT`) result to a WebSocket client one stash batch at a +/// time, flushing between batches so a slow client applies real backpressure +/// and only one batch is resident. This gives a large `SELECT` over WebSocket +/// the same bounded memory profile as pgwire `SELECT`, and mirrors the +/// `Subscribe` arm of `WebSocket::add_result`. +/// +/// On success returns the `(is_err, msgs, stmt_logging)` triple that +/// `add_result` folds into its response. An `Err` means a write to the socket +/// failed, so the server should disconnect. `add_result` turns that into a +/// cancellation to match pgwire. +/// +/// The `Rows` descriptor is sent lazily, right before the first batch of rows +/// or an empty successful result, but never before an error. So a query that +/// fails before producing any rows emits only an `Error`. +async fn stream_ws_peek_rows( + ws: &mut WebSocket, + client: &mut SessionClient, + desc: &RelationDesc, + rows_stream: &mut RecordFirstRowStream, + max_result_size: usize, +) -> Result< + ( + bool, + Vec, + Option<(StatementEndedExecutionReason, ExecuteContextGuard)>, + ), + Error, +> { + let mut datum_vec = mz_repr::DatumVec::new(); + let mut result_size: usize = 0; + let mut rows_returned: usize = 0; + let mut sent_rows_desc = false; + loop { + // Bind before matching so the `Option` (which has a + // significant `Drop`) is not a temporary living for the whole match. + let res = await_rows(ws, client, rows_stream.recv()).await?; + match res { + Some(PeekResponseUnary::Rows(mut rows)) => { + if let Err(err) = verify_datum_desc(desc, &mut rows) { + return Ok(ws_peek_result( + client, + true, + vec![WebSocketResponse::Error(err.into())], + )); + } + // The header waits until a batch has passed `verify_datum_desc`, + // so a query that fails before producing any rows emits only an + // `Error`. Sending it before the loop, as the `Subscribe` arm + // does, would put a `Rows` header in front of that `Error`. + if !sent_rows_desc { + send_ws_response(ws, WebSocketResponse::Rows(desc.into())).await?; + sent_rows_desc = true; + } + let types = &desc.typ().column_types; + while let Some(row) = rows.next() { + result_size = result_size.saturating_add(row.byte_len()); + if result_size > max_result_size { + use bytesize::ByteSize; + return Ok(ws_peek_result( + client, + true, + vec![WebSocketResponse::Error( + AdapterError::ResultSize(format!( + "result exceeds max size of {}", + ByteSize::b(u64::cast_from(max_result_size)) + )) + .into(), + )], + )); + } + let datums = datum_vec.borrow_with(row); + send_ws_response( + ws, + WebSocketResponse::Row( + datums + .iter() + .enumerate() + .map(|(i, d)| { + TypedDatum::new(*d, &types[i]) + .json(&JsonNumberPolicy::ConvertNumberToString) + }) + .collect(), + ), + ) + .await?; + rows_returned += 1; + } + } + Some(PeekResponseUnary::Error(error)) => { + return Ok(ws_peek_result( + client, + true, + vec![WebSocketResponse::Error( + Error::Unstructured(anyhow!(error)).into(), + )], + )); + } + Some(PeekResponseUnary::DependencyDropped(dep)) => { + return Ok(ws_peek_result( + client, + true, + vec![WebSocketResponse::Error( + dep.to_concurrent_dependency_drop().into(), + )], + )); + } + Some(PeekResponseUnary::Canceled) => { + return Ok(ws_peek_result( + client, + true, + vec![WebSocketResponse::Error(AdapterError::Canceled.into())], + )); + } + None => { + // An empty successful result still owes the client a `Rows` + // descriptor before `CommandComplete`. + if !sent_rows_desc { + send_ws_response(ws, WebSocketResponse::Rows(desc.into())).await?; + } + return Ok(ws_peek_result( + client, + false, + vec![WebSocketResponse::CommandComplete(format!( + "SELECT {rows_returned}" + ))], + )); + } + } + } +} + +/// Packages a terminal result of `stream_ws_peek_rows`, appending any notices +/// still buffered on the session after `msgs`. +/// +/// The streaming loop forwards notices live through `await_rows`' `recv_notice` +/// select, but the terminal `recv() -> None` can race a still-buffered notice +/// and end the loop first. Draining here restores the buffered path's guarantee +/// that all notices reach the client, after `CommandComplete` or `Error`. +fn ws_peek_result( + client: &mut SessionClient, + is_err: bool, + mut msgs: Vec, +) -> ( + bool, + Vec, + Option<(StatementEndedExecutionReason, ExecuteContextGuard)>, +) { + msgs.extend( + make_notices(client) + .into_iter() + .map(WebSocketResponse::Notice), + ); + (is_err, msgs, None) +} + async fn send_and_retire( res: StatementResult, client: &mut SessionClient, @@ -1614,7 +1822,7 @@ async fn execute_stmt( instance_id, strategy, } => { - let max_query_result_size = + let max_result_size = usize::cast_from(client.get_system_vars().await.max_result_size()); let rows_stream = RecordFirstRowStream::new( @@ -1625,33 +1833,25 @@ async fn execute_stmt( Some(strategy), ); - SqlResult::rows( - sender, - client, + StatementResult::Rows { + desc: desc.relation_desc.expect("RelationDesc must exist"), rows_stream, - max_query_result_size, - &desc.relation_desc.expect("RelationDesc must exist"), - ) - .await? - .into() + max_result_size, + } } ExecuteResponse::SendingRowsImmediate { rows } => { - let max_query_result_size = + let max_result_size = usize::cast_from(client.get_system_vars().await.max_result_size()); let rows = futures::stream::once(futures::future::ready(PeekResponseUnary::Rows(rows))); let rows_stream = RecordFirstRowStream::new(Box::new(rows), execute_started, client, None, None); - SqlResult::rows( - sender, - client, + StatementResult::Rows { + desc: desc.relation_desc.expect("RelationDesc must exist"), rows_stream, - max_query_result_size, - &desc.relation_desc.expect("RelationDesc must exist"), - ) - .await? - .into() + max_result_size, + } } ExecuteResponse::Subscribing { rx, @@ -1660,13 +1860,7 @@ async fn execute_stmt( } => StatementResult::Subscribe { tag: "SUBSCRIBE".into(), desc: desc.relation_desc.unwrap(), - rx: RecordFirstRowStream::new( - Box::new(UnboundedReceiverStream::new(rx)), - execute_started, - client, - Some(instance_id), - None, - ), + rx: RecordFirstRowStream::new(rx, execute_started, client, Some(instance_id), None), ctx_extra, }, res @ (ExecuteResponse::Fetch { .. } diff --git a/src/environmentd/src/http/static/js/cluster-replica.jsx b/src/environmentd/src/http/static/js/cluster-replica.jsx new file mode 100644 index 0000000000000..2155b2378fdd4 --- /dev/null +++ b/src/environmentd/src/http/static/js/cluster-replica.jsx @@ -0,0 +1,139 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +'use strict'; + +// NOTE: this file and the page script that consumes it are separate classic +// scripts sharing one global scope, so a top-level `const { useState } = React` +// here would collide with the same declaration in the page script. Reach +// through `React` instead. + +/** + * The cluster replica picker shared by the memory visualizer pages. + * + * Renders `props.children(clusterName, replicaName)` once a replica is settled, + * which is the one named in the URL if there is one and otherwise the one + * chosen below. The choice is mirrored back into the URL, so a page can be + * linked to a specific replica. + */ +function ClusterReplicaView(props) { + const [currentClusterName, setCurrentClusterName] = React.useState(null); + const [currentReplicaName, setCurrentReplicaName] = React.useState(null); + const [replicas, setReplicas] = React.useState(null); + const [loading, setLoading] = React.useState(true); + const [error, setError] = React.useState(false); + + // The first statement names the session's cluster, which is the starting + // point for the choice below. `mz_clusters.id` starts with `u` for the + // clusters a user created, which is what separates them from the builtins. + const queryClusterReplicas = ` + SELECT current_setting('cluster'); + + SELECT + clusters.name AS cluster_name, + replicas.name AS replica_name, + clusters.id LIKE 'u%' AS user_cluster + FROM + mz_catalog.mz_cluster_replicas replicas + LEFT JOIN mz_catalog.mz_clusters clusters ON clusters.id = replicas.cluster_id + ORDER BY cluster_name ASC, replica_name ASC + `; + + React.useEffect(() => { + const search = new URLSearchParams(location.search); + const clusterName = search.get('cluster_name'); + const replicaName = search.get('replica_name'); + if (clusterName) { + setCurrentClusterName(clusterName); + } + if (replicaName) { + setCurrentReplicaName(replicaName); + } + + query(queryClusterReplicas) + .then((data) => { + const [sessionClusterTable, replicasTable] = data.results; + const sessionCluster = sessionClusterTable.rows[0][0]; + const replicas = replicasTable.rows.map( + ([clusterName, replicaName, userCluster]) => ({ + clusterName, + replicaName, + userCluster, + }) + ); + setReplicas(replicas); + if (!replicaName && replicas.length > 0) { + // A user cluster is preferred over the session's own, because this + // page is normally reached through a proxy that authenticates as + // `mz_support`, whose default cluster is the builtin + // `mz_catalog_server`. Nobody opens the dataflow visualizer to look + // at the catalog server, and its introspection relations are an + // order of magnitude more expensive to query than a small user + // cluster's. Rows are ordered by cluster name and then replica name, + // so each fallback lands on the first replica of the first cluster + // that qualifies. + const preferred = + replicas.find( + (r) => r.userCluster && r.clusterName === sessionCluster + ) || + replicas.find((r) => r.userCluster) || + replicas[0]; + setCurrentClusterName(preferred.clusterName); + setCurrentReplicaName(preferred.replicaName); + } + setLoading(false); + }) + .catch((error) => { + setError(error); + setLoading(false); + }); + }, []); + + React.useEffect(() => { + if (!currentReplicaName) return; + const params = new URLSearchParams(location.search); + params.set('cluster_name', currentClusterName); + params.set('replica_name', currentReplicaName); + window.history.replaceState({}, '', `${location.pathname}?${params}`); + }, [currentClusterName, currentReplicaName]); + + return ( +
+ {loading ? ( +
Loading...
+ ) : error ? ( +
error: {String(error)}
+ ) : ( +
+ + + {props.children(currentClusterName, currentReplicaName)} +
+ )} +
+ ); +} diff --git a/src/environmentd/src/http/static/js/hierarchical-memory.jsx b/src/environmentd/src/http/static/js/hierarchical-memory.jsx index 9cb0119643e79..ebaecf32894eb 100644 --- a/src/environmentd/src/http/static/js/hierarchical-memory.jsx +++ b/src/environmentd/src/http/static/js/hierarchical-memory.jsx @@ -17,102 +17,11 @@ function formatNameForQuery(name) { const { useState, useEffect } = React; -function ClusterReplicaView() { - const [currentClusterName, setCurrentClusterName] = useState(null); - const [currentReplicaName, setCurrentReplicaName] = useState(null); - const [sqlResponse, setSqlResponse] = useState(null); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(false); - - const queryClusterReplicas = ` - SELECT - clusters.name AS cluster_name, replicas.name AS replica_name - FROM - mz_catalog.mz_cluster_replicas replicas - LEFT JOIN mz_catalog.mz_clusters clusters ON clusters.id = replicas.cluster_id - ORDER BY cluster_name ASC, replica_name ASC - `; - - useEffect(() => { - const search = new URLSearchParams(location.search); - const clusterName = search.get('cluster_name'); - const replicaName = search.get('replica_name'); - if (clusterName) { - setCurrentClusterName(clusterName); - } - if (replicaName) { - setCurrentReplicaName(replicaName); - } - query(queryClusterReplicas) - .then((data) => { - const results = data.results[0].rows; - setSqlResponse(results); - if (!replicaName && results.length > 0) { - if(results.some( - result => ('default' == result[0]) && ('r1' == result[1]))) { - setCurrentClusterName('default'); - setCurrentReplicaName('r1'); - } else { - setCurrentClusterName(results[0][0]); - setCurrentReplicaName(results[0][1]); - } - } - setLoading(false); - }) - .catch((error) => { - setError(error); - setLoading(false); - }); - }, []); - - useEffect(() => { - if (!currentReplicaName) return; - const params = new URLSearchParams(location.search); - params.set('cluster_name', currentClusterName); - params.set('replica_name', currentReplicaName); - window.history.replaceState({}, '', `${location.pathname}?${params}`); - }, [currentClusterName, currentReplicaName]); - - return ( -
- {loading ? ( -
Loading...
- ) : error ? ( -
error: {String(error)}
- ) : ( -
- - - -
- )} -
- ); -} - function Dataflows(props) { - const [stats, setStats] = useState(null); const [addrs, setAddrs] = useState(null); const [records, setRecords] = useState(null); const [opers, setOpers] = useState(null); const [chans, setChans] = useState(null); - const [view, setView] = useState(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(false); const [page, setPage] = useState(null); @@ -174,14 +83,6 @@ function Dataflows(props) { ) setRecords(records); - try { - const view = await getCreateView(stats.name); - setView(view); - } catch (error) { - console.debug('could not get create view:', error); - setView(null); - } - console.log("Loaded"); setLoading(false); }; @@ -337,68 +238,6 @@ function Dataflows(props) { } -async function getCreateView(dataflow_name) { - // dataflow_name is the full name of the dataflow operator. It is generally - // of the form "Dataflow: ..". We will use a - // regex to parse these out and use them to get the fully qualified view name - // which we will use with SHOW CREATE VIEW to show the SQL that created this - // dataflow. - // - // There are known problems with this method. It doesn't know anything about - // SQL parsing or escaping, assumes the dataflow operator's name is of a very - // specific shape, and assumes a CREATE VIEW statement made an index which made - // this dataflow. So we assume that problems can happen at any level here and - // will cleanly bail if anything doesn't exactly match what we want. In that - // case we will not show the SQL. This is intended to be good enough for most - // users for now. - const match = dataflow_name.match(/^Dataflow: (.*)\.(.*)\.(.*)$/); - if (!match) { - throw 'unknown dataflow name pattern'; - } - const view_name_table = await query(` - SELECT - d.name AS database, s.schema, s.view - FROM - mz_catalog.mz_databases AS d - JOIN ( - SELECT - s.database_id, s.name AS schema, v.view - FROM - mz_catalog.mz_schemas AS s - JOIN ( - SELECT - name AS view, schema_id - FROM - mz_catalog.mz_views - WHERE - id - = ( - SELECT - DISTINCT idx.on_id - FROM - mz_catalog.mz_databases AS db, - mz_catalog.mz_schemas AS sc, - mz_catalog.mz_indexes AS idx - WHERE - db.name = '${match[1]}' - AND sc.name = '${match[2]}' - AND idx.name = '${match[3]}' - ) - ) - AS v ON s.id = v.schema_id - ) - AS s ON d.id = s.database_id; - `); - if (view_name_table.rows.length !== 1) { - throw 'could not determine view'; - } - const name = view_name_table.rows[0]; - const create_table = await query( - `SHOW CREATE VIEW "${name[0]}"."${name[1]}"."${name[2]}"` - ); - return { name: create_table.rows[0][0], create: create_table.rows[0][1] }; -} - function addrStr(addr) { return addr.join(', '); } @@ -423,4 +262,11 @@ function toggle_active(e) { } const content = document.getElementById('content2'); -ReactDOM.render(, content); +ReactDOM.render( + + {(clusterName, replicaName) => ( + + )} + , + content +); diff --git a/src/environmentd/src/http/static/js/memory.jsx b/src/environmentd/src/http/static/js/memory.jsx index 2e6b822cb3000..5d3f72de23842 100644 --- a/src/environmentd/src/http/static/js/memory.jsx +++ b/src/environmentd/src/http/static/js/memory.jsx @@ -17,96 +17,6 @@ function formatNameForQuery(name) { const { useState, useEffect } = React; -function ClusterReplicaView() { - const [currentClusterName, setCurrentClusterName] = useState(null); - const [currentReplicaName, setCurrentReplicaName] = useState(null); - const [sqlResponse, setSqlResponse] = useState(null); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(false); - - const queryClusterReplicas = ` - SELECT - clusters.name AS cluster_name, replicas.name AS replica_name - FROM - mz_catalog.mz_cluster_replicas replicas - LEFT JOIN mz_catalog.mz_clusters clusters ON clusters.id = replicas.cluster_id - ORDER BY cluster_name ASC, replica_name ASC - `; - - useEffect(() => { - const search = new URLSearchParams(location.search); - const clusterName = search.get('cluster_name'); - const replicaName = search.get('replica_name'); - if (clusterName) { - setCurrentClusterName(clusterName); - } - if (replicaName) { - setCurrentReplicaName(replicaName); - } - - query(queryClusterReplicas) - .then((data) => { - const results = data.results[0].rows; - setSqlResponse(results); - if (!replicaName && results.length > 0) { - if(results.some( - result => ('default' == result[0]) && ('r1' == result[1]))) { - setCurrentClusterName('default'); - setCurrentReplicaName('r1'); - } else { - setCurrentClusterName(results[0][0]); - setCurrentReplicaName(results[0][1]); - } - } - setLoading(false); - }) - .catch((error) => { - setError(error); - setLoading(false); - }); - }, []); - - useEffect(() => { - if (!currentReplicaName) return; - const params = new URLSearchParams(location.search); - params.set('cluster_name', currentClusterName); - params.set('replica_name', currentReplicaName); - window.history.replaceState({}, '', `${location.pathname}?${params}`); - }, [currentClusterName, currentReplicaName]); - - return ( -
- {loading ? ( -
Loading...
- ) : error ? ( -
error: {String(error)}
- ) : ( -
- - - -
- )} -
- ); -} - function Views(props) { const [currentDataflow, setCurrentDataflow] = useState(null); const [includeSystemCatalog, setIncludeSystemCatalog] = useState(false); @@ -691,7 +601,9 @@ async function getCreateView(dataflow_name) { if (!match) { throw 'unknown dataflow name pattern'; } - const view_name_table = await query(` + const { + results: [view_name_table], + } = await query(` SELECT d.name AS database, s.schema, s.view FROM @@ -729,7 +641,9 @@ async function getCreateView(dataflow_name) { throw 'could not determine view'; } const name = view_name_table.rows[0]; - const create_table = await query(`SHOW CREATE VIEW "${name[0]}"."${name[1]}"."${name[2]}"`); + const { + results: [create_table], + } = await query(`SHOW CREATE VIEW "${name[0]}"."${name[1]}"."${name[2]}"`); return { name: create_table.rows[0][0], create: create_table.rows[0][1] }; } @@ -758,4 +672,11 @@ function dispNs(ns) { } const content = document.getElementById('content'); -ReactDOM.render(, content); +ReactDOM.render( + + {(clusterName, replicaName) => ( + + )} + , + content +); diff --git a/src/environmentd/templates/hierarchical-memory.html b/src/environmentd/templates/hierarchical-memory.html index 1e867a3490f5e..285c7548ad4ca 100644 --- a/src/environmentd/templates/hierarchical-memory.html +++ b/src/environmentd/templates/hierarchical-memory.html @@ -51,6 +51,7 @@ {# GraphViz in WASM #} {# Zlib compression #} + {# Shared replica picker #} {% endblock %} diff --git a/src/environmentd/templates/memory.html b/src/environmentd/templates/memory.html index ded02ab5c06b4..80dea44bad1ce 100644 --- a/src/environmentd/templates/memory.html +++ b/src/environmentd/templates/memory.html @@ -11,6 +11,7 @@ {# GraphViz in WASM #} {# Zlib compression #} + {# Shared replica picker #} {% endblock %} diff --git a/src/environmentd/tests/server.rs b/src/environmentd/tests/server.rs index 4c1c8bb2d1ecc..acfd4cb088da8 100644 --- a/src/environmentd/tests/server.rs +++ b/src/environmentd/tests/server.rs @@ -1297,6 +1297,299 @@ fn test_ws_notifies_for_bad_options() { }; } +/// A `SELECT` over `/api/ws` emits a `Rows` descriptor, then one `Row` message +/// per row, then `CommandComplete`. This pins the wire shape, which is the same +/// shape a buffered implementation would produce. What proves the rows are +/// emitted incrementally is `test_ws_select_streams_rows_then_size_error`. +#[mz_ore::test] +#[cfg_attr(miri, ignore)] // too slow +fn test_ws_select_row_message_shape() { + let server = test_util::TestHarness::default().start_blocking(); + let ws_url = server.ws_addr(); + let (mut ws, _resp) = tungstenite::connect(ws_url).unwrap(); + test_util::auth_with_ws(&mut ws, BTreeMap::default()).unwrap(); + + let query = "SELECT * FROM generate_series(1, 1000)"; + let json = serde_json::json!({ "query": query }); + ws.send(Message::Text(json.to_string().into())).unwrap(); + + let mut read_msg = || -> WebSocketResponse { + let msg = ws.read().unwrap(); + let msg = msg.into_text().expect("response should be text"); + serde_json::from_str(&msg).unwrap() + }; + + match read_msg() { + WebSocketResponse::CommandStarting(_) => {} + other => panic!("expected CommandStarting, got {other:?}"), + } + match read_msg() { + WebSocketResponse::Rows(desc) => assert_eq!(desc.columns.len(), 1), + other => panic!("expected Rows description, got {other:?}"), + } + + // Every row arrives as its own `Row` message before `CommandComplete`. + let mut rows_seen = 0; + loop { + match read_msg() { + WebSocketResponse::Row(_) => rows_seen += 1, + WebSocketResponse::Notice(_) => continue, + WebSocketResponse::CommandComplete(tag) => { + assert_eq!(tag, "SELECT 1000"); + break; + } + other => panic!("unexpected message before CommandComplete: {other:?}"), + } + } + assert_eq!(rows_seen, 1000); +} + +/// Regression test for SQL-428: a `SELECT` over `/api/ws` emits rows as it +/// streams them, so a result that outgrows `max_result_size` mid-flight arrives +/// as some `Row` messages followed by an `Error`. An implementation that +/// buffered the result would size it before emitting anything and send the +/// `Error` alone, so this is the assertion that pins streaming. +/// +/// NOTE: clients must therefore tolerate rows that precede an error, which is +/// the contract pgwire has always had. +#[mz_ore::test] +#[cfg_attr(miri, ignore)] // too slow +#[allow(clippy::disallowed_methods)] +fn test_ws_select_streams_rows_then_size_error() { + let server = test_util::TestHarness::default().start_blocking(); + + // `max_result_size` is a system parameter, so lower it over the internal + // (system) port. A normal session cannot. + let mut system = server.connect_internal(postgres::NoTls).unwrap(); + system + .batch_execute("ALTER SYSTEM SET max_result_size = '1MB'") + .unwrap(); + + let (mut ws, _resp) = tungstenite::connect(server.ws_addr()).unwrap(); + test_util::auth_with_ws(&mut ws, BTreeMap::default()).unwrap(); + + // Each row is ~10 KB, so the cap trips around row 100 of 500. + const ROWS: usize = 500; + let query = format!("SELECT repeat('x', 10000) FROM generate_series(1, {ROWS})"); + let json = serde_json::json!({ "query": query }); + ws.send(Message::Text(json.to_string().into())).unwrap(); + + let mut read_msg = || -> WebSocketResponse { + let msg = ws.read().unwrap(); + let msg = msg.into_text().expect("response should be text"); + serde_json::from_str(&msg).unwrap() + }; + + match read_msg() { + WebSocketResponse::CommandStarting(_) => {} + other => panic!("expected CommandStarting, got {other:?}"), + } + match read_msg() { + WebSocketResponse::Rows(desc) => assert_eq!(desc.columns.len(), 1), + other => panic!("expected Rows description, got {other:?}"), + } + + let mut rows_seen = 0; + let error = loop { + match read_msg() { + WebSocketResponse::Row(_) => rows_seen += 1, + WebSocketResponse::Notice(_) => continue, + WebSocketResponse::Error(err) => break err, + other => panic!("unexpected message before the size error: {other:?}"), + } + }; + + assert!( + error.message.contains("result exceeds max size"), + "expected a result-size error, got: {error:?}" + ); + assert!( + rows_seen > 0 && rows_seen < ROWS, + "expected a truncated stream of rows before the error, got {rows_seen}" + ); +} + +/// Regression test for SQL-428: the JSON `/api/sql` endpoint enforces +/// `max_result_size` on `Row::byte_len`, the same quantity the WebSocket +/// transport and pgwire use. A result that exceeds the cap fails cleanly with a +/// result-size error rather than being buffered whole, while a result under the +/// cap succeeds. +#[mz_ore::test] +#[cfg_attr(miri, ignore)] // too slow +#[allow(clippy::disallowed_methods)] +fn test_http_sql_result_size_limit() { + let server = test_util::TestHarness::default().start_blocking(); + + // Lower the cap to its minimum. `max_result_size` is a system parameter, so + // set it via the internal (system) port. A normal session cannot. + let mut system = server.connect_internal(postgres::NoTls).unwrap(); + system + .batch_execute("ALTER SYSTEM SET max_result_size = '1MB'") + .unwrap(); + + let http_url = Url::parse(&format!("http://{}/api/sql", server.http_local_addr())).unwrap(); + let post = |query: &str| -> serde_json::Value { + let json = serde_json::json!({ "query": query }); + let res = Client::new() + .post(http_url.clone()) + .json(&json) + .send() + .unwrap(); + res.json().unwrap() + }; + + // A result whose rows exceed the cap fails cleanly. + let body = post("SELECT * FROM generate_series(1, 500000)"); + let results = body.get("results").unwrap().as_array().unwrap(); + let err = results[0] + .get("error") + .and_then(|e| e.get("message")) + .and_then(|m| m.as_str()); + assert!( + err.map_or(false, |m| m.contains("result exceeds max size")), + "expected a result-size error, got: {body}" + ); + + // A result that fits under the cap succeeds. + let body = post("SELECT * FROM generate_series(1, 3)"); + let results = body.get("results").unwrap().as_array().unwrap(); + assert!( + results[0].get("error").is_none(), + "expected success, got: {body}" + ); + let rows = results[0].get("rows").unwrap().as_array().unwrap(); + assert_eq!(rows.len(), 3); +} + +/// Regression test for SQL-423: a SUBSCRIBE whose client stops reading is +/// retired once its coordinator-side buffer exceeds +/// `subscribe_max_buffered_bytes`, rather than growing environmentd's memory +/// without bound. The client sees a clean "fell behind" error. +#[mz_ore::test] +#[cfg_attr(miri, ignore)] // too slow +#[allow(clippy::disallowed_methods)] +fn test_subscribe_buffer_bound() { + let server = test_util::TestHarness::default() + .with_system_parameter_default( + "subscribe_max_buffered_bytes".to_string(), + "1024".to_string(), + ) + .start_blocking(); + + let mut writer = server.connect(postgres::NoTls).unwrap(); + writer.batch_execute("CREATE TABLE t (a bigint)").unwrap(); + + // Open a SUBSCRIBE over COPY but never read from it, so the coordinator-side + // buffer fills once the socket backs up. + let mut reader_client = server.connect(postgres::NoTls).unwrap(); + let mut copy = reader_client + .copy_out("COPY (SUBSCRIBE t) TO STDOUT") + .unwrap(); + + // Drive far more than the 1 KiB budget through the subscribe while the + // reader is not draining. + for _ in 0..200 { + writer + .batch_execute("INSERT INTO t SELECT generate_series(1, 1000)") + .unwrap(); + } + + // Draining now surfaces the terminal "fell behind" error rather than an + // unbounded backlog. The COPY read fails with the underlying DB error, whose + // message lives in the error's source chain (the top-level display is just + // "db error"). + let mut buf = Vec::new(); + let io_err = std::io::Read::read_to_end(&mut copy, &mut buf).unwrap_err(); + let mut chain = io_err.to_string(); + let mut source = std::error::Error::source(&io_err); + while let Some(err) = source { + chain.push_str(" | "); + chain.push_str(&err.to_string()); + source = err.source(); + } + assert!( + chain.contains("fell behind"), + "expected a fell-behind error, got: {chain}" + ); +} + +/// Regression test for SQL-423: a client that drains each batch as it arrives is +/// never retired, however many batches flow past the budget in total. A false +/// `SubscribeFellBehind` on a well-behaved client is the failure mode this +/// budget must not introduce. +/// +/// The budget here is 64 KiB and each batch costs at least the 1 KiB per-message +/// overhead, so a client that stopped draining would be retired within ~64 of +/// the 200 batches. +#[mz_ore::test] +#[cfg_attr(miri, ignore)] // too slow +#[allow(clippy::disallowed_methods)] +fn test_subscribe_keeping_up_client_not_retired() { + let server = test_util::TestHarness::default() + .with_system_parameter_default( + "subscribe_max_buffered_bytes".to_string(), + "65536".to_string(), + ) + .start_blocking(); + + let mut writer = server.connect(postgres::NoTls).unwrap(); + writer.batch_execute("CREATE TABLE t (a bigint)").unwrap(); + + let mut reader = server.connect(postgres::NoTls).unwrap(); + reader + .batch_execute("BEGIN; DECLARE c CURSOR FOR SUBSCRIBE t") + .unwrap(); + + // `FETCH` defaults to `WaitOnce`, so each round trip blocks until the batch + // for the insert it follows has arrived. That makes the drain deterministic + // rather than timing-dependent. + const BATCHES: usize = 200; + let mut rows_seen = 0; + for i in 0..BATCHES { + writer + .execute("INSERT INTO t VALUES ($1)", &[&i64::try_from(i).unwrap()]) + .unwrap(); + rows_seen += reader + .query("FETCH ALL c", &[]) + .expect("a client that keeps up must not be retired") + .len(); + } + assert_eq!(rows_seen, BATCHES); +} + +/// Regression test for SQL-423: a SUBSCRIBE whose snapshot is a single batch +/// larger than `subscribe_max_buffered_bytes` completes when the client reads it +/// promptly. The budget bounds accumulated backlog, not the size of any single +/// batch, and it is independent of `max_result_size`, which is what bounds an +/// individual batch. +#[mz_ore::test] +#[cfg_attr(miri, ignore)] // too slow +#[allow(clippy::disallowed_methods)] +fn test_subscribe_single_large_batch_not_retired() { + let server = test_util::TestHarness::default() + .with_system_parameter_default( + "subscribe_max_buffered_bytes".to_string(), + "1024".to_string(), + ) + .start_blocking(); + + let mut client = server.connect(postgres::NoTls).unwrap(); + + // Subscribe to a finite collection whose snapshot is a single batch far + // larger than the 1 KiB budget. The subscribe finishes once the snapshot is + // delivered, and a client reading it promptly (via `query`) keeps the + // backlog at that single batch, so it must stream to completion rather than + // be retired with a "fell behind" error. + let rows: usize = 10_000; + let returned = client + .query( + &format!("SUBSCRIBE TO (SELECT * FROM generate_series(1, {rows}))"), + &[], + ) + .expect("single large snapshot batch should stream, not be retired"); + assert_eq!(returned.len(), rows); +} + #[derive(Debug, Deserialize)] struct HttpResponse { results: Vec, @@ -1835,27 +2128,41 @@ async fn test_termination_races() { .unwrap(); let adapter_client = server.inner.adapter_client(); - let conn_id = adapter_client.new_conn_id().unwrap(); - let session = adapter_client.new_session( - SessionConfig { - conn_id, - uuid: Uuid::new_v4(), - user: "materialize".to_string(), - client_ip: None, - external_metadata_rx: None, - helm_chart_version: None, - authenticator_kind: AuthenticatorKind::None, - groups: None, - }, - Authenticated, - ); - // Box the future so that dropping it below drops the future itself, not - // just a reference to it. - let mut fut = Box::pin(adapter_client.startup(session)); - // The first poll sends the Startup command to the Coordinator and installs - // the cleanup guard around the response channel. - assert!(futures::poll!(&mut fut).is_pending()); + // The cleanup guard only runs on a response the startup future never observed, so + // the future has to be left pending on its first poll. That poll both sends the + // Startup command and polls the response channel, and the Coordinator runs on its + // own thread, so it can answer in between and resolve the future right away. Retry + // until the poll lands while the response is still in flight. A lost attempt + // leaves nothing behind, a failed startup registers no state in the Coordinator. + let mut fut = None; + for _ in 0..100 { + let conn_id = adapter_client.new_conn_id().unwrap(); + let session = adapter_client.new_session( + SessionConfig { + conn_id, + uuid: Uuid::new_v4(), + user: "materialize".to_string(), + client_ip: None, + external_metadata_rx: None, + helm_chart_version: None, + authenticator_kind: AuthenticatorKind::None, + groups: None, + }, + Authenticated, + ); + // Box the future so that dropping it below drops the future itself, not + // just a reference to it. + let mut candidate = Box::pin(adapter_client.startup(session)); + // The first poll sends the Startup command to the Coordinator and installs + // the cleanup guard around the response channel. + if futures::poll!(&mut candidate).is_pending() { + fut = Some(candidate); + break; + } + } + let fut = fut.expect("startup always answered before its first poll finished"); + // Wait for the Coordinator to process the Startup command and place the // startup error in the response channel. Commands on the client channel // are processed in order, so a completed round trip implies the Startup diff --git a/src/environmentd/tests/testdata/http/ws b/src/environmentd/tests/testdata/http/ws index 17dbeff38719b..23e7a86a396e7 100644 --- a/src/environmentd/tests/testdata/http/ws +++ b/src/environmentd/tests/testdata/http/ws @@ -413,7 +413,7 @@ ws-text fixid=true {"query": "SELECT 1 / 0 FROM mz_sources LIMIT 1"} ---- {"type":"Notice","payload":{"message":"{\n \"plans\": {\n \"raw\": {\n \"text\": \"Finish limit=1 output=[#0]\\n Project (#15)\\n Map ((1 / 0))\\n Get mz_catalog.mz_sources\\n\\nTarget cluster: mz_catalog_server\\n\",\n \"json\": {\n \"Project\": {\n \"input\": {\n \"Map\": {\n \"input\": {\n \"Get\": {\n \"id\": {\n \"Global\": {\n \n }\n },\n \"typ\": {\n \"column_types\": [\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"Oid\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": {\n \"Array\": \"MzAclItem\"\n },\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n }\n ],\n \"keys\": [\n [\n 0\n ],\n [\n 1\n ]\n ]\n }\n }\n },\n \"scalars\": [\n {\n \"CallBinary\": {\n \"func\": {\n \"DivInt32\": null\n },\n \"expr1\": {\n \"Literal\": [\n {\n \"data\": [\n 45,\n 1\n ]\n },\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n },\n null\n ]\n },\n \"expr2\": {\n \"Literal\": [\n {\n \"data\": [\n 44\n ]\n },\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n },\n null\n ]\n },\n \"name\": null\n }\n }\n ]\n }\n },\n \"outputs\": [\n 15\n ]\n }\n }\n },\n \"optimized\": {\n \"global\": {\n \"text\": \":\\n Finish limit=1 output=[#0]\\n ArrangeBy keys=[[#0]]\\n ReadGlobalFromSameDataflow \\n\\n:\\n Project (#15)\\n Map (error(\\\"division by zero\\\"))\\n ReadIndex on=mz_sources mz_sources_ind=[*** full scan ***]\\n\\nTarget cluster: mz_catalog_server\\n\",\n \"json\": {\n \"plans\": [\n {\n \"id\": \"\",\n \"plan\": {\n \"ArrangeBy\": {\n \"input\": {\n \"Get\": {\n \"id\": {\n \"Global\": {\n \n }\n },\n \"typ\": {\n \"column_types\": [\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n }\n ],\n \"keys\": []\n },\n \"access_strategy\": \"SameDataflow\"\n }\n },\n \"keys\": [\n [\n {\n \"Column\": [\n 0,\n null\n ]\n }\n ]\n ]\n }\n }\n },\n {\n \"id\": \"\",\n \"plan\": {\n \"Project\": {\n \"input\": {\n \"Map\": {\n \"input\": {\n \"Get\": {\n \"id\": {\n \"Global\": {\n \n }\n },\n \"typ\": {\n \"column_types\": [\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"UInt32\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": {\n \"Array\": \"MzAclItem\"\n },\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n }\n ],\n \"keys\": [\n [\n 0\n ],\n [\n 1\n ]\n ]\n },\n \"access_strategy\": {\n \"Index\": [\n [\n {\n \n },\n \"FullScan\"\n ]\n ]\n }\n }\n },\n \"scalars\": [\n {\n \"Literal\": [\n {\n \"Err\": \"DivisionByZero\"\n },\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n }\n ]\n }\n ]\n }\n },\n \"outputs\": [\n 15\n ]\n }\n }\n }\n ],\n \"sources\": []\n }\n },\n \"fast_path\": {\n \"text\": \"Explained Query (fast path):\\n Finish limit=1 output=[#0]\\n →Map/Filter/Project\\n Project: #15\\n Map: error(\\\"division by zero\\\")\\n →Indexed mz_catalog.mz_sources (using mz_catalog.mz_sources_ind)\\n\\nTarget cluster: mz_catalog_server\\n\",\n \"json\": {\n \"plans\": [\n {\n \"id\": \"Explained Query (fast path)\",\n \"plan\": {\n \"PeekExisting\": [\n {\n \n },\n {\n \n },\n null,\n {\n \"mfp\": {\n \"expressions\": [\n {\n \"Literal\": [\n {\n \"Err\": \"DivisionByZero\"\n },\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n }\n ]\n }\n ],\n \"predicates\": [],\n \"projection\": [\n 15\n ],\n \"input_arity\": 15\n }\n }\n ]\n }\n }\n ],\n \"sources\": []\n }\n }\n }\n },\n \"insights\": {\n \"imports\": {\n \"\": {\n \"name\": {\n \"schema\": \"mz_catalog\",\n \"item\": \"mz_sources_ind\"\n },\n \"type\": \"compute\"\n }\n },\n \"fast_path_clusters\": {},\n \"fast_path_limit\": null,\n \"persist_count\": []\n },\n \"cluster\": {\n \"name\": \"mz_catalog_server\",\n \"id\": {\n \n }\n },\n \"redacted_sql\": \"SELECT '' / '' FROM [ AS mz_catalog.mz_sources] LIMIT ''\"\n}","code":"MZ001","severity":"notice"}} -{"type":"CommandStarting","payload":{"has_rows":false,"is_streaming":false}} +{"type":"CommandStarting","payload":{"has_rows":true,"is_streaming":false}} {"type":"Error","payload":{"message":"division by zero","code":"XX000"}} {"type":"ReadyForQuery","payload":"I"} diff --git a/src/environmentd/tests/testdata/mcp/agent_query_tool b/src/environmentd/tests/testdata/mcp/agent_query_tool index 504f32a1b6b85..52342df008825 100644 --- a/src/environmentd/tests/testdata/mcp/agent_query_tool +++ b/src/environmentd/tests/testdata/mcp/agent_query_tool @@ -27,14 +27,14 @@ mcp-agent {"jsonrpc":"2.0","id":10,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"SELECT 1 AS val"}}} ---- 200 OK -{"jsonrpc":"2.0","id":10,"result":{"content":[{"type":"text","text":"[\n [\n \"1\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":10,"result":{"content":[{"type":"text","text":"[\n [\"1\"]\n]"}],"isError":false}} # SHOW statement should work. mcp-agent {"jsonrpc":"2.0","id":11,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"SHOW DATABASES"}}} ---- 200 OK -{"jsonrpc":"2.0","id":11,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\",\n \"\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":11,"result":{"content":[{"type":"text","text":"[\n [\"materialize\",\"\"]\n]"}],"isError":false}} # ============================================================================= # query tool - DDL rejected diff --git a/src/environmentd/tests/testdata/mcp/developer b/src/environmentd/tests/testdata/mcp/developer index dc9bc02b3a1bf..6bd5fcc707644 100644 --- a/src/environmentd/tests/testdata/mcp/developer +++ b/src/environmentd/tests/testdata/mcp/developer @@ -38,14 +38,14 @@ mcp-developer {"jsonrpc":"2.0","id":10,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT name FROM mz_databases WHERE name = 'materialize'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":10,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":10,"result":{"content":[{"type":"text","text":"[\n [\"materialize\"]\n]"}],"isError":false}} # Query with qualified schema should work. mcp-developer {"jsonrpc":"2.0","id":11,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT name FROM mz_catalog.mz_databases WHERE name = 'materialize'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":11,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":11,"result":{"content":[{"type":"text","text":"[\n [\"materialize\"]\n]"}],"isError":false}} # ============================================================================= # query_system_catalog - pg_catalog and information_schema @@ -56,14 +56,14 @@ mcp-developer {"jsonrpc":"2.0","id":12,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT typname FROM pg_catalog.pg_type WHERE typname = 'bool'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":12,"result":{"content":[{"type":"text","text":"[\n [\n \"bool\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":12,"result":{"content":[{"type":"text","text":"[\n [\"bool\"]\n]"}],"isError":false}} # information_schema should be accessible. mcp-developer {"jsonrpc":"2.0","id":13,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT table_schema FROM information_schema.tables WHERE table_name = 'mz_databases' AND table_schema = 'mz_catalog' LIMIT 1"}}} ---- 200 OK -{"jsonrpc":"2.0","id":13,"result":{"content":[{"type":"text","text":"[\n [\n \"mz_catalog\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":13,"result":{"content":[{"type":"text","text":"[\n [\"mz_catalog\"]\n]"}],"isError":false}} # ============================================================================= # query_system_catalog - non-system tables rejected diff --git a/src/environmentd/tests/testdata/mcp/developer_query_tool b/src/environmentd/tests/testdata/mcp/developer_query_tool index 724d56630c392..042c40e42bfcc 100644 --- a/src/environmentd/tests/testdata/mcp/developer_query_tool +++ b/src/environmentd/tests/testdata/mcp/developer_query_tool @@ -39,7 +39,7 @@ mcp-developer {"jsonrpc":"2.0","id":10,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"SELECT 1 AS val"}}} ---- 200 OK -{"jsonrpc":"2.0","id":10,"result":{"content":[{"type":"text","text":"[\n [\n \"1\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":10,"result":{"content":[{"type":"text","text":"[\n [\"1\"]\n]"}],"isError":false}} # SELECT against a system catalog table should work too: the developer query # tool is intentionally NOT restricted to mz_*. @@ -47,14 +47,14 @@ mcp-developer {"jsonrpc":"2.0","id":11,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"SELECT name FROM mz_databases WHERE name = 'materialize'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":11,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":11,"result":{"content":[{"type":"text","text":"[\n [\"materialize\"]\n]"}],"isError":false}} # SHOW statement should work. mcp-developer {"jsonrpc":"2.0","id":12,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"SHOW DATABASES"}}} ---- 200 OK -{"jsonrpc":"2.0","id":12,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\",\n \"\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":12,"result":{"content":[{"type":"text","text":"[\n [\"materialize\",\"\"]\n]"}],"isError":false}} # EXPLAIN should work — this is the motivating use case for the dev query tool # (EXPLAIN ANALYZE needs a cluster and so cannot use query_system_catalog). @@ -62,7 +62,7 @@ mcp-developer {"jsonrpc":"2.0","id":13,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"EXPLAIN SELECT 1"}}} ---- 200 OK -{"jsonrpc":"2.0","id":13,"result":{"content":[{"type":"text","text":"[\n [\n \"Explained Query (fast path):\\n →Constant (1 rows)\\n\\nTarget cluster: mz_catalog_server\\n\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":13,"result":{"content":[{"type":"text","text":"[\n [\"Explained Query (fast path):\\n →Constant (1 rows)\\n\\nTarget cluster: mz_catalog_server\\n\"]\n]"}],"isError":false}} # ============================================================================= # query - EXPLAIN ANALYZE @@ -92,14 +92,14 @@ mcp-developer {"jsonrpc":"2.0","id":14,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"EXPLAIN ANALYZE MEMORY FOR INDEX t_idx AS SQL"}}} ---- 200 OK -{"jsonrpc":"2.0","id":14,"result":{"content":[{"type":"text","text":"[\n [\n \"WITH\\n summary_memory AS\\n (\\n SELECT\\n mlm.global_id AS global_id,\\n mlm.lir_id AS lir_id,\\n sum(mas.size) AS total_memory,\\n sum(mas.records) AS total_records,\\n CASE\\n WHEN count(DISTINCT mas.worker_id) <> 0\\n THEN sum(mas.size) / count(DISTINCT mas.worker_id)\\n ELSE NULL\\n END\\n AS avg_memory,\\n CASE\\n WHEN count(DISTINCT mas.worker_id) <> 0\\n THEN\\n sum(mas.records) / count(DISTINCT mas.worker_id)\\n ELSE NULL\\n END\\n AS avg_records\\n FROM\\n mz_introspection.mz_lir_mapping AS mlm\\n CROSS JOIN generate_series((mlm.operator_id_start)::int8, (mlm.operator_id_end - 1)::int8) AS valid_id\\n JOIN\\n mz_introspection.mz_arrangement_sizes_per_worker AS mas\\n ON (mas.operator_id = valid_id)\\n GROUP BY mlm.global_id, mlm.lir_id\\n )\\nSELECT\\n repeat(' ', nesting * 2) || operator AS operator,\\n pg_size_pretty(sm.total_memory) AS total_memory,\\n sm.total_records AS total_records\\nFROM\\n mz_introspection.mz_lir_mapping AS mlm\\n LEFT JOIN summary_memory AS sm USING(global_id, lir_id)\\n JOIN\\n mz_introspection.mz_mappable_objects AS mo\\n ON (mlm.global_id = mo.global_id)\\nWHERE mo.name = 'materialize.public.t_idx'\\nORDER BY mlm.lir_id DESC;\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":14,"result":{"content":[{"type":"text","text":"[\n [\"WITH\\n summary_memory AS\\n (\\n SELECT\\n mlm.global_id AS global_id,\\n mlm.lir_id AS lir_id,\\n sum(mas.size) AS total_memory,\\n sum(mas.records) AS total_records,\\n CASE\\n WHEN count(DISTINCT mas.worker_id) <> 0\\n THEN sum(mas.size) / count(DISTINCT mas.worker_id)\\n ELSE NULL\\n END\\n AS avg_memory,\\n CASE\\n WHEN count(DISTINCT mas.worker_id) <> 0\\n THEN\\n sum(mas.records) / count(DISTINCT mas.worker_id)\\n ELSE NULL\\n END\\n AS avg_records\\n FROM\\n mz_introspection.mz_lir_mapping AS mlm\\n CROSS JOIN generate_series((mlm.operator_id_start)::int8, (mlm.operator_id_end - 1)::int8) AS valid_id\\n JOIN\\n mz_introspection.mz_arrangement_sizes_per_worker AS mas\\n ON (mas.operator_id = valid_id)\\n GROUP BY mlm.global_id, mlm.lir_id\\n )\\nSELECT\\n repeat(' ', nesting * 2) || operator AS operator,\\n pg_size_pretty(sm.total_memory) AS total_memory,\\n sm.total_records AS total_records\\nFROM\\n mz_introspection.mz_lir_mapping AS mlm\\n LEFT JOIN summary_memory AS sm USING(global_id, lir_id)\\n JOIN\\n mz_introspection.mz_mappable_objects AS mo\\n ON (mlm.global_id = mo.global_id)\\nWHERE mo.name = 'materialize.public.t_idx'\\nORDER BY mlm.lir_id DESC;\"]\n]"}],"isError":false}} # EXPLAIN ANALYZE CLUSTER (Statement::ExplainAnalyzeCluster). mcp-developer {"jsonrpc":"2.0","id":15,"method":"tools/call","params":{"name":"query","arguments":{"cluster":"quickstart","sql_query":"EXPLAIN ANALYZE CLUSTER MEMORY AS SQL"}}} ---- 200 OK -{"jsonrpc":"2.0","id":15,"result":{"content":[{"type":"text","text":"[\n [\n \"WITH\\n per_operator_memory_totals AS\\n (\\n SELECT\\n mlm.global_id AS global_id,\\n mlm.lir_id AS lir_id,\\n sum(mas.size) AS total_memory,\\n sum(mas.records) AS total_records\\n FROM\\n mz_introspection.mz_lir_mapping AS mlm\\n CROSS JOIN generate_series((mlm.operator_id_start)::int8, (mlm.operator_id_end - 1)::int8) AS valid_id\\n JOIN\\n mz_introspection.mz_arrangement_sizes_per_worker AS mas\\n ON (mas.operator_id = valid_id)\\n GROUP BY mlm.global_id, mlm.lir_id\\n ),\\n object_memory_totals AS\\n (\\n SELECT\\n pomt.global_id AS global_id,\\n sum(pomt.total_memory) AS total_memory,\\n sum(pomt.total_records) AS total_records\\n FROM per_operator_memory_totals AS pomt\\n GROUP BY pomt.global_id\\n )\\nSELECT\\n mo.name AS object,\\n mo.global_id AS global_id,\\n pg_size_pretty(omt.total_memory) AS total_memory,\\n omt.total_records AS total_records\\nFROM\\n mz_introspection.mz_mappable_objects AS mo\\n LEFT JOIN object_memory_totals AS omt USING(global_id)\\nORDER BY\\n omt.total_memory DESC NULLS LAST,\\n total_records DESC NULLS LAST,\\n mo.name DESC;\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":15,"result":{"content":[{"type":"text","text":"[\n [\"WITH\\n per_operator_memory_totals AS\\n (\\n SELECT\\n mlm.global_id AS global_id,\\n mlm.lir_id AS lir_id,\\n sum(mas.size) AS total_memory,\\n sum(mas.records) AS total_records\\n FROM\\n mz_introspection.mz_lir_mapping AS mlm\\n CROSS JOIN generate_series((mlm.operator_id_start)::int8, (mlm.operator_id_end - 1)::int8) AS valid_id\\n JOIN\\n mz_introspection.mz_arrangement_sizes_per_worker AS mas\\n ON (mas.operator_id = valid_id)\\n GROUP BY mlm.global_id, mlm.lir_id\\n ),\\n object_memory_totals AS\\n (\\n SELECT\\n pomt.global_id AS global_id,\\n sum(pomt.total_memory) AS total_memory,\\n sum(pomt.total_records) AS total_records\\n FROM per_operator_memory_totals AS pomt\\n GROUP BY pomt.global_id\\n )\\nSELECT\\n mo.name AS object,\\n mo.global_id AS global_id,\\n pg_size_pretty(omt.total_memory) AS total_memory,\\n omt.total_records AS total_records\\nFROM\\n mz_introspection.mz_mappable_objects AS mo\\n LEFT JOIN object_memory_totals AS omt USING(global_id)\\nORDER BY\\n omt.total_memory DESC NULLS LAST,\\n total_records DESC NULLS LAST,\\n mo.name DESC;\"]\n]"}],"isError":false}} # ============================================================================= # query - writes and DDL rejected @@ -182,7 +182,7 @@ mcp-developer {"jsonrpc":"2.0","id":50,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT name FROM mz_databases WHERE name = 'materialize'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":50,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":50,"result":{"content":[{"type":"text","text":"[\n [\"materialize\"]\n]"}],"isError":false}} # ============================================================================= # query - cluster_replica validation diff --git a/src/environmentd/tests/testdata/mcp/developer_response_limit b/src/environmentd/tests/testdata/mcp/developer_response_limit index 91b87971d29f3..ab0ecaa87d60f 100644 --- a/src/environmentd/tests/testdata/mcp/developer_response_limit +++ b/src/environmentd/tests/testdata/mcp/developer_response_limit @@ -17,7 +17,7 @@ mcp-developer {"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT name FROM mz_databases WHERE name = 'materialize'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"[\n [\n \"materialize\"\n ]\n]"}],"isError":false}} +{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"[\n [\"materialize\"]\n]"}],"isError":false}} # Responses over the limit are rejected with an error telling the agent to # narrow its query. repeat() gives a deterministic response size. @@ -25,4 +25,4 @@ mcp-developer {"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"query_system_catalog","arguments":{"sql_query":"SELECT repeat('x', 5000) FROM mz_catalog.mz_databases WHERE name = 'materialize'"}}} ---- 200 OK -{"jsonrpc":"2.0","id":2,"error":{"code":-32603,"message":"Query execution failed: Response size (5018 bytes) exceeds the 1024 byte limit. Use LIMIT or WHERE to narrow your query.","data":{"error_type":"ExecutionError"}}} +{"jsonrpc":"2.0","id":2,"error":{"code":-32603,"message":"Query execution failed: Response size (5010 bytes) exceeds the 1024 byte limit. Use LIMIT or WHERE to narrow your query.","data":{"error_type":"ExecutionError"}}} diff --git a/src/expr/src/scalar/func/impls/jsonb.rs b/src/expr/src/scalar/func/impls/jsonb.rs index ba36ef77374e2..287027f3eedca 100644 --- a/src/expr/src/scalar/func/impls/jsonb.rs +++ b/src/expr/src/scalar/func/impls/jsonb.rs @@ -458,7 +458,15 @@ fn parse_catalog_create_sql<'a>(a: &'a str) -> Result { "connection" } - CreateView(_) => "view", + CreateView(stmt) => { + let mut definition = stmt.definition.query.to_ast_string_stable(); + // PostgreSQL appends a semicolon in `pg_views.definition`, we + // do the same for compatibility's sake. + definition.push(';'); + info.insert("definition", json!(definition)); + + "view" + } CreateMaterializedView(stmt) => { let Some(in_cluster) = stmt.in_cluster else { return Err("missing IN CLUSTER".into()); @@ -475,7 +483,13 @@ fn parse_catalog_create_sql<'a>(a: &'a str) -> Result { "materialized-view" } - CreateTable(_) | CreateTableFromSource(_) => "table", + CreateTable(_) => "table", + CreateTableFromSource(stmt) => { + let source_id = get_item_id(stmt.source)?; + info.insert("source_id", json!(source_id)); + + "table" + } CreateSource(stmt) => { let Some(in_cluster) = stmt.in_cluster else { return Err("missing IN CLUSTER".into()); @@ -2071,6 +2085,246 @@ mod tests { fn sink_arm_leaves_other_item_types_alone() { let sql = "CREATE VIEW \"materialize\".\"public\".\"v\" AS SELECT 1"; let out = super::parse_catalog_create_sql(sql).expect("ok"); - assert_eq!(as_serde(out), json!({ "type": "view" })); + assert_eq!( + as_serde(out), + json!({ "type": "view", "definition": "SELECT 1;" }) + ); + } + + // --- parse_catalog_create_sql -------------------------------------------- + + /// `type` for a `create_sql`, or the error message if parsing failed. + fn item_type(sql: &str) -> Result { + match super::parse_catalog_create_sql(sql) { + Ok(out) => match as_serde(out) { + serde_json::Value::Object(mut m) => match m.remove("type") { + Some(serde_json::Value::String(s)) => Ok(s), + other => panic!("no string `type` key: {other:?}"), + }, + other => panic!("not a JSON object: {other:?}"), + }, + Err(EvalError::InvalidCatalogJson(msg)) => Err(msg.to_string()), + Err(e) => panic!("unexpected error variant: {e:?}"), + } + } + + fn view_sql(query: &str) -> String { + format!("CREATE VIEW \"materialize\".\"public\".\"v\" AS {query}") + } + + /// `definition` for a `CREATE VIEW` whose query is `query`. + fn view_definition(query: &str) -> String { + match as_serde(super::parse_catalog_create_sql(&view_sql(query)).expect("ok")) { + serde_json::Value::Object(mut m) => match m.remove("definition") { + Some(serde_json::Value::String(s)) => s, + other => panic!("no string `definition` key: {other:?}"), + }, + other => panic!("not a JSON object: {other:?}"), + } + } + + /// `mz_tables` and `mz_views` select rows by + /// `parse_catalog_create_sql(...)->>'type'`, and the function runs over + /// every `Item` row in the catalog, so a statement kind that changes its + /// reported type silently gains or loses rows in those relations. Pin the + /// type of every kind the catalog can hold. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` + fn catalog_item_type_per_statement_kind() { + let cases = [ + ( + "CREATE TABLE \"materialize\".\"public\".\"t\" (a int4)", + "table", + ), + // A table created from a source, and a webhook table, are both + // `table`, so both land in mz_tables. + ( + "CREATE TABLE \"materialize\".\"public\".\"tbl\" \ + FROM SOURCE [u1 AS \"materialize\".\"public\".\"src\"] \ + (REFERENCE = \"topic\") FORMAT TEXT", + "table", + ), + ( + "CREATE TABLE \"materialize\".\"public\".\"wht\" FROM WEBHOOK BODY FORMAT JSON", + "table", + ), + ( + "CREATE VIEW \"materialize\".\"public\".\"v\" AS SELECT 1", + "view", + ), + ( + "CREATE MATERIALIZED VIEW \"materialize\".\"public\".\"mv\" \ + IN CLUSTER [u1] AS SELECT 1", + "materialized-view", + ), + ( + "CREATE SOURCE \"materialize\".\"public\".\"lg\" \ + IN CLUSTER [u1] FROM LOAD GENERATOR COUNTER", + "source", + ), + ( + "CREATE SOURCE \"materialize\".\"public\".\"wh\" \ + IN CLUSTER [u1] FROM WEBHOOK BODY FORMAT JSON", + "source", + ), + ( + "CREATE SUBSOURCE \"materialize\".\"public\".\"sub\" (id int4) \ + OF SOURCE [u1 AS \"materialize\".\"public\".\"src\"]", + "subsource", + ), + ( + "CREATE SUBSOURCE \"materialize\".\"public\".\"progress\" (id int4) \ + WITH (PROGRESS)", + "subsource", + ), + ( + "CREATE SINK \"materialize\".\"public\".\"snk\" IN CLUSTER [u1] \ + FROM [u1 AS \"materialize\".\"public\".\"t\"] \ + INTO KAFKA CONNECTION [u2 AS \"materialize\".\"public\".\"c\"] \ + (TOPIC 'tp') FORMAT JSON ENVELOPE DEBEZIUM", + "sink", + ), + ( + "CREATE INDEX \"i\" IN CLUSTER [u1] \ + ON [u1 AS \"materialize\".\"public\".\"t\"] (\"a\")", + "index", + ), + ( + "CREATE TYPE \"materialize\".\"public\".\"ty\" AS LIST (ELEMENT TYPE = int4)", + "type", + ), + ( + "CREATE SECRET \"materialize\".\"public\".\"s\" AS 'x'", + "secret", + ), + ( + "CREATE CONNECTION \"materialize\".\"public\".\"c\" \ + TO KAFKA (BROKER 'b', SECURITY PROTOCOL PLAINTEXT)", + "connection", + ), + ]; + for (sql, expected) in cases { + assert_eq!(item_type(sql).as_deref(), Ok(expected), "for {sql}"); + } + } + + /// `mz_views.definition` is produced here. It used to be produced by + /// `pack_view_update` in the adapter, so the exact rendering is a + /// compatibility surface: `pg_views.definition` reads it. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` + fn catalog_view_definition() { + // Identifiers and function names come back fully quoted, literals + // untouched, and PostgreSQL's trailing semicolon is appended. + assert_eq!(view_definition("SELECT 1"), "SELECT 1;"); + assert_eq!( + view_definition("WITH c AS (SELECT 1 AS a) SELECT a FROM c"), + "WITH \"c\" AS (SELECT 1 AS \"a\") SELECT \"a\" FROM \"c\";" + ); + assert_eq!( + view_definition("SELECT 1 UNION ALL SELECT 2"), + "SELECT 1 UNION ALL SELECT 2;" + ); + assert_eq!( + view_definition("SELECT (SELECT max(a) FROM [u1 AS \"materialize\".\"public\".\"t\"])"), + "SELECT (SELECT \"max\"(\"a\") FROM [u1 AS \"materialize\".\"public\".\"t\"]);" + ); + // Identifiers needing quotes, an embedded double quote, non-ASCII, an + // embedded single quote in a literal, and ORDER BY all survive. + assert_eq!( + view_definition( + "SELECT \"a b\", \"héllo\", \"q\"\"x\" \ + FROM [u1 AS \"materialize\".\"public\".\"t\"] \ + WHERE s = 'lit''eral' AND n = 42 ORDER BY 1" + ), + "SELECT \"a b\", \"héllo\", \"q\"\"x\" \ + FROM [u1 AS \"materialize\".\"public\".\"t\"] \ + WHERE \"s\" = 'lit''eral' AND \"n\" = 42 ORDER BY 1;" + ); + } + + /// The rendering must be a fixed point: `pg_views` consumers re-issue + /// `definition` as the body of a new view, so a second pass through the + /// parser has to produce the identical string. The trailing `;` is part of + /// what gets re-parsed. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` + fn catalog_view_definition_is_idempotent() { + for query in [ + "SELECT 1", + "WITH c AS (SELECT 1 AS a) SELECT a FROM c", + "SELECT 1 UNION ALL SELECT 2", + "SELECT \"a b\", \"q\"\"x\" FROM [u1 AS \"materialize\".\"public\".\"t\"] \ + WHERE s = 'lit''eral' ORDER BY 1", + ] { + let once = view_definition(query); + assert_eq!( + view_definition(&once), + once, + "not a fixed point for {query}" + ); + } + } + + /// `mz_tables.source_id` comes from this key. A table with no source must + /// omit it entirely, so the MV's `->>'source_id'` yields SQL NULL. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` + fn catalog_table_source_id() { + let from_source = as_serde( + super::parse_catalog_create_sql( + "CREATE TABLE \"materialize\".\"public\".\"tbl\" \ + FROM SOURCE [u1 AS \"materialize\".\"public\".\"src\"] \ + (REFERENCE = \"topic\") FORMAT TEXT", + ) + .expect("ok"), + ); + assert_eq!(from_source, json!({ "type": "table", "source_id": "u1" })); + + for sql in [ + "CREATE TABLE \"materialize\".\"public\".\"t\" (a int4)", + "CREATE TABLE \"materialize\".\"public\".\"wht\" FROM WEBHOOK BODY FORMAT JSON", + ] { + assert_eq!( + as_serde(super::parse_catalog_create_sql(sql).expect("ok")), + json!({ "type": "table" }), + "for {sql}" + ); + } + } + + /// Every error here is fatal to the whole of `mz_tables`/`mz_views`, not to + /// one row: the MVs call this function inside their `WHERE` clause, so an + /// item the parser rejects makes the relation unreadable for everyone. + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` + fn catalog_create_sql_errors() { + assert_eq!( + item_type("this is not sql"), + Err( + "failed to parse create_sql: Expected a keyword at the beginning of a statement, \ + found identifier \"this\"" + .to_string() + ) + ); + assert_eq!( + item_type("CREATE TABLE t (a int4); CREATE TABLE u (b int4)"), + Err("expected a single statement, found 2".to_string()) + ); + // A statement that is not a CREATE of a catalog item, e.g. if a future + // change persists something else in an Item record. + assert_eq!( + item_type("SELECT 1"), + Err("not a CREATE item statement".to_string()) + ); + // Catalog `create_sql` always names items by id. An unresolved name + // means the record was written wrong. + assert_eq!( + item_type( + "CREATE TABLE \"materialize\".\"public\".\"tbl\" \ + FROM SOURCE src (REFERENCE = \"topic\")" + ), + Err("unresolved item name".to_string()) + ); } } diff --git a/src/pgrepr-consts/src/oid.rs b/src/pgrepr-consts/src/oid.rs index 7a4716922990f..e906f223f34bd 100644 --- a/src/pgrepr-consts/src/oid.rs +++ b/src/pgrepr-consts/src/oid.rs @@ -429,12 +429,12 @@ pub const MV_MZ_SCHEMAS_OID: u32 = 16703; pub const TABLE_MZ_COLUMNS_OID: u32 = 16704; pub const MV_MZ_INDEXES_OID: u32 = 16705; pub const TABLE_MZ_INDEX_COLUMNS_OID: u32 = 16706; -pub const TABLE_MZ_TABLES_OID: u32 = 16707; +pub const MV_MZ_TABLES_OID: u32 = 16707; pub const MV_MZ_CONNECTIONS_OID: u32 = 16708; pub const MV_MZ_SSH_TUNNEL_CONNECTIONS_OID: u32 = 16709; pub const MV_MZ_SOURCES_OID: u32 = 16710; pub const MV_MZ_SINKS_OID: u32 = 16711; -pub const TABLE_MZ_VIEWS_OID: u32 = 16712; +pub const MV_MZ_VIEWS_OID: u32 = 16712; pub const MV_MZ_MATERIALIZED_VIEWS_OID: u32 = 16713; pub const TABLE_MZ_TYPES_OID: u32 = 16714; pub const TABLE_MZ_TYPE_PG_METADATA_OID: u32 = 16715; @@ -822,3 +822,7 @@ pub const FUNC_PARSE_CONNECTION_DETAILS_OID: u32 = 17112; pub const FUNC_MZ_AWS_ACCOUNT_ID_OID: u32 = 17113; pub const FUNC_MZ_AWS_EXTERNAL_ID_PREFIX_OID: u32 = 17114; pub const FUNC_MZ_AWS_CONNECTION_ROLE_ARN_OID: u32 = 17115; +pub const VIEW_MZ_OBJECT_GRAPH_EDGES_OID: u32 = 17116; +pub const INDEX_MZ_OBJECT_GRAPH_EDGES_IND_OID: u32 = 17117; +pub const VIEW_MZ_BUILTIN_TABLES_OID: u32 = 17118; +pub const VIEW_MZ_BUILTIN_VIEWS_OID: u32 = 17119; diff --git a/src/pgwire/src/protocol.rs b/src/pgwire/src/protocol.rs index cfe140bfac742..d98e75820ae4c 100644 --- a/src/pgwire/src/protocol.rs +++ b/src/pgwire/src/protocol.rs @@ -66,7 +66,6 @@ use tokio::io::{self, AsyncRead, AsyncWrite}; use tokio::select; use tokio::time::{self}; use tokio_metrics::TaskMetrics; -use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{Instrument, debug, debug_span, info, warn}; use uuid::Uuid; @@ -2245,7 +2244,7 @@ where row_desc, portal_name, InProgressRows::new(RecordFirstRowStream::new( - Box::new(UnboundedReceiverStream::new(rx)), + rx, execute_started, &self.adapter_client, Some(instance_id), @@ -2302,7 +2301,7 @@ where format, row_desc, RecordFirstRowStream::new( - Box::new(UnboundedReceiverStream::new(rx)), + rx, execute_started, &self.adapter_client, Some(instance_id), diff --git a/src/sql/src/plan/error.rs b/src/sql/src/plan/error.rs index 4644b2e497a50..ec848436bf9dc 100644 --- a/src/sql/src/plan/error.rs +++ b/src/sql/src/plan/error.rs @@ -31,6 +31,7 @@ use mz_sql_parser::ast::display::AstDisplay; use mz_sql_parser::ast::{IdentError, UnresolvedItemName}; use mz_sql_parser::parser::{ParserError, ParserStatementError}; use mz_sql_server_util::SqlServerError; +use mz_storage_types::connections::InvalidAwsPrivatelinkServiceName; use mz_storage_types::sources::ExternalReferenceResolutionError; use crate::catalog::{ @@ -237,6 +238,7 @@ pub enum PlanError { name: String, supported_azs: BTreeSet, }, + InvalidPrivatelinkServiceName(InvalidAwsPrivatelinkServiceName), DuplicatePrivatelinkAvailabilityZone { duplicate_azs: BTreeSet, }, @@ -457,6 +459,7 @@ impl PlanError { let supported_azs_str = supported_azs.iter().join("\n "); Some(format!("Did you supply an availability zone name instead of an ID? Known availability zone IDs:\n {}", supported_azs_str)) } + Self::InvalidPrivatelinkServiceName(err) => Some(err.hint()), Self::DuplicatePrivatelinkAvailabilityZone { duplicate_azs, ..} => { let duplicate_azs = duplicate_azs.iter().join("\n "); Some(format!("Duplicated availability zones:\n {}", duplicate_azs)) @@ -762,6 +765,7 @@ impl fmt::Display for PlanError { }) }, Self::InvalidPrivatelinkAvailabilityZone { name, ..} => write!(f, "invalid AWS PrivateLink availability zone {}", name.quoted()), + Self::InvalidPrivatelinkServiceName(err) => err.fmt(f), Self::DuplicatePrivatelinkAvailabilityZone {..} => write!(f, "connection cannot contain duplicate availability zones"), Self::InvalidSchemaName => write!(f, "no valid schema selected"), Self::ItemAlreadyExists { name, item_type } => write!(f, "{item_type} {} already exists", name.quoted()), diff --git a/src/storage-types/src/connections.rs b/src/storage-types/src/connections.rs index 7f76c8ff2b5f4..b5977fcaba52b 100644 --- a/src/storage-types/src/connections.rs +++ b/src/storage-types/src/connections.rs @@ -11,6 +11,7 @@ use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; use std::net::SocketAddr; use std::sync::Arc; use std::time::SystemTime; @@ -50,6 +51,7 @@ use mz_ore::error::ErrorExt; use mz_ore::future::{InTask, OreFutureExt}; use mz_ore::netio::resolve_address; use mz_ore::num::NonNeg; +use mz_ore::str::StrExt; use mz_repr::{CatalogItemId, GlobalId}; use mz_secrets::SecretsReader; use mz_sql_parser::ast::ConnectionRulePattern; @@ -543,6 +545,8 @@ pub enum ConnectionValidationError { Aws(#[from] AwsConnectionValidationError), #[error(transparent)] Gcp(#[from] gcp::GcpConnectionValidationError), + #[error(transparent)] + AwsPrivatelinkServiceName(#[from] InvalidAwsPrivatelinkServiceName), #[error("{}", .0.display_with_causes())] Other(#[from] anyhow::Error), } @@ -556,6 +560,7 @@ impl ConnectionValidationError { ConnectionValidationError::SqlServer(e) => e.detail(), ConnectionValidationError::Aws(e) => e.detail(), ConnectionValidationError::Gcp(e) => e.detail(), + ConnectionValidationError::AwsPrivatelinkServiceName(_) => None, ConnectionValidationError::Other(_) => None, } } @@ -568,6 +573,7 @@ impl ConnectionValidationError { ConnectionValidationError::SqlServer(e) => e.hint(), ConnectionValidationError::Aws(e) => e.hint(), ConnectionValidationError::Gcp(e) => e.hint(), + ConnectionValidationError::AwsPrivatelinkServiceName(e) => Some(e.hint()), ConnectionValidationError::Other(_) => None, } } @@ -1014,6 +1020,51 @@ impl AlterCompatible for AwsPrivatelinkConnection { } } +/// A `SERVICE NAME` that cannot name an AWS VPC endpoint service. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct InvalidAwsPrivatelinkServiceName { + pub name: String, +} + +impl fmt::Display for InvalidAwsPrivatelinkServiceName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "invalid AWS PrivateLink service name {}", + self.name.quoted() + ) + } +} + +impl std::error::Error for InvalidAwsPrivatelinkServiceName {} + +impl InvalidAwsPrivatelinkServiceName { + /// Explains how to find the right value. + pub fn hint(&self) -> String { + "SERVICE NAME must name an AWS VPC endpoint service, for example \ + `com.amazonaws.vpce.us-east-1.vpce-svc-0e123abc123198abc`. Endpoint service names are \ + listed in the AWS console under VPC > Endpoint services." + .into() + } +} + +impl AwsPrivatelinkConnection { + /// Checks that `service_name` could name an AWS VPC endpoint service. + /// + /// Every endpoint service name starts with `com.amazonaws.`, whether the + /// service is customer-owned (`com.amazonaws.vpce..vpce-svc-`) + /// or AWS-managed (`com.amazonaws..`). Only the prefix is + /// checked, so a name AWS would accept is never rejected here. + pub fn check_service_name(service_name: &str) -> Result<(), InvalidAwsPrivatelinkServiceName> { + if service_name.starts_with("com.amazonaws.") { + return Ok(()); + } + Err(InvalidAwsPrivatelinkServiceName { + name: service_name.to_string(), + }) + } +} + #[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] pub struct KafkaTlsConfig { pub identity: Option, @@ -3213,12 +3264,16 @@ impl AwsPrivatelinkConnection { &self, id: CatalogItemId, storage_configuration: &StorageConfiguration, - ) -> Result<(), anyhow::Error> { + ) -> Result<(), ConnectionValidationError> { + // An endpoint for an unusable service name reports a misleading + // condition (missing availability zones), so check the name first. + Self::check_service_name(&self.service_name)?; + let Some(ref cloud_resource_reader) = storage_configuration .connection_context .cloud_resource_reader else { - return Err(anyhow!("AWS PrivateLink connections are unsupported")); + return Err(anyhow!("AWS PrivateLink connections are unsupported").into()); }; // No need to optionally run this in a task, as we are just validating from envd. @@ -3231,8 +3286,8 @@ impl AwsPrivatelinkConnection { match availability { Some(condition) if condition.status == "True" => Ok(()), - Some(condition) => Err(anyhow!("{}", condition.message)), - None => Err(anyhow!("Endpoint availability is unknown")), + Some(condition) => Err(anyhow!("{}", condition.message).into()), + None => Err(anyhow!("Endpoint availability is unknown").into()), } } @@ -3240,3 +3295,38 @@ impl AwsPrivatelinkConnection { false } } + +#[cfg(test)] +mod tests { + use super::*; + + #[mz_ore::test] + fn test_check_service_name() { + // Customer-owned and AWS-managed endpoint services are both accepted, + // as is anything else that could be an endpoint service name. + for name in [ + "com.amazonaws.vpce.us-east-1.vpce-svc-0e123abc123198abc", + "com.amazonaws.vpce.test.vpce-svc-e2e-test", + "com.amazonaws.us-east-1.s3", + "com.amazonaws.anything", + ] { + assert_eq!( + AwsPrivatelinkConnection::check_service_name(name), + Ok(()), + "expected {name} to be accepted" + ); + } + + for name in [ + "", + "com.amazonaws", + "vpce-svc-0e123abc123198abc", + "my-db-lb-0123456789abcdef.elb.eu-central-1.amazonaws.com", + "db.internal.example.org", + ] { + let err = AwsPrivatelinkConnection::check_service_name(name) + .expect_err("service name should be rejected"); + assert_eq!(err.name, name); + } + } +} diff --git a/test/0dt/mzcompose.py b/test/0dt/mzcompose.py index 2a010b4a085f2..84b520e0cf296 100644 --- a/test/0dt/mzcompose.py +++ b/test/0dt/mzcompose.py @@ -1787,7 +1787,7 @@ def get_persist_shard_id(item_id: str, service: str) -> str: ) mz_tables_gid = c.sql_query( - "SELECT id FROM mz_tables WHERE name = 'mz_tables'", + "SELECT id FROM mz_materialized_views WHERE name = 'mz_tables'", service="mz_old", )[0][0] mv_gid = c.sql_query( @@ -1817,7 +1817,7 @@ def get_persist_shard_id(item_id: str, service: str) -> str: c.await_mz_deployment_status(DeploymentStatus.IS_LEADER, "mz_new") new_mz_tables_gid = c.sql_query( - "SELECT id FROM mz_tables WHERE name = 'mz_tables'", + "SELECT id FROM mz_materialized_views WHERE name = 'mz_tables'", service="mz_new", reuse_connection=False, )[0][0] @@ -1870,7 +1870,7 @@ def get_persist_shard_id(item_id: str, service: str) -> str: ) mz_tables_gid = c.sql_query( - "SELECT id FROM mz_tables WHERE name = 'mz_tables'", + "SELECT id FROM mz_materialized_views WHERE name = 'mz_tables'", service="mz_old", )[0][0] mv_gid = c.sql_query( @@ -1901,7 +1901,7 @@ def get_persist_shard_id(item_id: str, service: str) -> str: c.await_mz_deployment_status(DeploymentStatus.IS_LEADER, "mz_new") new_mz_tables_gid = c.sql_query( - "SELECT id FROM mz_tables WHERE name = 'mz_tables'", + "SELECT id FROM mz_materialized_views WHERE name = 'mz_tables'", service="mz_new", reuse_connection=False, )[0][0] diff --git a/test/cluster-spec-sheet/mzcompose.py b/test/cluster-spec-sheet/mzcompose.py index c02e246a7af7b..500aa80f1621c 100644 --- a/test/cluster-spec-sheet/mzcompose.py +++ b/test/cluster-spec-sheet/mzcompose.py @@ -2111,6 +2111,9 @@ def inner() -> None: class SourceIngestionScenario(ClusterScalingScenario): + # 1.1.0: MySQL RDS instance type changed + VERSION: str = "1.1.0" + def name(self) -> str: return "source_ingestion" diff --git a/test/cluster/mzcompose.py b/test/cluster/mzcompose.py index c879958dd3a3a..6842f11696900 100644 --- a/test/cluster/mzcompose.py +++ b/test/cluster/mzcompose.py @@ -4268,7 +4268,7 @@ def workflow_test_subscribe_hydration_status( FROM mz_internal.mz_subscriptions s, unnest(s.referenced_object_ids) as sroi(id) JOIN mz_introspection.mz_compute_hydration_times_per_worker h ON h.export_id = s.id - JOIN mz_tables t ON (t.id = sroi.id) + JOIN mz_materialized_views t ON (t.id = sroi.id) WHERE t.name = 'mz_tables' true """)) @@ -4283,7 +4283,7 @@ def workflow_test_subscribe_hydration_status( FROM mz_internal.mz_subscriptions s, unnest(s.referenced_object_ids) as sroi(id) JOIN mz_introspection.mz_compute_hydration_times_per_worker h ON h.export_id = s.id - JOIN mz_tables t ON (t.id = sroi.id) + JOIN mz_materialized_views t ON (t.id = sroi.id) WHERE t.name = 'mz_tables' """)) @@ -6003,9 +6003,9 @@ def workflow_test_adhoc_system_indexes( WHERE i.name = 'mz_test_idx1' """) assert output[0] == ("u1", "mz_tables", "mz_catalog_server"), output - output = c.sql_query("EXPLAIN SELECT * FROM mz_tables WHERE char_length(name) = 9") + output = c.sql_query("EXPLAIN SELECT * FROM mz_tables WHERE char_length(name) = 8") assert "mz_test_idx1" in output[0][0], output - output = c.sql_query("SELECT * FROM mz_tables WHERE char_length(name) = 9") + output = c.sql_query("SELECT * FROM mz_tables WHERE char_length(name) = 8") assert len(output) > 0 # The system user should be able to create a new index on an unstable diff --git a/test/dataflow-visualizer/mzcompose.py b/test/dataflow-visualizer/mzcompose.py index 1f3edc78a2184..e19475f6bc8ce 100644 --- a/test/dataflow-visualizer/mzcompose.py +++ b/test/dataflow-visualizer/mzcompose.py @@ -12,12 +12,41 @@ Tests the /memory, /hierarchical-memory, and /metrics-viz endpoints on port 6878. """ +import time + from materialize import MZ_ROOT from materialize.buildkite import is_in_buildkite, upload_artifact from materialize.mzcompose.composition import Composition from materialize.mzcompose.service import Service from materialize.mzcompose.services.materialized import Materialized +# The cluster replica the tests visualize. `quickstart` holds the fixture index +# below and little else, which keeps the introspection queries behind the +# visualizer an order of magnitude cheaper than on `mz_catalog_server`, where +# they scan every builtin dataflow. +FIXTURE_CLUSTER = "quickstart" +FIXTURE_REPLICA = "r1" + +# The index whose dataflow the tests expand. Without it they would have to pick +# whatever sits at the top of the dataflow table, and that is a moving target: +# the table is ordered by record count descending, a dataflow that arranges +# nothing reports NULL rather than zero there, and NULLs sort first. Those are +# the transient `introspection-subscribe-*` dataflows and the storage +# `command_sequencer`. A transient one can be gone by the time the page queries +# its operators, and the page then reports an unknown dataflow id. +# +# The index key is load-bearing as well. An arrangement embeds the debug +# formatting of its key in its operator name, so indexing a named column yields +# `ArrangeBy[[Column(0, "id")]]`. Those double quotes have to survive into a DOT +# label, which is what the quoted-name test asserts. +# +# So is indexing a view rather than the table directly: the /memory page shows +# the SQL that created a dataflow by resolving the index back to a view, and +# that lookup finds nothing for an index on a table. +FIXTURE_TABLE = "visualizer_fixture" +FIXTURE_VIEW = "visualizer_fixture_view" +FIXTURE_INDEX = "visualizer_fixture_idx" + SERVICES = [ Materialized(), Service( @@ -29,6 +58,10 @@ ], "environment": [ "MZ_HOST=materialized", + f"FIXTURE_CLUSTER={FIXTURE_CLUSTER}", + f"FIXTURE_REPLICA={FIXTURE_REPLICA}", + f"FIXTURE_VIEW={FIXTURE_VIEW}", + f"FIXTURE_INDEX={FIXTURE_INDEX}", ], }, ), @@ -38,6 +71,7 @@ def workflow_default(c: Composition) -> None: """Run dataflow visualizer E2E tests""" c.up("materialized") + create_fixture_dataflow(c) try: c.run("playwright", "/workdir/run-tests.sh") finally: @@ -47,3 +81,47 @@ def workflow_default(c: Composition) -> None: ) if traces_path.exists() and is_in_buildkite(): upload_artifact(traces_path) + + +def create_fixture_dataflow(c: Composition) -> None: + """Install the index the browser tests visualize, and wait for its dataflow. + + The DDL runs over pgwire as `materialize`. The visualizer's own `/api/sql` + endpoint on the internal HTTP port runs as `anonymous_http_user`, which has + no CREATE privilege, so the tests cannot set this up themselves. + + Idempotent, so that re-running the workflow against a composition that is + already up does not have to start from `down -v`. + """ + c.sql(f""" + CREATE TABLE IF NOT EXISTS {FIXTURE_TABLE} (id int, other int); + CREATE VIEW IF NOT EXISTS {FIXTURE_VIEW} AS + SELECT id, other FROM {FIXTURE_TABLE}; + CREATE INDEX IF NOT EXISTS {FIXTURE_INDEX} + IN CLUSTER {FIXTURE_CLUSTER} ON {FIXTURE_VIEW} (id); + INSERT INTO {FIXTURE_TABLE} VALUES (1, 1); + """) + + # The introspection sources report the new dataflow only on their next tick, + # so without this the tests can open a page whose table does not list it yet. + deadline = time.time() + 60 + with c.sql_connection( + startup_params={"cluster": FIXTURE_CLUSTER, "cluster_replica": FIXTURE_REPLICA} + ) as conn: + cursor = conn.cursor() + while True: + cursor.execute( + b"SELECT count(*) FROM mz_introspection.mz_records_per_dataflow " + b"WHERE name LIKE %s", + (f"Dataflow: %.{FIXTURE_INDEX}",), + ) + row = cursor.fetchone() + assert row is not None + if row[0] > 0: + return + if time.time() > deadline: + raise AssertionError( + f"index {FIXTURE_INDEX} did not show up in " + f"mz_records_per_dataflow on {FIXTURE_CLUSTER}.{FIXTURE_REPLICA}" + ) + time.sleep(1) diff --git a/test/dataflow-visualizer/package.json b/test/dataflow-visualizer/package.json index 1de766f156d0e..e7fb2e2902897 100644 --- a/test/dataflow-visualizer/package.json +++ b/test/dataflow-visualizer/package.json @@ -10,6 +10,7 @@ "devDependencies": { "@playwright/test": "1.62.1", "@types/d3": "7.4.3", + "@types/node": "26.2.0", "@types/react": "16.14.69", "@types/react-dom": "16.9.25", "typescript": "7.0.2" diff --git a/test/dataflow-visualizer/playwright.config.ts b/test/dataflow-visualizer/playwright.config.ts index 32740ad9ec589..5e0290792171c 100644 --- a/test/dataflow-visualizer/playwright.config.ts +++ b/test/dataflow-visualizer/playwright.config.ts @@ -11,7 +11,12 @@ import { defineConfig } from '@playwright/test'; export default defineConfig({ testDir: './tests', - fullyParallel: false, // Run serially since we share a single Materialize instance + // Every test shares one Materialize instance, and the introspection queries + // behind the visualizer are unindexed scans. Two workers on a CI agent turn + // those into multi-second waits that trip the assertion timeouts below, so + // pin the suite to a single worker rather than only serializing each file. + fullyParallel: false, + workers: 1, retries: 1, reporter: 'list', use: { @@ -22,5 +27,16 @@ export default defineConfig({ : 'http://localhost:6878', trace: 'on-first-retry', }, - timeout: 30000, + expect: { + // The visualizer pages load in stages, each rendering its own `Loading...` + // while its introspection query is in flight. Those queries are unindexed + // scans, so a busy CI agent can take seconds over each one. Set the budget + // once here rather than per assertion. + timeout: 30000, + }, + // A test can wait out two page loads in a row (the dataflow list, then one + // dataflow's operators), so this has to exceed the sum of the assertion + // timeouts a single test can accumulate. Otherwise a slow page reports as an + // opaque test timeout instead of naming the element it was waiting for. + timeout: 90000, }); diff --git a/test/dataflow-visualizer/tests/fixture.ts b/test/dataflow-visualizer/tests/fixture.ts new file mode 100644 index 0000000000000..cf746fb3ed022 --- /dev/null +++ b/test/dataflow-visualizer/tests/fixture.ts @@ -0,0 +1,52 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file at the root of this repository. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +// The dataflow the browser tests visualize. `mzcompose.py` creates the index +// and waits for its dataflow to reach introspection before Playwright starts, +// and passes these names in the environment so both ends agree on one +// definition. See the comments there for why the tests pin a dataflow instead +// of expanding whichever row the page happens to list first. +// +// The fallbacks keep `npx playwright test` usable against a Materialize started +// by hand, as long as the same fixture exists there. +// +// The fixture lives on `quickstart`, the only user cluster in the composition, +// which is also the one an unpinned page has to select. That is not a +// coincidence: the visualizer exists to show you your own dataflows. + +export const FIXTURE_CLUSTER = process.env.FIXTURE_CLUSTER ?? 'quickstart'; +export const FIXTURE_REPLICA = process.env.FIXTURE_REPLICA ?? 'r1'; +export const FIXTURE_VIEW = + process.env.FIXTURE_VIEW ?? 'visualizer_fixture_view'; +export const FIXTURE_INDEX = + process.env.FIXTURE_INDEX ?? 'visualizer_fixture_idx'; + +// How the fixture's dataflow is named in the visualizer's dataflow table. +export const FIXTURE_DATAFLOW = `Dataflow: materialize.public.${FIXTURE_INDEX}`; + +/** A page URL that pins the cluster replica the page should show. */ +export function replicaPage( + path: string, + cluster: string, + replica: string +): string { + const params = new URLSearchParams({ + cluster_name: cluster, + replica_name: replica, + }); + return `${path}?${params}`; +} + +/** + * A page URL pinned to the fixture's replica, so a test never depends on which + * replica the page picks by default. + */ +export function fixturePage(path: string): string { + return replicaPage(path, FIXTURE_CLUSTER, FIXTURE_REPLICA); +} diff --git a/test/dataflow-visualizer/tests/hierarchical-memory.spec.ts b/test/dataflow-visualizer/tests/hierarchical-memory.spec.ts index 9a34ef7325af2..1607f1c988ba1 100644 --- a/test/dataflow-visualizer/tests/hierarchical-memory.spec.ts +++ b/test/dataflow-visualizer/tests/hierarchical-memory.spec.ts @@ -7,7 +7,22 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { test, expect } from '@playwright/test'; +import { test, expect, Page } from '@playwright/test'; +import { FIXTURE_INDEX, fixturePage } from './fixture'; + +/** + * Wait for the page to render its dataflow graphs. + * + * Scope graphs live inside collapsed `.content` divs, so assert on presence + * rather than visibility. The graphs are the page's actual output: the dropdown + * appearing only means the replica list came back, and asserting on that alone + * passes while the rest of the page still says `Loading...`. + */ +async function expectGraphs(page: Page) { + await expect + .poll(() => page.locator('#content2 svg').count()) + .toBeGreaterThan(0); +} test.describe('/hierarchical-memory page', () => { test('page loads without crashing', async ({ page }) => { @@ -28,9 +43,8 @@ test.describe('/hierarchical-memory page', () => { await page.goto('/hierarchical-memory'); // The dropdown should appear after initial data loads - // Use a longer timeout as the page queries the database const dropdown = page.locator('#cluster_replica'); - await expect(dropdown).toBeVisible({ timeout: 20000 }); + await expect(dropdown).toBeVisible(); // Should have at least one option (check count, not visibility since options are inside select) const options = dropdown.locator('option'); @@ -46,24 +60,20 @@ test.describe('/hierarchical-memory page', () => { await page.goto('/hierarchical-memory'); - // Wait for dropdown to appear (indicates initial load complete) - const dropdown = page.locator('#cluster_replica'); - await expect(dropdown).toBeVisible({ timeout: 20000 }); - - // The page queries data and renders - give it time - await page.waitForTimeout(3000); - - await expect(dropdown).toBeVisible(); + await expectGraphs(page); + await expect(page.locator('#cluster_replica')).toBeVisible(); await expect(page.locator('#content2')).not.toContainText('error:'); expect(pageErrors.map(String)).toEqual([]); }); - test('URL updates with cluster parameters after selection', async ({ page }) => { + test('URL updates with cluster parameters after selection', async ({ + page, + }) => { await page.goto('/hierarchical-memory'); // Wait for dropdown const dropdown = page.locator('#cluster_replica'); - await expect(dropdown).toBeVisible({ timeout: 20000 }); + await expect(dropdown).toBeVisible(); // URL should have been updated with cluster params await expect(page).toHaveURL(/cluster_name=/); @@ -72,45 +82,23 @@ test.describe('/hierarchical-memory page', () => { test('renders graphs for operator names containing double quotes', async ({ page, - request, }) => { // An arrangement operator embeds the debug formatting of its key in its - // name, so an index on a named column is called something like + // name, so the fixture index on a named column is called something like // `ArrangeBy[[Column(0, "id")]]`. Those double quotes have to be escaped // before they reach a DOT label, otherwise they terminate the label early // and GraphViz rejects the whole graph. - for (const query of [ - 'CREATE TABLE IF NOT EXISTS quoted_name_regression (id int, other int)', - 'CREATE INDEX IF NOT EXISTS quoted_name_regression_idx ON quoted_name_regression (id)', - ]) { - const response = await request.post('/api/sql', { data: { query } }); - // /api/sql reports SQL failures in the body, not the status code. - const body = response.ok() ? await response.json() : null; - const failure = !body - ? `HTTP ${response.status()}` - : body.results?.find((result) => result.error)?.error?.message; - // The endpoint runs as an unprivileged role, so DDL may be refused - // depending on how the environment is configured. Skip visibly rather - // than reporting a visualizer bug that isn't one. - test.skip(!!failure, `could not set up test index: ${failure}`); - } - const pageErrors: Error[] = []; page.on('pageerror', (error) => pageErrors.push(error)); - await page.goto( - '/hierarchical-memory?cluster_name=quickstart&replica_name=r1' - ); - - const content = page.locator('#content2'); + await page.goto(fixturePage('/hierarchical-memory')); - // Scope graphs live inside collapsed `.content` divs, so assert on - // presence rather than visibility. - await expect - .poll(() => content.locator('svg').count(), { timeout: 20000 }) - .toBeGreaterThan(0); + await expectGraphs(page); - // The arrangement label rendered, quotes and all. + // The fixture's dataflow rendered, and with it the arrangement label, + // quotes and all. + const content = page.locator('#content2'); + await expect(content).toContainText(FIXTURE_INDEX); await expect(content).toContainText('ArrangeBy'); await expect(content).not.toContainText('error:'); expect(pageErrors.map(String)).toEqual([]); @@ -120,7 +108,7 @@ test.describe('/hierarchical-memory page', () => { await page.goto('/hierarchical-memory'); const dropdown = page.locator('#cluster_replica'); - await expect(dropdown).toBeVisible({ timeout: 20000 }); + await expect(dropdown).toBeVisible(); const options = dropdown.locator('option'); const optionCount = await options.count(); @@ -131,18 +119,18 @@ test.describe('/hierarchical-memory page', () => { return; } - // Get initial URL - const initialUrl = page.url(); + // Pick an option other than the one the page settled on, so the URL has to + // change rather than being rewritten to the same value. + const selected = await dropdown.inputValue(); + const values = await options.evaluateAll((els) => + els.map((el) => (el as HTMLOptionElement).value) + ); + const newValue = values.find((value) => value !== selected)!; + const [clusterName, replicaName] = JSON.parse(newValue); - // Select a different option - const secondOption = options.nth(1); - const newValue = await secondOption.getAttribute('value'); - if (newValue) { - await dropdown.selectOption(newValue); + await dropdown.selectOption(newValue); - // URL should update - await page.waitForTimeout(1000); - expect(page.url()).not.toBe(initialUrl); - } + await expect(page).toHaveURL(new RegExp(`cluster_name=${clusterName}`)); + await expect(page).toHaveURL(new RegExp(`replica_name=${replicaName}`)); }); }); diff --git a/test/dataflow-visualizer/tests/memory.spec.ts b/test/dataflow-visualizer/tests/memory.spec.ts index f39e2aa57c99d..c268a4adc3e3a 100644 --- a/test/dataflow-visualizer/tests/memory.spec.ts +++ b/test/dataflow-visualizer/tests/memory.spec.ts @@ -7,7 +7,43 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { test, expect } from '@playwright/test'; +import { test, expect, Page } from '@playwright/test'; +import { + FIXTURE_CLUSTER, + FIXTURE_DATAFLOW, + FIXTURE_REPLICA, + FIXTURE_VIEW, + fixturePage, +} from './fixture'; + +/** Open a /memory page and wait for its dataflow table to render. */ +async function openMemoryPage(page: Page, url: string) { + await page.goto(url); + await expect(page.locator('text=Loading...')).toBeHidden(); +} + +/** + * Expand the fixture dataflow. + * + * Addressing the row by name matters: the table is ordered by record count + * descending, a dataflow that arranges nothing reports NULL rather than zero + * there, and NULLs sort first. Those are the transient + * `introspection-subscribe-*` dataflows and the storage `command_sequencer`. + * Expanding a transient one races its teardown, and the page then reports an + * unknown dataflow id instead of a graph. + */ +async function expandFixtureDataflow(page: Page) { + const row = page + .locator('table.dataflows tbody tr') + .filter({ hasText: FIXTURE_DATAFLOW }); + await expect(row).toBeVisible(); + await row.locator('button').click(); +} + +/** The panel one expanded dataflow renders into. */ +function vizSection(page: Page) { + return page.locator('div:has(> h3)').filter({ hasText: 'Name:' }); +} test.describe('/memory page', () => { test('page loads without errors', async ({ page }) => { @@ -18,23 +54,15 @@ test.describe('/memory page', () => { } }); - await page.goto('/memory'); + await openMemoryPage(page, '/memory'); - // Wait for loading to complete - await expect(page.locator('text=Loading...')).toBeHidden({ timeout: 10000 }); - - // Check for no console errors (excluding expected debug messages) - const unexpectedErrors = consoleErrors.filter( - (err) => !err.includes('could not get create view') - ); - expect(unexpectedErrors).toHaveLength(0); + expect(consoleErrors).toHaveLength(0); }); test('cluster replica dropdown is populated', async ({ page }) => { - await page.goto('/memory'); - - // Wait for loading to complete - await expect(page.locator('text=Loading...')).toBeHidden({ timeout: 10000 }); + // Deliberately not pinned to a replica: this covers the page picking one + // for itself when the URL names none. + await openMemoryPage(page, '/memory'); // Check that the dropdown exists and has options const dropdown = page.locator('#cluster_replica'); @@ -45,11 +73,22 @@ test.describe('/memory page', () => { expect(optionCount).toBeGreaterThan(0); }); - test('dataflow table renders', async ({ page }) => { - await page.goto('/memory'); + test('an unpinned page selects a user cluster', async ({ page }) => { + // Replicas are listed in cluster then replica name order, so a page that + // cannot identify a user cluster silently settles on the first of them, + // which is a builtin one. Those dataflows are not what a user opening this + // page came to look at, and their introspection relations are an order of + // magnitude more expensive to query. + await openMemoryPage(page, '/memory'); - // Wait for loading to complete - await expect(page.locator('text=Loading...')).toBeHidden({ timeout: 10000 }); + await expect(page.locator('#cluster_replica')).toHaveValue( + JSON.stringify([FIXTURE_CLUSTER, FIXTURE_REPLICA]) + ); + await expect(page).toHaveURL(new RegExp(`cluster_name=${FIXTURE_CLUSTER}`)); + }); + + test('dataflow table renders', async ({ page }) => { + await openMemoryPage(page, fixturePage('/memory')); // Check that the table exists with expected headers const table = page.locator('table.dataflows'); @@ -59,50 +98,44 @@ test.describe('/memory page', () => { await expect(table.locator('th:has-text("dataflow id")')).toBeVisible(); await expect(table.locator('th:has-text("index name")')).toBeVisible(); await expect(table.locator('th:has-text("records")')).toBeVisible(); - }); - test('clicking dataflow expand button shows visualization', async ({ page }) => { - await page.goto('/memory'); - - // Wait for loading to complete - await expect(page.locator('text=Loading...')).toBeHidden({ timeout: 10000 }); - - // Find the first expand button in the table - const expandButton = page.locator('table.dataflows tbody button').first(); + // The fixture index is what the tests below expand. + await expect( + table.locator('tbody tr').filter({ hasText: FIXTURE_DATAFLOW }) + ).toBeVisible(); + }); - // Skip test if no dataflows are present - if ((await expandButton.count()) === 0) { - test.skip(); - return; - } + test('clicking dataflow expand button shows visualization', async ({ + page, + }) => { + await openMemoryPage(page, fixturePage('/memory')); + await expandFixtureDataflow(page); - await expandButton.click(); + const viz = vizSection(page); + await expect(viz).toBeVisible(); + await expect(viz).toContainText(FIXTURE_DATAFLOW); + }); - // Wait for the visualization section to appear - const vizSection = page.locator('div:has(> h3)').filter({ hasText: 'Name:' }); - await expect(vizSection).toBeVisible({ timeout: 15000 }); + test('expanded dataflow shows the SQL that created it', async ({ page }) => { + // The page resolves a dataflow back to the view its index was built on and + // renders that view's SHOW CREATE VIEW. It is the only part of the page + // that reaches outside `mz_introspection`, so nothing else catches it + // breaking. + await openMemoryPage(page, fixturePage('/memory')); + await expandFixtureDataflow(page); + + const viz = vizSection(page); + await expect(viz).toContainText(`View: materialize.public.${FIXTURE_VIEW}`); + await expect(viz).toContainText('CREATE VIEW'); }); test('graphviz renders SVG when dataflow is expanded', async ({ page }) => { - await page.goto('/memory'); - - // Wait for loading to complete - await expect(page.locator('text=Loading...')).toBeHidden({ timeout: 10000 }); - - // Find the first expand button - const expandButton = page.locator('table.dataflows tbody button').first(); - - // Skip test if no dataflows are present - if ((await expandButton.count()) === 0) { - test.skip(); - return; - } - - await expandButton.click(); + await openMemoryPage(page, fixturePage('/memory')); + await expandFixtureDataflow(page); // Wait for SVG to be rendered by graphviz const svg = page.locator('svg').first(); - await expect(svg).toBeVisible({ timeout: 15000 }); + await expect(svg).toBeVisible(); // Verify it's a valid graphviz SVG (has graph elements) const graphElement = svg.locator('g.graph, g.node, g.edge').first(); @@ -110,10 +143,7 @@ test.describe('/memory page', () => { }); test('include system catalog checkbox works', async ({ page }) => { - await page.goto('/memory'); - - // Wait for loading to complete - await expect(page.locator('text=Loading...')).toBeHidden({ timeout: 10000 }); + await openMemoryPage(page, fixturePage('/memory')); const checkbox = page.locator('#include_system_catalog'); await expect(checkbox).toBeVisible(); diff --git a/test/dataflow-visualizer/tsconfig.typecheck.json b/test/dataflow-visualizer/tsconfig.typecheck.json index cfeef96aed893..6620e793d6062 100644 --- a/test/dataflow-visualizer/tsconfig.typecheck.json +++ b/test/dataflow-visualizer/tsconfig.typecheck.json @@ -9,14 +9,17 @@ "lib": ["ES2020", "DOM"], "strict": false, "moduleResolution": "bundler", - // The checked files are plain browser scripts with no imports, so nothing - // pulls `@types/*` in. List them explicitly to get `React` and `d3` as - // ambient globals. - "types": ["react", "react-dom", "d3"] + // The browser scripts have no imports, so nothing pulls `@types/*` in. + // List them explicitly to get `React` and `d3` as ambient globals, plus + // `node` for the `process.env` the Playwright specs read. + "types": ["react", "react-dom", "d3", "node"] }, "include": [ "../../src/environmentd/src/http/static/js/prometheus.js", "../../src/environmentd/src/http/static/js/metrics.jsx", - "../../src/environmentd/src/http/static/js/fetch.js" + "../../src/environmentd/src/http/static/js/fetch.js", + "../../src/environmentd/src/http/static/js/cluster-replica.jsx", + "tests/**/*.ts", + "playwright.config.ts" ] } diff --git a/test/launchdarkly-flag-consistency/mzcompose.py b/test/launchdarkly-flag-consistency/mzcompose.py index 074467a98fc16..bd8d9cb6baac4 100644 --- a/test/launchdarkly-flag-consistency/mzcompose.py +++ b/test/launchdarkly-flag-consistency/mzcompose.py @@ -526,6 +526,7 @@ "compute_subscribe_snapshot_optimization", "enable_cast_elimination", "enable_compute_correction_v2", + "enable_compute_error_distinct", "enable_compute_temporal_bucketing", "enable_new_outer_join_lowering", "enable_upsert_v2", diff --git a/test/sqllogictest/alter.slt b/test/sqllogictest/alter.slt index 44f3aa6575187..1935ee5ff34e5 100644 --- a/test/sqllogictest/alter.slt +++ b/test/sqllogictest/alter.slt @@ -24,7 +24,7 @@ ALTER SYSTEM SET enable_rbac_checks TO false; ---- COMPLETE 0 -query error must be owner of TABLE mz_catalog.mz_tables +query error mz_tables is a materialized view not a table ALTER TABLE mz_tables RENAME TO foo; query error must be owner of SOURCE mz_internal.mz_storage_shards @@ -33,7 +33,7 @@ ALTER SOURCE mz_internal.mz_storage_shards RENAME TO foo; simple conn=mz_system,user=mz_system ALTER TABLE mz_tables RENAME TO foo; ---- -db error: ERROR: system item 'mz_catalog.mz_tables' cannot be modified +db error: ERROR: mz_tables is a materialized view not a table simple conn=mz_system,user=mz_system ALTER SOURCE mz_internal.mz_storage_shards RENAME TO foo; diff --git a/test/sqllogictest/autogenerated/mz_internal.slt b/test/sqllogictest/autogenerated/mz_internal.slt index c09f62672307c..66c3808a23067 100644 --- a/test/sqllogictest/autogenerated/mz_internal.slt +++ b/test/sqllogictest/autogenerated/mz_internal.slt @@ -352,6 +352,12 @@ SELECT name, type, comment FROM objects WHERE schema = 'mz_internal' AND object object_id text The␠ID␠of␠the␠dependent␠object.␠Corresponds␠to␠`mz_objects.id`. referenced_object_id text The␠ID␠of␠the␠referenced␠object.␠Corresponds␠to␠`mz_objects.id`. +query TTT +SELECT name, type, comment FROM objects WHERE schema = 'mz_internal' AND object = 'mz_object_graph_edges' ORDER BY position +---- +object_id text The␠ID␠of␠the␠dependent␠object.␠Corresponds␠to␠`mz_objects.id`. +dependency_id text The␠ID␠of␠the␠object␠it␠depends␠on.␠Corresponds␠to␠`mz_objects.id`. + query TTT SELECT name, type, comment FROM objects WHERE schema = 'mz_internal' AND object = 'mz_object_fully_qualified_names' ORDER BY position ---- @@ -780,6 +786,8 @@ mz_aws_privatelink_connection_status_history mz_aws_privatelink_connection_statuses mz_builtin_materialized_views mz_builtin_sources +mz_builtin_tables +mz_builtin_views mz_catalog_raw mz_cluster_auto_scaling_strategies mz_cluster_deployment_lineage @@ -829,6 +837,7 @@ mz_object_arrangement_sizes mz_object_dependencies mz_object_fully_qualified_names mz_object_global_ids +mz_object_graph_edges mz_object_history mz_object_lifetimes mz_object_oid_alias diff --git a/test/sqllogictest/catalog_server_explain.slt b/test/sqllogictest/catalog_server_explain.slt index ee7a28553dc06..6a2ed84d1f888 100644 --- a/test/sqllogictest/catalog_server_explain.slt +++ b/test/sqllogictest/catalog_server_explain.slt @@ -1558,6 +1558,103 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN INDEX "mz_internal"."mz_object_graph_edges_ind"; +---- +mz_internal.mz_object_graph_edges_ind: + →Arrange (#0{object_id}) + →Arranged mz_internal.mz_object_graph_edges + +mz_internal.mz_object_graph_edges: + →With + cte l0 = + →Arrange (#0{id}) + →Fused with Child Map/Filter/Project + Project: #1 + Filter: ((#4{type} = "sink") OR (#4{type} = "index") OR (#4{type} = "table") OR (#4{type} = "source") OR (#4{type} = "materialized-view")) + →Arranged mz_catalog.mz_objects + Key: (#2{schema_id}) + cte l1 = + →Arranged mz_catalog.mz_sources + →Return + →Distinct GroupAggregate + →Union + →Delta Join [%0[#0{object_id}] » %1:l0[#0{id}] » %2:l0[#0{id}]] [%1:l0[#0{id}] » %0[#1{dependency_id}] » %2:l0[#0{id}]] [%2:l0[#0{id}] » %0[#0{object_id}] » %1:l0[#0{id}]] + →Arrange (#0{object_id}) (#1{dependency_id}) + →Union + →Unarranged Raw Stream + →Fused with Parent Map/Filter/Project + Project: #1, #0 + →Arranged mz_internal.mz_compute_dependencies + →Delta Join [%0:mz_object_dependencies[#0{object_id}] » %1:mz_sinks[#0{id}] » %2[#0{id}]] [%1:mz_sinks[#0{id}] » %0:mz_object_dependencies[#0{object_id}] » %2[#0{id}]] [%2[#0{id}] » %0:mz_object_dependencies[#1{referenced_object_id}] » %1:mz_sinks[#0{id}]] + →Arrange (#1{referenced_object_id}) + →Arranged mz_internal.mz_object_dependencies + →Arranged mz_catalog.mz_sinks + →Arrange (#0{id}) + →Union + →Fused with Child Map/Filter/Project + Project: #1 + →Arranged mz_catalog.mz_tables + Key: (#2{schema_id}) + →Fused with Child Map/Filter/Project + Project: #0 + →Arranged mz_catalog.mz_sources + Key: (#0{id}) + →Fused with Child Map/Filter/Project + Project: #1 + →Arranged mz_catalog.mz_views + Key: (#2{schema_id}) + →Fused with Child Map/Filter/Project + Project: #0 + →Arranged mz_catalog.mz_materialized_views + Key: (#0{id}) + →Arranged l0 + →Arranged l0 + →Delta Join [%0:mz_object_dependencies[#0{object_id}] » %2:l1[#0{id}] » %1:l1[#0{id}]] [%1:l1[#0{id}] » %0:mz_object_dependencies[#1{referenced_object_id}] » %2:l1[#0{id}]] [%2:l1[#0{id}] » %0:mz_object_dependencies[#0{object_id}] » %1:l1[#0{id}]] + path %0: + after %2: + Project: #0, #1 + Filter: (#5{type} = "subsource") + path %1: + after %0: + Project: #1, #0 + Filter: like["u%"](#0{referenced_object_id}) + after %2: + Project: #0, #1 + Filter: (#5{type} = "subsource") + path %2: + after %0: + Filter: like["u%"](#1{referenced_object_id}) + →Arrange (#0{object_id}) (#1{referenced_object_id}) + →Fused with Child Map/Filter/Project + Filter: like["u%"](#1{referenced_object_id}) + →Arranged mz_internal.mz_object_dependencies + Key: (#0{object_id}) + →Arranged l1 + →Arranged l1 + →Differential Join %1:l1[#0{id}] » %0:mz_tables[#1{source_id}] + →Arrange (#1{source_id}) + →Fused with Child Map/Filter/Project + Project: #1, #8 + Filter: (#8{source_id}) IS NOT NULL + →Arranged mz_catalog.mz_tables + Key: (#2{schema_id}) + →Arranged l1 + +Used Indexes: + - mz_catalog.mz_sources_ind (*** full scan ***, differential join, delta join lookup) + - mz_catalog.mz_sinks_ind (delta join lookup) + - mz_catalog.mz_materialized_views_ind (*** full scan ***) + - mz_internal.mz_object_dependencies_ind (*** full scan ***, delta join 1st input (full scan)) + - mz_internal.mz_compute_dependencies_ind (*** full scan ***) + - mz_catalog.mz_tables_ind (*** full scan ***) + - mz_catalog.mz_objects_ind (*** full scan ***) + - mz_catalog.mz_views_ind (*** full scan ***) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN INDEX "mz_internal"."mz_object_history_ind"; ---- @@ -4811,7 +4908,7 @@ mz_catalog.mz_indexes: Project: #2, #0, #1, #3, #4, #7, #5, #6 Map: "s1" →Arrange (#1{name}) (#2{on_schema}, #3{on_name}) - →Constant (80 rows) + →Constant (81 rows) →Arrange (empty key) (#1{name}) →Fused with Child Map/Filter/Project Project: #4, #3 @@ -5011,7 +5108,7 @@ mz_catalog.mz_materialized_views: Project: #4, #0, #5, #1, #6, #2, #9, #8, #3, #7 Map: {=r/s1, s1=r/s1}, "s1" →Arrange (#1{schema_name}, #2{name}) - →Constant (40 rows) + →Constant (42 rows) →Arrange (empty key) (#0{schema_name}) (#0{schema_name}, #1{name}) →Fused with Child Map/Filter/Project Project: #4, #3, #5 @@ -5332,6 +5429,110 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_tables"; +---- +mz_catalog.mz_tables: + →Union + →Fused with Child Map/Filter/Project + Project: #6..=#11, #4, #12, #13 + Filter: ("Item" = #3) AND ("table" = (#5 ->> "type")) + Map: (((#2 -> "definition") -> "V1") ->> "create_sql"), parse_catalog_create_sql(#4), parse_catalog_id((#1 -> "gid")), text_to_oid((#2 ->> "oid")), case when ((#2 ->> "ephemeral_owner_session")) IS NULL then parse_catalog_id(((#0{data} -> "value") -> "schema_id")) else "0" end, (#2 ->> "name"), parse_catalog_id((#2 -> "owner_id")), parse_catalog_privileges((#2 -> "privileges")), redact_sql(#4), (#5 ->> "source_id") + →Read mz_internal.mz_catalog_raw + →Delta Join [%0[#1{schema_name}, #2{name}] » %1:mz_catalog_raw[#0{schema_name}, #1{name}] » %2:mz_schemas[#1{name}]] [%1:mz_catalog_raw[#0{schema_name}] » %0[#1{schema_name}, #2{name}] » %2:mz_schemas[#1{name}]] [%2:mz_schemas[#1{name}] » %1:mz_catalog_raw[#0{schema_name}] » %0[#1{schema_name}, #2{name}]] + path %0: + Final closure: + Project: #3, #0, #4, #1, #5, #2, #6, #6, #6 + Map: "s1", null + path %1: + Final closure: + Project: #3, #0, #4, #1, #5, #2, #6, #6, #6 + Map: "s1", null + path %2: + Final closure: + Project: #3, #0, #4, #1, #5, #2, #6, #6, #6 + Map: "s1", null + →Arrange (#1{schema_name}, #2{name}) + →Constant (29 rows) + →Arrange (#0{schema_name}) (#0{schema_name}, #1{name}) + →Fused with Child Map/Filter/Project + Project: #5, #4, #6 + Filter: ("1" = (#1 ->> "object_type")) AND ("GidMapping" = #3) AND (#4) IS NOT NULL AND (#5) IS NOT NULL + Map: (#1 ->> "object_name"), (#1 ->> "schema_name"), ("s" || (#2 ->> "catalog_id")) + →Read mz_internal.mz_catalog_raw + →Arrange (#1{name}) + →Fused with Child Map/Filter/Project + Project: #1, #3 + Filter: (#0{database_id}) IS NULL + →Arranged mz_catalog.mz_schemas + Key: (#2{database_id}) + +Source mz_internal.mz_catalog_raw + project=(#0..=#3) + map=((#0{data} -> "key"), (#0{data} -> "value"), (#0{data} ->> "kind")) + +Used Indexes: + - mz_catalog.mz_schemas_ind (*** full scan ***) + +Target cluster: mz_catalog_server + +EOF + +query T multiline +EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_views"; +---- +mz_catalog.mz_views: + →Union + →Fused with Child Map/Filter/Project + Project: #6..=#12, #4, #13 + Filter: ("Item" = #3) AND ("view" = (#5 ->> "type")) + Map: (((#2 -> "definition") -> "V1") ->> "create_sql"), parse_catalog_create_sql(#4), parse_catalog_id((#1 -> "gid")), text_to_oid((#2 ->> "oid")), case when ((#2 ->> "ephemeral_owner_session")) IS NULL then parse_catalog_id(((#0{data} -> "value") -> "schema_id")) else "0" end, (#2 ->> "name"), (#5 ->> "definition"), parse_catalog_id((#2 -> "owner_id")), parse_catalog_privileges((#2 -> "privileges")), redact_sql(#4) + →Read mz_internal.mz_catalog_raw + →Delta Join [%0[#1{schema_name}, #2{name}] » %1:mz_catalog_raw[#0{schema_name}, #1{name}] » %2:mz_schemas[#1{name}]] [%1:mz_catalog_raw[#0{schema_name}] » %0[#1{schema_name}, #2{name}] » %2:mz_schemas[#1{name}]] [%2:mz_schemas[#1{name}] » %1:mz_catalog_raw[#0{schema_name}] » %0[#1{schema_name}, #2{name}]] + path %0: + Final closure: + Project: #5, #0, #6, #1, #2, #8, #3, #4, #7 + Map: "s1" + path %1: + after %0: + Project: #3, #1, #4..=#6, #0, #2, #7 + Map: redact_sql(#6{create_sql}) + Final closure: + Project: #5, #0, #6, #1, #2, #8, #3, #4, #7 + Map: "s1" + path %2: + after %0: + Project: #4, #1, #5..=#7, #2, #3, #8 + Map: redact_sql(#7{create_sql}) + Final closure: + Project: #5, #0, #6, #1, #2, #8, #3, #4, #7 + Map: "s1" + →Arrange (#1{schema_name}, #2{name}) + →Constant (193 rows) + →Arrange (#0{schema_name}) (#0{schema_name}, #1{name}) + →Fused with Child Map/Filter/Project + Project: #5, #4, #6 + Filter: ("4" = (#1 ->> "object_type")) AND ("GidMapping" = #3) AND (#4) IS NOT NULL AND (#5) IS NOT NULL + Map: (#1 ->> "object_name"), (#1 ->> "schema_name"), ("s" || (#2 ->> "catalog_id")) + →Read mz_internal.mz_catalog_raw + →Arrange (#1{name}) + →Fused with Child Map/Filter/Project + Project: #1, #3 + Filter: (#0{database_id}) IS NULL + →Arranged mz_catalog.mz_schemas + Key: (#2{database_id}) + +Source mz_internal.mz_catalog_raw + project=(#0..=#3) + map=((#0{data} -> "key"), (#0{data} -> "value"), (#0{data} ->> "kind")) + +Used Indexes: + - mz_catalog.mz_schemas_ind (*** full scan ***) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_aws_connections"; ---- @@ -7991,7 +8192,7 @@ query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_materialized_views"; ---- Explained Query (fast path): - →Constant (40 rows) + →Constant (42 rows) Target cluster: mz_catalog_server @@ -8007,6 +8208,26 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_tables"; +---- +Explained Query (fast path): + →Constant (29 rows) + +Target cluster: mz_catalog_server + +EOF + +query T multiline +EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_views"; +---- +Explained Query (fast path): + →Constant (193 rows) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_cluster_deployment_lineage"; ---- @@ -9903,6 +10124,20 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN SELECT * FROM "mz_internal"."mz_object_graph_edges"; +---- +Explained Query (fast path): + →Map/Filter/Project + →Indexed mz_internal.mz_object_graph_edges (using mz_internal.mz_object_graph_edges_ind) + +Used Indexes: + - mz_internal.mz_object_graph_edges_ind (*** full scan ***) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_object_history"; ---- diff --git a/test/sqllogictest/cluster.slt b/test/sqllogictest/cluster.slt index 5766418e810a0..10cb6649b6d62 100644 --- a/test/sqllogictest/cluster.slt +++ b/test/sqllogictest/cluster.slt @@ -420,7 +420,7 @@ CREATE CLUSTER test REPLICAS (foo (SIZE 'scale=1,workers=1')); query I SELECT COUNT(name) FROM mz_indexes; ---- -304 +305 statement ok DROP CLUSTER test CASCADE @@ -428,7 +428,7 @@ DROP CLUSTER test CASCADE query T SELECT COUNT(name) FROM mz_indexes; ---- -272 +273 simple conn=mz_system,user=mz_system ALTER CLUSTER quickstart OWNER TO materialize diff --git a/test/sqllogictest/cockroach/srfs.slt b/test/sqllogictest/cockroach/srfs.slt index b8a169c3836e1..10592810e8f52 100644 --- a/test/sqllogictest/cockroach/srfs.slt +++ b/test/sqllogictest/cockroach/srfs.slt @@ -1472,6 +1472,7 @@ mz_object_arrangement_size_history 2 mz_object_arrangement_size_history 4 mz_object_arrangement_sizes 1 mz_object_dependencies 1 +mz_object_graph_edges 1 mz_object_history 1 mz_object_lifetimes 1 mz_object_transitive_dependencies 1 diff --git a/test/sqllogictest/error_semantics.slt b/test/sqllogictest/error_semantics.slt index 967ca375be35c..3462e888974f3 100644 --- a/test/sqllogictest/error_semantics.slt +++ b/test/sqllogictest/error_semantics.slt @@ -93,3 +93,95 @@ select coalesce(a, (select a/b from test)) from test; query error Evaluation error: division by zero select *, case when a = 5 then (select a/b from test) else 7 end from test; + +# A chain of diamond-shaped CTEs over an erroring expression. Each level reads the level +# below it twice, so an undeduplicated error collection doubles its multiplicity per level +# and overflows `Diff` well before the chain ends. The multiplicity of an error is not +# observable, so the only thing to assert is that the error still comes back, and does so +# without an overflow (which CI catches as a panic). +# +# `union` rather than `union all` is what makes this reach the overflow: the distinct arranges +# the error collection, so duplicates consolidate into one record carrying a doubling diff. +# Under `union all` nothing consolidates between levels, the records duplicate physically +# instead, and the query exhausts memory without ever overflowing. + +statement ok +create table diamond (a int, b int) + +statement ok +insert into diamond values (1, 0) + +query error division by zero +with + c0 as (select a, a / b as q from diamond), + c1 as (select a, q from c0 where a = 1 union select q, a from c0 where a = 2), + c2 as (select a, q from c1 where a = 1 union select q, a from c1 where a = 2), + c3 as (select a, q from c2 where a = 1 union select q, a from c2 where a = 2), + c4 as (select a, q from c3 where a = 1 union select q, a from c3 where a = 2), + c5 as (select a, q from c4 where a = 1 union select q, a from c4 where a = 2), + c6 as (select a, q from c5 where a = 1 union select q, a from c5 where a = 2), + c7 as (select a, q from c6 where a = 1 union select q, a from c6 where a = 2), + c8 as (select a, q from c7 where a = 1 union select q, a from c7 where a = 2), + c9 as (select a, q from c8 where a = 1 union select q, a from c8 where a = 2), + c10 as (select a, q from c9 where a = 1 union select q, a from c9 where a = 2), + c11 as (select a, q from c10 where a = 1 union select q, a from c10 where a = 2), + c12 as (select a, q from c11 where a = 1 union select q, a from c11 where a = 2), + c13 as (select a, q from c12 where a = 1 union select q, a from c12 where a = 2), + c14 as (select a, q from c13 where a = 1 union select q, a from c13 where a = 2), + c15 as (select a, q from c14 where a = 1 union select q, a from c14 where a = 2), + c16 as (select a, q from c15 where a = 1 union select q, a from c15 where a = 2), + c17 as (select a, q from c16 where a = 1 union select q, a from c16 where a = 2), + c18 as (select a, q from c17 where a = 1 union select q, a from c17 where a = 2), + c19 as (select a, q from c18 where a = 1 union select q, a from c18 where a = 2), + c20 as (select a, q from c19 where a = 1 union select q, a from c19 where a = 2), + c21 as (select a, q from c20 where a = 1 union select q, a from c20 where a = 2), + c22 as (select a, q from c21 where a = 1 union select q, a from c21 where a = 2), + c23 as (select a, q from c22 where a = 1 union select q, a from c22 where a = 2), + c24 as (select a, q from c23 where a = 1 union select q, a from c23 where a = 2), + c25 as (select a, q from c24 where a = 1 union select q, a from c24 where a = 2), + c26 as (select a, q from c25 where a = 1 union select q, a from c25 where a = 2), + c27 as (select a, q from c26 where a = 1 union select q, a from c26 where a = 2), + c28 as (select a, q from c27 where a = 1 union select q, a from c27 where a = 2), + c29 as (select a, q from c28 where a = 1 union select q, a from c28 where a = 2), + c30 as (select a, q from c29 where a = 1 union select q, a from c29 where a = 2), + c31 as (select a, q from c30 where a = 1 union select q, a from c30 where a = 2), + c32 as (select a, q from c31 where a = 1 union select q, a from c31 where a = 2), + c33 as (select a, q from c32 where a = 1 union select q, a from c32 where a = 2), + c34 as (select a, q from c33 where a = 1 union select q, a from c33 where a = 2), + c35 as (select a, q from c34 where a = 1 union select q, a from c34 where a = 2), + c36 as (select a, q from c35 where a = 1 union select q, a from c35 where a = 2), + c37 as (select a, q from c36 where a = 1 union select q, a from c36 where a = 2), + c38 as (select a, q from c37 where a = 1 union select q, a from c37 where a = 2), + c39 as (select a, q from c38 where a = 1 union select q, a from c38 where a = 2), + c40 as (select a, q from c39 where a = 1 union select q, a from c39 where a = 2), + c41 as (select a, q from c40 where a = 1 union select q, a from c40 where a = 2), + c42 as (select a, q from c41 where a = 1 union select q, a from c41 where a = 2), + c43 as (select a, q from c42 where a = 1 union select q, a from c42 where a = 2), + c44 as (select a, q from c43 where a = 1 union select q, a from c43 where a = 2), + c45 as (select a, q from c44 where a = 1 union select q, a from c44 where a = 2), + c46 as (select a, q from c45 where a = 1 union select q, a from c45 where a = 2), + c47 as (select a, q from c46 where a = 1 union select q, a from c46 where a = 2), + c48 as (select a, q from c47 where a = 1 union select q, a from c47 where a = 2), + c49 as (select a, q from c48 where a = 1 union select q, a from c48 where a = 2), + c50 as (select a, q from c49 where a = 1 union select q, a from c49 where a = 2), + c51 as (select a, q from c50 where a = 1 union select q, a from c50 where a = 2), + c52 as (select a, q from c51 where a = 1 union select q, a from c51 where a = 2), + c53 as (select a, q from c52 where a = 1 union select q, a from c52 where a = 2), + c54 as (select a, q from c53 where a = 1 union select q, a from c53 where a = 2), + c55 as (select a, q from c54 where a = 1 union select q, a from c54 where a = 2), + c56 as (select a, q from c55 where a = 1 union select q, a from c55 where a = 2), + c57 as (select a, q from c56 where a = 1 union select q, a from c56 where a = 2), + c58 as (select a, q from c57 where a = 1 union select q, a from c57 where a = 2), + c59 as (select a, q from c58 where a = 1 union select q, a from c58 where a = 2), + c60 as (select a, q from c59 where a = 1 union select q, a from c59 where a = 2), + c61 as (select a, q from c60 where a = 1 union select q, a from c60 where a = 2), + c62 as (select a, q from c61 where a = 1 union select q, a from c61 where a = 2), + c63 as (select a, q from c62 where a = 1 union select q, a from c62 where a = 2), + c64 as (select a, q from c63 where a = 1 union select q, a from c63 where a = 2), + c65 as (select a, q from c64 where a = 1 union select q, a from c64 where a = 2), + c66 as (select a, q from c65 where a = 1 union select q, a from c65 where a = 2), + c67 as (select a, q from c66 where a = 1 union select q, a from c66 where a = 2), + c68 as (select a, q from c67 where a = 1 union select q, a from c67 where a = 2), + c69 as (select a, q from c68 where a = 1 union select q, a from c68 where a = 2), + c70 as (select a, q from c69 where a = 1 union select q, a from c69 where a = 2) +select * from c70 diff --git a/test/sqllogictest/information_schema_tables.slt b/test/sqllogictest/information_schema_tables.slt index 3f1d1a0dca450..6aa277b21008a 100644 --- a/test/sqllogictest/information_schema_tables.slt +++ b/test/sqllogictest/information_schema_tables.slt @@ -250,7 +250,7 @@ MATERIALIZED VIEW materialize mz_catalog mz_tables -BASE TABLE +MATERIALIZED VIEW materialize mz_catalog mz_timezone_abbreviations @@ -266,7 +266,7 @@ BASE TABLE materialize mz_catalog mz_views -BASE TABLE +MATERIALIZED VIEW materialize mz_internal mz_activity_log_thinned @@ -297,6 +297,14 @@ mz_builtin_sources VIEW materialize mz_internal +mz_builtin_tables +VIEW +materialize +mz_internal +mz_builtin_views +VIEW +materialize +mz_internal mz_catalog_raw SOURCE materialize @@ -493,6 +501,10 @@ mz_object_global_ids BASE TABLE materialize mz_internal +mz_object_graph_edges +VIEW +materialize +mz_internal mz_object_history VIEW materialize diff --git a/test/sqllogictest/mz_catalog_server_index_accounting.slt b/test/sqllogictest/mz_catalog_server_index_accounting.slt index 95138d7704663..6e9b993db5d58 100644 --- a/test/sqllogictest/mz_catalog_server_index_accounting.slt +++ b/test/sqllogictest/mz_catalog_server_index_accounting.slt @@ -37,107 +37,108 @@ mz_arrangement_heap_capacity_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangemen mz_arrangement_heap_size_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_heap_size_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_heap_size_raw"␠("operator_id",␠"worker_id") mz_arrangement_records_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_records_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_records_raw"␠("operator_id",␠"worker_id") mz_arrangement_sharing_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_sharing_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_sharing_raw"␠("operator_id",␠"worker_id") -mz_cluster_auto_scaling_strategies_ind CREATE␠INDEX␠"mz_cluster_auto_scaling_strategies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s518␠AS␠"mz_internal"."mz_cluster_auto_scaling_strategies"]␠("cluster_id") -mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s765␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") +mz_cluster_auto_scaling_strategies_ind CREATE␠INDEX␠"mz_cluster_auto_scaling_strategies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s520␠AS␠"mz_internal"."mz_cluster_auto_scaling_strategies"]␠("cluster_id") +mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s768␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") mz_cluster_prometheus_metrics_s2_primary_idx CREATE␠INDEX␠"mz_cluster_prometheus_metrics_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_cluster_prometheus_metrics"␠("process_id",␠"metric_name",␠"labels") -mz_cluster_reconfigurations_ind CREATE␠INDEX␠"mz_cluster_reconfigurations_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s517␠AS␠"mz_internal"."mz_cluster_reconfigurations"]␠("cluster_id") -mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s759␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") -mz_cluster_replica_history_ind CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s615␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at") -mz_cluster_replica_metrics_history_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s523␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id") -mz_cluster_replica_metrics_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s524␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id") -mz_cluster_replica_name_history_ind CREATE␠INDEX␠"mz_cluster_replica_name_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s616␠AS␠"mz_internal"."mz_cluster_replica_name_history"]␠("id") -mz_cluster_replica_size_internal_ind CREATE␠INDEX␠"mz_cluster_replica_size_internal_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s513␠AS␠"mz_internal"."mz_cluster_replica_size_internal"]␠("size") -mz_cluster_replica_sizes_ind CREATE␠INDEX␠"mz_cluster_replica_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s512␠AS␠"mz_catalog"."mz_cluster_replica_sizes"]␠("size") -mz_cluster_replica_status_history_ind CREATE␠INDEX␠"mz_cluster_replica_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s525␠AS␠"mz_internal"."mz_cluster_replica_status_history"]␠("replica_id") -mz_cluster_replica_statuses_ind CREATE␠INDEX␠"mz_cluster_replica_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s526␠AS␠"mz_internal"."mz_cluster_replica_statuses"]␠("replica_id") -mz_cluster_replicas_ind CREATE␠INDEX␠"mz_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s522␠AS␠"mz_catalog"."mz_cluster_replicas"]␠("id") -mz_clusters_ind CREATE␠INDEX␠"mz_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s514␠AS␠"mz_catalog"."mz_clusters"]␠("id") -mz_columns_ind CREATE␠INDEX␠"mz_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s485␠AS␠"mz_catalog"."mz_columns"]␠("name") -mz_comments_ind CREATE␠INDEX␠"mz_comments_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s541␠AS␠"mz_internal"."mz_comments"]␠("id") +mz_cluster_reconfigurations_ind CREATE␠INDEX␠"mz_cluster_reconfigurations_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s519␠AS␠"mz_internal"."mz_cluster_reconfigurations"]␠("cluster_id") +mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s762␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") +mz_cluster_replica_history_ind CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s617␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at") +mz_cluster_replica_metrics_history_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s525␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id") +mz_cluster_replica_metrics_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s526␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id") +mz_cluster_replica_name_history_ind CREATE␠INDEX␠"mz_cluster_replica_name_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s618␠AS␠"mz_internal"."mz_cluster_replica_name_history"]␠("id") +mz_cluster_replica_size_internal_ind CREATE␠INDEX␠"mz_cluster_replica_size_internal_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s515␠AS␠"mz_internal"."mz_cluster_replica_size_internal"]␠("size") +mz_cluster_replica_sizes_ind CREATE␠INDEX␠"mz_cluster_replica_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s514␠AS␠"mz_catalog"."mz_cluster_replica_sizes"]␠("size") +mz_cluster_replica_status_history_ind CREATE␠INDEX␠"mz_cluster_replica_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s527␠AS␠"mz_internal"."mz_cluster_replica_status_history"]␠("replica_id") +mz_cluster_replica_statuses_ind CREATE␠INDEX␠"mz_cluster_replica_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s528␠AS␠"mz_internal"."mz_cluster_replica_statuses"]␠("replica_id") +mz_cluster_replicas_ind CREATE␠INDEX␠"mz_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s524␠AS␠"mz_catalog"."mz_cluster_replicas"]␠("id") +mz_clusters_ind CREATE␠INDEX␠"mz_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s516␠AS␠"mz_catalog"."mz_clusters"]␠("id") +mz_columns_ind CREATE␠INDEX␠"mz_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s487␠AS␠"mz_catalog"."mz_columns"]␠("name") +mz_comments_ind CREATE␠INDEX␠"mz_comments_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s543␠AS␠"mz_internal"."mz_comments"]␠("id") mz_compute_dataflow_global_ids_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_dataflow_global_ids_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_dataflow_global_ids_per_worker"␠("id",␠"worker_id",␠"global_id") -mz_compute_dependencies_ind CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s741␠AS␠"mz_internal"."mz_compute_dependencies"]␠("dependency_id") +mz_compute_dependencies_ind CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s743␠AS␠"mz_internal"."mz_compute_dependencies"]␠("dependency_id") mz_compute_error_counts_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_error_counts_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_error_counts_raw"␠("export_id",␠"worker_id") mz_compute_exports_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_exports_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_exports_per_worker"␠("export_id",␠"worker_id") mz_compute_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_frontiers_per_worker"␠("export_id",␠"worker_id") -mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s750␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") +mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") mz_compute_hydration_times_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_hydration_times_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_hydration_times_per_worker"␠("export_id",␠"worker_id") mz_compute_import_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_import_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_import_frontiers_per_worker"␠("export_id",␠"import_id",␠"worker_id") mz_compute_lir_mapping_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_lir_mapping_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_lir_mapping_per_worker"␠("global_id",␠"lir_id",␠"worker_id") mz_compute_operator_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_durations_histogram_raw"␠("id",␠"worker_id",␠"duration_ns") mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_hydration_statuses_per_worker"␠("export_id",␠"lir_id",␠"worker_id") -mz_connections_ind CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s520␠AS␠"mz_catalog"."mz_connections"]␠("schema_id") -mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s746␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") -mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s745␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") -mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s744␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") -mz_databases_ind CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s483␠AS␠"mz_catalog"."mz_databases"]␠("name") +mz_connections_ind CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s522␠AS␠"mz_catalog"."mz_connections"]␠("schema_id") +mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s749␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") +mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s748␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") +mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s747␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") +mz_databases_ind CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s485␠AS␠"mz_catalog"."mz_databases"]␠("name") mz_dataflow_addresses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_addresses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_addresses_per_worker"␠("id",␠"worker_id") mz_dataflow_channels_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_channels_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_channels_per_worker"␠("id",␠"worker_id") mz_dataflow_operator_reachability_raw_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operator_reachability_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operator_reachability_raw"␠("id",␠"worker_id",␠"source",␠"port",␠"update_type",␠"time") mz_dataflow_operators_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operators_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operators_per_worker"␠("id",␠"worker_id") -mz_frontiers_ind CREATE␠INDEX␠"mz_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s732␠AS␠"mz_internal"."mz_frontiers"]␠("object_id") -mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s761␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") -mz_indexes_ind CREATE␠INDEX␠"mz_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s486␠AS␠"mz_catalog"."mz_indexes"]␠("id") -mz_kafka_sources_ind CREATE␠INDEX␠"mz_kafka_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s480␠AS␠"mz_catalog"."mz_kafka_sources"]␠("id") -mz_materialized_views_ind CREATE␠INDEX␠"mz_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s544␠AS␠"mz_catalog"."mz_materialized_views"]␠("id") +mz_frontiers_ind CREATE␠INDEX␠"mz_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s734␠AS␠"mz_internal"."mz_frontiers"]␠("object_id") +mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s764␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") +mz_indexes_ind CREATE␠INDEX␠"mz_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s488␠AS␠"mz_catalog"."mz_indexes"]␠("id") +mz_kafka_sources_ind CREATE␠INDEX␠"mz_kafka_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s482␠AS␠"mz_catalog"."mz_kafka_sources"]␠("id") +mz_materialized_views_ind CREATE␠INDEX␠"mz_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s546␠AS␠"mz_catalog"."mz_materialized_views"]␠("id") mz_message_batch_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_batch_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_batch_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_batch_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") -mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s835␠AS␠"mz_internal"."mz_notices"]␠("id") -mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") -mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") -mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s751␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") -mz_object_dependencies_ind CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s481␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id") -mz_object_history_ind CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s555␠AS␠"mz_internal"."mz_object_history"]␠("id") -mz_object_lifetimes_ind CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s556␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id") -mz_object_transitive_dependencies_ind CREATE␠INDEX␠"mz_object_transitive_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s572␠AS␠"mz_internal"."mz_object_transitive_dependencies"]␠("object_id") -mz_objects_ind CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s552␠AS␠"mz_catalog"."mz_objects"]␠("schema_id") +mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s839␠AS␠"mz_internal"."mz_notices"]␠("id") +mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s756␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") +mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s756␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") +mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s754␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") +mz_object_dependencies_ind CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s483␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id") +mz_object_graph_edges_ind CREATE␠INDEX␠"mz_object_graph_edges_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s745␠AS␠"mz_internal"."mz_object_graph_edges"]␠("object_id") +mz_object_history_ind CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s557␠AS␠"mz_internal"."mz_object_history"]␠("id") +mz_object_lifetimes_ind CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s558␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id") +mz_object_transitive_dependencies_ind CREATE␠INDEX␠"mz_object_transitive_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s574␠AS␠"mz_internal"."mz_object_transitive_dependencies"]␠("object_id") +mz_objects_ind CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s554␠AS␠"mz_catalog"."mz_objects"]␠("schema_id") mz_peek_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_peek_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_peek_durations_histogram_raw"␠("worker_id",␠"type",␠"duration_ns") -mz_recent_activity_log_thinned_ind CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s716␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash") -mz_recent_sql_text_ind CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s712␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash") -mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s828␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") -mz_roles_ind CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s504␠AS␠"mz_catalog"."mz_roles"]␠("id") +mz_recent_activity_log_thinned_ind CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s718␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash") +mz_recent_sql_text_ind CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s714␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash") +mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s832␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") +mz_roles_ind CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s506␠AS␠"mz_catalog"."mz_roles"]␠("id") mz_scheduling_elapsed_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_elapsed_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_elapsed_raw"␠("id",␠"worker_id") mz_scheduling_parks_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_parks_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_parks_histogram_raw"␠("worker_id",␠"slept_for_ns",␠"requested_ns") -mz_schemas_ind CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s484␠AS␠"mz_catalog"."mz_schemas"]␠("database_id") -mz_secrets_ind CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s519␠AS␠"mz_catalog"."mz_secrets"]␠("name") -mz_show_all_objects_ind CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s600␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id") -mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s763␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") -mz_show_clusters_ind CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s602␠AS␠"mz_internal"."mz_show_clusters"]␠("name") -mz_show_columns_ind CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s601␠AS␠"mz_internal"."mz_show_columns"]␠("id") -mz_show_connections_ind CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s610␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id") -mz_show_databases_ind CREATE␠INDEX␠"mz_show_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s604␠AS␠"mz_internal"."mz_show_databases"]␠("name") -mz_show_indexes_ind CREATE␠INDEX␠"mz_show_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s614␠AS␠"mz_internal"."mz_show_indexes"]␠("schema_id") -mz_show_materialized_views_ind CREATE␠INDEX␠"mz_show_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s613␠AS␠"mz_internal"."mz_show_materialized_views"]␠("schema_id") -mz_show_roles_ind CREATE␠INDEX␠"mz_show_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s609␠AS␠"mz_internal"."mz_show_roles"]␠("name") -mz_show_schemas_ind CREATE␠INDEX␠"mz_show_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s605␠AS␠"mz_internal"."mz_show_schemas"]␠("database_id") -mz_show_secrets_ind CREATE␠INDEX␠"mz_show_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s603␠AS␠"mz_internal"."mz_show_secrets"]␠("schema_id") -mz_show_sinks_ind CREATE␠INDEX␠"mz_show_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s612␠AS␠"mz_internal"."mz_show_sinks"]␠("schema_id") -mz_show_sources_ind CREATE␠INDEX␠"mz_show_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s611␠AS␠"mz_internal"."mz_show_sources"]␠("schema_id") -mz_show_tables_ind CREATE␠INDEX␠"mz_show_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s606␠AS␠"mz_internal"."mz_show_tables"]␠("schema_id") -mz_show_types_ind CREATE␠INDEX␠"mz_show_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s608␠AS␠"mz_internal"."mz_show_types"]␠("schema_id") -mz_show_views_ind CREATE␠INDEX␠"mz_show_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s607␠AS␠"mz_internal"."mz_show_views"]␠("schema_id") -mz_sink_statistics_ind CREATE␠INDEX␠"mz_sink_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s729␠AS␠"mz_internal"."mz_sink_statistics"]␠("id",␠"replica_id") -mz_sink_status_history_ind CREATE␠INDEX␠"mz_sink_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s701␠AS␠"mz_internal"."mz_sink_status_history"]␠("sink_id") -mz_sink_statuses_ind CREATE␠INDEX␠"mz_sink_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s702␠AS␠"mz_internal"."mz_sink_statuses"]␠("id") -mz_sinks_ind CREATE␠INDEX␠"mz_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s496␠AS␠"mz_catalog"."mz_sinks"]␠("id") -mz_source_statistics_ind CREATE␠INDEX␠"mz_source_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s727␠AS␠"mz_internal"."mz_source_statistics"]␠("id",␠"replica_id") -mz_source_statistics_with_history_ind CREATE␠INDEX␠"mz_source_statistics_with_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s725␠AS␠"mz_internal"."mz_source_statistics_with_history"]␠("id",␠"replica_id") -mz_source_status_history_ind CREATE␠INDEX␠"mz_source_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s703␠AS␠"mz_internal"."mz_source_status_history"]␠("source_id") -mz_source_statuses_ind CREATE␠INDEX␠"mz_source_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s720␠AS␠"mz_internal"."mz_source_statuses"]␠("id") -mz_sources_ind CREATE␠INDEX␠"mz_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s489␠AS␠"mz_catalog"."mz_sources"]␠("id") -mz_tables_ind CREATE␠INDEX␠"mz_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s488␠AS␠"mz_catalog"."mz_tables"]␠("schema_id") -mz_types_ind CREATE␠INDEX␠"mz_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s498␠AS␠"mz_catalog"."mz_types"]␠("schema_id") -mz_views_ind CREATE␠INDEX␠"mz_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s497␠AS␠"mz_catalog"."mz_views"]␠("schema_id") -mz_wallclock_global_lag_recent_history_ind CREATE␠INDEX␠"mz_wallclock_global_lag_recent_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s736␠AS␠"mz_internal"."mz_wallclock_global_lag_recent_history"]␠("object_id") -mz_webhook_sources_ind CREATE␠INDEX␠"mz_webhook_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s542␠AS␠"mz_internal"."mz_webhook_sources"]␠("id") -pg_attrdef_all_databases_ind CREATE␠INDEX␠"pg_attrdef_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s641␠AS␠"mz_internal"."pg_attrdef_all_databases"]␠("oid",␠"adrelid",␠"adnum",␠"adbin",␠"adsrc") -pg_attribute_all_databases_ind CREATE␠INDEX␠"pg_attribute_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s634␠AS␠"mz_internal"."pg_attribute_all_databases"]␠("attrelid",␠"attname",␠"atttypid",␠"attlen",␠"attnum",␠"atttypmod",␠"attnotnull",␠"atthasdef",␠"attidentity",␠"attgenerated",␠"attisdropped",␠"attcollation",␠"database_name",␠"pg_type_database_name") -pg_authid_core_ind CREATE␠INDEX␠"pg_authid_core_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s651␠AS␠"mz_internal"."pg_authid_core"]␠("rolname") -pg_class_all_databases_ind CREATE␠INDEX␠"pg_class_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s622␠AS␠"mz_internal"."pg_class_all_databases"]␠("relname") -pg_description_all_databases_ind CREATE␠INDEX␠"pg_description_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s631␠AS␠"mz_internal"."pg_description_all_databases"]␠("objoid",␠"classoid",␠"objsubid",␠"description",␠"oid_database_name",␠"class_database_name") -pg_namespace_all_databases_ind CREATE␠INDEX␠"pg_namespace_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s619␠AS␠"mz_internal"."pg_namespace_all_databases"]␠("nspname") -pg_type_all_databases_ind CREATE␠INDEX␠"pg_type_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s628␠AS␠"mz_internal"."pg_type_all_databases"]␠("oid") +mz_schemas_ind CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s486␠AS␠"mz_catalog"."mz_schemas"]␠("database_id") +mz_secrets_ind CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s521␠AS␠"mz_catalog"."mz_secrets"]␠("name") +mz_show_all_objects_ind CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s602␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id") +mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s766␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") +mz_show_clusters_ind CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s604␠AS␠"mz_internal"."mz_show_clusters"]␠("name") +mz_show_columns_ind CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s603␠AS␠"mz_internal"."mz_show_columns"]␠("id") +mz_show_connections_ind CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s612␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id") +mz_show_databases_ind CREATE␠INDEX␠"mz_show_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s606␠AS␠"mz_internal"."mz_show_databases"]␠("name") +mz_show_indexes_ind CREATE␠INDEX␠"mz_show_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s616␠AS␠"mz_internal"."mz_show_indexes"]␠("schema_id") +mz_show_materialized_views_ind CREATE␠INDEX␠"mz_show_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s615␠AS␠"mz_internal"."mz_show_materialized_views"]␠("schema_id") +mz_show_roles_ind CREATE␠INDEX␠"mz_show_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s611␠AS␠"mz_internal"."mz_show_roles"]␠("name") +mz_show_schemas_ind CREATE␠INDEX␠"mz_show_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s607␠AS␠"mz_internal"."mz_show_schemas"]␠("database_id") +mz_show_secrets_ind CREATE␠INDEX␠"mz_show_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s605␠AS␠"mz_internal"."mz_show_secrets"]␠("schema_id") +mz_show_sinks_ind CREATE␠INDEX␠"mz_show_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s614␠AS␠"mz_internal"."mz_show_sinks"]␠("schema_id") +mz_show_sources_ind CREATE␠INDEX␠"mz_show_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s613␠AS␠"mz_internal"."mz_show_sources"]␠("schema_id") +mz_show_tables_ind CREATE␠INDEX␠"mz_show_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s608␠AS␠"mz_internal"."mz_show_tables"]␠("schema_id") +mz_show_types_ind CREATE␠INDEX␠"mz_show_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s610␠AS␠"mz_internal"."mz_show_types"]␠("schema_id") +mz_show_views_ind CREATE␠INDEX␠"mz_show_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s609␠AS␠"mz_internal"."mz_show_views"]␠("schema_id") +mz_sink_statistics_ind CREATE␠INDEX␠"mz_sink_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s731␠AS␠"mz_internal"."mz_sink_statistics"]␠("id",␠"replica_id") +mz_sink_status_history_ind CREATE␠INDEX␠"mz_sink_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s703␠AS␠"mz_internal"."mz_sink_status_history"]␠("sink_id") +mz_sink_statuses_ind CREATE␠INDEX␠"mz_sink_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s704␠AS␠"mz_internal"."mz_sink_statuses"]␠("id") +mz_sinks_ind CREATE␠INDEX␠"mz_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s498␠AS␠"mz_catalog"."mz_sinks"]␠("id") +mz_source_statistics_ind CREATE␠INDEX␠"mz_source_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s729␠AS␠"mz_internal"."mz_source_statistics"]␠("id",␠"replica_id") +mz_source_statistics_with_history_ind CREATE␠INDEX␠"mz_source_statistics_with_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s727␠AS␠"mz_internal"."mz_source_statistics_with_history"]␠("id",␠"replica_id") +mz_source_status_history_ind CREATE␠INDEX␠"mz_source_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s705␠AS␠"mz_internal"."mz_source_status_history"]␠("source_id") +mz_source_statuses_ind CREATE␠INDEX␠"mz_source_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s722␠AS␠"mz_internal"."mz_source_statuses"]␠("id") +mz_sources_ind CREATE␠INDEX␠"mz_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s491␠AS␠"mz_catalog"."mz_sources"]␠("id") +mz_tables_ind CREATE␠INDEX␠"mz_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s490␠AS␠"mz_catalog"."mz_tables"]␠("schema_id") +mz_types_ind CREATE␠INDEX␠"mz_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s500␠AS␠"mz_catalog"."mz_types"]␠("schema_id") +mz_views_ind CREATE␠INDEX␠"mz_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s499␠AS␠"mz_catalog"."mz_views"]␠("schema_id") +mz_wallclock_global_lag_recent_history_ind CREATE␠INDEX␠"mz_wallclock_global_lag_recent_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s738␠AS␠"mz_internal"."mz_wallclock_global_lag_recent_history"]␠("object_id") +mz_webhook_sources_ind CREATE␠INDEX␠"mz_webhook_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s544␠AS␠"mz_internal"."mz_webhook_sources"]␠("id") +pg_attrdef_all_databases_ind CREATE␠INDEX␠"pg_attrdef_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s643␠AS␠"mz_internal"."pg_attrdef_all_databases"]␠("oid",␠"adrelid",␠"adnum",␠"adbin",␠"adsrc") +pg_attribute_all_databases_ind CREATE␠INDEX␠"pg_attribute_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s636␠AS␠"mz_internal"."pg_attribute_all_databases"]␠("attrelid",␠"attname",␠"atttypid",␠"attlen",␠"attnum",␠"atttypmod",␠"attnotnull",␠"atthasdef",␠"attidentity",␠"attgenerated",␠"attisdropped",␠"attcollation",␠"database_name",␠"pg_type_database_name") +pg_authid_core_ind CREATE␠INDEX␠"pg_authid_core_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s653␠AS␠"mz_internal"."pg_authid_core"]␠("rolname") +pg_class_all_databases_ind CREATE␠INDEX␠"pg_class_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s624␠AS␠"mz_internal"."pg_class_all_databases"]␠("relname") +pg_description_all_databases_ind CREATE␠INDEX␠"pg_description_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s633␠AS␠"mz_internal"."pg_description_all_databases"]␠("objoid",␠"classoid",␠"objsubid",␠"description",␠"oid_database_name",␠"class_database_name") +pg_namespace_all_databases_ind CREATE␠INDEX␠"pg_namespace_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s621␠AS␠"mz_internal"."pg_namespace_all_databases"]␠("nspname") +pg_type_all_databases_ind CREATE␠INDEX␠"pg_type_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s630␠AS␠"mz_internal"."pg_type_all_databases"]␠("oid") # Record all transitive dependencies (tables, sources, views, mvs) of indexes on # the mz_catalog_server cluster. @@ -224,6 +225,16 @@ mz_builtin_materialized_views name mz_builtin_materialized_views oid mz_builtin_materialized_views privileges mz_builtin_materialized_views schema_name +mz_builtin_tables name +mz_builtin_tables oid +mz_builtin_tables privileges +mz_builtin_tables schema_name +mz_builtin_views create_sql +mz_builtin_views definition +mz_builtin_views name +mz_builtin_views oid +mz_builtin_views privileges +mz_builtin_views schema_name mz_catalog_raw data mz_cluster_auto_scaling_strategies cluster_id mz_cluster_auto_scaling_strategies state @@ -489,6 +500,8 @@ mz_map_types key_id mz_map_types key_modifiers mz_map_types value_id mz_map_types value_modifiers +mz_materialization_dependencies dependency_id +mz_materialization_dependencies object_id mz_materialized_views cluster_id mz_materialized_views create_sql mz_materialized_views definition @@ -532,6 +545,8 @@ mz_object_arrangement_sizes replica_id mz_object_arrangement_sizes size mz_object_dependencies object_id mz_object_dependencies referenced_object_id +mz_object_graph_edges dependency_id +mz_object_graph_edges object_id mz_object_history cluster_id mz_object_history created_at mz_object_history dropped_at diff --git a/test/sqllogictest/mz_tables.slt b/test/sqllogictest/mz_tables.slt new file mode 100644 index 0000000000000..487d2ade49020 --- /dev/null +++ b/test/sqllogictest/mz_tables.slt @@ -0,0 +1,223 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Behavioural tests for mz_catalog.mz_tables. It is a BuiltinMaterializedView +# over mz_internal.mz_catalog_raw (see MZ_TABLES in +# src/catalog/src/builtin/mz_catalog.rs) with two union branches, user tables +# and builtin tables, and it derives every column from the durable catalog JSON +# through parse_catalog_id, parse_catalog_privileges, redact_sql and +# parse_catalog_create_sql (src/expr/src/scalar/func/impls/jsonb.rs). +# +# Before that it was a BuiltinTable populated by pack_table_update in the +# adapter, so the values pinned here are a compatibility surface: +# information_schema.tables, pg_tables and pg_class all read through it. +# +# Assertions filter by object name rather than counting rows, so the file also +# holds under --auto-index-selects, which creates extra views. + +mode cockroach + +# Stable object ids, so create_sql can name dependencies by id. +reset-server + +# --- user_tables branch ------------------------------------------------------- + +# `t` is the first object created after the reset, so it is u1 and create_sql +# below can name it. +statement ok +CREATE TABLE t (a int, b text NOT NULL) + +query T +SELECT id FROM mz_tables WHERE name = 't' +---- +u1 + +query T +SELECT schema_id = (SELECT id FROM mz_schemas WHERE name = 'public' AND database_id IS NOT NULL) +FROM mz_tables WHERE name = 't' +---- +true + +query T +SELECT owner_id = (SELECT id FROM mz_roles WHERE name = 'materialize') +FROM mz_tables WHERE name = 't' +---- +true + +query T multiline +SELECT create_sql FROM mz_tables WHERE name = 't' +---- +CREATE TABLE "materialize"."public"."t" ("a" [s20 AS "pg_catalog"."int4"], "b" [s46 AS "pg_catalog"."text"] NOT NULL) +EOF + +query T multiline +SELECT redacted_create_sql FROM mz_tables WHERE name = 't' +---- +CREATE TABLE materialize.public.t (a [s20 AS pg_catalog.int4], b [s46 AS pg_catalog.text] NOT NULL) +EOF + +# A table with no source omits `source_id` from the parsed create_sql, so the +# column must be NULL rather than absent or empty. +query T +SELECT source_id IS NULL FROM mz_tables WHERE name = 't' +---- +true + +# --- Cross-check against the in-memory catalog -------------------------------- + +# The oid column is read out of the durable JSON, while a regclass cast resolves +# the name through the in-memory catalog. Two independent paths that must agree. +# +# NB create_sql cannot be compared against SHOW CREATE TABLE the same way: SHOW +# humanizes item ids back to names and pretty-prints, so it is deliberately not +# the stored string. +query T +SELECT oid = 't'::regclass::oid FROM mz_tables WHERE name = 't' +---- +true + +# --- source_id, for a table created from a source ----------------------------- + +statement ok +CREATE SOURCE lg FROM LOAD GENERATOR COUNTER + +statement ok +CREATE TABLE lg_tbl FROM SOURCE lg (REFERENCE counter) + +query T +SELECT source_id = (SELECT id FROM mz_sources WHERE name = 'lg') +FROM mz_tables WHERE name = 'lg_tbl' +---- +true + +# --- Temporary tables --------------------------------------------------------- + +# Temporary items are durable catalog items tagged with their owning session, +# parented to a sentinel schema id shared by every session. The MV maps that +# sentinel to schema_id '0', which is what the old SchemaSpecifier::Temporary +# populator printed (src/sql/src/names.rs). + +statement ok +CREATE TEMP TABLE tt (x int) + +query TTT +SELECT id LIKE 'u%', schema_id, create_sql IS NOT NULL FROM mz_tables WHERE name = 'tt' +---- +true 0 true + +# Two sessions may each hold a temporary table of the same name: name +# uniqueness is scoped by the owning session. Both rows show up here, since +# mz_tables reports every item and per-session visibility lives in name +# resolution, not in this MV. +simple conn=other +CREATE TEMP TABLE tt (y text); +---- +COMPLETE 0 + +query I +SELECT count(*) FROM mz_tables WHERE name = 'tt' +---- +2 + +query T +SELECT array_agg(DISTINCT schema_id) FROM mz_tables WHERE name = 'tt' +---- +{0} + +# --- builtin_tables branch ---------------------------------------------------- + +query TTTT +SELECT id LIKE 's%', owner_id, create_sql IS NULL, source_id IS NULL +FROM mz_tables WHERE name = 'mz_columns' +---- +true s1 true true + +query T +SELECT schema_id = (SELECT id FROM mz_schemas WHERE name = 'mz_catalog' AND database_id IS NULL) +FROM mz_tables WHERE name = 'mz_columns' +---- +true + +# Exactly once: the user and builtin branches must not both claim a row. +query I +SELECT count(*) FROM mz_tables WHERE name = 'mz_columns' +---- +1 + +# Every builtin table the catalog generates must be reported. +query T +SELECT array_agg(name ORDER BY name) FROM ( + SELECT name FROM mz_internal.mz_builtin_tables + EXCEPT + SELECT name FROM mz_tables WHERE id LIKE 's%' +) +---- +NULL + +# ...and nothing else: every builtin row traces back to the reporter. +query T +SELECT array_agg(name ORDER BY name) FROM ( + SELECT name FROM mz_tables WHERE id LIKE 's%' + EXCEPT + SELECT name FROM mz_internal.mz_builtin_tables +) +---- +NULL + +# The reporter view backing this MV is itself a builtin view, reported through +# mz_views with placeholder SQL. See mz_views.slt for the full contract. +query I +SELECT count(*) FROM mz_views WHERE name = 'mz_builtin_tables' +---- +1 + +# --- Robustness over the whole catalog ---------------------------------------- + +# The MV runs parse_catalog_create_sql over every Item row in the catalog, in +# its WHERE clause, so a single item whose create_sql the parser rejects makes +# the entire relation unreadable rather than dropping one row. Create one item +# of every kind reachable here and confirm the relation still resolves. + +statement ok +CREATE VIEW v AS SELECT a FROM t + +statement ok +CREATE MATERIALIZED VIEW mv AS SELECT count(*) FROM t + +statement ok +CREATE INDEX t_idx ON t (a) + +statement ok +CREATE TYPE ty AS LIST (ELEMENT TYPE = int4) + +statement ok +CREATE SECRET sec AS 'hunter2' + +query T +SELECT count(*) > 0 FROM mz_tables +---- +true + +# --- NOT NULL invariants ------------------------------------------------------ + +# The MV declares ASSERT NOT NULL for these columns. +query I +SELECT count(*) +FROM mz_tables +WHERE id IS NULL + OR oid IS NULL + OR schema_id IS NULL + OR name IS NULL + OR owner_id IS NULL + OR privileges IS NULL +---- +0 + +statement ok +DROP TABLE t CASCADE diff --git a/test/sqllogictest/mz_views.slt b/test/sqllogictest/mz_views.slt new file mode 100644 index 0000000000000..5fd82a0c8a6b5 --- /dev/null +++ b/test/sqllogictest/mz_views.slt @@ -0,0 +1,266 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Behavioural tests for mz_catalog.mz_views. It is a BuiltinMaterializedView +# over mz_internal.mz_catalog_raw (see MZ_VIEWS in +# src/catalog/src/builtin/mz_catalog.rs) with two union branches, user views and +# builtin views, and it derives every column from the durable catalog JSON +# through parse_catalog_id, parse_catalog_privileges, redact_sql and +# parse_catalog_create_sql (src/expr/src/scalar/func/impls/jsonb.rs). +# +# Before that it was a BuiltinTable populated by pack_view_update in the +# adapter, so the values pinned here are a compatibility surface. The +# `definition` column in particular is what pg_views.definition exposes. +# +# Assertions filter by object name rather than counting rows, so the file also +# holds under --auto-index-selects, which creates extra views. + +mode cockroach + +# Stable object ids, so create_sql can name dependencies by id. +reset-server + +# --- user_views branch -------------------------------------------------------- + +# `t` is the first object created after the reset, so it is u1 and the +# definitions below can name it. +statement ok +CREATE TABLE t (a int, b text NOT NULL) + +statement ok +CREATE VIEW v AS SELECT a, b FROM t WHERE a > 42 + +query T +SELECT schema_id = (SELECT id FROM mz_schemas WHERE name = 'public' AND database_id IS NOT NULL) +FROM mz_views WHERE name = 'v' +---- +true + +query T +SELECT owner_id = (SELECT id FROM mz_roles WHERE name = 'materialize') +FROM mz_views WHERE name = 'v' +---- +true + +# definition is the view's query rendered stably, with the trailing semicolon +# PostgreSQL puts in pg_views.definition. +query T multiline +SELECT definition FROM mz_views WHERE name = 'v' +---- +SELECT "a", "b" FROM [u1 AS "materialize"."public"."t"] WHERE "a" > 42; +EOF + +query T multiline +SELECT create_sql FROM mz_views WHERE name = 'v' +---- +CREATE VIEW "materialize"."public"."v" AS SELECT "a", "b" FROM [u1 AS "materialize"."public"."t"] WHERE "a" > 42 +EOF + +# Redaction replaces literals but keeps structure and resolves quoting. +query T multiline +SELECT redacted_create_sql FROM mz_views WHERE name = 'v' +---- +CREATE VIEW materialize.public.v AS SELECT a, b FROM [u1 AS materialize.public.t] WHERE a > '' +EOF + +# --- Cross-check against the in-memory catalog -------------------------------- + +# The oid column is read out of the durable JSON, while a regclass cast resolves +# the name through the in-memory catalog. Two independent paths that must agree. +# +# NB create_sql cannot be compared against SHOW CREATE VIEW the same way: SHOW +# humanizes item ids back to names and pretty-prints, so it is deliberately not +# the stored string. +query T +SELECT oid = 'v'::regclass::oid FROM mz_views WHERE name = 'v' +---- +true + +# definition must be a fixed point: consumers re-issue it as the body of a new +# view, so planning it again and rendering it again has to reproduce it exactly. +# This goes out through the planner and back through the durable catalog, so it +# is an end-to-end check on the rendering rather than a self-comparison. +statement ok +CREATE VIEW v_roundtrip AS SELECT "a", "b" FROM [u1 AS "materialize"."public"."t"] WHERE "a" > 42; + +query T +SELECT (SELECT definition FROM mz_views WHERE name = 'v') + = (SELECT definition FROM mz_views WHERE name = 'v_roundtrip') +---- +true + +# --- Temporary views ---------------------------------------------------------- + +# Temporary items are durable catalog items tagged with their owning session, +# parented to a sentinel schema id shared by every session. The MV maps that +# sentinel to schema_id '0', which is what the old SchemaSpecifier::Temporary +# populator printed (src/sql/src/names.rs). + +statement ok +CREATE TEMP VIEW tv AS SELECT a FROM t + +query TTT +SELECT id LIKE 'u%', schema_id, create_sql IS NOT NULL FROM mz_views WHERE name = 'tv' +---- +true 0 true + +query T multiline +SELECT definition FROM mz_views WHERE name = 'tv' +---- +SELECT "a" FROM [u1 AS "materialize"."public"."t"]; +EOF + +# Two sessions may each hold a temporary view of the same name: name uniqueness +# is scoped by the owning session. Both rows show up here, since mz_views +# reports every item and per-session visibility lives in name resolution, not in +# this MV. +simple conn=other +CREATE TEMP VIEW tv AS SELECT 1 AS other; +---- +COMPLETE 0 + +query I +SELECT count(*) FROM mz_views WHERE name = 'tv' +---- +2 + +query T +SELECT array_agg(DISTINCT schema_id) FROM mz_views WHERE name = 'tv' +---- +{0} + +# --- builtin_views branch ----------------------------------------------------- + +query TTT +SELECT id LIKE 's%', owner_id, create_sql IS NOT NULL +FROM mz_views WHERE name = 'mz_objects' +---- +true s1 true + +query T +SELECT schema_id = (SELECT id FROM mz_schemas WHERE name = 'mz_catalog' AND database_id IS NULL) +FROM mz_views WHERE name = 'mz_objects' +---- +true + +# Exactly once: the user and builtin branches must not both claim a row. +query I +SELECT count(*) FROM mz_views WHERE name = 'mz_objects' +---- +1 + +# Every builtin view the catalog generates must be reported. +query T +SELECT array_agg(name ORDER BY name) FROM ( + SELECT name FROM mz_internal.mz_builtin_views + EXCEPT + SELECT name FROM mz_views WHERE id LIKE 's%' +) +---- +NULL + +# ...and nothing else: every builtin row traces back to the reporter. +query T +SELECT array_agg(name ORDER BY name) FROM ( + SELECT name FROM mz_views WHERE id LIKE 's%' + EXCEPT + SELECT name FROM mz_internal.mz_builtin_views +) +---- +NULL + +# --- Generated reporter views --------------------------------------------------- + +# The builtin branch reads mz_internal.mz_builtin_views, a generated constant +# view that lists every builtin view, including itself and the other generated +# mz_builtin_* views. The generated views are listed with a short placeholder +# instead of their real SQL: mz_builtin_views cannot textually contain itself, +# and the other reporters embed metadata about every builtin object, so +# re-embedding their SQL would produce enormous rows. + +# The placeholder rows are exactly the generated views, and nothing else is +# elided. +query T +SELECT name FROM mz_views WHERE definition LIKE '%definition elided%' AND id LIKE 's%' ORDER BY name +---- +mz_builtin_materialized_views +mz_builtin_sources +mz_builtin_tables +mz_builtin_views + +query T multiline +SELECT definition FROM mz_views WHERE name = 'mz_builtin_views' +---- +SELECT ''; +EOF + +query T multiline +SELECT create_sql FROM mz_views WHERE name = 'mz_builtin_views' +---- +CREATE VIEW "mz_internal"."mz_builtin_views" AS SELECT '' +EOF + +# The placeholder must stay parseable: redact_sql errors on unparseable input, +# and an error would poison the whole materialized view, not just this row. +query T multiline +SELECT redacted_create_sql FROM mz_views WHERE name = 'mz_builtin_views' +---- +CREATE VIEW mz_internal.mz_builtin_views AS SELECT '' +EOF + +# --- Robustness over the whole catalog ---------------------------------------- + +# The MV runs parse_catalog_create_sql over every Item row in the catalog, in +# its WHERE clause, so a single item whose create_sql the parser rejects makes +# the entire relation unreadable rather than dropping one row. Create one item +# of every kind reachable here and confirm the relation still resolves. + +statement ok +CREATE MATERIALIZED VIEW mv AS SELECT count(*) FROM t + +statement ok +CREATE INDEX t_idx ON t (a) + +statement ok +CREATE TYPE ty AS LIST (ELEMENT TYPE = int4) + +statement ok +CREATE SECRET sec AS 'hunter2' + +statement ok +CREATE SOURCE lg FROM LOAD GENERATOR COUNTER + +statement ok +CREATE TABLE lg_tbl FROM SOURCE lg (REFERENCE counter) + +query T +SELECT count(*) > 0 FROM mz_views +---- +true + +# --- NOT NULL invariants ------------------------------------------------------ + +# The MV declares ASSERT NOT NULL for every column. +query I +SELECT count(*) +FROM mz_views +WHERE id IS NULL + OR oid IS NULL + OR schema_id IS NULL + OR name IS NULL + OR definition IS NULL + OR owner_id IS NULL + OR privileges IS NULL + OR create_sql IS NULL + OR redacted_create_sql IS NULL +---- +0 + +statement ok +DROP TABLE t CASCADE diff --git a/test/sqllogictest/object_ownership.slt b/test/sqllogictest/object_ownership.slt index fb332383cf59d..531367a9b063d 100644 --- a/test/sqllogictest/object_ownership.slt +++ b/test/sqllogictest/object_ownership.slt @@ -2159,7 +2159,7 @@ db error: ERROR: cannot alter item mz_introspection.mz_dataflow_operators becaus simple conn=mz_system,user=mz_system ALTER TABLE mz_views OWNER TO mz_system ---- -db error: ERROR: cannot alter item mz_catalog.mz_views because it is required by the database system +db error: ERROR: mz_views is a materialized view not a table simple conn=mz_system,user=mz_system ALTER VIEW mz_relations OWNER TO mz_system diff --git a/test/sqllogictest/oid.slt b/test/sqllogictest/oid.slt index a563054773926..1d99040fdb6d9 100644 --- a/test/sqllogictest/oid.slt +++ b/test/sqllogictest/oid.slt @@ -1245,3 +1245,7 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17113 mz_aws_account_id 17114 mz_aws_external_id_prefix 17115 mz_aws_connection_role_arn +17116 mz_object_graph_edges +17117 mz_object_graph_edges_ind +17118 mz_builtin_tables +17119 mz_builtin_views diff --git a/test/sqllogictest/show_create_system_objects.slt b/test/sqllogictest/show_create_system_objects.slt index c63218213bd9e..d5d5d466ad6c8 100644 --- a/test/sqllogictest/show_create_system_objects.slt +++ b/test/sqllogictest/show_create_system_objects.slt @@ -12,5 +12,5 @@ mode cockroach query error cannot show create for system object mz_internal\.mz_source_statistics SHOW CREATE SOURCE mz_internal.mz_source_statistics -query error cannot show create for system object mz_catalog\.mz_tables -SHOW CREATE TABLE mz_tables +query error cannot show create for system object mz_catalog\.mz_columns +SHOW CREATE TABLE mz_columns diff --git a/test/testdrive-old-kafka-src-syntax/top-1-monotonic.td b/test/testdrive-old-kafka-src-syntax/top-1-monotonic.td index 203471e5eada5..c05dd0cc6c4c8 100644 --- a/test/testdrive-old-kafka-src-syntax/top-1-monotonic.td +++ b/test/testdrive-old-kafka-src-syntax/top-1-monotonic.td @@ -314,3 +314,35 @@ $ kafka-ingest format=avro topic=top1 schema=${schema} timestamp=5 "Dataflow: order_by_desc_limit" MonotonicTop1 "Dataflow: order_by_limit" "Arranged MonotonicTop1 partial [val: empty]" "Dataflow: order_by_limit" MonotonicTop1 +"Dataflow: group_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_in_top_1" "Arrange errors" +"Dataflow: group_by_in_top_1" "Distinct errors" +"Dataflow: group_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: group_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: group_by_limit" "Arrange bundle err" +"Dataflow: group_by_limit" "Distinct errors[[]]" +"Dataflow: group_by_order_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_order_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_order_by_in_top_1" "Arrange errors" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: limit_only" "Arrange bundle err" +"Dataflow: limit_only" "Distinct errors[[]]" +"Dataflow: order_by_desc_limit" "Arrange bundle err" +"Dataflow: order_by_desc_limit" "Distinct errors[[]]" +"Dataflow: order_by_limit" "Arrange bundle err" +"Dataflow: order_by_limit" "Distinct errors[[]]" +"Dataflow: group_by_in_top_1" "Arrange errors" +"Dataflow: group_by_in_top_1" "Distinct errors" +"Dataflow: group_by_limit" "Arrange errors" +"Dataflow: group_by_limit" "Distinct errors" +"Dataflow: group_by_order_by_in_top_1" "Arrange errors" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors" +"Dataflow: limit_only" "Arrange errors" +"Dataflow: limit_only" "Distinct errors" +"Dataflow: order_by_desc_limit" "Arrange errors" +"Dataflow: order_by_desc_limit" "Distinct errors" +"Dataflow: order_by_limit" "Arrange errors" +"Dataflow: order_by_limit" "Distinct errors" diff --git a/test/testdrive-old-kafka-src-syntax/update.td b/test/testdrive-old-kafka-src-syntax/update.td index 7038ec7920bca..33339603145e0 100644 --- a/test/testdrive-old-kafka-src-syntax/update.td +++ b/test/testdrive-old-kafka-src-syntax/update.td @@ -76,7 +76,7 @@ contains:invalid input syntax for type double precision contains:cannot mutate materialized view ! UPDATE mz_tables SET a = 1 -contains:cannot mutate system table +contains:cannot mutate materialized view 'mz_catalog.mz_tables' ! UPDATE t SET a = 1 contains:unknown column a diff --git a/test/testdrive/catalog.td b/test/testdrive/catalog.td index 336e31057b87a..1ecf361507c03 100644 --- a/test/testdrive/catalog.td +++ b/test/testdrive/catalog.td @@ -522,9 +522,7 @@ mz_map_types "" mz_operators "" mz_pseudo_types "" mz_role_auth "" -mz_tables "" mz_types "" -mz_views "" > SHOW VIEWS FROM mz_catalog name comment @@ -561,6 +559,8 @@ mz_sources mz_catalog_server "" mz_ssh_tunnel_connections mz_catalog_server "" mz_system_privileges mz_catalog_server "" mz_materialized_views mz_catalog_server "" +mz_tables mz_catalog_server "" +mz_views mz_catalog_server "" # Check default sources, tables, and views in mz_catalog_unstable. @@ -625,6 +625,8 @@ name comment mz_activity_log_thinned "" mz_builtin_materialized_views "" mz_builtin_sources "" +mz_builtin_tables "" +mz_builtin_views "" mz_cluster_deployment_lineage "" mz_cluster_replica_history "" mz_cluster_replica_metrics "" @@ -646,6 +648,7 @@ mz_mcp_data_products "" mz_notices "" mz_notices_redacted "" mz_object_fully_qualified_names "" +mz_object_graph_edges "" mz_object_history "" mz_object_lifetimes "" mz_object_oid_alias "" @@ -830,11 +833,11 @@ test_table "" # `SHOW TABLES` and `mz_tables` should agree. > SELECT COUNT(*) FROM mz_tables WHERE id LIKE 's%' -31 +29 # There is one entry in mz_indexes for each field_number/expression of the index. > SELECT COUNT(id) FROM mz_indexes WHERE id LIKE 's%' -272 +273 # Create a second schema with the same table name as above > CREATE SCHEMA tester2 diff --git a/test/testdrive/connection-create-drop.td b/test/testdrive/connection-create-drop.td index 0cbe43171f76d..3a2f7dbffe97f 100644 --- a/test/testdrive/connection-create-drop.td +++ b/test/testdrive/connection-create-drop.td @@ -532,6 +532,9 @@ contains: BROKERS must contain at least one static broker address > DROP CONNECTION kafka_matching_ok +! ALTER CONNECTION pl_for_matching SET (SERVICE NAME 'my-db-lb-0123456789abcdef.elb.eu-central-1.amazonaws.com') +contains: invalid AWS PrivateLink service name "my-db-lb-0123456789abcdef.elb.eu-central-1.amazonaws.com" + $ postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr} ALTER SYSTEM RESET enable_kafka_broker_matching_rules; ALTER SYSTEM RESET max_aws_privatelink_connections; @@ -549,3 +552,17 @@ contains: unknown catalog item 'foo' AVAILABILITY ZONES ('use1-az1', 'use1-az2') ) contains: AWS PrivateLink + +# SERVICE NAME must name a VPC endpoint service, not the DNS name of the target. +! CREATE CONNECTION privatelink_elb_hostname TO AWS PRIVATELINK ( + SERVICE NAME 'my-db-lb-0123456789abcdef.elb.eu-central-1.amazonaws.com', + AVAILABILITY ZONES () + ) +contains: invalid AWS PrivateLink service name "my-db-lb-0123456789abcdef.elb.eu-central-1.amazonaws.com" +hint: SERVICE NAME must name an AWS VPC endpoint service + +! CREATE CONNECTION privatelink_not_a_service_name TO AWS PRIVATELINK ( + SERVICE NAME 'vpce-svc-0e123abc123198abc', + AVAILABILITY ZONES () + ) +contains: invalid AWS PrivateLink service name "vpce-svc-0e123abc123198abc" diff --git a/test/testdrive/dataflow-dependencies.td b/test/testdrive/dataflow-dependencies.td index f8be9eb9b8e65..d5d715d3902b9 100644 --- a/test/testdrive/dataflow-dependencies.td +++ b/test/testdrive/dataflow-dependencies.td @@ -199,3 +199,61 @@ snk_cleanup t1 LEFT JOIN mz_objects object ON dep.object_id = object.id LEFT JOIN mz_objects import ON dep.dependency_id = import.id WHERE object_id LIKE 'u%' + +# Test `mz_internal.mz_object_graph_edges`, the console workflow-graph edge set: +# the dependencies between maintained objects, plus the edges that stitch a +# source to the subsources/tables carrying its data. Like the relations above it +# is updated asynchronously, so these rely on testdrive's retry feature. + +> CREATE TABLE base (a int) +> CREATE VIEW unmaintained AS SELECT * FROM base +> CREATE MATERIALIZED VIEW mvg AS SELECT * FROM unmaintained +> CREATE DEFAULT INDEX ON mvg +> CREATE SINK snk_graph + IN CLUSTER ${arg.single-replica-cluster} + FROM mvg + INTO KAFKA CONNECTION kafka_conn (TOPIC 'testdrive-sink-graph-${testdrive.seed}') + FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_conn + ENVELOPE DEBEZIUM + +# The intermediate view is not a maintained object, so it carries no edge: the +# chain is index -> mvg -> base, and sink -> mvg alongside it. +> SELECT object.name, dependency.name + FROM mz_internal.mz_object_graph_edges e + JOIN mz_objects object ON e.object_id = object.id + JOIN mz_objects dependency ON e.dependency_id = dependency.id + WHERE object.name IN ('mvg', 'mvg_primary_idx', 'snk_graph') + ORDER BY object.name, dependency.name +mvg base +mvg_primary_idx mvg +snk_graph mvg + +> CREATE SOURCE src_sub IN CLUSTER ${arg.single-replica-cluster} FROM LOAD GENERATOR AUCTION FOR ALL TABLES +> CREATE MATERIALIZED VIEW mv_bids AS SELECT count(*) FROM bids + +# `FOR ALL TABLES` creates subsources, which are stitched to their parent source +# (bids -> src_sub), an edge that mz_materialization_dependencies does not carry, +# alongside the maintained dependency mv_bids -> bids. +> SELECT object.name, dependency.name + FROM mz_internal.mz_object_graph_edges e + JOIN mz_objects object ON e.object_id = object.id + JOIN mz_objects dependency ON e.dependency_id = dependency.id + WHERE object.name IN ('bids', 'mv_bids') + ORDER BY object.name, dependency.name +bids src_sub +mv_bids bids + +> CREATE SOURCE src_tbl IN CLUSTER ${arg.single-replica-cluster} FROM LOAD GENERATOR AUCTION (UP TO 100) +> CREATE TABLE bids_tbl FROM SOURCE src_tbl (REFERENCE bids) + +# A table fed by a source is stitched to it through `mz_tables.source_id`. +> SELECT object.name, dependency.name + FROM mz_internal.mz_object_graph_edges e + JOIN mz_objects object ON e.object_id = object.id + JOIN mz_objects dependency ON e.dependency_id = dependency.id + WHERE object.name = 'bids_tbl' +bids_tbl src_tbl + +> DROP SOURCE src_sub CASCADE +> DROP SOURCE src_tbl CASCADE +> DROP TABLE base CASCADE diff --git a/test/testdrive/indexes.td b/test/testdrive/indexes.td index 8c9dc0dc73b3c..d54a1752031be 100644 --- a/test/testdrive/indexes.td +++ b/test/testdrive/indexes.td @@ -345,6 +345,7 @@ mz_object_arrangement_size_history_object_ind mz_object_arrangemen mz_object_arrangement_size_history_ts_ind mz_object_arrangement_size_history mz_catalog_server {collection_timestamp} "" mz_object_arrangement_sizes_ind mz_object_arrangement_sizes mz_catalog_server {replica_id} "" mz_object_dependencies_ind mz_object_dependencies mz_catalog_server {object_id} "" +mz_object_graph_edges_ind mz_object_graph_edges mz_catalog_server {object_id} "" mz_object_history_ind mz_object_history mz_catalog_server {id} "" mz_object_lifetimes_ind mz_object_lifetimes mz_catalog_server {id} "" mz_object_transitive_dependencies_ind mz_object_transitive_dependencies mz_catalog_server {object_id} "" diff --git a/test/testdrive/introspection-sources.td b/test/testdrive/introspection-sources.td index 1300e662974f6..d836ae00ba8d4 100644 --- a/test/testdrive/introspection-sources.td +++ b/test/testdrive/introspection-sources.td @@ -489,7 +489,7 @@ mv_div_by_zero 1 JOIN mz_objects o ON (c.export_id = o.id) ORDER BY name idx1_div_by_zero 3 -idx2_div_by_zero 3 +idx2_div_by_zero 1 mv_div_by_zero 3 > DELETE FROM zeros @@ -508,7 +508,7 @@ mv_div_by_zero 3 JOIN mz_objects o ON (c.export_id = o.id) ORDER BY name idx1_div_by_zero 2 -idx2_div_by_zero 2 +idx2_div_by_zero 1 mv_div_by_zero 2 > DROP INDEX idx1_div_by_zero @@ -517,7 +517,7 @@ mv_div_by_zero 2 FROM mz_introspection.mz_compute_error_counts c JOIN mz_objects o ON (c.export_id = o.id) ORDER BY name -idx2_div_by_zero 2 +idx2_div_by_zero 1 mv_div_by_zero 2 > DROP MATERIALIZED VIEW mv_div_by_zero diff --git a/test/testdrive/kafka-avro-upsert-sinks.td b/test/testdrive/kafka-avro-upsert-sinks.td index b7d7b347a3393..c2bf9965ce7a2 100644 --- a/test/testdrive/kafka-avro-upsert-sinks.td +++ b/test/testdrive/kafka-avro-upsert-sinks.td @@ -447,3 +447,9 @@ contains:upsert key could not be validated as unique "Dataflow: materialize.public.input_with_deletions_keyed" "Reduced Fallibly MinsMaxesHierarchical" "Dataflow: materialize.public.input_with_deletions_keyed" "Reduced Fallibly MinsMaxesHierarchical" "Dataflow: materialize.public.input_with_deletions_keyed" "Reduced Fallibly MinsMaxesHierarchical" +"Dataflow: materialize.public.input_keyed" "Arrange errors" +"Dataflow: materialize.public.input_keyed" "Distinct errors" +"Dataflow: materialize.public.input_keyed_ab" "Arrange errors" +"Dataflow: materialize.public.input_keyed_ab" "Distinct errors" +"Dataflow: materialize.public.input_with_deletions_keyed" "Arrange errors" +"Dataflow: materialize.public.input_with_deletions_keyed" "Distinct errors" diff --git a/test/testdrive/materializations.td b/test/testdrive/materializations.td index 8fe17b5c20289..38280d337ba54 100644 --- a/test/testdrive/materializations.td +++ b/test/testdrive/materializations.td @@ -496,3 +496,15 @@ c d "Dataflow: materialize.public.test7" AccumulableErrorCheck "Dataflow: materialize.public.test7" "ArrangeAccumulable [val: empty]" "Dataflow: materialize.public.test7" ReduceAccumulable +"Dataflow: materialize.public.test1" "Arrange errors" +"Dataflow: materialize.public.test1" "Distinct errors" +"Dataflow: materialize.public.test2" "Arrange errors" +"Dataflow: materialize.public.test2" "Distinct errors" +"Dataflow: materialize.public.test4" "Arrange errors" +"Dataflow: materialize.public.test4" "Distinct errors" +"Dataflow: materialize.public.test6" "Arrange errors" +"Dataflow: materialize.public.test6" "Distinct errors" +"Dataflow: materialize.public.test7" "Arrange bundle err" +"Dataflow: materialize.public.test7" "Arrange errors" +"Dataflow: materialize.public.test7" "Distinct errors" +"Dataflow: materialize.public.test7" "Distinct errors[[]]" diff --git a/test/testdrive/monotonic.td b/test/testdrive/monotonic.td index 7428d3de28feb..eea687c6ea778 100644 --- a/test/testdrive/monotonic.td +++ b/test/testdrive/monotonic.td @@ -243,3 +243,11 @@ Target cluster: quickstart "Dataflow: materialize.public.i8_primary_idx" "ArrangeBy[[Column(0, \"b\")]]-errors" "Dataflow: materialize.public.i9_primary_idx" "ArrangeBy[[Column(0, \"b\")]]" "Dataflow: materialize.public.i9_primary_idx" "ArrangeBy[[Column(0, \"b\")]]-errors" +"Dataflow: materialize.public.m1" "Arrange errors" +"Dataflow: materialize.public.m1" "Distinct errors" +"Dataflow: materialize.public.monotonic_fused" "Arrange errors" +"Dataflow: materialize.public.monotonic_fused" "Distinct errors" +"Dataflow: materialize.public.monotonic_max" "Arrange errors" +"Dataflow: materialize.public.monotonic_max" "Distinct errors" +"Dataflow: materialize.public.monotonic_min" "Arrange errors" +"Dataflow: materialize.public.monotonic_min" "Distinct errors" diff --git a/test/testdrive/temporary.td b/test/testdrive/temporary.td index 4a4e7e2e615e5..70e23e45a895d 100644 --- a/test/testdrive/temporary.td +++ b/test/testdrive/temporary.td @@ -241,13 +241,13 @@ contains:unknown catalog item 'temp_v' contains:cannot create temporary item in non-temporary schema # Regression test: a persistent (non-temporary) item must not be allowed to -# depend on a TEMPORARY object. Temporary objects live only in the in-memory, -# session-scoped catalog and are never persisted, so a persisted item that -# references one is a dangling catalog entry. The coordinator applies durable -# items before temporary ones (it assumes the invariant holds, see -# apply.rs), so on the next catalog rebuild deserializing the persisted item -# fails to resolve the temporary id and panics with -# PlanError(InvalidId(...)): invalid persisted SQL. +# depend on a TEMPORARY object. A temporary object is durable, but it is owned +# by the session that created it and is reclaimed when that session ends, or at +# the next writable catalog open if the process died first. A persisted item +# referencing one therefore becomes a dangling catalog entry as soon as the +# owning session goes away, and on the next catalog rebuild deserializing it +# fails to resolve the id and panics with PlanError(InvalidId(...)): invalid +# persisted SQL. # # CREATE enforces this via ErrorKind::InvalidTemporaryDependency. ALTER SINK # ... SET FROM went through a separate catalog path that skipped the check, diff --git a/test/testdrive/top-1-monotonic.td b/test/testdrive/top-1-monotonic.td index 13eecd6fdb4bc..ddd2d4376bfc9 100644 --- a/test/testdrive/top-1-monotonic.td +++ b/test/testdrive/top-1-monotonic.td @@ -311,3 +311,35 @@ $ kafka-ingest format=avro topic=top1 schema=${schema} timestamp=5 "Dataflow: group_by_order_by_in_top_1" "Arranged DistinctBy" "Dataflow: group_by_order_by_in_top_1" DistinctBy "Dataflow: group_by_order_by_in_top_1" DistinctByErrorCheck +"Dataflow: group_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_in_top_1" "Arrange errors" +"Dataflow: group_by_in_top_1" "Distinct errors" +"Dataflow: group_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: group_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: group_by_limit" "Arrange bundle err" +"Dataflow: group_by_limit" "Distinct errors[[]]" +"Dataflow: group_by_order_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_order_by_in_top_1" "Arrange bundle err" +"Dataflow: group_by_order_by_in_top_1" "Arrange errors" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors[[Column(0)]]" +"Dataflow: limit_only" "Arrange bundle err" +"Dataflow: limit_only" "Distinct errors[[]]" +"Dataflow: order_by_desc_limit" "Arrange bundle err" +"Dataflow: order_by_desc_limit" "Distinct errors[[]]" +"Dataflow: order_by_limit" "Arrange bundle err" +"Dataflow: order_by_limit" "Distinct errors[[]]" +"Dataflow: group_by_in_top_1" "Arrange errors" +"Dataflow: group_by_in_top_1" "Distinct errors" +"Dataflow: group_by_limit" "Arrange errors" +"Dataflow: group_by_limit" "Distinct errors" +"Dataflow: group_by_order_by_in_top_1" "Arrange errors" +"Dataflow: group_by_order_by_in_top_1" "Distinct errors" +"Dataflow: limit_only" "Arrange errors" +"Dataflow: limit_only" "Distinct errors" +"Dataflow: order_by_desc_limit" "Arrange errors" +"Dataflow: order_by_desc_limit" "Distinct errors" +"Dataflow: order_by_limit" "Arrange errors" +"Dataflow: order_by_limit" "Distinct errors" diff --git a/test/testdrive/unified-compute-introspection.td b/test/testdrive/unified-compute-introspection.td index 9e23821a8caec..8b8ae48ba5017 100644 --- a/test/testdrive/unified-compute-introspection.td +++ b/test/testdrive/unified-compute-introspection.td @@ -52,7 +52,7 @@ mv_div_by_zero r1 1 JOIN mz_cluster_replicas r ON c.replica_id = r.id WHERE c.count != 0 idx1_div_by_zero r1 3 -idx2_div_by_zero r1 3 +idx2_div_by_zero r1 1 mv_div_by_zero r1 3 > DELETE FROM zeros @@ -73,7 +73,7 @@ mv_div_by_zero r1 3 JOIN mz_cluster_replicas r ON c.replica_id = r.id WHERE c.count != 0 idx1_div_by_zero r1 2 -idx2_div_by_zero r1 2 +idx2_div_by_zero r1 1 mv_div_by_zero r1 2 > ALTER CLUSTER test SET (REPLICATION FACTOR 2) @@ -85,8 +85,8 @@ mv_div_by_zero r1 2 WHERE c.count != 0 idx1_div_by_zero r1 2 idx1_div_by_zero r2 2 -idx2_div_by_zero r1 2 -idx2_div_by_zero r2 2 +idx2_div_by_zero r1 1 +idx2_div_by_zero r2 1 mv_div_by_zero r1 2 mv_div_by_zero r2 2 @@ -100,7 +100,7 @@ mv_div_by_zero r2 2 JOIN mz_cluster_replicas r ON c.replica_id = r.id WHERE c.count != 0 idx1_div_by_zero r1 2 -idx2_div_by_zero r1 2 +idx2_div_by_zero r1 1 mv_div_by_zero r1 2 # Test that error logging is retracted when objects are dropped. @@ -112,7 +112,7 @@ mv_div_by_zero r1 2 JOIN mz_objects o ON c.object_id = o.id JOIN mz_cluster_replicas r ON c.replica_id = r.id WHERE c.count != 0 -idx2_div_by_zero r1 2 +idx2_div_by_zero r1 1 mv_div_by_zero r1 2 > DROP MATERIALIZED VIEW mv_div_by_zero diff --git a/test/testdrive/update.td b/test/testdrive/update.td index 1bcb28cbcf16d..8ff37126e8a33 100644 --- a/test/testdrive/update.td +++ b/test/testdrive/update.td @@ -76,7 +76,7 @@ contains:invalid input syntax for type double precision contains:cannot mutate materialized view ! UPDATE mz_tables SET a = 1 -contains:cannot mutate system table +contains:cannot mutate materialized view 'mz_catalog.mz_tables' ! UPDATE t SET a = 1 contains:unknown column a