diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b6627bc720ee0..17fb47ecb2f8b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -97,6 +97,8 @@ updates: - /misc/images/openssh-static - /misc/images/tini-static - /misc/images/distroless-prod-base + - /misc/mcp-materialize + - /misc/mcp-materialize-agents - /ci/builder - /test schedule: diff --git a/AGENTS.md b/AGENTS.md index bffc572450387..a7beb5751ec78 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -157,4 +157,27 @@ works goes in an inline `//` at the decision point, or in a module-level `//!` when it is about how the pieces fit together. Document a struct field only when its meaning is subtle. +Every fact gets exactly one owning comment: the decision point for reasoning, +the mechanism for mechanism docs, the public setter or constant for config +semantics. Everywhere else either points at the owner or says nothing. The +tell is the same clause appearing near-verbatim in a module doc, a struct doc, +and an inline comment; when that happens, pick the owner and cut the rest. In +particular: don't restate a callee's documented contract at its call sites, +don't narrate in one function's doc what a sibling's doc already owns, and +don't duplicate an assert message in a comment on the same line. + +Budget doc-comment paragraphs by decisions: each paragraph should carry a +distinct decision or invariant, not re-argue one documented elsewhere or +enumerate the item's callers or fields. Skip speculative comments about what +a future implementation could do (one `TODO` at the owning seam is enough), +and skip comments explaining an absence when the surrounding docs already +imply it. No ASCII section-divider banners (`// ----- helpers -----`); item +placement carries the structure. Performance claims in comments should state +a constraint or a measured number, not unverifiable color ("the compiler can +autovectorize this"). + +The same economy applies to tests: a test whose name and assert messages +state the property needs no doc comment. Keep test docs for non-obvious +setup, fixtures whose shape encodes the scenario, and multi-phase protocols. + Mark counterintuitive gotchas with `NOTE:` and future work with `TODO:`. diff --git a/Cargo.lock b/Cargo.lock index ce9398cd42d20..6b1758610a670 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6797,6 +6797,7 @@ version = "0.0.0" dependencies = [ "anyhow", "async-trait", + "bincode", "bytesize", "chrono", "derivative", @@ -7487,10 +7488,12 @@ dependencies = [ name = "mz-metrics" version = "0.0.0" dependencies = [ + "anyhow", "lgalloc", "libc", "mz-dyncfg", "mz-ore", + "nix 0.31.3", "paste", "prometheus", "thiserror 2.0.18", @@ -8569,6 +8572,7 @@ dependencies = [ "md-5", "mz-adapter-types", "mz-catalog", + "mz-compute-types", "mz-controller", "mz-dyncfgs", "mz-environmentd", diff --git a/doc/user/content/headless/sql-command-privileges/create-table.md b/doc/user/content/headless/sql-command-privileges/create-table.md index c0f97fdb916f2..505b7a1c8e55c 100644 --- a/doc/user/content/headless/sql-command-privileges/create-table.md +++ b/doc/user/content/headless/sql-command-privileges/create-table.md @@ -5,3 +5,7 @@ headless: true - `USAGE` privileges on all types used in the table definition. - `USAGE` privileges on the schemas that all types in the statement are contained in. +- For `CREATE TABLE ... FROM SOURCE`, `SELECT` privileges on the source and + `USAGE` privileges on its schema. `SELECT` on a source permits attaching any + reference that source ingests, so grant it only to roles that should be able + to read all of the source's data. diff --git a/doc/user/content/manage/monitor/replica-resource-usage.md b/doc/user/content/manage/monitor/replica-resource-usage.md new file mode 100644 index 0000000000000..328fcd8a626ea --- /dev/null +++ b/doc/user/content/manage/monitor/replica-resource-usage.md @@ -0,0 +1,129 @@ +--- +title: "Replica resource usage" +description: "The per-process memory, swap and disk observations a cluster replica reports about itself, and how to interpret them." +menu: + main: + parent: "monitor" + identifier: "monitor-replica-resource-usage" + weight: 18 +--- + +Every process of a cluster replica reports its own resource usage through +[`mz_introspection.mz_cluster_replica_resource_usage`](/reference/system-catalog/mz_introspection/#mz_cluster_replica_resource_usage). +Each row is one measurement, taken from one source, reported as that source +gave it. Sources measure overlapping but distinct quantities, and the +differences between them are informative, so no row is a combination of two +others. Deciding which number is "the" memory usage of a replica, or how close +it is to its limit, is left to queries over the relation. + +A metric whose name ends in `peak` is a high-water mark since the process +started, and the rest are instantaneous. Peaks the operating system maintains +itself are exact, and are unaffected by how often the replica reads them. Peaks +folded from samples are marked as such below and can miss a spike shorter than +the sampling interval, which makes them lower bounds. An observation the +replica could not read is absent rather than zero, so which metrics appear +depends on the platform and the kernel version. + +## Sources + +| Source | Reads | Measures | +|---------------|-----------------------------------------|-------------------------------------------------------------------------------------------| +| `cgroup` | the process's cgroup v2 interface files | the whole container, and the accounting that limit enforcement and the OOM killer act on | +| `proc_status` | `/proc/self/status` | this process only, with resident memory broken down by backing | +| `rusage` | `getrusage(RUSAGE_SELF)` | this process only | +| `statvfs` | the replica's scratch filesystem | the filesystem as a whole, absent where disk is provided as swap | + +## Metrics + +| Source | Metric | Meaning | +|---------------|-------------------|--------------------------------------------------------------------------------------------------------------------------------| +| `cgroup` | `memory_current` | Memory charged to the cgroup: anonymous, page cache, kernel and socket memory. | +| `cgroup` | `memory_peak` | High-water mark of `memory_current`, maintained by the kernel. | +| `cgroup` | `memory_max` | The cgroup's memory limit. | +| `cgroup` | `swap_current` | Swap charged to the cgroup, including pages already read back whose swap slot is still allocated. | +| `cgroup` | `swap_peak` | High-water mark of `swap_current`, maintained by the kernel. | +| `cgroup` | `swap_max` | The cgroup's swap limit. | +| `cgroup` | `anon` | The part of `memory_current` backed by no file. | +| `cgroup` | `file` | Page cache charged to the cgroup. | +| `cgroup` | `shmem` | Shared memory and tmpfs pages. | +| `cgroup` | `swapcached` | Pages resident in memory whose swap slot is still allocated. Counted in both `anon` and `swap_current`. | +| `cgroup` | `kernel` | Kernel memory charged to the cgroup. | +| `cgroup` | `slab` | Kernel slab allocations, part of `kernel`. | +| `cgroup` | `sock` | Socket buffer memory. | +| `cgroup` | `events_max` | Times an allocation hit `memory_max`. See the caveat below before using this as a limit-hit signal. | +| `cgroup` | `events_oom_kill` | Processes in the cgroup killed by the OOM killer. | +| `proc_status` | `vm_rss` | Resident set size of this process, the sum of `rss_anon`, `rss_file` and `rss_shmem`. | +| `proc_status` | `rss_anon` | Resident memory backed by no file. The replica's own memory. | +| `proc_status` | `rss_file` | Resident file-backed memory, largely this binary's text. Shared between replicas and charged to whichever cgroup first faulted it in. | +| `proc_status` | `rss_shmem` | Resident shared memory. | +| `proc_status` | `vm_swap` | This process's pages currently in swap. Excludes swap-cached pages, so it reads below `cgroup` `swap_current`. | +| `proc_status` | `vm_swap_peak` | Maximum `vm_swap` over samples, so a lower bound on the true peak. | +| `proc_status` | `heap` | `vm_rss` plus `vm_swap`, the quantity a replica is limited on. | +| `proc_status` | `heap_peak` | Maximum `heap` over samples, so a lower bound on the true peak. | +| `rusage` | `max_rss` | Peak resident set size. Maintained by the kernel, but refreshed only at internal checkpoints, so it can read below a concurrent `vm_rss`. | +| `statvfs` | `fs_used` | Used bytes of the filesystem, which on a shared filesystem counts writes this replica never made. | +| `statvfs` | `fs_used_peak` | Maximum `fs_used` over samples, so a lower bound on the true peak. | + +## Interpreting + +Values from different sources are not interchangeable, and adding them together +generally produces a number that means nothing. In particular: + +* **How close is this replica to its memory limit?** Compare `cgroup` + `memory_current` against `memory_max`. **Did it ever reach it?** Compare + `memory_peak` against `memory_max`. A `memory_peak` at the limit means the + replica ran out of RAM and spilled to swap, even if the current reading is + comfortable. +* **How close is it to its heap limit?** Compare `proc_status` `heap` against + [`mz_internal.mz_cluster_replica_metrics`](/reference/system-catalog/mz_internal/#mz_cluster_replica_metrics)'s + `heap_limit`. `heap_peak` bounds the high-water mark from below, and no source + bounds it from above: the `cgroup` peaks describe a smaller quantity, and + `max_rss` lags. +* **Do not use `events_max` as a limit-hit signal.** Where swap is configured it + stays at zero even for a replica pinned at its ceiling, because reclaim + succeeds by swapping instead of failing. `events_oom_kill` does report kills. +* **How much memory does this replica itself account for?** Use `proc_status` + `rss_anon`. Do not use `vm_rss`: it includes `rss_file`, which is charged to + another cgroup and so runs a roughly constant amount above the replica's own + charge. +* **Do not add `memory_current` and `swap_current`.** A page read back from swap + is counted in both, and `swapcached` reports how much is in that state. +* **Where disk is provided as swap**, disk usage appears as `swap_current` and + there are no `statvfs` rows at all. +* **These readings live and die with the replica process.** A restart resets + every peak. For history that survives restarts, see + [`mz_internal.mz_cluster_replica_metrics_history`](/reference/system-catalog/mz_internal/#mz_cluster_replica_metrics_history). + +## Example + +Introspection relations are replica-local: a query reads the replica that +serves it, so pin both the cluster and the replica. This reports how close each +process came to a memory-limiter kill, comparing the quantity the limiter +enforces against the limit it enforces: + +```mzsql +SET cluster = ; +SET cluster_replica = ; + +SELECT + u.process_id, + round((max(u.value) FILTER (WHERE u.metric = 'heap'))::numeric / 1073741824, 2) AS heap_gib, + round((max(u.value) FILTER (WHERE u.metric = 'heap_peak'))::numeric / 1073741824, 2) AS heap_peak_gib, + round(m.heap_limit::numeric / 1073741824, 2) AS limit_gib, + round(100 * (max(u.value) FILTER (WHERE u.metric = 'heap_peak'))::numeric / m.heap_limit, 1) AS peak_pct +FROM mz_introspection.mz_cluster_replica_resource_usage u +JOIN mz_cluster_replicas r + ON r.name = current_setting('cluster_replica') + AND r.cluster_id = (SELECT id FROM mz_clusters WHERE name = current_setting('cluster')) +JOIN mz_internal.mz_cluster_replica_metrics m + ON m.replica_id = r.id AND m.process_id = u.process_id +WHERE u.source = 'proc_status' +GROUP BY u.process_id, m.heap_limit +ORDER BY u.process_id; +``` + +`peak_pct` is a lower bound, because `heap_peak` is: a spike shorter than the +sampling interval can slip through it. No source provides a matching upper +bound. The query stays within `proc_status` deliberately. The `cgroup` metrics +measure a different quantity, and `memory_current` plus `swap_current` +double-counts every swap-cached page. diff --git a/doc/user/content/reference/system-catalog/mz_internal.md b/doc/user/content/reference/system-catalog/mz_internal.md index 8d6f3347be034..39fbf312341a4 100644 --- a/doc/user/content/reference/system-catalog/mz_internal.md +++ b/doc/user/content/reference/system-catalog/mz_internal.md @@ -204,6 +204,13 @@ The `mz_cluster_replica_metrics` view gives the last known CPU and RAM utilizati for all processes of all extant cluster replicas. At this time, we do not make any guarantees about the exactness or freshness of these numbers. +They are sampled roughly once a minute, so a spike shorter than the sampling interval is not +visible here at all. For a view of a single replica sampled every few seconds, including high-water +marks that survive a spike the sampling missed, see [Replica resource +usage](/manage/monitor/replica-resource-usage/). + +Where a replica's disk is provided as swap rather than as a filesystem, `disk_bytes` reports swap +usage. | Field | Type | Meaning @@ -223,6 +230,11 @@ The `mz_cluster_replica_metrics_history` table records resource utilization metr for all processes of all extant cluster replicas. At this time, we do not make any guarantees about the exactness or freshness of these numbers. +They are sampled roughly once a minute, so a spike shorter than the sampling interval leaves no +trace. Unlike +[`mz_introspection.mz_cluster_replica_resource_usage`](/reference/system-catalog/mz_introspection/#mz_cluster_replica_resource_usage), +which is sampled every few seconds but is replica-local and resets when a replica restarts, this +history is retained across restarts. | Field | Type | Meaning diff --git a/doc/user/content/reference/system-catalog/mz_introspection.md b/doc/user/content/reference/system-catalog/mz_introspection.md index f238ce5e8240b..bf048dddc15b2 100644 --- a/doc/user/content/reference/system-catalog/mz_introspection.md +++ b/doc/user/content/reference/system-catalog/mz_introspection.md @@ -170,6 +170,27 @@ Summaries are flattened into separate quantile, sum, and count rows. | `value` | [`double precision`] | The numeric value of the metric. | | `help` | [`text`] | The help string describing the metric. | +## `mz_cluster_replica_resource_usage` + +The `mz_cluster_replica_resource_usage` source reports the resource usage of each process of a +cluster replica, as one row per measurement source and metric. Each row is what that source +reported, without interpretation: sources measure overlapping but distinct quantities, so combining +them into a single figure for memory usage, or for how close a replica is to its limit, is left to +queries over this relation. + +The replica samples its sources every few seconds, and reports a high-water mark alongside the +instantaneous value where one is available. For the sources and metrics that appear here, how to +interpret them, and an example query, see [Replica resource +usage](/manage/monitor/replica-resource-usage/). + + +| Field | Type | Meaning | +|--------------|-----------|----------------------------------------------------------------------| +| `process_id` | [`uint8`] | The ID of the process within the replica. | +| `source` | [`text`] | The measurement source, for example `cgroup` or `rusage`. | +| `metric` | [`text`] | What the source measured, for example `memory_current`. | +| `value` | [`uint8`] | The reported value, in bytes for a size and as a count otherwise. | + ## `mz_dataflows` The `mz_dataflows` view describes the [dataflows][dataflow] in the system. diff --git a/doc/user/data/metrics.yml b/doc/user/data/metrics.yml index 034f404364004..f8af94bdea893 100644 --- a/doc/user/data/metrics.yml +++ b/doc/user/data/metrics.yml @@ -1402,6 +1402,13 @@ metrics: help: user CPU time used source: src/metrics/src/rusage.rs visibility: internal +- name: mz_metrics_resource_usage + help: Resource usage observations, by source and metric. + labels: + - metric + - source + source: src/metrics/src/usage.rs + visibility: internal - name: mz_metrics_update_duration_bucket help: The time it took to update lgalloc stats labels: diff --git a/misc/mcp-materialize-agents/Dockerfile b/misc/mcp-materialize-agents/Dockerfile index 0b9b9ca8faed4..07c1c89ba62c5 100644 --- a/misc/mcp-materialize-agents/Dockerfile +++ b/misc/mcp-materialize-agents/Dockerfile @@ -7,7 +7,9 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -FROM python:3.13-alpine +# Pinned by rolling tag + multi-arch index digest so dependabot refreshes the +# digest for security fixes. +FROM python:3.13-alpine@sha256:540c7d91f98ff6880174c40e99067bf5941eb54d818a7a5e094d188b196a934d RUN pip install uv diff --git a/misc/mcp-materialize/Dockerfile b/misc/mcp-materialize/Dockerfile index 0b9b9ca8faed4..07c1c89ba62c5 100644 --- a/misc/mcp-materialize/Dockerfile +++ b/misc/mcp-materialize/Dockerfile @@ -7,7 +7,9 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -FROM python:3.13-alpine +# Pinned by rolling tag + multi-arch index digest so dependabot refreshes the +# digest for security fixes. +FROM python:3.13-alpine@sha256:540c7d91f98ff6880174c40e99067bf5941eb54d818a7a5e094d188b196a934d RUN pip install uv diff --git a/misc/python/materialize/mzcompose/__init__.py b/misc/python/materialize/mzcompose/__init__.py index 0244c6f41f21d..a3e7aad96db01 100644 --- a/misc/python/materialize/mzcompose/__init__.py +++ b/misc/python/materialize/mzcompose/__init__.py @@ -139,6 +139,11 @@ def get_minimal_system_parameters( # End of list (ordered by name) } + if version >= MzVersion.parse_mz("v26.40.0-dev"): + # Exercise the row-limit check without constraining normal test queries. + config["compute_peek_row_iteration_limit"] = "1000000000" + config["enable_compute_peek_row_iteration_limit"] = "true" + if version < MzVersion.parse_mz("v0.163.0-dev"): config["enable_compute_active_dataflow_cancelation"] = "true" @@ -800,6 +805,7 @@ def get_default_system_parameters( "mz_metrics_lgalloc_map_refresh_interval", "mz_metrics_lgalloc_refresh_interval", "mz_metrics_rusage_refresh_interval", + "mz_metrics_usage_refresh_interval", "compute_peek_response_stash_batch_max_runs", "compute_peek_response_stash_read_batch_size_bytes", "compute_peek_response_stash_read_memory_budget_bytes", diff --git a/misc/python/materialize/parallel_workload/action.py b/misc/python/materialize/parallel_workload/action.py index 405cce9e78092..65dda1529e115 100644 --- a/misc/python/materialize/parallel_workload/action.py +++ b/misc/python/materialize/parallel_workload/action.py @@ -3064,6 +3064,10 @@ def __init__( self.flags_with_values["enable_compute_peek_response_stash"] = ( BOOLEAN_FLAG_VALUES ) + self.flags_with_values["enable_compute_peek_row_iteration_limit"] = ( + BOOLEAN_FLAG_VALUES + ) + self.flags_with_values["compute_peek_row_iteration_limit"] = ["1000000000"] self.flags_with_values["compute_peek_response_stash_threshold_bytes"] = [ "0", # "force enabled" "1048576", # 1 MiB, an in-between value @@ -3388,6 +3392,7 @@ def __init__( "mz_metrics_lgalloc_map_refresh_interval", "mz_metrics_lgalloc_refresh_interval", "mz_metrics_rusage_refresh_interval", + "mz_metrics_usage_refresh_interval", "compute_peek_stash_num_batches", "compute_peek_stash_batch_size", "compute_peek_response_stash_batch_max_runs", diff --git a/src/adapter/src/active_compute_sink.rs b/src/adapter/src/active_compute_sink.rs index 556b87544d8a9..880ad456bcc42 100644 --- a/src/adapter/src/active_compute_sink.rs +++ b/src/adapter/src/active_compute_sink.rs @@ -279,7 +279,10 @@ impl ActiveSubscribe { mz_ore::iter::consolidate_update_iter(merged) } Err(s) => { - self.send(PeekResponseUnary::Error(s), 0); + self.send( + PeekResponseUnary::Error(AdapterError::Unstructured(anyhow::Error::msg(s))), + 0, + ); return true; } }; @@ -507,13 +510,10 @@ impl ActiveSubscribe { ActiveComputeSinkRetireReason::BufferExceeded { buffered_bytes, max_buffered_bytes, - } => PeekResponseUnary::Error( - AdapterError::SubscribeFellBehind { - buffered_bytes, - max_buffered_bytes, - } - .to_string(), - ), + } => PeekResponseUnary::Error(AdapterError::SubscribeFellBehind { + buffered_bytes, + max_buffered_bytes, + }), }; self.send(message, 0); } diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index 394a479d54202..a96caf5281975 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -424,6 +424,23 @@ static MIGRATIONS: LazyLock> = LazyLock::new(|| { MZ_CATALOG_SCHEMA, "mz_views", ), + // Required because we added the `mz_cluster_replica_resource_usage` builtin log. + // make_mz_indexes and make_mz_sources inline the builtin-log set as + // VALUES, so adding one changes both MVs' SQL fingerprints. See the NOTE + // above: this version must stay at the workspace's current dev version + // until the change ships. + MigrationStep::replacement( + "26.40.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_indexes", + ), + MigrationStep::replacement( + "26.40.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_sources", + ), ] }); diff --git a/src/adapter/src/coord/catalog_implications.rs b/src/adapter/src/coord/catalog_implications.rs index 432865aa6c757..fdbe8f259fe65 100644 --- a/src/adapter/src/coord/catalog_implications.rs +++ b/src/adapter/src/coord/catalog_implications.rs @@ -39,7 +39,7 @@ use mz_catalog::memory::objects::{ }; use mz_cloud_resources::VpcEndpointConfig; use mz_compute_client::logging::LogVariant; -use mz_compute_client::protocol::response::PeekResponse; +use mz_compute_client::protocol::response::{PeekError, PeekResponse}; use mz_controller::clusters::{ClusterRole, ReplicaConfig}; use mz_controller_types::{ClusterId, ReplicaId}; use mz_ore::collections::CollectionExt; @@ -1001,7 +1001,9 @@ impl Coordinator { if !peeks_to_drop.is_empty() { for (dep, uuid) in peeks_to_drop { if let Some(pending_peek) = self.remove_pending_peek(&uuid) { - let cancel_reason = PeekResponse::Error(dep.query_terminated_error()); + let cancel_reason = PeekResponse::Error(PeekError::unstructured( + dep.query_terminated_error(), + )); self.controller .compute .cancel_peek(pending_peek.cluster_id, uuid, cancel_reason) diff --git a/src/adapter/src/coord/peek.rs b/src/adapter/src/coord/peek.rs index f593fa539a696..a8d653495210f 100644 --- a/src/adapter/src/coord/peek.rs +++ b/src/adapter/src/coord/peek.rs @@ -85,13 +85,9 @@ pub(crate) struct PendingPeek { #[derive(Debug)] pub enum PeekResponseUnary { Rows(Box), - Error(String), + Error(AdapterError), Canceled, /// A dependency was dropped during execution. - /// - /// N.B. This is a bit of a workaround for the fact that our Error variant - /// is unstructured and right now we specifically care about this error and - /// need to render differently based on context. DependencyDropped(DroppedDependency), } @@ -1049,7 +1045,7 @@ impl crate::coord::Coordinator { let rows = match result { Ok(rows) => rows, Err(e) => { - yield PeekResponseUnary::Error(e.to_string()); + yield PeekResponseUnary::Error(AdapterError::Unstructured(anyhow::anyhow!(e))); return; } }; @@ -1064,7 +1060,11 @@ impl crate::coord::Coordinator { &duration_histogram, ) { Ok((rows, _size_bytes)) => yield PeekResponseUnary::Rows(Box::new(rows)), - Err(e) => yield PeekResponseUnary::Error(e), + Err(e) => { + yield PeekResponseUnary::Error(AdapterError::Unstructured( + anyhow::Error::msg(e), + )) + } } } PeekResponse::Stashed(response) => { @@ -1221,7 +1221,11 @@ impl crate::coord::Coordinator { match result_rows { Ok(result_rows) => yield PeekResponseUnary::Rows(Box::new(result_rows)), - Err(e) => yield PeekResponseUnary::Error(e), + Err(e) => { + yield PeekResponseUnary::Error(AdapterError::Unstructured( + anyhow::Error::msg(e), + )) + } } } @@ -1236,7 +1240,7 @@ impl crate::coord::Coordinator { yield PeekResponseUnary::Canceled; } PeekResponse::Error(e) => { - yield PeekResponseUnary::Error(e); + yield PeekResponseUnary::Error(e.into()); } } }) diff --git a/src/adapter/src/coord/sequencer/inner.rs b/src/adapter/src/coord/sequencer/inner.rs index 7f1d4dcd8b025..687d79025579d 100644 --- a/src/adapter/src/coord/sequencer/inner.rs +++ b/src/adapter/src/coord/sequencer/inner.rs @@ -3053,9 +3053,7 @@ impl Coordinator { }; } PeekResponseUnary::Canceled => break Err(AdapterError::Canceled), - PeekResponseUnary::Error(e) => { - break Err(AdapterError::Unstructured(anyhow!(e))); - } + PeekResponseUnary::Error(e) => break Err(e), PeekResponseUnary::DependencyDropped(dep) => { break Err(dep.to_concurrent_dependency_drop()); } diff --git a/src/adapter/src/error.rs b/src/adapter/src/error.rs index 6f000e8b120ff..84efa804474d9 100644 --- a/src/adapter/src/error.rs +++ b/src/adapter/src/error.rs @@ -36,7 +36,7 @@ use mz_sql::rbac; use mz_sql::session::vars::VarError; use mz_storage_types::connections::ConnectionValidationError; use mz_storage_types::controller::StorageError; -use mz_storage_types::errors::CollectionMissing; +use mz_storage_types::errors::{CollectionMissing, DataflowError}; use smallvec::SmallVec; use timely::progress::Antichain; use tokio::sync::oneshot; @@ -69,6 +69,8 @@ pub enum AdapterError { DuplicateCursor(String), /// An error while evaluating an expression. Eval(EvalError), + /// An error produced while executing a dataflow. + Dataflow(Box), /// An error occurred while planning the statement. Explain(ExplainError), /// The ID allocator exhausted all valid IDs. @@ -201,6 +203,11 @@ pub enum AdapterError { /// The `subscribe_max_buffered_bytes` budget that was exceeded. max_buffered_bytes: usize, }, + /// A query exceeded the configured compute peek row iteration limit. + PeekRowIterationLimitExceeded { + /// The configured per-worker limit. + limit: usize, + }, /// The specified feature is not permitted in safe mode. SafeModeViolation(String), /// The current transaction had the wrong set of write locks. @@ -524,6 +531,15 @@ fn eval_error_code(err: &EvalError) -> SqlState { } } +fn dataflow_error_code(error: &DataflowError) -> SqlState { + match error { + DataflowError::EvalError(error) => eval_error_code(error), + DataflowError::DecodeError(_) + | DataflowError::SourceError(_) + | DataflowError::EnvelopeError(_) => SqlState::INTERNAL_ERROR, + } +} + impl AdapterError { pub fn into_response(self, severity: Severity) -> ErrorResponse { ErrorResponse { @@ -551,6 +567,10 @@ impl AdapterError { } AdapterError::Catalog(c) => c.detail(), AdapterError::Eval(e) => e.detail(), + AdapterError::Dataflow(e) => match &**e { + DataflowError::EvalError(e) => e.detail(), + _ => None, + }, AdapterError::RelationOutsideTimeDomain { relations, names } => Some(format!( "The following relations in the query are outside the transaction's time domain:\n{}\n{}", relations @@ -590,6 +610,11 @@ impl AdapterError { objects. Reduce the number of dependencies, or raise the \ read_then_write_max_dependencies system parameter." )), + AdapterError::PeekRowIterationLimitExceeded { limit } => Some(format!( + "The query attempted to examine more than {limit} rows on a single compute \ + worker. This limit prevents long-running SELECT queries from delaying other \ + work on the cluster." + )), AdapterError::SafeModeViolation(_) => Some( "The Materialize server you are connected to is running in \ safe mode, which limits the features that are available." @@ -768,6 +793,10 @@ impl AdapterError { without buffering, or raise the subscribe_max_buffered_bytes system variable." .to_string(), ), + AdapterError::Dataflow(e) => match &**e { + DataflowError::EvalError(e) => e.hint(), + _ => None, + }, AdapterError::AlterClusterUnmanagedWhileReconfiguring => Some( "Cancel the reconfiguration by altering the cluster back to its current \ configuration, or wait for it to settle, then convert." @@ -831,6 +860,13 @@ impl AdapterError { statement_timeout = '120s'`." .into(), ), + AdapterError::PeekRowIterationLimitExceeded { .. } => Some( + "Reduce the number of rows the query must examine, for example by querying an \ + indexed, more selective result. Queries with `LIMIT` and no `ORDER BY` can also \ + stop early. To permit this query, increase \ + `compute_peek_row_iteration_limit`." + .into(), + ), AdapterError::PlanError(e) => e.hint(), AdapterError::UnallowedOnCluster { cluster, .. } => { (cluster != MZ_CATALOG_SERVER_CLUSTER.name).then(|| @@ -906,6 +942,7 @@ impl AdapterError { // exhaustively so the catch-all `INTERNAL_ERROR` no longer applies // to errors that are really the user's fault. See SQL-326. AdapterError::Eval(e) => eval_error_code(e), + AdapterError::Dataflow(e) => dataflow_error_code(e), AdapterError::Explain(_) => SqlState::INTERNAL_ERROR, AdapterError::IdExhaustionError => SqlState::INTERNAL_ERROR, AdapterError::Internal(_) => SqlState::INTERNAL_ERROR, @@ -973,6 +1010,7 @@ impl AdapterError { AdapterError::ResourceExhaustion { .. } => SqlState::INSUFFICIENT_RESOURCES, AdapterError::ResultSize(_) => SqlState::OUT_OF_MEMORY, AdapterError::SubscribeFellBehind { .. } => SqlState::OUT_OF_MEMORY, + AdapterError::PeekRowIterationLimitExceeded { .. } => SqlState::PROGRAM_LIMIT_EXCEEDED, AdapterError::SafeModeViolation(_) => SqlState::INTERNAL_ERROR, AdapterError::SubscribeOnlyTransaction => SqlState::INVALID_TRANSACTION_STATE, AdapterError::Optimizer(e) => match e { @@ -1240,6 +1278,7 @@ impl fmt::Display for AdapterError { write!(f, "cursor {} already exists", name.quoted()) } AdapterError::Eval(e) => e.fmt(f), + AdapterError::Dataflow(e) => e.fmt(f), AdapterError::Explain(e) => e.fmt(f), AdapterError::IdExhaustionError => f.write_str("ID allocator exhausted all valid IDs"), AdapterError::Internal(e) => write!(f, "internal error: {}", e), @@ -1280,6 +1319,12 @@ impl fmt::Display for AdapterError { "selection has too many transitive dependencies to validate (limit {max_rw_dependencies})" ) } + AdapterError::PeekRowIterationLimitExceeded { limit } => { + write!( + f, + "query exceeded the configured row iteration limit of {limit} rows" + ) + } AdapterError::ReplaceMaterializedViewSealed { name } => { write!( f, @@ -1627,6 +1672,20 @@ impl From for AdapterError { } } +impl From for AdapterError { + fn from(error: mz_compute_client::protocol::response::PeekError) -> Self { + use mz_compute_client::protocol::response::PeekError; + + match error { + PeekError::Dataflow(error) => AdapterError::Dataflow(error), + PeekError::Unstructured(error) => AdapterError::Unstructured(anyhow::Error::msg(error)), + PeekError::RowIterationLimitExceeded { limit } => { + AdapterError::PeekRowIterationLimitExceeded { limit } + } + } + } +} + impl From for AdapterError { fn from(e: ExplainError) -> AdapterError { match e { @@ -1761,3 +1820,47 @@ impl From for AdapterError { } impl Error for AdapterError {} + +#[cfg(test)] +mod tests { + use super::*; + + #[mz_ore::test] + fn peek_row_iteration_limit_error_is_user_facing() { + let response = AdapterError::PeekRowIterationLimitExceeded { limit: 1000 } + .into_response(Severity::Error); + + assert_eq!(response.code, SqlState::PROGRAM_LIMIT_EXCEEDED); + assert_eq!( + response.message, + "query exceeded the configured row iteration limit of 1000 rows" + ); + assert_eq!( + response.detail.as_deref(), + Some( + "The query attempted to examine more than 1000 rows on a single compute worker. \ + This limit prevents long-running SELECT queries from delaying other work on the \ + cluster." + ) + ); + assert!( + response + .hint + .as_deref() + .is_some_and(|hint| hint.contains("compute_peek_row_iteration_limit")) + ); + } + + #[mz_ore::test] + fn structured_dataflow_error_preserves_message_and_code() { + use mz_compute_client::protocol::response::PeekError; + + let dataflow_error = DataflowError::from(EvalError::DivisionByZero); + let expected_message = dataflow_error.to_string(); + let response = + AdapterError::from(PeekError::from(dataflow_error)).into_response(Severity::Error); + + assert_eq!(response.code, SqlState::DIVISION_BY_ZERO); + assert_eq!(response.message, expected_message); + } +} diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs index 0a4c6d914c209..a3dbda548f476 100644 --- a/src/catalog/src/builtin.rs +++ b/src/catalog/src/builtin.rs @@ -1108,6 +1108,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::Log(&MZ_COMPUTE_EXPORTS_PER_WORKER), Builtin::Log(&MZ_COMPUTE_DATAFLOW_GLOBAL_IDS_PER_WORKER), Builtin::Log(&MZ_CLUSTER_PROMETHEUS_METRICS), + Builtin::Log(&MZ_CLUSTER_REPLICA_RESOURCE_USAGE), Builtin::Log(&MZ_MESSAGE_COUNTS_RECEIVED_RAW), Builtin::Log(&MZ_MESSAGE_COUNTS_SENT_RAW), Builtin::Log(&MZ_MESSAGE_BATCH_COUNTS_RECEIVED_RAW), diff --git a/src/catalog/src/builtin/mz_introspection.rs b/src/catalog/src/builtin/mz_introspection.rs index d5ad93b478809..2e3bdf9764d02 100644 --- a/src/catalog/src/builtin/mz_introspection.rs +++ b/src/catalog/src/builtin/mz_introspection.rs @@ -248,6 +248,21 @@ pub static MZ_CLUSTER_PROMETHEUS_METRICS: LazyLock = LazyLock::new(| }), }); +pub static MZ_CLUSTER_REPLICA_RESOURCE_USAGE: LazyLock = LazyLock::new(|| BuiltinLog { + name: "mz_cluster_replica_resource_usage", + schema: MZ_INTROSPECTION_SCHEMA, + oid: oid::LOG_MZ_CLUSTER_REPLICA_RESOURCE_USAGE_OID, + variant: LogVariant::Compute(ComputeLog::ResourceUsage), + access: vec![PUBLIC_SELECT], + ontology: Some(Ontology { + entity_name: "cluster_replica_resource_usage", + description: "Resource usage of each process of the cluster replica, as reported by each \ + measurement source.", + links: &const { [] }, + column_semantic_types: &[], + }), +}); + pub static MZ_COMPUTE_FRONTIERS_PER_WORKER: LazyLock = LazyLock::new(|| BuiltinLog { name: "mz_compute_frontiers_per_worker", schema: MZ_INTROSPECTION_SCHEMA, diff --git a/src/catalog/src/durable/transaction.rs b/src/catalog/src/durable/transaction.rs index c27c43fadcc93..a83d5dd6def40 100644 --- a/src/catalog/src/durable/transaction.rs +++ b/src/catalog/src/durable/transaction.rs @@ -1021,6 +1021,7 @@ impl<'a> Transaction<'a> { LogVariant::Compute(ComputeLog::DataflowGlobal) => 31, LogVariant::Compute(ComputeLog::OperatorHydrationStatus) => 32, LogVariant::Compute(ComputeLog::PrometheusMetrics) => 33, + LogVariant::Compute(ComputeLog::ResourceUsage) => 34, }; let mut id: u64 = u64::from(cluster_variant) << 56; diff --git a/src/clusterd/src/lib.rs b/src/clusterd/src/lib.rs index 4d202a6fcaabd..87e9d932c0b29 100644 --- a/src/clusterd/src/lib.rs +++ b/src/clusterd/src/lib.rs @@ -268,7 +268,12 @@ async fn run(args: Args) -> Result<(), anyhow::Error> { emit_boot_diagnostics!(&BUILD_INFO); mz_alloc::register_metrics_into(&metrics_registry).await; - mz_metrics::register_metrics_into(&metrics_registry, mz_dyncfgs::all_dyncfgs()).await; + mz_metrics::register_metrics_into( + &metrics_registry, + mz_dyncfgs::all_dyncfgs(), + args.scratch_directory.clone(), + ) + .await; if let Some(heap_limit) = args.heap_limit { mz_compute::memory_limiter::start_limiter(heap_limit, &metrics_registry); diff --git a/src/clusterd/src/usage_metrics.rs b/src/clusterd/src/usage_metrics.rs index fbd648c59d49c..08f407e5613d4 100644 --- a/src/clusterd/src/usage_metrics.rs +++ b/src/clusterd/src/usage_metrics.rs @@ -15,7 +15,7 @@ use std::path::PathBuf; use serde::Serialize; -use tracing::{debug, error}; +use tracing::error; /// A system usage metrics collector. pub(crate) struct Collector { @@ -38,26 +38,14 @@ impl Collector { } fn collect_disk_usage(&self) -> Option { - let Some(root) = &self.disk_root else { - return None; - }; - - let stat = match nix::sys::statvfs::statvfs(root) { - Ok(stat) => stat, + let root = self.disk_root.as_deref()?; + match mz_metrics::usage::disk_usage(root) { + Ok(bytes) => Some(bytes), Err(err) => { error!("statvfs error: {err}"); - return None; + None } - }; - - // `fsblkcnt_t` is a `u32` on macOS but a `u64` on Linux. - #[allow(clippy::useless_conversion)] - let used_blocks = u64::from(stat.blocks() - stat.blocks_available()); - let used_bytes = used_blocks * stat.fragment_size(); - - debug!("disk usage: {used_bytes}"); - - Some(used_bytes) + } } } @@ -77,6 +65,7 @@ mod linux { use anyhow::{anyhow, bail}; use mz_compute::memory_limiter; + use mz_metrics::usage::ProcStatus; use mz_ore::cast::CastInto; use tracing::{debug, error}; @@ -84,7 +73,7 @@ mod linux { pub fn collect_heap_usage() -> (Option, Option) { use mz_ore::cast::CastInto; - match memory_limiter::ProcStatus::from_proc() { + match ProcStatus::from_proc() { Ok(status) => { let memory_bytes = status.vm_rss.cast_into(); let swap_bytes = status.vm_swap.cast_into(); diff --git a/src/compute-client/Cargo.toml b/src/compute-client/Cargo.toml index 37910388b869c..a35e3063f7f9a 100644 --- a/src/compute-client/Cargo.toml +++ b/src/compute-client/Cargo.toml @@ -41,6 +41,7 @@ tracing.workspace = true uuid = { workspace = true, features = ["serde", "v4"] } [dev-dependencies] +bincode.workspace = true futures.workspace = true [features] diff --git a/src/compute-client/src/controller.rs b/src/compute-client/src/controller.rs index 405cb3aa417fa..ce1dd5f808feb 100644 --- a/src/compute-client/src/controller.rs +++ b/src/compute-client/src/controller.rs @@ -172,7 +172,7 @@ impl PeekNotification { result_size: u64::cast_from(result_size), } } - PeekResponse::Error(err) => Self::Error(err.clone()), + PeekResponse::Error(err) => Self::Error(err.to_string()), PeekResponse::Canceled => Self::Canceled, } } diff --git a/src/compute-client/src/controller/instance.rs b/src/compute-client/src/controller/instance.rs index f5490bd179e35..2f444d89f4755 100644 --- a/src/compute-client/src/controller/instance.rs +++ b/src/compute-client/src/controller/instance.rs @@ -64,8 +64,8 @@ use crate::protocol::command::{ }; use crate::protocol::history::ComputeCommandHistory; use crate::protocol::response::{ - ComputeResponse, CopyToResponse, FrontiersResponse, PeekResponse, StatusResponse, - SubscribeBatch, SubscribeResponse, + ComputeResponse, CopyToResponse, FrontiersResponse, PeekError as ProtocolPeekError, + PeekResponse, StatusResponse, SubscribeBatch, SubscribeResponse, }; #[derive(Error, Debug)] @@ -1362,7 +1362,8 @@ impl Instance { self.deliver_response(response); } for uuid in to_drop { - let response = PeekResponse::Error(ERROR_TARGET_REPLICA_FAILED.into()); + let response = + PeekResponse::Error(ProtocolPeekError::unstructured(ERROR_TARGET_REPLICA_FAILED)); self.finish_peek(uuid, response); } diff --git a/src/compute-client/src/logging.rs b/src/compute-client/src/logging.rs index d3b788af63efe..86df318074e85 100644 --- a/src/compute-client/src/logging.rs +++ b/src/compute-client/src/logging.rs @@ -184,6 +184,8 @@ pub enum ComputeLog { DataflowGlobal, /// Prometheus metrics gathered from the metrics registry. PrometheusMetrics, + /// Resource usage observations of each replica process. + ResourceUsage, } impl LogVariant { @@ -413,6 +415,14 @@ impl LogVariant { .with_column("help", SqlScalarType::String.nullable(false)) .with_key(vec![0, 1, 3]) .finish(), + + LogVariant::Compute(ComputeLog::ResourceUsage) => RelationDesc::builder() + .with_column("process_id", SqlScalarType::UInt64.nullable(false)) + .with_column("source", SqlScalarType::String.nullable(false)) + .with_column("metric", SqlScalarType::String.nullable(false)) + .with_column("value", SqlScalarType::UInt64.nullable(false)) + .with_key(vec![0, 1, 2]) + .finish(), } } } diff --git a/src/compute-client/src/protocol/command.rs b/src/compute-client/src/protocol/command.rs index 3c3f35980c249..b69bca23fffca 100644 --- a/src/compute-client/src/protocol/command.rs +++ b/src/compute-client/src/protocol/command.rs @@ -233,12 +233,13 @@ pub enum ComputeCommand { /// After receiving a `Peek` command, the replica must eventually produce a single /// [`PeekResponse`]: /// - /// * For peeks that were not cancelled: either [`Rows`] or [`Error`]. - /// * For peeks that were cancelled: either [`Rows`], or [`Error`], or [`Canceled`]. + /// * For peeks that were not cancelled: [`Rows`], [`Stashed`], or [`Error`]. + /// * For peeks that were cancelled: any response above, or [`Canceled`]. /// /// [`PeekResponse`]: super::response::PeekResponse /// [`PeekResponse::Error`]: super::response::PeekResponse::Error /// [`Rows`]: super::response::PeekResponse::Rows + /// [`Stashed`]: super::response::PeekResponse::Stashed /// [`Error`]: super::response::PeekResponse::Error /// [`Canceled`]: super::response::PeekResponse::Canceled Peek(Box), diff --git a/src/compute-client/src/protocol/response.rs b/src/compute-client/src/protocol/response.rs index fd62e744bffac..1fc18ab694332 100644 --- a/src/compute-client/src/protocol/response.rs +++ b/src/compute-client/src/protocol/response.rs @@ -9,12 +9,16 @@ //! Compute protocol responses. +use std::fmt; + +use mz_expr::EvalError; use mz_expr::row::RowCollection; use mz_ore::cast::CastFrom; use mz_ore::tracing::OpenTelemetryContext; use mz_persist_client::batch::ProtoBatch; use mz_persist_types::ShardId; use mz_repr::{GlobalId, RelationDesc, Timestamp, UpdateCollection}; +use mz_storage_types::errors::DataflowError; use serde::{Deserialize, Serialize}; use timely::progress::frontier::Antichain; use uuid::Uuid; @@ -187,6 +191,11 @@ impl FrontiersResponse { /// /// Note that each `Peek` expects to generate exactly one `PeekResponse`, i.e. /// we expect a 1:1 contract between `Peek` and `PeekResponse`. +/// +/// Encoded with bincode, which identifies a variant by its position, so both ends of a connection +/// have to agree on this declaration. They do: the CTP handshake refuses a peer whose version +/// differs from its own (`mz_service::transport`), so a replica never speaks to a controller that +/// declares a different shape, and the encoding is free to change with the type. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum PeekResponse { /// Returned rows of a successful peek. @@ -194,7 +203,7 @@ pub enum PeekResponse { /// Results of the peek were stashed in persist batches. Stashed(Box), /// Error of an unsuccessful peek. - Error(String), + Error(PeekError), /// The peek was canceled. Canceled, } @@ -210,6 +219,58 @@ impl PeekResponse { } } +/// The error of an unsuccessful peek. +/// +/// The variant decides what the user sees: a `Dataflow` error keeps the structure the dataflow +/// produced and so gets the same SQLSTATE constant folding would have assigned, a +/// `RowIterationLimitExceeded` names a limit the user can raise, and an `Unstructured` error is +/// reported as an internal error. Prefer the structured variants whenever the source has one. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub enum PeekError { + /// An error produced while executing the dataflow, for example evaluating an expression over + /// a collection. + Dataflow(Box), + /// An error from the peek machinery itself, with no structured form. + Unstructured(String), + /// A worker examined more rows than `compute_peek_row_iteration_limit` allows. + RowIterationLimitExceeded { + /// The limit that was in effect, in rows. + limit: usize, + }, +} + +impl PeekError { + /// Constructs an unstructured peek error. + pub fn unstructured(message: impl Into) -> Self { + Self::Unstructured(message.into()) + } +} + +impl fmt::Display for PeekError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Dataflow(error) => error.fmt(f), + Self::Unstructured(error) => f.write_str(error), + Self::RowIterationLimitExceeded { limit } => write!( + f, + "query exceeded the configured row iteration limit of {limit} rows" + ), + } + } +} + +impl From for PeekError { + fn from(error: DataflowError) -> Self { + Self::Dataflow(Box::new(error)) + } +} + +impl From for PeekError { + fn from(error: EvalError) -> Self { + Self::Dataflow(Box::new(error.into())) + } +} + /// Response from a peek whose results have been stashed into persist. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct StashedPeekResponse { diff --git a/src/compute-client/src/service.rs b/src/compute-client/src/service.rs index 019bcab56fcc6..70c5cb56126ee 100644 --- a/src/compute-client/src/service.rs +++ b/src/compute-client/src/service.rs @@ -27,8 +27,8 @@ use uuid::Uuid; use crate::protocol::command::ComputeCommand; use crate::protocol::response::{ - ComputeResponse, CopyToResponse, FrontiersResponse, PeekResponse, StashedPeekResponse, - SubscribeBatch, SubscribeResponse, + ComputeResponse, CopyToResponse, FrontiersResponse, PeekError, PeekResponse, + StashedPeekResponse, SubscribeBatch, SubscribeResponse, }; /// A client to a compute server. @@ -106,7 +106,7 @@ pub struct PartitionedComputeState { /// The compute protocol requires that exactly one response is emitted for each peek. This /// property ensures that a) we can eventually drop the tracking state maintained for a peek /// and b) we won't re-initialize tracking for a peek we have already served. - peek_responses: BTreeMap)>, + peek_responses: BTreeMap, /// For each in-progress copy-to the response data received so far, and the set of shards that /// provided responses already. /// @@ -223,19 +223,14 @@ impl PartitionedComputeState { response: PeekResponse, otel_ctx: OpenTelemetryContext, ) -> Option { - let (merged, ready_shards) = self.peek_responses.entry(uuid).or_insert(( - PeekResponse::Rows(vec![RowCollection::default()]), - BTreeSet::new(), - )); - - let first = ready_shards.insert(shard_id); - assert!(first, "duplicate peek response"); - - let resp1 = mem::replace(merged, PeekResponse::Canceled); - *merged = merge_peek_responses(resp1, response, self.max_result_size); - - if ready_shards.len() == self.parts { - let (response, _) = self.peek_responses.remove(&uuid).unwrap(); + let pending = self + .peek_responses + .entry(uuid) + .or_insert_with(PendingPeek::new); + pending.absorb(shard_id, response, self.max_result_size); + + if pending.ready_shards.len() == self.parts { + let response = self.peek_responses.remove(&uuid).unwrap().response; Some(ComputeResponse::PeekResponse(uuid, response, otel_ctx)) } else { None @@ -565,32 +560,67 @@ impl PendingSubscribe { } } +/// Accumulates the per-worker responses to one peek into the single response the controller +/// hands upwards. +#[derive(Debug)] +struct PendingPeek { + /// The responses merged so far. + response: PeekResponse, + /// Inline result bytes seen so far, across all shards. + /// + /// Tracked separately from `response` because a worker's rows are dropped as soon as any + /// worker reports an error. Without this the aggregate size check would depend on the order + /// the responses happen to arrive in. + inline_byte_len: usize, + /// The shards that have provided responses. + ready_shards: BTreeSet, +} + +impl PendingPeek { + fn new() -> Self { + Self { + response: PeekResponse::Rows(vec![RowCollection::default()]), + inline_byte_len: 0, + ready_shards: BTreeSet::new(), + } + } + + fn absorb(&mut self, shard_id: usize, response: PeekResponse, max_result_size: u64) { + let first = self.ready_shards.insert(shard_id); + assert!(first, "duplicate peek response"); + + self.inline_byte_len = self + .inline_byte_len + .saturating_add(response.inline_byte_len()); + let current = mem::replace(&mut self.response, PeekResponse::Canceled); + self.response = merge_peek_responses(current, response); + + // Merging eagerly is what keeps the controller's memory bounded, so the size check has to + // happen on every response rather than once at the end. + if self.inline_byte_len > max_result_size.cast_into() { + // NOTE: Tests match on this exact message, so nothing else may produce it. + let error = PeekError::unstructured(format!( + "total result exceeds max size of {}", + ByteSize::b(max_result_size) + )); + let current = mem::replace(&mut self.response, PeekResponse::Canceled); + self.response = merge_peek_responses(current, PeekResponse::Error(error)); + } + } +} + /// Merge two [`PeekResponse`]s. -fn merge_peek_responses( - resp1: PeekResponse, - resp2: PeekResponse, - max_result_size: u64, -) -> PeekResponse { +fn merge_peek_responses(resp1: PeekResponse, resp2: PeekResponse) -> PeekResponse { use PeekResponse::*; // Cancelations and errors short-circuit. Cancelations take precedence over errors. let (resp1, resp2) = match (resp1, resp2) { (Canceled, _) | (_, Canceled) => return Canceled, + (Error(e1), Error(e2)) => return Error(merge_peek_errors(e1, e2)), (Error(e), _) | (_, Error(e)) => return Error(e), resps => resps, }; - let total_byte_len = resp1.inline_byte_len() + resp2.inline_byte_len(); - if total_byte_len > max_result_size.cast_into() { - // Note: We match on this specific error message in tests so it's important that - // nothing else returns the same string. - let err = format!( - "total result exceeds max size of {}", - ByteSize::b(max_result_size) - ); - return Error(err); - } - match (resp1, resp2) { (Rows(mut rows1), Rows(rows2)) => { rows1.extend(rows2); @@ -625,14 +655,14 @@ fn merge_peek_responses( "shard IDs of stashed responses do not match: \ {shard_id1} != {shard_id2}" ); - return Error("internal error".into()); + return Error(PeekError::unstructured("internal error")); } if relation_desc1 != relation_desc2 { soft_panic_or_log!( "relation descs of stashed responses do not match: \ {relation_desc1:?} != {relation_desc2:?}" ); - return Error("internal error".into()); + return Error(PeekError::unstructured("internal error")); } batches1.append(&mut batches2); @@ -650,3 +680,25 @@ fn merge_peek_responses( _ => unreachable!("handled above"), } } + +/// Merge two [`PeekError`]s into the one we report. +/// +/// A row-iteration-limit error only wins against another one, and then the smaller limit wins so +/// the choice does not depend on which worker answered first. Any other error outranks it: a +/// query that both overran the limit and failed for a real reason should report the real reason. +fn merge_peek_errors(error1: PeekError, error2: PeekError) -> PeekError { + match (error1, error2) { + ( + PeekError::RowIterationLimitExceeded { limit: limit1 }, + PeekError::RowIterationLimitExceeded { limit: limit2 }, + ) => PeekError::RowIterationLimitExceeded { + limit: limit1.min(limit2), + }, + (PeekError::RowIterationLimitExceeded { .. }, error) + | (error, PeekError::RowIterationLimitExceeded { .. }) => error, + (error, _) => error, + } +} + +#[cfg(test)] +mod tests; diff --git a/src/compute-client/src/service/tests.rs b/src/compute-client/src/service/tests.rs new file mode 100644 index 0000000000000..c9b06dbe246b2 --- /dev/null +++ b/src/compute-client/src/service/tests.rs @@ -0,0 +1,76 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//! Tests of how a peek's responses from several replicas are merged. + +use super::*; +use std::num::NonZeroUsize; + +use mz_repr::Row; + +#[mz_ore::test] +fn pending_peek_response_precedence() { + let rows = PeekResponse::Rows(vec![RowCollection::default()]); + let error = PeekResponse::Error(PeekError::unstructured("dataflow error")); + let row_limit = |limit| PeekResponse::Error(PeekError::RowIterationLimitExceeded { limit }); + + let mut pending = PendingPeek::new(); + pending.absorb(0, rows.clone(), u64::MAX); + pending.absorb(1, row_limit(1000), u64::MAX); + pending.absorb(2, row_limit(500), u64::MAX); + assert_eq!(pending.response, row_limit(500)); + + let mut pending = PendingPeek::new(); + pending.absorb(0, rows, u64::MAX); + pending.absorb(1, row_limit(1000), u64::MAX); + pending.absorb(2, error.clone(), u64::MAX); + assert_eq!(pending.response, error); + + let mut pending = PendingPeek::new(); + pending.absorb( + 0, + PeekResponse::Error(PeekError::unstructured("dataflow error")), + u64::MAX, + ); + pending.absorb(3, PeekResponse::Canceled, u64::MAX); + assert_eq!(pending.response, PeekResponse::Canceled); +} + +#[mz_ore::test] +fn peek_max_size_wins_over_row_iteration_limit_in_every_order() { + let row = RowCollection::new(vec![(Row::default(), NonZeroUsize::new(1).unwrap())], &[]); + let rows = PeekResponse::Rows(vec![row]); + let max_result_size = u64::try_from(rows.inline_byte_len()).unwrap(); + let responses = [ + rows.clone(), + PeekResponse::Error(PeekError::RowIterationLimitExceeded { limit: 1000 }), + rows, + ]; + let permutations = [ + [0, 1, 2], + [0, 2, 1], + [1, 0, 2], + [1, 2, 0], + [2, 0, 1], + [2, 1, 0], + ]; + let expected = PeekResponse::Error(PeekError::unstructured(format!( + "total result exceeds max size of {}", + ByteSize::b(max_result_size) + ))); + + for permutation in permutations { + let mut pending = PendingPeek::new(); + for (shard_id, response_index) in permutation.into_iter().enumerate() { + pending.absorb(shard_id, responses[response_index].clone(), max_result_size); + } + + assert_eq!(pending.response, expected, "{permutation:?}"); + } +} diff --git a/src/compute-types/src/dyncfgs.rs b/src/compute-types/src/dyncfgs.rs index 2f7a2c7586549..9d2c8ce0e119a 100644 --- a/src/compute-types/src/dyncfgs.rs +++ b/src/compute-types/src/dyncfgs.rs @@ -643,6 +643,22 @@ pub const PEEK_STASH_BATCH_SIZE: Config = Config::new( ParameterScope::Environment, ); +/// Whether compute should stop peeks that iterate over too many rows. +pub const ENABLE_PEEK_ROW_ITERATION_LIMIT: Config = Config::new( + "enable_compute_peek_row_iteration_limit", + false, + "Whether compute should stop peeks that exceed compute_peek_row_iteration_limit.", + ParameterScope::Environment, +); + +/// The maximum number of rows a peek may iterate over on each worker. +pub const PEEK_ROW_ITERATION_LIMIT: Config = Config::new( + "compute_peek_row_iteration_limit", + 1000, + "The maximum number of rows a peek may iterate over on each worker when enable_compute_peek_row_iteration_limit is enabled. Does not apply once a peek's results move to the peek stash.", + ParameterScope::Environment, +); + /// The collection interval for the Prometheus metrics introspection source. /// /// Set to zero to disable scraping and retract any existing data. @@ -728,6 +744,8 @@ pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet { .add(&PEEK_RESPONSE_STASH_READ_MEMORY_BUDGET_BYTES) .add(&PEEK_STASH_NUM_BATCHES) .add(&PEEK_STASH_BATCH_SIZE) + .add(&ENABLE_PEEK_ROW_ITERATION_LIMIT) + .add(&PEEK_ROW_ITERATION_LIMIT) .add(&COMPUTE_PROMETHEUS_INTROSPECTION_SCRAPE_INTERVAL) .add(&SUBSCRIBE_SNAPSHOT_OPTIMIZATION) .add(&MV_SINK_ADVANCE_PERSIST_FRONTIERS) diff --git a/src/compute/src/compute_state.rs b/src/compute/src/compute_state.rs index 1569592ede151..41f927b4eb29b 100644 --- a/src/compute/src/compute_state.rs +++ b/src/compute/src/compute_state.rs @@ -26,15 +26,16 @@ use mz_compute_client::protocol::command::{ }; use mz_compute_client::protocol::history::ComputeCommandHistory; use mz_compute_client::protocol::response::{ - ComputeResponse, CopyToResponse, FrontiersResponse, PeekResponse, SubscribeResponse, + ComputeResponse, CopyToResponse, FrontiersResponse, PeekError, PeekResponse, SubscribeResponse, }; use mz_compute_types::dataflows::DataflowDescription; use mz_compute_types::dyncfgs::{ - ENABLE_PEEK_RESPONSE_STASH, PEEK_RESPONSE_STASH_BATCH_MAX_RUNS, - PEEK_RESPONSE_STASH_THRESHOLD_BYTES, PEEK_STASH_BATCH_SIZE, PEEK_STASH_NUM_BATCHES, + ENABLE_PEEK_RESPONSE_STASH, ENABLE_PEEK_ROW_ITERATION_LIMIT, + PEEK_RESPONSE_STASH_BATCH_MAX_RUNS, PEEK_RESPONSE_STASH_THRESHOLD_BYTES, + PEEK_ROW_ITERATION_LIMIT, PEEK_STASH_BATCH_SIZE, PEEK_STASH_NUM_BATCHES, }; use mz_compute_types::plan::render_plan::RenderPlan; -use mz_dyncfg::ConfigSet; +use mz_dyncfg::{ConfigSet, ConfigValHandle}; use mz_expr::row::RowCollection; use mz_expr::{RowComparator, SafeMfpPlan}; use mz_ore::cast::{CastFrom, CastLossy}; @@ -79,6 +80,80 @@ use crate::server::{ComputeInstanceContext, ResponseSender}; mod peek_result_iterator; mod peek_stash; +/// Cheap handles on the dyncfgs that bound how many rows a peek may examine. +/// +/// The limit is read through handles rather than captured once, because `UpdateConfiguration` +/// applies to peeks that are already in flight. +#[derive(Clone, Debug)] +struct PeekRowIterationConfig { + enabled: ConfigValHandle, + limit: ConfigValHandle, +} + +impl PeekRowIterationConfig { + fn new(config: &ConfigSet) -> Self { + Self { + enabled: ENABLE_PEEK_ROW_ITERATION_LIMIT.handle(config), + limit: PEEK_ROW_ITERATION_LIMIT.handle(config), + } + } + + fn current_limit(&self) -> Option { + self.enabled.get().then(|| self.limit.get()) + } +} + +/// Counts the rows a peek has examined on this worker and fails it once that exceeds the limit. +/// +/// A "row" here is a record the worker had to look at, not a record it returned. Records that a +/// literal constraint or the MFP throws away, and records that consolidate to zero, cost scan +/// time all the same, so they count too. +/// +/// Exactly `limit` rows are allowed. The peek only fails when it asks for the row after that. +#[derive(Debug)] +pub(crate) struct PeekRowIterationTracker { + limit: Option, + rows_iterated: usize, +} + +impl PeekRowIterationTracker { + fn new(limit: Option, rows_iterated: usize) -> Self { + Self { + limit, + rows_iterated, + } + } + + /// Adopts a new limit without forgetting the rows already examined. + /// + /// Rows counted while the feature was off still count, so turning it on mid-scan accounts for + /// the work the peek has already caused. + fn set_limit(&mut self, limit: Option) { + self.limit = limit; + } + + fn rows_iterated(&self) -> usize { + self.rows_iterated + } + + fn track_next(&mut self) -> Result<(), PeekError> { + if let Some(limit) = self.limit + && self.rows_iterated >= limit + { + return Err(PeekError::RowIterationLimitExceeded { limit }); + } + + self.rows_iterated = self.rows_iterated.saturating_add(1); + Ok(()) + } +} + +fn peek_row_iteration_limit(config: &ConfigSet) -> Option { + ENABLE_PEEK_ROW_ITERATION_LIMIT + .get(config) + .then(|| PEEK_ROW_ITERATION_LIMIT.get(config)) +} + /// Worker-local state that is maintained across dataflows. /// /// This state is restricted to the COMPUTE state, the deterministic, idempotent work @@ -782,6 +857,7 @@ impl<'a> ActiveComputeState<'a> { metadata, usize::cast_from(self.compute_state.max_result_size), self.timely_worker, + PeekRowIterationConfig::new(&self.compute_state.worker_config), ) } }; @@ -1037,6 +1113,9 @@ impl<'a> ActiveComputeState<'a> { PendingPeek::Index(peek) => { let start = Instant::now(); + let row_iteration_limit = + peek_row_iteration_limit(&self.compute_state.worker_config); + let peek_stash_eligible = peek .peek .finishing @@ -1087,6 +1166,7 @@ impl<'a> ActiveComputeState<'a> { self.compute_state.max_result_size, peek_stash_enabled && peek_stash_eligible, peek_stash_threshold_bytes, + row_iteration_limit, &metrics, ); @@ -1102,6 +1182,9 @@ impl<'a> ActiveComputeState<'a> { let _span = span!(parent: &peek.span, Level::DEBUG, "process_stash_peek").entered(); + // NOTE: The row iteration limit does not follow a peek into the stash. The + // stash restarts the scan and produces in bounded bursts, so a stashed + // peek may examine any number of rows. let peek_stash_batch_max_runs = PEEK_RESPONSE_STASH_BATCH_MAX_RUNS .get(&self.compute_state.worker_config); @@ -1360,6 +1443,7 @@ impl PendingPeek { metadata: CollectionMetadata, max_result_size: usize, timely_worker: &TimelyWorker, + row_iteration_config: PeekRowIterationConfig, ) -> Self { let active_worker = { // Choose the worker that does the actual peek arbitrarily but consistently. @@ -1397,6 +1481,7 @@ impl PendingPeek { mfp_plan, max_result_size, max_results_needed, + row_iteration_config, ) .await } else { @@ -1404,7 +1489,7 @@ impl PendingPeek { }; let result = match result { Ok(rows) => PeekResponse::Rows(vec![RowCollection::new(rows, &order_by)]), - Err(e) => PeekResponse::Error(e.to_string()), + Err(error) => PeekResponse::Error(error), }; match result_tx.send((result, start.elapsed())) { Ok(()) => {} @@ -1468,11 +1553,12 @@ impl PersistPeek { mfp_plan: SafeMfpPlan, max_result_size: usize, mut limit_remaining: usize, - ) -> Result, String> { + row_iteration_config: PeekRowIterationConfig, + ) -> Result, PeekError> { let client = persist_clients .open(metadata.persist_location) .await - .map_err(|e| e.to_string())?; + .map_err(|e| PeekError::unstructured(e.to_string()))?; let mut reader: ReadHandle = client .open_leased_reader( @@ -1483,7 +1569,7 @@ impl PersistPeek { USE_CRITICAL_SINCE_SNAPSHOT.get(client.dyncfgs()), ) .await - .map_err(|e| e.to_string())?; + .map_err(|e| PeekError::unstructured(e.to_string()))?; // If we are using txn-wal for this collection, then the upper might // be advanced lazily and we have to go through txn-wal for reads. @@ -1509,7 +1595,9 @@ impl PersistPeek { ) .await .map_err(|since| { - format!("attempted to peek at {as_of}, but the since has advanced to {since:?}") + PeekError::unstructured(format!( + "attempted to peek at {as_of}, but the since has advanced to {since:?}" + )) })?; // Re-used state for processing and building rows. @@ -1518,6 +1606,7 @@ impl PersistPeek { let mut row_builder = Row::default(); let arena = RowArena::new(); let mut total_size = 0usize; + let mut row_iteration_tracker = PeekRowIterationTracker::new(None, 0); let literal_len = match &literal_constraint { None => 0, @@ -1529,7 +1618,12 @@ impl PersistPeek { break; }; for (data, _, d) in batch { - let row = data.map_err(|e| e.to_string())?; + // Count before literal and MFP filtering because the Persist row + // has already been read and must still be examined. + row_iteration_tracker.set_limit(row_iteration_config.current_limit()); + row_iteration_tracker.track_next()?; + + let row = data.map_err(PeekError::from)?; if let Some(literal) = &literal_constraint { match row.iter().take(literal_len).cmp(literal.iter()) { @@ -1544,11 +1638,11 @@ impl PersistPeek { shard = %metadata.data_shard, diff = d, ?row, "persist peek encountered negative multiplicities", ); - format!( + PeekError::unstructured(format!( "Invalid data in source, \ saw retractions ({}) for row that does not exist: {:?}", -d, row, - ) + )) })?; let Some(count) = NonZeroUsize::new(count) else { continue; @@ -1557,16 +1651,16 @@ impl PersistPeek { let eval_result = mfp_plan .evaluate_into(&mut datum_local, &arena, &mut row_builder) .map(|row| row.cloned()) - .map_err(|e| e.to_string())?; + .map_err(PeekError::from)?; if let Some(row) = eval_result { total_size = total_size .saturating_add(row.byte_len()) .saturating_add(std::mem::size_of::()); if total_size > max_result_size { - return Err(format!( + return Err(PeekError::unstructured(format!( "result exceeds max size of {}", ByteSize::b(u64::cast_from(max_result_size)) - )); + ))); } result.push((row, count)); limit_remaining = limit_remaining.saturating_sub(count.get()); @@ -1625,6 +1719,7 @@ impl IndexPeek { max_result_size: u64, peek_stash_eligible: bool, peek_stash_threshold_bytes: usize, + row_iteration_limit: Option, metrics: &IndexPeekMetrics<'_>, ) -> PeekStatus { let method_start = Instant::now(); @@ -1645,7 +1740,7 @@ impl IndexPeek { read_frontier.elements(), self.peek.timestamp, ); - return PeekStatus::Ready(PeekResponse::Error(error)); + return PeekStatus::Ready(PeekResponse::Error(PeekError::unstructured(error))); } metrics @@ -1656,6 +1751,7 @@ impl IndexPeek { max_result_size, peek_stash_eligible, peek_stash_threshold_bytes, + row_iteration_limit, metrics, ); @@ -1672,14 +1768,20 @@ impl IndexPeek { max_result_size: u64, peek_stash_eligible: bool, peek_stash_threshold_bytes: usize, + row_iteration_limit: Option, metrics: &IndexPeekMetrics<'_>, ) -> PeekStatus { let error_scan_start = Instant::now(); // Check if there exist any errors and, if so, return whatever one we // find first. + let mut row_iteration_tracker = PeekRowIterationTracker::new(row_iteration_limit, 0); let (mut cursor, storage) = self.trace_bundle.errs_mut().cursor(); while cursor.key_valid(&storage) { + if let Err(error) = row_iteration_tracker.track_next() { + return PeekStatus::Ready(PeekResponse::Error(error)); + } + let mut copies = Diff::ZERO; cursor.map_times(&storage, |time, diff| { if time.less_equal(&self.peek.timestamp) { @@ -1692,14 +1794,15 @@ impl IndexPeek { target = %self.peek.target.id(), diff = %copies, %error, "index peek encountered negative multiplicities in error trace", ); - return PeekStatus::Ready(PeekResponse::Error(format!( + return PeekStatus::Ready(PeekResponse::Error(PeekError::unstructured(format!( "Invalid data in source errors, \ saw retractions ({}) for row that does not exist: {}", -copies, error, - ))); + )))); } if copies.is_positive() { - return PeekStatus::Ready(PeekResponse::Error(cursor.key(&storage).to_string())); + let error = cursor.key(&storage).deserialize(); + return PeekStatus::Ready(PeekResponse::Error(error.into())); } cursor.step_key(&storage); } @@ -1714,6 +1817,8 @@ impl IndexPeek { max_result_size, peek_stash_eligible, peek_stash_threshold_bytes, + row_iteration_limit, + row_iteration_tracker.rows_iterated(), metrics, ) } @@ -1725,6 +1830,8 @@ impl IndexPeek { max_result_size: u64, peek_stash_eligible: bool, peek_stash_threshold_bytes: usize, + row_iteration_limit: Option, + rows_iterated: usize, metrics: &IndexPeekMetrics<'_>, ) -> PeekStatus where @@ -1751,6 +1858,8 @@ impl IndexPeek { peek.timestamp, peek.literal_constraints.clone().as_deref_mut(), oks_handle, + row_iteration_limit, + rows_iterated, ); metrics @@ -1790,10 +1899,10 @@ impl IndexPeek { return PeekStatus::UsePeekStash; } if total_size > max_result_size { - return PeekStatus::Ready(PeekResponse::Error(format!( + return PeekStatus::Ready(PeekResponse::Error(PeekError::unstructured(format!( "result exceeds max size of {}", ByteSize::b(u64::cast_from(max_result_size)) - ))); + )))); } results.push((row, copies)); @@ -2118,3 +2227,6 @@ impl CollectionState { let _ = self.read_only_tx.send(false); } } + +#[cfg(test)] +mod tests; diff --git a/src/compute/src/compute_state/peek_result_iterator.rs b/src/compute/src/compute_state/peek_result_iterator.rs index 8e3a2967693fb..6d7c20c71d562 100644 --- a/src/compute/src/compute_state/peek_result_iterator.rs +++ b/src/compute/src/compute_state/peek_result_iterator.rs @@ -12,18 +12,20 @@ use std::ops::Range; use differential_dataflow::trace::cursor::{BatchCursor, BatchKey, CursorList}; use differential_dataflow::trace::implementations::BatchContainer; use differential_dataflow::trace::{Cursor, Navigable, TraceReader}; +use mz_compute_client::protocol::response::PeekError; /// The merged cursor a [`TraceReader::cursor`] hands out over all of a trace's batches: a /// [`CursorList`] over the per-batch cursors. type TraceCursor = CursorList>; /// Backing storage for a [`TraceCursor`]: the batches the cursor borrows from. type TraceStorage = Vec<::Batch>; -use mz_ore::result::ResultExt; use mz_repr::fixed_length::ExtendDatums; use mz_repr::{DatumVec, Diff, GlobalId, Row, RowArena}; use timely::order::PartialOrder; -pub struct PeekResultIterator +use super::PeekRowIterationTracker; + +pub(super) struct PeekResultIterator where Tr: TraceReader, { @@ -37,6 +39,8 @@ where datum_vec: DatumVec, literals: Option>, rows_processed: usize, + row_iteration_tracker: PeekRowIterationTracker, + exhausted: bool, } /// Helper to handle literals in peeks @@ -121,12 +125,14 @@ where DiffGat<'a> = &'a Diff, >, { - pub fn new( + pub(super) fn new( target_id: GlobalId, map_filter_project: mz_expr::SafeMfpPlan, peek_timestamp: mz_repr::Timestamp, literal_constraints: Option<&mut [Row]>, trace_reader: &mut Tr, + row_iteration_limit: Option, + rows_iterated: usize, ) -> Self { let (mut cursor, storage) = trace_reader.cursor(); let literals = literal_constraints @@ -142,6 +148,8 @@ where datum_vec: DatumVec::new(), literals, rows_processed: 0, + row_iteration_tracker: PeekRowIterationTracker::new(row_iteration_limit, rows_iterated), + exhausted: false, } } @@ -180,25 +188,84 @@ where DiffGat<'a> = &'a Diff, >, { - type Item = Result<(Row, NonZeroI64), String>; + type Item = Result<(Row, NonZeroI64), PeekError>; fn next(&mut self) -> Option { + let mut fuel = usize::MAX; + match self.step(&mut fuel) { + Step::Row(row) => Some(row), + Step::Done => None, + Step::OutOfFuel => unreachable!("stepped with unbounded fuel"), + } + } +} + +/// The outcome of a single fueled [`PeekResultIterator::step`]. +pub enum Step { + /// A result row, or the error that ended the scan. + Row(Result<(Row, NonZeroI64), PeekError>), + /// The cursor is exhausted. Further steps also return `Done`. + Done, + /// The fuel ran out before a row was found. The cursor sits at the next + /// position to attempt, so stepping again resumes exactly there. + OutOfFuel, +} + +impl PeekResultIterator +where + Tr: TraceReader, + for<'a> BatchCursor: Cursor< + Key<'a>: ExtendDatums + Eq, + KeyContainer: BatchContainer, + Val<'a>: ExtendDatums, + TimeGat<'a>: PartialOrder, + DiffGat<'a> = &'a Diff, + >, +{ + /// Advances the cursor until it produces a row, the cursor is exhausted, + /// or `fuel` runs out, whichever comes first. Decrements `fuel` by the + /// number of cursor positions visited. + /// + /// Fuel is charged per cursor position, not per row returned, so a + /// selective `map_filter_project` cannot starve the caller of yield + /// points. Returning with fuel left over means the cursor is exhausted. + pub fn step(&mut self, fuel: &mut usize) -> Step { + if self.exhausted { + return Step::Done; + } + let result = loop { + if *fuel == 0 { + return Step::OutOfFuel; + } + *fuel -= 1; + if self.literals_exhausted() { - return None; + return Step::Done; } if !self.cursor.key_valid(&self.storage) { - return None; + return Step::Done; } if !self.cursor.val_valid(&self.storage) { let exhausted = self.step_key(); if exhausted { - return None; + return Step::Done; } } + // Filtered and zero-multiplicity rows still consume worker time, so + // they count against the budget before evaluation. + // + // Failing here leaves the cursor where it is, so latch the iterator + // shut. Otherwise a caller that polls again would get the same error + // forever rather than an end. + if let Err(error) = self.row_iteration_tracker.track_next() { + self.exhausted = true; + return Step::Row(Err(error)); + } + self.rows_processed = self.rows_processed.saturating_add(1); match self.extract_current_row() { Ok(Some(row)) => break Ok(row), @@ -212,25 +279,13 @@ where self.cursor.step_val(&self.storage); - Some(result) + Step::Row(result) } -} -impl PeekResultIterator -where - Tr: TraceReader, - for<'a> BatchCursor: Cursor< - Key<'a>: ExtendDatums + Eq, - KeyContainer: BatchContainer, - Val<'a>: ExtendDatums, - TimeGat<'a>: PartialOrder, - DiffGat<'a> = &'a Diff, - >, -{ /// Extracts and returns the row currently pointed at by our cursor. Returns /// `Ok(None)` if our MapFilterProject evaluates to `None`. Also returns any /// errors that arise from evaluating the MapFilterProject. - fn extract_current_row(&mut self) -> Result, String> { + fn extract_current_row(&mut self) -> Result, PeekError> { // TODO: This arena could be maintained and reused for longer, // but it wasn't clear at what interval we should flush // it to ensure we don't accidentally spike our memory use. @@ -260,7 +315,7 @@ where .map_filter_project .evaluate_into(&mut borrow, &arena, &mut self.row_builder) .map(|row| row.cloned()) - .map_err_to_string_with_causes()? + .map_err(PeekError::from)? { let mut copies = Diff::ZERO; self.cursor.map_times(&self.storage, |time, diff| { @@ -274,11 +329,11 @@ where target = %self.target_id, diff = %copies, ?row, "index peek encountered negative multiplicities in ok trace", ); - return Err(format!( + return Err(PeekError::unstructured(format!( "Invalid data in source, \ saw retractions ({}) for row that does not exist: {:?}", -copies, row, - )); + ))); } else { copies.into_inner() }; diff --git a/src/compute/src/compute_state/peek_stash.rs b/src/compute/src/compute_state/peek_stash.rs index 214d9f371f64f..b9f70ce00425d 100644 --- a/src/compute/src/compute_state/peek_stash.rs +++ b/src/compute/src/compute_state/peek_stash.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use mz_compute_client::protocol::command::Peek; -use mz_compute_client::protocol::response::{PeekResponse, StashedPeekResponse}; +use mz_compute_client::protocol::response::{PeekError, PeekResponse, StashedPeekResponse}; use mz_expr::row::RowCollection; use mz_ore::cast::CastFrom; use mz_ore::task::AbortOnDropHandle; @@ -45,7 +45,7 @@ pub struct StashingPeek { /// We can't give a PeekResultIterator to our async upload task because the /// underlying trace reader is not Send/Sync. So we need to use a channel to /// send result rows from the worker thread to the async background task. - rows_tx: Option, String>>>, + rows_tx: Option, PeekError>>>, /// The result of the background task, eventually. pub result: oneshot::Receiver<(PeekResponse, Duration)>, /// The `tracing::Span` tracking this peek's operation @@ -80,6 +80,8 @@ impl StashingPeek { peek.timestamp, peek.literal_constraints.as_deref_mut(), oks_handle, + None, + 0, ); let rows_needed_by_finishing = peek.finishing.num_rows_needed(); @@ -102,7 +104,7 @@ impl StashingPeek { let result = match result { Ok(peek_response) => peek_response, - Err(e) => PeekResponse::Error(e.to_string()), + Err(e) => PeekResponse::Error(PeekError::unstructured(e.to_string())), }; match result_tx.send((result, start.elapsed())) { Ok(()) => {} @@ -130,7 +132,7 @@ impl StashingPeek { peek_uuid: Uuid, relation_desc: RelationDesc, max_rows: Option, // The number of rows needed by the RowSetFinishing's offset + limit - mut rows_rx: tokio::sync::mpsc::Receiver, String>>, + mut rows_rx: tokio::sync::mpsc::Receiver, PeekError>>, ) -> Result { let client = persist_clients .open(persist_location) diff --git a/src/compute/src/compute_state/tests.rs b/src/compute/src/compute_state/tests.rs new file mode 100644 index 0000000000000..73508ae6a5e8b --- /dev/null +++ b/src/compute/src/compute_state/tests.rs @@ -0,0 +1,48 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//! Tests of the compute state's own helpers, as opposed to the peek sweep it drives. + +use mz_dyncfg::ConfigUpdates; + +use super::*; + +#[mz_ore::test] +fn row_iteration_limit_observes_updates_and_disabled_rows() { + let config = mz_dyncfgs::all_dyncfgs(); + let row_iteration_config = PeekRowIterationConfig::new(&config); + let mut tracker = PeekRowIterationTracker::new(row_iteration_config.current_limit(), 0); + + tracker.track_next().unwrap(); + tracker.track_next().unwrap(); + + let mut updates = ConfigUpdates::default(); + updates.add(&PEEK_ROW_ITERATION_LIMIT, 3); + updates.add(&ENABLE_PEEK_ROW_ITERATION_LIMIT, true); + updates.apply(&config); + tracker.set_limit(row_iteration_config.current_limit()); + tracker.track_next().unwrap(); + + let mut updates = ConfigUpdates::default(); + updates.add(&ENABLE_PEEK_ROW_ITERATION_LIMIT, false); + updates.apply(&config); + tracker.set_limit(row_iteration_config.current_limit()); + tracker.track_next().unwrap(); + + let mut updates = ConfigUpdates::default(); + updates.add(&PEEK_ROW_ITERATION_LIMIT, 5); + updates.add(&ENABLE_PEEK_ROW_ITERATION_LIMIT, true); + updates.apply(&config); + tracker.set_limit(row_iteration_config.current_limit()); + tracker.track_next().unwrap(); + assert_eq!( + tracker.track_next(), + Err(PeekError::RowIterationLimitExceeded { limit: 5 }) + ); +} diff --git a/src/compute/src/logging.rs b/src/compute/src/logging.rs index b7414f8b2fbe6..e8352b134ca9a 100644 --- a/src/compute/src/logging.rs +++ b/src/compute/src/logging.rs @@ -14,20 +14,21 @@ mod differential; pub(super) mod initialize; mod prometheus; mod reachability; +mod resource_usage; mod timely; use std::any::Any; use std::collections::BTreeMap; use std::marker::PhantomData; use std::rc::Rc; -use std::time::Duration; +use std::time::{Duration, Instant}; use ::timely::container::{CapacityContainerBuilder, PushInto}; use ::timely::dataflow::Stream; use ::timely::dataflow::channels::pact::Pipeline; use ::timely::dataflow::operators::capture::{Event, EventLink, EventPusher}; use ::timely::dataflow::operators::generic::Session; -use ::timely::dataflow::operators::{InputCapability, Operator}; +use ::timely::dataflow::operators::{Capability, CapabilityTrait, InputCapability, Operator}; use ::timely::progress::Timestamp as TimelyTimestamp; use ::timely::scheduling::Activator; use ::timely::{Container, ContainerBuilder}; @@ -211,6 +212,74 @@ impl PermutedRowPacker { } } +/// Downgrade `cap` to the next logging-interval boundary and schedule the operator's next +/// activation there. Returns the time the capability now holds. +/// +/// `now` and `start_offset` must be the ones the logging dataflow was constructed with, so that +/// every collection in it reports on the same boundaries. Scheduling off the boundary rather than +/// off a fixed delay keeps the output frontier progressing at the logging rate without drifting +/// from wall-clock elapsed time. +/// +/// NOTE: downgrading the capability asserts the collection is complete up to the new time, so an +/// operator that samples less often than the logging interval publishes a stale value rather than +/// withholding it. +pub(super) fn downgrade_to_interval_boundary( + cap: &mut Capability, + activator: &Activator, + now: Instant, + start_offset: Duration, + interval_ms: u128, +) -> Timestamp { + let elapsed = now.elapsed().as_millis(); + let time_ms: u128 = ((elapsed + start_offset.as_millis()) / interval_ms + 1) * interval_ms; + let ts: Timestamp = time_ms.try_into().expect("must fit"); + cap.downgrade(&ts); + + let next_boundary_ms = time_ms - start_offset.as_millis(); + let next_activation = + now + Duration::from_millis(next_boundary_ms.try_into().expect("must fit")); + activator.activate_after(next_activation.saturating_duration_since(Instant::now())); + + ts +} + +/// Emit the difference between two snapshots of a sampled source as updates at `ts`. +/// +/// A sampled source reports its whole state on every read, so a changed value has to be expressed +/// as a retraction of the previous one paired with an insertion of the new one. A key absent from +/// `current` is retracted without a replacement, so a source that stops being readable drops out of +/// the collection rather than lingering at its last value. +/// +/// `pack` takes the packer as an argument instead of closing over it, because it hands back rows +/// borrowed from it. +pub(super) fn emit_snapshot_diff( + session: &mut Session<'_, '_, Timestamp, CB, P>, + packer: &mut PermutedRowPacker, + prev: &BTreeMap, + current: &BTreeMap, + ts: Timestamp, + pack: F, +) where + K: Ord, + V: PartialEq, + CB: ContainerBuilder + for<'a> PushInto<((&'a RowRef, &'a RowRef), Timestamp, Diff)>, + P: CapabilityTrait, + F: for<'a> Fn(&'a mut PermutedRowPacker, &K, &V) -> (&'a RowRef, &'a RowRef), +{ + for (key, value) in prev { + if current.get(key) != Some(value) { + let row = pack(packer, key, value); + session.give((row, ts, Diff::MINUS_ONE)); + } + } + for (key, value) in current { + if prev.get(key) != Some(value) { + let row = pack(packer, key, value); + session.give((row, ts, Diff::ONE)); + } + } +} + /// Information about a collection exported from a logging dataflow. struct LogCollection { /// Trace handle providing access to the logged records. diff --git a/src/compute/src/logging/initialize.rs b/src/compute/src/logging/initialize.rs index 860f303a5b864..245cef517d309 100644 --- a/src/compute/src/logging/initialize.rs +++ b/src/compute/src/logging/initialize.rs @@ -182,6 +182,17 @@ impl LoggingContext<'_> { ); collections.extend(prometheus_collections); + let super::resource_usage::Return { + collections: resource_usage_collections, + } = super::resource_usage::construct( + scope, + self.config, + self.now, + self.start_offset, + self.workers_per_process, + ); + collections.extend(resource_usage_collections); + let errs = scope.scoped("logging errors", |scope| { let collection: KeyCollection<_, DataflowErrorSer, Diff> = VecCollection::empty(scope).into(); diff --git a/src/compute/src/logging/prometheus.rs b/src/compute/src/logging/prometheus.rs index 431107ad292be..9c71433fdcac4 100644 --- a/src/compute/src/logging/prometheus.rs +++ b/src/compute/src/logging/prometheus.rs @@ -19,7 +19,7 @@ use mz_ore::cast::{CastFrom, CastLossy}; use mz_ore::collections::CollectionExt; use mz_ore::metrics::MetricsRegistry; use mz_ore::soft_panic_or_log; -use mz_repr::{Datum, Diff, Timestamp}; +use mz_repr::{Datum, Timestamp}; use mz_timely_util::columnar::batcher; use mz_timely_util::columnar::builder::ColumnBuilder; use mz_timely_util::columnar::{Col2ValBatcher, columnar_exchange}; @@ -30,7 +30,10 @@ use timely::dataflow::operators::generic::OutputBuilder; use timely::dataflow::operators::generic::builder_rc::OperatorBuilder; use crate::extensions::arrange::MzArrangeCore; -use crate::logging::{ComputeLog, LogCollection, LogVariant, PermutedRowPacker}; +use crate::logging::{ + ComputeLog, LogCollection, LogVariant, PermutedRowPacker, downgrade_to_interval_boundary, + emit_snapshot_diff, +}; use crate::typedefs::RowRowSpine; use mz_row_spine::RowRowBuilder; @@ -88,24 +91,8 @@ pub(super) fn construct( move |_frontiers| { let Some(cap) = &mut cap else { return }; - // Advance the capability to the next logging interval boundary. - // This keeps the output frontier progressing at the logging - // rate, even when scrapes happen less frequently. Note that - // advancing the frontier implies the data is up-to-date, but - // the metrics snapshot may be stale by up to the scrape - // interval when it exceeds the logging interval. - let elapsed = now.elapsed().as_millis(); - let time_ms: u128 = - ((elapsed + start_offset.as_millis()) / interval_ms + 1) * interval_ms; - let ts: Timestamp = time_ms.try_into().expect("must fit"); - cap.downgrade(&ts); - - // Schedule the next activation at the interval boundary - // to avoid drift from wall-clock elapsed time. - let next_boundary_ms = time_ms - start_offset.as_millis(); - let next_activation = - now + Duration::from_millis(next_boundary_ms.try_into().expect("must fit")); - activator.activate_after(next_activation.saturating_duration_since(Instant::now())); + let ts = + downgrade_to_interval_boundary(cap, &activator, now, start_offset, interval_ms); // Only scrape when the scrape interval has elapsed. // The operator wakes every logging interval to advance the @@ -131,44 +118,18 @@ pub(super) fn construct( // Diff against previous snapshot and emit packed Row pairs. let mut output = output.activate(); let mut session = output.session_with_builder(&cap); - - // Retract entries that were removed or changed. - for (key, old_val) in &prev_snapshot { - match new_snapshot.get(key) { - Some(new_val) if new_val == old_val => {} - _ => { - let (row_key, row_val) = pack_row( - &mut packer, - &key.0, - old_val.1, - &key.1, - old_val.0, - &old_val.2, - process_id, - ); - session.give(((row_key, row_val), ts, Diff::MINUS_ONE)); - } - } - } - - // Insert entries that are new or changed. - for (key, new_val) in &new_snapshot { - match prev_snapshot.get(key) { - Some(old_val) if old_val == new_val => {} - _ => { - let (row_key, row_val) = pack_row( - &mut packer, - &key.0, - new_val.1, - &key.1, - new_val.0, - &new_val.2, - process_id, - ); - session.give(((row_key, row_val), ts, Diff::ONE)); - } - } - } + emit_snapshot_diff( + &mut session, + &mut packer, + &prev_snapshot, + &new_snapshot, + ts, + |packer, key, value| { + pack_row( + packer, &key.0, value.1, &key.1, value.0, &value.2, process_id, + ) + }, + ); prev_snapshot = new_snapshot; } diff --git a/src/compute/src/logging/resource_usage.rs b/src/compute/src/logging/resource_usage.rs new file mode 100644 index 0000000000000..6d1cf773afbdd --- /dev/null +++ b/src/compute/src/logging/resource_usage.rs @@ -0,0 +1,143 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//! Logging dataflow for the resource usage of the replica's processes. +//! +//! The observations are read by `mz_metrics::usage`, on a task that keeps sampling while the +//! timely workers are busy. This dataflow only reports them, so a worker that stalls delays the +//! report without losing an observation: the values it reads are either kernel-maintained +//! high-water marks or the sampler's own folded peaks, neither of which a late read can miss. +//! +//! One row per `(process_id, source, metric)`, so a metric that moves every sample does not drag +//! the stable ones through a retraction with it. + +use std::collections::BTreeMap; +use std::rc::Rc; +use std::time::{Duration, Instant}; + +use mz_metrics::usage::{MetricKey, observations}; +use mz_ore::cast::CastFrom; +use mz_ore::collections::CollectionExt; +use mz_repr::{Datum, Timestamp}; +use mz_row_spine::RowRowBuilder; +use mz_timely_util::columnar::builder::ColumnBuilder; +use mz_timely_util::columnar::{Col2ValBatcher, batcher, columnar_exchange}; +use timely::dataflow::Scope; +use timely::dataflow::channels::pact::ExchangeCore; +use timely::dataflow::operators::generic::OutputBuilder; +use timely::dataflow::operators::generic::builder_rc::OperatorBuilder; + +use crate::extensions::arrange::MzArrangeCore; +use crate::logging::{ + ComputeLog, LogCollection, LogVariant, PermutedRowPacker, downgrade_to_interval_boundary, + emit_snapshot_diff, +}; +use crate::typedefs::RowRowSpine; + +/// The return type of [`construct`]. +pub(super) struct Return { + /// Collections to export. + pub collections: BTreeMap, +} + +/// Constructs the logging dataflow fragment for process resource usage. +pub(super) fn construct( + scope: Scope<'_, Timestamp>, + config: &mz_compute_client::logging::LoggingConfig, + now: Instant, + start_offset: Duration, + workers_per_process: usize, +) -> Return { + let variant = LogVariant::Compute(ComputeLog::ResourceUsage); + let mut collections = BTreeMap::new(); + let interval_ms = std::cmp::max(1, config.interval.as_millis()); + + if !config.index_logs.contains_key(&variant) { + return Return { collections }; + } + + let process_id = scope.index() / workers_per_process; + let enable = scope.index() % workers_per_process == 0; + + let mut builder = OperatorBuilder::new("ResourceUsage".to_string(), scope.clone()); + let (output, stream) = builder.new_output(); + let mut output = OutputBuilder::<_, ColumnBuilder<_>>::from(output); + + let operator_info = builder.operator_info(); + builder.build(move |capabilities| { + // Usage is per-process, so only one worker per process reports it. Drop the capability for + // disabled workers so the frontier can advance without this operator holding it back. + let mut cap = enable.then_some(capabilities.into_element()); + let activator = scope.activator_for(operator_info.address); + + let mut prev: BTreeMap = BTreeMap::new(); + let mut packer = PermutedRowPacker::new(ComputeLog::ResourceUsage); + + move |_frontiers| { + let Some(cap) = &mut cap else { return }; + + // The capability is downgraded on this operator's own timer rather than on the + // sampler's, so a sampler that stops ticking cannot freeze this collection's frontier. + let ts = + downgrade_to_interval_boundary(cap, &activator, now, start_offset, interval_ms); + + let current = observations().unwrap_or_default(); + if prev == current { + return; + } + + let mut output = output.activate(); + let mut session = output.session_with_builder(&cap); + emit_snapshot_diff( + &mut session, + &mut packer, + &prev, + ¤t, + ts, + |packer, key, value| pack_row(packer, process_id, *key, *value), + ); + + prev = current; + } + }); + + let exchange = ExchangeCore::, _>::new_core( + columnar_exchange::, + ); + let trace = stream + .mz_arrange_core::< + _, + batcher::Chunker<_>, + Col2ValBatcher<_, _, _, _>, + RowRowBuilder<_, _>, + RowRowSpine<_, _>, + >(exchange, "Arrange ResourceUsage") + .trace; + let token: Rc = Rc::new(()); + let collection = LogCollection { trace, token }; + collections.insert(variant, collection); + + Return { collections } +} + +/// Pack one observation into key/value row pairs. +fn pack_row( + packer: &mut PermutedRowPacker, + process_id: usize, + (source, metric): MetricKey, + value: u64, +) -> (&mz_repr::RowRef, &mz_repr::RowRef) { + packer.pack_by_index(|row_packer, index| match index { + 0 => row_packer.push(Datum::UInt64(u64::cast_from(process_id))), + 1 => row_packer.push(Datum::String(source)), + 2 => row_packer.push(Datum::String(metric)), + 3 => row_packer.push(Datum::UInt64(value)), + _ => unreachable!("unexpected column index {index}"), + }) +} diff --git a/src/compute/src/memory_limiter.rs b/src/compute/src/memory_limiter.rs index 947f0cb9ec867..cf7e07d903c54 100644 --- a/src/compute/src/memory_limiter.rs +++ b/src/compute/src/memory_limiter.rs @@ -15,11 +15,11 @@ use std::sync::Mutex; use std::time::{Duration, Instant}; -use anyhow::Context; use mz_compute_types::dyncfgs::{ MEMORY_LIMITER_BURST_FACTOR, MEMORY_LIMITER_INTERVAL, MEMORY_LIMITER_USAGE_BIAS, }; use mz_dyncfg::ConfigSet; +use mz_metrics::usage::ProcStatus; use mz_ore::cast::{CastFrom, CastLossy}; use mz_ore::metric; use mz_ore::metrics::{MetricsRegistry, UIntGauge}; @@ -193,10 +193,7 @@ impl LimiterTask { Err(err) } #[cfg(not(target_os = "linux"))] - Err(_err) => Ok(ProcStatus { - vm_rss: 0, - vm_swap: 0, - }), + Err(_err) => Ok(ProcStatus::default()), } } @@ -204,7 +201,9 @@ impl LimiterTask { fn check(&mut self) -> Result<(), anyhow::Error> { debug!("checking memory limits"); - let ProcStatus { vm_rss, vm_swap } = Self::current_utilization()?; + let ProcStatus { + vm_rss, vm_swap, .. + } = Self::current_utilization()?; let memory_limit = self.config.memory_limit; let burst_budget_remaining = self.burst_budget_remaining; @@ -315,14 +314,6 @@ impl LimiterMetrics { } } -/// Helper for reading and parsing `/proc/self/status` on Linux. -pub struct ProcStatus { - /// Resident Set Size (RSS) in bytes. - pub vm_rss: usize, - /// Swap memory in bytes. - pub vm_swap: usize, -} - #[cfg(test)] mod tests { use super::*; @@ -383,34 +374,3 @@ mod tests { assert_eq!(task.last_check, stale); } } - -impl ProcStatus { - /// Populate a new `ProcStatus` with information in /proc/self/status. - pub fn from_proc() -> anyhow::Result { - let contents = std::fs::read_to_string("/proc/self/status")?; - let mut vm_rss = 0; - let mut vm_swap = 0; - - for line in contents.lines() { - if line.starts_with("VmRSS:") { - vm_rss = line - .split_whitespace() - .nth(1) - .ok_or_else(|| anyhow::anyhow!("failed to parse VmRSS"))? - .parse::() - .context("failed to parse VmRSS")? - * 1024 - } else if line.starts_with("VmSwap:") { - vm_swap = line - .split_whitespace() - .nth(1) - .ok_or_else(|| anyhow::anyhow!("failed to parse VmSwap"))? - .parse::() - .context("failed to parse VmSwap")? - * 1024; - } - } - - Ok(Self { vm_rss, vm_swap }) - } -} diff --git a/src/environmentd/src/environmentd/main.rs b/src/environmentd/src/environmentd/main.rs index 60a744a57d0d2..73d1ea5350307 100644 --- a/src/environmentd/src/environmentd/main.rs +++ b/src/environmentd/src/environmentd/main.rs @@ -755,6 +755,8 @@ fn run(mut args: Args) -> Result<(), anyhow::Error> { runtime.block_on(mz_metrics::register_metrics_into( &metrics_registry, mz_dyncfgs::all_dyncfgs(), + // environmentd has no scratch directory, so it tracks no disk usage. + None, )); // Initialize fail crate for failpoint support diff --git a/src/environmentd/src/http/mcp.rs b/src/environmentd/src/http/mcp.rs index e91207586626e..9094b164cf4a5 100644 --- a/src/environmentd/src/http/mcp.rs +++ b/src/environmentd/src/http/mcp.rs @@ -751,6 +751,14 @@ fn endpoint_instructions( } else { "" }; + // Same reasoning for the introspection rule: only route to the + // `query` tool when it is exposed, otherwise say that + // cluster-targeted introspection is unavailable here. + let introspection_rule = if query_tool_enabled { + "- mz_introspection relations (for example mz_dataflow_arrangement_sizes) are cluster-scoped and query_system_catalog answers about the session's default cluster: read them only through the query tool with its cluster argument (and cluster_replica on a multi-replica cluster), never through query_system_catalog" + } else { + "- mz_introspection relations (for example mz_dataflow_arrangement_sizes) are cluster-scoped and query_system_catalog answers about the session's default cluster only; cluster-targeted introspection is unavailable on this server (the query tool is disabled), so say so rather than reporting another cluster's numbers" + }; Some(format!( "You are connected to the Materialize developer MCP server for troubleshooting and observability.\n\n\ Tools:\n\ @@ -765,7 +773,8 @@ fn endpoint_instructions( Key rules:\n\ - mz_source_statuses and mz_sink_statuses use `last_status_change_at` (NOT `updated_at`)\n\ - mz_cluster_replica_utilization only has `replica_id` — JOIN with mz_cluster_replicas and mz_clusters to get names\n\ - - Do NOT query mz_introspection.mz_dataflow_arrangement_sizes — it is cluster-scoped and has uint8/text type mismatches\n\ + {introspection_rule}\n\ + - mz_dataflow_arrangement_sizes.id and mz_dataflows.id are dataflow ids (uint8), not catalog ids: do not JOIN them to mz_catalog.mz_objects.id (text); join through mz_introspection.mz_compute_exports (dataflow_id to export_id) instead, or match the name column, which reads Dataflow: ..\n\ - Use SHOW COLUMNS FROM to verify column names if unsure", )) } @@ -2256,6 +2265,24 @@ mod tests { tool_names.contains(&"query"), "query tool should be present on developer when enabled" ); + + let instructions = endpoint_instructions(McpEndpointType::Developer, true, true) + .expect("developer instructions must be present"); + assert!( + instructions.contains("through the query tool with its cluster argument"), + "instructions must route mz_introspection through query: {instructions}", + ); + } + + #[mz_ore::test] + fn test_developer_instructions_query_tool_disabled() { + let instructions = endpoint_instructions(McpEndpointType::Developer, false, true) + .expect("developer instructions must be present"); + assert!( + instructions.contains("cluster-targeted introspection is unavailable") + && !instructions.contains("through the query tool"), + "instructions must not route to the hidden query tool: {instructions}", + ); } // ── Response size cap tests ──────────────────────────────────────── diff --git a/src/environmentd/src/http/sql.rs b/src/environmentd/src/http/sql.rs index d47c8db4e3561..73628a708f0f0 100644 --- a/src/environmentd/src/http/sql.rs +++ b/src/environmentd/src/http/sql.rs @@ -675,7 +675,7 @@ impl SqlResult { let mut sql_rows = match peek_response { PeekResponseUnary::Rows(rows) => rows, PeekResponseUnary::Error(e) => { - return Ok(SqlResult::err(client, Error::Unstructured(anyhow!(e)))); + return Ok(SqlResult::err(client, e)); } PeekResponseUnary::DependencyDropped(dep) => { return Ok(SqlResult::err(client, dep.to_concurrent_dependency_drop())); @@ -1116,12 +1116,11 @@ impl ResultSender for WebSocket { } } } - Some(PeekResponseUnary::Error(error)) => { + Some(PeekResponseUnary::Error(err)) => { + let error = err.to_string(); break ( true, - vec![WebSocketResponse::Error( - Error::Unstructured(anyhow!(error.clone())).into(), - )], + vec![WebSocketResponse::Error(err.into())], Some((StatementEndedExecutionReason::Errored { error }, ctx_extra)), ); } @@ -1307,9 +1306,7 @@ async fn stream_ws_peek_rows( return Ok(ws_peek_result( client, true, - vec![WebSocketResponse::Error( - Error::Unstructured(anyhow!(error)).into(), - )], + vec![WebSocketResponse::Error(error.into())], )); } Some(PeekResponseUnary::DependencyDropped(dep)) => { diff --git a/src/environmentd/tests/sql.rs b/src/environmentd/tests/sql.rs index 7524fa51ff225..7280a221a9126 100644 --- a/src/environmentd/tests/sql.rs +++ b/src/environmentd/tests/sql.rs @@ -2567,6 +2567,52 @@ fn test_parse_error_codes() { } } +#[mz_ore::test] +#[allow(clippy::disallowed_methods)] +fn test_dataflow_error_codes() { + let server = test_util::TestHarness::default().start_blocking(); + let mut client = server.connect(postgres::NoTls).unwrap(); + + client + .batch_execute("CREATE TABLE t (a int4, b int4)") + .unwrap(); + client.batch_execute("INSERT INTO t VALUES (1, 0)").unwrap(); + + let cases: &[(&str, &SqlState)] = &[ + ("SELECT a / b FROM t", &SqlState::DIVISION_BY_ZERO), + ( + "SELECT 2147483647 + a FROM t", + &SqlState::NUMERIC_VALUE_OUT_OF_RANGE, + ), + ]; + + for (query, expected) in cases { + let err = client.query_one(*query, &[]).unwrap_err().unwrap_db_error(); + assert_eq!( + err.code(), + *expected, + "unexpected SQLSTATE {} for query `{query}`: {}", + err.code().code(), + err.message(), + ); + } + + client + .batch_execute("CREATE MATERIALIZED VIEW mv AS SELECT a / b AS x FROM t;") + .unwrap(); + let err = client + .query_one("SELECT * FROM mv", &[]) + .unwrap_err() + .unwrap_db_error(); + assert_eq!( + err.code(), + &SqlState::DIVISION_BY_ZERO, + "unexpected SQLSTATE {} reading from materialized view: {}", + err.code().code(), + err.message(), + ); +} + #[mz_ore::test] #[allow(clippy::disallowed_methods)] fn test_emit_timestamp_notice() { diff --git a/src/environmentd/tests/testdata/http/ws b/src/environmentd/tests/testdata/http/ws index 23e7a86a396e7..2d744ff369472 100644 --- a/src/environmentd/tests/testdata/http/ws +++ b/src/environmentd/tests/testdata/http/ws @@ -414,7 +414,7 @@ ws-text fixid=true ---- {"type":"Notice","payload":{"message":"{\n \"plans\": {\n \"raw\": {\n \"text\": \"Finish limit=1 output=[#0]\\n Project (#15)\\n Map ((1 / 0))\\n Get mz_catalog.mz_sources\\n\\nTarget cluster: mz_catalog_server\\n\",\n \"json\": {\n \"Project\": {\n \"input\": {\n \"Map\": {\n \"input\": {\n \"Get\": {\n \"id\": {\n \"Global\": {\n \n }\n },\n \"typ\": {\n \"column_types\": [\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"Oid\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": {\n \"Array\": \"MzAclItem\"\n },\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n }\n ],\n \"keys\": [\n [\n 0\n ],\n [\n 1\n ]\n ]\n }\n }\n },\n \"scalars\": [\n {\n \"CallBinary\": {\n \"func\": {\n \"DivInt32\": null\n },\n \"expr1\": {\n \"Literal\": [\n {\n \"data\": [\n 45,\n 1\n ]\n },\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n },\n null\n ]\n },\n \"expr2\": {\n \"Literal\": [\n {\n \"data\": [\n 44\n ]\n },\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n },\n null\n ]\n },\n \"name\": null\n }\n }\n ]\n }\n },\n \"outputs\": [\n 15\n ]\n }\n }\n },\n \"optimized\": {\n \"global\": {\n \"text\": \":\\n Finish limit=1 output=[#0]\\n ArrangeBy keys=[[#0]]\\n ReadGlobalFromSameDataflow \\n\\n:\\n Project (#15)\\n Map (error(\\\"division by zero\\\"))\\n ReadIndex on=mz_sources mz_sources_ind=[*** full scan ***]\\n\\nTarget cluster: mz_catalog_server\\n\",\n \"json\": {\n \"plans\": [\n {\n \"id\": \"\",\n \"plan\": {\n \"ArrangeBy\": {\n \"input\": {\n \"Get\": {\n \"id\": {\n \"Global\": {\n \n }\n },\n \"typ\": {\n \"column_types\": [\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n }\n ],\n \"keys\": []\n },\n \"access_strategy\": \"SameDataflow\"\n }\n },\n \"keys\": [\n [\n {\n \"Column\": [\n 0,\n null\n ]\n }\n ]\n ]\n }\n }\n },\n {\n \"id\": \"\",\n \"plan\": {\n \"Project\": {\n \"input\": {\n \"Map\": {\n \"input\": {\n \"Get\": {\n \"id\": {\n \"Global\": {\n \n }\n },\n \"typ\": {\n \"column_types\": [\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"UInt32\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": false\n },\n {\n \"scalar_type\": {\n \"Array\": \"MzAclItem\"\n },\n \"nullable\": false\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n },\n {\n \"scalar_type\": \"String\",\n \"nullable\": true\n }\n ],\n \"keys\": [\n [\n 0\n ],\n [\n 1\n ]\n ]\n },\n \"access_strategy\": {\n \"Index\": [\n [\n {\n \n },\n \"FullScan\"\n ]\n ]\n }\n }\n },\n \"scalars\": [\n {\n \"Literal\": [\n {\n \"Err\": \"DivisionByZero\"\n },\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n }\n ]\n }\n ]\n }\n },\n \"outputs\": [\n 15\n ]\n }\n }\n }\n ],\n \"sources\": []\n }\n },\n \"fast_path\": {\n \"text\": \"Explained Query (fast path):\\n Finish limit=1 output=[#0]\\n →Map/Filter/Project\\n Project: #15\\n Map: error(\\\"division by zero\\\")\\n →Indexed mz_catalog.mz_sources (using mz_catalog.mz_sources_ind)\\n\\nTarget cluster: mz_catalog_server\\n\",\n \"json\": {\n \"plans\": [\n {\n \"id\": \"Explained Query (fast path)\",\n \"plan\": {\n \"PeekExisting\": [\n {\n \n },\n {\n \n },\n null,\n {\n \"mfp\": {\n \"expressions\": [\n {\n \"Literal\": [\n {\n \"Err\": \"DivisionByZero\"\n },\n {\n \"scalar_type\": \"Int32\",\n \"nullable\": false\n }\n ]\n }\n ],\n \"predicates\": [],\n \"projection\": [\n 15\n ],\n \"input_arity\": 15\n }\n }\n ]\n }\n }\n ],\n \"sources\": []\n }\n }\n }\n },\n \"insights\": {\n \"imports\": {\n \"\": {\n \"name\": {\n \"schema\": \"mz_catalog\",\n \"item\": \"mz_sources_ind\"\n },\n \"type\": \"compute\"\n }\n },\n \"fast_path_clusters\": {},\n \"fast_path_limit\": null,\n \"persist_count\": []\n },\n \"cluster\": {\n \"name\": \"mz_catalog_server\",\n \"id\": {\n \n }\n },\n \"redacted_sql\": \"SELECT '' / '' FROM [ AS mz_catalog.mz_sources] LIMIT ''\"\n}","code":"MZ001","severity":"notice"}} {"type":"CommandStarting","payload":{"has_rows":true,"is_streaming":false}} -{"type":"Error","payload":{"message":"division by zero","code":"XX000"}} +{"type":"Error","payload":{"message":"Evaluation error: division by zero","code":"22012"}} {"type":"ReadyForQuery","payload":"I"} ws-text rows=2 fixtimestamp=true diff --git a/src/environmentd/tests/testdata/mcp/developer b/src/environmentd/tests/testdata/mcp/developer index 6bd5fcc707644..609ba2de29dc7 100644 --- a/src/environmentd/tests/testdata/mcp/developer +++ b/src/environmentd/tests/testdata/mcp/developer @@ -16,7 +16,7 @@ mcp-developer {"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"test-client","version":"1.0.0"}}} ---- 200 OK -{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-11-25","capabilities":{"tools":{}},"serverInfo":{"name":"materialize-mcp-developer","version":""},"instructions":"You are connected to the Materialize developer MCP server for troubleshooting and observability.\n\nTools:\n- query_system_catalog: read-only SELECT/SHOW/EXPLAIN restricted to system catalog tables (mz_*, pg_catalog, information_schema). No cluster argument; prefer this for most catalog lookups.\n\nIMPORTANT: Before writing queries, discover table schemas using the mz_ontology tables:\n- mz_internal.mz_ontology_entity_types: what catalog entities exist and which tables they map to\n- mz_internal.mz_ontology_link_types: relationships between entities (foreign keys, metrics, etc.)\n- mz_internal.mz_ontology_properties: column names, types, and descriptions for each entity\n- mz_internal.mz_ontology_semantic_types: typed ID domains (CatalogItemId, ReplicaId, etc.)\n\nUse these to find the correct tables, join paths, and column names instead of guessing.\n\nKey rules:\n- mz_source_statuses and mz_sink_statuses use `last_status_change_at` (NOT `updated_at`)\n- mz_cluster_replica_utilization only has `replica_id` — JOIN with mz_cluster_replicas and mz_clusters to get names\n- Do NOT query mz_introspection.mz_dataflow_arrangement_sizes — it is cluster-scoped and has uint8/text type mismatches\n- Use SHOW COLUMNS FROM
to verify column names if unsure"}} +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-11-25","capabilities":{"tools":{}},"serverInfo":{"name":"materialize-mcp-developer","version":""},"instructions":"You are connected to the Materialize developer MCP server for troubleshooting and observability.\n\nTools:\n- query_system_catalog: read-only SELECT/SHOW/EXPLAIN restricted to system catalog tables (mz_*, pg_catalog, information_schema). No cluster argument; prefer this for most catalog lookups.\n\nIMPORTANT: Before writing queries, discover table schemas using the mz_ontology tables:\n- mz_internal.mz_ontology_entity_types: what catalog entities exist and which tables they map to\n- mz_internal.mz_ontology_link_types: relationships between entities (foreign keys, metrics, etc.)\n- mz_internal.mz_ontology_properties: column names, types, and descriptions for each entity\n- mz_internal.mz_ontology_semantic_types: typed ID domains (CatalogItemId, ReplicaId, etc.)\n\nUse these to find the correct tables, join paths, and column names instead of guessing.\n\nKey rules:\n- mz_source_statuses and mz_sink_statuses use `last_status_change_at` (NOT `updated_at`)\n- mz_cluster_replica_utilization only has `replica_id` — JOIN with mz_cluster_replicas and mz_clusters to get names\n- mz_introspection relations (for example mz_dataflow_arrangement_sizes) are cluster-scoped and query_system_catalog answers about the session's default cluster only; cluster-targeted introspection is unavailable on this server (the query tool is disabled), so say so rather than reporting another cluster's numbers\n- mz_dataflow_arrangement_sizes.id and mz_dataflows.id are dataflow ids (uint8), not catalog ids: do not JOIN them to mz_catalog.mz_objects.id (text); join through mz_introspection.mz_compute_exports (dataflow_id to export_id) instead, or match the name column, which reads Dataflow: ..\n- Use SHOW COLUMNS FROM
to verify column names if unsure"}} # ============================================================================= # Tools List diff --git a/src/environmentd/tests/testdata/mcp/developer_query_tool b/src/environmentd/tests/testdata/mcp/developer_query_tool index 042c40e42bfcc..c98092ad08953 100644 --- a/src/environmentd/tests/testdata/mcp/developer_query_tool +++ b/src/environmentd/tests/testdata/mcp/developer_query_tool @@ -17,7 +17,7 @@ mcp-developer {"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"test-client","version":"1.0.0"}}} ---- 200 OK -{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-11-25","capabilities":{"tools":{}},"serverInfo":{"name":"materialize-mcp-developer","version":""},"instructions":"You are connected to the Materialize developer MCP server for troubleshooting and observability.\n\nTools:\n- query_system_catalog: read-only SELECT/SHOW/EXPLAIN restricted to system catalog tables (mz_*, pg_catalog, information_schema). No cluster argument; prefer this for most catalog lookups.\n- query: read-only SELECT/SHOW/EXPLAIN that can also reach user objects on a named cluster. Use this for EXPLAIN ANALYZE and for inspecting user objects directly.\n\nIMPORTANT: Before writing queries, discover table schemas using the mz_ontology tables:\n- mz_internal.mz_ontology_entity_types: what catalog entities exist and which tables they map to\n- mz_internal.mz_ontology_link_types: relationships between entities (foreign keys, metrics, etc.)\n- mz_internal.mz_ontology_properties: column names, types, and descriptions for each entity\n- mz_internal.mz_ontology_semantic_types: typed ID domains (CatalogItemId, ReplicaId, etc.)\n\nUse these to find the correct tables, join paths, and column names instead of guessing.\n\nKey rules:\n- mz_source_statuses and mz_sink_statuses use `last_status_change_at` (NOT `updated_at`)\n- mz_cluster_replica_utilization only has `replica_id` — JOIN with mz_cluster_replicas and mz_clusters to get names\n- Do NOT query mz_introspection.mz_dataflow_arrangement_sizes — it is cluster-scoped and has uint8/text type mismatches\n- Use SHOW COLUMNS FROM
to verify column names if unsure"}} +{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-11-25","capabilities":{"tools":{}},"serverInfo":{"name":"materialize-mcp-developer","version":""},"instructions":"You are connected to the Materialize developer MCP server for troubleshooting and observability.\n\nTools:\n- query_system_catalog: read-only SELECT/SHOW/EXPLAIN restricted to system catalog tables (mz_*, pg_catalog, information_schema). No cluster argument; prefer this for most catalog lookups.\n- query: read-only SELECT/SHOW/EXPLAIN that can also reach user objects on a named cluster. Use this for EXPLAIN ANALYZE and for inspecting user objects directly.\n\nIMPORTANT: Before writing queries, discover table schemas using the mz_ontology tables:\n- mz_internal.mz_ontology_entity_types: what catalog entities exist and which tables they map to\n- mz_internal.mz_ontology_link_types: relationships between entities (foreign keys, metrics, etc.)\n- mz_internal.mz_ontology_properties: column names, types, and descriptions for each entity\n- mz_internal.mz_ontology_semantic_types: typed ID domains (CatalogItemId, ReplicaId, etc.)\n\nUse these to find the correct tables, join paths, and column names instead of guessing.\n\nKey rules:\n- mz_source_statuses and mz_sink_statuses use `last_status_change_at` (NOT `updated_at`)\n- mz_cluster_replica_utilization only has `replica_id` — JOIN with mz_cluster_replicas and mz_clusters to get names\n- mz_introspection relations (for example mz_dataflow_arrangement_sizes) are cluster-scoped and query_system_catalog answers about the session's default cluster: read them only through the query tool with its cluster argument (and cluster_replica on a multi-replica cluster), never through query_system_catalog\n- mz_dataflow_arrangement_sizes.id and mz_dataflows.id are dataflow ids (uint8), not catalog ids: do not JOIN them to mz_catalog.mz_objects.id (text); join through mz_introspection.mz_compute_exports (dataflow_id to export_id) instead, or match the name column, which reads Dataflow: ..\n- Use SHOW COLUMNS FROM
to verify column names if unsure"}} # ============================================================================= # Tools List - query tool enabled diff --git a/src/metrics/Cargo.toml b/src/metrics/Cargo.toml index af03146bef6c3..8f40864dd6f1c 100644 --- a/src/metrics/Cargo.toml +++ b/src/metrics/Cargo.toml @@ -10,15 +10,20 @@ publish = false workspace = true [dependencies] +anyhow.workspace = true lgalloc.workspace = true libc.workspace = true mz-dyncfg = { path = "../dyncfg" } mz-ore = { path = "../ore", features = ["metrics"] } +nix.workspace = true paste.workspace = true prometheus.workspace = true thiserror.workspace = true tokio.workspace = true tracing.workspace = true +[dev-dependencies] +mz-ore = { path = "../ore", features = ["metrics", "test"] } + [features] default = [] diff --git a/src/metrics/src/dyncfgs.rs b/src/metrics/src/dyncfgs.rs index f75c44a314ee6..22f4f5ed2b1b2 100644 --- a/src/metrics/src/dyncfgs.rs +++ b/src/metrics/src/dyncfgs.rs @@ -37,10 +37,24 @@ pub(crate) const MZ_METRICS_RUSAGE_REFRESH_INTERVAL: Config = Config:: ParameterScope::Replica, ); +/// How frequently to sample process resource usage. +/// +/// This interval bounds how short a spike can be and still be observed, but only for the sources +/// with no kernel-side high-water mark. A kernel-maintained peak such as `cgroup memory.peak` is +/// exact no matter how rarely it is read. Deliberately separate from `memory_limiter_interval`, +/// which governs OOM-kill behavior and must not be retuned for introspection's sake. +pub(crate) const MZ_METRICS_USAGE_REFRESH_INTERVAL: Config = Config::new( + "mz_metrics_usage_refresh_interval", + Duration::from_secs(5), + "How frequently to sample process resource usage. A zero duration disables sampling.", + ParameterScope::Replica, +); + /// Adds the full set of all storage `Config`s. pub fn all_dyncfgs(configs: ConfigSet) -> ConfigSet { configs .add(&MZ_METRICS_LGALLOC_MAP_REFRESH_INTERVAL) .add(&MZ_METRICS_LGALLOC_REFRESH_INTERVAL) .add(&MZ_METRICS_RUSAGE_REFRESH_INTERVAL) + .add(&MZ_METRICS_USAGE_REFRESH_INTERVAL) } diff --git a/src/metrics/src/lib.rs b/src/metrics/src/lib.rs index d024044763a31..ff0df44a62f44 100644 --- a/src/metrics/src/lib.rs +++ b/src/metrics/src/lib.rs @@ -17,6 +17,7 @@ #![warn(missing_docs, missing_debug_implementations)] +use std::path::PathBuf; use std::time::Duration; use mz_dyncfg::{ConfigSet, ConfigUpdates}; @@ -28,6 +29,7 @@ pub use dyncfgs::all_dyncfgs; mod dyncfgs; pub mod lgalloc; pub mod rusage; +pub mod usage; /// Handle to metrics defined in this crate. #[derive(Debug)] @@ -36,6 +38,7 @@ pub struct Metrics { lgalloc: MetricsTask, lgalloc_map: MetricsTask, rusage: MetricsTask, + usage: MetricsTask, } static METRICS: std::sync::Mutex> = std::sync::Mutex::new(None); @@ -47,8 +50,15 @@ static METRICS: std::sync::Mutex> = std::sync::Mutex::new(None); /// remove the shared static mutex and make this function return a handle to the metrics. /// /// This function is async, because it needs to be called from a tokio runtime context. +/// +/// `disk_root` is a directory whose filesystem usage should be tracked, or `None` for processes +/// that do not use disk. #[allow(clippy::unused_async)] -pub async fn register_metrics_into(metrics_registry: &MetricsRegistry, config_set: ConfigSet) { +pub async fn register_metrics_into( + metrics_registry: &MetricsRegistry, + config_set: ConfigSet, + disk_root: Option, +) { let update_duration_metric = metrics_registry.register(mz_ore::metric!( name: "mz_metrics_update_duration", help: "The time it took to update lgalloc stats", @@ -75,10 +85,18 @@ pub async fn register_metrics_into(metrics_registry: &MetricsRegistry, config_se &update_duration_metric, ); + let usage = Metrics::new_metrics_task( + metrics_registry, + |registry| usage::register_metrics_into(registry, disk_root), + dyncfgs::MZ_METRICS_USAGE_REFRESH_INTERVAL, + &update_duration_metric, + ); + *METRICS.lock().expect("lock poisoned") = Some(Metrics { lgalloc, lgalloc_map, rusage, + usage, config_set, }); } @@ -146,6 +164,7 @@ impl Metrics { self.lgalloc.update_dyncfg(&self.config_set); self.lgalloc_map.update_dyncfg(&self.config_set); self.rusage.update_dyncfg(&self.config_set); + self.usage.update_dyncfg(&self.config_set); } fn new_metrics_task( diff --git a/src/metrics/src/rusage.rs b/src/metrics/src/rusage.rs index f1d97fded526a..c5b59cab95b5c 100644 --- a/src/metrics/src/rusage.rs +++ b/src/metrics/src/rusage.rs @@ -133,6 +133,22 @@ metrics! { (ru_nivcsw, "involuntary context switches", "_total", Unitless) } +/// Read this process's peak resident set size, in bytes. +/// +/// The kernel maintains this high-water mark itself, so it cannot miss a short-lived spike the +/// way a sampled maximum can. +pub(crate) fn max_rss_bytes() -> Result { + let rusage = unsafe { + let mut rusage = std::mem::zeroed(); + let ret = libc::getrusage(libc::RUSAGE_SELF, &mut rusage); + if ret < 0 { + return Err(std::io::Error::last_os_error()); + } + rusage + }; + Ok(::from(rusage.ru_maxrss)) +} + /// Register a task to read rusage stats. pub(crate) fn register_metrics_into(metrics_registry: &MetricsRegistry) -> RuMetrics { RuMetrics::new(metrics_registry) diff --git a/src/metrics/src/usage.rs b/src/metrics/src/usage.rs new file mode 100644 index 0000000000000..ddea17fd67f21 --- /dev/null +++ b/src/metrics/src/usage.rs @@ -0,0 +1,449 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +//! Resource usage observations for the current process. +//! +//! This module is a mechanism, not a policy. It reports what each source says, under that +//! source's own name, and never combines two sources into a third figure. The discrepancies +//! between sources carry information: cgroup memory far above `VmRSS` means page cache or kernel +//! memory is charged to the replica, which is exactly the case a single fused "memory" number +//! hides. Deciding which source answers "how much memory is this replica using" belongs to the +//! SQL views built on top. +//! +//! Peaks are observations too. `cgroup memory.peak` and `getrusage`'s `ru_maxrss` are high-water +//! marks the kernel maintains itself, so they carry no sampling error and outlive anything short +//! of the process exiting. Only a source with no kernel-side peak gets one folded here, reported +//! under a distinct metric name so a caller can tell an exact peak from a sampled one. + +use std::collections::BTreeMap; +use std::convert::Infallible; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use anyhow::Context; +use mz_ore::cast::CastFrom; +use mz_ore::cgroup::CgroupV2; +use mz_ore::metric; +use mz_ore::metrics::{MetricsRegistry, raw}; +use tracing::{debug, info}; + +use crate::MetricsUpdate; + +/// Identifies one observation: which reader produced it, and what it measures. +/// +/// Static strings rather than an enum, so adding a metric touches only the sampler. +pub type MetricKey = (&'static str, &'static str); + +/// The most recent observations, keyed by [`MetricKey`]. +/// +/// Published as a whole map rather than as independent cells so a reader always sees one +/// self-consistent sample. Reading a set of independent atomics could mix two samples into a +/// combination that never existed, for instance a peak below the current value it bounds. +static OBSERVATIONS: Mutex>> = Mutex::new(None); + +/// Source names, as reported alongside each observation. +pub mod source { + /// This process's cgroup v2 interface files. + pub const CGROUP: &str = "cgroup"; + /// `getrusage(RUSAGE_SELF)`. + pub const RUSAGE: &str = "rusage"; + /// `/proc/self/status`. + pub const PROC_STATUS: &str = "proc_status"; + /// `statvfs` on the scratch directory's filesystem. + pub const STATVFS: &str = "statvfs"; +} + +/// Return the most recent resource usage observations of this process. +/// +/// `None` until the sampler has taken its first sample, and forever in a process that never +/// registered one. An observation the sampler could not read is absent from the map, never +/// present as zero. +pub fn observations() -> Option> { + OBSERVATIONS.lock().expect("poisoned").clone() +} + +/// Observations that get a peak folded in this process, because their source has no kernel-side +/// high-water mark. Each is a maximum over samples and therefore a lower bound on the true peak. +const DERIVED_PEAKS: &[(MetricKey, &str)] = &[ + ((source::STATVFS, "fs_used"), "fs_used_peak"), + // Load-bearing only below the kernel version that provides `memory.swap.peak`. + ((source::PROC_STATUS, "vm_swap"), "vm_swap_peak"), + // A lower bound on the peak of the quantity the memory limiter enforces. No upper bound is + // available: the cgroup peaks describe a smaller quantity, excluding resident file-backed + // pages, and `ru_maxrss` is refreshed at kernel checkpoints and has been seen reading below + // the concurrent `vm_rss`. + ((source::PROC_STATUS, "heap"), "heap_peak"), +]; + +/// Sampler of resource usage observations, driven by the metrics update task. +/// +/// Sampling happens here rather than in a compute logging operator because a saturated timely +/// worker stops scheduling its logging operators during exactly the episodes we most want +/// sampled. Folding the derived peaks here also keeps them alive across a teardown and rebuild of +/// the logging dataflow, which an operator-local fold would lose. +pub(crate) struct UsageMetrics { + /// This process's cgroup, if it has a v2 one with the memory controller enabled. + cgroup: Option, + /// Directory whose filesystem usage is tracked, if disk is in use. + disk_root: Option, + /// Peaks folded here, for the sources listed in [`DERIVED_PEAKS`]. + derived_peaks: BTreeMap, + gauges: raw::UIntGaugeVec, +} + +impl UsageMetrics { + fn new(registry: &MetricsRegistry, disk_root: Option) -> Self { + // Readings taken from the wrong cgroup look plausible rather than absent, and diagnosing + // that otherwise takes access to the container. Reported once, at registration. + let cgroup = CgroupV2::detect(); + match &cgroup { + Some(cgroup) => info!( + dir = %cgroup.path().display(), + "reading resource usage from cgroup v2", + ), + None => info!("no cgroup v2 with a memory controller; cgroup usage unavailable"), + } + + Self { + cgroup, + disk_root, + derived_peaks: BTreeMap::new(), + gauges: registry.register(metric!( + name: "mz_metrics_resource_usage", + help: "Resource usage observations, by source and metric.", + var_labels: ["source", "metric"], + )), + } + } + + /// Read every source, without interpreting any of them. + fn sample(&self) -> BTreeMap { + let mut out = BTreeMap::new(); + let mut put = |source: &'static str, metric: &'static str, value: Option| { + if let Some(value) = value { + out.insert((source, metric), value); + } + }; + + if let Some(cgroup) = &self.cgroup { + // `memory.peak` and `memory.swap.peak` are absent on kernels too old to provide them + // and read as `None` there. `memory.current` is the accounting that limit enforcement + // and the OOM killer act on, which is why it is worth reporting next to `vm_rss`. + let files: &[(&'static str, &str)] = &[ + ("memory_current", "memory.current"), + ("memory_peak", "memory.peak"), + ("memory_max", "memory.max"), + ("swap_current", "memory.swap.current"), + ("swap_peak", "memory.swap.peak"), + ("swap_max", "memory.swap.max"), + ]; + for &(metric, file) in files { + put(source::CGROUP, metric, cgroup.read_u64(file)); + } + + // `oom_kill` counts kills inside this cgroup and `max` counts times the limit was + // hit, which together answer whether a replica died of memory pressure. + let keyed: &[(&'static str, &str, &str)] = &[ + ("anon", "memory.stat", "anon"), + ("file", "memory.stat", "file"), + ("shmem", "memory.stat", "shmem"), + // Pages held in memory with their swap slot still allocated. They are charged + // twice, once as `anon` here and once in `memory.swap.current`, and they are the + // whole of the difference between that and `proc_status vm_swap`. + ("swapcached", "memory.stat", "swapcached"), + ("kernel", "memory.stat", "kernel"), + ("slab", "memory.stat", "slab"), + ("sock", "memory.stat", "sock"), + ("events_max", "memory.events", "max"), + ("events_oom_kill", "memory.events", "oom_kill"), + ]; + for &(metric, file, key) in keyed { + put(source::CGROUP, metric, cgroup.read_keyed_u64(file, key)); + } + } + + put(source::RUSAGE, "max_rss", max_rss_bytes()); + + match ProcStatus::from_proc() { + Ok(status) => { + put(source::PROC_STATUS, "vm_rss", Some(status.rss())); + put(source::PROC_STATUS, "vm_swap", Some(status.swap())); + // The quantity the memory limiter enforces against `--heap-limit`. Reported as + // its own observation because the kernel maintains no combined memory-plus-swap + // peak, so a peak of the sum has to be folded from samples of the sum. It is one + // source added to itself, not two sources fused. + put(source::PROC_STATUS, "heap", Some(status.heap())); + // Decomposes `vm_rss`. `rss_file` is the part charged to another cgroup, so it + // explains the gap between `vm_rss` and `cgroup memory_current`. + put(source::PROC_STATUS, "rss_anon", Some(status.rss_anon())); + put(source::PROC_STATUS, "rss_file", Some(status.rss_file())); + put(source::PROC_STATUS, "rss_shmem", Some(status.rss_shmem())); + } + Err(err) => debug!("failed to read /proc/self/status: {err}"), + } + + if let Some(root) = self.disk_root.as_deref() { + // NOTE: filesystem-wide used bytes, not this process's usage. Named for what it is, + // since on a shared filesystem it counts writes this replica never made. + match disk_usage(root) { + Ok(bytes) => put(source::STATVFS, "fs_used", Some(bytes)), + Err(err) => debug!("statvfs on {} failed: {err}", root.display()), + } + } + + out + } + + /// Fold the derived peaks over `sample`, adding them to it. + fn fold_derived_peaks(&mut self, sample: &mut BTreeMap) { + for ((source, metric), peak_metric) in DERIVED_PEAKS { + let Some(&value) = sample.get(&(*source, *metric)) else { + continue; + }; + let peak = self + .derived_peaks + .entry((source, peak_metric)) + .and_modify(|peak| *peak = (*peak).max(value)) + .or_insert(value); + sample.insert((source, peak_metric), *peak); + } + } +} + +impl MetricsUpdate for UsageMetrics { + type Error = Infallible; + const NAME: &'static str = "usage"; + + fn update(&mut self) -> Result<(), Self::Error> { + let mut sample = self.sample(); + self.fold_derived_peaks(&mut sample); + + for ((source, metric), value) in &sample { + self.gauges.with_label_values(&[source, metric]).set(*value); + } + + *OBSERVATIONS.lock().expect("poisoned") = Some(sample); + + Ok(()) + } +} + +/// Register the resource usage sampler. +/// +/// `disk_root` is a directory on the filesystem whose usage should be tracked, or `None` if this +/// process does not use disk. +pub(crate) fn register_metrics_into( + registry: &MetricsRegistry, + disk_root: Option, +) -> UsageMetrics { + UsageMetrics::new(registry, disk_root) +} + +/// Return the used bytes of the filesystem containing `root`. +/// +/// Callers decide how to report a failure. The sampler polls this on a short interval, so logging +/// an error here would repeat for as long as the directory is unavailable. +pub fn disk_usage(root: &Path) -> Result { + let stat = nix::sys::statvfs::statvfs(root)?; + + // `fsblkcnt_t` is a `u32` on macOS but a `u64` on Linux. + #[allow(clippy::useless_conversion)] + let used_blocks = u64::from(stat.blocks() - stat.blocks_available()); + let used_bytes = used_blocks * stat.fragment_size(); + + debug!("disk usage: {used_bytes}"); + + Ok(used_bytes) +} + +/// Return this process's peak resident set size, in bytes. +/// +/// This is the kernel's own high-water mark, so unlike a sampled maximum it cannot miss a +/// short-lived spike. +fn max_rss_bytes() -> Option { + match crate::rusage::max_rss_bytes() { + Ok(bytes) => u64::try_from(bytes).ok(), + Err(err) => { + debug!("getrusage failed: {err}"); + None + } + } +} + +/// Memory usage of the current process, read from `/proc/self/status`. +/// +/// The `rss_*` fields decompose `vm_rss`. The decomposition is load-bearing rather than +/// decorative: `rss_file` counts pages of file-backed mappings, most of it this binary's own text, +/// and those pages are charged to whichever cgroup first faulted them in. On a Kubernetes node +/// that is the runtime that unpacked the image, not the replica, so `vm_rss` runs a roughly +/// constant amount above the replica's own cgroup charge. +#[derive(Clone, Copy, Debug, Default)] +pub struct ProcStatus { + /// Resident Set Size (RSS) in bytes. + pub vm_rss: usize, + /// Swap memory in bytes. + pub vm_swap: usize, + /// Resident anonymous memory in bytes. + pub rss_anon: usize, + /// Resident file-backed memory in bytes. + pub rss_file: usize, + /// Resident shared memory in bytes. + pub rss_shmem: usize, +} + +impl ProcStatus { + /// Read a new `ProcStatus` from `/proc/self/status`. + /// + /// Fails on platforms without a Linux-style procfs. + pub fn from_proc() -> anyhow::Result { + let contents = std::fs::read_to_string("/proc/self/status")?; + let mut status = Self::default(); + + for line in contents.lines() { + let (field, target) = match line.split_once(':') { + Some(("VmRSS", rest)) => ("VmRSS", (&mut status.vm_rss, rest)), + Some(("VmSwap", rest)) => ("VmSwap", (&mut status.vm_swap, rest)), + Some(("RssAnon", rest)) => ("RssAnon", (&mut status.rss_anon, rest)), + Some(("RssFile", rest)) => ("RssFile", (&mut status.rss_file, rest)), + Some(("RssShmem", rest)) => ("RssShmem", (&mut status.rss_shmem, rest)), + _ => continue, + }; + let (slot, rest) = target; + *slot = parse_kib(rest).with_context(|| format!("failed to parse {field}"))?; + } + + Ok(status) + } + + /// Memory (RAM) usage, in bytes. + pub fn rss(&self) -> u64 { + u64::cast_from(self.vm_rss) + } + + /// Swap usage, in bytes. + pub fn swap(&self) -> u64 { + u64::cast_from(self.vm_swap) + } + + /// Heap (RAM + swap) usage, in bytes. + pub fn heap(&self) -> u64 { + self.rss().saturating_add(self.swap()) + } + + /// Resident anonymous memory, in bytes. + pub fn rss_anon(&self) -> u64 { + u64::cast_from(self.rss_anon) + } + + /// Resident file-backed memory, in bytes. + pub fn rss_file(&self) -> u64 { + u64::cast_from(self.rss_file) + } + + /// Resident shared memory, in bytes. + pub fn rss_shmem(&self) -> u64 { + u64::cast_from(self.rss_shmem) + } +} + +/// Parse the value part of a `/proc/self/status` line reporting a size in KiB, returning bytes. +fn parse_kib(rest: &str) -> anyhow::Result { + let kib: usize = rest + .split_whitespace() + .next() + .ok_or_else(|| anyhow::anyhow!("missing value: {rest}"))? + .parse()?; + Ok(kib * 1024) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn metrics_for_test() -> UsageMetrics { + UsageMetrics::new(&MetricsRegistry::new(), None) + } + + /// Something must be measurable on the platforms we test on, otherwise the other tests here + /// assert nothing. + #[mz_ore::test] + fn sample_is_not_empty() { + let metrics = metrics_for_test(); + assert!(!metrics.sample().is_empty()); + } + + /// A derived peak must rise with a higher observation and survive a lower one. + /// + /// Drives the fold over a synthetic sample rather than through `update`, so the assertion does + /// not depend on this machine's disk usage actually moving. + #[mz_ore::test] + fn derived_peaks_are_monotonic() { + let mut metrics = metrics_for_test(); + let key = (source::STATVFS, "fs_used"); + let peak_key = (source::STATVFS, "fs_used_peak"); + + let mut fold = |value| { + let mut sample = BTreeMap::from_iter([(key, value)]); + metrics.fold_derived_peaks(&mut sample); + sample[&peak_key] + }; + + assert_eq!(fold(100), 100, "first observation sets the peak"); + assert_eq!(fold(200), 200, "a higher observation raises the peak"); + assert_eq!(fold(50), 200, "a lower observation must not lower the peak"); + } + + /// `vm_rss` must decompose exactly into its three parts, since a caller comparing `rss_file` + /// against a cgroup charge relies on the decomposition being complete. + #[mz_ore::test] + #[cfg_attr(not(target_os = "linux"), ignore = "requires a Linux procfs")] + fn vm_rss_decomposes() { + let status = ProcStatus::from_proc().expect("procfs available"); + assert_eq!( + status.rss(), + status.rss_anon() + status.rss_file() + status.rss_shmem(), + "vm_rss {} != anon {} + file {} + shmem {}", + status.rss(), + status.rss_anon(), + status.rss_file(), + status.rss_shmem(), + ); + } + + /// `heap` must be exactly the sum the memory limiter compares against its limit, since a + /// caller reading `heap_peak` to ask how close a replica came to a kill relies on it. + #[mz_ore::test] + #[cfg_attr(not(target_os = "linux"), ignore = "requires a Linux procfs")] + fn heap_is_rss_plus_swap() { + let metrics = metrics_for_test(); + let sample = metrics.sample(); + + let get = |metric| sample[&(source::PROC_STATUS, metric)]; + assert_eq!(get("heap"), get("vm_rss") + get("vm_swap")); + } + + /// A source that cannot be read is absent, never zero. + #[mz_ore::test] + fn unmeasured_observation_is_absent() { + let metrics = metrics_for_test(); + let sample = metrics.sample(); + + // No `disk_root` was configured, so nothing from `statvfs` may appear. + assert!(sample.keys().all(|(source, _)| *source != source::STATVFS)); + } + + /// A derived peak is only published for a source that was actually read. + #[mz_ore::test] + fn derived_peak_needs_an_observation() { + let mut metrics = metrics_for_test(); + let mut sample = BTreeMap::new(); + metrics.fold_derived_peaks(&mut sample); + assert!(sample.is_empty()); + } +} diff --git a/src/ore/src/cgroup.rs b/src/ore/src/cgroup.rs index 318546655248a..d1d851409103f 100644 --- a/src/ore/src/cgroup.rs +++ b/src/ore/src/cgroup.rs @@ -14,6 +14,11 @@ // limitations under the License. //! Linux cgroup detection utilities. +//! +//! NOTE: this module must stay free of non-`std` dependencies. It is compiled unconditionally, +//! including into feature-reduced builds such as the wasm32 one, where `mz_ore`'s optional +//! dependencies are absent. Reaching for `tracing` here breaks that build. Callers that want the +//! resolved cgroup reported should log [`CgroupV2::path`] themselves. use std::fs::File; use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; @@ -172,6 +177,69 @@ pub fn detect_memory_limit() -> Option { read_v1_memory_limit(&cgroups, &v1_mounts) } +/// The directory holding this process's cgroup v2 interface files. +/// +/// Resolving the directory walks `/proc/self/mountinfo` and `/proc/self/cgroup`, so callers that +/// read repeatedly should [`CgroupV2::detect`] once and keep the handle. +#[derive(Clone, Debug)] +pub struct CgroupV2 { + dir: PathBuf, +} + +impl CgroupV2 { + /// Resolve this process's cgroup v2 directory, if it has one with the memory controller + /// enabled. + /// + /// Returns `None` on a v1-only hierarchy, a mixed hierarchy, and on non-Linux platforms. + pub fn detect() -> Option { + let (v2_mounts, _v1_mounts) = parse_proc_self_mountinfo()?; + let cgroups = parse_proc_self_cgroup()?; + + // cgroups v2 supports only a single cgroup per process. + let mount = v2_mounts.first()?; + if mount.root != cgroups.first()?.root { + // Mixed v1/v2 hierarchies are not supported. + return None; + } + + let dir = &mount.mount_point; + let controllers = std::fs::read_to_string(dir.join("cgroup.controllers")).ok()?; + if !controllers.trim().split(' ').any(|c| c == "memory") { + return None; + } + + Some(Self { dir: dir.clone() }) + } + + /// The resolved directory. + /// + /// Readings taken from the wrong cgroup look plausible rather than absent, so callers are + /// expected to report this once so that a misresolution is diagnosable without container + /// access. This crate does not log it itself, since `tracing` is an optional dependency here. + pub fn path(&self) -> &Path { + &self.dir + } + + /// Read an interface file holding a single integer, in bytes or as a count. + /// + /// Returns `None` when the file is absent, which is how a kernel too old to provide it + /// presents, and when it holds a non-integer. `memory.max` and friends read as `None` when + /// unlimited, since they then hold the literal `max`. + pub fn read_u64(&self, file: &str) -> Option { + let contents = std::fs::read_to_string(self.dir.join(file)).ok()?; + contents.trim().parse().ok() + } + + /// Read a `key value` interface file, returning the value for `key`. + pub fn read_keyed_u64(&self, file: &str, key: &str) -> Option { + let contents = std::fs::read_to_string(self.dir.join(file)).ok()?; + contents.lines().find_map(|line| { + let (name, value) = line.split_once(' ')?; + (name == key).then(|| value.trim().parse().ok())? + }) + } +} + #[cfg(test)] mod tests { use std::path::PathBuf; diff --git a/src/ore/src/lib.rs b/src/ore/src/lib.rs index fd9b1f40da1f0..e1ea84d82fca1 100644 --- a/src/ore/src/lib.rs +++ b/src/ore/src/lib.rs @@ -30,6 +30,7 @@ pub mod bits; #[cfg(feature = "bytes")] pub mod bytes; pub mod cast; +pub mod cgroup; #[cfg_attr(nightly_doc_features, doc(cfg(feature = "async")))] #[cfg(feature = "async")] pub mod channel; diff --git a/src/ore/src/memory.rs b/src/ore/src/memory.rs index 76a2d73f0f86a..72ec906635dde 100644 --- a/src/ore/src/memory.rs +++ b/src/ore/src/memory.rs @@ -15,14 +15,6 @@ //! Physical memory introspection. -// Only this probe uses the cgroup helpers, so the module lives here rather -// than at the crate root. The file carries more surface than the probe -// needs, hence the dead-code allowance. -#[cfg(target_os = "linux")] -#[path = "cgroup.rs"] -#[allow(dead_code)] -mod cgroup; - /// Returns the physical memory available to this process in bytes: the /// host's RAM, clamped by the cgroup memory limit when one is set. Both /// cgroup v1 and v2 are honored, resolved through `/proc/self/mountinfo` @@ -81,7 +73,7 @@ fn host_memory_bytes() -> Option { /// The RAM limit of the cgroup governing this process, if any. #[cfg(target_os = "linux")] fn cgroup_memory_max() -> Option { - cgroup::detect_memory_limit()?.max + crate::cgroup::detect_memory_limit()?.max } #[cfg(not(target_os = "linux"))] diff --git a/src/pgrepr-consts/src/oid.rs b/src/pgrepr-consts/src/oid.rs index c2f7d677821a6..aa820195e6f1a 100644 --- a/src/pgrepr-consts/src/oid.rs +++ b/src/pgrepr-consts/src/oid.rs @@ -829,3 +829,4 @@ pub const VIEW_MZ_BUILTIN_VIEWS_OID: u32 = 17119; pub const MV_MZ_METRIC_SINKS_OID: u32 = 17120; pub const INDEX_MZ_METRIC_SINKS_IND_OID: u32 = 17121; pub const TABLE_MZ_OBJECT_HYDRATION_HISTORY_OID: u32 = 17122; +pub const LOG_MZ_CLUSTER_REPLICA_RESOURCE_USAGE_OID: u32 = 17123; diff --git a/src/pgwire/src/protocol.rs b/src/pgwire/src/protocol.rs index d98e75820ae4c..acb485fe6bf7e 100644 --- a/src/pgwire/src/protocol.rs +++ b/src/pgwire/src/protocol.rs @@ -2580,7 +2580,7 @@ where None => FetchResult::Rows(None), Some(PeekResponseUnary::Rows(rows)) => FetchResult::Rows(Some(rows)), Some(PeekResponseUnary::Error(err)) => { - FetchResult::Error(ErrorResponse::error(SqlState::INTERNAL_ERROR, err)) + FetchResult::Error(err.into_response(Severity::Error)) } Some(PeekResponseUnary::DependencyDropped(dep)) => { FetchResult::Error( @@ -2844,11 +2844,10 @@ where e = self.conn.wait_closed() => return Err(e), batch = stream.recv() => match batch { None => break, - Some(PeekResponseUnary::Error(text)) => { - let err = - ErrorResponse::error(SqlState::INTERNAL_ERROR, text.clone()); + Some(PeekResponseUnary::Error(err)) => { + let text = err.to_string(); return self - .send_error_and_get_state(err) + .send_error_and_get_state(err.into_response(Severity::Error)) .await .map(|state| (state, SendRowsEndedReason::Errored { error: text })); } diff --git a/src/sql/src/rbac.rs b/src/sql/src/rbac.rs index 54fd7548f38a8..cd1bec5d67f14 100644 --- a/src/sql/src/rbac.rs +++ b/src/sql/src/rbac.rs @@ -31,7 +31,8 @@ use crate::names::{ }; use crate::plan::{self, PlanKind}; use crate::plan::{ - DataSourceDesc, Explainee, MutationKind, Plan, SideEffectingFunc, UpdatePrivilege, + DataSourceDesc, Explainee, MutationKind, Plan, SideEffectingFunc, TableDataSource, + UpdatePrivilege, }; use crate::session::metadata::SessionMetadata; use crate::session::user::{MZ_SUPPORT_ROLE_ID, MZ_SYSTEM_ROLE_ID, SUPPORT_USER, SYSTEM_USER}; @@ -635,17 +636,33 @@ fn generate_rbac_requirements( } Plan::CreateTable(plan::CreateTablePlan { name, - table: _, + table, if_not_exists: _, - }) => RbacRequirements { - privileges: vec![( + }) => { + let mut privileges = vec![( SystemObjectId::Object(name.qualifiers.clone().into()), AclMode::CREATE, role_id, - )], - item_usage: &CREATE_ITEM_USAGE, - ..Default::default() - }, + )]; + // `CREATE TABLE ... FROM SOURCE` reads the source's data, so it + // requires `SELECT` on the source. + if let TableDataSource::DataSource { + desc: DataSourceDesc::IngestionExport { ingestion_id, .. }, + timeline: _, + } = &table.data_source + { + privileges.extend(generate_read_privileges( + catalog, + iter::once(*ingestion_id), + role_id, + )); + } + RbacRequirements { + privileges, + item_usage: &CREATE_ITEM_USAGE, + ..Default::default() + } + } Plan::CreateView(plan::CreateViewPlan { name, view: _, diff --git a/src/sqllogictest/Cargo.toml b/src/sqllogictest/Cargo.toml index b7716bb67d05d..ffa780e442c05 100644 --- a/src/sqllogictest/Cargo.toml +++ b/src/sqllogictest/Cargo.toml @@ -23,6 +23,7 @@ maplit.workspace = true md-5.workspace = true mz-adapter-types = { path = "../adapter-types" } mz-catalog = { path = "../catalog" } +mz-compute-types = { path = "../compute-types" } mz-controller = { path = "../controller" } mz-dyncfgs = { path = "../dyncfgs" } mz-environmentd = { path = "../environmentd", default-features = false } diff --git a/src/sqllogictest/src/bin/sqllogictest.rs b/src/sqllogictest/src/bin/sqllogictest.rs index 415e8b361714a..5309150dca1c1 100644 --- a/src/sqllogictest/src/bin/sqllogictest.rs +++ b/src/sqllogictest/src/bin/sqllogictest.rs @@ -21,6 +21,7 @@ use std::process::ExitCode; use chrono::Utc; use clap::ArgAction; use mz_adapter_types::dyncfgs::ENABLE_BACKGROUND_ALTER_CLUSTER; +use mz_compute_types::dyncfgs::{ENABLE_PEEK_ROW_ITERATION_LIMIT, PEEK_ROW_ITERATION_LIMIT}; use mz_orchestrator_tracing::{StaticTracingConfig, TracingCliArgs}; use mz_ore::cli::{self, CliConfig, KeyValueArg}; use mz_ore::metrics::MetricsRegistry; @@ -186,6 +187,16 @@ async fn main() -> ExitCode { .entry(ENABLE_BACKGROUND_ALTER_CLUSTER.name().to_string()) .or_insert_with(|| "true".to_string()); + // Keep the guard enabled in the suite without constraining normal test queries. + for (name, value) in [ + (ENABLE_PEEK_ROW_ITERATION_LIMIT.name(), "true"), + (PEEK_ROW_ITERATION_LIMIT.name(), "1000000000"), + ] { + system_parameter_defaults + .entry(name.to_string()) + .or_insert_with(|| value.to_string()); + } + let config = RunConfig { stdout: &OutputStream::new(io::stdout(), args.timestamps), stderr: &OutputStream::new(io::stderr(), args.timestamps), diff --git a/test/cluster/mzcompose.py b/test/cluster/mzcompose.py index e687825fb2b64..804ff0e6bd5b6 100644 --- a/test/cluster/mzcompose.py +++ b/test/cluster/mzcompose.py @@ -8188,6 +8188,86 @@ def workflow_test_prometheus_metrics(c: Composition) -> None: """)) +def workflow_test_resource_usage(c: Composition) -> None: + """Test that mz_cluster_replica_resource_usage reports observations per replica process.""" + + c.up("materialized") + + # Sample fast, so the test does not have to wait out the default interval. + c.sql( + "ALTER SYSTEM SET mz_metrics_usage_refresh_interval = '1s';", + port=6877, + user="mz_system", + ) + c.sql("CREATE CLUSTER cluster1 SIZE 'scale=2,workers=2';") + + def observations() -> dict[tuple[int, str, str], int]: + with c.sql_cursor() as cursor: + cursor.execute(b"SET cluster = cluster1") + # `process_id` and `value` are `uint8`, which psycopg has no adapter for and hands + # back as strings. Cast so the comparisons below are numeric, not lexicographic. + cursor.execute(b""" + SELECT process_id::int8, source, metric, value::int8 + FROM mz_introspection.mz_cluster_replica_resource_usage + """) + return {(r[0], r[1], r[2]): r[3] for r in cursor.fetchall()} + + # Both processes must report, and every process must report the same set of metrics: the two + # run the same binary in the same environment, so a metric readable on one and not the other + # means a reader failed rather than that the source is unavailable. + for _ in range(60): + before = observations() + processes = {process_id for process_id, _, _ in before} + per_process = { + process_id: {(s, m) for p, s, m in before if p == process_id} + for process_id in processes + } + if processes == {0, 1} and len(set(map(frozenset, per_process.values()))) == 1: + break + time.sleep(1) + else: + assert False, f"resource usage not reported for both processes: {before}" + + # `rusage` and `proc_status` are available on any Linux replica, so their absence is a bug + # rather than an unsupported configuration. cgroup metrics are deliberately not asserted: + # `memory.peak` and `memory.swap.peak` depend on the kernel version. + metrics = {(source, metric) for _, source, metric in before} + for required in [("rusage", "max_rss"), ("proc_status", "vm_rss")]: + assert required in metrics, f"{required} missing from {sorted(metrics)}" + + # Every reported value must be a plain number. Nothing is allowed to surface as a sentinel. + for key, value in before.items(): + assert value >= 0, f"{key} reported {value}" + + # Do some work to push usage up. Peaks must not go backwards, whether or not this particular + # workload moves them. + c.sql(""" + SET cluster = cluster1; + CREATE TABLE t (a int); + INSERT INTO t SELECT generate_series(1, 500000); + CREATE MATERIALIZED VIEW mv AS SELECT count(*) FROM t; + """) + with c.sql_cursor() as cursor: + cursor.execute(b"SET cluster = cluster1") + cursor.execute(b"SELECT * FROM mv") + cursor.fetchall() + + # Give the sampler a few intervals to observe the new usage. + time.sleep(5) + + after = observations() + + # Peaks are monotonic, whether the kernel maintains them or the sampler folds them. Current + # values are free to fall, so they are deliberately not checked here. + peaks = [key for key in before if key[2].endswith("peak") or key[2] == "max_rss"] + assert peaks, f"no peak metrics reported: {sorted(before)}" + for key in peaks: + assert key in after, f"{key} stopped being reported" + assert ( + after[key] >= before[key] + ), f"{key} peak fell from {before[key]} to {after[key]}" + + def workflow_test_metrics_null_label(c: Composition) -> None: """SQL-198: `/metrics/mz_usage` must not abort environmentd when a Prometheus label column is SQL NULL. An unorchestrated cluster replica has diff --git a/test/launchdarkly-flag-consistency/mzcompose.py b/test/launchdarkly-flag-consistency/mzcompose.py index da92bb3bf07ee..5b207a6254080 100644 --- a/test/launchdarkly-flag-consistency/mzcompose.py +++ b/test/launchdarkly-flag-consistency/mzcompose.py @@ -210,6 +210,7 @@ compute_flat_map_fuel compute_logical_backpressure_max_retained_capabilities compute_mv_sink_advance_persist_frontiers + compute_peek_row_iteration_limit compute_peek_response_stash_batch_max_runs compute_peek_response_stash_read_batch_size_bytes compute_peek_response_stash_read_memory_budget_bytes @@ -243,6 +244,7 @@ enable_coalesce_case_transform enable_columnar_merge_batcher enable_compute_half_join2 + enable_compute_peek_row_iteration_limit enable_compute_render_fueled_as_specific_collection enable_date_bin_hopping enable_default_connection_validation diff --git a/test/sqllogictest/autogenerated/mz_introspection.slt b/test/sqllogictest/autogenerated/mz_introspection.slt index a914ba4344620..b1f881892146d 100644 --- a/test/sqllogictest/autogenerated/mz_introspection.slt +++ b/test/sqllogictest/autogenerated/mz_introspection.slt @@ -101,6 +101,14 @@ labels map value double␠precision help text +query TT +SELECT name, type FROM objects WHERE schema = 'mz_introspection' AND object = 'mz_cluster_replica_resource_usage' ORDER BY position +---- +process_id uint8 +source text +metric text +value uint8 + query TTT SELECT name, type, comment FROM objects WHERE schema = 'mz_introspection' AND object = 'mz_dataflows' ORDER BY position ---- @@ -271,6 +279,7 @@ mz_arrangement_sharing_raw mz_arrangement_sizes mz_arrangement_sizes_per_worker mz_cluster_prometheus_metrics +mz_cluster_replica_resource_usage mz_compute_dataflow_global_ids_per_worker mz_compute_error_counts mz_compute_error_counts_per_worker diff --git a/test/sqllogictest/catalog_server_explain.slt b/test/sqllogictest/catalog_server_explain.slt index 769ed420cb0ab..b06437fb06918 100644 --- a/test/sqllogictest/catalog_server_explain.slt +++ b/test/sqllogictest/catalog_server_explain.slt @@ -4986,7 +4986,7 @@ mz_catalog.mz_indexes: Filter: ("2" = (#1 ->> "object_type")) AND ("mz_introspection" = (#1 ->> "schema_name")) AND ((#1 ->> "object_name")) IS NOT NULL AND ("GidMapping" = #2) →Read mz_internal.mz_catalog_raw →Arrange (#0{log_name}) - →Constant (32 rows) + →Constant (33 rows) Source mz_internal.mz_catalog_raw project=(#0..=#2) @@ -5392,7 +5392,7 @@ mz_catalog.mz_sources: Project: #4, #0, #5, #1, #2, #6..=#12, #3, #13, #14 Map: null, null, null, null, null, null, "s1", null, null →Arrange (#1{schema_name}, #2{name}) - →Constant (56 rows) + →Constant (57 rows) →Arrange (#0{schema_name}) (#0{schema_name}, #1{name}) →Fused with Child Map/Filter/Project Project: #4, #3, #5 @@ -8236,7 +8236,7 @@ query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_sources"; ---- Explained Query (fast path): - →Constant (56 rows) + →Constant (57 rows) Target cluster: mz_catalog_server @@ -10238,7 +10238,7 @@ query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_ontology_entity_types"; ---- Explained Query (fast path): - →Constant (136 rows) + →Constant (137 rows) Target cluster: mz_catalog_server @@ -10262,7 +10262,7 @@ Explained Query: cte l0 = →Differential Join %1:mz_schemas[#0{id}] » %2:mz_objects[#2{schema_id}] » %0[#0{schema_name}, #1{table_name}] » %3:mz_columns[#0{id}] →Arrange (#0{schema_name}, #1{table_name}) - →Constant (136 rows) + →Constant (137 rows) →Arrange (#0{id}) →Fused with Child Map/Filter/Project Project: #1, #3 diff --git a/test/sqllogictest/cluster.slt b/test/sqllogictest/cluster.slt index 7d38047d85087..b793028eeabfd 100644 --- a/test/sqllogictest/cluster.slt +++ b/test/sqllogictest/cluster.slt @@ -201,6 +201,9 @@ bar mz_arrangement_sharing_raw mz_arrangement_sharing_raw_u7_primary_idx 2 w bar mz_cluster_prometheus_metrics mz_cluster_prometheus_metrics_u7_primary_idx 1 process_id NULL false bar mz_cluster_prometheus_metrics mz_cluster_prometheus_metrics_u7_primary_idx 2 metric_name NULL false bar mz_cluster_prometheus_metrics mz_cluster_prometheus_metrics_u7_primary_idx 3 labels NULL false +bar mz_cluster_replica_resource_usage mz_cluster_replica_resource_usage_u7_primary_idx 1 process_id NULL false +bar mz_cluster_replica_resource_usage mz_cluster_replica_resource_usage_u7_primary_idx 2 source NULL false +bar mz_cluster_replica_resource_usage mz_cluster_replica_resource_usage_u7_primary_idx 3 metric NULL false bar mz_compute_dataflow_global_ids_per_worker mz_compute_dataflow_global_ids_per_worker_u7_primary_idx 1 id NULL false bar mz_compute_dataflow_global_ids_per_worker mz_compute_dataflow_global_ids_per_worker_u7_primary_idx 2 worker_id NULL false bar mz_compute_dataflow_global_ids_per_worker mz_compute_dataflow_global_ids_per_worker_u7_primary_idx 3 global_id NULL false @@ -407,7 +410,7 @@ DROP CLUSTER foo, foo2, foo3, foo4 CASCADE query I SELECT COUNT(name) FROM mz_indexes WHERE cluster_id = 'u1'; ---- -32 +33 query I SELECT COUNT(name) FROM mz_indexes WHERE cluster_id <> 'u1' AND cluster_id NOT LIKE 's%'; @@ -420,7 +423,7 @@ CREATE CLUSTER test REPLICAS (foo (SIZE 'scale=1,workers=1')); query I SELECT COUNT(name) FROM mz_indexes; ---- -306 +313 statement ok DROP CLUSTER test CASCADE @@ -428,7 +431,7 @@ DROP CLUSTER test CASCADE query T SELECT COUNT(name) FROM mz_indexes; ---- -274 +280 simple conn=mz_system,user=mz_system ALTER CLUSTER quickstart OWNER TO materialize diff --git a/test/sqllogictest/cockroach/srfs.slt b/test/sqllogictest/cockroach/srfs.slt index 629a18d555394..e8e5aa17e4007 100644 --- a/test/sqllogictest/cockroach/srfs.slt +++ b/test/sqllogictest/cockroach/srfs.slt @@ -1164,6 +1164,24 @@ mz_cluster_replica_history 7 mz_cluster_replica_metrics 1 mz_cluster_replica_metrics_history 1 mz_cluster_replica_name_history 2 +mz_cluster_replica_resource_usage 1 +mz_cluster_replica_resource_usage 1 +mz_cluster_replica_resource_usage 1 +mz_cluster_replica_resource_usage 1 +mz_cluster_replica_resource_usage 1 +mz_cluster_replica_resource_usage 1 +mz_cluster_replica_resource_usage 2 +mz_cluster_replica_resource_usage 2 +mz_cluster_replica_resource_usage 2 +mz_cluster_replica_resource_usage 2 +mz_cluster_replica_resource_usage 2 +mz_cluster_replica_resource_usage 2 +mz_cluster_replica_resource_usage 3 +mz_cluster_replica_resource_usage 3 +mz_cluster_replica_resource_usage 3 +mz_cluster_replica_resource_usage 3 +mz_cluster_replica_resource_usage 3 +mz_cluster_replica_resource_usage 3 mz_cluster_replica_size_internal 1 mz_cluster_replica_sizes 1 mz_cluster_replica_status_history 1 diff --git a/test/sqllogictest/distinct_arrangements.slt b/test/sqllogictest/distinct_arrangements.slt index 05d66f9d9a694..e3d7d108b4dcd 100644 --- a/test/sqllogictest/distinct_arrangements.slt +++ b/test/sqllogictest/distinct_arrangements.slt @@ -1124,6 +1124,7 @@ Arrange Differential(BatcherRecords) Arrange Differential(BatcherSize) Arrange Differential(Sharing) Arrange PrometheusMetrics +Arrange ResourceUsage Arrange Timely(Addresses) Arrange Timely(BatchesReceived) Arrange Timely(BatchesSent) diff --git a/test/sqllogictest/information_schema_tables.slt b/test/sqllogictest/information_schema_tables.slt index 2081606984765..4d0d5d7ac2a25 100644 --- a/test/sqllogictest/information_schema_tables.slt +++ b/test/sqllogictest/information_schema_tables.slt @@ -945,6 +945,10 @@ mz_cluster_prometheus_metrics SOURCE materialize mz_introspection +mz_cluster_replica_resource_usage +SOURCE +materialize +mz_introspection mz_compute_dataflow_global_ids_per_worker SOURCE materialize diff --git a/test/sqllogictest/introspection/relations.slt b/test/sqllogictest/introspection/relations.slt index b8acec4390bb0..5a36a7cee42a8 100644 --- a/test/sqllogictest/introspection/relations.slt +++ b/test/sqllogictest/introspection/relations.slt @@ -136,6 +136,7 @@ Arrange␠Differential(BatcherRecords) ArrangementSize alloc::vec::Vec)>>>> Arrange␠Differential(Sharing) ArrangementSize alloc::vec::Vec)>>>> Arrange␠PrometheusMetrics ArrangementSize alloc::vec::Vec)>>>> +Arrange␠ResourceUsage ArrangementSize alloc::vec::Vec)>>>> Arrange␠Timely(Addresses) ArrangementSize alloc::vec::Vec)>>>> Arrange␠Timely(BatchesReceived) ArrangementSize alloc::vec::Vec)>>>> Arrange␠Timely(BatchesSent) ArrangementSize alloc::vec::Vec)>>>> @@ -192,6 +193,7 @@ Replay␠differential␠logs Differential␠Logging␠Demux alloc::vec::Vec<(c Replay␠reachability␠logs FlatMapReachability mz_timely_util::columnar::Column<(core::time::Duration,␠(usize,␠alloc::vec::Vec<(usize,␠usize,␠bool,␠mz_repr::timestamp::Timestamp,␠mz_ore::overflowing::Overflowing)>))> Replay␠storage␠timely␠logs Concatenate alloc::vec::Vec<(core::time::Duration,␠timely::logging::TimelyEvent)> Replay␠timely␠logs Concatenate alloc::vec::Vec<(core::time::Duration,␠timely::logging::TimelyEvent)> +ResourceUsage Arrange␠ResourceUsage mz_timely_util::columnar::Column<((mz_repr::row::Row,␠mz_repr::row::Row),␠mz_repr::timestamp::Timestamp,␠mz_ore::overflowing::Overflowing)> Timely␠Logging␠Demux Consolidate␠Timely(Addresses) alloc::vec::Vec<((usize,␠alloc::vec::Vec),␠mz_repr::timestamp::Timestamp,␠mz_ore::overflowing::Overflowing)> Timely␠Logging␠Demux Consolidate␠Timely(BatchesReceived) alloc::vec::Vec<((mz_compute::logging::timely::MessageDatum,␠()),␠mz_repr::timestamp::Timestamp,␠mz_ore::overflowing::Overflowing)> Timely␠Logging␠Demux Consolidate␠Timely(BatchesSent) alloc::vec::Vec<((mz_compute::logging::timely::MessageDatum,␠()),␠mz_repr::timestamp::Timestamp,␠mz_ore::overflowing::Overflowing)> @@ -245,8 +247,8 @@ GROUP BY type; 1 mz_timely_util::columnar::Column<(core::time::Duration,␠(usize,␠alloc::vec::Vec<(usize,␠usize,␠bool,␠mz_repr::timestamp::Timestamp,␠mz_ore::overflowing::Overflowing)>))> 1 mz_timely_util::columnar::Column<(core::time::Duration,␠mz_compute::logging::compute::ComputeEvent)> 3 alloc::vec::Vec<(core::time::Duration,␠timely::logging::TimelyEvent)> -32 alloc::vec::Vec)>>>> -32 mz_timely_util::columnar::Column<((mz_repr::row::Row,␠mz_repr::row::Row),␠mz_repr::timestamp::Timestamp,␠mz_ore::overflowing::Overflowing)> +33 alloc::vec::Vec)>>>> +33 mz_timely_util::columnar::Column<((mz_repr::row::Row,␠mz_repr::row::Row),␠mz_repr::timestamp::Timestamp,␠mz_ore::overflowing::Overflowing)> 4 alloc::vec::Vec<((mz_compute::logging::timely::MessageDatum,␠()),␠mz_repr::timestamp::Timestamp,␠mz_ore::overflowing::Overflowing)> 4 alloc::vec::Vec)>>> 8 alloc::vec::Vec<((usize,␠()),␠mz_repr::timestamp::Timestamp,␠mz_ore::overflowing::Overflowing)> diff --git a/test/sqllogictest/max_result_size.slt b/test/sqllogictest/max_result_size.slt index e165c3e4f16c9..671446a1d3bfe 100644 --- a/test/sqllogictest/max_result_size.slt +++ b/test/sqllogictest/max_result_size.slt @@ -9,6 +9,94 @@ mode cockroach +simple conn=mz_system,user=mz_system +ALTER SYSTEM SET compute_peek_row_iteration_limit TO 2; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +ALTER SYSTEM SET enable_compute_peek_row_iteration_limit TO false; +---- +COMPLETE 0 + +statement ok +CREATE CLUSTER peek_limit_cluster SIZE 'scale=1,workers=1'; + +statement ok +SET cluster TO 'peek_limit_cluster'; + +statement ok +CREATE TABLE peek_iteration_limit (a int); + +statement ok +INSERT INTO peek_iteration_limit VALUES (1), (2); + +statement ok +CREATE INDEX peek_iteration_limit_idx ON peek_iteration_limit (a); + +simple conn=mz_system,user=mz_system +ALTER SYSTEM SET enable_compute_peek_row_iteration_limit TO true; +---- +COMPLETE 0 + +# A peek may examine exactly the configured number of rows. +query I +SELECT a FROM peek_iteration_limit ORDER BY a; +---- +1 +2 + +simple conn=mz_system,user=mz_system +ALTER SYSTEM SET enable_compute_peek_row_iteration_limit TO false; +---- +COMPLETE 0 + +statement ok +INSERT INTO peek_iteration_limit VALUES (3); + +query I +SELECT a FROM peek_iteration_limit ORDER BY a; +---- +1 +2 +3 + +simple conn=mz_system,user=mz_system +ALTER SYSTEM SET enable_compute_peek_row_iteration_limit TO true; +---- +COMPLETE 0 + +query error query exceeded the configured row iteration limit of 2 rows +SELECT a FROM peek_iteration_limit ORDER BY a; + +simple conn=mz_system,user=mz_system +ALTER SYSTEM SET compute_peek_row_iteration_limit TO 3; +---- +COMPLETE 0 + +query I +SELECT a FROM peek_iteration_limit ORDER BY a; +---- +1 +2 +3 + +simple conn=mz_system,user=mz_system +ALTER SYSTEM RESET compute_peek_row_iteration_limit; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +ALTER SYSTEM RESET enable_compute_peek_row_iteration_limit; +---- +COMPLETE 0 + +statement ok +SET cluster TO 'quickstart'; + +statement ok +DROP CLUSTER peek_limit_cluster CASCADE; + simple conn=mz_system,user=mz_system ALTER SYSTEM SET max_result_size TO 1; ---- diff --git a/test/sqllogictest/mz_catalog_server_index_accounting.slt b/test/sqllogictest/mz_catalog_server_index_accounting.slt index 1cc1103fdf6bc..e8c638c152d60 100644 --- a/test/sqllogictest/mz_catalog_server_index_accounting.slt +++ b/test/sqllogictest/mz_catalog_server_index_accounting.slt @@ -37,109 +37,110 @@ mz_arrangement_heap_capacity_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangemen mz_arrangement_heap_size_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_heap_size_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_heap_size_raw"␠("operator_id",␠"worker_id") mz_arrangement_records_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_records_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_records_raw"␠("operator_id",␠"worker_id") mz_arrangement_sharing_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_sharing_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_sharing_raw"␠("operator_id",␠"worker_id") -mz_cluster_auto_scaling_strategies_ind CREATE␠INDEX␠"mz_cluster_auto_scaling_strategies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s520␠AS␠"mz_internal"."mz_cluster_auto_scaling_strategies"]␠("cluster_id") -mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s769␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") +mz_cluster_auto_scaling_strategies_ind CREATE␠INDEX␠"mz_cluster_auto_scaling_strategies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s521␠AS␠"mz_internal"."mz_cluster_auto_scaling_strategies"]␠("cluster_id") +mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s770␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") mz_cluster_prometheus_metrics_s2_primary_idx CREATE␠INDEX␠"mz_cluster_prometheus_metrics_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_cluster_prometheus_metrics"␠("process_id",␠"metric_name",␠"labels") -mz_cluster_reconfigurations_ind CREATE␠INDEX␠"mz_cluster_reconfigurations_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s519␠AS␠"mz_internal"."mz_cluster_reconfigurations"]␠("cluster_id") -mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s763␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") -mz_cluster_replica_history_ind CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s618␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at") -mz_cluster_replica_metrics_history_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s525␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id") -mz_cluster_replica_metrics_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s526␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id") -mz_cluster_replica_name_history_ind CREATE␠INDEX␠"mz_cluster_replica_name_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s619␠AS␠"mz_internal"."mz_cluster_replica_name_history"]␠("id") -mz_cluster_replica_size_internal_ind CREATE␠INDEX␠"mz_cluster_replica_size_internal_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s515␠AS␠"mz_internal"."mz_cluster_replica_size_internal"]␠("size") -mz_cluster_replica_sizes_ind CREATE␠INDEX␠"mz_cluster_replica_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s514␠AS␠"mz_catalog"."mz_cluster_replica_sizes"]␠("size") -mz_cluster_replica_status_history_ind CREATE␠INDEX␠"mz_cluster_replica_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s527␠AS␠"mz_internal"."mz_cluster_replica_status_history"]␠("replica_id") -mz_cluster_replica_statuses_ind CREATE␠INDEX␠"mz_cluster_replica_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s528␠AS␠"mz_internal"."mz_cluster_replica_statuses"]␠("replica_id") -mz_cluster_replicas_ind CREATE␠INDEX␠"mz_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s524␠AS␠"mz_catalog"."mz_cluster_replicas"]␠("id") -mz_clusters_ind CREATE␠INDEX␠"mz_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s516␠AS␠"mz_catalog"."mz_clusters"]␠("id") -mz_columns_ind CREATE␠INDEX␠"mz_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s487␠AS␠"mz_catalog"."mz_columns"]␠("name") -mz_comments_ind CREATE␠INDEX␠"mz_comments_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s543␠AS␠"mz_internal"."mz_comments"]␠("id") +mz_cluster_reconfigurations_ind CREATE␠INDEX␠"mz_cluster_reconfigurations_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s520␠AS␠"mz_internal"."mz_cluster_reconfigurations"]␠("cluster_id") +mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s764␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") +mz_cluster_replica_history_ind CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s619␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at") +mz_cluster_replica_metrics_history_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s526␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id") +mz_cluster_replica_metrics_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s527␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id") +mz_cluster_replica_name_history_ind CREATE␠INDEX␠"mz_cluster_replica_name_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s620␠AS␠"mz_internal"."mz_cluster_replica_name_history"]␠("id") +mz_cluster_replica_resource_usage_s2_primary_idx CREATE␠INDEX␠"mz_cluster_replica_resource_usage_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_cluster_replica_resource_usage"␠("process_id",␠"source",␠"metric") +mz_cluster_replica_size_internal_ind CREATE␠INDEX␠"mz_cluster_replica_size_internal_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s516␠AS␠"mz_internal"."mz_cluster_replica_size_internal"]␠("size") +mz_cluster_replica_sizes_ind CREATE␠INDEX␠"mz_cluster_replica_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s515␠AS␠"mz_catalog"."mz_cluster_replica_sizes"]␠("size") +mz_cluster_replica_status_history_ind CREATE␠INDEX␠"mz_cluster_replica_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s528␠AS␠"mz_internal"."mz_cluster_replica_status_history"]␠("replica_id") +mz_cluster_replica_statuses_ind CREATE␠INDEX␠"mz_cluster_replica_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s529␠AS␠"mz_internal"."mz_cluster_replica_statuses"]␠("replica_id") +mz_cluster_replicas_ind CREATE␠INDEX␠"mz_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s525␠AS␠"mz_catalog"."mz_cluster_replicas"]␠("id") +mz_clusters_ind CREATE␠INDEX␠"mz_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s517␠AS␠"mz_catalog"."mz_clusters"]␠("id") +mz_columns_ind CREATE␠INDEX␠"mz_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s488␠AS␠"mz_catalog"."mz_columns"]␠("name") +mz_comments_ind CREATE␠INDEX␠"mz_comments_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s544␠AS␠"mz_internal"."mz_comments"]␠("id") mz_compute_dataflow_global_ids_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_dataflow_global_ids_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_dataflow_global_ids_per_worker"␠("id",␠"worker_id",␠"global_id") -mz_compute_dependencies_ind CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s744␠AS␠"mz_internal"."mz_compute_dependencies"]␠("dependency_id") +mz_compute_dependencies_ind CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s745␠AS␠"mz_internal"."mz_compute_dependencies"]␠("dependency_id") mz_compute_error_counts_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_error_counts_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_error_counts_raw"␠("export_id",␠"worker_id") mz_compute_exports_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_exports_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_exports_per_worker"␠("export_id",␠"worker_id") mz_compute_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_frontiers_per_worker"␠("export_id",␠"worker_id") -mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s754␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") +mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s755␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") mz_compute_hydration_times_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_hydration_times_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_hydration_times_per_worker"␠("export_id",␠"worker_id") mz_compute_import_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_import_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_import_frontiers_per_worker"␠("export_id",␠"import_id",␠"worker_id") mz_compute_lir_mapping_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_lir_mapping_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_lir_mapping_per_worker"␠("global_id",␠"lir_id",␠"worker_id") mz_compute_operator_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_durations_histogram_raw"␠("id",␠"worker_id",␠"duration_ns") mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_hydration_statuses_per_worker"␠("export_id",␠"lir_id",␠"worker_id") -mz_connections_ind CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s522␠AS␠"mz_catalog"."mz_connections"]␠("schema_id") -mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s750␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") -mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s749␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") -mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s748␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") -mz_databases_ind CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s485␠AS␠"mz_catalog"."mz_databases"]␠("name") +mz_connections_ind CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s523␠AS␠"mz_catalog"."mz_connections"]␠("schema_id") +mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s751␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") +mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s750␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") +mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s749␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") +mz_databases_ind CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s486␠AS␠"mz_catalog"."mz_databases"]␠("name") mz_dataflow_addresses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_addresses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_addresses_per_worker"␠("id",␠"worker_id") mz_dataflow_channels_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_channels_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_channels_per_worker"␠("id",␠"worker_id") mz_dataflow_operator_reachability_raw_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operator_reachability_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operator_reachability_raw"␠("id",␠"worker_id",␠"source",␠"port",␠"update_type",␠"time") mz_dataflow_operators_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operators_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operators_per_worker"␠("id",␠"worker_id") -mz_frontiers_ind CREATE␠INDEX␠"mz_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s735␠AS␠"mz_internal"."mz_frontiers"]␠("object_id") -mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s765␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") -mz_indexes_ind CREATE␠INDEX␠"mz_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s488␠AS␠"mz_catalog"."mz_indexes"]␠("id") -mz_kafka_sources_ind CREATE␠INDEX␠"mz_kafka_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s482␠AS␠"mz_catalog"."mz_kafka_sources"]␠("id") -mz_materialized_views_ind CREATE␠INDEX␠"mz_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s547␠AS␠"mz_catalog"."mz_materialized_views"]␠("id") +mz_frontiers_ind CREATE␠INDEX␠"mz_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s736␠AS␠"mz_internal"."mz_frontiers"]␠("object_id") +mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s766␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") +mz_indexes_ind CREATE␠INDEX␠"mz_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s489␠AS␠"mz_catalog"."mz_indexes"]␠("id") +mz_kafka_sources_ind CREATE␠INDEX␠"mz_kafka_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s483␠AS␠"mz_catalog"."mz_kafka_sources"]␠("id") +mz_materialized_views_ind CREATE␠INDEX␠"mz_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s548␠AS␠"mz_catalog"."mz_materialized_views"]␠("id") mz_message_batch_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_batch_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_batch_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_batch_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") -mz_metric_sinks_ind CREATE␠INDEX␠"mz_metric_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s545␠AS␠"mz_internal"."mz_metric_sinks"]␠("id") -mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s842␠AS␠"mz_internal"."mz_notices"]␠("id") -mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s757␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") -mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s757␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") -mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s755␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") -mz_object_dependencies_ind CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s483␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id") -mz_object_graph_edges_ind CREATE␠INDEX␠"mz_object_graph_edges_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s746␠AS␠"mz_internal"."mz_object_graph_edges"]␠("object_id") -mz_object_history_ind CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s558␠AS␠"mz_internal"."mz_object_history"]␠("id") -mz_object_lifetimes_ind CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s559␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id") -mz_object_transitive_dependencies_ind CREATE␠INDEX␠"mz_object_transitive_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s575␠AS␠"mz_internal"."mz_object_transitive_dependencies"]␠("object_id") -mz_objects_ind CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s555␠AS␠"mz_catalog"."mz_objects"]␠("schema_id") +mz_metric_sinks_ind CREATE␠INDEX␠"mz_metric_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s546␠AS␠"mz_internal"."mz_metric_sinks"]␠("id") +mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s843␠AS␠"mz_internal"."mz_notices"]␠("id") +mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s758␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") +mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s758␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") +mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s756␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") +mz_object_dependencies_ind CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s484␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id") +mz_object_graph_edges_ind CREATE␠INDEX␠"mz_object_graph_edges_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s747␠AS␠"mz_internal"."mz_object_graph_edges"]␠("object_id") +mz_object_history_ind CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s559␠AS␠"mz_internal"."mz_object_history"]␠("id") +mz_object_lifetimes_ind CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s560␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id") +mz_object_transitive_dependencies_ind CREATE␠INDEX␠"mz_object_transitive_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s576␠AS␠"mz_internal"."mz_object_transitive_dependencies"]␠("object_id") +mz_objects_ind CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s556␠AS␠"mz_catalog"."mz_objects"]␠("schema_id") mz_peek_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_peek_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_peek_durations_histogram_raw"␠("worker_id",␠"type",␠"duration_ns") -mz_recent_activity_log_thinned_ind CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s719␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash") -mz_recent_sql_text_ind CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s715␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash") -mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s834␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") -mz_roles_ind CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s506␠AS␠"mz_catalog"."mz_roles"]␠("id") +mz_recent_activity_log_thinned_ind CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s720␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash") +mz_recent_sql_text_ind CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s716␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash") +mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s835␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") +mz_roles_ind CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s507␠AS␠"mz_catalog"."mz_roles"]␠("id") mz_scheduling_elapsed_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_elapsed_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_elapsed_raw"␠("id",␠"worker_id") mz_scheduling_parks_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_parks_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_parks_histogram_raw"␠("worker_id",␠"slept_for_ns",␠"requested_ns") -mz_schemas_ind CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s486␠AS␠"mz_catalog"."mz_schemas"]␠("database_id") -mz_secrets_ind CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s521␠AS␠"mz_catalog"."mz_secrets"]␠("name") -mz_show_all_objects_ind CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s603␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id") -mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s767␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") -mz_show_clusters_ind CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s605␠AS␠"mz_internal"."mz_show_clusters"]␠("name") -mz_show_columns_ind CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s604␠AS␠"mz_internal"."mz_show_columns"]␠("id") -mz_show_connections_ind CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s613␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id") -mz_show_databases_ind CREATE␠INDEX␠"mz_show_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s607␠AS␠"mz_internal"."mz_show_databases"]␠("name") -mz_show_indexes_ind CREATE␠INDEX␠"mz_show_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s617␠AS␠"mz_internal"."mz_show_indexes"]␠("schema_id") -mz_show_materialized_views_ind CREATE␠INDEX␠"mz_show_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s616␠AS␠"mz_internal"."mz_show_materialized_views"]␠("schema_id") -mz_show_roles_ind CREATE␠INDEX␠"mz_show_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s612␠AS␠"mz_internal"."mz_show_roles"]␠("name") -mz_show_schemas_ind CREATE␠INDEX␠"mz_show_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s608␠AS␠"mz_internal"."mz_show_schemas"]␠("database_id") -mz_show_secrets_ind CREATE␠INDEX␠"mz_show_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s606␠AS␠"mz_internal"."mz_show_secrets"]␠("schema_id") -mz_show_sinks_ind CREATE␠INDEX␠"mz_show_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s615␠AS␠"mz_internal"."mz_show_sinks"]␠("schema_id") -mz_show_sources_ind CREATE␠INDEX␠"mz_show_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s614␠AS␠"mz_internal"."mz_show_sources"]␠("schema_id") -mz_show_tables_ind CREATE␠INDEX␠"mz_show_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s609␠AS␠"mz_internal"."mz_show_tables"]␠("schema_id") -mz_show_types_ind CREATE␠INDEX␠"mz_show_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s611␠AS␠"mz_internal"."mz_show_types"]␠("schema_id") -mz_show_views_ind CREATE␠INDEX␠"mz_show_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s610␠AS␠"mz_internal"."mz_show_views"]␠("schema_id") -mz_sink_statistics_ind CREATE␠INDEX␠"mz_sink_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s732␠AS␠"mz_internal"."mz_sink_statistics"]␠("id",␠"replica_id") -mz_sink_status_history_ind CREATE␠INDEX␠"mz_sink_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s704␠AS␠"mz_internal"."mz_sink_status_history"]␠("sink_id") -mz_sink_statuses_ind CREATE␠INDEX␠"mz_sink_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s705␠AS␠"mz_internal"."mz_sink_statuses"]␠("id") -mz_sinks_ind CREATE␠INDEX␠"mz_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s498␠AS␠"mz_catalog"."mz_sinks"]␠("id") -mz_source_statistics_ind CREATE␠INDEX␠"mz_source_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s730␠AS␠"mz_internal"."mz_source_statistics"]␠("id",␠"replica_id") -mz_source_statistics_with_history_ind CREATE␠INDEX␠"mz_source_statistics_with_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s728␠AS␠"mz_internal"."mz_source_statistics_with_history"]␠("id",␠"replica_id") -mz_source_status_history_ind CREATE␠INDEX␠"mz_source_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s706␠AS␠"mz_internal"."mz_source_status_history"]␠("source_id") -mz_source_statuses_ind CREATE␠INDEX␠"mz_source_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s723␠AS␠"mz_internal"."mz_source_statuses"]␠("id") -mz_sources_ind CREATE␠INDEX␠"mz_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s491␠AS␠"mz_catalog"."mz_sources"]␠("id") -mz_tables_ind CREATE␠INDEX␠"mz_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s490␠AS␠"mz_catalog"."mz_tables"]␠("schema_id") -mz_types_ind CREATE␠INDEX␠"mz_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s500␠AS␠"mz_catalog"."mz_types"]␠("schema_id") -mz_views_ind CREATE␠INDEX␠"mz_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s499␠AS␠"mz_catalog"."mz_views"]␠("schema_id") -mz_wallclock_global_lag_recent_history_ind CREATE␠INDEX␠"mz_wallclock_global_lag_recent_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s739␠AS␠"mz_internal"."mz_wallclock_global_lag_recent_history"]␠("object_id") -mz_webhook_sources_ind CREATE␠INDEX␠"mz_webhook_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s544␠AS␠"mz_internal"."mz_webhook_sources"]␠("id") -pg_attrdef_all_databases_ind CREATE␠INDEX␠"pg_attrdef_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s644␠AS␠"mz_internal"."pg_attrdef_all_databases"]␠("oid",␠"adrelid",␠"adnum",␠"adbin",␠"adsrc") -pg_attribute_all_databases_ind CREATE␠INDEX␠"pg_attribute_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s637␠AS␠"mz_internal"."pg_attribute_all_databases"]␠("attrelid",␠"attname",␠"atttypid",␠"attlen",␠"attnum",␠"atttypmod",␠"attnotnull",␠"atthasdef",␠"attidentity",␠"attgenerated",␠"attisdropped",␠"attcollation",␠"database_name",␠"pg_type_database_name") -pg_authid_core_ind CREATE␠INDEX␠"pg_authid_core_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s654␠AS␠"mz_internal"."pg_authid_core"]␠("rolname") -pg_class_all_databases_ind CREATE␠INDEX␠"pg_class_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s625␠AS␠"mz_internal"."pg_class_all_databases"]␠("relname") -pg_description_all_databases_ind CREATE␠INDEX␠"pg_description_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s634␠AS␠"mz_internal"."pg_description_all_databases"]␠("objoid",␠"classoid",␠"objsubid",␠"description",␠"oid_database_name",␠"class_database_name") -pg_namespace_all_databases_ind CREATE␠INDEX␠"pg_namespace_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s622␠AS␠"mz_internal"."pg_namespace_all_databases"]␠("nspname") -pg_type_all_databases_ind CREATE␠INDEX␠"pg_type_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s631␠AS␠"mz_internal"."pg_type_all_databases"]␠("oid") +mz_schemas_ind CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s487␠AS␠"mz_catalog"."mz_schemas"]␠("database_id") +mz_secrets_ind CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s522␠AS␠"mz_catalog"."mz_secrets"]␠("name") +mz_show_all_objects_ind CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s604␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id") +mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s768␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") +mz_show_clusters_ind CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s606␠AS␠"mz_internal"."mz_show_clusters"]␠("name") +mz_show_columns_ind CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s605␠AS␠"mz_internal"."mz_show_columns"]␠("id") +mz_show_connections_ind CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s614␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id") +mz_show_databases_ind CREATE␠INDEX␠"mz_show_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s608␠AS␠"mz_internal"."mz_show_databases"]␠("name") +mz_show_indexes_ind CREATE␠INDEX␠"mz_show_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s618␠AS␠"mz_internal"."mz_show_indexes"]␠("schema_id") +mz_show_materialized_views_ind CREATE␠INDEX␠"mz_show_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s617␠AS␠"mz_internal"."mz_show_materialized_views"]␠("schema_id") +mz_show_roles_ind CREATE␠INDEX␠"mz_show_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s613␠AS␠"mz_internal"."mz_show_roles"]␠("name") +mz_show_schemas_ind CREATE␠INDEX␠"mz_show_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s609␠AS␠"mz_internal"."mz_show_schemas"]␠("database_id") +mz_show_secrets_ind CREATE␠INDEX␠"mz_show_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s607␠AS␠"mz_internal"."mz_show_secrets"]␠("schema_id") +mz_show_sinks_ind CREATE␠INDEX␠"mz_show_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s616␠AS␠"mz_internal"."mz_show_sinks"]␠("schema_id") +mz_show_sources_ind CREATE␠INDEX␠"mz_show_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s615␠AS␠"mz_internal"."mz_show_sources"]␠("schema_id") +mz_show_tables_ind CREATE␠INDEX␠"mz_show_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s610␠AS␠"mz_internal"."mz_show_tables"]␠("schema_id") +mz_show_types_ind CREATE␠INDEX␠"mz_show_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s612␠AS␠"mz_internal"."mz_show_types"]␠("schema_id") +mz_show_views_ind CREATE␠INDEX␠"mz_show_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s611␠AS␠"mz_internal"."mz_show_views"]␠("schema_id") +mz_sink_statistics_ind CREATE␠INDEX␠"mz_sink_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s733␠AS␠"mz_internal"."mz_sink_statistics"]␠("id",␠"replica_id") +mz_sink_status_history_ind CREATE␠INDEX␠"mz_sink_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s705␠AS␠"mz_internal"."mz_sink_status_history"]␠("sink_id") +mz_sink_statuses_ind CREATE␠INDEX␠"mz_sink_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s706␠AS␠"mz_internal"."mz_sink_statuses"]␠("id") +mz_sinks_ind CREATE␠INDEX␠"mz_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s499␠AS␠"mz_catalog"."mz_sinks"]␠("id") +mz_source_statistics_ind CREATE␠INDEX␠"mz_source_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s731␠AS␠"mz_internal"."mz_source_statistics"]␠("id",␠"replica_id") +mz_source_statistics_with_history_ind CREATE␠INDEX␠"mz_source_statistics_with_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s729␠AS␠"mz_internal"."mz_source_statistics_with_history"]␠("id",␠"replica_id") +mz_source_status_history_ind CREATE␠INDEX␠"mz_source_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s707␠AS␠"mz_internal"."mz_source_status_history"]␠("source_id") +mz_source_statuses_ind CREATE␠INDEX␠"mz_source_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s724␠AS␠"mz_internal"."mz_source_statuses"]␠("id") +mz_sources_ind CREATE␠INDEX␠"mz_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s492␠AS␠"mz_catalog"."mz_sources"]␠("id") +mz_tables_ind CREATE␠INDEX␠"mz_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s491␠AS␠"mz_catalog"."mz_tables"]␠("schema_id") +mz_types_ind CREATE␠INDEX␠"mz_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s501␠AS␠"mz_catalog"."mz_types"]␠("schema_id") +mz_views_ind CREATE␠INDEX␠"mz_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s500␠AS␠"mz_catalog"."mz_views"]␠("schema_id") +mz_wallclock_global_lag_recent_history_ind CREATE␠INDEX␠"mz_wallclock_global_lag_recent_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s740␠AS␠"mz_internal"."mz_wallclock_global_lag_recent_history"]␠("object_id") +mz_webhook_sources_ind CREATE␠INDEX␠"mz_webhook_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s545␠AS␠"mz_internal"."mz_webhook_sources"]␠("id") +pg_attrdef_all_databases_ind CREATE␠INDEX␠"pg_attrdef_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s645␠AS␠"mz_internal"."pg_attrdef_all_databases"]␠("oid",␠"adrelid",␠"adnum",␠"adbin",␠"adsrc") +pg_attribute_all_databases_ind CREATE␠INDEX␠"pg_attribute_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s638␠AS␠"mz_internal"."pg_attribute_all_databases"]␠("attrelid",␠"attname",␠"atttypid",␠"attlen",␠"attnum",␠"atttypmod",␠"attnotnull",␠"atthasdef",␠"attidentity",␠"attgenerated",␠"attisdropped",␠"attcollation",␠"database_name",␠"pg_type_database_name") +pg_authid_core_ind CREATE␠INDEX␠"pg_authid_core_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s655␠AS␠"mz_internal"."pg_authid_core"]␠("rolname") +pg_class_all_databases_ind CREATE␠INDEX␠"pg_class_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s626␠AS␠"mz_internal"."pg_class_all_databases"]␠("relname") +pg_description_all_databases_ind CREATE␠INDEX␠"pg_description_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s635␠AS␠"mz_internal"."pg_description_all_databases"]␠("objoid",␠"classoid",␠"objsubid",␠"description",␠"oid_database_name",␠"class_database_name") +pg_namespace_all_databases_ind CREATE␠INDEX␠"pg_namespace_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s623␠AS␠"mz_internal"."pg_namespace_all_databases"]␠("nspname") +pg_type_all_databases_ind CREATE␠INDEX␠"pg_type_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s632␠AS␠"mz_internal"."pg_type_all_databases"]␠("oid") # Record all transitive dependencies (tables, sources, views, mvs) of indexes on # the mz_catalog_server cluster. @@ -285,6 +286,10 @@ mz_cluster_replica_name_history id mz_cluster_replica_name_history new_name mz_cluster_replica_name_history occurred_at mz_cluster_replica_name_history previous_name +mz_cluster_replica_resource_usage metric +mz_cluster_replica_resource_usage process_id +mz_cluster_replica_resource_usage source +mz_cluster_replica_resource_usage value mz_cluster_replica_size_internal disk_bytes mz_cluster_replica_size_internal size mz_cluster_replica_size_internal swap_enabled diff --git a/test/sqllogictest/mzcompose.py b/test/sqllogictest/mzcompose.py index e134b53f9e28b..f9e143e3467e2 100644 --- a/test/sqllogictest/mzcompose.py +++ b/test/sqllogictest/mzcompose.py @@ -694,6 +694,7 @@ def compileFastSltConfig() -> SltRunConfig: "test/sqllogictest/quote_ident.slt", "test/sqllogictest/quoting.slt", "test/sqllogictest/range.slt", + "test/sqllogictest/rbac_create_table_from_source.slt", "test/sqllogictest/rbac_enabled.slt", "test/sqllogictest/record.slt", "test/sqllogictest/recursion_limit.slt", diff --git a/test/sqllogictest/oid.slt b/test/sqllogictest/oid.slt index f7a72291710a0..ba84f1540219a 100644 --- a/test/sqllogictest/oid.slt +++ b/test/sqllogictest/oid.slt @@ -1252,3 +1252,4 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17120 mz_metric_sinks 17121 mz_metric_sinks_ind 17122 mz_object_hydration_history +17123 mz_cluster_replica_resource_usage diff --git a/test/sqllogictest/persist-fast-path.slt b/test/sqllogictest/persist-fast-path.slt index a61cf5a061787..c0882dd0cedea 100644 --- a/test/sqllogictest/persist-fast-path.slt +++ b/test/sqllogictest/persist-fast-path.slt @@ -244,7 +244,57 @@ ALTER SYSTEM RESET enable_compute_peek_response_stash ---- COMPLETE 0 -# Does not apply when an index exists. +# Persist-backed peeks enforce the same row iteration limit. +# +# The limit is per worker, so this needs a single-worker cluster to be +# deterministic. Spread across several workers the same rows are walked a share +# at a time and no single walk reaches the limit, which also makes a one-shot +# query disagree with the indexed view --auto-index-selects wraps it in. +statement ok +CREATE CLUSTER peek_iteration_limit_cluster SIZE 'scale=1,workers=1' + +statement ok +SET cluster TO 'peek_iteration_limit_cluster' + +simple conn=mz_system,user=mz_system +ALTER SYSTEM SET enable_compute_peek_row_iteration_limit TO true; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +ALTER SYSTEM SET compute_peek_row_iteration_limit TO 2; +---- +COMPLETE 0 + +query error query exceeded the configured row iteration limit of 2 rows +SELECT a FROM large_rows LIMIT 3; + +simple conn=mz_system,user=mz_system +ALTER SYSTEM SET compute_peek_row_iteration_limit TO 3; +---- +COMPLETE 0 + +query I rowsort +SELECT a FROM large_rows LIMIT 3; +---- +1 +2 +3 + +simple conn=mz_system,user=mz_system +ALTER SYSTEM RESET compute_peek_row_iteration_limit; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +ALTER SYSTEM RESET enable_compute_peek_row_iteration_limit; +---- +COMPLETE 0 + +statement ok +RESET cluster + +# The Persist fast path does not apply when an index exists. statement ok CREATE DEFAULT INDEX ON numbers; diff --git a/test/sqllogictest/pg_catalog_user.slt b/test/sqllogictest/pg_catalog_user.slt index d892d2d48689a..b09c3b5f32729 100644 --- a/test/sqllogictest/pg_catalog_user.slt +++ b/test/sqllogictest/pg_catalog_user.slt @@ -27,7 +27,7 @@ CREATE ROLE "materialize@foocorp.io" WITH LOGIN query TIBBBBTTT rowsort SELECT usename, usesysid, usecreatedb, usesuper, userepl, usebypassrls, passwd, valuntil, useconfig FROM pg_user; ---- -materialize@foocorp.io 20196 false NULL false false ******** NULL NULL +materialize@foocorp.io 20202 false NULL false false ******** NULL NULL mz_support 16662 false true false false ******** NULL NULL mz_system 16661 true true false false ******** NULL NULL diff --git a/test/sqllogictest/rbac_create_table_from_source.slt b/test/sqllogictest/rbac_create_table_from_source.slt new file mode 100644 index 0000000000000..9459abfdf0c20 --- /dev/null +++ b/test/sqllogictest/rbac_create_table_from_source.slt @@ -0,0 +1,167 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# `CREATE TABLE ... FROM SOURCE` reads the source's data, so it requires the +# same read privileges as reading the source directly: `SELECT` on the source +# and `USAGE` on its schema. + +mode cockroach + +reset-server + +simple conn=mz_system,user=mz_system +ALTER SYSTEM SET enable_rbac_checks TO true; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +CREATE ROLE attacker; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +CREATE SCHEMA attacker_schema; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +GRANT USAGE, CREATE ON SCHEMA attacker_schema TO attacker; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +GRANT USAGE ON DATABASE materialize TO attacker; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +GRANT USAGE ON CLUSTER quickstart TO attacker; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +CREATE SCHEMA victim_schema; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +CREATE SOURCE victim_schema.victim_auction FROM LOAD GENERATOR AUCTION (AS OF 300, UP TO 301) FOR ALL TABLES; +---- +COMPLETE 0 + +# The attacker can name the objects but holds no read privilege on the source. + +simple conn=mz_system,user=mz_system +GRANT USAGE ON SCHEMA victim_schema TO attacker; +---- +COMPLETE 0 + +# Reading an export directly is denied. + +simple conn=attacker,user=attacker +SELECT count(*) FROM victim_schema.users; +---- +db error: ERROR: permission denied for SOURCE "materialize.victim_schema.users" +DETAIL: The 'attacker' role needs SELECT privileges on SOURCE "materialize.victim_schema.users" + +# Attaching a reference to an attacker-owned table is denied for the same +# reason: it reads the source's data. + +simple conn=attacker,user=attacker +CREATE TABLE attacker_schema.laundered FROM SOURCE victim_schema.victim_auction (REFERENCE "auction"."users"); +---- +db error: ERROR: permission denied for SOURCE "materialize.victim_schema.victim_auction" +DETAIL: The 'attacker' role needs SELECT privileges on SOURCE "materialize.victim_schema.victim_auction" + +# `CREATE` on the destination schema alone is not enough even when the source +# is readable, so both halves of the requirement are pinned. + +simple conn=mz_system,user=mz_system +CREATE ROLE no_create; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +GRANT USAGE ON DATABASE materialize TO no_create; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +GRANT USAGE ON SCHEMA victim_schema, attacker_schema TO no_create; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +GRANT SELECT ON victim_schema.victim_auction TO no_create; +---- +COMPLETE 0 + +simple conn=no_create,user=no_create +CREATE TABLE attacker_schema.nope FROM SOURCE victim_schema.victim_auction (REFERENCE "auction"."users"); +---- +db error: ERROR: permission denied for SCHEMA "materialize.attacker_schema" +DETAIL: The 'no_create' role needs CREATE privileges on SCHEMA "materialize.attacker_schema" + +# With `SELECT` on the source, the attach is allowed, so the check denies the +# attack without blocking legitimate attachment. + +simple conn=mz_system,user=mz_system +GRANT SELECT ON victim_schema.victim_auction TO attacker; +---- +COMPLETE 0 + +simple conn=attacker,user=attacker +CREATE TABLE attacker_schema.authorized FROM SOURCE victim_schema.victim_auction (REFERENCE "auction"."users"); +---- +COMPLETE 0 + +# The outcome does not depend on unrelated catalog state. A second role with the +# same grants can attach the same reference even though another team's table now +# exports it too, and nothing another team does to its own tables can revoke +# this role's ability to run the same statement. + +simple conn=mz_system,user=mz_system +CREATE ROLE other_team; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +CREATE SCHEMA other_team_schema; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +GRANT USAGE, CREATE ON SCHEMA other_team_schema TO other_team; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +GRANT USAGE ON DATABASE materialize TO other_team; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +GRANT USAGE ON CLUSTER quickstart TO other_team; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +GRANT USAGE ON SCHEMA victim_schema TO other_team; +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +GRANT SELECT ON victim_schema.victim_auction TO other_team; +---- +COMPLETE 0 + +simple conn=other_team,user=other_team +CREATE TABLE other_team_schema.mine FROM SOURCE victim_schema.victim_auction (REFERENCE "auction"."users"); +---- +COMPLETE 0 diff --git a/test/sqllogictest/regclass.slt b/test/sqllogictest/regclass.slt index d3816146ae8ea..7257fab6c0108 100644 --- a/test/sqllogictest/regclass.slt +++ b/test/sqllogictest/regclass.slt @@ -35,12 +35,12 @@ CREATE MATERIALIZED VIEW s.m AS SELECT * FROM s.t; query T SELECT 't'::regclass::oid::int ---- -20196 +20202 query T SELECT 's.t'::regclass::oid::int ---- -20197 +20203 query T SELECT 't'::regclass = 's.t'::regclass @@ -73,7 +73,7 @@ t query T SELECT 't'::regclass::oid::int ---- -20197 +20203 query T SELECT 'public.t'::regclass::text; @@ -101,12 +101,12 @@ d.public.t query T SELECT 'm'::regclass::oid::int ---- -20201 +20207 query T SELECT 's.m'::regclass::oid::int ---- -20202 +20208 query T SELECT 'm'::regclass = 's.m'::regclass @@ -296,7 +296,7 @@ true query T SELECT 'materialize.public.t'::regclass::oid::int ---- -20196 +20202 query error relation "t" does not exist SELECT 't'::regclass::oid::int diff --git a/test/sqllogictest/regtype.slt b/test/sqllogictest/regtype.slt index 8ba6b5b88e8f3..98aa4a37e3671 100644 --- a/test/sqllogictest/regtype.slt +++ b/test/sqllogictest/regtype.slt @@ -142,12 +142,12 @@ CREATE TYPE d.public.t AS LIST (ELEMENT TYPE = int4); query T SELECT 't'::regtype::oid::int ---- -20197 +20203 query T SELECT 's.t'::regtype::oid::int ---- -20198 +20204 query T SELECT 't'::regtype = 's.t'::regtype diff --git a/test/testdrive/catalog.td b/test/testdrive/catalog.td index 3e4ad1606a6a3..d783af6de9cce 100644 --- a/test/testdrive/catalog.td +++ b/test/testdrive/catalog.td @@ -763,6 +763,7 @@ mz_compute_import_frontiers_per_worker log "" mz_compute_lir_mapping_per_worker log "" mz_compute_operator_durations_histogram_raw log "" mz_compute_operator_hydration_statuses_per_worker log "" +mz_cluster_replica_resource_usage log "" mz_cluster_prometheus_metrics log "" mz_dataflow_addresses_per_worker log "" mz_dataflow_channels_per_worker log "" @@ -839,7 +840,7 @@ test_table "" # There is one entry in mz_indexes for each field_number/expression of the index. > SELECT COUNT(id) FROM mz_indexes WHERE id LIKE 's%' -274 +280 # Create a second schema with the same table name as above > CREATE SCHEMA tester2 diff --git a/test/testdrive/indexes.td b/test/testdrive/indexes.td index 6368143730dbf..ab3beb5266589 100644 --- a/test/testdrive/indexes.td +++ b/test/testdrive/indexes.td @@ -322,6 +322,7 @@ mz_compute_import_frontiers_per_worker_s2_primary_idx mz_compute_import_fr mz_compute_lir_mapping_per_worker_s2_primary_idx mz_compute_lir_mapping_per_worker mz_catalog_server {global_id,lir_id,worker_id} "" mz_compute_operator_durations_histogram_raw_s2_primary_idx mz_compute_operator_durations_histogram_raw mz_catalog_server {id,worker_id,duration_ns} "" mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx mz_compute_operator_hydration_statuses_per_worker mz_catalog_server {export_id,lir_id,worker_id} "" +mz_cluster_replica_resource_usage_s2_primary_idx mz_cluster_replica_resource_usage mz_catalog_server {process_id,source,metric} "" mz_cluster_prometheus_metrics_s2_primary_idx mz_cluster_prometheus_metrics mz_catalog_server {process_id,metric_name,labels} "" mz_connections_ind mz_connections mz_catalog_server {schema_id} "" mz_console_cluster_utilization_overview_24h_ind mz_console_cluster_utilization_overview_24h mz_catalog_server {cluster_id} "" diff --git a/test/workload-replay/system_catalog_identifiers.txt b/test/workload-replay/system_catalog_identifiers.txt index 32959e9a05343..354f1df0e5662 100644 --- a/test/workload-replay/system_catalog_identifiers.txt +++ b/test/workload-replay/system_catalog_identifiers.txt @@ -665,6 +665,13 @@ mz_builtin_sources mz_catalog_raw mz_cluster_deployment_lineage mz_cluster_deployment_lineage_ind +mz_cluster_replica_resource_usage +mz_cluster_replica_resource_usage_s1_primary_idx +mz_cluster_replica_resource_usage_s2_primary_idx +mz_cluster_replica_resource_usage_s3_primary_idx +mz_cluster_replica_resource_usage_s4_primary_idx +mz_cluster_replica_resource_usage_s5_primary_idx +mz_cluster_replica_resource_usage_u1_primary_idx mz_cluster_prometheus_metrics mz_cluster_prometheus_metrics_s1_primary_idx mz_cluster_prometheus_metrics_s2_primary_idx