diff --git a/Cargo.lock b/Cargo.lock index f6250503ef9d5..67c2707b6f438 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3683,7 +3683,6 @@ dependencies = [ "tokio", "tokio-postgres", "tokio-stream", - "tokio-util", "tonic", "tonic-prost", "tonic-prost-build", @@ -6159,7 +6158,6 @@ dependencies = [ "timely", "tokio", "tokio-postgres", - "tokio-stream", "tracing", "tracing-core", "tracing-opentelemetry", @@ -6181,7 +6179,6 @@ dependencies = [ "serde", "serde_json", "timely", - "tracing", ] [[package]] @@ -6215,7 +6212,6 @@ dependencies = [ "mz-repr", "num-traits", "ordered-float 5.1.0", - "serde", "serde_json", "uuid", ] @@ -6297,12 +6293,10 @@ version = "0.0.0" dependencies = [ "aws-sdk-glue", "aws-smithy-mocks", - "aws-smithy-runtime-api", "aws-types", "mz-ore", "thiserror 2.0.18", "tokio", - "tracing", "uuid", ] @@ -6313,10 +6307,7 @@ dependencies = [ "anyhow", "async-trait", "aws-config", - "aws-credential-types", "aws-sdk-secretsmanager", - "aws-types", - "futures", "mz-aws-util", "mz-repr", "mz-secrets", @@ -6338,7 +6329,6 @@ dependencies = [ "bytes", "bytesize", "futures", - "http 1.4.2", "hyper 0.14.32", "hyper-tls 0.5.0", "mz-ore", @@ -6364,7 +6354,6 @@ dependencies = [ "hickory-resolver", "humantime", "hyper 1.9.0", - "hyper-openssl", "hyper-util", "jsonwebtoken 10.3.0", "launchdarkly-server-sdk", @@ -6434,7 +6423,6 @@ dependencies = [ "bytes", "bytesize", "chrono", - "clap", "derivative", "differential-dataflow", "futures", @@ -6473,7 +6461,6 @@ dependencies = [ "prometheus", "proptest", "proptest-derive", - "prost", "rand 0.9.4", "semver", "serde", @@ -6568,9 +6555,7 @@ dependencies = [ name = "mz-cloud-api" version = "0.0.0" dependencies = [ - "anyhow", "chrono", - "mz-frontegg-auth", "mz-frontegg-client", "reqwest 0.12.28", "serde", @@ -6675,10 +6660,8 @@ dependencies = [ "mz-alloc-default", "mz-build-info", "mz-cloud-resources", - "mz-cluster", "mz-cluster-client", "mz-compute", - "mz-compute-client", "mz-dyncfgs", "mz-http-util", "mz-metrics", @@ -6688,7 +6671,6 @@ dependencies = [ "mz-prof-http", "mz-service", "mz-storage", - "mz-storage-client", "mz-storage-types", "mz-timely-util", "mz-txn-wal", @@ -6820,7 +6802,6 @@ name = "mz-compute-types" version = "0.0.0" dependencies = [ "columnar", - "columnation", "differential-dataflow", "itertools 0.14.0", "mz-dyncfg", @@ -6841,7 +6822,6 @@ dependencies = [ "bytesize", "chrono", "futures", - "itertools 0.14.0", "mz-build-info", "mz-cluster-client", "mz-compute-client", @@ -6852,8 +6832,6 @@ dependencies = [ "mz-orchestrator", "mz-ore", "mz-persist-client", - "mz-persist-types", - "mz-proto", "mz-repr", "mz-service", "mz-storage-client", @@ -6894,7 +6872,6 @@ dependencies = [ "mz-ore", "mz-postgres-util", "mz-server-core", - "mz-sql-parser", "mz-tls-util", "postgres-openssl", "regex", @@ -6968,23 +6945,13 @@ dependencies = [ name = "mz-durable-cache" version = "0.0.0" dependencies = [ - "async-trait", - "bytes", "differential-dataflow", - "futures", - "itertools 0.14.0", - "mz-dyncfg", "mz-ore", "mz-persist-client", "mz-persist-types", - "mz-timely-util", - "prometheus", - "prost", - "serde", "timely", "tokio", "tracing", - "uuid", ] [[package]] @@ -7152,7 +7119,6 @@ dependencies = [ "serde_urlencoded", "shell-words", "similar-asserts", - "stacker", "sysctl", "tempfile", "thiserror 2.0.18", @@ -7164,7 +7130,6 @@ dependencies = [ "tower 0.5.3", "tower-http", "tower-sessions", - "tower-sessions-memory-store", "tracing", "tracing-capture", "tracing-opentelemetry", @@ -7181,7 +7146,6 @@ dependencies = [ "aho-corasick", "anyhow", "aws-lc-rs", - "bytes", "bytesize", "chrono", "chrono-tz", @@ -7192,7 +7156,6 @@ dependencies = [ "datadriven", "dec", "derivative", - "differential-dataflow", "encoding", "enum-iterator", "fallible-iterator 0.2.0", @@ -7313,10 +7276,8 @@ version = "0.0.0" dependencies = [ "jsonwebtoken 10.3.0", "mz-frontegg-auth", - "mz-ore", "reqwest 0.12.28", "serde", - "serde_json", "thiserror 2.0.18", "tokio", "url", @@ -7357,7 +7318,6 @@ dependencies = [ "axum-extra", "headers", "http 1.4.2", - "hyper 1.9.0", "include_dir", "mime", "mz-ore", @@ -7418,7 +7378,6 @@ dependencies = [ "clap", "crossbeam", "fancy-regex", - "futures", "itertools 0.14.0", "mz-avro", "mz-build-tools", @@ -7426,7 +7385,6 @@ dependencies = [ "mz-ore", "mz-ssh-util", "num_cpus", - "prost", "prost-build", "rand 0.9.4", "rdkafka", @@ -7500,7 +7458,6 @@ dependencies = [ "prometheus", "thiserror 2.0.18", "tokio", - "tokio-util", "tracing", ] @@ -7529,7 +7486,6 @@ dependencies = [ "aws-types", "chrono", "fancy-regex", - "indexmap 2.11.4", "itertools 0.14.0", "maplit", "mysql_async", @@ -7552,7 +7508,6 @@ dependencies = [ "tokio", "tracing", "url", - "uuid", ] [[package]] @@ -7579,12 +7534,9 @@ dependencies = [ "jsonwebtoken 10.3.0", "mz-ore", "openssl", - "reqwest 0.12.28", "serde", "serde_json", "tokio", - "tracing", - "uuid", ] [[package]] @@ -7672,10 +7624,7 @@ dependencies = [ "mz-tracing", "opentelemetry", "opentelemetry_sdk", - "sentry-tracing", - "tracing", "tracing-capture", - "tracing-subscriber", ] [[package]] @@ -7716,10 +7665,8 @@ dependencies = [ "sha2", "thiserror 2.0.18", "tokio", - "tokio-postgres", "tower-http", "tracing", - "urlencoding", "uuid", ] @@ -7831,7 +7778,6 @@ dependencies = [ "fail", "futures-util", "itertools 0.14.0", - "md-5", "mz-aws-util", "mz-build-tools", "mz-dyncfg", @@ -7840,10 +7786,8 @@ dependencies = [ "mz-persist-types", "mz-postgres-client", "mz-proto", - "openssl", "openssl-sys", "parquet", - "postgres-openssl", "postgres-protocol", "prometheus", "proptest", @@ -7946,7 +7890,6 @@ dependencies = [ "prost-build", "serde", "serde_json", - "timely", "tracing", "uuid", ] @@ -8028,7 +7971,6 @@ dependencies = [ "base64 0.22.1", "byteorder", "bytes", - "bytesize", "csv-core", "enum-kinds", "futures", @@ -8094,18 +8036,15 @@ name = "mz-postgres-util" version = "0.0.0" dependencies = [ "anyhow", - "itertools 0.14.0", "mz-build-tools", "mz-cloud-resources", "mz-ore", "mz-proto", "mz-repr", - "mz-sql-parser", "mz-ssh-util", "mz-tls-util", "openssh", "openssl", - "postgres-openssl", "postgres_array", "proptest", "proptest-derive", @@ -8125,7 +8064,6 @@ dependencies = [ "anyhow", "backtrace", "flate2", - "itertools 0.14.0", "jemalloc_pprof", "mz-build-tools", "mz-ore", @@ -8194,7 +8132,6 @@ dependencies = [ "serde", "serde_json", "url", - "uuid", ] [[package]] @@ -8261,7 +8198,6 @@ dependencies = [ "tracing", "tracing-core", "tracing-subscriber", - "url", "uuid", ] @@ -8279,7 +8215,6 @@ dependencies = [ "prometheus", "rocksdb", "serde", - "serde_json", "tempfile", "thiserror 2.0.18", "tokio", @@ -8315,7 +8250,6 @@ name = "mz-s3-datagen" version = "0.0.0" dependencies = [ "anyhow", - "aws-config", "aws-sdk-s3", "bytefmt", "clap", @@ -8416,24 +8350,17 @@ version = "0.0.0" dependencies = [ "anyhow", "array-concat", - "aws-sdk-sts", - "bitflags 1.3.2", "chrono", - "clap", "derivative", "dynfmt", "enum-kinds", "fail", - "globset", "hex", - "http 1.4.2", "humantime", - "iceberg", "imbl", "ipnet", "itertools 0.14.0", "maplit", - "mysql_async", "mz-adapter-types", "mz-arrow-util", "mz-audit-log", @@ -8442,7 +8369,6 @@ dependencies = [ "mz-build-info", "mz-ccsr", "mz-cloud-provider", - "mz-cloud-resources", "mz-compute-types", "mz-controller-types", "mz-dyncfg", @@ -8469,7 +8395,6 @@ dependencies = [ "mz-ssh-util", "mz-storage-types", "mz-tracing", - "mz-txn-wal", "paste", "prometheus", "proptest", @@ -8486,7 +8411,6 @@ dependencies = [ "tokio", "tokio-postgres", "tracing", - "tracing-subscriber", "uncased", "uuid", "version-compare", @@ -8530,12 +8454,10 @@ dependencies = [ "mz-sql-lexer", "mz-sql-parser", "mz-walkabout", - "phf 0.13.1", "serde", "smallvec", "thiserror 2.0.18", "tracing", - "uncased", "unicode-width 0.2.2", ] @@ -8608,14 +8530,11 @@ dependencies = [ "maplit", "md-5", "mz-adapter-types", - "mz-authenticator", - "mz-build-info", "mz-catalog", "mz-controller", "mz-dyncfgs", "mz-environmentd", "mz-license-keys", - "mz-orchestrator", "mz-orchestrator-process", "mz-orchestrator-tracing", "mz-ore", @@ -8652,7 +8571,6 @@ version = "0.0.0" dependencies = [ "anyhow", "aws-lc-rs", - "futures", "itertools 0.14.0", "mz-ore", "openssh", @@ -8663,11 +8581,8 @@ dependencies = [ "serde_json", "ssh-key", "tempfile", - "thiserror 2.0.18", "tokio", - "tokio-stream", "tracing", - "zeroize", ] [[package]] @@ -8676,16 +8591,12 @@ version = "0.0.0" dependencies = [ "anyhow", "arrow", - "arrow-ipc", "async-stream", "async-trait", - "aws-credential-types", - "aws-sdk-sts", "axum", "bincode", "bytemuck", "bytes", - "bytesize", "chrono", "clap", "columnar", @@ -8737,7 +8648,6 @@ dependencies = [ "pin-project", "postgres-replication", "prometheus", - "prost", "rand 0.8.5", "rdkafka", "regex", @@ -8805,7 +8715,6 @@ version = "0.0.0" dependencies = [ "anyhow", "async-trait", - "bytes", "chrono", "derivative", "differential-dataflow", @@ -8819,21 +8728,15 @@ dependencies = [ "mz-ore", "mz-persist-client", "mz-persist-types", - "mz-proto", "mz-repr", "mz-service", "mz-storage-client", "mz-storage-types", "mz-timely-util", - "mz-tls-util", "mz-txn-wal", - "proptest", - "prost", - "serde", "serde_json", "timely", "tokio", - "tokio-postgres", "tokio-stream", "tracing", "uuid", @@ -8864,14 +8767,12 @@ dependencies = [ "itertools 0.14.0", "mz-arrow-util", "mz-aws-util", - "mz-dyncfg", "mz-expr", "mz-ore", "mz-persist-client", "mz-persist-types", "mz-pgcopy", "mz-pgrepr", - "mz-proto", "mz-repr", "mz-storage-types", "mz-timely-util", @@ -8879,7 +8780,6 @@ dependencies = [ "parquet", "prometheus", "proptest", - "prost", "reqwest 0.12.28", "sentry", "serde", @@ -8887,10 +8787,8 @@ dependencies = [ "thiserror 2.0.18", "timely", "tokio", - "tokio-stream", "tokio-util", "tracing", - "url", "uuid", ] @@ -9005,11 +8903,9 @@ dependencies = [ "chrono", "clap", "duckdb", - "flate2", "futures", "globset", "hex", - "http 1.4.2", "humantime", "itertools 0.14.0", "junit-report", @@ -9023,7 +8919,6 @@ dependencies = [ "mz-build-tools", "mz-catalog", "mz-ccsr", - "mz-expr", "mz-interchange", "mz-kafka-util", "mz-license-keys", @@ -9039,12 +8934,10 @@ dependencies = [ "mz-storage-types", "mz-tls-util", "parquet", - "postgres-protocol", "postgres_array", "prost", "prost-build", "prost-reflect", - "prost-types", "rand 0.9.4", "rdkafka", "regex", @@ -9056,12 +8949,10 @@ dependencies = [ "similar 2.7.0", "tempfile", "termcolor", - "tiberius", "time", "tokio", "tokio-postgres", "tokio-stream", - "tokio-util", "tonic", "tonic-prost", "tonic-prost-build", @@ -9078,16 +8969,13 @@ version = "0.0.0" dependencies = [ "ahash 0.8.12", "allocation-counter", - "bincode", "bytemuck", "columnar", "columnation", "criterion", "differential-dataflow", - "either", "futures-util", "itertools 0.14.0", - "lgalloc", "lz4_flex", "mz-ore", "num-traits", @@ -9132,12 +9020,9 @@ dependencies = [ "anyhow", "mz-ore", "openssl", - "openssl-sys", "postgres-openssl", "thiserror 2.0.18", - "tokio", "tokio-postgres", - "tracing", ] [[package]] @@ -9177,7 +9062,6 @@ dependencies = [ name = "mz-txn-wal" version = "0.0.0" dependencies = [ - "async-trait", "bytes", "crossbeam-channel", "differential-dataflow", @@ -10025,7 +9909,6 @@ dependencies = [ "timely", "tokio", "tracing", - "url", "uuid", ] diff --git a/bin/unused-deps b/bin/unused-deps index 673d0d9bfc3e8..1a9b5cc3c82f7 100755 --- a/bin/unused-deps +++ b/bin/unused-deps @@ -11,18 +11,4 @@ # # unused-deps — prints unused dependencies for all Rust crates. -# Requires https://github.com/est31/cargo-udeps and a working nightly toolchain. -# -# cargo-udeps occasionally produces false positives, especially with -# dependencies that are used only on certain platforms. To ignore these -# dependencies, see: -# https://github.com/est31/cargo-udeps#ignoring-some-of-the-dependencies. - -set -euo pipefail - -cd "$(dirname "$0")/.." - -. misc/shlib/shlib.bash - -export RUSTUP_TOOLCHAIN=${RUSTUP_TOOLCHAIN:=nightly} -cargo udeps --workspace --all-targets "$@" +exec "$(dirname "$0")"/pyactivate -m materialize.cli.unused-deps "$@" diff --git a/ci/mkpipeline.py b/ci/mkpipeline.py index b953ce447118b..586b3afdfb103 100644 --- a/ci/mkpipeline.py +++ b/ci/mkpipeline.py @@ -1238,7 +1238,7 @@ def add_cargo_test_dependency( return for step in steps(pipeline): - if step.get("id") in ("cargo-test", "miri-test"): + if step.get("id") == "cargo-test": step["depends_on"] = ( "build-x86_64" if "x86" in step["agents"]["queue"] else "build-aarch64" ) diff --git a/ci/nightly/pipeline.template.yml b/ci/nightly/pipeline.template.yml index 4716f7e1ae149..42705f5ee2800 100644 --- a/ci/nightly/pipeline.template.yml +++ b/ci/nightly/pipeline.template.yml @@ -107,7 +107,7 @@ steps: - id: unused-deps label: Unused dependencies - command: bin/ci-builder run nightly bin/unused-deps + command: bin/ci-builder run stable bin/unused-deps inputs: - Cargo.lock - Cargo.toml @@ -163,7 +163,7 @@ steps: - id: miri-test topics: [kafka, mysql, postgres] label: ":rust: Miri test (full)" - depends_on: build-aarch64 + depends_on: [] timeout_in_minutes: 60 plugins: - ./ci/plugins/mzcompose: @@ -587,7 +587,7 @@ steps: - id: ssh-connection-extended topics: [kafka, kafka-sink, mysql, postgres, sql-server, ssh-tunnel] label: Extended SSH connection - depends_on: build-aarch64 + depends_on: build-x86_64 timeout_in_minutes: 45 plugins: - ./ci/plugins/mzcompose: diff --git a/ci/test/cargo-test/mzcompose.py b/ci/test/cargo-test/mzcompose.py index 9bcad2023d249..8c6e9b1ab7bee 100644 --- a/ci/test/cargo-test/mzcompose.py +++ b/ci/test/cargo-test/mzcompose.py @@ -123,6 +123,19 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: if os.path.exists(junit_path): os.remove(junit_path) + coverage = ui.env_is_truthy("CI_COVERAGE_ENABLED") + + # Miri excludes the network-based tests, so it needs none of the services + # below and pulls none of their mzbuild images. Dispatch before `c.up()` so + # the Buildkite step has nothing to depend on the build for. + if not coverage: + if args.miri_full: + run_miri_slow(dict(os.environ)) + return + if args.miri_fast: + run_miri_fast(dict(os.environ)) + return + c.up( "kafka", "schema-registry", @@ -168,7 +181,6 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: FDB_CLUSTER_FILE=fdb_cluster_file.name, ) - coverage = ui.env_is_truthy("CI_COVERAGE_ENABLED") sanitizer = Sanitizer[os.getenv("CI_SANITIZER", "none")] metadata = json.loads( @@ -179,10 +191,6 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: if coverage: run_coverage_test(args, env) - elif args.miri_full: - run_miri_slow(env) - elif args.miri_fast: - run_miri_fast(env) elif sanitizer != Sanitizer.none: run_sanitizer(args, env, metadata, sanitizer) else: diff --git a/deny.toml b/deny.toml index 0c7fdc9c2645f..1bb62cceccbf7 100644 --- a/deny.toml +++ b/deny.toml @@ -301,6 +301,8 @@ ignore = [ "RUSTSEC-2026-0195", # `bitmaps` is unmaintained "RUSTSEC-2026-0247", + # `aws-sdk-s3` and `launchdarkly-server-sdk` both require `lru ^0.16.3` in their latest releases + "RUSTSEC-2026-0253", ] # Must be manually kept in sync with about.toml. diff --git a/doc/developer/generated/adapter/catalog/_module.md b/doc/developer/generated/adapter/catalog/_module.md index f05a1c38f46d6..a094c3edc4bd5 100644 --- a/doc/developer/generated/adapter/catalog/_module.md +++ b/doc/developer/generated/adapter/catalog/_module.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog.rs -revision: a60edac7f1 +revision: fd1dd6e62b --- # adapter::catalog diff --git a/doc/developer/generated/adapter/catalog/apply.md b/doc/developer/generated/adapter/catalog/apply.md index b888012aea3f3..c7593445dafa0 100644 --- a/doc/developer/generated/adapter/catalog/apply.md +++ b/doc/developer/generated/adapter/catalog/apply.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/apply.rs -revision: fca741734d +revision: b9097f8a3d --- # adapter::catalog::apply diff --git a/doc/developer/generated/adapter/catalog/builtin_table_updates.md b/doc/developer/generated/adapter/catalog/builtin_table_updates.md index b292275e94ebe..e417139a15005 100644 --- a/doc/developer/generated/adapter/catalog/builtin_table_updates.md +++ b/doc/developer/generated/adapter/catalog/builtin_table_updates.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/builtin_table_updates.rs -revision: 34813f0a3f +revision: fd1dd6e62b --- # adapter::catalog::builtin_table_updates @@ -10,5 +10,5 @@ For each catalog object type, a corresponding `pack_*` function converts a `Cata `BuiltinTableUpdate` wraps either a `Row`-level update or a `ProtoBatch` for bulk ingestion, both resolved to a `CatalogItemId` before being written. `pack_role_auth_update` writes a row into `mz_role_auth` containing the role's password hash and the timestamp it was last updated. `mz_roles`, `mz_role_parameters`, `mz_clusters`, `mz_cluster_replicas`, `mz_cluster_schedules`, `mz_default_privileges`, `mz_system_privileges`, and `mz_internal.mz_comments` are materialized views backed by `mz_internal.mz_catalog_raw`; rows for these objects are not produced by `pack_*` functions here. `MZ_CLUSTER_REPLICA_SIZE_INTERNAL` (a `BuiltinTable`) receives rows via `pack_cluster_replica_size_update` and retains entries for disabled sizes; the `mz_cluster_replicas` MV LEFT JOINs it to resolve the `disk` column. -`CatalogItem::Secret`, `CatalogItem::Log`, `CatalogItem::Connection`, and `CatalogItem::Index` do not produce direct table updates for `MZ_SECRETS`, `MZ_SOURCES`, `MZ_CONNECTIONS`, or `MZ_INDEXES` from `pack_*` functions; `mz_connections`, `mz_secrets`, and `mz_sources` are backed by catalog views or materialized views rather than builtin tables, so `CatalogItem::Secret` and `CatalogItem::Log` return empty update vecs and `pack_connection_update` only emits rows for the connection-type-specific subtables (e.g. `MZ_KAFKA_CONNECTIONS`, `MZ_SSH_TUNNEL_CONNECTIONS`). `ConnectionDetails::GlueSchemaRegistry` and `ConnectionDetails::Gcp` are handled in `pack_connection_update` alongside `Csr`, `Postgres`, `MySql`, and `SqlServer` as types that do not produce their own subtable row. `CatalogItem::Source` does not produce rows into `MZ_SOURCES` (a materialized view over `mz_catalog_raw`), `mz_postgres_sources` (a materialized view), or `mz_kafka_sources` (a materialized view); for ingestion sources, `pack_item_update` returns an empty vec. Sources only emit rows for source-type-specific subtables such as postgres/kafka source-table mappings and subsource/progress metadata. `CatalogItem::Index` does not produce rows into `MZ_INDEXES` (a materialized view); `pack_index_update` does not write to `MZ_INDEXES`. +`CatalogItem::Secret`, `CatalogItem::Log`, `CatalogItem::Connection`, and `CatalogItem::Index` do not produce direct table updates from `pack_*` functions. `CatalogItem::Secret` and `CatalogItem::Log` return empty update vecs. `CatalogItem::Connection` returns an empty vec because all connection-detail subtables (`mz_kafka_connections`, `mz_ssh_tunnel_connections`, `mz_aws_connections`, `mz_aws_privatelink_connections`) are now materialized views over `mz_catalog_raw` derived from each connection's persisted `create_sql`. `CatalogItem::Source` does not produce rows into `MZ_SOURCES` (a materialized view over `mz_catalog_raw`), `mz_postgres_sources` (a materialized view), or `mz_kafka_sources` (a materialized view); for ingestion sources, `pack_item_update` returns an empty vec. Source-type subtable details such as `mz_postgres_source_tables`, `mz_mysql_source_tables`, `mz_sql_server_source_tables`, and `mz_kafka_source_tables` are also materialized views over `mz_catalog_raw`, so ingestion exports and old-syntax subsources do not emit rows from `pack_*` functions either. Sources only emit rows for webhook sources and source-specific progress/subsource metadata. `CatalogItem::Index` does not produce rows into `MZ_INDEXES` (a materialized view); `pack_index_update` does not write to `MZ_INDEXES`. The `notice` sub-module handles the special `mz_recent_activity_log` and related notice tables separately. diff --git a/doc/developer/generated/adapter/catalog/open.md b/doc/developer/generated/adapter/catalog/open.md index 2245ee04b37f8..b72ef54340a05 100644 --- a/doc/developer/generated/adapter/catalog/open.md +++ b/doc/developer/generated/adapter/catalog/open.md @@ -1,13 +1,14 @@ --- source: src/adapter/src/catalog/open.rs -revision: fca741734d +revision: 30210e9ef7 --- # adapter::catalog::open Implements `Catalog::open`, the full catalog initialisation sequence run at environment startup. The function opens the durable catalog store, runs AST migrations (`catalog::migrate`), applies all persisted `StateUpdate` diffs to build the in-memory `CatalogState`, bootstraps built-in objects (schemas, roles, clusters, tables, views, materialized views, functions), runs builtin-schema migrations (see `open::builtin_schema_migration`), and returns a ready `Catalog` along with the initial builtin-table updates to write. +After applying all updates, `state.system_config().sync_dyncfgs()` is called explicitly to mirror the effective `SystemVars` values into the dyncfg `ConfigSet`; `apply_updates` only syncs when a durable `SystemConfiguration` update is present, so a deployment configured purely via `system_parameter_default` would otherwise leave the `ConfigSet` at compile-time defaults for any startup-only read. `add_new_remove_old_builtin_items_migration` drops comments under every relation-style `CommentObjectId` variant (Table, View, MaterializedView, Source) for a given id, not just the current one. This handles cases where a builtin's type changes (e.g., Table to MaterializedView) but its catalog id is preserved. `add_new_remove_old_builtin_clusters_migration` emits audit log events when creating or dropping builtin clusters so those operations are visible in `mz_audit_events`. -`add_new_remove_old_builtin_cluster_replicas_migration` allocates builtin replica IDs via `txn.allocate_system_replica_id()` within the open transaction (single-source, safe because there is no coordinator at that point) and emits audit log events when creating or dropping builtin cluster replicas so those operations are visible in `mz_audit_events`. +`reconcile_builtin_cluster_replicas` reconciles each builtin cluster's replica set against the cluster's own durable `replication_factor`, creating missing replicas (allocating IDs via `txn.allocate_system_replica_id()`, single-source and safe because there is no coordinator at that point) and dropping surplus ones. Bootstrap flags seed `size` and `replication_factor` only when a cluster is first created, so an `ALTER CLUSTER` against a builtin cluster persists across restarts. Audit log events are emitted for every create and drop so those operations are visible in `mz_audit_events`. `remove_pending_cluster_replicas_migration` emits audit log events when dropping pending replicas so the drops are visible in `mz_audit_events`. diff --git a/doc/developer/generated/adapter/catalog/open/_module.md b/doc/developer/generated/adapter/catalog/open/_module.md index b59913fd7d8fb..c6194c3010590 100644 --- a/doc/developer/generated/adapter/catalog/open/_module.md +++ b/doc/developer/generated/adapter/catalog/open/_module.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/open.rs -revision: fca741734d +revision: 30210e9ef7 --- # adapter::catalog::open diff --git a/doc/developer/generated/adapter/catalog/open/builtin_schema_migration.md b/doc/developer/generated/adapter/catalog/open/builtin_schema_migration.md index e18491fc0b469..0f04ecc5c35e8 100644 --- a/doc/developer/generated/adapter/catalog/open/builtin_schema_migration.md +++ b/doc/developer/generated/adapter/catalog/open/builtin_schema_migration.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/open/builtin_schema_migration.rs -revision: 34813f0a3f +revision: fd1dd6e62b --- # adapter::catalog::open::builtin_schema_migration @@ -15,6 +15,7 @@ A `Replacement` step at version `26.31.0-dev.0` covers `mz_catalog.mz_cluster_re A `Replacement` step at version `26.32.0-dev.0` covers `mz_internal.mz_comments`, reflecting its conversion from `BuiltinTable` to `BuiltinMaterializedView`. A second `Replacement` step at version `26.32.0-dev.0` covers `mz_catalog.mz_indexes`, required because adding the console cluster utilization overview builtin indexes changed the VALUES set inlined by `make_mz_indexes`, altering the MV's SQL fingerprint. A `Replacement` step at version `26.33.0-dev.0` covers `mz_catalog.mz_audit_events`, reflecting its conversion from `BuiltinTable` to `BuiltinMaterializedView`. Three `Replacement` steps at version `26.34.0-dev.0` cover: `mz_catalog.mz_indexes` (because adding the `mz_cluster_reconfigurations_ind` and `mz_cluster_auto_scaling_strategies_ind` builtin indexes changes the VALUES set inlined by `make_mz_indexes`, altering the MV's SQL fingerprint); `mz_internal.mz_postgres_sources` (reflecting its conversion from `BuiltinTable` to `BuiltinMaterializedView`); and `mz_catalog.mz_kafka_sources` (reflecting the same conversion). +Eight `Replacement` steps at version `26.37.0-dev.0` cover: `mz_internal.mz_postgres_source_tables`, `mz_internal.mz_mysql_source_tables`, `mz_internal.mz_sql_server_source_tables`, and `mz_internal.mz_kafka_source_tables` (reflecting their conversion from `BuiltinTable` to `BuiltinMaterializedView`); and `mz_catalog.mz_kafka_connections`, `mz_catalog.mz_ssh_tunnel_connections`, `mz_internal.mz_aws_connections`, and `mz_catalog.mz_aws_privatelink_connections` (reflecting their conversion from `BuiltinTable` to `BuiltinMaterializedView`, where each is now derived from its connection's persisted `create_sql` via `mz_catalog_raw`). When applying replacement migrations, `mz_storage_usage_by_shard` is excluded from data-destroying replacement plans to preserve billing data. When the source and target versions differ and the source version is a dev build, `Migration::run` forces evolution-mode migration even without an explicit `force_migration` config, avoiding version-based filter failures in dev environments. In forced dev-to-dev migrations, `migrate_evolve_one` skips builtins that do not yet have a shard registered. Brand-new builtins in a given version may not have their shards allocated until the leader completes bootstrap; excluding them avoids "missing shard ID" errors on read-only replicas. diff --git a/doc/developer/generated/adapter/catalog/state.md b/doc/developer/generated/adapter/catalog/state.md index aafc512664a23..a4d1f7e3c371a 100644 --- a/doc/developer/generated/adapter/catalog/state.md +++ b/doc/developer/generated/adapter/catalog/state.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/state.rs -revision: 3713eed996 +revision: efe0c63d45 --- # adapter::catalog::state diff --git a/doc/developer/generated/adapter/catalog/transact.md b/doc/developer/generated/adapter/catalog/transact.md index f8ba0868316be..ecc46854433a3 100644 --- a/doc/developer/generated/adapter/catalog/transact.md +++ b/doc/developer/generated/adapter/catalog/transact.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/transact.rs -revision: 6eeaca032b +revision: 38a95cefe2 --- # adapter::catalog::transact diff --git a/doc/developer/generated/adapter/client.md b/doc/developer/generated/adapter/client.md index 98b2f56c4b55e..2850b9cfcff7a 100644 --- a/doc/developer/generated/adapter/client.md +++ b/doc/developer/generated/adapter/client.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/client.rs -revision: 443b75e6b4 +revision: ff68428395 --- # adapter::client @@ -17,4 +17,5 @@ During `startup`, `Client` constructs a `PeekClient` from the `StartupResponse` `RecordFirstRowStream` is an adapter stream that records the timestamp of the first row for metrics. `TimeoutType` enumerates session-level timeouts (currently `IdleInTransactionSession`); the `Timeout` struct manages active timeout tasks and delivers expired timeouts through `SessionClient::recv_timeout`. `SessionClient::statement_arrival_logging_enabled` checks the `enable_statement_arrival_logging` system variable via the session's catalog snapshot. +`SessionClient::extended_protocol_implicit_transaction_enabled` checks the `enable_extended_protocol_implicit_transaction` system variable via the session's catalog snapshot; pgwire consults this when deciding whether an implicit write transaction in the extended-protocol pipeline should span to the Sync message. `redact_sql_for_logging(sql) -> String` parses SQL and re-serialises it with literals redacted (the same redaction the statement log applies). If the text does not parse or exceeds the statement batch size limit, a placeholder with the byte length is returned instead. diff --git a/doc/developer/generated/adapter/command.md b/doc/developer/generated/adapter/command.md index cb2a321b9eeea..5a9f8fb8620b9 100644 --- a/doc/developer/generated/adapter/command.md +++ b/doc/developer/generated/adapter/command.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/command.rs -revision: 5dc2cf510a +revision: ff68428395 --- # adapter::command diff --git a/doc/developer/generated/adapter/config/frontend.md b/doc/developer/generated/adapter/config/frontend.md index 95acb02df328b..6a5b29da62cce 100644 --- a/doc/developer/generated/adapter/config/frontend.md +++ b/doc/developer/generated/adapter/config/frontend.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/config/frontend.rs -revision: 34effa9dc0 +revision: 26305d8cb0 --- # adapter::config::frontend diff --git a/doc/developer/generated/adapter/coord/_module.md b/doc/developer/generated/adapter/coord/_module.md index 9d47643917c8a..3865c58cbc3a8 100644 --- a/doc/developer/generated/adapter/coord/_module.md +++ b/doc/developer/generated/adapter/coord/_module.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord.rs -revision: e156fe6b04 +revision: 38a95cefe2 --- # adapter::coord @@ -9,7 +9,7 @@ The core coordinator: `coord.rs` defines the `Coordinator` struct (the central s The coordinator owns the `Catalog`, active compute sinks, pending peeks, read-policy manager, timeline oracles, and all inter-subsystem handles (controller, storage collections, secrets, orchestrator). The `Coordinator` struct holds a `reconcile_now: Arc` field that wakes the cluster controller task to reconcile immediately after catalog transactions that change durable cluster state, rather than waiting out the tick interval. The file also defines `IdPool`, a pre-allocated pool of user `GlobalId` integers that amortizes per-DDL persist writes by reserving batches of IDs at once; the pool is owned by the coordinator and access is serialized through its single-threaded event loop. `ArrangementSizeRecord` is a struct for rows destined for `mz_object_arrangement_size_history`, carrying `replica_id`, `object_id`, `size`, and `hydration_complete`; records are prepared off-thread by the arrangement sizes snapshot task and stamped with a collection timestamp on the coordinator loop at write time. -Child modules partition the coordinator's responsibilities: `command_handler` handles external `Command` messages; `message_handler` handles internal async `Message` responses; `sequencer` executes SQL plans; `appends` manages table and builtin-table writes; `catalog_implications` derives and applies downstream effects from catalog state changes, including pushing replica-scoped dyncfg override layers to the compute controller via `push_replica_dyncfg_overrides`; `ddl` wraps catalog transactions; `peek` and `read_policy` manage query execution and compaction; `read_then_write` implements the read-then-write protocol for DML (INSERT, UPDATE, DELETE) including dependency validation; `group_sync` computes the diff between sync-managed and manually-granted role memberships for SSO group synchronization; `timestamp_selection` and `timeline` handle temporal reasoning; `catalog_serving` serves catalog snapshots; `info_metrics` owns the catalog `*_info` Prometheus series background task; `cluster_controller` scaffolds the cluster controller task that reconciles durable cluster state with the live controller; and supporting modules cover cluster scheduling, unified compute introspection subscribes, consistency checking, index management, and statement logging. +Child modules partition the coordinator's responsibilities: `command_handler` handles external `Command` messages; `message_handler` handles internal async `Message` responses; `sequencer` executes SQL plans; `appends` manages table and builtin-table writes; `catalog_implications` derives and applies downstream effects from catalog state changes, including pushing replica-scoped dyncfg override layers to the compute controller via `push_replica_dyncfg_overrides`; `ddl` wraps catalog transactions; `peek` and `read_policy` manage query execution and compaction; `read_then_write` implements the read-then-write protocol for DML (INSERT, UPDATE, DELETE) including dependency validation; `group_sync` computes the diff between sync-managed and manually-granted role memberships for SSO group synchronization; `timestamp_selection` and `timeline` handle temporal reasoning; `catalog_serving` serves catalog snapshots; `info_metrics` owns the catalog `*_info` Prometheus series background task; `cluster_controller` scaffolds the cluster controller task that reconciles durable cluster state with the live controller; and supporting modules cover unified compute introspection subscribes, consistency checking, index management, and statement logging. The `Coordinator` struct holds a `scoped_frontend: Option>` field populated by `Command::InstallScopedSystemParameterFrontend`. `reconcile_scoped_system_parameters` persists the diff between the current durable working copy and the desired `ScopedParameters` via `Op::UpdateScopedSystemParameters`, which also updates the in-memory working copy and introspection relations. `scoped_overrides_create_op` evaluates scoped overrides for freshly-created clusters and replicas from explicit `ClusterEvalContext` / `ReplicaEvalContext` values and returns an `Op::UpdateScopedSystemParameters` to fold into the same transaction that creates them, so the committed diff drives the replica-scoped controller push before `create_replica`. `push_replica_dyncfg_overrides` propagates the replica-scoped overrides from the catalog working copy to the compute controller's per-replica dyncfg layer. `cluster_scoped_optimizer_overrides` retrieves the cluster-coherent `OptimizerFeatureOverrides` from the catalog working copy for use at plan time. The `Coordinator` struct holds a `catalog_info_metrics_registry: MetricsRegistry` field used to hand the metrics registry to the catalog info-metrics background task spawned at bootstrap. Bootstrap handles derived builtin storage collections (builtin MVs) separately: after registering input-less collections in dependency order, it bumps their sinces based on transitive dependency frontiers to satisfy as-of selection invariants. Bootstrap restores compaction policies for all MV version IDs (via `global_ids()`, not just the write ID) so that no version blocks capability propagation through the `primary` ownership chain and pins compaction. When applying replacement MVs, each `CollectionDescription` receives a `primary` field pointing to its predecessor's latest collection ID, chaining shard ownership from the oldest version through each replacement in order. Bootstrap also calls `push_replica_dyncfg_overrides` once before dataflows are rendered so that replica-scoped dyncfg flags take effect on the first configuration of each replica, and calls `spawn_cluster_controller_task` to start the cluster controller background task. diff --git a/doc/developer/generated/adapter/coord/appends.md b/doc/developer/generated/adapter/coord/appends.md index a131d1e4d0d10..291693e547e9e 100644 --- a/doc/developer/generated/adapter/coord/appends.md +++ b/doc/developer/generated/adapter/coord/appends.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/appends.rs -revision: a60edac7f1 +revision: b5624d20bb --- # adapter::coord::appends @@ -9,5 +9,6 @@ Implements all append operations executed by the coordinator: group-commit of us `group_commit_initiate` coalesces pending table write requests into a single timestamped batch; `write_and_append_builtin_table_updates` drives updates to system catalog tables. `BuiltinTableAppendNotify` is a pinned future that resolves when a builtin-table write completes. `BuiltinTableAppendCompletion` is a completion handle wrapping a `BuiltinTableAppendNotify`; callers obtain one to use as a response barrier and convert it to a notify via `into_notify`. `GroupCommitWriteLocks` manages the locking primitives used to serialise group commits. `group_commit` always performs the append call even when there are no user writes, relying on the storage layer to periodically advance the upper of all tables; it does not iterate catalog entries to inject empty advancement entries per table. -`UserWriteResponder` is an enum with a single `Session(PendingTxn)` variant that wraps the per-session `PendingTxn` for `PendingWriteTxn::User`; the `User` variant's `pending_txn` field is replaced by `responder: UserWriteResponder`. The write timestamp for `User` writes is picked by the oracle during group commit; the write lock is either handed off from the submitting session (`write_locks: Some(..)`) or acquired during group commit (`write_locks: None`). -`GroupCommitter` serializes runtime txns-shard writes off the coordinator loop via `TableWriteCmd` variants: `GroupCommit`, `Register`, and `Forget`. `DeferredOp` represents an operation awaiting a resource; `DeferredOp::Plan` wraps a `DeferredPlan` that must uniquely hold write locks (e.g. UPDATE), while `DeferredOp::Write` wraps a `DeferredWrite` for blind inserts that can be optimistically retried. `GroupCommitRequest` accumulates appends, responses, statement-logging IDs, notifies, write locks, and permits for a single commit batch. +`UserWriteResponder` is an enum with two variants: `Session(PendingTxn)` wraps the per-session `PendingTxn` for coordinator-sequenced writes, and `Internal` carries a `WriteTarget` (item id plus the `GlobalId` generation current at planning time) and an `InternalWriteResponder` for frontend-sequenced blind writes. The write timestamp for `Session` writes is picked by the oracle during group commit; the write lock is either handed off from the submitting session (`write_locks: Some(..)`) or acquired during group commit (`write_locks: None`). +`TableWriteCmd` has a `TimestampedWrite` variant (in addition to `GroupCommit`, `Register`, and `Forget`) that carries a caller-chosen timestamp; the group committer accepts or refuses it based on oracle eligibility and reports the outcome via `WriteResult`. `WriteResult` is an enum whose variants are `Success { timestamp }`, `TimestampPassed { target_timestamp, next_eligible_timestamp }`, `Canceled`, `ReadOnly`, `TargetChanged`, and `Indeterminate`. `InternalWriteResponder` wraps the `oneshot::Sender`; its `Drop` impl sends `WriteResult::Indeterminate` so the waiting session task never hangs on coordinator shutdown. `WriteTarget` pairs a `CatalogItemId` with the `GlobalId` generation current at planning time; the group committer refuses the write if the table's latest generation has moved on. +`GroupCommitter` serializes runtime txns-shard writes off the coordinator loop via `TableWriteCmd` variants. `DeferredOp` represents an operation awaiting a resource; `DeferredOp::Plan` wraps a `DeferredPlan` that must uniquely hold write locks (e.g. UPDATE), while `DeferredOp::Write` wraps a `DeferredWrite` for blind inserts that can be optimistically retried. `GroupCommitRequest` accumulates appends, responses, statement-logging IDs, notifies, write locks, and permits for a single commit batch. diff --git a/doc/developer/generated/adapter/coord/catalog_implications.md b/doc/developer/generated/adapter/coord/catalog_implications.md index db867c6f56458..953f332c7efd4 100644 --- a/doc/developer/generated/adapter/coord/catalog_implications.md +++ b/doc/developer/generated/adapter/coord/catalog_implications.md @@ -1,12 +1,12 @@ --- source: src/adapter/src/coord/catalog_implications.rs -revision: a60edac7f1 +revision: efe0c63d45 --- # adapter::coord::catalog_implications Derives and applies downstream effects from catalog state changes following the pipeline `StateUpdateKind -> ParsedStateUpdate -> CatalogImplication`. `ParsedStateUpdate` enriches raw catalog diffs with in-memory object representations; `CatalogImplication` is both a state machine for absorbing multiple updates to the same object and the final command applied to controllers. -`apply_catalog_implications` reacts to consolidated catalog updates by creating or dropping compute dataflows and storage collections, updating read policies, advancing compaction frontiers, managing VPC endpoints, creating and dropping clusters and replicas, applying source alterations, cancelling affected peeks and subscribes, and sending builtin-table updates. When any `ReplicaSystemConfiguration` update is present in the batch, `apply_catalog_implications` calls `push_replica_dyncfg_overrides` after cluster creation but before replica creation, re-pushing the complete per-replica dyncfg override layer from the catalog working copy so new replicas receive their overrides on their first configuration. -After applying implications, `apply_catalog_implications` calls `reconcile_now.notify_one()` when `should_reconcile_now` returns true, waking the cluster controller task to reconcile immediately rather than waiting out its tick interval. `should_reconcile_now` returns true when any committed diff contains a `Cluster` or `ClusterReplica` update, keying the wake off the committed diff so it fires regardless of whether this node or another writer applied the change. Environment-wide system-config changes are not represented as catalog implications and do not trigger a wake; the controller picks those up on its next tick. +`apply_catalog_implications` reacts to consolidated catalog updates by creating or dropping compute dataflows and storage collections, updating read policies, advancing compaction frontiers, managing VPC endpoints, creating and dropping clusters and replicas, applying source alterations, cancelling affected peeks and subscribes, and sending builtin-table updates. When any `ReplicaSystemConfiguration` update is present in the batch, `apply_catalog_implications` calls `push_replica_dyncfg_overrides` after cluster creation but before replica creation, re-pushing the complete per-replica dyncfg override layer from the catalog working copy so new replicas receive their overrides on their first configuration. When any `SystemConfiguration` update is present in the batch, `apply_catalog_implications` calls `notify_all_callbacks` on `SystemVars` to re-run all registered callbacks against the committed values, including on follower `environmentd` instances that only replay the committed diff. +After applying implications, `apply_catalog_implications` calls `reconcile_now.notify_one()` when `should_reconcile_now` returns true, waking the cluster controller task to reconcile immediately rather than waiting out its tick interval. `should_reconcile_now` returns true when any committed diff contains a `Cluster` or `ClusterReplica` update, keying the wake off the committed diff so it fires regardless of whether this node or another writer applied the change. Environment-wide system-config changes are not represented as cluster-controller wakeup triggers and do not trigger a wake; the controller picks those up on its next tick. The child module `parsed_state_updates` handles the parsing step. diff --git a/doc/developer/generated/adapter/coord/catalog_implications/parsed_state_updates.md b/doc/developer/generated/adapter/coord/catalog_implications/parsed_state_updates.md index c258735dbf869..5c9eecd2e9322 100644 --- a/doc/developer/generated/adapter/coord/catalog_implications/parsed_state_updates.md +++ b/doc/developer/generated/adapter/coord/catalog_implications/parsed_state_updates.md @@ -1,11 +1,13 @@ --- source: src/adapter/src/coord/catalog_implications/parsed_state_updates.rs -revision: 5d24e1fd58 +revision: efe0c63d45 --- # adapter::coord::catalog_implications::parsed_state_updates Defines `ParsedStateUpdate` and `ParsedStateUpdateKind`, which enrich raw `StateUpdateKind` diffs with in-memory object representations to make `apply_catalog_implications` easier to work with. -The `ParsedStateUpdateKind` enum covers `Item`, `TemporaryItem`, `Cluster`, `ClusterReplica`, `IntrospectionSourceIndex`, and `ReplicaSystemConfiguration` variants. -The `ReplicaSystemConfiguration` variant carries the raw durable row and is emitted whenever a `StateUpdateKind::ReplicaSystemConfiguration` change is seen; the implication handler re-pushes the complete per-replica dyncfg layer from the catalog working copy rather than acting on individual rows, so the variant exists mainly to make the change visible in tracing. `StateUpdateKind::ClusterSystemConfiguration` changes are not represented because cluster-scoped parameter overrides are read at plan time and require no controller action. +The `ParsedStateUpdateKind` enum covers `Item`, `TemporaryItem`, `Cluster`, `ClusterReplica`, `IntrospectionSourceIndex`, `ReplicaSystemConfiguration`, and `SystemConfiguration` variants. +The `ReplicaSystemConfiguration` variant carries the raw durable row and is emitted whenever a `StateUpdateKind::ReplicaSystemConfiguration` change is seen; the implication handler re-pushes the complete per-replica dyncfg layer from the catalog working copy rather than acting on individual rows, so the variant exists mainly to make the change visible in tracing. +The `SystemConfiguration` variant carries the raw durable row and is emitted whenever a `StateUpdateKind::SystemConfiguration` change is seen; the implication handler re-runs all `SystemVars` callbacks against the committed values rather than acting on individual vars, so the variant exists mainly to make the change visible in tracing. +`StateUpdateKind::ClusterSystemConfiguration` changes are not represented because cluster-scoped parameter overrides are read at plan time and require no controller action. The `parse_state_update` function converts a `StateUpdate` into a `ParsedStateUpdate` by looking up parsed representations from the current `CatalogState`. diff --git a/doc/developer/generated/adapter/coord/cluster_controller.md b/doc/developer/generated/adapter/coord/cluster_controller.md index b503c14d2acc3..978c58b753ce5 100644 --- a/doc/developer/generated/adapter/coord/cluster_controller.md +++ b/doc/developer/generated/adapter/coord/cluster_controller.md @@ -1,22 +1,22 @@ --- source: src/adapter/src/coord/cluster_controller.rs -revision: 6eeaca032b +revision: 767b86caef --- # adapter::coord::cluster_controller Driver and glue for the `mz-cluster-controller` reconciler. -The controller crate is pure and knows nothing about the Coordinator. This module is the half of the `ClusterControllerCtx` boundary that does: it runs the controller as a separate task and implements the ctx by marshaling each pull/apply to the Coordinator over the internal command channel, because the catalog and live compute/storage signals are reachable only from the coordinator loop. The two whole-tick reads are batched; the per-cluster live signals are pulled on demand, so a tick's round-trips scale with the number of managed clusters that need a live signal, not with a constant. +The controller crate is pure and knows nothing about the Coordinator. This module is the half of the `ClusterControllerCtx` boundary that does: it runs the controller as a separate task and implements the ctx by marshaling each pull/apply to the Coordinator over the internal command channel, because the catalog and live compute/storage signals are reachable only from the coordinator loop. Whole-tick reads are batched. Refresh-window catalog inputs are pulled one cluster at a time and completed with one shared oracle read. The remaining per-cluster live signals are pulled on demand, so steady clusters do not pay for signals they do not use. -Everything here is gated by `ENABLE_CLUSTER_CONTROLLER` (default on). With the gate off the task does not tick, so the legacy scheduling and graceful paths remain the sole writers of the replica set. With the gate on the controller owns the *user* managed-cluster replica set; the legacy entry points no-op. System/builtin clusters are never controller-owned: the catalog's bootstrap migration owns their replicas. +The controller owns the replica set of every managed cluster, user and system alike. A builtin cluster's config-implied replicas are additionally materialized by `reconcile_builtin_cluster_replicas` at catalog open, which derives the same target from the same cluster config, so the two converge rather than compete. Key types and functions: -* `ClusterControllerRequest` — the enum of requests the controller task marshals to the Coordinator: `ManagedClusterIds`, `ClusterStates` (batched read; reply also carries `now`), `HydratedReplicas` (per-cluster live signal a strategy pulls on demand), `HasHydratableObjects` (per-cluster live signal reporting whether the cluster has any dataflow-backed objects bound to it), `RefreshWindowInputs` (per-cluster live signal supplying the oracle read timestamp, compaction estimate, and bound REFRESH MV frontiers for scheduled clusters; `None` for non-scheduled clusters), `Apply` (tick's decision batch), `TickInterval`. Each variant carries a oneshot for the reply. +* `ClusterControllerRequest` — the enum of requests the controller task marshals to the Coordinator: `ManagedClusterIds`, `ClusterStates` (batched read; reply also carries `now`), `HydratedReplicas` (per-cluster live signal a strategy pulls on demand), `HasHydratableObjects` (per-cluster live signal reporting whether the cluster has any dataflow-backed objects bound to it), `RefreshWindowClusterInputs` (catalog and storage refresh-window inputs for one scheduled cluster; `None` if the cluster no longer qualifies at pull time), `RefreshWindowReadTs` (one timestamp-oracle read completing a refresh-window input batch), `Apply` (tick's decision batch), `TickInterval`. Each variant carries a oneshot for the reply. * `CoordCtx` — the controller-task side of the `ClusterControllerCtx` boundary. Marshals every ctx call to the Coordinator via `internal_cmd_tx`. Latches `now` from the most recent `ClusterStates` reply so both phases of a tick see a consistent time. -* `Coordinator::spawn_cluster_controller_task` — spawns the controller task. The task loops: reads the tick interval via a `TickInterval` request, then waits for either the sleep to elapse or a `reconcile_now` notification (sent after catalog transactions that change durable cluster state), then calls `ClusterController::reconcile`. The gate and tick interval are re-read each iteration so runtime changes take effect without a restart. Called once at bootstrap. The `ClusterController` is constructed with a shared dyncfg handle so flag flips take effect without a restart. -* `Coordinator::handle_cluster_controller_request` — the Coordinator-side handler for `Message::ClusterControllerRequest`. The controller is inactive when the gate is off or while the deployment is in read-only mode; when inactive, `ManagedClusterIds` returns an empty list and `Apply` is rejected, so no catalog state is read or written. Dispatches each variant to the appropriate catalog/controller reads or applies the decision batch via `apply_cluster_decisions`. For `HydratedReplicas`, starts per-replica hydration checks on the coordinator loop (via `start_hydration_checks`) then waits for compute's replies off-loop in a spawned task. For `HasHydratableObjects`, delegates to `cluster_has_hydratable_objects`. For `RefreshWindowInputs`, reads catalog and storage inputs on the coordinator loop then completes the oracle timestamp read in a spawned task. +* `Coordinator::spawn_cluster_controller_task` — spawns the controller task. The task loops: reads the tick interval via a `TickInterval` request, then waits for either the sleep to elapse or a `reconcile_now` notification (sent after catalog transactions that change durable cluster state), then calls `ClusterController::reconcile`. The tick interval is re-read each iteration so runtime changes take effect without a restart. Called once at bootstrap. The `ClusterController` is constructed with a shared dyncfg handle so flag flips take effect without a restart. +* `Coordinator::handle_cluster_controller_request` — the Coordinator-side handler for `Message::ClusterControllerRequest`. The controller is inactive while the deployment is in read-only mode; when inactive, `ManagedClusterIds` returns an empty list and `Apply` is rejected, so no catalog state is read or written. Dispatches each variant to the appropriate catalog/controller reads or applies the decision batch via `apply_cluster_decisions`. For `HydratedReplicas`, starts per-replica hydration checks on the coordinator loop (via `start_hydration_checks`) then waits for compute's replies off-loop in a spawned task. For `HasHydratableObjects`, delegates to `cluster_has_hydratable_objects`. For `RefreshWindowClusterInputs`, reads catalog and storage inputs on the coordinator loop. For `RefreshWindowReadTs`, performs the oracle timestamp read in a spawned task (the oracle read is a network round-trip that must not block the coordinator loop). * `Coordinator::start_hydration_checks` — checks that replicas are online, storage-hydrated, and known to the compute controller for a set of replicas on a cluster. Returns only replicas whose processes are all online, that are already storage-hydrated, and that are known to the compute controller. Replica-pinned materialized views (those with `IN CLUSTER ... REPLICA`) are excluded from the hydration check for replicas they are not pinned to, so a graceful reconfiguration does not wait forever for a new replica to hydrate an MV bound to a replica being replaced. * `Coordinator::cluster_has_hydratable_objects` — returns whether any object bound to the cluster is hydratable (dataflow-backed). Backs the `HasHydratableObjects` ctx pull; a `false` answer prevents the burst strategy from arming when there is nothing to hydrate. * `Coordinator::apply_cluster_decisions` — transacts a decision batch in four steps: collect per-cluster compare-and-append guards, pre-allocate replica ids for creates, build mutation ops, then commit guards and mutations together. Returns `ApplyOutcome::Rejected` if any step finds the batch incoherent (missing cluster, allocation failure, or a concurrent `ALTER` that moved cluster state), `ApplyOutcome::ResourceExhausted` if the batch exceeds the resource budget, and `Applied` otherwise. Create decisions carry a strategy attribution translated to a `ReplicaCreateDropReason` via `audit_reason_for_create`: `CreateReason::Baseline` maps to `Manual`, `GracefulReconfiguration` maps to `GracefulReconfiguration`, `HydrationBurst` maps to `HydrationBurst`, and `OnRefresh` maps to `OnRefresh` carrying the window decision through to the audit detail. All drops are uniformly attributed `Retired`. `build_mutation_ops` checks whether the target replica of a `DropReplica` decision still exists in the catalog before building the drop op; if it has already been removed (e.g. by a concurrent user `DROP CLUSTER`), the function returns `None` to reject the batch, avoiding a panic from resource-limit validation running against a missing replica. diff --git a/doc/developer/generated/adapter/coord/cluster_scheduling.md b/doc/developer/generated/adapter/coord/cluster_scheduling.md deleted file mode 100644 index 2f005d8a59328..0000000000000 --- a/doc/developer/generated/adapter/coord/cluster_scheduling.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -source: src/adapter/src/coord/cluster_scheduling.rs -revision: 6eeaca032b ---- - -# adapter::coord::cluster_scheduling - -Implements the legacy automated cluster scheduling policy: `SCHEDULE = ON REFRESH`, which keeps track of per-cluster hydration time estimates and uses them to schedule cluster replica creation and teardown. -Both `check_scheduling_policies` and `handle_scheduling_decisions` are no-ops when `ENABLE_CLUSTER_CONTROLLER` (default on) is set: the controller's `OnRefreshStrategy` is then the sole authority over scheduled clusters, so the legacy policy must not also toggle their replication factor. -When the gate is off, `check_scheduling_policies` is called on a timer from the coordinator's main loop and sends `Message::SchedulingDecisions` to drive `handle_scheduling_decisions`. `handle_scheduling_decisions` sums decisions across all policies per cluster and turns replicas on or off as needed, but skips any cluster that has a pending (graceful-reconfiguration) replica, because that reconfiguration owns the replica set until it finalizes. When turning a cluster off it drops replicas by id rather than by altering the replication factor, so it can retire replicas left behind by the controller after a gate-off. When the factor is already 0 but owned replicas exist (controller handoff scenario), it adopts one replica and retires any surplus in the same transaction. diff --git a/doc/developer/generated/adapter/coord/command_handler.md b/doc/developer/generated/adapter/coord/command_handler.md index 29b00f61bb6a7..d45404b9eff8d 100644 --- a/doc/developer/generated/adapter/coord/command_handler.md +++ b/doc/developer/generated/adapter/coord/command_handler.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/command_handler.rs -revision: 443b75e6b4 +revision: ff68428395 --- # adapter::coord::command_handler diff --git a/doc/developer/generated/adapter/coord/message_handler.md b/doc/developer/generated/adapter/coord/message_handler.md index 20baf071f7374..aa0b0b3a25f0a 100644 --- a/doc/developer/generated/adapter/coord/message_handler.md +++ b/doc/developer/generated/adapter/coord/message_handler.md @@ -1,12 +1,12 @@ --- source: src/adapter/src/coord/message_handler.rs -revision: a60edac7f1 +revision: 38a95cefe2 --- # adapter::coord::message_handler Implements `Coordinator::handle_message`, the main dispatch for internal `Message` variants flowing through the coordinator's event loop. -Handles controller responses (compute peek results, subscribe batches, copy-to responses, watch-set notifications), timer ticks (group commit, timeline advancement, cluster scheduling, storage usage collection and pruning), staged-pipeline continuations (peek, create index, create view, create materialized view, subscribe, introspection subscribe, explain timestamp, secret, cluster), linearized read delivery, deferred statement execution, private-link VPC endpoint events, and cluster controller requests (`Message::ClusterControllerRequest`, dispatched to `handle_cluster_controller_request`). +Handles controller responses (compute peek results, subscribe batches, copy-to responses, watch-set notifications), timer ticks (group commit, timeline advancement, storage usage collection and pruning), staged-pipeline continuations (peek, create index, create view, create materialized view, subscribe, introspection subscribe, explain timestamp, secret, cluster), linearized read delivery, deferred statement execution, private-link VPC endpoint events, and cluster controller requests (`Message::ClusterControllerRequest`, dispatched to `handle_cluster_controller_request`). This is the heart of the coordinator's reactive loop; every asynchronous response from the storage/compute layers arrives here. `storage_usage_update` obtains a write timestamp from the oracle, allocates a single durable batch id via `Catalog::allocate_storage_usage_id`, builds `BuiltinTableUpdate` rows via `pack_storage_usage_update` for each shard, and submits them via `builtin_table_update().execute()` without going through `catalog_transact_inner`. In read-only mode, `storage_usage_fetch` logs an info message and reschedules via `Message::StorageUsageSchedule` without performing any shard scan or writes. diff --git a/doc/developer/generated/adapter/coord/read_then_write.md b/doc/developer/generated/adapter/coord/read_then_write.md index 991789041d564..df86fbfe9f669 100644 --- a/doc/developer/generated/adapter/coord/read_then_write.md +++ b/doc/developer/generated/adapter/coord/read_then_write.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/read_then_write.rs -revision: 3713eed996 +revision: 50f9dad7b2 --- # adapter::coord::read_then_write @@ -12,3 +12,5 @@ An object is considered invalid if it is a source, secret, or connection; a non- Additionally, any view or materialized view whose optimized expression contains a call to `mz_now()` is rejected, because the timestamp produced during the read phase would differ from the write timestamp and could yield inconsistent results. User tables (that are not source exports), user-defined views, user-defined materialized views, functions, and types are accepted. On failure, the function returns `AdapterError::Unsupported` (for `mz_now()` usage), `AdapterError::ReadThenWriteDependencyLimitExceeded` (when the dependency bound is exceeded), or `AdapterError::InvalidTableMutationSelection` with the offending object's name and type. + +`Coordinator::handle_create_internal_subscribe` creates a subscribe that introspection does not see (`internal: true` on `ActiveSubscribe`). It takes ownership of `read_holds` and drops them only after the dataflow is shipped, preventing the `since` from advancing past `as_of` in the interim. The dataflow is shipped via `try_ship_dataflow` before the sink is registered, so a failure (e.g. a dependency dropped since optimization) leaves nothing to unwind. Results are delivered through a `response_tx` oneshot; if the receiver is gone when the handler runs, the internal subscribe is immediately retired via `drop_internal_subscribe`. `Coordinator::drop_internal_subscribe` cancels the dataflow on the compute side by delegating to `drop_compute_sink`. diff --git a/doc/developer/generated/adapter/coord/sequencer.md b/doc/developer/generated/adapter/coord/sequencer.md index 857fa7b30b4c0..e1e7d958d1570 100644 --- a/doc/developer/generated/adapter/coord/sequencer.md +++ b/doc/developer/generated/adapter/coord/sequencer.md @@ -1,11 +1,10 @@ --- source: src/adapter/src/coord/sequencer.rs -revision: d0e1746c64 +revision: 38a95cefe2 --- # adapter::coord::sequencer Top-level sequencer module: implements `Coordinator::sequence_plan`, which matches on each `Plan` variant and dispatches to the appropriate `sequence_*` method. The module also contains shared utilities used across statement types: `statistics_oracle` (builds a cardinality-estimate oracle for query optimization), `eval_copy_to_uri` (validates the COPY TO URI, accepting `s3://` and `gs://` schemes), `check_log_reads` (validates introspection-source reads and enforces replica targeting on multi-replica clusters), `emit_optimizer_notices` (forwards optimizer notices to the session), `explain_pushdown_future_inner` (computes filter-pushdown statistics for EXPLAIN FILTER PUSHDOWN), `explain_plan_inner` (generates EXPLAIN PLAN output), and the `return_if_err!` macro (short-circuits on error, sending a response to the client). -`cancel_carried_reconfiguration` is re-exported from the `inner` sub-module for use by callers outside the sequencer tree. The `inner` sub-module holds most per-statement implementations; this file ties them together and handles generic concerns like RBAC checks and transaction validation. diff --git a/doc/developer/generated/adapter/coord/sequencer/_module.md b/doc/developer/generated/adapter/coord/sequencer/_module.md index b91193062b126..953fe5f576802 100644 --- a/doc/developer/generated/adapter/coord/sequencer/_module.md +++ b/doc/developer/generated/adapter/coord/sequencer/_module.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/sequencer.rs -revision: d0e1746c64 +revision: 38a95cefe2 --- # adapter::coord::sequencer diff --git a/doc/developer/generated/adapter/coord/sequencer/inner.md b/doc/developer/generated/adapter/coord/sequencer/inner.md index 90d1ca6bda194..691a3e0421ec7 100644 --- a/doc/developer/generated/adapter/coord/sequencer/inner.md +++ b/doc/developer/generated/adapter/coord/sequencer/inner.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/sequencer/inner.rs -revision: 6eeaca032b +revision: a4e7587de2 --- # adapter::coord::sequencer::inner diff --git a/doc/developer/generated/adapter/coord/sequencer/inner/_module.md b/doc/developer/generated/adapter/coord/sequencer/inner/_module.md index 79cce6c865969..669dc669c2cf8 100644 --- a/doc/developer/generated/adapter/coord/sequencer/inner/_module.md +++ b/doc/developer/generated/adapter/coord/sequencer/inner/_module.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/sequencer/inner.rs -revision: 6eeaca032b +revision: a4e7587de2 --- # adapter::coord::sequencer::inner diff --git a/doc/developer/generated/adapter/coord/sequencer/inner/cluster.md b/doc/developer/generated/adapter/coord/sequencer/inner/cluster.md index c9c7d19949378..e55bfa8c84b71 100644 --- a/doc/developer/generated/adapter/coord/sequencer/inner/cluster.md +++ b/doc/developer/generated/adapter/coord/sequencer/inner/cluster.md @@ -1,17 +1,18 @@ --- source: src/adapter/src/coord/sequencer/inner/cluster.rs -revision: ac496d5de8 +revision: a4e7587de2 --- # adapter::coord::sequencer::inner::cluster Implements sequencing for cluster-related DDL: `CREATE CLUSTER`, `CREATE CLUSTER REPLICA`, `DROP CLUSTER`, `DROP CLUSTER REPLICA`, `ALTER CLUSTER`, and `ALTER CLUSTER REPLICA RENAME`. Handles managed and unmanaged replica configurations, validates size parameters against the allowed replica-size map, and drives the scheduling-policy state after cluster alteration. Managed cluster variants carry `auto_scaling_strategy`, `reconfiguration`, and `burst` fields (all `None` by default) in addition to the existing scheduling and optimizer-override fields. The durable `ReplicaLocation::Managed` record stores a list of availability zones (`availability_zones: Vec`) rather than an optional single zone; user-pinned `AVAILABILITY ZONE` clauses produce a one-element list, and the cluster's AZ pool is stamped in at provisioning time. +`CREATE CLUSTER` and `CREATE CLUSTER REPLICA` accept an `IF NOT EXISTS` clause. When the named cluster or replica already exists, the statement succeeds and emits an `ObjectAlreadyExists` notice (SQLSTATE 42710) rather than an error; the existing object is left completely untouched. Replica IDs are pre-allocated out-of-band via `Catalog::allocate_replica_ids` before the catalog transaction, dispatching to user or system ID space based on the owning cluster's ID type. This mirrors how cluster and item IDs are allocated, so nothing allocates a replica ID in-apply. Cluster and replica creation and deletion are applied through the catalog implications pipeline rather than directly in this module. For `CREATE CLUSTER` (both managed and unmanaged variants), `scoped_overrides_create_op` is called with `ClusterEvalContext` and `ReplicaEvalContext` values built from plan data and pre-allocated replica IDs, and the resulting `Op::UpdateScopedSystemParameters` is folded into the same catalog transaction that creates the cluster. This makes the committed diff drive the replica-scoped controller push before `create_replica`, so the new cluster's optimizer and each new replica's render-frozen flags take effect before any dataflow renders, rather than falling back to environment-wide values until the next sync tick. `CREATE CLUSTER` validates the `HYDRATION SIZE` of an `AUTO SCALING STRATEGY` against the allowed replica sizes for the session role, just as `SIZE` itself is validated. This prevents a typo from failing silently at burst-arm time and prevents a size-restricted role from bursting at a size it may not `CREATE` with. -When `ENABLE_CLUSTER_CONTROLLER` is on and the cluster is user-owned, a shape-changing managed-to-managed `ALTER CLUSTER` (one that changes `SIZE`, `AVAILABILITY ZONES`, or `INTROSPECTION`) reshapes into a durable `reconfiguration` record via `reshape_alter_cluster_managed` instead of going through the legacy 3-stage machine. The record carries the full target config shape, a deadline, and an `on_timeout` action (`ROLLBACK` by default). Non-shape fields (`workload_class`, `schedule`, `auto_scaling_strategy`, etc.) are applied to the realized config immediately. The controller converges the replica set onto the target and cuts the realized config over at hydration. `ClusterStage::AwaitReconfiguration` (carrying `AlterClusterAwaitReconfiguration`) is a foreground wait-shim that polls the durable record until it reaches a terminal status, then reports success or failure depending on whether the realized config reached the target; it is used when `ENABLE_BACKGROUND_ALTER_CLUSTER` is off. The legacy foreground `AlterClusterWaitForHydrated` readiness check additionally requires every pending replica to be online (per `mz_cluster_replica_statuses`) before declaring the reconfiguration ready to cut over, so a cluster hosting only single-replica sources (which are never placed on a pending replica until cut-over) does not declare readiness vacuously the moment the pending replicas are created. +A shape-changing managed-to-managed `ALTER CLUSTER` (one that changes `SIZE`, `AVAILABILITY ZONES`, or `INTROSPECTION`) reshapes into a durable `reconfiguration` record via `reshape_alter_cluster_managed` instead of going through the legacy 3-stage machine. The record carries the full target config shape, a deadline, and an `on_timeout` action (`ROLLBACK` by default). Non-shape fields (`workload_class`, `schedule`, `auto_scaling_strategy`, etc.) are applied to the realized config immediately. The controller converges the replica set onto the target and cuts the realized config over at hydration. `ClusterStage::AwaitReconfiguration` (carrying `AlterClusterAwaitReconfiguration`) is a foreground wait-shim that polls the durable record until it reaches a terminal status, then reports success or failure depending on whether the realized config reached the target; it is used when `ENABLE_BACKGROUND_ALTER_CLUSTER` is off. The legacy foreground `AlterClusterWaitForHydrated` readiness check additionally requires every pending replica to be online (per `mz_cluster_replica_statuses`) before declaring the reconfiguration ready to cut over, so a cluster hosting only single-replica sources (which are never placed on a pending replica until cut-over) does not declare readiness vacuously the moment the pending replicas are created. When a reconfiguration is in flight, a subsequent shape-changing `ALTER` folds onto the in-flight target via `fold_reconfiguration_target`: dimensions the `ALTER` set explicitly take the new value, while dimensions left `Unchanged` keep the in-flight target's value (not the realized config's value, which would silently revert the in-flight transition). The `deadline` and `on_timeout` are handled separately: when the folding `ALTER` carries an explicit `WAIT` clause, the clause's deadline and action overwrite the in-flight record's (allowing a later `ALTER` to steer the deadline without discarding hydration progress already made on the target); when the folding `ALTER` carries no `WAIT` clause, the in-flight record's `deadline` and `on_timeout` are preserved so that an unrelated shape-changing `ALTER` does not silently reset the user-set deadline. When there is no reconfiguration in flight and no `WAIT` clause, the system-default timeout and implicit `ROLLBACK` action apply. An `ALTER` back to the realized shape cancels the record. Changing the replication factor while a reconfiguration is in flight is refused with `AdapterError::AlterClusterReplicationFactorWhileReconfiguring`. `ALTER CLUSTER` validates a newly set or changed `AUTO SCALING STRATEGY`'s `HYDRATION SIZE` like `SIZE` itself: it must name a real replica size the session role may use. Only a changed strategy is checked, so an existing policy does not block unrelated `ALTER`s if the size allow-list later shrinks. Additionally, if a reconfiguration is in flight, `HYDRATION SIZE` must differ from the in-flight reconfiguration's target `SIZE`: equality would end the reshape with a no-op burst shape and a stored statement that fails its own re-plan. Unmanaged clusters reject `AUTO SCALING STRATEGY` changes with an error, mirroring how other managed-only options are handled. Converting a cluster to unmanaged while a hydration burst is in flight is refused with `AdapterError::AlterClusterUnmanagedWhileBursting`. The unmanaged variant carries no burst field, so converting would drop the record without a `Finished` audit event and strand the burst replica as an ordinary unmanaged replica nothing ever tears down. diff --git a/doc/developer/generated/adapter/coord/sequencer/inner/copy_from.md b/doc/developer/generated/adapter/coord/sequencer/inner/copy_from.md index 69d5e6f94dab4..56941ac13f6ae 100644 --- a/doc/developer/generated/adapter/coord/sequencer/inner/copy_from.md +++ b/doc/developer/generated/adapter/coord/sequencer/inner/copy_from.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/sequencer/inner/copy_from.rs -revision: e156fe6b04 +revision: b9097f8a3d --- # adapter::coord::sequencer::inner::copy_from diff --git a/doc/developer/generated/adapter/coord/sql.md b/doc/developer/generated/adapter/coord/sql.md index 17a8d5f78d43b..d4623d883f0e2 100644 --- a/doc/developer/generated/adapter/coord/sql.md +++ b/doc/developer/generated/adapter/coord/sql.md @@ -1,10 +1,10 @@ --- source: src/adapter/src/coord/sql.rs -revision: a60edac7f1 +revision: 50f9dad7b2 --- # adapter::coord::sql Implements coordinator utility methods for the SQL layer: `plan_statement` resolves names against the catalog and invokes the SQL planner; it takes `&ResolvedIds` and returns `(Plan, ResolvedIds)` where the second element contains IDs from SQL-implemented function bodies, kept separate from the statement's dependencies; `declare` and `describe` support cursor declaration and the pgwire Describe flow; `verify_prepared_statement` and `verify_portal` re-check prepared statement and portal validity after catalog changes. Also contains `clear_transaction`, `clear_connection`, `add_active_compute_sink`, and `remove_active_compute_sink` for per-connection and per-sink bookkeeping. -`add_active_compute_sink` and `remove_active_compute_sink` skip the `mz_subscriptions` builtin table update for `ActiveSubscribe` entries whose `internal` flag is set. +`add_active_compute_sink` and `remove_active_compute_sink` treat `ActiveSubscribe` entries with `internal: true` differently from regular subscribes: they skip the `mz_subscriptions` builtin table update and increment/decrement the `active_internal_subscribes` gauge rather than `active_subscribes`. diff --git a/doc/developer/generated/adapter/coord/statement_logging.md b/doc/developer/generated/adapter/coord/statement_logging.md index 68c2ecd294ba5..6a20093ec13ec 100644 --- a/doc/developer/generated/adapter/coord/statement_logging.md +++ b/doc/developer/generated/adapter/coord/statement_logging.md @@ -1,11 +1,11 @@ --- source: src/adapter/src/coord/statement_logging.rs -revision: a60edac7f1 +revision: ff68428395 --- # adapter::coord::statement_logging Implements the coordinator-side statement logging: `WatchSet` registration and resolution (triggering log writes once storage and compute frontiers advance past the execution timestamp), and the actual writes to `mz_statement_execution_history` and `mz_prepared_statement_history`. `handle_statement_logging_watch_set` is called from the message handler when watched frontiers advance, completing deferred statement log entries. -`end_statement_execution` is idempotent: the first end wins, and later duplicate ends for the same statement are ignored with a `tracing::warn!`. Duplicate ends are legitimate under async cancellation: ownership of the end-of-execution log is handed from the frontend to the coordinator at dispatch time, and a client disconnect can drop the frontend future after the coordinator registers a peek but before the frontend defuses its logging guard, leaving both sides holding end ownership. +`end_statement_execution` is idempotent: the first end wins, and later duplicate ends for the same statement are ignored with a `tracing::warn!`. Duplicate ends are legitimate under async cancellation: ownership of the end-of-execution log is handed from the frontend to the coordinator at dispatch time, and a client disconnect can drop the frontend future after the coordinator registers a peek but before the frontend defuses its logging guard, leaving both sides holding end ownership. `end_statement_execution` takes an explicit `ended_at: EpochMillis` parameter rather than sampling the clock internally: a statement sequenced off the coordinator loop finishes in its session task and reports the end as a message, so the coordinator must use the time the statement actually finished rather than when it processes the end message. `PreparedStatementLoggingInfo::AlreadyLogged` carries both the `uuid` and the statement `kind`, so that `end_statement_execution` can detect secret statements (`kind.is_secret()`) and replace the error message with a fixed redaction string rather than persisting any error text that may embed secret material. The `kind` is captured from the session at `begin_statement_execution` time via `session.qcell_ro(logging).kind()` and passed through `create_began_execution_record`. diff --git a/doc/developer/generated/adapter/coord/timestamp_selection.md b/doc/developer/generated/adapter/coord/timestamp_selection.md index fb5274459c6de..092aa8adf96eb 100644 --- a/doc/developer/generated/adapter/coord/timestamp_selection.md +++ b/doc/developer/generated/adapter/coord/timestamp_selection.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/timestamp_selection.rs -revision: 1c17d34993 +revision: 253293ef87 --- # adapter::coord::timestamp_selection diff --git a/doc/developer/generated/adapter/frontend_peek.md b/doc/developer/generated/adapter/frontend_peek.md index ede4c1aad36ee..aba8be82b7477 100644 --- a/doc/developer/generated/adapter/frontend_peek.md +++ b/doc/developer/generated/adapter/frontend_peek.md @@ -1,12 +1,12 @@ --- source: src/adapter/src/frontend_peek.rs -revision: 9b2fd0bbc6 +revision: ff68428395 --- # adapter::frontend_peek Implements the "frontend peek" sequencing path, where SELECT query optimization and fast-path execution are performed in the pgwire connection task rather than the coordinator's main loop. -`PeekClient::try_frontend_peek` is the main entry point; it verifies the portal, delegates statement logging setup to `PeekClient::begin_statement_logging` (which returns a `StatementLoggingGuard` owning end-of-execution logging), threads the guard through `try_frontend_peek_inner` — dispatch sites that hand a statement to the coordinator for asynchronous completion defuse it — and retires any still-armed guard with the execution's outcome. +`PeekClient::try_frontend_peek` is the main entry point; it receives an `ExecutionLogging` slot owning the end-of-execution obligation, threads the slot through `try_frontend_peek_inner` — dispatch sites that hand a statement to the coordinator for asynchronous completion defuse the slot — and retires any still-armed slot with the execution's outcome. `try_frontend_peek_inner` sequences the full pipeline — name resolution, planning, cluster selection, RBAC checks, timeline context determination, timestamp determination with read-hold acquisition, optimization (in a `spawn_blocking` task), and finally execution — handling `SELECT`, `EXPLAIN` (plan and pushdown), `COPY TO S3`, and `SUBSCRIBE` statement types. Fast-path peeks (constant, arrangement, or persist) are executed directly via `implement_fast_path_peek_plan`; slow-path dataflow plans are dispatched to the coordinator via `Command::ExecuteSlowPathPeek`. SUBSCRIBE statements are dispatched via `Command::ExecuteSubscribe`; COPY TO S3 performs an S3 preflight check via `Command::CopyToPreflight` before dispatching `Command::ExecuteCopyTo`. diff --git a/doc/developer/generated/adapter/metrics.md b/doc/developer/generated/adapter/metrics.md index 34e23c7264e8f..8fe81d12e48f5 100644 --- a/doc/developer/generated/adapter/metrics.md +++ b/doc/developer/generated/adapter/metrics.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/metrics.rs -revision: b644395dd7 +revision: 38a95cefe2 --- # adapter::metrics @@ -13,3 +13,5 @@ The `mz_time_to_first_row_seconds` histogram carries an `application_name` label `Metrics` includes `catalog_transact_seconds` (a `HistogramVec` labeled by `method`) for timing catalog transact methods, `catalog_transact_phase_seconds` (a `HistogramVec` labeled by `phase`) for fine-grained per-phase timing within a catalog transaction (phases overlap and do not sum to `catalog_transact_seconds`), `apply_catalog_implications_seconds` for timing catalog implication application, and `group_commit_catalog_upper_seconds` for timing catalog shard upper advances during group commits and table register/forget operations. Several public metrics carry `MetricTag` annotations for categorization: `mz_query_total`, `mz_active_sessions`, `mz_active_subscribes`, and `mz_adapter_commands` carry `MetricTag::Environment`. Helper functions `session_type_label_value`, `statement_type_label_value`, and `subscribe_output_label_value` produce the label strings used for partitioning these metrics. +`Metrics` includes a `subscribe_outputs` `IntCounterVec` (labeled via `subscribe_output_label_value`) counting subscribe output rows; `SessionMetrics` vends per-call counters from it via `Metrics::subscribe_outputs`. +`Metrics` includes `active_internal_subscribes: IntGaugeVec` (labeled by `session_type`) tracking the number of active internal subscribes, which serve frontend-sequenced read-then-write operations. Internal subscribes are not reflected in `active_subscribes` or in the `mz_subscriptions` builtin table. diff --git a/doc/developer/generated/adapter/notice.md b/doc/developer/generated/adapter/notice.md index 28d8a61bbb450..1d871c2fecab6 100644 --- a/doc/developer/generated/adapter/notice.md +++ b/doc/developer/generated/adapter/notice.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/notice.rs -revision: 47b3fad674 +revision: a4e7587de2 --- # adapter::notice diff --git a/doc/developer/generated/adapter/optimize/_module.md b/doc/developer/generated/adapter/optimize/_module.md index 182492f3e3d1c..7c6ba0ef4636b 100644 --- a/doc/developer/generated/adapter/optimize/_module.md +++ b/doc/developer/generated/adapter/optimize/_module.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/optimize.rs -revision: 94054eb165 +revision: b9097f8a3d --- # adapter::optimize @@ -9,4 +9,4 @@ Defines the high-level optimizer interface used throughout the coordinator: the Each child module (`peek`, `index`, `materialized_view`, `subscribe`, `copy_to`, `view`) implements the full optimization pipeline for one statement type as a sequence of `Optimize` impls that transform opaque stage-result structs; `dataflows` provides the shared `DataflowBuilder` utility. A private `metric_sink` module handles the metric sink optimization path. `OptimizerError` includes a `RestrictedFunction(UnmaterializableFunc)` variant for unmaterializable functions blocked by the `restrict_to_user_objects` session variable; it carries a hint directing users to contact their administrator. `OptimizerConfig` can be overridden from `ExplainContext` (for `EXPLAIN ... WITH(...)`) or from `OptimizerFeatureOverrides` (for `CLUSTER ... FEATURES(...)`), keeping the optimizer API stable while allowing ad-hoc experimentation. -`PeekOptimizer` is an enum that carries either a `peek::Optimizer` (for `SELECT` and `EXPLAIN`) or a `copy_to::Optimizer` (for `COPY TO`) through the shared peek sequencing state machine. Both statement types go through identical surrounding stages (timestamp selection, read holds, off-thread optimization), and `PeekOptimizer` lets that shared machinery carry either optimizer without caring which one it is. The companion `PeekGlobalLirPlan` enum tags the optimization result with the same two variants so that downstream stages can recover the concrete plan type. The internal `LirDataflowDescription` type alias resolves to `DataflowDescription`. The shared optimization pipeline steps (HIR-to-local-MIR, timestamp resolution, global-MIR-to-LIR) are factored into the `pub(crate) optimize_oneshot` helper, which is also used by the frontend peek path in `frontend_peek.rs`. +`PeekOptimizer` is an enum that carries either a `peek::Optimizer` (for `SELECT` and `EXPLAIN`) or a `copy_to::Optimizer` (for `COPY TO`) through the shared peek sequencing state machine. Both statement types go through identical surrounding stages (timestamp selection, read holds, off-thread optimization), and `PeekOptimizer` lets that shared machinery carry either optimizer without caring which one it is. The companion `PeekGlobalLirPlan` enum tags the optimization result with the same two variants so that downstream stages can recover the concrete plan type. The `LirDataflowDescription` type alias (`pub type`) resolves to `DataflowDescription`. The shared optimization pipeline steps (HIR-to-local-MIR, timestamp resolution, global-MIR-to-LIR) are factored into the `pub(crate) optimize_oneshot` helper, which is also used by the frontend peek path in `frontend_peek.rs`. diff --git a/doc/developer/generated/adapter/optimize/metric_sink.md b/doc/developer/generated/adapter/optimize/metric_sink.md new file mode 100644 index 0000000000000..03de73d5324ea --- /dev/null +++ b/doc/developer/generated/adapter/optimize/metric_sink.md @@ -0,0 +1,26 @@ +--- +source: src/adapter/src/optimize/metric_sink.rs +revision: 94054eb165 +--- + +# adapter::optimize::metric_sink + +Row-wise MIR shaping for `MetricSink` sources. + +The central export is `shape_metric_sink_source`, which takes a source relation's `GlobalId` and `RelationDesc` and produces a `(MirRelationExpr, RelationDesc)` pair. The returned expression extends the source relation with four new scalar columns via a `Map`, then projects down to exactly seven canonical columns: + +| Column | Type | Semantics | +|---|---|---| +| `metric_name` | `String` (nullable) | Prometheus metric name | +| `metric_type` | `String` (non-null) | Raw type string from the source | +| `labels` | `Map` (non-null after coalesce) | Label key-value pairs | +| `value` | `Float64` (nullable) | Metric value | +| `help` | `String` (non-null after coalesce) | Metric help text | +| `metric_kind` | `Int32` (nullable) | `0` for gauge, `1` for counter, `NULL` for unsupported types | +| `name_valid` | `Bool` (nullable) | Whether `metric_name` matches the Prometheus name grammar | + +`labels` and `help` are coalesced to their identity elements (`{}` and `""`) so the compute-side operator never sees nulls for those columns. `metric_name` and `value` remain nullable because they have no meaningful identity element. The two classification columns (`metric_kind`, `name_valid`) are precomputed in MIR so the operator's hot path does not parse `metric_type` strings or run regexp checks per row. + +No rows are filtered: the operator needs every row, including invalid ones, to count `skipped` and `null_values` metrics. Only per-row, stateless shaping moves to MIR. Cross-row logic (dedup, collision detection, family-conflict counting) stays in `mz_compute::sink::metric_sink` because it requires frontier-gated fold state that a `Map` expression cannot express. + +`METRIC_NAME_PATTERN` is the Prometheus metric name grammar regexp (`^[a-zA-Z_:][a-zA-Z0-9_:]*$`), stored as a constant for use in the MIR `IsRegexpMatchCaseSensitive` scalar. Both `METRIC_NAME_PATTERN` and `shape_metric_sink_source` carry `#[allow(dead_code)]` because the SQL planner caller does not yet exist; they are placed here alongside the compute operator that reads the column contract they define. diff --git a/doc/developer/generated/adapter/optimize/subscribe.md b/doc/developer/generated/adapter/optimize/subscribe.md index c89047043611a..da15fbc688368 100644 --- a/doc/developer/generated/adapter/optimize/subscribe.md +++ b/doc/developer/generated/adapter/optimize/subscribe.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/optimize/subscribe.rs -revision: 43a933b189 +revision: b9097f8a3d --- # adapter::optimize::subscribe diff --git a/doc/developer/generated/adapter/optimize/view.md b/doc/developer/generated/adapter/optimize/view.md index f7d2706c2575a..c4ce84c0d6937 100644 --- a/doc/developer/generated/adapter/optimize/view.md +++ b/doc/developer/generated/adapter/optimize/view.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/optimize/view.rs -revision: 1189146e6e +revision: b9097f8a3d --- # adapter::optimize::view diff --git a/doc/developer/generated/adapter/peek_client.md b/doc/developer/generated/adapter/peek_client.md index a7ebe8fbad991..fc48bcb3a2fb4 100644 --- a/doc/developer/generated/adapter/peek_client.md +++ b/doc/developer/generated/adapter/peek_client.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/peek_client.rs -revision: dbd2c3fc06 +revision: ff68428395 --- # adapter::peek_client @@ -10,5 +10,5 @@ This allows the frontend peek sequencing path (and other callers) to issue peeks `PeekClient::catalog_snapshot` serves the catalog from the session-side `catalog_cache` when the catalog's transient revision is unchanged (verified via `Catalog::transient_revision_is_current`), avoiding a coordinator round-trip on cache hits. On a miss it sends `Command::CatalogSnapshot`, re-populates `catalog_cache` with a `Weak` downgrade of the returned `Arc`, and records the round-trip latency in `mz_catalog_snapshot_seconds` (labeled by `context`). Both hits and misses increment `mz_catalog_snapshot_cache` (labeled `context` and `result`). The `catalog_cache` holds a `Weak` so that an idle session does not prevent an old catalog version from being freed. `PeekClient::new` seeds `catalog_cache` from the `Arc` returned by `Command::Startup`, so the session's first statements do not require a round-trip. `CollectionLookupError` enumerates errors that can occur when consulting storage or compute frontiers for timestamp selection. -`begin_statement_logging` sets up statement logging for a frontend-sequenced operation: it begins a new execution log entry for fresh statements, or inherits and retires the outer context for nested executions (e.g., EXECUTE/FETCH), and returns a `StatementLoggingGuard`. -`StatementLoggingGuard` is an RAII guard that ensures every statement for which `BeganExecution` was logged also receives a corresponding `EndedExecution`: if dropped without being defused or retired, it emits `StatementEndedExecutionReason::Aborted`. Callers call `retire` to log a terminal outcome explicitly, or `defuse` (taking `&mut self`) to hand off logging responsibility to another component (e.g. the coordinator for streaming peek responses) at the exact handoff point; afterwards the guard is inert. For non-sampled statements the guard carries no id and is a no-op. +`ExecutionLogging` is the session task's slot for the end-of-execution obligation of the statement it is running. One slot is held for the whole `SessionClient::execute` call with exactly one retirement site. An empty slot means no log entry exists for this execution, so a fallback to the coordinator lets it begin its own. An occupied slot means the session task owes an end event: `retire` pays it, `release` transfers it to the coordinator, and `defuse` discards it. For non-sampled statements the slot carries no id and all operations are no-ops. `ExecutionLogging::adopt` constructs the slot from an optional outer `ExecuteContextGuard` (for FETCH/EXECUTE callers). `ExecutionLogging::take_over` records that the session task, not the coordinator, is executing a given statement: it bumps the counters `Coordinator::handle_execute` would have bumped, makes sure a log entry exists (inheriting the adopted one or creating one via the private `begin_statement_logging`), and marks the slot so the coordinator must not run the statement. `TakeOver` distinguishes `StatementToRun` (the statement executing here; coordinator must not run it) from `UnrolledExecute` (a SQL `EXECUTE` being unrolled; coordinator must not count it). +The private `StatementLoggingGuard` is the inner RAII guard that enforces the end-of-execution invariant: if dropped without being defused or retired, it emits `StatementEndedExecutionReason::Aborted`. `ExecutionLogging` wraps one optional `StatementLoggingGuard`. diff --git a/doc/developer/generated/adapter/session.md b/doc/developer/generated/adapter/session.md index dd613841c15f0..6b5a102515ba8 100644 --- a/doc/developer/generated/adapter/session.md +++ b/doc/developer/generated/adapter/session.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/session.rs -revision: a60edac7f1 +revision: b5624d20bb --- # adapter::session @@ -10,4 +10,5 @@ A `Session` tracks the connection ID and UUID, the current transaction state (`T `SessionConfig` includes an `authenticator_kind: AuthenticatorKind` field that records which authenticator was used for the connection, and a `groups: Option>` field that carries JWT group claims from OIDC authentication for group-to-role sync; non-OIDC paths set this to `None`. `Transaction` holds the current transaction's operations (`TransactionOps`), a plan context, and optional write-lock guards; `TransactionStatus` is the state machine covering `Default`, `Started`, `InTransaction`, `InTransactionImplicit`, and `Failed` states. `TransactionStatus::allows_writes` permits adding write operations only when the current ops are `TransactionOps::None` (and the transaction is not explicitly `ReadOnly`) or `TransactionOps::Peeks` whose timestamp context is constant (no timestamp assigned) and whose access mode is not `ReadOnly`. A read-only transaction is rejected even when all prior peeks were constant, preventing a constant-peek transaction from silently transitioning into a write transaction when the transaction was opened with `BEGIN READ ONLY` or an equivalent access-mode constraint. +`TransactionStatus::may_span_pipeline` returns `true` for a `Started` transaction whose ops are `TransactionOps::Writes`; when true, the extended-protocol pipeline in pgwire keeps the implicit write transaction open until the client's Sync message so the pipeline commits or rolls back as a unit, matching PostgreSQL's behavior. All other states and op kinds return `false`. The module also defines `RowBatchStream` (the channel type for streaming subscribe results to pgwire) and `PreparedStatement` (a parsed, described statement with optional logging info). diff --git a/doc/developer/generated/avro/decode.md b/doc/developer/generated/avro/decode.md index a3411d6876505..07f0b586c5d9e 100644 --- a/doc/developer/generated/avro/decode.md +++ b/doc/developer/generated/avro/decode.md @@ -1,6 +1,6 @@ --- source: src/avro/src/decode.rs -revision: e6b08eaf07 +revision: 18d73eb54d --- Implements schema-driven binary decoding via a visitor-style trait hierarchy. @@ -8,3 +8,4 @@ Implements schema-driven binary decoding via a visitor-style trait hierarchy. `GeneralDeserializer` drives the decode loop by matching the resolved `SchemaNode` to the appropriate decoding path; `give_value` feeds a pre-built `Value` back through an `AvroDecode` implementation, used when supplying default field values. `TrivialDecoder` and `ValueDecoder` (re-exported from `public_decoders`) provide ready-made implementations: the former discards data, the latter produces `Value`. The `Skip` trait extends `Read` with an efficient seek-forward operation, and `AvroRead` is a blanket alias combining `Read + Skip`. +`bound_block_object_count` validates the object count declared in an OCF block header against the block's decompressed payload: when the schema has a proven positive wire-byte floor (`min_encoded_len`), the count times that floor must not exceed the payload length; when the schema is zero-width (such as `null` or a record of only `null` fields), the count weighted by `min_value_nodes` must not exceed `MAX_VALUE_NODES`. This check is a standalone guard on the block header and is not charged against the per-datum `DECODE_NODES` budget. diff --git a/doc/developer/generated/avro/reader.md b/doc/developer/generated/avro/reader.md index 4877901e59f51..1f28e3e752452 100644 --- a/doc/developer/generated/avro/reader.md +++ b/doc/developer/generated/avro/reader.md @@ -1,6 +1,6 @@ --- source: src/avro/src/reader.rs -revision: 5c40ac7338 +revision: 253293ef87 --- Provides the high-level `Reader` type for reading Avro Object Container Files (OCF). @@ -8,6 +8,6 @@ Provides the high-level `Reader` type for reading Avro Object Container Files (O `Block` and `BlockIter` give lower-level access to raw compressed blocks for callers that need finer control over I/O or custom decoding. `from_avro_datum` decodes a single bare datum (no OCF framing) from a byte slice using a provided schema. `SchemaResolver` is an internal struct used by `schema::resolve_schemas` to walk writer/reader schema pairs and produce resolved `SchemaPiece` trees. -Block object counts and byte lengths read from the wire are validated through `util::safe_len` before use, rejecting negative values (which wrap to a huge `usize`) and counts that exceed a safe bound; this prevents a crafted Avro file with a large block count and a zero-byte schema from causing the reader to spin decoding billions of empty values. +Block object counts and byte lengths read from the wire are validated through `util::safe_len` before use, rejecting negative values (which wrap to a huge `usize`) and counts that exceed a safe bound. After decompression, `bound_block_object_count` (from `decode`) checks the object count against the decompressed payload length, preventing a crafted block with a large count and a zero-byte schema (such as `null`) from causing the reader to spin decoding empty values. The internal buffer is also truncated to exactly the current block's payload after each `fill_buf` call, so a block shorter than its predecessor cannot read stale bytes from the previous block's tail as though they were values in the current block. When union resolution fails for a nested named type, the resolver rolls back all named nodes at indices `>= resolved_idx` (truncating `named` and removing entries from `reader_to_resolved_names` and `indices`), rather than only popping the last entry; this avoids leaving `None` placeholder entries that would panic on a later `Option::unwrap`. Union variant matching during schema resolution uses `match_promote_writer` and `match_ref_promote_writer`/`match_ref_promote_reader` (defined on `UnionSchema` in `schema.rs`) so that numeric promotion across union variants — for example, resolving a writer's `["null", "int"]` against a reader's `["null", "double"]` — is handled without error. diff --git a/doc/developer/generated/catalog-debug/_crate.md b/doc/developer/generated/catalog-debug/_crate.md index a593edde617ab..39769c921ba0e 100644 --- a/doc/developer/generated/catalog-debug/_crate.md +++ b/doc/developer/generated/catalog-debug/_crate.md @@ -1,6 +1,6 @@ --- source: src/catalog-debug/src/main.rs -revision: 584bb9030c +revision: 253293ef87 --- # mz-catalog-debug diff --git a/doc/developer/generated/catalog-protos/objects_v90.md b/doc/developer/generated/catalog-protos/objects_v90.md new file mode 100644 index 0000000000000..089d452d0fd16 --- /dev/null +++ b/doc/developer/generated/catalog-protos/objects_v90.md @@ -0,0 +1,19 @@ +--- +source: src/catalog-protos/src/objects_v90.rs +revision: fca741734d +--- + +# mz-catalog-protos::objects_v90 + +Frozen snapshot of catalog object type definitions at schema version 90, identical to `objects.rs` at the time v90 was declared current. +This snapshot serves as the migration target for v89->v90 upgrades and as the migration source for v90->v91. + +Relative to v89, this snapshot adds an `arrangement_compression: bool` field in three places: + +- `ManagedCluster` — controls whether arrangement compression is enabled for the cluster's managed replicas. +- `ReconfigurationTarget` — the in-flight reconfiguration target for a managed cluster inherits the same field, so the target state is complete. +- `ReplicaConfig` — each cluster replica's configuration independently carries the flag. + +All three fields are backfilled as `false` by the v89->v90 migration (`catalog::durable::upgrade::v89_to_v90`). Unmanaged clusters have no `ManagedCluster` variant and are not rewritten. All other catalog types are identical to v89. + +`derive(Arbitrary)` on all types is gated behind `#[cfg_attr(any(test, feature = "proptest"), derive(Arbitrary))]`; the `proptest_derive` import is similarly conditional. diff --git a/doc/developer/generated/catalog/builtin/_module.md b/doc/developer/generated/catalog/builtin/_module.md index ec85da77921b0..a8ab9b3e2bbd8 100644 --- a/doc/developer/generated/catalog/builtin/_module.md +++ b/doc/developer/generated/catalog/builtin/_module.md @@ -1,13 +1,13 @@ --- source: src/catalog/src/builtin.rs -revision: 34813f0a3f +revision: fd1dd6e62b --- # catalog::builtin -Defines every built-in catalog object hardcoded into Materialize: system tables (`BuiltinTable`), views (`BuiltinView`), materialized views (`BuiltinMaterializedView`), indexes (`BuiltinIndex`), types (`BuiltinType`), sources (`BuiltinSource`), logs (`BuiltinLog`), connections (`BuiltinConnection`), cluster configs, roles, and schemas. +Defines every built-in catalog object hardcoded into Materialize: system tables (`BuiltinTable`), views (`BuiltinView`), materialized views (`BuiltinMaterializedView`), indexes (`BuiltinIndex`), types (`BuiltinType`), sources (`BuiltinSource`), logs (`BuiltinLog`), connections (`BuiltinConnection`), cluster definitions (`BuiltinCluster`), roles, and schemas. `BuiltinClusterReplica` and `BUILTIN_CLUSTER_REPLICAS` are absent; the durable managed cluster config (specifically `replication_factor`) is the single source of truth for each builtin cluster's replica set, and `reconcile_builtin_cluster_replicas` in the adapter converges it at catalog open. Key types include `Builtin` (a generic enum with variants Log, Table, View, MaterializedView, Type, Func, Source, Index, and Connection, each wrapping the corresponding builtin struct), `BUILTINS` (the exhaustive static list used at catalog open time), and constants like `BUILTIN_PREFIXES` and `RUNTIME_ALTERABLE_FINGERPRINT_SENTINEL`. -`MZ_DATABASES`, `MZ_CLUSTERS`, `MZ_CLUSTER_REPLICAS`, `MZ_CLUSTER_SCHEDULES`, `MZ_CLUSTER_RECONFIGURATIONS`, `MZ_CLUSTER_AUTO_SCALING_STRATEGIES`, `MZ_KAFKA_SOURCES`, and `MZ_POSTGRES_SOURCES` are all `BuiltinMaterializedView` objects backed by queries over the catalog. `MZ_CLUSTER_RECONFIGURATIONS` and `MZ_CLUSTER_AUTO_SCALING_STRATEGIES` each have a corresponding `BuiltinIndex` (`MZ_CLUSTER_RECONFIGURATIONS_IND` and `MZ_CLUSTER_AUTO_SCALING_STRATEGIES_IND` respectively) on `cluster_id`, registered in `BUILTINS_STATIC` alongside the other index entries. `MZ_CLUSTER_REPLICA_SIZE_INTERNAL` is a `BuiltinTable` that retains size-map entries including disabled sizes, allowing the `MZ_CLUSTER_REPLICAS` materialized view to resolve the `disk` column via LEFT JOIN even for replicas whose sizes have been disabled. +`MZ_DATABASES`, `MZ_CLUSTERS`, `MZ_CLUSTER_REPLICAS`, `MZ_CLUSTER_SCHEDULES`, `MZ_CLUSTER_RECONFIGURATIONS`, `MZ_CLUSTER_AUTO_SCALING_STRATEGIES`, `MZ_KAFKA_SOURCES`, `MZ_POSTGRES_SOURCES`, `MZ_POSTGRES_SOURCE_TABLES`, `MZ_MYSQL_SOURCE_TABLES`, `MZ_SQL_SERVER_SOURCE_TABLES`, `MZ_KAFKA_SOURCE_TABLES`, `MZ_KAFKA_CONNECTIONS`, `MZ_SSH_TUNNEL_CONNECTIONS`, `MZ_AWS_CONNECTIONS`, and `MZ_AWS_PRIVATELINK_CONNECTIONS` are all `BuiltinMaterializedView` objects backed by queries over the catalog. `MZ_CLUSTER_RECONFIGURATIONS` and `MZ_CLUSTER_AUTO_SCALING_STRATEGIES` each have a corresponding `BuiltinIndex` (`MZ_CLUSTER_RECONFIGURATIONS_IND` and `MZ_CLUSTER_AUTO_SCALING_STRATEGIES_IND` respectively) on `cluster_id`, registered in `BUILTINS_STATIC` alongside the other index entries. `MZ_CLUSTER_REPLICA_SIZE_INTERNAL` is a `BuiltinTable` that retains size-map entries including disabled sizes, allowing the `MZ_CLUSTER_REPLICAS` materialized view to resolve the `disk` column via LEFT JOIN even for replicas whose sizes have been disabled. `MZ_CONNECTIONS`, `MZ_SECRETS`, `MZ_SOURCES`, `MZ_DEFAULT_PRIVILEGES`, and `MZ_SYSTEM_PRIVILEGES` are `BuiltinMaterializedView` objects backed by queries over `mz_internal.mz_catalog_raw`. `MZ_SOURCES` is generated dynamically via `make_mz_sources()` in the `builtin` submodule rather than declared as a static, since its SQL must enumerate all builtin and log sources. `MZ_INDEXES` is also a `BuiltinMaterializedView` generated dynamically via `make_mz_indexes()` in the `mz_catalog` submodule; it inlines the full set of builtin indexes and logs as `VALUES` so that its SQL fingerprint changes whenever a builtin index or log is added or removed, triggering an automatic `MigrationStep::replacement`. Each builtin struct carries an optional `ontology` field of type `Option` that marks the object as a catalog ontology entity and provides entity-level metadata. diff --git a/doc/developer/generated/catalog/builtin/builtin.md b/doc/developer/generated/catalog/builtin/builtin.md index aae82eb1755d4..f07f9eb391f37 100644 --- a/doc/developer/generated/catalog/builtin/builtin.md +++ b/doc/developer/generated/catalog/builtin/builtin.md @@ -1,13 +1,13 @@ --- source: src/catalog/src/builtin/builtin.rs -revision: 9693291dd4 +revision: fd1dd6e62b --- # catalog::builtin::builtin Generates builtin views that expose information about builtin catalog objects. -The public entry point is `builtins(builtin_items)`, which iterates over a slice of `Builtin` values and produces additional `Builtin::View` and `Builtin::MaterializedView` entries to be included in the static builtin list. +The public entry point is `builtins(builtin_items)`, which iterates over a slice of `Builtin` values and produces additional `Builtin::View` entries to be included in the static builtin list. The module generates two catalog objects: - The `mz_builtin_materialized_views` builtin view (in `mz_internal`), which reports every builtin materialized view with columns: `oid`, `schema_name`, `name`, `cluster_name`, `definition`, `privileges`, and `create_sql`. Its SQL is constructed dynamically by serializing each `BuiltinMaterializedView`'s metadata into a `VALUES` clause. diff --git a/doc/developer/generated/catalog/builtin/mz_catalog.md b/doc/developer/generated/catalog/builtin/mz_catalog.md index 6ef8f8499b9af..a0f18467f531b 100644 --- a/doc/developer/generated/catalog/builtin/mz_catalog.md +++ b/doc/developer/generated/catalog/builtin/mz_catalog.md @@ -1,6 +1,6 @@ --- source: src/catalog/src/builtin/mz_catalog.rs -revision: 34813f0a3f +revision: fd1dd6e62b --- # catalog::builtin::mz_catalog @@ -11,9 +11,9 @@ This module exports 76 public items: types, tables, materialized views, views, s **Types** — Materialize-specific pseudo-types and their array forms: `TYPE_LIST`, `TYPE_MAP`, `TYPE_ANYCOMPATIBLELIST`, `TYPE_ANYCOMPATIBLEMAP`; unsigned integer types `TYPE_UINT2/4/8` and their arrays; `TYPE_MZ_TIMESTAMP`, `TYPE_MZ_TIMESTAMP_ARRAY`; `TYPE_MZ_ACL_ITEM`, `TYPE_MZ_ACL_ITEM_ARRAY`. -**Tables** — `BuiltinTable` statics covering connector-specific metadata: `MZ_ICEBERG_SINKS`, `MZ_KAFKA_SINKS`, `MZ_KAFKA_CONNECTIONS`, and more. +**Tables** — `BuiltinTable` statics covering connector-specific metadata: `MZ_ICEBERG_SINKS`, `MZ_KAFKA_SINKS`, and more. -**Materialized views** — Core catalog entities backed by queries over `mz_internal.mz_catalog_raw`: `MZ_DATABASES`, `MZ_SCHEMAS`, `MZ_CONNECTIONS`, `MZ_SECRETS`, `MZ_TABLES`, `MZ_COLUMNS`, `MZ_VIEWS`, `MZ_SOURCES`, `MZ_SINKS`, `MZ_CLUSTERS`, `MZ_CLUSTER_REPLICAS`, `MZ_ROLES`, `MZ_ROLE_PARAMETERS`, `MZ_OBJECTS`, `MZ_ALL_OBJECTS`, `MZ_DEFAULT_PRIVILEGES`, `MZ_SYSTEM_PRIVILEGES`, `MZ_AUDIT_EVENTS`, `MZ_KAFKA_SOURCES`, and others. `MZ_KAFKA_SOURCES` is a `BuiltinMaterializedView` (OID constant `MV_MZ_KAFKA_SOURCES_OID`) that derives Kafka source details (topic and computed group ID) by parsing the raw catalog create SQL via `mz_internal.parse_kafka_source_details`; the `group_id_prefix` column holds the full computed `group.id`, not just the user-supplied prefix. `MZ_AUDIT_EVENTS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_AUDIT_EVENTS_OID`) whose `event_type` column covers `create`, `drop`, `alter`, `grant`, `revoke`, and `comment`, and whose `object_type` column covers `cluster`, `cluster-replica`, `connection`, `database`, `func`, `index`, `materialized-view`, `role`, `secret`, `schema`, `sink`, `source`, `table`, `type`, `view`, `system`, `continual-task`, and `network-policy`. `MZ_CLUSTERS` and `MZ_CLUSTER_REPLICAS` are `BuiltinMaterializedView` objects that derive cluster and replica metadata from the durable catalog raw source; `MZ_CLUSTER_REPLICAS` resolves the `disk` column via a LEFT JOIN against `mz_cluster_replica_size_internal` (which retains rows for disabled sizes), so replicas with unknown sizes return `NULL` for `disk` rather than causing an error. `MZ_DEFAULT_PRIVILEGES` decodes `DefaultPrivileges` rows from `mz_catalog_raw`, mapping the proto enum numeric `object_type` to its SQL display name via a CASE expression. `MZ_SYSTEM_PRIVILEGES` decodes `SystemPrivileges` rows and unnests them into `mz_aclitem` values. These carry `Ontology` annotations with entity names, descriptions, and `OntologyLink` relationships for the catalog graph. `MZ_INDEXES` is also a `BuiltinMaterializedView` but is generated dynamically via the `pub(super) make_mz_indexes` function rather than declared as a static, so it is not counted among the module's exported statics. +**Materialized views** — Core catalog entities backed by queries over `mz_internal.mz_catalog_raw`: `MZ_DATABASES`, `MZ_SCHEMAS`, `MZ_CONNECTIONS`, `MZ_SECRETS`, `MZ_TABLES`, `MZ_COLUMNS`, `MZ_VIEWS`, `MZ_SOURCES`, `MZ_SINKS`, `MZ_CLUSTERS`, `MZ_CLUSTER_REPLICAS`, `MZ_ROLES`, `MZ_ROLE_PARAMETERS`, `MZ_OBJECTS`, `MZ_ALL_OBJECTS`, `MZ_DEFAULT_PRIVILEGES`, `MZ_SYSTEM_PRIVILEGES`, `MZ_AUDIT_EVENTS`, `MZ_KAFKA_SOURCES`, and others. `MZ_KAFKA_CONNECTIONS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_KAFKA_CONNECTIONS_OID`) that reads `Item` rows from `mz_catalog_raw`, pulls broker addresses and the progress topic out of the persisted `create_sql` via `parse_connection_details`, and keeps only kafka connections. The default progress topic is reconstructed in SQL to match `KafkaConnection::progress_topic` in the storage types crate. `MZ_SSH_TUNNEL_CONNECTIONS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_SSH_TUNNEL_CONNECTIONS_OID`) that derives SSH tunnel connection details (public keys) by parsing `create_sql` via `parse_connection_details`. `MZ_AWS_PRIVATELINK_CONNECTIONS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_AWS_PRIVATELINK_CONNECTIONS_OID`) that reconstructs the principal ARN from the environment-context functions `mz_aws_account_id()` and `mz_aws_external_id_prefix()`; the ARN format mirrors `AwsPrincipalContext::to_principal_string`. It keeps a `WHERE principal IS NOT NULL` guard so no row is emitted when the AWS context is absent, matching the packer it replaces. `MZ_KAFKA_SOURCES` is a `BuiltinMaterializedView` (OID constant `MV_MZ_KAFKA_SOURCES_OID`) that derives Kafka source details (topic and computed group ID) by parsing the raw catalog create SQL via `mz_internal.parse_kafka_source_details`; the `group_id_prefix` column holds the full computed `group.id`, not just the user-supplied prefix. `MZ_AUDIT_EVENTS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_AUDIT_EVENTS_OID`) whose `event_type` column covers `create`, `drop`, `alter`, `grant`, `revoke`, and `comment`, and whose `object_type` column covers `cluster`, `cluster-replica`, `connection`, `database`, `func`, `index`, `materialized-view`, `role`, `secret`, `schema`, `sink`, `source`, `table`, `type`, `view`, `system`, `continual-task`, and `network-policy`. `MZ_CLUSTERS` and `MZ_CLUSTER_REPLICAS` are `BuiltinMaterializedView` objects that derive cluster and replica metadata from the durable catalog raw source; `MZ_CLUSTER_REPLICAS` resolves the `disk` column via a LEFT JOIN against `mz_cluster_replica_size_internal` (which retains rows for disabled sizes), so replicas with unknown sizes return `NULL` for `disk` rather than causing an error. `MZ_DEFAULT_PRIVILEGES` decodes `DefaultPrivileges` rows from `mz_catalog_raw`, mapping the proto enum numeric `object_type` to its SQL display name via a CASE expression. `MZ_SYSTEM_PRIVILEGES` decodes `SystemPrivileges` rows and unnests them into `mz_aclitem` values. These carry `Ontology` annotations with entity names, descriptions, and `OntologyLink` relationships for the catalog graph. `MZ_INDEXES` is also a `BuiltinMaterializedView` but is generated dynamically via the `pub(super) make_mz_indexes` function rather than declared as a static, so it is not counted among the module's exported statics. **Views** — Additional SQL views for derived catalog information such as `MZ_TIMEZONE_ABBREVIATIONS` and `MZ_TIMEZONE_NAMES`. diff --git a/doc/developer/generated/catalog/builtin/mz_internal.md b/doc/developer/generated/catalog/builtin/mz_internal.md index 90b5ebb43da99..7d6451ffa205f 100644 --- a/doc/developer/generated/catalog/builtin/mz_internal.md +++ b/doc/developer/generated/catalog/builtin/mz_internal.md @@ -1,6 +1,6 @@ --- source: src/catalog/src/builtin/mz_internal.rs -revision: b341dc47d6 +revision: fd1dd6e62b --- # catalog::builtin::mz_internal @@ -11,9 +11,9 @@ This is the largest builtin submodule, exporting 199 public items: sources, tabl **Sources** (`BuiltinSource`) — Backed by `DataSourceDesc::Catalog` or `IntrospectionType`. Key items include `MZ_CATALOG_RAW` (the raw persist-backed catalog source, system-only access), and storage statistics/status history sources such as `MZ_SOURCE_STATISTICS_RAW`, `MZ_SINK_STATISTICS_RAW`, `MZ_SOURCE_STATUS_HISTORY`, `MZ_SINK_STATUS_HISTORY`, `MZ_STATEMENT_EXECUTION_HISTORY`, `MZ_SESSION_HISTORY`, `MZ_SQL_TEXT`, `MZ_PREPARED_STATEMENT_HISTORY`, and replica metrics/status history sources. -**Tables** (`BuiltinTable`) — Connector-specific and internal metadata tables such as `MZ_MYSQL_SOURCES`, and others tracking sink/source details. `MZ_CLUSTER_SCHEDULES` is a `BuiltinMaterializedView` (not a table) backed by a query over `mz_internal.mz_catalog_raw` that derives cluster scheduling configuration from the durable catalog JSON. +**Tables** (`BuiltinTable`) — Connector-specific and internal metadata tables such as `MZ_MYSQL_SOURCES`, and others tracking sink/source details. `MZ_CLUSTER_SCHEDULES` is a `BuiltinMaterializedView` (not a table) backed by a query over `mz_internal.mz_catalog_raw` that derives cluster scheduling configuration from the durable catalog JSON. `MZ_POSTGRES_SOURCE_TABLES`, `MZ_MYSQL_SOURCE_TABLES`, `MZ_SQL_SERVER_SOURCE_TABLES`, and `MZ_KAFKA_SOURCE_TABLES` are `BuiltinMaterializedView` objects (not tables) backed by queries over `mz_catalog_raw`. -**Materialized views** (`BuiltinMaterializedView`) — Derived catalog views backed by queries over `mz_catalog_raw` and other sources: aggregated statistics, lag histograms, and other derived metrics. `MZ_POSTGRES_SOURCES` is a `BuiltinMaterializedView` (OID constant `MV_MZ_POSTGRES_SOURCES_OID`) that derives PostgreSQL source details (`replication_slot`, `timeline_id`) by parsing the raw catalog create SQL via `mz_internal.parse_postgres_source_details`; the MV filters to postgres sources first via `parse_catalog_create_sql` because the detail decoder must not be applied to non-postgres rows. `MZ_OVERRIDDEN_SYSTEM_PARAMETERS` projects the `system_configurations` durable collection out of `mz_catalog_raw`, exposing environment-wide system parameter overrides as `(name, value)` pairs with `PUBLIC_SELECT` access and an `Ontology` annotation; only parameters with an explicit override appear (defaults are absent). `MZ_CLUSTER_SYSTEM_PARAMETERS` projects the `cluster_system_configurations` durable collection out of `mz_catalog_raw`, exposing per-cluster system parameter overrides keyed by `(cluster_id, name)`. `MZ_REPLICA_SYSTEM_PARAMETERS` does the same for the `replica_system_configurations` collection, exposing per-replica overrides keyed by `(replica_id, name)`. `MZ_COMMENTS` is a `BuiltinMaterializedView` backed by a query over `mz_catalog_raw` that reads `Comment` entries, decodes each `proto::CommentObject` variant via CASE expressions, and projects `(id, object_type, object_sub_id, comment)` rows; new `CommentObject` variants must be added to both CASE expressions in its SQL. `MZ_CLUSTER_RECONFIGURATIONS` (OID `MV_MZ_CLUSTER_RECONFIGURATIONS_OID`) is a `BuiltinMaterializedView` with a unique key on `cluster_id` that exposes the latest graceful reconfiguration record per managed cluster, with columns `cluster_id`, `status`, `deadline`, `on_timeout`, `target` (JSONB), and `changes` (JSONB). Terminal records are retained after settling so the outcome stays inspectable until a later reconfiguration overwrites it. The `status` column maps serde variant names from `ReconfigurationStatus` to kebab-case (e.g. `InProgress` -> `in-progress`); unmapped variants pass through verbatim. `changes` diffs `target` against the realized config per dimension. `MZ_CLUSTER_AUTO_SCALING_STRATEGIES` (OID `MV_MZ_CLUSTER_AUTO_SCALING_STRATEGIES_OID`) is a `BuiltinMaterializedView` with a unique key on `cluster_id` that exposes the configured autoscaling strategy and in-flight state per managed cluster; `state` is keyed by strategy type so a future strategy's state is an additional key rather than a schema change. Both views have corresponding `BuiltinIndex` constants (`MZ_CLUSTER_RECONFIGURATIONS_IND`, `MZ_CLUSTER_AUTO_SCALING_STRATEGIES_IND`) on `cluster_id`. +**Materialized views** (`BuiltinMaterializedView`) — Derived catalog views backed by queries over `mz_catalog_raw` and other sources: aggregated statistics, lag histograms, and other derived metrics. `MZ_POSTGRES_SOURCES` is a `BuiltinMaterializedView` (OID constant `MV_MZ_POSTGRES_SOURCES_OID`) that derives PostgreSQL source details (`replication_slot`, `timeline_id`) by parsing the raw catalog create SQL via `mz_internal.parse_postgres_source_details`; the MV filters to postgres sources first via `parse_catalog_create_sql` because the detail decoder must not be applied to non-postgres rows. `MZ_POSTGRES_SOURCE_TABLES` (OID `MV_MZ_POSTGRES_SOURCE_TABLES_OID`), `MZ_MYSQL_SOURCE_TABLES` (OID `MV_MZ_MYSQL_SOURCE_TABLES_OID`), `MZ_SQL_SERVER_SOURCE_TABLES` (OID `MV_MZ_SQL_SERVER_SOURCE_TABLES_OID`), and `MZ_KAFKA_SOURCE_TABLES` (OID `MV_MZ_KAFKA_SOURCE_TABLES_OID`) are `BuiltinMaterializedView` objects that derive source-export (source table) metadata by parsing catalog `create_sql` via `mz_internal.parse_source_export_details` and joining `mz_catalog.mz_sources` to filter by parent source type. Each has columns `id`, `schema_name`, and `table_name` (postgres, mysql, sql-server) or `id`, `topic`, `envelope_type`, `key_format`, and `value_format` (kafka). The postgres and sql-server external references are three-part (`[database, schema, table]`), so `schema_name` and `table_name` come from positions 1 and 2. MySQL references are two-part (`[schema, table]`), so positions 0 and 1. For kafka source tables, `envelope_type` is not nullable: Kafka defaults to `ENVELOPE NONE` when the clause is omitted, so the SQL defaults to `'none'` via `COALESCE` for rows where the parser helper returns NULL. These four MVs are marked `is_retained_metrics_object: true`. `MZ_AWS_CONNECTIONS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_AWS_CONNECTIONS_OID`) that reads `Item` rows from `mz_catalog_raw`, pulls create_sql-derived fields (endpoint, region, credentials, assume-role options) via `parse_connection_details`, and keeps only aws connections. The three context-derived columns (`principal`, `external_id`, `example_trust_policy`) apply to assume-role connections only and are reconstructed from the plan-time AWS context functions (`mz_aws_account_id`, `mz_aws_external_id_prefix`, `mz_aws_connection_role_arn`). On an environment without the AWS context, assume-role connections are excluded entirely, matching the packer this view replaces. `MZ_OVERRIDDEN_SYSTEM_PARAMETERS` projects the `system_configurations` durable collection out of `mz_catalog_raw`, exposing environment-wide system parameter overrides as `(name, value)` pairs with `PUBLIC_SELECT` access and an `Ontology` annotation; only parameters with an explicit override appear (defaults are absent). `MZ_CLUSTER_SYSTEM_PARAMETERS` projects the `cluster_system_configurations` durable collection out of `mz_catalog_raw`, exposing per-cluster system parameter overrides keyed by `(cluster_id, name)`. `MZ_REPLICA_SYSTEM_PARAMETERS` does the same for the `replica_system_configurations` collection, exposing per-replica overrides keyed by `(replica_id, name)`. `MZ_COMMENTS` is a `BuiltinMaterializedView` backed by a query over `mz_catalog_raw` that reads `Comment` entries, decodes each `proto::CommentObject` variant via CASE expressions, and projects `(id, object_type, object_sub_id, comment)` rows; new `CommentObject` variants must be added to both CASE expressions in its SQL. `MZ_CLUSTER_RECONFIGURATIONS` (OID `MV_MZ_CLUSTER_RECONFIGURATIONS_OID`) is a `BuiltinMaterializedView` with a unique key on `cluster_id` that exposes the latest graceful reconfiguration record per managed cluster, with columns `cluster_id`, `status`, `deadline`, `on_timeout`, `target` (JSONB), and `changes` (JSONB). Terminal records are retained after settling so the outcome stays inspectable until a later reconfiguration overwrites it. The `status` column maps serde variant names from `ReconfigurationStatus` to kebab-case (e.g. `InProgress` -> `in-progress`); unmapped variants pass through verbatim. `changes` diffs `target` against the realized config per dimension. `MZ_CLUSTER_AUTO_SCALING_STRATEGIES` (OID `MV_MZ_CLUSTER_AUTO_SCALING_STRATEGIES_OID`) is a `BuiltinMaterializedView` with a unique key on `cluster_id` that exposes the configured autoscaling strategy and in-flight state per managed cluster; `state` is keyed by strategy type so a future strategy's state is an additional key rather than a schema change. Both views have corresponding `BuiltinIndex` constants (`MZ_CLUSTER_RECONFIGURATIONS_IND`, `MZ_CLUSTER_AUTO_SCALING_STRATEGIES_IND`) on `cluster_id`. **Views** (`BuiltinView`) — 82 entries covering the full `mz_internal` SQL surface: cluster introspection, compute operator metrics, peek durations, arrangement sizes, source/sink status, wall-clock lag, statement execution history, privilege management, dependency graph views, and more. `MZ_SOURCE_STATUSES` and `MZ_SINK_STATUSES` filter their `latest_per_replica_events` CTE to include events from still-existing replicas or the global sentinel, and additionally retain rows where `status = 'paused'` because a per-replica `paused` event is only written when the replica is dropped and acts as a terminal drop marker. Precedence ties in `latest_events` are broken by `occurred_at DESC` in addition to status precedence number, so a dropped replica's `paused` event wins over an older source-global `paused`. `MZ_SHOW_COLUMNS` applies an `object_type` predicate on its comment join to guard against stale comment rows from type changes. `MZ_SHOW_ALL_OBJECTS` and `MZ_SHOW_CLUSTER_REPLICAS` move the `object_type` filter from a WHERE clause into the LEFT JOIN ON condition so that rows with mismatched comment types are excluded from the join rather than the result set. `MZ_MCP_DATA_PRODUCTS` and `MZ_MCP_DATA_PRODUCT_DETAILS` similarly scope their comment joins with `object_type` predicates in the ON clause. `MZ_SHOW_CLUSTERS` includes an `activity` column (nullable string) that summarizes any in-flight reconfiguration or autoscaling burst: it joins `mz_cluster_reconfigurations` (filtering to `status = 'in-progress'`) and `mz_cluster_auto_scaling_strategies` (checking for a non-null `state`) and produces a human-readable summary of the dimensions being changed and the burst size; `NULL` when the cluster is steady. Views carry inline SQL queries over `mz_catalog`, `mz_introspection`, and `mz_internal` tables. Access levels vary: most grant `PUBLIC_SELECT`; sensitive views use `MONITOR_SELECT`, `MONITOR_REDACTED_SELECT`, `SUPPORT_SELECT`, or `ANALYTICS_SELECT`. `MZ_MCP_DATA_PRODUCTS` and `MZ_MCP_DATA_PRODUCT_DETAILS` are MCP-agent-facing views listing data products and their column/key details that are exempted from `restrict_to_user_objects` blocking. The `cluster` column in both views is null unless the session role has `USAGE` on the object's index/compute cluster (checked via `mz_show_my_cluster_privileges`); materialized views still appear regardless because they serve from persist, but plain indexed views without at least one usable index cluster are excluded. `MZ_MCP_DATA_PRODUCT_DETAILS` includes a `hydration` column (JSONB) reporting readiness across the cluster's replicas, with fields `hydrated` (bool), `replica_count` (int), and `hydrated_replica_count` (int); the hydration join uses the real cluster name even when the `cluster` column is null. `MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW` (1-hour buckets, 14-day retention), `MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_24H` (5-minute buckets, 24-hour retention), and `MZ_CONSOLE_CLUSTER_UTILIZATION_OVERVIEW_3H` (un-binned raw samples, 3-hour retention) are three cluster utilization views for the Console's time-range graphs; their SQL bodies are generated by shared helpers so their fingerprints stay in sync with the Console's equivalent ad-hoc query. `MZ_BUILTIN_SOURCES` is a view in `mz_internal` listing builtin and log sources that do not appear in `mz_catalog_raw`; user sources are exposed via `mz_catalog.mz_sources`. diff --git a/doc/developer/generated/catalog/config.md b/doc/developer/generated/catalog/config.md index 0f93e36aed277..99f82fcfcfaa1 100644 --- a/doc/developer/generated/catalog/config.md +++ b/doc/developer/generated/catalog/config.md @@ -1,6 +1,6 @@ --- source: src/catalog/src/config.rs -revision: a8f4526d28 +revision: fd1dd6e62b --- # catalog::config @@ -8,4 +8,4 @@ revision: a8f4526d28 Defines `Config` and `StateConfig`, the structs used to open and configure the catalog. `Config` bundles the durable storage backend, a metrics registry, and the `StateConfig`. `StateConfig` collects all initialization parameters: build info, environment ID, boot timestamp, cluster replica size map, builtin cluster configs, system parameter defaults, feature flags, connection context, expression cache override, helm chart version, license key, and more. -Also defines `ClusterReplicaSizeMap` (mapping user-facing size strings to `ReplicaAllocation`), `BuiltinItemMigrationConfig`, and `AwsPrincipalContext` for resource allocation and connection configuration. `ReplicaAllocation` includes a `family: Option` field identifying the coarse size family (e.g. `"D"` or `"legacy"`) used as the `replica_size_family` attribute when evaluating replica-local scoped feature flags; when `None`, the family is derived from `is_cc`. +Also defines `ClusterReplicaSizeMap` (mapping user-facing size strings to `ReplicaAllocation`), `BuiltinItemMigrationConfig`, and `AwsPrincipalContext` for resource allocation and connection configuration. `ReplicaAllocation` includes a `family: Option` field identifying the coarse size family (e.g. `"D"` or `"legacy"`) used as the `replica_size_family` attribute when evaluating replica-local scoped feature flags; when `None`, the family is derived from `is_cc`. `AwsPrincipalContext::to_principal_string` constructs the IAM role ARN in the format `arn:aws:iam:::role/mz__`; the `mz_catalog.mz_aws_privatelink_connections` materialized view reconstructs this same format in SQL from the `mz_aws_account_id()` and `mz_aws_external_id_prefix()` plan-time functions, so the two must be kept in sync. diff --git a/doc/developer/generated/catalog/durable/_module.md b/doc/developer/generated/catalog/durable/_module.md index 22efee414577b..a996201da8c45 100644 --- a/doc/developer/generated/catalog/durable/_module.md +++ b/doc/developer/generated/catalog/durable/_module.md @@ -1,6 +1,6 @@ --- source: src/catalog/src/durable.rs -revision: a60edac7f1 +revision: 9acb77622f --- # catalog::durable diff --git a/doc/developer/generated/catalog/durable/objects/_module.md b/doc/developer/generated/catalog/durable/objects/_module.md index 95a1881d1ff7e..699be3f7d7607 100644 --- a/doc/developer/generated/catalog/durable/objects/_module.md +++ b/doc/developer/generated/catalog/durable/objects/_module.md @@ -1,11 +1,11 @@ --- source: src/catalog/src/durable/objects.rs -revision: fca741734d +revision: 9acb77622f --- # catalog::durable::objects Defines the on-disk representation of every catalog entity as split key-value pairs (e.g., `DatabaseKey`/`DatabaseValue`) and their combined Rust structs (e.g., `Database`). The `DurableType` trait converts between the combined struct and its key-value pair, enabling encoding and decoding for persist. -Public re-exports (via `crate::durable`) expose the combined structs (`Cluster`, `ClusterSystemConfiguration`, `Item`, `Role`, `ReplicaSystemConfiguration`, `Schema`, `BurstState`, `ReconfigurationState`, `ReconfigurationStatus`, `ReconfigurationTarget`, etc.) to other modules; the key-value types remain internal. +Public re-exports (via `crate::durable`) expose the combined structs (`Cluster`, `ClusterConfig`, `ClusterVariant`, `ClusterVariantManaged`, `ClusterSystemConfiguration`, `Item`, `Role`, `ReplicaSystemConfiguration`, `Schema`, `BurstState`, `ReconfigurationState`, `ReconfigurationStatus`, `ReconfigurationTarget`, etc.) to other modules; the key-value types remain internal. The `serialization` and `state_update` submodules handle protobuf conversion and the update-lifecycle pipeline respectively. diff --git a/doc/developer/generated/catalog/durable/upgrade/v89_to_v90.md b/doc/developer/generated/catalog/durable/upgrade/v89_to_v90.md new file mode 100644 index 0000000000000..2b26dce8bf2b2 --- /dev/null +++ b/doc/developer/generated/catalog/durable/upgrade/v89_to_v90.md @@ -0,0 +1,21 @@ +--- +source: src/catalog/src/durable/upgrade/v89_to_v90.rs +revision: fca741734d +--- + +# catalog::durable::upgrade::v89_to_v90 + +Migrates catalog state from version 89 to version 90. + +The only structural change is adding `arrangement_compression: bool` to managed clusters, their in-flight reconfiguration targets, and cluster replicas. Every affected record is backfilled with `false`. + +The `upgrade` function accepts a full v89 state snapshot and returns one `MigrationAction::Update` entry for each managed cluster and each cluster replica. Unmanaged clusters have no `ManagedCluster` variant and are passed through without a migration action. All other record types are unchanged. + +The three migration helpers correspond to the three affected record types: + +- `migrate_cluster` — rewrites a managed `Cluster` record, delegating managed-variant fields to `migrate_managed`. +- `migrate_managed` — sets `arrangement_compression: false` on a `ManagedCluster` and forwards any in-flight reconfiguration to `migrate_reconfiguration`. +- `migrate_reconfiguration` — sets `arrangement_compression: false` on the `ReconfigurationTarget` inside a `ReconfigurationState`. +- `migrate_replica` — sets `arrangement_compression: false` on the `ReplicaConfig` of a `ClusterReplica`. + +Structurally-compatible fields (those shared verbatim between v89 and v90) are transferred via `JsonCompatible::convert`, which is macro-asserted for each type pair at the top of the file. diff --git a/doc/developer/generated/catalog/memory/objects.md b/doc/developer/generated/catalog/memory/objects.md index 0465b9d3840bc..615ff8195e708 100644 --- a/doc/developer/generated/catalog/memory/objects.md +++ b/doc/developer/generated/catalog/memory/objects.md @@ -1,6 +1,6 @@ --- source: src/catalog/src/memory/objects.rs -revision: fca741734d +revision: 447da2b53e --- # catalog::memory::objects @@ -11,6 +11,7 @@ Key types include `CatalogItem` (an enum over Table, Source, Log, View, Material `Index` and `MaterializedView` additionally carry `optimized_plan` (global MIR dataflow), `physical_plan` (`DataflowDescription`, i.e. the LIR dataflow), and `dataflow_metainfo` (optimizer notices) fields, all `#[serde(skip)]` since they are populated after catalog replay and are not part of the durable representation. `CatalogItem` exposes `optimized_plan()`, `physical_plan()`, and `dataflow_metainfo()` accessors for the plan fields, and `plan_fields_mut()` for mutable access to all three plan fields at once on plan-bearing items (`Index`, `MaterializedView`); it returns `None` for other item kinds. `CatalogItem::set_create_sql` overwrites the `create_sql` of a view, index, or table without replanning, used during temporary item updates to keep the byte-identical `create_sql` for retraction consolidation; it panics for item kinds that cannot be temporary. `CatalogItem::update_timestamp_interval` updates the timestamp interval for a source and returns the previous `WithOptionValue` if one was set, or `None`; it returns `Err(())` if called on a non-source item. `CatalogItem::is_hydratable` returns `true` for items that run a dataflow on cluster replicas and therefore have hydration state: indexes, materialized views, sinks, and ingestion sources. Non-ingestion sources (webhooks, ingestion exports) are excluded. `CatalogEntry` exposes `item_mut()` for in-place mutation. `ClusterVariantManaged` carries `auto_scaling_strategy` (`Option`), `reconfiguration` (`Option`), and `burst` (`Option`) in addition to the base cluster shape fields. `ClusterVariantManaged::has_unwarranted_burst_record` returns `true` when the in-flight `burst` record is no longer consistent with the current config: the `ON HYDRATION` policy was removed or re-sized, or `replication_factor` was set to 0. `ReconfigurationState` is the in-memory mirror of the durable type, recording the target config, deadline, `on_timeout` action, and `status` (`ReconfigurationStatus`: `InProgress`, `Finalized`, `TimedOut`, `Cancelled`, `ResourceExhausted`) for a graceful reconfiguration. `BurstState` is the in-memory mirror of the durable burst-state record, carrying `burst_size`, `linger_duration`, and an optional `steady_hydrated_at` timestamp. +`Cluster::try_to_plan` rebuilds the `CreateClusterPlan` that the original `CREATE CLUSTER` statement would have produced. Clusters are stored as a structured config rather than SQL text, so `SHOW CREATE CLUSTER` uses this method as the first half of the config-to-statement roundtrip; `unplan_create_cluster` then converts the plan back into a statement. The rebuilt plan always sets `if_not_exists: false` because `IF NOT EXISTS` is a property of the original statement, not of the cluster itself. `ClusterReplicaProcessStatus` records the process status, a cumulative `restart_count` mirrored from the orchestrator, and the time of the most recent change to either field. `Schema::has_items` returns true if any of `self.items`, `self.types`, or `self.functions` is non-empty, so DROP SCHEMA without CASCADE is rejected when the schema contains only types or functions. `StateUpdate` and `StateUpdateKind` represent in-memory deltas applied during catalog replay and incremental updates. `StateUpdateKind` includes `ClusterSystemConfiguration` and `ReplicaSystemConfiguration` variants carrying the corresponding durable objects. `BootstrapStateUpdateKind` mirrors `StateUpdateKind` for the bootstrap phase and includes the same two variants; it converts to/from `StateUpdateKind` via `From`/`TryFrom` implementations. diff --git a/doc/developer/generated/cloud-resources/crd/_module.md b/doc/developer/generated/cloud-resources/crd/_module.md index 85749f9974994..8273615da44b9 100644 --- a/doc/developer/generated/cloud-resources/crd/_module.md +++ b/doc/developer/generated/cloud-resources/crd/_module.md @@ -1,6 +1,6 @@ --- source: src/cloud-resources/src/crd.rs -revision: b088d23e1d +revision: 253293ef87 --- # cloud-resources::crd diff --git a/doc/developer/generated/cloud-resources/crd/materialize.md b/doc/developer/generated/cloud-resources/crd/materialize.md index fc7f0fbb4e89e..c0bcd1c975d14 100644 --- a/doc/developer/generated/cloud-resources/crd/materialize.md +++ b/doc/developer/generated/cloud-resources/crd/materialize.md @@ -1,6 +1,6 @@ --- source: src/cloud-resources/src/crd/materialize.rs -revision: f5669fe128 +revision: 253293ef87 --- # cloud-resources::crd::materialize diff --git a/doc/developer/generated/cloud-resources/crd/vpc_endpoint.md b/doc/developer/generated/cloud-resources/crd/vpc_endpoint.md index 6dea955a9d8b1..1cfe633834e0c 100644 --- a/doc/developer/generated/cloud-resources/crd/vpc_endpoint.md +++ b/doc/developer/generated/cloud-resources/crd/vpc_endpoint.md @@ -1,6 +1,6 @@ --- source: src/cloud-resources/src/crd/vpc_endpoint.rs -revision: 4267863081 +revision: 253293ef87 --- # cloud-resources::crd::vpc_endpoint diff --git a/doc/developer/generated/cluster-controller/_crate.md b/doc/developer/generated/cluster-controller/_crate.md index aa1a37271cc6f..a6881d6f3b717 100644 --- a/doc/developer/generated/cluster-controller/_crate.md +++ b/doc/developer/generated/cluster-controller/_crate.md @@ -1,6 +1,6 @@ --- source: src/cluster-controller/src/lib.rs -revision: 6eeaca032b +revision: 767b86caef --- # mz-cluster-controller diff --git a/doc/developer/generated/cluster-controller/ctx.md b/doc/developer/generated/cluster-controller/ctx.md index 7149454be44f6..1e76c6cba5e83 100644 --- a/doc/developer/generated/cluster-controller/ctx.md +++ b/doc/developer/generated/cluster-controller/ctx.md @@ -1,6 +1,6 @@ --- source: src/cluster-controller/src/ctx.rs -revision: 6eeaca032b +revision: 767b86caef --- # cluster-controller::ctx @@ -11,11 +11,13 @@ The boundary between the controller and its environment. Key types: -* `ClusterControllerCtx` (trait) — seven async methods: `now`, `managed_cluster_ids`, `cluster_states`, `hydrated_replicas`, `has_hydratable_objects`, `refresh_window_inputs`, `apply`. Reads are batched to bound round-trips in a separate-task deployment. `hydrated_replicas` returns the subset of given replicas that are online and have all current collections hydrated; it is only called when a strategy declares it needs hydration via `SignalRequest`. `has_hydratable_objects` returns whether a cluster has at least one dataflow-backed object (index, materialized view, ingestion source, or sink); it is only called when a strategy declares `hydratable_objects` in its `SignalRequest`. `refresh_window_inputs` pulls the read timestamp, compaction estimate, and bound REFRESH MV frontiers/schedules for a scheduled cluster; called only for clusters the on-refresh strategy needs to evaluate, returning `None` if the cluster has moved off a scheduled schedule mid-tick. +* `ClusterControllerCtx` (trait) — one synchronous method (`now`) and six async methods (`managed_cluster_ids`, `cluster_states`, `hydrated_replicas`, `has_hydratable_objects`, `refresh_window_inputs`, `apply`). Reads are batched to bound round-trips in a separate-task deployment. `hydrated_replicas` returns the subset of given replicas that are online and have all current collections hydrated; it is only called when a strategy declares it needs hydration via `SignalRequest`. `has_hydratable_objects` returns whether a cluster has at least one dataflow-backed object (index, materialized view, ingestion source, or sink); it is only called when a strategy declares `hydratable_objects` in its `SignalRequest`. `refresh_window_inputs` pulls the read timestamp, compaction estimate, and bound REFRESH MV frontiers/schedules for a batch of scheduled clusters; called only for clusters the on-refresh strategy needs to evaluate, returning `None` when the batch fails or no requested cluster has valid inputs. * `ClusterState` — durable config plus observed replicas of one managed cluster for one tick. Carries `cluster_id`, `size`, `replication_factor`, `availability_zones`, `logging`, `arrangement_compression`, `schedule` (the `ClusterSchedule` driving whether the baseline or on-refresh strategy owns the replica set), optional `auto_scaling_policy`, `reconfiguration` and `burst` records, and `replicas`. Unmanaged clusters are not represented. * `ObservedReplica` — a replica that actually exists: `replica_id`, `name`, `shape` (optional; `None` for unmanaged-location replicas), `internal`, `billed_as`, `pending`. `ObservedReplica::owned_shape` returns the replica's shape only when the controller owns it: `INTERNAL`, `BILLED AS`, and pending replicas return `None` and are excluded from the desired/actual diff, though their names still block the name generator. * `RefreshMvInfo` — one REFRESH materialized view bound to a scheduled cluster. Carries the MV's `GlobalId`, its storage write frontier (`write_frontier: Antichain`), and its `refresh_schedule`. Used by the on-refresh strategy to determine whether a refresh is due and whether compaction time is still needed. -* `RefreshWindowInputs` — the live signals the on-refresh strategy reads to decide whether a scheduled cluster is inside a refresh window: `read_ts` (the local oracle read timestamp), `compaction_estimate` (how long after a refresh an MV is estimated to still need Persist compaction), and `refresh_mvs` (the bound REFRESH MVs). Pulled on demand only for scheduled clusters. +* `RefreshWindowClusterInputs` — the catalog and storage inputs for one scheduled cluster's refresh window: `compaction_estimate` (how long after a refresh an MV is estimated to still need Persist compaction) and `refresh_mvs` (the bound REFRESH MVs). +* `RefreshWindowInputsBatch` — the inputs gathered for one reconciliation phase across a batch of scheduled clusters. Carries a shared `read_ts` (one oracle read per phase) and a `cluster_inputs` map keyed by cluster. A cluster absent from the map has unavailable inputs and must not be reconciled during that phase. +* `RefreshWindowInputs` — the fulfilled live signal the on-refresh strategy uses for one cluster: `read_ts` (the shared oracle read timestamp), `compaction_estimate`, and `refresh_mvs`. Pulled on demand only for scheduled clusters. * `RefreshWindowDecision` — the on-refresh strategy's per-tick window decision. Lists which bound REFRESH MVs keep the cluster on (`objects_needing_refresh`, `objects_needing_compaction`) and the `hydration_time_estimate` the window was widened by. The window is open iff either list is non-empty. Carried inside `CreateReason::OnRefresh` on the create decisions an open window produces. * `CreateReason` — why a strategy desires a replica slot: the audit attribution a create decision carries. Variants: `Baseline` (audited as manual create), `GracefulReconfiguration`, `HydrationBurst`, `OnRefresh(RefreshWindowDecision)` (embeds the window decision so the audit detail appears iff this reason wins). When several strategies desire the same shape, `CreateReason::outranks` picks the winner: `GracefulReconfiguration` > `HydrationBurst` > `OnRefresh` > `Baseline`. * `StateWrite` — the durable mutations a strategy's `update_state` requests: cut-overs (`new_size`, `new_replication_factor`, `new_availability_zones`, `new_logging`, `new_arrangement_compression`) and record writes/clears (`reconfiguration`, `burst`). `None` fields are no-ops. diff --git a/doc/developer/generated/cluster-controller/strategy.md b/doc/developer/generated/cluster-controller/strategy.md index 32f9cc6360949..9afdaaaa92029 100644 --- a/doc/developer/generated/cluster-controller/strategy.md +++ b/doc/developer/generated/cluster-controller/strategy.md @@ -1,6 +1,6 @@ --- source: src/cluster-controller/src/strategy.rs -revision: 6eeaca032b +revision: 38a95cefe2 --- # cluster-controller::strategy diff --git a/doc/developer/generated/compute-types/dyncfgs.md b/doc/developer/generated/compute-types/dyncfgs.md index 7ccca474ee80b..0aa8768a82096 100644 --- a/doc/developer/generated/compute-types/dyncfgs.md +++ b/doc/developer/generated/compute-types/dyncfgs.md @@ -1,6 +1,6 @@ --- source: src/compute-types/src/dyncfgs.rs -revision: 93dcb0ef5a +revision: bd299ae61e --- # compute-types::dyncfgs diff --git a/doc/developer/generated/compute-types/explain/text.md b/doc/developer/generated/compute-types/explain/text.md index 8d4cb88a45a01..dc03c62027f2d 100644 --- a/doc/developer/generated/compute-types/explain/text.md +++ b/doc/developer/generated/compute-types/explain/text.md @@ -1,6 +1,6 @@ --- source: src/compute-types/src/explain/text.rs -revision: 58b5ff70c8 +revision: 253293ef87 --- # compute-types::explain::text @@ -9,5 +9,5 @@ Implements `DisplayText` for `LirRelationExpr` nodes, rendering the LIR plan tre Each `LirRelationExpr` variant maps to an uppercase header line; sub-plan details follow as indented `key=value` pairs or lowercase field lines. Supports both a default and a `verbose_syntax` mode; the latter is kept in sync with the `Display` implementation for `RenderPlanExprHumanizer`. Delegates join, reduce, threshold, and top-k sub-plan formatting to helper impls defined in the same file. -Join nodes are labeled `→Differential Cross Join` or `→Delta Cross Join` when any stage is a cross product (empty lookup key), and `→Differential Join` or `→Delta Join` otherwise. The private `fmt_join_chain` helper formats the join chain as `%pos:name[key]` entries, annotating each input with its relation name (if the underlying `LirRelationNode::Get` can be identified) and key columns. +Join nodes are labeled `→Differential Cross Join` when any stage is a cross product (empty lookup key), and `→Differential Join` otherwise, for linear joins. Delta joins use `→Delta Cross Join` or `→Delta Join`, and when reduced to a single path (one-shot joins) use `→One-Shot Delta Cross Join` or `→One-Shot Delta Join`. The private `fmt_join_chain` helper formats the join chain as `%pos:name[key]` entries, annotating each input with its relation name (if the underlying `LirRelationNode::Get` can be identified) and key columns. `Reduce` and `TopK` nodes whose `temporal_bucketing_strategy` is `TemporalBucketing` are prefixed with `Temporally-Bucketed ` in the output (e.g., `→Temporally-Bucketed Accumulable GroupAggregate`). A `Union` node where any input uses `TemporalBucketing` is similarly labeled `Temporally-Bucketed Union`. The verbose mode also emits a `temporal_bucketing_strategy=` line for `ArrangeBy` nodes whose strategy is non-`Direct`. diff --git a/doc/developer/generated/compute/compute_state/_module.md b/doc/developer/generated/compute/compute_state/_module.md index 4aca550fa6139..f2cce6ec066c3 100644 --- a/doc/developer/generated/compute/compute_state/_module.md +++ b/doc/developer/generated/compute/compute_state/_module.md @@ -1,6 +1,6 @@ --- source: src/compute/src/compute_state.rs -revision: 93dcb0ef5a +revision: 783a693265 --- # mz-compute::compute_state @@ -11,6 +11,7 @@ Contains the per-worker `ComputeState` and `ActiveComputeState` types that hold When `handle_create_instance` is called, it first applies `InstanceConfig::initial_config` to the worker configuration so that create-time setup observes controller-synced dyncfg values rather than defaults, then calls `apply_worker_config` to ensure state consistency before initialization. When `initialize_logging` is called on `CreateInstance`, the `storage_log_reader` is taken from `ComputeState` and forwarded to the logging setup so the timely logging dataflow can replay storage worker events. In read-only mode, the output frontier for collections excludes the write frontier (which can't be advanced by the dataflow), preventing stalled progress reporting. +Index peek processing is instrumented via `IndexPeekMetrics`, which carries references to per-phase histograms including `row_iteration_rows` (arrangement rows evaluated by the result iterator, including those filtered by MFP) and `result_sort_rows` (input rows across all thinning sort passes) alongside the existing duration histograms. ## Submodules diff --git a/doc/developer/generated/compute/compute_state/peek_result_iterator.md b/doc/developer/generated/compute/compute_state/peek_result_iterator.md index ee953ce8401c1..65cff4bb8630c 100644 --- a/doc/developer/generated/compute/compute_state/peek_result_iterator.md +++ b/doc/developer/generated/compute/compute_state/peek_result_iterator.md @@ -1,10 +1,11 @@ --- source: src/compute/src/compute_state/peek_result_iterator.rs -revision: 1c55de49eb +revision: 783a693265 --- # mz-compute::compute_state::peek_result_iterator Provides `PeekResultIterator`, an `Iterator` that extracts `(Row, NonZeroI64)` pairs from a `TraceReader` while applying a `SafeMfpPlan` and optional literal key constraints. Literal constraints are sorted and used to seek the trace cursor directly to matching keys, avoiding a full scan; the `Literals` helper manages the cursor seek state. +The iterator tracks `rows_processed`, the total number of arrangement rows (key-value pairs) it has evaluated including those filtered out by the MFP; callers retrieve this via `rows_processed()` after iteration to record row-workload metrics. This iterator is the hot path for index peeks and is used both inline in `ComputeState` and as a row source for the async peek stash upload task. diff --git a/doc/developer/generated/compute/metrics.md b/doc/developer/generated/compute/metrics.md index 73799b193f4e3..4b8a241734b0f 100644 --- a/doc/developer/generated/compute/metrics.md +++ b/doc/developer/generated/compute/metrics.md @@ -1,11 +1,12 @@ --- source: src/compute/src/metrics.rs -revision: 03de4421ba +revision: 783a693265 --- # mz-compute::metrics Defines `ComputeMetrics` (process-level) and `WorkerMetrics` (per-worker) Prometheus metric structs for the compute replica. -Tracks arrangement maintenance time, timely step durations, persist peek and stashed peek latencies, command handling durations, index peek latencies broken down by phase (`seek_fulfillment`, `error_scan`, `cursor_setup`, `row_iteration`, `result_sort`, `frontier_check`, `row_collection`, and total), reconciliation outcomes, replica expiration timestamps, collection hydration counts, and subscribe snapshot optimizations. +Tracks arrangement maintenance time, timely step durations, persist peek and stashed peek latencies, command handling durations, index peek latencies broken down by phase (`seek_fulfillment`, `error_scan`, `cursor_setup`, `row_iteration`, `result_sort`, `frontier_check`, `row_collection`, and total), row-count histograms for index peek row iteration (`row_iteration_rows`) and result sorting (`result_sort_rows`), reconciliation outcomes, replica expiration timestamps, collection hydration counts, and subscribe snapshot optimizations. +Index peek row-count histograms use exponential buckets starting at 1 with a factor of 2 across 25 steps, with an explicit zero bucket prepended, so both empty and large result sets are captured accurately. `CollectionMetrics` wraps `WorkerMetrics` to track the hydration state of a single collection and decrements the collection count gauge on drop. A `workload_class` label is optionally injected into all metrics via a registry post-processor. diff --git a/doc/developer/generated/compute/render/_module.md b/doc/developer/generated/compute/render/_module.md index e2c41eddbd60e..cabb347627f78 100644 --- a/doc/developer/generated/compute/render/_module.md +++ b/doc/developer/generated/compute/render/_module.md @@ -1,6 +1,6 @@ --- source: src/compute/src/render.rs -revision: d43cd78803 +revision: 253293ef87 --- # mz-compute::render diff --git a/doc/developer/generated/compute/sink/metric_sink.md b/doc/developer/generated/compute/sink/metric_sink.md new file mode 100644 index 0000000000000..4b8c393e7613e --- /dev/null +++ b/doc/developer/generated/compute/sink/metric_sink.md @@ -0,0 +1,34 @@ +--- +source: src/compute/src/sink/metric_sink.rs +revision: 94054eb165 +--- + +# mz-compute::sink::metric_sink + +Render arm for `MetricSinkConnection`: the compute-side operator that funnels a source collection's rows into the process's Prometheus registry. + +## Architecture + +The operator routes all data to one worker per process (chosen by hashing the sink's `GlobalId`), accumulates updates into a `SinkState`, and exposes that state to the registry through a `SinkCollector`. `SinkState` is shared between the timely operator (the sole writer) and `SinkCollector::collect` (the reader) via `Arc>`. Both sides hold the lock only across a short synchronous section: the operator has no `await` points and the collector clones out only what it needs before releasing the lock. + +## Key types + +**`SinkState`** — the full working and published state for one metric sink. Incoming updates are buffered by timestamp in `pending_ok`/`pending_err` and folded into `working` only once the combined ok+err frontier has closed that timestamp. `working` holds a signed multiplicity per full row identity (`RowKey`). `published` is rebuilt from the live set of `working` on each healthy activation. + +**`RowKey`** — `(metric_name, labels, value_bits, metric_kind, name_valid, help)`. The name and labels lead the tuple so that a `BTreeMap` keeps all rows of one `(metric_name, labels)` series adjacent for efficient collision detection. + +**`MetricKind`** — `Gauge` or `Counter`, recovered from the `metric_kind` column the planner's `shape_metric_sink_source` already computed (`0` = gauge, `1` = counter). + +**`SinkCollector`** — a `prometheus::core::Collector` that exposes six companion gauges (`mz_metric_sink_frontier_ms`, `mz_metric_sink_errors`, `mz_metric_sink_skipped`, `mz_metric_sink_conflicts`, `mz_metric_sink_collisions`, `mz_metric_sink_null_values`) plus the user-defined series built dynamically as `MetricFamily` protos. The companion gauges carry a `sink` const label so each sink instance has distinct `Desc` ids at registration time. + +## Collision and conflict semantics + +A *collision* is a `(metric_name, labels)` series with more than one distinct live non-null value. The winner is the row with the numerically smallest value, breaking ties by metric kind then help string. A *conflict* is a published series whose `metric_type`/`help` disagree with the family's winning type/help (determined by the lexicographically smallest label vector within the family). A series all of whose live values are null is absent from `published` and counted in `null_values`. + +## Publication freeze + +While the sink's error count (`errors`) is nonzero, `published` is not rebuilt; it stays at the last healthy snapshot. `working` continues integrating closed timestamps during the freeze so that the next healthy `publish_if_healthy` call reflects everything that happened while frozen. + +## Planner contract + +The source relation is expected to carry the seven canonical columns produced by `mz_adapter::optimize::metric_sink::shape_metric_sink_source`: `metric_name`, `metric_type`, `labels` (non-null), `value`, `help` (non-null), `metric_kind`, and `name_valid`. `ColumnIndices::resolve` panics if a required column is missing; the SQL planner enforces this contract once the `CREATE METRIC SINK` planning path exists. diff --git a/doc/developer/generated/dyncfg-launchdarkly/_crate.md b/doc/developer/generated/dyncfg-launchdarkly/_crate.md index 53d8ef527854f..19c6026e07748 100644 --- a/doc/developer/generated/dyncfg-launchdarkly/_crate.md +++ b/doc/developer/generated/dyncfg-launchdarkly/_crate.md @@ -1,6 +1,6 @@ --- source: src/dyncfg-launchdarkly/src/lib.rs -revision: 34effa9dc0 +revision: 26305d8cb0 --- # mz-dyncfg-launchdarkly diff --git a/doc/developer/generated/environmentd/_crate.md b/doc/developer/generated/environmentd/_crate.md index d13fd2b29e1ca..cbbd33954fa78 100644 --- a/doc/developer/generated/environmentd/_crate.md +++ b/doc/developer/generated/environmentd/_crate.md @@ -1,6 +1,6 @@ --- source: src/environmentd/src/lib.rs -revision: 443b75e6b4 +revision: d92a868ee9 --- # environmentd diff --git a/doc/developer/generated/environmentd/http/console.md b/doc/developer/generated/environmentd/http/console.md index 027e3b4dddb7a..ba9e4d6da7a63 100644 --- a/doc/developer/generated/environmentd/http/console.md +++ b/doc/developer/generated/environmentd/http/console.md @@ -1,6 +1,6 @@ --- source: src/environmentd/src/http/console.rs -revision: 964fd6a0ab +revision: 253293ef87 --- # environmentd::http::console diff --git a/doc/developer/generated/environmentd/http/mcp.md b/doc/developer/generated/environmentd/http/mcp.md index 335fbe005e1a8..04ef6aa47edae 100644 --- a/doc/developer/generated/environmentd/http/mcp.md +++ b/doc/developer/generated/environmentd/http/mcp.md @@ -1,13 +1,13 @@ --- source: src/environmentd/src/http/mcp.rs -revision: 11ed4b65a6 +revision: 0ee3c26409 --- # environmentd::http::mcp Implements Model Context Protocol (MCP) HTTP handlers that expose Materialize data products to AI agents via JSON-RPC 2.0. Provides two endpoints: `/api/mcp/agent` (tools: `get_data_products`, `get_data_product_details`, `read_data_product`, `query`) for user data products discovered via `mz_internal.mz_mcp_data_products`, and `/api/mcp/developer` (tools: `query_system_catalog`, `query`) for read-only access to system catalog tables in schemas from `SYSTEM_SCHEMAS` (excluding `mz_unsafe`). The developer endpoint's `query` tool accepts an optional `cluster_replica` parameter to pin execution to a named replica (e.g. for `EXPLAIN ANALYZE` on a cluster with multiple replicas); the agent endpoint's `query` tool does not expose this parameter. -Each endpoint is gated by a dynamic feature flag (`ENABLE_MCP_AGENT`, `ENABLE_MCP_DEVELOPER`, `ENABLE_MCP_AGENT_QUERY_TOOL`, `ENABLE_MCP_DEVELOPER_QUERY_TOOL`, `ENABLE_MCP_AGENT_READ_DATA_PRODUCT_TOOL`); `ENABLE_MCP_AGENT_READ_DATA_PRODUCT_TOOL` gates the `read_data_product` tool independently of the per-endpoint query tool flag. Response size is bounded by `MCP_MAX_RESPONSE_SIZE`. Request execution is bounded by `MCP_REQUEST_TIMEOUT`, a dynamic config that controls how long a tool call may run before a clean JSON-RPC timeout error is returned to the caller. +Each endpoint is gated by a dynamic feature flag (`ENABLE_MCP_AGENT`, `ENABLE_MCP_DEVELOPER`, `ENABLE_MCP_AGENT_QUERY_TOOL`, `ENABLE_MCP_DEVELOPER_QUERY_TOOL`, `ENABLE_MCP_AGENT_READ_DATA_PRODUCT_TOOL`); `ENABLE_MCP_AGENT_READ_DATA_PRODUCT_TOOL` gates the `read_data_product` tool independently of the per-endpoint query tool flag. Response size is bounded by `MCP_MAX_RESPONSE_SIZE`. Request execution is bounded by `MCP_REQUEST_TIMEOUT`, a dynamic config that controls how long a tool call may run before a clean JSON-RPC timeout error is returned to the caller. The catalog snapshot fetched at the start of each request (to read feature flags and dyncfgs) is itself bounded by the compiled-in default of `MCP_REQUEST_TIMEOUT`; if the snapshot stalls the handler returns `503 Service Unavailable` rather than hanging past the configured timeout. Each request tags the session's `application_name` with `mz_mcp_agents` or `mz_mcp_developer` (via `set_default`, so a caller-supplied value still takes precedence) to make MCP-originated sessions visible in `mz_session_history` and `mz_statement_execution_history`. Implements MCP protocol version `2025-11-25`: the `initialize` response includes a `protocolVersion` field set to `MCP_PROTOCOL_VERSION`, tool definitions carry `title`, `annotations` (`ToolAnnotations` with `readOnlyHint`, `destructiveHint`, `idempotentHint`, `openWorldHint`), and `ToolContentResult` includes an `isError` field (always `false` since tool failures surface as JSON-RPC errors rather than as tool-error content). Both endpoints' `initialize` responses include usage instructions (via `endpoint_instructions`). The developer endpoint guides AI agents to use `mz_internal.mz_ontology_*` tables for discovering table schemas, join paths, and column names before writing queries; when `ENABLE_MCP_DEVELOPER_QUERY_TOOL` is on, the instructions also advertise the `query` tool. The agent endpoint instructs agents to prefer indexed objects (served from memory) over unindexed materialized views, and that `read_data_product` automatically routes the read to the cluster recorded in the data product catalog so indexes are used. The `cluster` field in `mz_mcp_data_products` is null when the session role lacks `USAGE` on the object's index/compute cluster; in that case `read_data_product` without a cluster override runs on the session's default cluster (safe: only materialized views appear this way and they serve from persist). Instructions also tell agents to set the `cluster` parameter only when intentionally targeting a different cluster (e.g. one with larger or more replicas). The agent endpoint also instructs agents to check the `hydration` object returned by `get_data_product_details` before reading: if `hydrated` is false and `replica_count > 0` the dataflow is warming up and a read would block until it catches up; if `replica_count` is 0 the cluster has no replicas and reading cannot make progress. diff --git a/doc/developer/generated/environmentd/telemetry.md b/doc/developer/generated/environmentd/telemetry.md index c1cb247bc9a3d..d038cfb68efce 100644 --- a/doc/developer/generated/environmentd/telemetry.md +++ b/doc/developer/generated/environmentd/telemetry.md @@ -1,6 +1,6 @@ --- source: src/environmentd/src/telemetry.rs -revision: 4e2b6c0984 +revision: d92a868ee9 --- # environmentd::telemetry diff --git a/doc/developer/generated/environmentd/test_util.md b/doc/developer/generated/environmentd/test_util.md index 8bb037c08f76f..2c24b6d8c9423 100644 --- a/doc/developer/generated/environmentd/test_util.md +++ b/doc/developer/generated/environmentd/test_util.md @@ -1,10 +1,10 @@ --- source: src/environmentd/src/test_util.rs -revision: 34effa9dc0 +revision: fd1dd6e62b --- # environmentd::test_util Provides test infrastructure for integration tests against a full `environmentd` process. -Exposes `TestHarness` (builder for configuring a test server with options for data directory, TLS, authentication, cluster replica sizes, system parameters, and propagation of persist/Kafka/schema-registry URLs), `TestServerWithRuntime` (the running server plus Tokio runtime), and many helpers for connecting via PostgreSQL, HTTP, and WebSocket; for inspecting catalog state; and for controlling timing. +Exposes `TestHarness` (builder for configuring a test server with options for data directory, TLS, authentication, cluster replica sizes, builtin cluster replication factors, system parameters, and propagation of persist/Kafka/schema-registry URLs), `TestServerWithRuntime` (the running server plus Tokio runtime), and many helpers for connecting via PostgreSQL, HTTP, and WebSocket; for inspecting catalog state; and for controlling timing. Only compiled when the `test` feature is enabled. diff --git a/doc/developer/generated/expr-parser/printer.md b/doc/developer/generated/expr-parser/printer.md new file mode 100644 index 0000000000000..ab52d26a19a00 --- /dev/null +++ b/doc/developer/generated/expr-parser/printer.md @@ -0,0 +1,25 @@ +--- +source: src/expr-parser/src/printer.rs +revision: 94ee2d5448 +--- + +# mz-expr-parser::printer + +Renders a `MirScalarExpr` to text in the syntax accepted by the expr-parser's own `try_parse_scalar`. Parsing the output reconstructs an equal expression, so the module provides a round-trip guarantee for the expression types it supports. + +## Key export + +**`print_scalar(expr: &MirScalarExpr) -> Result`** — the primary entry point. Returns `Err` for datum types or error literal variants the text syntax cannot represent (for example, non-internal `EvalError` variants or scalar types not covered by `print_repr_scalar_type`). + +## Rendering rules + +- **`Column(i)`** → `#i` +- **`Literal(Ok(row), typ)`** → delegates to `print_literal`, which formats datums with explicit type casts (`1::integer`, `"hello"`, `true`, etc.). +- **`Literal(Err(Internal(msg)))`** → `error("internal error: ...")`. Other error variants are unsupported. +- **`CallUnmaterializable(func)`** → `func()` +- **`CallUnary`** — `IsNull` renders as `((expr) IS NULL)`. `RecordGet[i]` renders as `record_get[i](expr)`. `CastInt32ToNumeric` with a scale renders as `cast_int32_to_numeric[scale](expr)`. All other unary functions render as `variant_name(expr)`. +- **`CallBinary`** — comparison operators with a unique variant (`Eq`, `NotEq`, `Lt`, `Lte`, `Gt`, `Gte`) render as infix `(expr1 op expr2)`. All others render as `variant_name(expr1, expr2)`. +- **`CallVariadic`** — `And`/`Or` with at least two arguments and no nested call of the same function render as infix `(a AND b ...)` / `(a OR b ...)`. `RecordCreate` renders as `record_create["field1", ...](args)`. `ListCreate` renders as `list_create[elem_type](args)`. All others render as `variant_name(args)`. +- **`If { cond, then, els }`** → `case when cond then then else els end` + +Unlike the EXPLAIN printer, this renderer uses exact variant names (via `variant_name()`) rather than SQL operator symbols, so the round-trip through the parser is lossless. diff --git a/doc/developer/generated/expr/scalar/_module.md b/doc/developer/generated/expr/scalar/_module.md index 0d19be00ec7f0..8d2afc1483865 100644 --- a/doc/developer/generated/expr/scalar/_module.md +++ b/doc/developer/generated/expr/scalar/_module.md @@ -1,6 +1,6 @@ --- source: src/expr/src/scalar.rs -revision: 94ee2d5448 +revision: 7dcee13cec --- # mz-expr::scalar diff --git a/doc/developer/generated/expr/scalar/func/impls/jsonb.md b/doc/developer/generated/expr/scalar/func/impls/jsonb.md index 7480ad66e879a..332dd7f142885 100644 --- a/doc/developer/generated/expr/scalar/func/impls/jsonb.md +++ b/doc/developer/generated/expr/scalar/func/impls/jsonb.md @@ -1,9 +1,9 @@ --- source: src/expr/src/scalar/func/impls/jsonb.rs -revision: 94ee2d5448 +revision: fd1dd6e62b --- # mz-expr::scalar::func::impls::jsonb Provides scalar function implementations for `jsonb` datums: subscript operators, `jsonb_array_elements`, `jsonb_each`, `jsonb_object_keys`, type-coercion casts, `jsonb_build_object`, `jsonb_build_array`, `jsonb_pretty`, and containment checks. -Also provides `parse_catalog_id` (converts catalog JSON-serialized IDs into string format like `u1`, `s2`, `p`), `parse_catalog_privileges` (converts catalog JSON-serialized privilege arrays into `mz_aclitem[]`), `parse_catalog_acl_mode` (converts a catalog JSON-serialized `AclMode` bitflags object of shape `{"bitflags": }` into a PostgreSQL ACL char-code string, e.g. `{"bitflags": 514}` → `"ar"`), `parse_catalog_create_sql` (parses a catalog `create_sql` string into a JSONB object with type and additional fields depending on the statement: for connections, a `connection_type` field; for materialized views, `cluster_id` and `definition`; for sources (`CreateSource`), `source_type`, `cluster_id`, `connection_id`, `envelope_type`, `key_format`, and `value_format`; for webhook sources, `source_type` and `cluster_id`; for subsources, `source_type` and `of_source_id`), `parse_postgres_source_details` (decodes a Postgres source `create_sql` to extract `slot` and `timeline_id` from the hex-encoded `DETAILS` proto option, returning `{"slot": , "timeline_id": }`; returns JSON `null` for non-Postgres-source statements), and `parse_kafka_source_details` (extracts `topic`, `group_id_prefix`, and `connection_id` from a Kafka source `create_sql`, returning `{"topic": , "group_id_prefix": , "connection_id": }`; returns JSON `null` for non-Kafka-source statements). Helper functions include `jsonb_datum_to_u64`, `jsonb_datum_to_acl_mode` (shared decoder for `AclMode` objects used by both `parse_catalog_privileges` and `parse_catalog_acl_mode`), and `jsonb_datum_to_role_id`. The `PostgresPublicationDetailsSubset` struct is a local prost message that re-declares the two proto tags needed (`slot` at tag 2, `timeline_id` at tag 3) to avoid a dependency cycle with `mz-storage-types`. +Also provides `parse_catalog_id` (converts catalog JSON-serialized IDs into string format like `u1`, `s2`, `p`), `parse_catalog_privileges` (converts catalog JSON-serialized privilege arrays into `mz_aclitem[]`), `parse_catalog_acl_mode` (converts a catalog JSON-serialized `AclMode` bitflags object of shape `{"bitflags": }` into a PostgreSQL ACL char-code string, e.g. `{"bitflags": 514}` → `"ar"`), `parse_catalog_create_sql` (parses a catalog `create_sql` string into a JSONB object with type and additional fields depending on the statement: for connections, a `connection_type` field; for materialized views, `cluster_id` and `definition`; for sources (`CreateSource`), `source_type`, `cluster_id`, `connection_id`, `envelope_type`, `key_format`, and `value_format`; for webhook sources, `source_type` and `cluster_id`; for subsources, `source_type` and `of_source_id`), `parse_postgres_source_details` (decodes a Postgres source `create_sql` to extract `slot` and `timeline_id` from the hex-encoded `DETAILS` proto option, returning `{"slot": , "timeline_id": }`; returns JSON `null` for non-Postgres-source statements), and `parse_kafka_source_details` (extracts `topic`, `group_id_prefix`, and `connection_id` from a Kafka source `create_sql`, returning `{"topic": , "group_id_prefix": , "connection_id": }`; returns JSON `null` for non-Kafka-source statements), and `parse_source_export_details` (extracts source-export metadata from a `CREATE TABLE ... FROM SOURCE` or non-progress `CREATE SUBSOURCE ... OF SOURCE ...` statement, returning `{"source_id": , "external_reference": [, ...], "envelope_type": , "key_format": , "value_format": }`; `envelope_type`, `key_format`, and `value_format` are always null for `CREATE SUBSOURCE` statements; returns JSON `null` for progress subsources and any non-source-export statement). Helper functions include `jsonb_datum_to_u64`, `jsonb_datum_to_acl_mode` (shared decoder for `AclMode` objects used by both `parse_catalog_privileges` and `parse_catalog_acl_mode`), and `jsonb_datum_to_role_id`. The `PostgresPublicationDetailsSubset` struct is a local prost message that re-declares the two proto tags needed (`slot` at tag 2, `timeline_id` at tag 3) to avoid a dependency cycle with `mz-storage-types`. diff --git a/doc/developer/generated/expr/scalar/func/impls/timestamp.md b/doc/developer/generated/expr/scalar/func/impls/timestamp.md index 99b61e868a841..2e4f0f87ff157 100644 --- a/doc/developer/generated/expr/scalar/func/impls/timestamp.md +++ b/doc/developer/generated/expr/scalar/func/impls/timestamp.md @@ -1,8 +1,8 @@ --- source: src/expr/src/scalar/func/impls/timestamp.rs -revision: 94ee2d5448 +revision: 38293e9a93 --- # mz-expr::scalar::func::impls::timestamp -Provides scalar function implementations for `timestamp` and `timestamptz` datums: casts to/from string, date, time, numeric epoch, and between the two variants; interval arithmetic; `date_trunc`; time-zone conversions; and date-part extraction. +Provides scalar function implementations for `timestamp` and `timestamptz` datums: casts to/from string, date, time, and between the two variants; precision adjustment (`AdjustTimestampPrecision`, `AdjustTimestampTzPrecision`); `to_char` formatting (`ToCharTimestamp`, `ToCharTimestampTz`); interval arithmetic; `date_trunc`; time-zone conversions; and date-part extraction. diff --git a/doc/developer/generated/expr/scalar/func/unary.md b/doc/developer/generated/expr/scalar/func/unary.md index e9b4c6d10d554..6fd8c885edfd1 100644 --- a/doc/developer/generated/expr/scalar/func/unary.md +++ b/doc/developer/generated/expr/scalar/func/unary.md @@ -1,6 +1,6 @@ --- source: src/expr/src/scalar/func/unary.rs -revision: 34813f0a3f +revision: fd1dd6e62b --- # mz-expr::scalar::func::unary @@ -8,5 +8,5 @@ revision: 34813f0a3f Defines the `LazyUnaryFunc` and `EagerUnaryFunc` traits, and uses `derive_unary\!` to generate the `UnaryFunc` enum. `LazyUnaryFunc` is the core trait all unary scalar functions must implement (either directly or via the blanket impl from `EagerUnaryFunc`); it requires `eval`, `output_sql_type`, `propagates_nulls`, `introduces_nulls`, `preserves_uniqueness`, `inverse`, `is_monotone`, `is_eliminable_cast`, and optionally `could_error`. `EagerUnaryFunc` is a higher-level trait using associated `Input`/`Output` types that implement `InputDatumType`/`OutputDatumType`; null propagation, null introduction, and error properties are inferred automatically from the type signatures. A blanket `impl LazyUnaryFunc for T` bridges the two traits. -The `derive_unary\!` invocation includes variants for all built-in unary functions, including `ParseCatalogId`, `ParseCatalogPrivileges`, `ParseCatalogAclMode`, `ParseCatalogCreateSql`, and `ParseCatalogAuditLogDetails` for catalog JSON deserialization, `ParseKafkaSourceDetails` and `ParsePostgresSourceDetails` for extracting source-specific fields from catalog `create_sql` strings, and `RedactSql` for SQL redaction. +The `derive_unary\!` invocation includes variants for all built-in unary functions, including `ParseCatalogId`, `ParseCatalogPrivileges`, `ParseCatalogAclMode`, `ParseCatalogCreateSql`, and `ParseCatalogAuditLogDetails` for catalog JSON deserialization, `ParseKafkaSourceDetails` and `ParsePostgresSourceDetails` for extracting source-specific fields from catalog `create_sql` strings, `ParseSourceExportDetails` for extracting source-export metadata (parent source ID, external reference, envelope, and key/value formats) from a catalog `create_sql` string, and `RedactSql` for SQL redaction. These properties drive null-propagation, type inference, and optimizer rewrites throughout the system. diff --git a/doc/developer/generated/expr/scalar/like_pattern.md b/doc/developer/generated/expr/scalar/like_pattern.md index 1b6140a323573..eead7d713883e 100644 --- a/doc/developer/generated/expr/scalar/like_pattern.md +++ b/doc/developer/generated/expr/scalar/like_pattern.md @@ -1,6 +1,6 @@ --- source: src/expr/src/scalar/like_pattern.rs -revision: 94ee2d5448 +revision: d14129847b --- # mz-expr::scalar::like_pattern diff --git a/doc/developer/generated/frontegg-mock/server.md b/doc/developer/generated/frontegg-mock/server.md index 4b78f4ed19d2b..9f558f3a50c63 100644 --- a/doc/developer/generated/frontegg-mock/server.md +++ b/doc/developer/generated/frontegg-mock/server.md @@ -1,6 +1,6 @@ --- source: src/frontegg-mock/src/server.rs -revision: 88b6028c79 +revision: 253293ef87 --- # frontegg-mock::server diff --git a/doc/developer/generated/interchange/envelopes.md b/doc/developer/generated/interchange/envelopes.md index a2f273e402862..125fdfc68d5de 100644 --- a/doc/developer/generated/interchange/envelopes.md +++ b/doc/developer/generated/interchange/envelopes.md @@ -1,13 +1,14 @@ --- source: src/interchange/src/envelopes.rs -revision: 2d842fe6d1 +revision: cd0f3534ba --- # interchange::envelopes -Provides two functions for walking arrangement batches and emitting `DiffPair`s: +Provides three functions for walking arrangement batches and emitting `DiffPair`s: * `for_each_diff_pair` — walks a batch and invokes a callback for each `DiffPair` at each `(key, timestamp)`. Thin wrapper around `iter_diff_pairs`. +* `for_each_diff_pair_async` — async, fallible variant of `for_each_diff_pair`. Accepts optional inclusive lower and exclusive upper time bounds and invokes an async fallible callback for each pair. Thin wrapper around `iter_diff_pairs`. * `iter_diff_pairs` — walks a batch and emits, for each key, the `DiffPair`s at each timestamp. Accepts optional inclusive lower and exclusive upper time bounds; updates outside the range are ignored. Keys with no updates in range are omitted entirely. The per-key iterator owns its data and can be held or consumed after the outer iterator has advanced. An update with multiplicity `n` fans out into `n` pairs lazily, cloning the value as pairs are consumed; memory held per key is proportional to the number of distinct updates, not the fan-out. No ordering is guaranteed across keys. Both functions are generic over a batch type `B` (bounded by `BatchReader