[pull] main from MaterializeInc:main

bin/unused-deps

@@ -11,18 +11,4 @@
 #
 # unused-deps — prints unused dependencies for all Rust crates.
 
-# Requires https://github.com/est31/cargo-udeps and a working nightly toolchain.
-#
-# cargo-udeps occasionally produces false positives, especially with
-# dependencies that are used only on certain platforms. To ignore these
-# dependencies, see:
-# https://github.com/est31/cargo-udeps#ignoring-some-of-the-dependencies.
-
-set -euo pipefail
-
-cd "$(dirname "$0")/.."
-
-. misc/shlib/shlib.bash
-
-export RUSTUP_TOOLCHAIN=${RUSTUP_TOOLCHAIN:=nightly}
-cargo udeps --workspace --all-targets "$@"
+exec "$(dirname "$0")"/pyactivate -m materialize.cli.unused-deps "$@"

ci/mkpipeline.py

@@ -1238,7 +1238,7 @@ def add_cargo_test_dependency(
         return
 
     for step in steps(pipeline):
-        if step.get("id") in ("cargo-test", "miri-test"):
+        if step.get("id") == "cargo-test":
             step["depends_on"] = (
                 "build-x86_64" if "x86" in step["agents"]["queue"] else "build-aarch64"
             )

ci/nightly/pipeline.template.yml

@@ -107,7 +107,7 @@ steps:
 
       - id: unused-deps
         label: Unused dependencies
-        command: bin/ci-builder run nightly bin/unused-deps
+        command: bin/ci-builder run stable bin/unused-deps
         inputs:
           - Cargo.lock
           - Cargo.toml
@@ -163,7 +163,7 @@ steps:
   - id: miri-test
     topics: [kafka, mysql, postgres]
     label: ":rust: Miri test (full)"
-    depends_on: build-aarch64
+    depends_on: []
     timeout_in_minutes: 60
     plugins:
       - ./ci/plugins/mzcompose:
@@ -587,7 +587,7 @@ steps:
   - id: ssh-connection-extended
     topics: [kafka, kafka-sink, mysql, postgres, sql-server, ssh-tunnel]
     label: Extended SSH connection
-    depends_on: build-aarch64
+    depends_on: build-x86_64
     timeout_in_minutes: 45
     plugins:
       - ./ci/plugins/mzcompose:

ci/test/cargo-test/mzcompose.py

@@ -123,6 +123,19 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
     if os.path.exists(junit_path):
         os.remove(junit_path)
 
+    coverage = ui.env_is_truthy("CI_COVERAGE_ENABLED")
+
+    # Miri excludes the network-based tests, so it needs none of the services
+    # below and pulls none of their mzbuild images. Dispatch before `c.up()` so
+    # the Buildkite step has nothing to depend on the build for.
+    if not coverage:
+        if args.miri_full:
+            run_miri_slow(dict(os.environ))
+            return
+        if args.miri_fast:
+            run_miri_fast(dict(os.environ))
+            return
+
     c.up(
         "kafka",
         "schema-registry",
@@ -168,7 +181,6 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
         FDB_CLUSTER_FILE=fdb_cluster_file.name,
     )
 
-    coverage = ui.env_is_truthy("CI_COVERAGE_ENABLED")
     sanitizer = Sanitizer[os.getenv("CI_SANITIZER", "none")]
 
     metadata = json.loads(
@@ -179,10 +191,6 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None:
 
     if coverage:
         run_coverage_test(args, env)
-    elif args.miri_full:
-        run_miri_slow(env)
-    elif args.miri_fast:
-        run_miri_fast(env)
     elif sanitizer != Sanitizer.none:
         run_sanitizer(args, env, metadata, sanitizer)
     else:

deny.toml

@@ -301,6 +301,8 @@ ignore = [
     "RUSTSEC-2026-0195",
     # `bitmaps` is unmaintained
     "RUSTSEC-2026-0247",
+    # `aws-sdk-s3` and `launchdarkly-server-sdk` both require `lru ^0.16.3` in their latest releases
+    "RUSTSEC-2026-0253",
 ]
 
 # Must be manually kept in sync with about.toml.

doc/developer/generated/adapter/catalog/_module.md

@@ -1,6 +1,6 @@
 ---
 source: src/adapter/src/catalog.rs
-revision: a60edac7f1
+revision: fd1dd6e62b
 ---
 
 # adapter::catalog

doc/developer/generated/adapter/catalog/apply.md

@@ -1,6 +1,6 @@
 ---
 source: src/adapter/src/catalog/apply.rs
-revision: fca741734d
+revision: b9097f8a3d
 ---
 
 # adapter::catalog::apply

doc/developer/generated/adapter/catalog/builtin_table_updates.md

@@ -1,6 +1,6 @@
 ---
 source: src/adapter/src/catalog/builtin_table_updates.rs
-revision: 34813f0a3f
+revision: fd1dd6e62b
 ---
 
 # adapter::catalog::builtin_table_updates
@@ -10,5 +10,5 @@ For each catalog object type, a corresponding `pack_*` function converts a `Cata
 `BuiltinTableUpdate` wraps either a `Row`-level update or a `ProtoBatch` for bulk ingestion, both resolved to a `CatalogItemId` before being written.
 `pack_role_auth_update` writes a row into `mz_role_auth` containing the role's password hash and the timestamp it was last updated. `mz_roles`, `mz_role_parameters`, `mz_clusters`, `mz_cluster_replicas`, `mz_cluster_schedules`, `mz_default_privileges`, `mz_system_privileges`, and `mz_internal.mz_comments` are materialized views backed by `mz_internal.mz_catalog_raw`; rows for these objects are not produced by `pack_*` functions here.
 `MZ_CLUSTER_REPLICA_SIZE_INTERNAL` (a `BuiltinTable`) receives rows via `pack_cluster_replica_size_update` and retains entries for disabled sizes; the `mz_cluster_replicas` MV LEFT JOINs it to resolve the `disk` column.
-`CatalogItem::Secret`, `CatalogItem::Log`, `CatalogItem::Connection`, and `CatalogItem::Index` do not produce direct table updates for `MZ_SECRETS`, `MZ_SOURCES`, `MZ_CONNECTIONS`, or `MZ_INDEXES` from `pack_*` functions; `mz_connections`, `mz_secrets`, and `mz_sources` are backed by catalog views or materialized views rather than builtin tables, so `CatalogItem::Secret` and `CatalogItem::Log` return empty update vecs and `pack_connection_update` only emits rows for the connection-type-specific subtables (e.g. `MZ_KAFKA_CONNECTIONS`, `MZ_SSH_TUNNEL_CONNECTIONS`). `ConnectionDetails::GlueSchemaRegistry` and `ConnectionDetails::Gcp` are handled in `pack_connection_update` alongside `Csr`, `Postgres`, `MySql`, and `SqlServer` as types that do not produce their own subtable row. `CatalogItem::Source` does not produce rows into `MZ_SOURCES` (a materialized view over `mz_catalog_raw`), `mz_postgres_sources` (a materialized view), or `mz_kafka_sources` (a materialized view); for ingestion sources, `pack_item_update` returns an empty vec. Sources only emit rows for source-type-specific subtables such as postgres/kafka source-table mappings and subsource/progress metadata. `CatalogItem::Index` does not produce rows into `MZ_INDEXES` (a materialized view); `pack_index_update` does not write to `MZ_INDEXES`.
+`CatalogItem::Secret`, `CatalogItem::Log`, `CatalogItem::Connection`, and `CatalogItem::Index` do not produce direct table updates from `pack_*` functions. `CatalogItem::Secret` and `CatalogItem::Log` return empty update vecs. `CatalogItem::Connection` returns an empty vec because all connection-detail subtables (`mz_kafka_connections`, `mz_ssh_tunnel_connections`, `mz_aws_connections`, `mz_aws_privatelink_connections`) are now materialized views over `mz_catalog_raw` derived from each connection's persisted `create_sql`. `CatalogItem::Source` does not produce rows into `MZ_SOURCES` (a materialized view over `mz_catalog_raw`), `mz_postgres_sources` (a materialized view), or `mz_kafka_sources` (a materialized view); for ingestion sources, `pack_item_update` returns an empty vec. Source-type subtable details such as `mz_postgres_source_tables`, `mz_mysql_source_tables`, `mz_sql_server_source_tables`, and `mz_kafka_source_tables` are also materialized views over `mz_catalog_raw`, so ingestion exports and old-syntax subsources do not emit rows from `pack_*` functions either. Sources only emit rows for webhook sources and source-specific progress/subsource metadata. `CatalogItem::Index` does not produce rows into `MZ_INDEXES` (a materialized view); `pack_index_update` does not write to `MZ_INDEXES`.
 The `notice` sub-module handles the special `mz_recent_activity_log` and related notice tables separately.

doc/developer/generated/adapter/catalog/open.md

@@ -1,13 +1,14 @@
 ---
 source: src/adapter/src/catalog/open.rs
-revision: fca741734d
+revision: 30210e9ef7
 ---
 
 # adapter::catalog::open
 
 Implements `Catalog::open`, the full catalog initialisation sequence run at environment startup.
 The function opens the durable catalog store, runs AST migrations (`catalog::migrate`), applies all persisted `StateUpdate` diffs to build the in-memory `CatalogState`, bootstraps built-in objects (schemas, roles, clusters, tables, views, materialized views, functions), runs builtin-schema migrations (see `open::builtin_schema_migration`), and returns a ready `Catalog` along with the initial builtin-table updates to write.
+After applying all updates, `state.system_config().sync_dyncfgs()` is called explicitly to mirror the effective `SystemVars` values into the dyncfg `ConfigSet`; `apply_updates` only syncs when a durable `SystemConfiguration` update is present, so a deployment configured purely via `system_parameter_default` would otherwise leave the `ConfigSet` at compile-time defaults for any startup-only read.
 `add_new_remove_old_builtin_items_migration` drops comments under every relation-style `CommentObjectId` variant (Table, View, MaterializedView, Source) for a given id, not just the current one. This handles cases where a builtin's type changes (e.g., Table to MaterializedView) but its catalog id is preserved.
 `add_new_remove_old_builtin_clusters_migration` emits audit log events when creating or dropping builtin clusters so those operations are visible in `mz_audit_events`.
-`add_new_remove_old_builtin_cluster_replicas_migration` allocates builtin replica IDs via `txn.allocate_system_replica_id()` within the open transaction (single-source, safe because there is no coordinator at that point) and emits audit log events when creating or dropping builtin cluster replicas so those operations are visible in `mz_audit_events`.
+`reconcile_builtin_cluster_replicas` reconciles each builtin cluster's replica set against the cluster's own durable `replication_factor`, creating missing replicas (allocating IDs via `txn.allocate_system_replica_id()`, single-source and safe because there is no coordinator at that point) and dropping surplus ones. Bootstrap flags seed `size` and `replication_factor` only when a cluster is first created, so an `ALTER CLUSTER` against a builtin cluster persists across restarts. Audit log events are emitted for every create and drop so those operations are visible in `mz_audit_events`.
 `remove_pending_cluster_replicas_migration` emits audit log events when dropping pending replicas so the drops are visible in `mz_audit_events`.

doc/developer/generated/adapter/catalog/open/_module.md

@@ -1,6 +1,6 @@
 ---
 source: src/adapter/src/catalog/open.rs
-revision: fca741734d
+revision: 30210e9ef7
 ---
 
 # adapter::catalog::open

doc/developer/generated/adapter/catalog/open/builtin_schema_migration.md

@@ -1,6 +1,6 @@
 ---
 source: src/adapter/src/catalog/open/builtin_schema_migration.rs
-revision: 34813f0a3f
+revision: fd1dd6e62b
 ---
 
 # adapter::catalog::open::builtin_schema_migration
@@ -15,6 +15,7 @@ A `Replacement` step at version `26.31.0-dev.0` covers `mz_catalog.mz_cluster_re
 A `Replacement` step at version `26.32.0-dev.0` covers `mz_internal.mz_comments`, reflecting its conversion from `BuiltinTable` to `BuiltinMaterializedView`. A second `Replacement` step at version `26.32.0-dev.0` covers `mz_catalog.mz_indexes`, required because adding the console cluster utilization overview builtin indexes changed the VALUES set inlined by `make_mz_indexes`, altering the MV's SQL fingerprint.
 A `Replacement` step at version `26.33.0-dev.0` covers `mz_catalog.mz_audit_events`, reflecting its conversion from `BuiltinTable` to `BuiltinMaterializedView`.
 Three `Replacement` steps at version `26.34.0-dev.0` cover: `mz_catalog.mz_indexes` (because adding the `mz_cluster_reconfigurations_ind` and `mz_cluster_auto_scaling_strategies_ind` builtin indexes changes the VALUES set inlined by `make_mz_indexes`, altering the MV's SQL fingerprint); `mz_internal.mz_postgres_sources` (reflecting its conversion from `BuiltinTable` to `BuiltinMaterializedView`); and `mz_catalog.mz_kafka_sources` (reflecting the same conversion).
+Eight `Replacement` steps at version `26.37.0-dev.0` cover: `mz_internal.mz_postgres_source_tables`, `mz_internal.mz_mysql_source_tables`, `mz_internal.mz_sql_server_source_tables`, and `mz_internal.mz_kafka_source_tables` (reflecting their conversion from `BuiltinTable` to `BuiltinMaterializedView`); and `mz_catalog.mz_kafka_connections`, `mz_catalog.mz_ssh_tunnel_connections`, `mz_internal.mz_aws_connections`, and `mz_catalog.mz_aws_privatelink_connections` (reflecting their conversion from `BuiltinTable` to `BuiltinMaterializedView`, where each is now derived from its connection's persisted `create_sql` via `mz_catalog_raw`).
 When applying replacement migrations, `mz_storage_usage_by_shard` is excluded from data-destroying replacement plans to preserve billing data.
 When the source and target versions differ and the source version is a dev build, `Migration::run` forces evolution-mode migration even without an explicit `force_migration` config, avoiding version-based filter failures in dev environments.
 In forced dev-to-dev migrations, `migrate_evolve_one` skips builtins that do not yet have a shard registered. Brand-new builtins in a given version may not have their shards allocated until the leader completes bootstrap; excluding them avoids "missing shard ID" errors on read-only replicas.

doc/developer/generated/adapter/catalog/state.md

@@ -1,6 +1,6 @@
 ---
 source: src/adapter/src/catalog/state.rs
-revision: 3713eed996
+revision: efe0c63d45
 ---
 
 # adapter::catalog::state

doc/developer/generated/adapter/catalog/transact.md

@@ -1,6 +1,6 @@
 ---
 source: src/adapter/src/catalog/transact.rs
-revision: 6eeaca032b
+revision: 38a95cefe2
 ---
 
 # adapter::catalog::transact

doc/developer/generated/adapter/client.md

@@ -1,6 +1,6 @@
 ---
 source: src/adapter/src/client.rs
-revision: 443b75e6b4
+revision: ff68428395
 ---
 
 # adapter::client
@@ -17,4 +17,5 @@ During `startup`, `Client` constructs a `PeekClient` from the `StartupResponse`
 `RecordFirstRowStream` is an adapter stream that records the timestamp of the first row for metrics.
 `TimeoutType` enumerates session-level timeouts (currently `IdleInTransactionSession`); the `Timeout` struct manages active timeout tasks and delivers expired timeouts through `SessionClient::recv_timeout`.
 `SessionClient::statement_arrival_logging_enabled` checks the `enable_statement_arrival_logging` system variable via the session's catalog snapshot.
+`SessionClient::extended_protocol_implicit_transaction_enabled` checks the `enable_extended_protocol_implicit_transaction` system variable via the session's catalog snapshot; pgwire consults this when deciding whether an implicit write transaction in the extended-protocol pipeline should span to the Sync message.
 `redact_sql_for_logging(sql) -> String` parses SQL and re-serialises it with literals redacted (the same redaction the statement log applies). If the text does not parse or exceeds the statement batch size limit, a placeholder with the byte length is returned instead.

doc/developer/generated/adapter/command.md

@@ -1,6 +1,6 @@
 ---
 source: src/adapter/src/command.rs
-revision: 5dc2cf510a
+revision: ff68428395
 ---
 
 # adapter::command

doc/developer/generated/adapter/config/frontend.md

@@ -1,6 +1,6 @@
 ---
 source: src/adapter/src/config/frontend.rs
-revision: 34effa9dc0
+revision: 26305d8cb0
 ---
 
 # adapter::config::frontend