[pull] main from MaterializeInc:main

AGENTS.md

@@ -99,10 +99,10 @@ Two files control license policy, **keep in sync**: `deny.toml` (`[licenses].all
   we need to record that knowledge. In general comments need to stand on their
   own and make sense from just looking at them and the code around it, not
   previous changes.
-* Avoid em-dashes and semicolons for structuring sentences, everywhere: code
-  comments, specs, design docs, all of it. Restructure with full stops and
-  commas instead. In most cases a sentence that wants an em-dash or semicolon
-  can just be split into two.
+* Avoid em-dashes for structuring sentences in any prose: code comments, specs,
+  design docs, all of it. Restructure with full stops and commas instead. The
+  same goes for semicolons, though one is fine where splitting would mangle the
+  sentence.
 * Our guidance applies both when writing new code or designs, or when we notice
   deviations in code or architecture that we are working on. At the same time,
   we want to keep our changes minimal so it's good to call out deviations and

ci/mkpipeline.py

@@ -761,7 +761,8 @@ def set_retry_on_agent_lost(pipeline: Any) -> None:
         retry.setdefault("automatic", []).extend(
             [
                 {
-                    "exit_status": -1,  # Agent lost or job timed out during checkout/setup
+                    "exit_status": -1,
+                    "signal_reason": "none",
                     "limit": 2,
                 },
                 {

ci/plugins/mzcompose/hooks/command

@@ -404,7 +404,7 @@ cleanup() {
     && [ "$BUILDKITE_LABEL" != "QA Canary Environment Base Load" ] \
     && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Canary Environment" ] \
     && [ "$BUILDKITE_LABEL" != "Parallel Benchmark against QA Benchmarking Staging Environment" ] \
-    && [ "$BUILDKITE_LABEL" != "Miri test (full)" ] \
+    && [ "$BUILDKITE_LABEL" != ":rust: Miri test (full)" ] \
     && [[ ! "$BUILDKITE_LABEL" =~ Terraform\ .* ]] \
     && [[ ! "$BUILDKITE_LABEL" =~ Orchestratord\ .* ]] \
     && [[ ! "$BUILDKITE_LABEL" =~ Cluster\ spec\ sheet.* ]]; then

console/src/platform/shell/Tutorial.tsx

@@ -53,7 +53,7 @@ import TutorialInsertionWidget from "./TutorialInsertionWidget";
 import TutorialSchemaWidget from "./TutorialSchemaWidget";
 
 /* Semantic version number used for instrumentation */
-const QUICKSTART_VERSION = "2.0.0";
+const QUICKSTART_VERSION = "2.1.0";
 
 type RunnableProps = {
   runCommand: (value: string) => void;
@@ -826,7 +826,7 @@ WHERE w2.amount > w1.amount
                 runCommand={runCommand}
                 value="CREATE INDEX wins_by_item_seller ON winning_bids (item, seller);
 CREATE INDEX wins_by_item_buyer ON winning_bids (item, buyer);"
-                title="View a sample row in flip_activities"
+                title="Create indexes on the winning_bids join keys"
               />
             </RunnableContainer>
             <TextContainer>

doc/user/content/integrations/mz-debug/_index.md

@@ -19,9 +19,8 @@ disable_list: true
 {{< tab "macOS" >}}
 
 ```shell
-ARCH=$(uname -m)
 sudo echo "Preparing to extract mz-debug..."
-curl -L "https://binaries.materialize.com/mz-debug-latest-$ARCH-apple-darwin.tar.gz" \
+curl -L "https://binaries.materialize.com/mz-debug-latest-arm64-apple-darwin.tar.gz" \
 | sudo tar -xzC /usr/local --strip-components=1
 ```
 {{</ tab >}}

doc/user/content/sql/functions/_index.md

@@ -113,7 +113,9 @@ Operator | Computes
 The regular expression syntax supported by Materialize is documented by the
 [Rust `regex` crate](https://docs.rs/regex/*/#syntax).
 The maximum length of a regular expression is 1 MiB in its raw form, and 10 MiB
-after compiling it.
+after compiling it. A regular expression may also contain at most 2000 character
+classes, counting each Unicode, Perl, or POSIX class such as `\p{L}`, `\d`, or
+`[[:alpha:]]`, and each range such as `a-z`.
 
 {{< warning >}}
 Materialize regular expressions are similar to, but not identical to, PostgreSQL

misc/dbt-materialize/CHANGELOG.md

@@ -10,7 +10,8 @@
 * Fix `deploy_init` when the `CI_TAG` environment variable is set. The
   deployment schema was tagged without passing the schema name, so the
   operation failed with `COMMENT ON SCHEMA ""`. The deployment schema is
-  now also created with a quoted name, matching how it is dropped.
+  now also created with a quoted name, matching how it is dropped and how its
+  grants and default privileges are copied.
 
 ## 1.9.11 - 2026-07-26
 

misc/dbt-materialize/dbt/include/materialize/macros/deploy/deploy_init.sql

@@ -170,7 +170,7 @@ SELECT
     WHEN object_owner = 'PUBLIC' THEN 'FOR ALL ROLES '
     ELSE 'FOR ROLE ' || quote_ident(object_owner) || ' '
   END ||
-  'IN SCHEMA {{ to }} '         ||
+  'IN SCHEMA ' || quote_ident({{ dbt.string_literal(to) }}) || ' ' ||
   'GRANT '  || privilege_type   || ' ' ||
   'ON '     || object_type      || 's ' ||
   CASE

misc/dbt-materialize/tests/adapter/test_deploy.py

@@ -64,14 +64,18 @@ def cleanup(self, project):
             project.run_sql("DROP ROLE IF EXISTS my_role")
         project.run_sql("DROP SCHEMA IF EXISTS blue_schema CASCADE")
         project.run_sql("DROP SCHEMA IF EXISTS green_schema CASCADE")
+        project.run_sql('DROP SCHEMA IF EXISTS "Green_Schema" CASCADE')
         project.run_sql("DROP CLUSTER IF EXISTS blue_cluster CASCADE")
         project.run_sql("DROP CLUSTER IF EXISTS green_cluster CASCADE")
 
-    def test_apply_schema_default_privileges(self, project):
+    # `deploy_init` creates the deployment schema with a quoted name, so a
+    # non-lowercase name has to survive into the generated statements too.
+    @pytest.mark.parametrize("to_schema", ["green_schema", "Green_Schema"])
+    def test_apply_schema_default_privileges(self, project, to_schema):
         project.run_sql("CREATE ROLE my_role")
         project.run_sql("GRANT my_role TO materialize")
         project.run_sql("CREATE SCHEMA blue_schema")
-        project.run_sql("CREATE SCHEMA green_schema")
+        project.run_sql(f'CREATE SCHEMA "{to_schema}"')
         project.run_sql(
             "ALTER DEFAULT PRIVILEGES FOR ROLE my_role IN SCHEMA blue_schema GRANT SELECT ON TABLES TO my_role"
         )
@@ -81,15 +85,15 @@ def test_apply_schema_default_privileges(self, project):
                 "run-operation",
                 "internal_copy_schema_default_privs",
                 "--args",
-                "{from: blue_schema, to: green_schema}",
+                f"{{from: blue_schema, to: {to_schema}}}",
             ]
         )
 
         result = project.run_sql(
-            """SELECT count(*) = 1
+            f"""SELECT count(*) = 1
              FROM mz_internal.mz_show_default_privileges
              WHERE database = current_database()
-                AND schema = 'green_schema'
+                AND schema = '{to_schema}'
                 AND grantee = 'my_role'
                 AND object_type = 'table'
                 AND privilege_type = 'SELECT'""",

misc/python/materialize/checks/all_checks/builtin_version_pin.py

@@ -36,7 +36,7 @@
 _STABLE_BUILTINS: dict[str, MzVersion | None] = {
     "mz_array_types": None,
     "mz_audit_events": MzVersion.parse_mz("v26.33.0"),
-    "mz_aws_privatelink_connections": None,
+    "mz_aws_privatelink_connections": MzVersion.parse_mz("v26.37.0"),
     "mz_base_types": None,
     "mz_cluster_replica_sizes": None,
     "mz_cluster_replicas": MzVersion.parse_mz("v26.30.0"),
@@ -45,17 +45,17 @@
     "mz_default_privileges": MzVersion.parse_mz("v26.30.0"),
     "mz_egress_ips": None,
     "mz_functions": None,
-    "mz_iceberg_sinks": None,
+    "mz_iceberg_sinks": MzVersion.parse_mz("v26.38.0"),
     "mz_index_columns": None,
-    "mz_kafka_connections": None,
-    "mz_kafka_sinks": None,
+    "mz_kafka_connections": MzVersion.parse_mz("v26.37.0"),
+    "mz_kafka_sinks": MzVersion.parse_mz("v26.38.0"),
     "mz_kafka_sources": MzVersion.parse_mz("v26.34.0"),
     "mz_list_types": None,
     "mz_map_types": None,
     "mz_operators": None,
     "mz_pseudo_types": None,
-    "mz_sinks": None,
-    "mz_ssh_tunnel_connections": None,
+    "mz_sinks": MzVersion.parse_mz("v26.38.0"),
+    "mz_ssh_tunnel_connections": MzVersion.parse_mz("v26.37.0"),
     "mz_system_privileges": MzVersion.parse_mz("v26.30.0"),
     "mz_tables": None,
     "mz_types": None,

src/adapter/src/catalog/builtin_table_updates.rs

@@ -17,16 +17,16 @@ use mz_catalog::SYSTEM_CONN_ID;
 use mz_catalog::builtin::{
     BuiltinTable, MZ_AGGREGATES, MZ_ARRAY_TYPES, MZ_BASE_TYPES, MZ_CLUSTER_REPLICA_SIZE_INTERNAL,
     MZ_CLUSTER_REPLICA_SIZES, MZ_COLUMNS, MZ_EGRESS_IPS, MZ_FUNCTIONS,
-    MZ_HISTORY_RETENTION_STRATEGIES, MZ_ICEBERG_SINKS, MZ_INDEX_COLUMNS, MZ_KAFKA_SINKS,
-    MZ_LICENSE_KEYS, MZ_LIST_TYPES, MZ_MAP_TYPES, MZ_MATERIALIZED_VIEW_REFRESH_STRATEGIES,
-    MZ_OBJECT_DEPENDENCIES, MZ_OBJECT_GLOBAL_IDS, MZ_OPERATORS, MZ_PSEUDO_TYPES, MZ_REPLACEMENTS,
-    MZ_ROLE_AUTH, MZ_SESSIONS, MZ_SINKS, MZ_SOURCE_REFERENCES, MZ_STORAGE_USAGE_BY_SHARD,
-    MZ_SUBSCRIPTIONS, MZ_TABLES, MZ_TYPE_PG_METADATA, MZ_TYPES, MZ_VIEWS, MZ_WEBHOOKS_SOURCES,
+    MZ_HISTORY_RETENTION_STRATEGIES, MZ_INDEX_COLUMNS, MZ_LICENSE_KEYS, MZ_LIST_TYPES,
+    MZ_MAP_TYPES, MZ_MATERIALIZED_VIEW_REFRESH_STRATEGIES, MZ_OBJECT_DEPENDENCIES,
+    MZ_OBJECT_GLOBAL_IDS, MZ_OPERATORS, MZ_PSEUDO_TYPES, MZ_REPLACEMENTS, MZ_ROLE_AUTH,
+    MZ_SESSIONS, MZ_SOURCE_REFERENCES, MZ_STORAGE_USAGE_BY_SHARD, MZ_SUBSCRIPTIONS, MZ_TABLES,
+    MZ_TYPE_PG_METADATA, MZ_TYPES, MZ_VIEWS, MZ_WEBHOOKS_SOURCES,
 };
 use mz_catalog::durable::SourceReferences;
 use mz_catalog::memory::error::Error;
 use mz_catalog::memory::objects::{
-    CatalogEntry, CatalogItem, DataSourceDesc, Func, Index, MaterializedView, Sink, Table,
+    CatalogEntry, CatalogItem, DataSourceDesc, Func, Index, MaterializedView, Table,
     TableDataSource, Type, View,
 };
 use mz_expr::MirScalarExpr;
@@ -50,7 +50,6 @@ use mz_sql::func::FuncImplCatalogDetails;
 use mz_sql::names::SchemaSpecifier;
 use mz_sql_parser::ast::display::AstDisplay;
 use mz_storage_client::client::TableData;
-use mz_storage_types::sinks::{IcebergSinkConnection, KafkaSinkConnection, StorageSinkConnection};
 use smallvec::smallvec;
 
 // DO NOT add any more imports from `crate` outside of `crate::catalog`.
@@ -192,9 +191,9 @@ impl CatalogState {
             CatalogItem::MaterializedView(mview) => {
                 self.pack_materialized_view_update(id, mview, diff)
             }
-            CatalogItem::Sink(sink) => {
-                self.pack_sink_update(id, oid, schema_id, name, owner_id, sink, diff)
-            }
+            // mz_sinks, mz_kafka_sinks and mz_iceberg_sinks read create_sql
+            // out of mz_catalog_raw, so there is nothing to pack here.
+            CatalogItem::Sink(_) => vec![],
             CatalogItem::Type(ty) => {
                 self.pack_type_update(id, oid, schema_id, name, owner_id, privileges, ty, diff)
             }
@@ -514,87 +513,6 @@ impl CatalogState {
         updates
     }
 
-    fn pack_sink_update(
-        &self,
-        id: CatalogItemId,
-        oid: u32,
-        schema_id: &SchemaSpecifier,
-        name: &str,
-        owner_id: &RoleId,
-        sink: &Sink,
-        diff: Diff,
-    ) -> Vec<BuiltinTableUpdate<&'static BuiltinTable>> {
-        let mut updates = vec![];
-        match &sink.connection {
-            StorageSinkConnection::Kafka(KafkaSinkConnection {
-                topic: topic_name, ..
-            }) => {
-                updates.push(BuiltinTableUpdate::row(
-                    &*MZ_KAFKA_SINKS,
-                    Row::pack_slice(&[
-                        Datum::String(&id.to_string()),
-                        Datum::String(topic_name.as_str()),
-                    ]),
-                    diff,
-                ));
-            }
-            StorageSinkConnection::Iceberg(IcebergSinkConnection {
-                namespace, table, ..
-            }) => {
-                updates.push(BuiltinTableUpdate::row(
-                    &*MZ_ICEBERG_SINKS,
-                    Row::pack_slice(&[
-                        Datum::String(&id.to_string()),
-                        Datum::String(namespace.as_str()),
-                        Datum::String(table.as_str()),
-                    ]),
-                    diff,
-                ));
-            }
-        };
-
-        let create_stmt = mz_sql::parse::parse(&sink.create_sql)
-            .unwrap_or_else(|_| panic!("create_sql cannot be invalid: {}", sink.create_sql))
-            .into_element()
-            .ast;
-
-        let envelope = sink.envelope();
-
-        // The combined format string is used for the deprecated `format` column.
-        let combined_format = sink.combined_format();
-        let (key_format, value_format) = match sink.formats() {
-            Some((key_format, value_format)) => (key_format, Some(value_format)),
-            None => (None, None),
-        };
-
-        updates.push(BuiltinTableUpdate::row(
-            &*MZ_SINKS,
-            Row::pack_slice(&[
-                Datum::String(&id.to_string()),
-                Datum::UInt32(oid),
-                Datum::String(&schema_id.to_string()),
-                Datum::String(name),
-                Datum::String(sink.connection.name()),
-                Datum::from(sink.connection_id().map(|id| id.to_string()).as_deref()),
-                // size column now deprecated w/o linked clusters
-                Datum::Null,
-                Datum::from(envelope),
-                // FIXME: These key/value formats are kinda leaky! Should probably live in
-                // the kafka sink table.
-                Datum::from(combined_format.as_ref().map(|f| f.as_ref())),
-                Datum::from(key_format),
-                Datum::from(value_format),
-                Datum::String(&sink.cluster_id.to_string()),
-                Datum::String(&owner_id.to_string()),
-                Datum::String(&sink.create_sql),
-                Datum::String(&create_stmt.to_ast_string_redacted()),
-            ]),
-            diff,
-        ));
-
-        updates
-    }
-
     fn pack_index_update(
         &self,
         id: CatalogItemId,

src/adapter/src/catalog/open/builtin_schema_migration.rs

@@ -101,12 +101,6 @@ static MIGRATIONS: LazyLock<Vec<MigrationStep>> = LazyLock::new(|| {
             MZ_INTERNAL_SCHEMA,
             "mz_cluster_replica_metrics_history",
         ),
-        MigrationStep::replacement(
-            "0.160.0",
-            CatalogItemType::Table,
-            MZ_CATALOG_SCHEMA,
-            "mz_sinks",
-        ),
         MigrationStep::replacement(
             "26.18.0-dev.0",
             CatalogItemType::MaterializedView,
@@ -355,6 +349,29 @@ static MIGRATIONS: LazyLock<Vec<MigrationStep>> = LazyLock::new(|| {
             MZ_CATALOG_SCHEMA,
             "mz_aws_privatelink_connections",
         ),
+        // The three sink tables became materialized views, which moves their
+        // fingerprints. The old `0.160.0` `mz_sinks` step had to go at the same
+        // time, because it names the `Table` description and that no longer
+        // resolves. Nothing is lost: anything that needed the old step upgrades
+        // from further back than this one, so this one covers it too.
+        MigrationStep::replacement(
+            "26.38.0-dev.0",
+            CatalogItemType::MaterializedView,
+            MZ_CATALOG_SCHEMA,
+            "mz_sinks",
+        ),
+        MigrationStep::replacement(
+            "26.38.0-dev.0",
+            CatalogItemType::MaterializedView,
+            MZ_CATALOG_SCHEMA,
+            "mz_kafka_sinks",
+        ),
+        MigrationStep::replacement(
+            "26.38.0-dev.0",
+            CatalogItemType::MaterializedView,
+            MZ_CATALOG_SCHEMA,
+            "mz_iceberg_sinks",
+        ),
     ]
 });
 

src/adapter/src/catalog/open/builtin_schema_migration_tests.rs

@@ -402,3 +402,18 @@ fn init_persist(sim: &mut turmoil::Sim) -> PersistLocation {
         consensus_uri: "turmoil://consensus:7000".parse().unwrap(),
     }
 }
+
+/// A step naming a builtin that no longer exists, or that changed
+/// `CatalogItemType`, panics `validate_migration_steps` at catalog open. That
+/// needs a catalog already on disk to reproduce, so the default CI suite never
+/// sees it and only the upgrade nightly does. Much cheaper to catch here.
+#[mz_ore::test]
+fn test_migration_steps_resolve_to_builtins() {
+    for step in MIGRATIONS.iter() {
+        assert!(
+            BUILTIN_LOOKUP.contains_key(&step.object),
+            "migration step for non-existent builtin: {:?}",
+            step.object
+        );
+    }
+}

src/catalog/src/builtin.rs

@@ -1107,11 +1107,11 @@ pub static BUILTINS_STATIC: LazyLock<Vec<Builtin<NameReference>>> = LazyLock::ne
         Builtin::Log(&MZ_COMPUTE_ERROR_COUNTS_RAW),
         Builtin::Log(&MZ_COMPUTE_HYDRATION_TIMES_PER_WORKER),
         Builtin::Log(&MZ_COMPUTE_OPERATOR_HYDRATION_STATUSES_PER_WORKER),
-        Builtin::Table(&MZ_KAFKA_SINKS),
+        Builtin::MaterializedView(&MZ_KAFKA_SINKS),
         Builtin::MaterializedView(&MZ_KAFKA_CONNECTIONS),
         Builtin::MaterializedView(&MZ_KAFKA_SOURCES),
         Builtin::Table(&MZ_OBJECT_DEPENDENCIES),
-        Builtin::Table(&MZ_ICEBERG_SINKS),
+        Builtin::MaterializedView(&MZ_ICEBERG_SINKS),
         Builtin::MaterializedView(&MZ_DATABASES),
         Builtin::MaterializedView(&MZ_SCHEMAS),
         Builtin::Table(&MZ_COLUMNS),
@@ -1125,7 +1125,7 @@ pub static BUILTINS_STATIC: LazyLock<Vec<Builtin<NameReference>>> = LazyLock::ne
         Builtin::MaterializedView(&MZ_MYSQL_SOURCE_TABLES),
         Builtin::MaterializedView(&MZ_SQL_SERVER_SOURCE_TABLES),
         Builtin::MaterializedView(&MZ_KAFKA_SOURCE_TABLES),
-        Builtin::Table(&MZ_SINKS),
+        Builtin::MaterializedView(&MZ_SINKS),
         Builtin::Table(&MZ_VIEWS),
         Builtin::Table(&MZ_TYPES),
         Builtin::Table(&MZ_TYPE_PG_METADATA),