From 8cca849ec4047c0e8f18d17d3073c273aff91ff2 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Aug 2026 14:01:48 +0200 Subject: [PATCH 1/6] ci: Nightly fixes (2026-08-06) (#38075) Mostly parallel-workload followups to https://github.com/MaterializeInc/materialize/pull/37590 and issues having been closed or moved to Linear from my sweep yesterday. Based on https://buildkite.com/materialize/nightly/builds/17774 --- .../pipeline.template.yml | 2 +- .../materialize/data_ingest/data_type.py | 10 + .../materialize/data_ingest/executor.py | 13 +- ...rnal_output_inconsistency_ignore_filter.py | 6 +- .../materialize/parallel_workload/action.py | 153 ++++++++++----- .../materialize/parallel_workload/database.py | 174 +++++++++++++++--- .../negative_accumulation_errors.py | 4 +- .../parallel_workload/parallel_workload.py | 101 ++++++---- .../materialize/parallel_workload/settings.py | 8 - .../materialize/parallel_workload/worker.py | 9 + .../pg_inconsistency_ignore_filter.py | 14 +- test/sqllogictest/boolean.slt | 9 +- test/sqllogictest/float.slt | 2 +- .../fold_vs_dataflow/1_numbers_dataflow.slt | 4 +- .../3_number_aggfns_dataflow.slt | 2 +- test/sqllogictest/unsigned_int.slt | 2 +- 16 files changed, 368 insertions(+), 145 deletions(-) diff --git a/ci/release-qualification/pipeline.template.yml b/ci/release-qualification/pipeline.template.yml index 99960bdd7575c..594ff61162afb 100644 --- a/ci/release-qualification/pipeline.template.yml +++ b/ci/release-qualification/pipeline.template.yml @@ -189,7 +189,7 @@ steps: plugins: - ./ci/plugins/mzcompose: composition: zippy - # TODO(def-) Increase number of actions when database-issues#7257 is fixed + # TODO(def-) Increase number of actions when SS-422 is fixed args: [--scenario=KafkaParallelInsert, --transaction-isolation=serializable, --actions=50000, --max-execution-time=4h] - group: Benchmarks diff --git a/misc/python/materialize/data_ingest/data_type.py b/misc/python/materialize/data_ingest/data_type.py index b9a246b41562e..762f96db54e22 100644 --- a/misc/python/materialize/data_ingest/data_type.py +++ b/misc/python/materialize/data_ingest/data_type.py @@ -1030,6 +1030,16 @@ def name(backend: Backend = Backend.MATERIALIZE) -> str: raise ValueError("Unsupported") elif backend == Backend.JSON: raise ValueError("Unsupported") + elif backend == Backend.MYSQL: + # Fractional-second precision spelled out, because a bare MySQL TIME + # has fsp 0 and *rounds* what it is given. The 23:59:59.999999 edge + # value above then lands as 24:00:00, which MySQL accepts (its TIME + # range is +/-838:59:59) but Materialize cannot represent: the source + # decodes TIME into a chrono::NaiveTime, so ingestion fails with + # "Couldn't convert the value Time(..) to a desired type" and the + # source has to be dropped and recreated. At fsp 6 nothing rounds and + # the value survives the round trip. + return "time(6)" else: return "time" diff --git a/misc/python/materialize/data_ingest/executor.py b/misc/python/materialize/data_ingest/executor.py index 41dca129ff0bb..b37f64aecc2ee 100644 --- a/misc/python/materialize/data_ingest/executor.py +++ b/misc/python/materialize/data_ingest/executor.py @@ -546,7 +546,12 @@ def create(self, logging_exe: Any | None = None) -> None: ) self.execute( cur, - f"""CREATE TABLE {identifier(self.table)} + # Qualified like the source above. Unqualified, the table lands + # in the connection's default schema (`public`) rather than + # `self.schema`, and a caller addressing it as + # `..`, as parallel-workload does, never + # resolves it. + f"""CREATE TABLE {identifier(self.database)}.{identifier(self.schema)}.{identifier(self.table)} FROM SOURCE {identifier(self.database)}.{identifier(self.schema)}.{identifier(self.source)} (REFERENCE {identifier(self.table)})""", ) @@ -696,7 +701,8 @@ def create(self, logging_exe: Any | None = None, prepopulate_rows: int = 0) -> N ) self.execute( cur, - f"""CREATE TABLE {identifier(self.table)} + # Qualified like the source above, see SqlServerExecutor.create. + f"""CREATE TABLE {identifier(self.database)}.{identifier(self.schema)}.{identifier(self.table)} FROM SOURCE {identifier(self.database)}.{identifier(self.schema)}.{identifier(self.source)} (REFERENCE mysql.{identifier(self.table)})""", ) @@ -826,7 +832,8 @@ def create(self, logging_exe: Any | None = None) -> None: ) self.execute( cur, - f"""CREATE TABLE {identifier(self.table)} + # Qualified like the source above, see SqlServerExecutor.create. + f"""CREATE TABLE {identifier(self.database)}.{identifier(self.schema)}.{identifier(self.table)} FROM SOURCE {identifier(self.database)}.{identifier(self.schema)}.{identifier(self.source)} (REFERENCE {identifier(self.table)})""", ) diff --git a/misc/python/materialize/output_consistency/ignore_filter/internal_output_inconsistency_ignore_filter.py b/misc/python/materialize/output_consistency/ignore_filter/internal_output_inconsistency_ignore_filter.py index 59eae8d957017..3b1f24ad959ba 100644 --- a/misc/python/materialize/output_consistency/ignore_filter/internal_output_inconsistency_ignore_filter.py +++ b/misc/python/materialize/output_consistency/ignore_filter/internal_output_inconsistency_ignore_filter.py @@ -84,7 +84,7 @@ def _matches_problematic_operation_or_function_invocation( isinstance(arg_type_spec, NumericReturnTypeSpec) and not arg_type_spec.only_integer ): - return YesIgnore("database-issues#4341") + return YesIgnore("DB-191") return NoIgnore() @@ -104,13 +104,13 @@ def _matches_problematic_function_invocation( "var_pop", }: if ExpressionCharacteristics.MAX_VALUE in all_involved_characteristics: - return YesIgnore("database-issues#4341") + return YesIgnore("DB-191") if ( ExpressionCharacteristics.DECIMAL in all_involved_characteristics and ExpressionCharacteristics.TINY_VALUE in all_involved_characteristics ): - return YesIgnore("database-issues#4341") + return YesIgnore("DB-191") return NoIgnore() diff --git a/misc/python/materialize/parallel_workload/action.py b/misc/python/materialize/parallel_workload/action.py index d314cc10db9c9..c585e9bccb09a 100644 --- a/misc/python/materialize/parallel_workload/action.py +++ b/misc/python/materialize/parallel_workload/action.py @@ -46,7 +46,6 @@ VarChar, ) from materialize.data_ingest.query_error import QueryError -from materialize.data_ingest.row import Operation from materialize.mzcompose import get_default_system_parameters from materialize.mzcompose.composition import Composition from materialize.mzcompose.services.materialized import ( @@ -243,6 +242,25 @@ def applicable(self, exe: Executor) -> bool: coverage check meaningful.""" return True + def insert_batch_size(self, exe: Executor, table: Table) -> int: + """How many rows the next insert into `table` may add, 0 once the table + sits at MAX_ROWS.""" + available = MAX_ROWS - table.num_rows + if available < 1: + return 0 + # A per-worker share of the budget, not the whole remainder: `num_rows` + # is read here but only bumped once the statement returns, so every + # worker would otherwise size a full-budget batch off the same stale + # counter and the table would overshoot MAX_ROWS by roughly a + # worker-count factor, which is what kept the cap from binding at all. + # A share bounds the overshoot at one extra round of concurrent batches. + # Charging the budget up front instead would bind exactly, but a claim + # is not given back when the statement's transaction rolls back, so the + # insert actions would stall against tables that read as full while + # holding far fewer rows. + share = max(1, MAX_ROWS // exe.db.num_threads) + return self.rng.randint(1, min(available, share)) + def create_system_connection( self, exe: Executor, num_attempts: int = 10 ) -> Connection: @@ -1066,10 +1084,13 @@ def run(self, exe: Executor) -> bool: return False table = self.rng.choice(tables) + num_rows = self.insert_batch_size(exe, table) + if not num_rows: + return False + column_names = ", ".join(column.name(True) for column in table.columns) column_values = [] - max_rows = min(100, MAX_ROWS - table.num_rows) - for i in range(self.rng.randrange(1, max_rows + 1)): + for i in range(num_rows): column_values.append( ", ".join(column.value(self.rng, True) for column in table.columns) ) @@ -1081,7 +1102,7 @@ def run(self, exe: Executor) -> bool: self.exe_prepared(query, f"insert{self.stmt_id}", exe) else: exe.execute(query, http=Http.RANDOM) - table.num_rows += len(column_values) + table.num_rows += num_rows exe.insert_table = table.table_id return True @@ -1137,10 +1158,11 @@ def run(self, exe: Executor) -> bool: f"({expression(column.data_type, source.columns, self.rng, kind=ExprKind.WRITE)})::{column.data_type.name()}" for column in table.columns ) - # `num_rows` can be stale, so clamp instead of trusting the filter - # above. The LIMIT keeps a self-insert from doubling the table on every + # The LIMIT keeps a self-insert from doubling the table on every # attempt. - limit = self.rng.randint(1, max(1, min(100, MAX_ROWS - table.num_rows))) + limit = self.insert_batch_size(exe, table) + if not limit: + return False query = ( f"INSERT INTO {table} ({column_names}) SELECT {expressions} FROM {source}" f" WHERE {expression(Boolean, source.columns, self.rng, kind=ExprKind.WRITE)}" @@ -1199,13 +1221,16 @@ def run(self, exe: Executor) -> bool: return False table = self.rng.choice(tables) + num_rows = self.insert_batch_size(exe, table) + if not num_rows: + return False + values = [] - max_rows = min(100, MAX_ROWS - table.num_rows) - for i in range(self.rng.randrange(1, max_rows + 1)): + for i in range(num_rows): values.append([column.value(self.rng, False) for column in table.columns]) query = f"COPY INTO {table} FROM STDIN" exe.copy(query, values) - table.num_rows += len(values) + table.num_rows += num_rows exe.insert_table = table.table_id return True @@ -1250,10 +1275,13 @@ def run(self, exe: Executor) -> bool: return False table = self.rng.choice(tables) + num_rows = self.insert_batch_size(exe, table) + if not num_rows: + return False + column_names = ", ".join(column.name(True) for column in table.columns) column_values = [] - max_rows = min(100, MAX_ROWS - table.num_rows) - for i in range(self.rng.randrange(1, max_rows + 1)): + for i in range(num_rows): column_values.append( ", ".join(column.value(self.rng, True) for column in table.columns) ) @@ -1280,7 +1308,7 @@ def run(self, exe: Executor) -> bool: self.exe_prepared(query, f"insert_returning{self.stmt_id}", exe) else: exe.execute(query, http=Http.RANDOM) - table.num_rows += len(column_values) + table.num_rows += num_rows exe.insert_table = table.table_id return True @@ -1319,16 +1347,28 @@ def run(self, exe: Executor) -> bool: class SourceInsertAction(Action): + """Feed one workload transaction to a random CDC source's upstream system. + + Unlike the table DML actions this carries no MAX_ROWS gate, because a + `data_ingest` workload bounds its own upstream table: the upsert workloads + only ever write key 0, and the delete-at-end-of-day workloads insert + `Records.SOME` keys and then delete exactly those keys again, so the table + oscillates and every cycle nets to zero. A gate could only ever misfire. + `next()` has already consumed a transaction by the time a row count could be + checked, so refusing to run it would drop one half of an insert/delete pair + and leave the count drifting away from the upstream table for the rest of the + run. A workload that grows without bound would need a different mechanism, + not a row count, since `MySqlSource.prepopulate_rows` already puts up to + 30,000 rows upstream before the source is even created.""" + def run(self, exe: Executor) -> bool: with exe.db.lock: - sources = [ - source - for source in exe.db.kafka_sources + sources = ( + exe.db.kafka_sources + exe.db.postgres_sources + exe.db.mysql_sources + exe.db.sql_server_sources - if source.num_rows < MAX_ROWS - ] + ) if not sources: return False source = self.rng.choice(sources) @@ -1341,14 +1381,7 @@ def run(self, exe: Executor) -> bool: ]: return False - transaction = next(source.generator) - for row_list in transaction.row_lists: - for row in row_list.rows: - if row.operation == Operation.INSERT: - source.num_rows += 1 - elif row.operation == Operation.DELETE: - source.num_rows -= 1 - source.executor.run(transaction, logging_exe=exe) + source.executor.run(next(source.generator), logging_exe=exe) return True @@ -1469,7 +1502,7 @@ def run(self, exe: Executor) -> bool: for t in exe.db.tables if t != table and (not t.temp or t in exe.temp_objects) ] - # TODO: Drop the RepeatRow gate once database-issues#9308 is fixed. + # TODO: Drop the RepeatRow gate once STG-36 is fixed. # DELETE .. USING lowers to a semijoin whose DistinctBy can leave the # target table with a net-negative row, and every later reader of that # table then surfaces the corruption. Tolerating that class outside @@ -2768,6 +2801,18 @@ def run(self, exe: Executor) -> bool: class FlipFlagsAction(Action): + # Shortest gap between two flips, fleet-wide. Every worker's action list + # carries this action, so weights alone put it at ~30 flips/s, which drove + # ~43k dyncfg applications per replica and most of a 132 MB services.log. + # What this action is after is a flag changing under a running dataflow, and + # one change per second is plenty for that. + MIN_INTERVAL_SEC = 1.0 + + # Guards `last_flip`, which is shared across workers since each holds its + # own instance of this action. + interval_lock = threading.Lock() + last_flip = 0.0 + def __init__( self, rng: random.Random, @@ -2927,10 +2972,10 @@ def __init__( BOOLEAN_FLAG_VALUES ) self.flags_with_values["cluster"] = ["quickstart", "dont_exist"] - # NOTE: enable_frontend_peek_sequencing is pinned off in - # ADDITIONAL_SYSTEM_PARAMETER_DEFAULTS (frontend-peek read-hold vs - # compaction race, https://linear.app/materializeinc/issue/SQL-520), so - # it is not flipped here. + self.flags_with_values["enable_frontend_peek_sequencing"] = [ + "true", + "false", + ] self.flags_with_values["enable_frontend_subscribes"] = [ "true", "false", @@ -3251,6 +3296,12 @@ def errors_to_ignore(self, exe: Executor) -> list[str]: ] + super().errors_to_ignore(exe) def run(self, exe: Executor) -> bool: + with FlipFlagsAction.interval_lock: + now = time.time() + if now - FlipFlagsAction.last_flip < FlipFlagsAction.MIN_INTERVAL_SEC: + return False + FlipFlagsAction.last_flip = now + # A tenth of the time set a random cluster's arrangement dictionary # compression instead of flipping a global flag. The per-cluster option # and the global `enable_arrangement_dictionary_compression_alpha` flag @@ -3895,6 +3946,20 @@ def run(self, exe: Executor) -> bool: f"{new_size} sat in-progress {read_ts_ms - int(record_deadline)}ms " f"past its deadline {record_deadline}" ) + # A settled record is retained with its terminal status, so a + # NULL from the LEFT JOIN means the record this ALTER wrote is + # gone, which no code path is allowed to do. Every writer of the + # durable field keeps the record and only moves its status: + # `reshape_alter_cluster_managed` writes InProgress or, for an + # ALTER back to the realized shape, Cancelled; + # `cancel_carried_reconfiguration` mutates the status in place; + # and the controller's three writes are Finalized, TimedOut and + # ResourceExhausted. `ReconfigurationWrite.record` is an Option + # documented as "None to clear it", but nothing constructs that. + # The relation also drops a cluster that stops being managed, and + # environmentd's bootstrap is the one place that resets the field + # outright, hence the scenario carve-outs in `applicable` and no + # `SET (MANAGED ...)` anywhere in the workload. if read_ts_ms > our_deadline_ms and status is None: raise ValueError( f"Reconfiguration record of cluster {cluster} is gone as of " @@ -5624,10 +5689,16 @@ def run(self, exe: Executor) -> bool: log = f"POST {url} Headers: {', '.join(headers_strs)} Body: {payload.encode('utf-8')}" exe.log(log) try: - source.num_rows += 1 result = requests.post(url, data=payload.encode(), headers=headers) if result.status_code != 200: raise QueryError(f"{result.status_code}: {result.text}", log) + # Count after the POST landed. A webhook source is append-only, + # so its row budget is never released, and counting rejected + # POSTs retires the source without a single row ever reaching + # it. A source whose POSTs all 404 (a concurrent cascading drop, + # a rename) then silently stops being posted to for the rest of + # the run. + source.num_rows += 1 except requests.exceptions.ConnectionError: # Expected when Mz is killed if exe.db.scenario not in ( @@ -6177,13 +6248,7 @@ def __init__( (CopyToStdoutAction, 20), (ShowAction, 10), (SystemCatalogReadAction, 10), - # TODO: Reenable once EXPLAIN FILTER PUSHDOWN can no longer panic the - # coordinator when a referenced compute collection is concurrently - # dropped. sequence_explain_pushdown -> acquire_read_holds().expect( - # "missing compute collection") at read_policy.rs:389 (normal peeks and - # EXPLAIN ANALYZE handle the drop gracefully). - # See https://linear.app/materializeinc/issue/SQL-519 - # (ExplainFilterPushdownAction, 5), + (ExplainFilterPushdownAction, 5), # PREPARED BUT DISABLED (see class docstrings): enabling these now just # re-detects known-unfixed coordinator bugs rather than finding new ones. # (DependencyConsistencyAction, 5), # TODO: enable once SQL-521 fixed @@ -6279,9 +6344,8 @@ def __init__( (DropLoadGeneratorSourceAction, 4), (CreateMultiLoadGeneratorSourceAction, 2), (DropMultiLoadGeneratorSourceAction, 2), - # TODO: Reenable when https://linear.app/materializeinc/issue/SS-307 is fixed - # (CreateMySqlSourceAction, 4), - # (DropMySqlSourceAction, 4), + (CreateMySqlSourceAction, 4), + (DropMySqlSourceAction, 4), (CreatePostgresSourceAction, 4), (DropPostgresSourceAction, 4), # TODO: Reenable when https://linear.app/materializeinc/issue/SS-290 is fixed @@ -6343,10 +6407,7 @@ def __init__( (DDLTransactionAction, 2), (SystemCatalogReadAction, 4), (ExplainAnalyzeAction, 4), - # TODO: Reenable with EXPLAIN FILTER PUSHDOWN's coordinator panic on a - # concurrently-dropped compute collection (read_policy.rs:389). - # See https://linear.app/materializeinc/issue/SQL-519 - # (ExplainFilterPushdownAction, 2), + (ExplainFilterPushdownAction, 2), (FlipFlagsAction, 2), # TODO: Reenable when https://linear.app/materializeinc/issue/SQL-405 is fixed. # (AlterTableAddColumnAction, 10), diff --git a/misc/python/materialize/parallel_workload/database.py b/misc/python/materialize/parallel_workload/database.py index b3d72d2fbb281..49ccfbabfe48e 100644 --- a/misc/python/materialize/parallel_workload/database.py +++ b/misc/python/materialize/parallel_workload/database.py @@ -61,17 +61,15 @@ MAX_COLUMNS = 50 MAX_INCLUDE_HEADERS = 5 -# The row count a view's join squares. A view over two tables joins them on a -# random boolean predicate, so the join is a cross product and every read of -# that view builds MAX_ROWS**2 wide intermediate rows, whatever LIMIT the -# reading query carries. Measured at 500 with only four columns, one -# `COPY (SELECT * FROM view WHERE .. LIMIT 100) TO 's3://..'` cost ~450 MiB of -# replica RSS, and pw tables carry up to MAX_COLUMNS of them. A handful of such -# reads at once is what grew a clusterd to 19 GiB and had the kernel OOM-killer -# take it, plus environmentd and the Kafka broker, out of the one cgroup all of -# a run's containers share (nightlies 17660-17701). 100 keeps enough rows for -# the DML, persist and index paths to be interesting while cutting the -# intermediate 25-fold. +# The row count a table is held at. Measured with a view whose join was a cross +# product, one `COPY (SELECT * FROM view WHERE .. LIMIT 100) TO 's3://..'` over +# 500-row bases cost ~450 MiB of replica RSS with only four columns, and pw +# tables carry up to MAX_COLUMNS of them. A handful of such reads at once is +# what grew a clusterd to 19 GiB and had the kernel OOM-killer take it, plus +# environmentd and the Kafka broker, out of the one cgroup all of a run's +# containers share (nightlies 17660-17701). 100 keeps enough rows for the DML, +# persist and index paths to be interesting. See also JOIN_KEY_TYPES, which +# keeps a view's join from squaring this in the first place. MAX_ROWS = 100 MAX_CLUSTERS = 4 MAX_CLUSTER_REPLICAS = 2 @@ -94,7 +92,6 @@ MAX_INITIAL_DBS = 1 MAX_INITIAL_SCHEMAS = 1 -MAX_INITIAL_CLUSTERS = 2 MAX_INITIAL_TABLES = 2 MAX_INITIAL_VIEWS = 2 MAX_INITIAL_ROLES = 1 @@ -105,6 +102,16 @@ MAX_INITIAL_POSTGRES_SOURCES = 1 MAX_INITIAL_LOADGEN_SOURCES = 1 +# Types eligible as the equality anchor of a two-base view's join, see +# `View.join_key_expr`. A type qualifies when Materialize has `=` for it (map +# has none) and, less obviously, when its random values collide often enough +# that the equality still lets rows through: every type here draws one of a +# handful of sentinels (the type's min or max, 0, 1, one of five fixed strings) +# about a fifth of the time, so on the order of a percent of the cross product +# survives. An equality on uuid or timestamp columns would bound the join just +# as well but always produce an empty view, which tests nothing. +JOIN_KEY_TYPES = NUMBER_TYPES + [Text] + class BodyFormat(Enum): TEXT = 1 @@ -333,9 +340,17 @@ def __init__( ) if base_object2: - self.on_expr = expression( + # The random boolean alone references an arbitrary subset of the + # columns, so it usually names only one side or neither, and the + # join plans as a cross product whose output squares MAX_ROWS. The + # equality anchor makes the output linear in the input instead. + # Cross joins stay covered by the ad-hoc SELECTs, which carry a + # LIMIT the optimizer can push into the join. + join_key = self.join_key_expr(base_object, base_object2, rng) + predicate = expression( Boolean, all_columns, rng, kind=ExprKind.MATERIALIZABLE ) + self.on_expr = f"{join_key} AND ({predicate})" self.union = rng.random() < 0.1 if self.union: @@ -354,6 +369,32 @@ def __init__( self.columns = [Column(rng, 0, Long, self)] self.union = False + @staticmethod + def join_key_expr(left: DBObject, right: DBObject, rng: random.Random) -> str: + """An equality between one column of each side, so that the join has a + key and its output stays linear in the input. + + Matches the join shape `SelectAction.generate_select_query` emits, with + two additions. A `JOIN_KEY_TYPES` pair wins over an equally valid pair of + some other type, whose values would never collide and leave the view + permanently empty. And a pair of objects sharing no column type at all, + which a query can answer by not joining but a two-base view cannot, falls + back to comparing the columns as text. That bounds the join just as well, + only with a near-empty result.""" + candidates = [ + (left_column, right_column) + for left_column in left.columns + for right_column in right.columns + # Materialize has no `=` for map. + if left_column.data_type is right_column.data_type + and left_column.data_type is not TextTextMap + ] + colliding = [pair for pair in candidates if pair[0].data_type in JOIN_KEY_TYPES] + if candidates: + left_column, right_column = rng.choice(colliding or candidates) + return f"{left_column} = {right_column}" + return f"{rng.choice(left.columns)}::text = {rng.choice(right.columns)}::text" + def name(self) -> str: if self.rename: return naughtify(f"v-{self.view_id}-{self.rename}") @@ -528,7 +569,6 @@ class KafkaSource(DBObject): lock: threading.Lock columns: list[KafkaColumn] schema: Schema - num_rows: int def __init__( self, @@ -542,7 +582,6 @@ def __init__( self.source_id = source_id self.cluster = cluster self.schema = schema - self.num_rows = 0 fields = [] for i in range(rng.randint(1, 10)): fields.append( @@ -759,7 +798,6 @@ class MySqlSource(DBObject): lock: threading.Lock columns: list[MySqlColumn] schema: Schema - num_rows: int def __init__( self, @@ -773,7 +811,6 @@ def __init__( self.source_id = source_id self.cluster = cluster self.schema = schema - self.num_rows = 0 # Bias toward a single-column key: only tables with a single-column # integer primary key take the parallel PK-range snapshot path, the # rest fall back to a whole-table read by one worker. @@ -823,7 +860,6 @@ class PostgresSource(DBObject): lock: threading.Lock columns: list[PostgresColumn] schema: Schema - num_rows: int def __init__( self, @@ -837,7 +873,6 @@ def __init__( self.source_id = source_id self.cluster = cluster self.schema = schema - self.num_rows = 0 fields = [] for i in range(rng.randint(1, 10)): fields.append( @@ -878,7 +913,6 @@ class SqlServerSource(DBObject): lock: threading.Lock columns: list[SqlServerColumn] schema: Schema - num_rows: int def __init__( self, @@ -893,7 +927,6 @@ def __init__( self.source_id = source_id self.cluster = cluster self.schema = schema - self.num_rows = 0 fields = [] for i in range(rng.randint(1, 10)): fields.append(Field(f"key{i}", rng.choice(DATA_TYPES_FOR_KEY), True)) @@ -1448,6 +1481,7 @@ class Database: seed: str sqlsmith_state: str flags: dict[str, str] + num_threads: int def __init__( self, @@ -1458,12 +1492,14 @@ def __init__( complexity: Complexity, scenario: Scenario, naughty_identifiers: bool, + num_threads: int, ): self.host = host self.ports = ports self.complexity = complexity self.scenario = scenario self.seed = seed + self.num_threads = num_threads set_naughty_identifiers(naughty_identifiers) self.s3_path = 0 @@ -1499,16 +1535,35 @@ def __init__( self.view_id = len(self.views) self.roles = [Role(i) for i in range(rng.randint(0, MAX_INITIAL_ROLES))] self.role_id = len(self.roles) - # At least one storage cluster required for WebhookSources + # Cluster 0 is the one cluster whose shape nothing churns: every cluster + # action skips it and operates on `clusters[1:]`, so it stays available + # for sources and sinks. Which cluster a given source or sink actually + # lands on is still random, both here and in the Create*Action paths. + # + # Each of those actions needs a cluster of a specific kind past cluster + # 0: managed for AlterClusterSetAction and FlipFlagsAction's per-cluster + # arrangement compression, unmanaged for Create/DropClusterReplicaAction. + # Drawing the count and the kinds the way the other initial objects are + # drawn covers both in one run out of eight, and the rest spend the whole + # run skipping. Nightly 17774 reported 369 AlterClusterSet, 129 + # ReconfigureCluster and 117 CreateClusterReplica attempts with not one + # success between them. So the kinds are fixed and only cluster 0's is + # left to the seed. self.clusters = [ Cluster( - i, - managed=rng.choice([True, False]), + cluster_id, + managed=managed, size=rng.choice(["scale=1,workers=1", "scale=1,workers=2"]), + # One replica each, which is what lets Create and + # DropClusterReplicaAction take turns on the unmanaged cluster: + # Create needs room below MAX_CLUSTER_REPLICAS, Drop needs more + # than one replica so the cluster keeps serving requests. replication_factor=1, introspection_interval="1s", ) - for i in range(rng.randint(1, MAX_INITIAL_CLUSTERS)) + for cluster_id, managed in enumerate( + [rng.choice([True, False]), True, False] + ) ] self.cluster_id = len(self.clusters) self.indexes = set() @@ -1517,6 +1572,9 @@ def __init__( for i in range(rng.randint(0, MAX_INITIAL_WEBHOOK_SOURCES)) ] self.webhook_source_id = len(self.webhook_sources) + # The CDC source lists are populated by `create_initial_sources`, not + # here: such a source's executor connects to the source's database on + # construction, and that database only exists once `create` has made it. self.kafka_sources = [] self.mysql_sources = [] self.postgres_sources = [] @@ -1621,6 +1679,70 @@ def __iter__(self): self.schemas + self.clusters + self.roles + self.db_objects() ).__iter__() + def create_initial_sources(self, exe: Executor) -> None: + """Populate the CDC source lists, once the databases they connect to + exist. + + SourceInsertAction is the only writer of a CDC source, and DML + complexity gives the DDL action list weight 0, so nothing there ever + runs a Create*SourceAction. Without a source from the start that action + cannot succeed once in a whole run, which is why the Kafka source is + unconditional rather than randomized down to zero like the other initial + objects. + + Skipped for a 0dt deploy. A source executor connects on construction and + picks one of the two environmentds at random every time it connects, and + the second one only comes up mid-run when ZeroDowntimeDeployAction + deploys it, so constructing a source here fails outright half the time. + That scenario runs at DDL complexity, so its actions create the sources + instead.""" + if self.scenario == Scenario.ZeroDowntimeDeploy: + return + self.kafka_sources = [ + KafkaSource( + i, + exe.rng.choice(self.clusters), + exe.rng.choice(self.schemas), + self.ports, + exe.rng, + ) + for i in range(MAX_INITIAL_KAFKA_SOURCES) + ] + self.kafka_source_id = len(self.kafka_sources) + self.postgres_sources = [ + PostgresSource( + i, + exe.rng.choice(self.clusters), + exe.rng.choice(self.schemas), + self.ports, + exe.rng, + ) + # A Postgres source is not expected to survive a backup & restore, + # see database-issues#6881 and + # CreatePostgresSourceAction.applicable. + for i in range( + 0 + if self.scenario == Scenario.BackupRestore + else exe.rng.randint(0, MAX_INITIAL_POSTGRES_SOURCES) + ) + ] + self.postgres_source_id = len(self.postgres_sources) + self.mysql_sources = [ + MySqlSource( + i, + exe.rng.choice(self.clusters), + exe.rng.choice(self.schemas), + self.ports, + exe.rng, + ) + for i in range(exe.rng.randint(0, MAX_INITIAL_MYSQL_SOURCES)) + ] + self.mysql_source_id = len(self.mysql_sources) + # MAX_INITIAL_SQL_SERVER_SOURCES stays unused: a SQL Server source + # drives CPU far above the other source kinds (SS-290), enough to have + # starved whole runs, and one in every run's initial database would put + # that back for good. + def create(self, exe: Executor, composition: Composition) -> None: for db in self.dbs: db.drop(exe) @@ -1706,6 +1828,8 @@ def create(self, exe: Executor, composition: Composition) -> None: print("Creating relations") + self.create_initial_sources(exe) + for relation in self: relation.create(exe) diff --git a/misc/python/materialize/parallel_workload/negative_accumulation_errors.py b/misc/python/materialize/parallel_workload/negative_accumulation_errors.py index 6f60be9596676..52eeb8a868315 100644 --- a/misc/python/materialize/parallel_workload/negative_accumulation_errors.py +++ b/misc/python/materialize/parallel_workload/negative_accumulation_errors.py @@ -12,12 +12,12 @@ Parallel Workload intentionally generates queries (via `repeat_row`) that can trigger these errors; this module is the single place to update when the -messages change. Source of truth: database-issues#9308 (section "For reference, +messages change. Source of truth: STG-36 (section "For reference, here is a list of places in the code / error msgs where we detect negative accumulations or similar issues"). """ -# Keep this list in sync with database-issues#9308. Each entry is a substring +# Keep this list in sync with STG-36. Each entry is a substring # that may appear in an error surfaced to the client. When the underlying # error messages are reworded, update this list in a single commit so the # rest of Parallel Workload picks up the new strings automatically. diff --git a/misc/python/materialize/parallel_workload/parallel_workload.py b/misc/python/materialize/parallel_workload/parallel_workload.py index 4a2740fc70842..addeca00435fe 100644 --- a/misc/python/materialize/parallel_workload/parallel_workload.py +++ b/misc/python/materialize/parallel_workload/parallel_workload.py @@ -119,7 +119,7 @@ def run( ).timestamp() database = Database( - rng, seed, host, ports, complexity, scenario, naughty_identifiers + rng, seed, host, ports, complexity, scenario, naughty_identifiers, num_threads ) system_conn = psycopg.connect( @@ -210,31 +210,48 @@ def run( database.create(Executor(rng, cur, None, database), composition) conn.close() + weights: list[float] + if complexity == Complexity.DDL: + weights = [60, 30, 30, 30, 100] + elif complexity == Complexity.DML: + weights = [60, 30, 30, 30, 0] + elif complexity == Complexity.Read: + weights = [60, 30, 0, 0, 0] + elif complexity == Complexity.DDLOnly: + weights = [0, 0, 0, 0, 100] + else: + raise ValueError(f"Unknown complexity {complexity}") + candidate_lists = [ + read_action_list, + fetch_action_list, + write_action_list, + dml_nontrans_action_list, + ddl_action_list, + ] + # Cover every list the complexity enables before sampling the rest. Drawing + # each worker independently leaves a list unmanned surprisingly often: at 8 + # threads the fetch, write and dml lists each go empty in ~36% of runs and + # the DDL list in ~2%, and an unmanned list is silent, the run just reports + # zero attempts for its actions. That is how nightly 17774's + # `--scenario=rename` run ended up with no DDL worker, so it renamed + # nothing, created no source (leaving HttpPostAction and SourceInsertAction + # with nothing to write to for 100k attempts each), and never dropped the + # initial cross-product view that then OOM-killed its replica. + # Heaviest-first so a thread count below the number of enabled lists still + # covers the ones the complexity cares about most, with the seed breaking + # ties among equal weights. + covered = [lst for lst, weight in zip(candidate_lists, weights) if weight] + rng.shuffle(covered) + covered.sort(key=lambda lst: -weights[candidate_lists.index(lst)]) + workers = [] threads = [] for i in range(num_threads): - weights: list[float] - if complexity == Complexity.DDL: - weights = [60, 30, 30, 30, 100] - elif complexity == Complexity.DML: - weights = [60, 30, 30, 30, 0] - elif complexity == Complexity.Read: - weights = [60, 30, 0, 0, 0] - elif complexity == Complexity.DDLOnly: - weights = [0, 0, 0, 0, 100] - else: - raise ValueError(f"Unknown complexity {complexity}") worker_rng = random.Random(rng.randrange(SEED_RANGE)) - action_list = worker_rng.choices( - [ - read_action_list, - fetch_action_list, - write_action_list, - dml_nontrans_action_list, - ddl_action_list, - ], - weights, - )[0] + if i < len(covered): + action_list = covered[i] + else: + action_list = worker_rng.choices(candidate_lists, weights)[0] actions = [ action_class(worker_rng, composition) for action_class in action_list.action_classes @@ -301,6 +318,7 @@ def run( autocommit=False, system=False, composition=composition, + off_session_work=True, ) workers.append(worker) thread = threading.Thread( @@ -329,6 +347,7 @@ def run( autocommit=False, system=False, composition=composition, + off_session_work=True, ) workers.append(worker) thread = threading.Thread( @@ -350,10 +369,11 @@ def run( autocommit=False, system=False, composition=composition, + off_session_work=True, ) workers.append(worker) thread = threading.Thread( - name="kill", + name="backup-restore", target=worker.run, args=(host, ports["materialized"], ports["http"], "materialize", database), daemon=True, @@ -435,11 +455,21 @@ def run( # loop (exe is None), which can only be hanging inside a connect call # to the server. One that never waits on the server is stuck in the # workload's own code, e.g. deadlocked on two object locks taken in - # opposite orders, which no timeout ever clears. Nothing legitimate - # pauses without a round trip for the 300s above (the longest such - # pause is BackupRestoreAction's 240s sleep), but a single sample can - # catch a worker between round trips, so only call a worker wedged if - # it stays off the server for a whole confirmation window. + # opposite orders, which no timeout ever clears. A single sample can + # catch a worker between round trips, so only call a worker wedged if it + # stays off the server for a whole confirmation window. + # + # Workers doing off-session work are left out entirely, in both + # directions. `last_status` tracks session round trips, and a scenario + # worker driving the composition makes none for as long as its action + # takes: BackupRestoreAction sleeps up to 240s and then runs a CRDB + # backup, a restore and a blocking `up("materialized")` that waits on + # environmentd's health check, which together outlast the 300s join + # window. Reading its stale status would report a healthy backup as a + # workload deadlock. Reporting it as on the server instead would be just + # as wrong the other way, because one worker on the server clears the + # verdict for all of them, so a real deadlock elsewhere would go + # unreported for the whole scenario. # # A tolerated server-side hang also explains every other worker: the # hung statement holds its actions' object locks (an `ALTER SINK ... @@ -451,19 +481,22 @@ def run( # diagnosed when no worker is on the server at all. Waiters and # deadlocks both resolve when the statement does, and calling a waiter # deadlocked costs the run its final checks. + def session_status(worker: Worker, thread: threading.Thread) -> str | None: + """This worker's last session status, or None when it has none worth + reading: already exited, not connected yet, or off-session work.""" + if not thread.is_alive() or worker.exe is None or worker.off_session_work: + return None + return worker.exe.last_status + def not_on_server() -> set[str]: return { t.name for w, t in alive - if t.is_alive() and w.exe is not None and w.exe.last_status != "running" + if (status := session_status(w, t)) is not None and status != "running" } def on_server() -> set[str]: - return { - t.name - for w, t in alive - if t.is_alive() and w.exe is not None and w.exe.last_status == "running" - } + return {t.name for w, t in alive if session_status(w, t) == "running"} wedged = not_on_server() confirm_until = time.time() + 60 diff --git a/misc/python/materialize/parallel_workload/settings.py b/misc/python/materialize/parallel_workload/settings.py index 4efdd32a3876e..8b1d306501944 100644 --- a/misc/python/materialize/parallel_workload/settings.py +++ b/misc/python/materialize/parallel_workload/settings.py @@ -58,14 +58,6 @@ def _missing_(cls, value): # it on outside that scenario is harmless: no Parallel Workload codegen # emits `repeat_row` unless the scenario is active. "enable_repeat_row": "true", - # TODO: Reenable once the frontend-peek path stops acquiring a read hold - # whose `since` has compacted past the chosen `as_of`. The soft-assert - # assert_read_holds_correct (frontend_peek.rs:1759) fires for peeks in - # multi-statement transactions ("... read hold at .. is not enough for - # as_of .."), panicking the coordinator. Peeks still work via the classic - # coordinator path with this off. - # See https://linear.app/materializeinc/issue/SQL-520 - "enable_frontend_peek_sequencing": "false", # 64 MiB, down from the 1 GiB default. A peek's result is materialized in # memory on the replica serving it before this bound errors it # (`to_error_if_exceeds`, compute_state.rs), and measured on a workload-like diff --git a/misc/python/materialize/parallel_workload/worker.py b/misc/python/materialize/parallel_workload/worker.py index f4eb99f0737a1..312394c226ee2 100644 --- a/misc/python/materialize/parallel_workload/worker.py +++ b/misc/python/materialize/parallel_workload/worker.py @@ -42,6 +42,13 @@ class Worker: ignored_errors: defaultdict[str, Counter[type[Action]]] composition: Composition | None occurred_exception: Exception | None + # Set for a worker whose action spends long stretches driving the mzcompose + # composition rather than its own session: sleeping, killing and restarting + # environmentd, backing up and restoring. `Executor.last_status` only tracks + # session round trips, so for such a worker it says nothing at all, and the + # end-of-run wedge check has to leave it out rather than read its stale + # status as evidence of a deadlock. + off_session_work: bool def __init__( self, @@ -53,12 +60,14 @@ def __init__( system: bool, composition: Composition | None, action_list: ActionList | None = None, + off_session_work: bool = False, ): self.rng = rng self.action_list = action_list self.actions = actions self.weights = weights self.end_time = end_time + self.off_session_work = off_session_work self.num_queries = Counter() # Unlike num_queries, these are never cleared: they feed the # end-of-run action coverage check. diff --git a/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py b/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py index 1f9688ed59558..9a06efe6a791c 100644 --- a/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py +++ b/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py @@ -35,7 +35,6 @@ matches_op_by_any_pattern, matches_op_by_pattern, matches_x_and_y, - matches_x_or_y, ) from materialize.output_consistency.ignore_filter.ignore_verdict import ( IgnoreVerdict, @@ -267,7 +266,7 @@ def _matches_problematic_function_invocation( if db_function.function_name_in_lower_case == "replace": # replace is not working properly with empty text; however, it is not possible to reliably determine if an # expression is an empty text, we therefore need to exclude the function completely - return YesIgnore("database-issues#6619: replace") + return YesIgnore("DB-189: replace") if db_function.function_name_in_lower_case == "regexp_replace": if expression.args[2].has_any_characteristic( @@ -992,17 +991,6 @@ def matches_nullif(expression: Expression) -> bool: ): return YesIgnore("database-issues#6623: mod") - if query_template.matches_specific_select_or_filter_expression( - col_index, - partial( - matches_x_or_y, - x=partial(matches_fun_by_name, function_name_in_lower_case="date_part"), - y=partial(matches_op_by_pattern, pattern="EXTRACT($ FROM $)"), - ), - True, - ): - return YesIgnore("database-issues#7089") - if query_template.matches_specific_select_or_filter_expression( col_index, matches_math_op_with_large_or_tiny_val, True ): diff --git a/test/sqllogictest/boolean.slt b/test/sqllogictest/boolean.slt index 32ab8874205a4..be1df8129b277 100644 --- a/test/sqllogictest/boolean.slt +++ b/test/sqllogictest/boolean.slt @@ -411,8 +411,7 @@ NULL 2 1 NULL 1 2 -# TODO: Reenable when database-issues#8807 is fixed -# query T -# SELECT CASE WHEN FALSE THEN 'short_string'::char(20) ELSE 'long_string_long_string'::char(30) END -# ---- -# long_string_long_string +query T +SELECT CASE WHEN FALSE THEN 'short_string'::char(20) ELSE 'long_string_long_string'::char(30) END +---- +long_string_long_string diff --git a/test/sqllogictest/float.slt b/test/sqllogictest/float.slt index f1b1dd21cf9d4..99954250f35d8 100644 --- a/test/sqllogictest/float.slt +++ b/test/sqllogictest/float.slt @@ -152,7 +152,7 @@ SELECT SUM(f1) FROM t1 ---- -282409603651671152154661355520.000 -# This *should* be zero, known issue https://github.com/MaterializeInc/database-issues/issues/4341 +# This *should* be zero, known issue DB-191 query T SELECT MIN(f1+f2)-SUM(f1) FROM t1 ---- diff --git a/test/sqllogictest/transform/fold_vs_dataflow/1_numbers_dataflow.slt b/test/sqllogictest/transform/fold_vs_dataflow/1_numbers_dataflow.slt index 41d9944eab7e4..790665ccd9131 100644 --- a/test/sqllogictest/transform/fold_vs_dataflow/1_numbers_dataflow.slt +++ b/test/sqllogictest/transform/fold_vs_dataflow/1_numbers_dataflow.slt @@ -22,7 +22,7 @@ statement ok INSERT INTO t_using_dataflow_rendering VALUES (1e-39::real, 1e-39::double, 1e-39::numeric); # DIFF TO CONSTANT FOLDING (SELECT on types [REAL, DOUBLE])! -# to be addressed with https://github.com/MaterializeInc/database-issues/issues/4341 +# to be addressed with DB-191 query RRR SELECT * FROM t_using_dataflow_rendering UNION ALL @@ -55,7 +55,7 @@ statement ok INSERT INTO t_using_dataflow_rendering VALUES (1e38::real, 1e38::double, 1e38::numeric); # DIFF TO CONSTANT FOLDING (SELECT on types [REAL, DOUBLE])! -# to be addressed with https://github.com/MaterializeInc/database-issues/issues/4341 +# to be addressed with DB-191 # The dataflow float sum accumulates in a fixed-point domain that cannot # represent values this large (|x| > ~1.01e31). Such values wrap modulo 2^128, # which is what makes cancelling sums correct (database-issues#11265) at the cost diff --git a/test/sqllogictest/transform/fold_vs_dataflow/3_number_aggfns_dataflow.slt b/test/sqllogictest/transform/fold_vs_dataflow/3_number_aggfns_dataflow.slt index cb61d9cf64caa..2334c5fc0d0d7 100644 --- a/test/sqllogictest/transform/fold_vs_dataflow/3_number_aggfns_dataflow.slt +++ b/test/sqllogictest/transform/fold_vs_dataflow/3_number_aggfns_dataflow.slt @@ -24,7 +24,7 @@ INSERT INTO t_using_dataflow_rendering VALUES (2, 2, 2, 32767::INT2, 1::INT2); # DIFF TO CONSTANT FOLDING ([SUM, AVG] with types [REAL, DOUBLE])! -# to be addressed with https://github.com/MaterializeInc/database-issues/issues/4341 +# to be addressed with DB-191 # The dataflow float sum accumulates in a fixed-point domain that cannot # represent values this large (|x| > ~1.01e31). Such values wrap modulo 2^128 # (database-issues#11265), so SUM/AVG over REAL/DOUBLE here are nonsensical; the diff --git a/test/sqllogictest/unsigned_int.slt b/test/sqllogictest/unsigned_int.slt index 93b083f3809ab..c7293decad5b3 100644 --- a/test/sqllogictest/unsigned_int.slt +++ b/test/sqllogictest/unsigned_int.slt @@ -1353,7 +1353,7 @@ SELECT STDDEV(a) FROM t8 ---- 4611686017711549096.944 -# Avoid overflow (known issue https://github.com/MaterializeInc/database-issues/issues/4341) +# Avoid overflow (known issue DB-191) statement ok DROP TABLE t8 From fd1dd6e62b034fc2b38938aabba024d7db11174d Mon Sep 17 00:00:00 2001 From: mtabebe Date: Thu, 6 Aug 2026 09:06:52 -0400 Subject: [PATCH 2/6] catalog: convert the connection-detail tables to materialized views (SQL-497) (#37725) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Converts the four connection-detail builtin tables to materialized views over mz_catalog_raw, so they are derived from each connection's persisted create_sql instead of being written by a Rust packer on every catalog change on the leader environmentd (the [SQL-118](https://linear.app/materializeinc/issue/SQL-118) / multi-envd motivation). - mz_kafka_connections - mz_ssh_tunnel_connections - mz_aws_connections - mz_aws_privatelink_connections Do this in two parts: 1. kafka + ssh — adds the parse_connection_details SQL helper and flips both tables to MVs. The kafka progress-topic default is reconstructed in SQL to match KafkaConnection::progress_topic. 2. aws + aws-privatelink — extends the helper with an aws branch and adds three plan-time fold functions (mz_aws_account_id, mz_aws_external_id_prefix, mz_aws_connection_role_arn) for the values that come from environment context rather than create_sql. The aws view reconstructs the assume-role principal, external id, and trust policy. The privatelink view reconstructs the principal ARN and keeps WHERE principal IS NOT NULL, so it emits no row when the AWS principal context is absent, matching the packer it replaces. RBAC note. The three new fold functions are deliberately not gated by restrict_to_user_objects, exactly like mz_environment_id. Testing: - parse_connection_details unit tests cover every connection kind and branch plus the null cases. - New lockdown SLTs pin the column shape, nullability, and wrong-type exclusion for each view. The mz_analytics builtin connection doubles as the SQL/Rust parity check for the aws assume-role columns. --------- Co-authored-by: Claude Opus 4.8 --- .../reference/system-catalog/mz_catalog.md | 2 +- .../reference/system-catalog/mz_internal.md | 2 +- src/adapter/src/catalog.rs | 138 ++++- .../src/catalog/builtin_table_updates.rs | 217 +------- src/adapter/src/catalog/open.rs | 4 + .../catalog/open/builtin_schema_migration.rs | 27 + src/adapter/src/catalog/state.rs | 1 + src/catalog/src/builtin.rs | 8 +- src/catalog/src/builtin/builtin.rs | 2 +- src/catalog/src/builtin/mz_catalog.rs | 311 +++++++---- src/catalog/src/builtin/mz_internal.rs | 268 ++++++---- src/catalog/src/config.rs | 4 + src/environmentd/src/test_util.rs | 21 +- src/environmentd/tests/sql.rs | 45 ++ src/expr/src/scalar/func/impls/jsonb.rs | 490 +++++++++++++++++- src/expr/src/scalar/func/unary.rs | 1 + .../src/project/compiler/typecheck/catalog.rs | 1 + src/pgrepr-consts/src/oid.rs | 24 +- src/sql/src/catalog.rs | 9 + src/sql/src/func.rs | 55 ++ src/storage-types/src/connections.rs | 5 + src/storage-types/src/connections/aws.rs | 8 + src/testdrive/src/action.rs | 42 +- .../sqllogictest/autogenerated/mz_catalog.slt | 2 +- .../autogenerated/mz_internal.slt | 2 +- test/sqllogictest/catalog_server_explain.slt | 112 +++- .../information_schema_tables.slt | 8 +- test/sqllogictest/mz_aws_connections.slt | 155 ++++++ .../mz_aws_privatelink_connections.slt | 85 +++ .../mz_catalog_server_index_accounting.slt | 160 +++--- test/sqllogictest/mz_kafka_connections.slt | 116 +++++ test/sqllogictest/mz_kafka_source_tables.slt | 2 +- .../mz_ssh_tunnel_connections.slt | 88 ++++ test/sqllogictest/oid.slt | 12 +- test/testdrive/catalog.td | 10 +- test/testdrive/mz-sources.td | 18 + 36 files changed, 1917 insertions(+), 538 deletions(-) create mode 100644 test/sqllogictest/mz_aws_connections.slt create mode 100644 test/sqllogictest/mz_aws_privatelink_connections.slt create mode 100644 test/sqllogictest/mz_kafka_connections.slt create mode 100644 test/sqllogictest/mz_ssh_tunnel_connections.slt diff --git a/doc/user/content/reference/system-catalog/mz_catalog.md b/doc/user/content/reference/system-catalog/mz_catalog.md index fc49cbcde6296..c10f9100761e3 100644 --- a/doc/user/content/reference/system-catalog/mz_catalog.md +++ b/doc/user/content/reference/system-catalog/mz_catalog.md @@ -562,7 +562,7 @@ Field | Type | Meaning `type` | [`text`] | The type of the source: `kafka`, `mysql`, `postgres`, `load-generator`, `progress`, or `subsource`. `connection_id` | [`text`] | The ID of the connection associated with the source, if any. Corresponds to [`mz_connections.id`](/reference/system-catalog/mz_catalog/#mz_connections). `size` | [`text`] | *Deprecated* The [size](/sql/create-source/#sizing-a-source) of the source. -`envelope_type` | [`text`] | For Kafka sources, the [envelope](/sql/create-source/kafka/#envelopes) type: `none`, `upsert`, or `debezium`. `NULL` for other source types. +`envelope_type` | [`text`] | For old-syntax Kafka sources, the [envelope](/sql/create-source/kafka/#envelopes) type: `none`, `upsert`, or `debezium`. `NULL` for new-syntax Kafka sources, whose envelopes are defined per source table (see `mz_kafka_source_tables`), and for other source types. `key_format` | [`text`] | For Kafka sources, the [format](/sql/create-source/kafka#syntax) of the Kafka message key: `avro`, `csv`, `regex`, `bytes`, `json`, `text`, or `NULL`. `value_format` | [`text`] | For Kafka sources, the [format](/sql/create-source/kafka#syntax) of the Kafka message value: `avro`, `csv`, `regex`, `bytes`, `json`, `text`. `NULL` for other source types. `cluster_id` | [`text`] | The ID of the cluster maintaining the source. Corresponds to [`mz_clusters.id`](/reference/system-catalog/mz_catalog/#mz_clusters). diff --git a/doc/user/content/reference/system-catalog/mz_internal.md b/doc/user/content/reference/system-catalog/mz_internal.md index 75812a14efcf5..bc2694d5c903f 100644 --- a/doc/user/content/reference/system-catalog/mz_internal.md +++ b/doc/user/content/reference/system-catalog/mz_internal.md @@ -806,7 +806,7 @@ table and the corresponding upstream Kafka topic being ingested. | ------------------- | ---------------- | -------- | | `id` | [`text`] | The ID of the table. Corresponds to [`mz_catalog.mz_tables.id`](../mz_catalog#mz_tables). | | `topic` | [`text`] | The topic being ingested. | -| `envelope_type` | [`text`] | The [envelope](/sql/create-source/kafka/#envelopes) type: `none`, `upsert`, or `debezium`. `NULL` for other source types. | +| `envelope_type` | [`text`] | The [envelope](/sql/create-source/kafka/#envelopes) type: `none`, `upsert`, or `debezium`. Defaults to `none` when the source table omits an explicit envelope. | | `key_format` | [`text`] | The [format](/sql/create-source/kafka/#syntax) of the Kafka message key: `avro`, `csv`, `regex`, `bytes`, `json`, `text`, or `NULL`. | | `value_format` | [`text`] | The [format](/sql/create-source/kafka/#syntax) of the Kafka message value: `avro`, `csv`, `regex`, `bytes`, `json`, `text`. `NULL` for other source types. | diff --git a/src/adapter/src/catalog.rs b/src/adapter/src/catalog.rs index 8b3a1f919da38..c66df7dd95262 100644 --- a/src/adapter/src/catalog.rs +++ b/src/adapter/src/catalog.rs @@ -32,7 +32,9 @@ use mz_catalog::builtin::{ BUILTIN_PREFIXES, BuiltinCluster, BuiltinLog, BuiltinSource, BuiltinTable, MZ_CATALOG_SERVER_CLUSTER, }; -use mz_catalog::config::{BuiltinItemMigrationConfig, ClusterReplicaSizeMap, Config, StateConfig}; +use mz_catalog::config::{ + AwsPrincipalContext, BuiltinItemMigrationConfig, ClusterReplicaSizeMap, Config, StateConfig, +}; #[cfg(test)] use mz_catalog::durable::CatalogError; use mz_catalog::durable::{ @@ -45,6 +47,7 @@ use mz_catalog::memory::objects::{ CatalogCollectionEntry, CatalogEntry, CatalogItem, Cluster, ClusterReplica, Database, NetworkPolicy, Role, RoleAuth, Schema, }; +use mz_cloud_resources::AwsExternalIdPrefix; use mz_compute_types::dataflows::DataflowDescription; use mz_controller::clusters::ReplicaLocation; use mz_controller_types::{ClusterId, ReplicaId}; @@ -333,6 +336,23 @@ impl ConnectionResolver for ConnCatalog<'_> { } } +/// AWS environment context for a debug catalog. +/// +/// The `mz_aws_connections` and `mz_aws_privatelink_connections` builtin +/// materialized views fold `mz_aws_account_id()`, `mz_aws_external_id_prefix()`, +/// and `mz_aws_connection_role_arn()` into their optimized expressions. A debug +/// catalog opened to be compared against a live environment (the testdrive +/// consistency check) must fold those functions to the same values the live +/// environment used, otherwise the optimized expressions diverge. This mirrors +/// how `environment_id` is threaded into the catalog copy. Fields left `None` +/// fold to SQL NULL, matching an environment without that context. +#[derive(Debug, Clone, Default)] +pub struct DebugAwsContext { + pub aws_account_id: Option, + pub aws_external_id_prefix: Option, + pub aws_connection_role_arn: Option, +} + impl Catalog { /// Returns the catalog's transient revision, which starts at 1 and is /// incremented on every change. This is not persisted to disk, and will @@ -378,6 +398,28 @@ impl Catalog { f(catalog).await } + /// Like [`Catalog::with_debug`], but folds the given AWS context into the + /// catalog. Used to exercise builtin materialized views that reproduce AWS + /// environment context in SQL (see [`DebugAwsContext`]). + pub async fn with_debug_aws_context(aws_context: DebugAwsContext, f: F) -> T + where + F: FnOnce(Catalog) -> Fut, + Fut: Future, + { + let persist_client = PersistClient::new_for_tests().await; + let organization_id = Uuid::new_v4(); + let bootstrap_args = test_bootstrap_args(); + let catalog = Self::open_debug_catalog_with_aws_context( + persist_client, + organization_id, + &bootstrap_args, + Some(aws_context), + ) + .await + .expect("can open debug catalog"); + f(catalog).await + } + /// Like [`Catalog::with_debug`], but the catalog created believes that bootstrap is still /// in progress. pub async fn with_debug_in_bootstrap(f: F) -> T @@ -441,6 +483,44 @@ impl Catalog { system_parameter_defaults, bootstrap_args, None, + None, + ) + .await + } + + /// Like [`Catalog::open_debug_catalog`], but folds `aws_context` into the + /// catalog (see [`DebugAwsContext`]). + /// + /// Calls `open_debug_catalog_inner` directly rather than delegating through + /// [`Catalog::open_debug_catalog`], to avoid adding an `async fn` layer to + /// the returned future. The catalog-open future is close to the crate + /// `recursion_limit`, and the extra nesting overflows it when computing the + /// layout of callers such as the `catalog` benchmark. + pub async fn open_debug_catalog_with_aws_context( + persist_client: PersistClient, + organization_id: Uuid, + bootstrap_args: &BootstrapArgs, + aws_context: Option, + ) -> Result { + let now = SYSTEM_TIME.clone(); + let environment_id = None; + let openable_storage = TestCatalogStateBuilder::new(persist_client.clone()) + .with_organization_id(organization_id) + .with_default_deploy_generation() + .build() + .await?; + let storage = openable_storage.open(now().into(), bootstrap_args).await?; + let system_parameter_defaults = BTreeMap::default(); + Self::open_debug_catalog_inner( + persist_client, + storage, + now, + environment_id, + &DUMMY_BUILD_INFO, + system_parameter_defaults, + bootstrap_args, + None, + aws_context, ) .await } @@ -473,6 +553,7 @@ impl Catalog { system_parameter_defaults, bootstrap_args, None, + None, ) .await } @@ -489,6 +570,7 @@ impl Catalog { build_info: &'static BuildInfo, bootstrap_args: &BootstrapArgs, enable_expression_cache_override: Option, + aws_context: Option, ) -> Result { let openable_storage = TestCatalogStateBuilder::new(persist_client.clone()) .with_organization_id(environment_id.organization_id()) @@ -510,6 +592,7 @@ impl Catalog { system_parameter_defaults, bootstrap_args, enable_expression_cache_override, + aws_context, ) .await } @@ -523,6 +606,7 @@ impl Catalog { system_parameter_defaults: BTreeMap, bootstrap_args: &BootstrapArgs, enable_expression_cache_override: Option, + aws_context: Option, ) -> Result { let metrics_registry = &MetricsRegistry::new(); let secrets_reader = Arc::new(InMemorySecretsController::new()); @@ -532,6 +616,37 @@ impl Catalog { let replica_size = &bootstrap_args.default_cluster_replica_size; let read_only = false; + // Fold the requested AWS context into the connection context and + // principal context. The builtin AWS connection views reproduce these + // values in SQL, so a catalog compared against a live environment must + // resolve them identically. When `aws_context` is `None` the default + // `for_tests` context is used unchanged. + let mut connection_context = ConnectionContext::for_tests(secrets_reader); + let aws_principal_context = match aws_context { + None => None, + Some(aws_context) => { + connection_context.aws_external_id_prefix = + aws_context.aws_external_id_prefix.as_deref().map(|prefix| { + AwsExternalIdPrefix::new_from_cli_argument_or_environment_variable(prefix) + .expect("infallible") + }); + connection_context.aws_connection_role_arn = aws_context.aws_connection_role_arn; + aws_context.aws_account_id.map(|aws_account_id| { + AwsPrincipalContext { + aws_account_id, + // Only `aws_account_id` reaches the `mz_aws_account_id()` + // fold. The external id prefix is read from the connection + // context above, so any placeholder works here. + aws_external_id_prefix: + AwsExternalIdPrefix::new_from_cli_argument_or_environment_variable( + "debug", + ) + .expect("infallible"), + } + }) + } + }; + let OpenCatalogResult { catalog, migrated_storage_collections_0dt: _, @@ -576,10 +691,10 @@ impl Catalog { remote_system_parameters: None, availability_zones: vec![], egress_addresses: vec![], - aws_principal_context: None, + aws_principal_context, aws_privatelink_availability_zones: None, http_host_name: None, - connection_context: ConnectionContext::for_tests(secrets_reader), + connection_context, builtin_item_migration_config: BuiltinItemMigrationConfig { persist_client: persist_client.clone(), read_only, @@ -2416,7 +2531,7 @@ mod tests { use mz_sql::session::vars::{SystemVars, VarInput}; use crate::catalog::state::LocalExpressionCache; - use crate::catalog::{Catalog, Op}; + use crate::catalog::{Catalog, DebugAwsContext, Op}; use crate::optimize::dataflows::{EvalTime, ExprPrep, ExprPrepOneShot}; use crate::session::Session; @@ -2845,7 +2960,20 @@ mod tests { #[mz_ore::test(tokio::test)] #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `TLS_client_method` on OS `linux` async fn verify_builtin_descs() { - Catalog::with_debug(|catalog| async move { + // Provide a full AWS context so the builtin AWS connection views resolve + // as they do in a cloud environment. Without an account id, + // `mz_aws_privatelink_connections.principal` folds to NULL, its + // `WHERE principal IS NOT NULL` filter is statically unsatisfiable, and + // the optimizer infers an extra empty key that the declared desc, which + // describes the cloud shape, does not carry. + let aws_context = DebugAwsContext { + aws_account_id: Some("123456789000".to_string()), + aws_external_id_prefix: Some("eb5cb59b-e2fe-41f3-87ca-d2176a495345".to_string()), + aws_connection_role_arn: Some( + "arn:aws:iam::123456789000:role/MaterializeConnection".to_string(), + ), + }; + Catalog::with_debug_aws_context(aws_context, |catalog| async move { let conn_catalog = catalog.for_system_session(); for builtin in BUILTINS::iter() { diff --git a/src/adapter/src/catalog/builtin_table_updates.rs b/src/adapter/src/catalog/builtin_table_updates.rs index 7c100ef46ef30..24ea3082ccf43 100644 --- a/src/adapter/src/catalog/builtin_table_updates.rs +++ b/src/adapter/src/catalog/builtin_table_updates.rs @@ -15,22 +15,19 @@ use mz_adapter_types::compaction::CompactionWindow; use mz_audit_log::VersionedStorageUsage; use mz_catalog::SYSTEM_CONN_ID; use mz_catalog::builtin::{ - BuiltinTable, MZ_AGGREGATES, MZ_ARRAY_TYPES, MZ_AWS_CONNECTIONS, - MZ_AWS_PRIVATELINK_CONNECTIONS, MZ_BASE_TYPES, MZ_CLUSTER_REPLICA_SIZE_INTERNAL, + BuiltinTable, MZ_AGGREGATES, MZ_ARRAY_TYPES, MZ_BASE_TYPES, MZ_CLUSTER_REPLICA_SIZE_INTERNAL, MZ_CLUSTER_REPLICA_SIZES, MZ_COLUMNS, MZ_EGRESS_IPS, MZ_FUNCTIONS, - MZ_HISTORY_RETENTION_STRATEGIES, MZ_ICEBERG_SINKS, MZ_INDEX_COLUMNS, MZ_KAFKA_CONNECTIONS, - MZ_KAFKA_SINKS, MZ_LICENSE_KEYS, MZ_LIST_TYPES, MZ_MAP_TYPES, - MZ_MATERIALIZED_VIEW_REFRESH_STRATEGIES, MZ_OBJECT_DEPENDENCIES, MZ_OBJECT_GLOBAL_IDS, - MZ_OPERATORS, MZ_PSEUDO_TYPES, MZ_REPLACEMENTS, MZ_ROLE_AUTH, MZ_SESSIONS, MZ_SINKS, - MZ_SOURCE_REFERENCES, MZ_SSH_TUNNEL_CONNECTIONS, MZ_STORAGE_USAGE_BY_SHARD, MZ_SUBSCRIPTIONS, - MZ_TABLES, MZ_TYPE_PG_METADATA, MZ_TYPES, MZ_VIEWS, MZ_WEBHOOKS_SOURCES, + MZ_HISTORY_RETENTION_STRATEGIES, MZ_ICEBERG_SINKS, MZ_INDEX_COLUMNS, MZ_KAFKA_SINKS, + MZ_LICENSE_KEYS, MZ_LIST_TYPES, MZ_MAP_TYPES, MZ_MATERIALIZED_VIEW_REFRESH_STRATEGIES, + MZ_OBJECT_DEPENDENCIES, MZ_OBJECT_GLOBAL_IDS, MZ_OPERATORS, MZ_PSEUDO_TYPES, MZ_REPLACEMENTS, + MZ_ROLE_AUTH, MZ_SESSIONS, MZ_SINKS, MZ_SOURCE_REFERENCES, MZ_STORAGE_USAGE_BY_SHARD, + MZ_SUBSCRIPTIONS, MZ_TABLES, MZ_TYPE_PG_METADATA, MZ_TYPES, MZ_VIEWS, MZ_WEBHOOKS_SOURCES, }; -use mz_catalog::config::AwsPrincipalContext; use mz_catalog::durable::SourceReferences; use mz_catalog::memory::error::Error; use mz_catalog::memory::objects::{ - CatalogEntry, CatalogItem, Connection, DataSourceDesc, Func, Index, MaterializedView, Sink, - Table, TableDataSource, Type, View, + CatalogEntry, CatalogItem, DataSourceDesc, Func, Index, MaterializedView, Sink, Table, + TableDataSource, Type, View, }; use mz_expr::MirScalarExpr; use mz_license_keys::ValidatedLicenseKey; @@ -51,13 +48,8 @@ use mz_sql::ast::{CreateIndexStatement, Statement}; use mz_sql::catalog::{CatalogType, TypeCategory}; use mz_sql::func::FuncImplCatalogDetails; use mz_sql::names::SchemaSpecifier; -use mz_sql::plan::{ConnectionDetails, SshKey}; use mz_sql_parser::ast::display::AstDisplay; use mz_storage_client::client::TableData; -use mz_storage_types::connections::KafkaConnection; -use mz_storage_types::connections::aws::{AwsAuth, AwsConnection}; -use mz_storage_types::connections::inline::ReferencedConnection; -use mz_storage_types::connections::string_or_secret::StringOrSecret; use mz_storage_types::sinks::{IcebergSinkConnection, KafkaSinkConnection, StorageSinkConnection}; use smallvec::smallvec; @@ -210,9 +202,11 @@ impl CatalogState { self.pack_func_update(id, schema_id, name, owner_id, func, diff) } CatalogItem::Log(_) | CatalogItem::Secret(_) => vec![], - CatalogItem::Connection(connection) => { - self.pack_connection_update(id, connection, diff) - } + // Connection details (mz_kafka_connections, mz_ssh_tunnel_connections, + // mz_aws_connections, mz_aws_privatelink_connections) are now derived + // from the persisted create_sql by materialized views over + // mz_catalog_raw, so connections need no special packing here. + CatalogItem::Connection(_) => vec![], }; if !entry.item().is_temporary() { @@ -393,191 +387,6 @@ impl CatalogState { )] } - fn pack_connection_update( - &self, - id: CatalogItemId, - connection: &Connection, - diff: Diff, - ) -> Vec> { - let mut updates = vec![]; - match connection.details { - ConnectionDetails::Kafka(ref kafka) => { - updates.extend(self.pack_kafka_connection_update(id, kafka, diff)); - } - ConnectionDetails::Aws(ref aws_config) => { - match self.pack_aws_connection_update(id, aws_config, diff) { - Ok(update) => { - updates.push(update); - } - Err(e) => { - tracing::error!(%id, %e, "failed writing row to mz_aws_connections table"); - } - } - } - ConnectionDetails::AwsPrivatelink(_) => { - if let Some(aws_principal_context) = self.aws_principal_context.as_ref() { - updates.push(self.pack_aws_privatelink_connection_update( - id, - aws_principal_context, - diff, - )); - } else { - tracing::error!(%id, "missing AWS principal context; cannot write row to mz_aws_privatelink_connections table"); - } - } - ConnectionDetails::Ssh { - ref key_1, - ref key_2, - .. - } => { - updates.push(self.pack_ssh_tunnel_connection_update(id, key_1, key_2, diff)); - } - ConnectionDetails::Csr(_) - | ConnectionDetails::GlueSchemaRegistry(_) - | ConnectionDetails::Gcp(_) - | ConnectionDetails::Postgres(_) - | ConnectionDetails::MySql(_) - | ConnectionDetails::SqlServer(_) - | ConnectionDetails::IcebergCatalog(_) => (), - }; - updates - } - - pub(crate) fn pack_ssh_tunnel_connection_update( - &self, - id: CatalogItemId, - key_1: &SshKey, - key_2: &SshKey, - diff: Diff, - ) -> BuiltinTableUpdate<&'static BuiltinTable> { - BuiltinTableUpdate::row( - &*MZ_SSH_TUNNEL_CONNECTIONS, - Row::pack_slice(&[ - Datum::String(&id.to_string()), - Datum::String(key_1.public_key().as_str()), - Datum::String(key_2.public_key().as_str()), - ]), - diff, - ) - } - - fn pack_kafka_connection_update( - &self, - id: CatalogItemId, - kafka: &KafkaConnection, - diff: Diff, - ) -> Vec> { - let progress_topic = kafka.progress_topic(&self.config.connection_context, id); - let mut row = Row::default(); - row.packer() - .try_push_array( - &[ArrayDimension { - lower_bound: 1, - length: kafka.brokers.len(), - }], - kafka - .brokers - .iter() - .map(|broker| Datum::String(&broker.address)), - ) - .expect("kafka.brokers is 1 dimensional, and its length is used for the array length"); - let brokers = row.unpack_first(); - vec![BuiltinTableUpdate::row( - &*MZ_KAFKA_CONNECTIONS, - Row::pack_slice(&[ - Datum::String(&id.to_string()), - brokers, - Datum::String(&progress_topic), - ]), - diff, - )] - } - - pub fn pack_aws_privatelink_connection_update( - &self, - connection_id: CatalogItemId, - aws_principal_context: &AwsPrincipalContext, - diff: Diff, - ) -> BuiltinTableUpdate<&'static BuiltinTable> { - let id = &MZ_AWS_PRIVATELINK_CONNECTIONS; - let row = Row::pack_slice(&[ - Datum::String(&connection_id.to_string()), - Datum::String(&aws_principal_context.to_principal_string(connection_id)), - ]); - BuiltinTableUpdate::row(id, row, diff) - } - - pub fn pack_aws_connection_update( - &self, - connection_id: CatalogItemId, - aws_config: &AwsConnection, - diff: Diff, - ) -> Result, anyhow::Error> { - let id = &MZ_AWS_CONNECTIONS; - - let mut access_key_id = None; - let mut access_key_id_secret_id = None; - let mut secret_access_key_secret_id = None; - let mut session_token = None; - let mut session_token_secret_id = None; - let mut assume_role_arn = None; - let mut assume_role_session_name = None; - let mut principal = None; - let mut external_id = None; - let mut example_trust_policy = None; - match &aws_config.auth { - AwsAuth::Credentials(credentials) => { - match &credentials.access_key_id { - StringOrSecret::String(s) => access_key_id = Some(s.as_str()), - StringOrSecret::Secret(s) => access_key_id_secret_id = Some(s.to_string()), - } - secret_access_key_secret_id = Some(credentials.secret_access_key.to_string()); - match credentials.session_token.as_ref() { - None => (), - Some(StringOrSecret::String(s)) => session_token = Some(s.as_str()), - Some(StringOrSecret::Secret(s)) => { - session_token_secret_id = Some(s.to_string()) - } - } - } - AwsAuth::AssumeRole(assume_role) => { - assume_role_arn = Some(assume_role.arn.as_str()); - assume_role_session_name = assume_role.session_name.as_deref(); - principal = self - .config - .connection_context - .aws_connection_role_arn - .as_deref(); - external_id = - Some(assume_role.external_id(&self.config.connection_context, connection_id)?); - example_trust_policy = { - let policy = assume_role - .example_trust_policy(&self.config.connection_context, connection_id)?; - let policy = Jsonb::from_serde_json(policy).expect("valid json"); - Some(policy.into_row()) - }; - } - } - - let row = Row::pack_slice(&[ - Datum::String(&connection_id.to_string()), - Datum::from(aws_config.endpoint.as_deref()), - Datum::from(aws_config.region.as_deref()), - Datum::from(access_key_id), - Datum::from(access_key_id_secret_id.as_deref()), - Datum::from(secret_access_key_secret_id.as_deref()), - Datum::from(session_token), - Datum::from(session_token_secret_id.as_deref()), - Datum::from(assume_role_arn), - Datum::from(assume_role_session_name), - Datum::from(principal), - Datum::from(external_id.as_deref()), - Datum::from(example_trust_policy.as_ref().map(|p| p.into_element())), - ]); - - Ok(BuiltinTableUpdate::row(id, row, diff)) - } - fn pack_view_update( &self, id: CatalogItemId, diff --git a/src/adapter/src/catalog/open.rs b/src/adapter/src/catalog/open.rs index 57e99d313dda8..afe4f8f1729c7 100644 --- a/src/adapter/src/catalog/open.rs +++ b/src/adapter/src/catalog/open.rs @@ -166,6 +166,10 @@ impl Catalog { build_info: config.build_info, now: config.now.clone(), connection_context: config.connection_context, + aws_account_id: config + .aws_principal_context + .as_ref() + .map(|c| c.aws_account_id.clone()), helm_chart_version: config.helm_chart_version, }, cluster_replica_sizes: config.cluster_replica_sizes, diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index 77da28cefcd23..58b4ec995a88a 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -328,6 +328,33 @@ static MIGRATIONS: LazyLock> = LazyLock::new(|| { MZ_INTERNAL_SCHEMA, "mz_kafka_source_tables", ), + // Converting the connection-detail builtin tables to materialized views + // changes their catalog fingerprint, so each needs an explicit + // replacement step. + MigrationStep::replacement( + "26.37.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_kafka_connections", + ), + MigrationStep::replacement( + "26.37.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_ssh_tunnel_connections", + ), + MigrationStep::replacement( + "26.37.0-dev.0", + CatalogItemType::MaterializedView, + MZ_INTERNAL_SCHEMA, + "mz_aws_connections", + ), + MigrationStep::replacement( + "26.37.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_aws_privatelink_connections", + ), ] }); diff --git a/src/adapter/src/catalog/state.rs b/src/adapter/src/catalog/state.rs index d4932ebe883be..63a704a30e6da 100644 --- a/src/adapter/src/catalog/state.rs +++ b/src/adapter/src/catalog/state.rs @@ -323,6 +323,7 @@ impl CatalogState { connection_context: ConnectionContext::for_tests(Arc::new( InMemorySecretsController::new(), )), + aws_account_id: None, helm_chart_version: None, }, cluster_replica_sizes: ClusterReplicaSizeMap::for_tests(), diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs index 2ec708348e946..3dfb9e2ae8774 100644 --- a/src/catalog/src/builtin.rs +++ b/src/catalog/src/builtin.rs @@ -1108,7 +1108,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::Log(&MZ_COMPUTE_HYDRATION_TIMES_PER_WORKER), Builtin::Log(&MZ_COMPUTE_OPERATOR_HYDRATION_STATUSES_PER_WORKER), Builtin::Table(&MZ_KAFKA_SINKS), - Builtin::Table(&MZ_KAFKA_CONNECTIONS), + Builtin::MaterializedView(&MZ_KAFKA_CONNECTIONS), Builtin::MaterializedView(&MZ_KAFKA_SOURCES), Builtin::Table(&MZ_OBJECT_DEPENDENCIES), Builtin::Table(&MZ_ICEBERG_SINKS), @@ -1150,7 +1150,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::MaterializedView(&MZ_CLUSTER_AUTO_SCALING_STRATEGIES), Builtin::MaterializedView(&MZ_SECRETS), Builtin::MaterializedView(&MZ_CONNECTIONS), - Builtin::Table(&MZ_SSH_TUNNEL_CONNECTIONS), + Builtin::MaterializedView(&MZ_SSH_TUNNEL_CONNECTIONS), Builtin::MaterializedView(&MZ_CLUSTER_REPLICAS), Builtin::Source(&MZ_CLUSTER_REPLICA_METRICS_HISTORY), Builtin::View(&MZ_CLUSTER_REPLICA_METRICS), @@ -1161,8 +1161,8 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::MaterializedView(&MZ_AUDIT_EVENTS), Builtin::Table(&MZ_STORAGE_USAGE_BY_SHARD), Builtin::Table(&MZ_EGRESS_IPS), - Builtin::Table(&MZ_AWS_PRIVATELINK_CONNECTIONS), - Builtin::Table(&MZ_AWS_CONNECTIONS), + Builtin::MaterializedView(&MZ_AWS_PRIVATELINK_CONNECTIONS), + Builtin::MaterializedView(&MZ_AWS_CONNECTIONS), Builtin::Table(&MZ_SUBSCRIPTIONS), Builtin::Table(&MZ_SESSIONS), Builtin::MaterializedView(&MZ_OVERRIDDEN_SYSTEM_PARAMETERS), diff --git a/src/catalog/src/builtin/builtin.rs b/src/catalog/src/builtin/builtin.rs index 51834a5f581f5..e62910c4ab1fe 100644 --- a/src/catalog/src/builtin/builtin.rs +++ b/src/catalog/src/builtin/builtin.rs @@ -338,7 +338,7 @@ SELECT * FROM builtin_sources"); ("size", "*Deprecated* The size of the source."), ( "envelope_type", - "For Kafka sources, the envelope type: `none`, `upsert`, or `debezium`. `NULL` for other source types.", + "For old-syntax Kafka sources, the envelope type: `none`, `upsert`, or `debezium`. `NULL` for new-syntax Kafka sources, whose envelopes are defined per source table (see `mz_kafka_source_tables`), and for other source types.", ), ( "key_format", diff --git a/src/catalog/src/builtin/mz_catalog.rs b/src/catalog/src/builtin/mz_catalog.rs index 3de16050b1f21..c2fb6d2a75be2 100644 --- a/src/catalog/src/builtin/mz_catalog.rs +++ b/src/catalog/src/builtin/mz_catalog.rs @@ -262,43 +262,88 @@ pub static MZ_KAFKA_SINKS: LazyLock = LazyLock::new(|| BuiltinTabl column_semantic_types: &[("id", SemanticType::CatalogItemId)], }), }); -pub static MZ_KAFKA_CONNECTIONS: LazyLock = LazyLock::new(|| BuiltinTable { - name: "mz_kafka_connections", - schema: MZ_CATALOG_SCHEMA, - oid: oid::TABLE_MZ_KAFKA_CONNECTIONS_OID, - desc: RelationDesc::builder() - .with_column("id", SqlScalarType::String.nullable(false)) - .with_column( - "brokers", - SqlScalarType::Array(Box::new(SqlScalarType::String)).nullable(false), - ) - .with_column("sink_progress_topic", SqlScalarType::String.nullable(false)) - .finish(), - column_comments: BTreeMap::from_iter([ - ("id", "The ID of the connection."), - ( - "brokers", - "The addresses of the Kafka brokers to connect to.", - ), - ( - "sink_progress_topic", - "The name of the Kafka topic where any sinks associated with this connection will track their progress information and other metadata. The contents of this topic are unspecified.", - ), - ]), - is_retained_metrics_object: false, - access: vec![PUBLIC_SELECT], - ontology: Some(Ontology { - entity_name: "kafka_connection", - description: "Kafka-specific connection configuration (brokers, progress topic)", - links: &const { - [OntologyLink { - name: "details_of", - target: "connection", - properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), - }] - }, - column_semantic_types: &[("id", SemanticType::CatalogItemId)], - }), +// Reads Item rows from `mz_catalog_raw`, pulls the broker addresses and any +// explicit progress topic out of the persisted `create_sql` via +// `parse_connection_details`, and keeps only kafka connections (the connection +// type is derived the same way `mz_connections` derives it). The default +// progress topic is reconstructed here rather than in the helper because it +// needs the environment id and the connection's own id. +pub static MZ_KAFKA_CONNECTIONS: LazyLock = LazyLock::new(|| { + BuiltinMaterializedView { + name: "mz_kafka_connections", + schema: MZ_CATALOG_SCHEMA, + oid: oid::MV_MZ_KAFKA_CONNECTIONS_OID, + desc: RelationDesc::builder() + .with_column("id", SqlScalarType::String.nullable(false)) + .with_column( + "brokers", + SqlScalarType::Array(Box::new(SqlScalarType::String)).nullable(false), + ) + .with_column("sink_progress_topic", SqlScalarType::String.nullable(false)) + .with_key(vec![0]) + .finish(), + column_comments: BTreeMap::from_iter([ + ("id", "The ID of the connection."), + ( + "brokers", + "The addresses of the Kafka brokers to connect to.", + ), + ( + "sink_progress_topic", + "The name of the Kafka topic where any sinks associated with this connection will track their progress information and other metadata. The contents of this topic are unspecified.", + ), + ]), + // The `sink_progress_topic` default must match + // `KafkaConnection::progress_topic` in + // src/storage-types/src/connections.rs. Keep the two in sync. + sql: " +IN CLUSTER mz_catalog_server +WITH ( + ASSERT NOT NULL id, + ASSERT NOT NULL brokers, + ASSERT NOT NULL sink_progress_topic +) AS +SELECT + mz_internal.parse_catalog_id(r.data->'key'->'gid') AS id, + ARRAY( + SELECT b.value + FROM jsonb_array_elements_text(details->'brokers') + WITH ORDINALITY AS b(value, ord) + ORDER BY b.ord + ) AS brokers, + COALESCE( + details->>'progress_topic', + '_materialize-progress-' || mz_environment_id() || '-' + || mz_internal.parse_catalog_id(r.data->'key'->'gid') + ) AS sink_progress_topic +FROM + mz_internal.mz_catalog_raw r, + LATERAL ( + SELECT mz_internal.parse_connection_details( + r.data->'value'->'definition'->'V1'->>'create_sql') + ) AS d(details) +WHERE + r.data->>'kind' = 'Item' AND + -- The connection_type filter selects the kind. A non-matching row yields a + -- NULL connection_type and is dropped here, so no `details IS NOT NULL` is + -- needed (parse_connection_details returns jsonb null, which passes it). + mz_internal.parse_catalog_create_sql( + r.data->'value'->'definition'->'V1'->>'create_sql')->>'connection_type' = 'kafka'", + is_retained_metrics_object: false, + access: vec![PUBLIC_SELECT], + ontology: Some(Ontology { + entity_name: "kafka_connection", + description: "Kafka-specific connection configuration (brokers, progress topic)", + links: &const { + [OntologyLink { + name: "details_of", + target: "connection", + properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), + }] + }, + column_semantic_types: &[("id", SemanticType::CatalogItemId)], + }), + } }); pub static MZ_KAFKA_SOURCES: LazyLock = LazyLock::new(|| { BuiltinMaterializedView { @@ -1112,41 +1157,71 @@ WHERE } }); -pub static MZ_SSH_TUNNEL_CONNECTIONS: LazyLock = LazyLock::new(|| BuiltinTable { - name: "mz_ssh_tunnel_connections", - schema: MZ_CATALOG_SCHEMA, - oid: oid::TABLE_MZ_SSH_TUNNEL_CONNECTIONS_OID, - desc: RelationDesc::builder() - .with_column("id", SqlScalarType::String.nullable(false)) - .with_column("public_key_1", SqlScalarType::String.nullable(false)) - .with_column("public_key_2", SqlScalarType::String.nullable(false)) - .finish(), - column_comments: BTreeMap::from_iter([ - ("id", "The ID of the connection."), - ( - "public_key_1", - "The first public key associated with the SSH tunnel.", - ), - ( - "public_key_2", - "The second public key associated with the SSH tunnel.", - ), - ]), - is_retained_metrics_object: false, - access: vec![PUBLIC_SELECT], - ontology: Some(Ontology { - entity_name: "ssh_tunnel_connection", - description: "SSH tunnel connection with public keys", - links: &const { - [OntologyLink { - name: "details_of", - target: "connection", - properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), - }] - }, - column_semantic_types: &[("id", SemanticType::CatalogItemId)], - }), -}); +// The two SSH public keys are stored verbatim in the persisted `create_sql` +// (they are generated, not user-supplied, and explicitly not redacted), so the +// view reads them straight out via `parse_connection_details` and keeps only +// ssh-tunnel connections. +pub static MZ_SSH_TUNNEL_CONNECTIONS: LazyLock = + LazyLock::new(|| BuiltinMaterializedView { + name: "mz_ssh_tunnel_connections", + schema: MZ_CATALOG_SCHEMA, + oid: oid::MV_MZ_SSH_TUNNEL_CONNECTIONS_OID, + desc: RelationDesc::builder() + .with_column("id", SqlScalarType::String.nullable(false)) + .with_column("public_key_1", SqlScalarType::String.nullable(false)) + .with_column("public_key_2", SqlScalarType::String.nullable(false)) + .with_key(vec![0]) + .finish(), + column_comments: BTreeMap::from_iter([ + ("id", "The ID of the connection."), + ( + "public_key_1", + "The first public key associated with the SSH tunnel.", + ), + ( + "public_key_2", + "The second public key associated with the SSH tunnel.", + ), + ]), + sql: " +IN CLUSTER mz_catalog_server +WITH ( + ASSERT NOT NULL id, + ASSERT NOT NULL public_key_1, + ASSERT NOT NULL public_key_2 +) AS +SELECT + mz_internal.parse_catalog_id(r.data->'key'->'gid') AS id, + details->>'public_key_1' AS public_key_1, + details->>'public_key_2' AS public_key_2 +FROM + mz_internal.mz_catalog_raw r, + LATERAL ( + SELECT mz_internal.parse_connection_details( + r.data->'value'->'definition'->'V1'->>'create_sql') + ) AS d(details) +WHERE + r.data->>'kind' = 'Item' AND + -- The connection_type filter selects the kind. A non-matching row yields a + -- NULL connection_type and is dropped here, so no `details IS NOT NULL` is + -- needed (parse_connection_details returns jsonb null, which passes it). + mz_internal.parse_catalog_create_sql( + r.data->'value'->'definition'->'V1'->>'create_sql')->>'connection_type' = 'ssh-tunnel'", + is_retained_metrics_object: false, + access: vec![PUBLIC_SELECT], + ontology: Some(Ontology { + entity_name: "ssh_tunnel_connection", + description: "SSH tunnel connection with public keys", + links: &const { + [OntologyLink { + name: "details_of", + target: "connection", + properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), + }] + }, + column_semantic_types: &[("id", SemanticType::CatalogItemId)], + }), + }); // mz_sources is generated dynamically in BUILTINS_STATIC via builtin::make_mz_sources() // with builtin source/log entries inlined as VALUES. See builtin/builtin.rs. pub static MZ_SINKS: LazyLock = LazyLock::new(|| { @@ -2706,36 +2781,68 @@ pub static MZ_EGRESS_IPS: LazyLock = LazyLock::new(|| BuiltinTable }), }); -pub static MZ_AWS_PRIVATELINK_CONNECTIONS: LazyLock = - LazyLock::new(|| BuiltinTable { - name: "mz_aws_privatelink_connections", - schema: MZ_CATALOG_SCHEMA, - oid: oid::TABLE_MZ_AWS_PRIVATELINK_CONNECTIONS_OID, - desc: RelationDesc::builder() - .with_column("id", SqlScalarType::String.nullable(false)) - .with_column("principal", SqlScalarType::String.nullable(false)) - .finish(), - column_comments: BTreeMap::from_iter([ - ("id", "The ID of the connection."), - ( - "principal", - "The AWS Principal that Materialize will use to connect to the VPC endpoint.", - ), - ]), - is_retained_metrics_object: false, - access: vec![PUBLIC_SELECT], - ontology: Some(Ontology { - entity_name: "aws_privatelink_connection", - description: "AWS PrivateLink connection configuration", - links: &const { - [OntologyLink { - name: "details_of", - target: "connection", - properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), - }] - }, - column_semantic_types: &[("id", SemanticType::CatalogItemId)], - }), +// The privatelink principal is entirely context-derived (there is nothing to +// parse out of create_sql), so this view selects the aws-privatelink +// connections and reconstructs the principal ARN from the plan-time AWS context +// functions. `principal` is NOT NULL and the removed packer emitted no row when +// the AWS principal context was absent. A view cannot skip a row, but the +// context functions fold to NULL without the context, so `WHERE principal IS +// NOT NULL` reproduces that skip (zero rows on envs without the context). +pub static MZ_AWS_PRIVATELINK_CONNECTIONS: LazyLock = + LazyLock::new(|| { + BuiltinMaterializedView { + name: "mz_aws_privatelink_connections", + schema: MZ_CATALOG_SCHEMA, + oid: oid::MV_MZ_AWS_PRIVATELINK_CONNECTIONS_OID, + desc: RelationDesc::builder() + .with_column("id", SqlScalarType::String.nullable(false)) + .with_column("principal", SqlScalarType::String.nullable(false)) + .with_key(vec![0]) + .finish(), + column_comments: BTreeMap::from_iter([ + ("id", "The ID of the connection."), + ( + "principal", + "The AWS Principal that Materialize will use to connect to the VPC endpoint.", + ), + ]), + // `principal` reproduces `AwsPrincipalContext::to_principal_string` in + // src/catalog/src/config.rs. Keep the two in sync. + sql: " +IN CLUSTER mz_catalog_server +WITH ( + ASSERT NOT NULL id, + ASSERT NOT NULL principal +) AS +SELECT id, principal FROM ( + SELECT + mz_internal.parse_catalog_id(r.data->'key'->'gid') AS id, + 'arn:aws:iam::' || mz_aws_account_id() || ':role/mz_' + || mz_aws_external_id_prefix() || '_' + || mz_internal.parse_catalog_id(r.data->'key'->'gid') AS principal + FROM mz_internal.mz_catalog_raw r + WHERE + r.data->>'kind' = 'Item' AND + mz_internal.parse_catalog_create_sql( + r.data->'value'->'definition'->'V1'->>'create_sql')->>'connection_type' + = 'aws-privatelink' +) +WHERE principal IS NOT NULL", + is_retained_metrics_object: false, + access: vec![PUBLIC_SELECT], + ontology: Some(Ontology { + entity_name: "aws_privatelink_connection", + description: "AWS PrivateLink connection configuration", + links: &const { + [OntologyLink { + name: "details_of", + target: "connection", + properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), + }] + }, + column_semantic_types: &[("id", SemanticType::CatalogItemId)], + }), + } }); pub static MZ_CLUSTER_REPLICA_FRONTIERS: LazyLock = diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs index 9ad99a5e417b1..c84850ec10ed5 100644 --- a/src/catalog/src/builtin/mz_internal.rs +++ b/src/catalog/src/builtin/mz_internal.rs @@ -321,7 +321,7 @@ pub static MZ_KAFKA_SOURCE_TABLES: LazyLock = LazyLock: desc: RelationDesc::builder() .with_column("id", SqlScalarType::String.nullable(false)) .with_column("topic", SqlScalarType::String.nullable(false)) - .with_column("envelope_type", SqlScalarType::String.nullable(true)) + .with_column("envelope_type", SqlScalarType::String.nullable(false)) .with_column("key_format", SqlScalarType::String.nullable(true)) .with_column("value_format", SqlScalarType::String.nullable(true)) .with_key(vec![0]) @@ -334,7 +334,7 @@ pub static MZ_KAFKA_SOURCE_TABLES: LazyLock = LazyLock: ("topic", "The topic being ingested."), ( "envelope_type", - "The envelope type: `none`, `upsert`, or `debezium`. `NULL` for other source types.", + "The envelope type: `none`, `upsert`, or `debezium`. Defaults to `none` when the source table omits an explicit envelope.", ), ( "key_format", @@ -361,7 +361,10 @@ WITH ( SELECT mz_internal.parse_catalog_id(r.data->'key'->'gid') AS id, details->'external_reference'->>0 AS topic, - details->>'envelope_type' AS envelope_type, + -- Kafka defaults to ENVELOPE NONE when the clause is omitted. The parser + -- helper is source-type agnostic and reports NULL for that case, so default + -- to 'none' here, where the join has already scoped rows to kafka. + COALESCE(details->>'envelope_type', 'none') AS envelope_type, details->>'key_format' AS key_format, details->>'value_format' AS value_format FROM @@ -2582,96 +2585,179 @@ pub static MZ_STORAGE_USAGE_BY_SHARD: LazyLock = LazyLock::new(|| }), }); -pub static MZ_AWS_CONNECTIONS: LazyLock = LazyLock::new(|| BuiltinTable { - name: "mz_aws_connections", - schema: MZ_INTERNAL_SCHEMA, - oid: oid::TABLE_MZ_AWS_CONNECTIONS_OID, - desc: RelationDesc::builder() - .with_column("id", SqlScalarType::String.nullable(false)) - .with_column("endpoint", SqlScalarType::String.nullable(true)) - .with_column("region", SqlScalarType::String.nullable(true)) - .with_column("access_key_id", SqlScalarType::String.nullable(true)) - .with_column( - "access_key_id_secret_id", - SqlScalarType::String.nullable(true), - ) - .with_column( - "secret_access_key_secret_id", - SqlScalarType::String.nullable(true), - ) - .with_column("session_token", SqlScalarType::String.nullable(true)) - .with_column( - "session_token_secret_id", - SqlScalarType::String.nullable(true), +// Reads Item rows from `mz_catalog_raw`, pulls the create_sql-derived fields +// (endpoint, region, credentials, assume-role options) out via +// `parse_connection_details`, and keeps only aws connections. The three +// context-derived columns (principal, external_id, example_trust_policy) apply +// to assume-role connections only and are reconstructed here from the plan-time +// AWS context functions, because they depend on the environment, not on +// create_sql. On an environment without the AWS context an assume-role +// connection is dropped entirely (see the WHERE guard below), matching the +// packer this view replaces, which bailed and wrote no row in that case. This +// is not the same as folding the columns to NULL: `example_trust_policy` uses +// `jsonb_build_object`, which keeps null values, so a context-less row would +// carry a non-NULL policy object full of nulls rather than SQL NULL. +pub static MZ_AWS_CONNECTIONS: LazyLock = LazyLock::new(|| { + BuiltinMaterializedView { + name: "mz_aws_connections", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::MV_MZ_AWS_CONNECTIONS_OID, + desc: RelationDesc::builder() + .with_column("id", SqlScalarType::String.nullable(false)) + .with_column("endpoint", SqlScalarType::String.nullable(true)) + .with_column("region", SqlScalarType::String.nullable(true)) + .with_column("access_key_id", SqlScalarType::String.nullable(true)) + .with_column( + "access_key_id_secret_id", + SqlScalarType::String.nullable(true), + ) + .with_column( + "secret_access_key_secret_id", + SqlScalarType::String.nullable(true), + ) + .with_column("session_token", SqlScalarType::String.nullable(true)) + .with_column( + "session_token_secret_id", + SqlScalarType::String.nullable(true), + ) + .with_column("assume_role_arn", SqlScalarType::String.nullable(true)) + .with_column( + "assume_role_session_name", + SqlScalarType::String.nullable(true), + ) + .with_column("principal", SqlScalarType::String.nullable(true)) + .with_column("external_id", SqlScalarType::String.nullable(true)) + .with_column("example_trust_policy", SqlScalarType::Jsonb.nullable(true)) + .with_key(vec![0]) + .finish(), + column_comments: BTreeMap::from_iter([ + ("id", "The ID of the connection."), + ("endpoint", "The value of the `ENDPOINT` option, if set."), + ("region", "The value of the `REGION` option, if set."), + ( + "access_key_id", + "The value of the `ACCESS KEY ID` option, if provided in line.", + ), + ( + "access_key_id_secret_id", + "The ID of the secret referenced by the `ACCESS KEY ID` option, if provided via a secret.", + ), + ( + "secret_access_key_secret_id", + "The ID of the secret referenced by the `SECRET ACCESS KEY` option, if set.", + ), + ( + "session_token", + "The value of the `SESSION TOKEN` option, if provided in line.", + ), + ( + "session_token_secret_id", + "The ID of the secret referenced by the `SESSION TOKEN` option, if provided via a secret.", + ), + ( + "assume_role_arn", + "The value of the `ASSUME ROLE ARN` option, if set.", + ), + ( + "assume_role_session_name", + "The value of the `ASSUME ROLE SESSION NAME` option, if set.", + ), + ( + "principal", + "The ARN of the AWS principal Materialize will use when assuming the provided role, if the connection is configured to use role assumption.", + ), + ( + "external_id", + "The external ID Materialize will use when assuming the provided role, if the connection is configured to use role assumption.", + ), + ( + "example_trust_policy", + "An example of an IAM role trust policy that allows this connection's principal and external ID to assume the role.", + ), + ]), + // `external_id` reproduces `AwsAssumeRole::external_id` and + // `example_trust_policy` reproduces `AwsAssumeRole::example_trust_policy` + // (both in src/storage-types/src/connections/aws.rs). Keep them in sync. + sql: " +IN CLUSTER mz_catalog_server +WITH ( + ASSERT NOT NULL id +) AS +SELECT + mz_internal.parse_catalog_id(r.data->'key'->'gid') AS id, + details->>'endpoint' AS endpoint, + details->>'region' AS region, + details->>'access_key_id' AS access_key_id, + details->>'access_key_id_secret_id' AS access_key_id_secret_id, + details->>'secret_access_key_secret_id' AS secret_access_key_secret_id, + details->>'session_token' AS session_token, + details->>'session_token_secret_id' AS session_token_secret_id, + details->>'assume_role_arn' AS assume_role_arn, + details->>'assume_role_session_name' AS assume_role_session_name, + CASE WHEN details->>'auth_kind' = 'assume-role' + THEN mz_aws_connection_role_arn() + END AS principal, + CASE WHEN details->>'auth_kind' = 'assume-role' + THEN 'mz_' || mz_aws_external_id_prefix() || '_' + || mz_internal.parse_catalog_id(r.data->'key'->'gid') + END AS external_id, + CASE WHEN details->>'auth_kind' = 'assume-role' + THEN jsonb_build_object( + 'Version', '2012-10-17', + 'Statement', jsonb_build_array(jsonb_build_object( + 'Effect', 'Allow', + 'Principal', jsonb_build_object('AWS', mz_aws_connection_role_arn()), + 'Action', 'sts:AssumeRole', + 'Condition', jsonb_build_object( + 'StringEquals', jsonb_build_object( + 'sts:ExternalId', + 'mz_' || mz_aws_external_id_prefix() || '_' + || mz_internal.parse_catalog_id(r.data->'key'->'gid') + ) + ) + )) ) - .with_column("assume_role_arn", SqlScalarType::String.nullable(true)) - .with_column( - "assume_role_session_name", - SqlScalarType::String.nullable(true), + END AS example_trust_policy +FROM + mz_internal.mz_catalog_raw r, + LATERAL ( + SELECT mz_internal.parse_connection_details( + r.data->'value'->'definition'->'V1'->>'create_sql') + ) AS d(details) +WHERE + r.data->>'kind' = 'Item' AND + -- The connection_type filter selects the kind. A non-matching row yields a + -- NULL connection_type and is dropped here, so no `details IS NOT NULL` is + -- needed (parse_connection_details returns jsonb null, which passes it). + mz_internal.parse_catalog_create_sql( + r.data->'value'->'definition'->'V1'->>'create_sql')->>'connection_type' = 'aws' AND + -- Drop assume-role connections when the AWS context is absent, matching the + -- packer this view replaces. `AwsAssumeRole::external_id` needs the external + -- ID prefix and `example_trust_policy` needs the connection role ARN. Either + -- one missing made the packer bail and write no row. Without this guard the + -- view would instead emit a row whose `example_trust_policy` is a non-NULL + -- JSON object full of nulls, because `jsonb_build_object` keeps null values. + ( + details->>'auth_kind' IS DISTINCT FROM 'assume-role' OR ( + mz_aws_external_id_prefix() IS NOT NULL AND + mz_aws_connection_role_arn() IS NOT NULL ) - .with_column("principal", SqlScalarType::String.nullable(true)) - .with_column("external_id", SqlScalarType::String.nullable(true)) - .with_column("example_trust_policy", SqlScalarType::Jsonb.nullable(true)) - .finish(), - column_comments: BTreeMap::from_iter([ - ("id", "The ID of the connection."), - ("endpoint", "The value of the `ENDPOINT` option, if set."), - ("region", "The value of the `REGION` option, if set."), - ( - "access_key_id", - "The value of the `ACCESS KEY ID` option, if provided in line.", - ), - ( - "access_key_id_secret_id", - "The ID of the secret referenced by the `ACCESS KEY ID` option, if provided via a secret.", - ), - ( - "secret_access_key_secret_id", - "The ID of the secret referenced by the `SECRET ACCESS KEY` option, if set.", - ), - ( - "session_token", - "The value of the `SESSION TOKEN` option, if provided in line.", - ), - ( - "session_token_secret_id", - "The ID of the secret referenced by the `SESSION TOKEN` option, if provided via a secret.", - ), - ( - "assume_role_arn", - "The value of the `ASSUME ROLE ARN` option, if set.", - ), - ( - "assume_role_session_name", - "The value of the `ASSUME ROLE SESSION NAME` option, if set.", - ), - ( - "principal", - "The ARN of the AWS principal Materialize will use when assuming the provided role, if the connection is configured to use role assumption.", - ), - ( - "external_id", - "The external ID Materialize will use when assuming the provided role, if the connection is configured to use role assumption.", - ), - ( - "example_trust_policy", - "An example of an IAM role trust policy that allows this connection's principal and external ID to assume the role.", - ), - ]), - is_retained_metrics_object: false, - access: vec![PUBLIC_SELECT], - ontology: Some(Ontology { - entity_name: "aws_connection", - description: "AWS connection configuration details", - links: &const { - [OntologyLink { - name: "details_of", - target: "connection", - properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), - }] - }, - column_semantic_types: &[], - }), + )", + is_retained_metrics_object: false, + access: vec![PUBLIC_SELECT], + ontology: Some(Ontology { + entity_name: "aws_connection", + description: "AWS connection configuration details", + links: &const { + [OntologyLink { + name: "details_of", + target: "connection", + properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), + }] + }, + column_semantic_types: &[], + }), + } }); pub static MZ_CLUSTER_REPLICA_METRICS_HISTORY: LazyLock = diff --git a/src/catalog/src/config.rs b/src/catalog/src/config.rs index f96ff2a2b0405..b11bba066073e 100644 --- a/src/catalog/src/config.rs +++ b/src/catalog/src/config.rs @@ -312,6 +312,10 @@ pub struct AwsPrincipalContext { } impl AwsPrincipalContext { + // NOTE: the `mz_catalog.mz_aws_privatelink_connections` builtin materialized + // view reconstructs this ARN format in SQL from the mz_aws_account_id() and + // mz_aws_external_id_prefix() functions (see MZ_AWS_PRIVATELINK_CONNECTIONS + // in src/catalog/src/builtin/mz_catalog.rs). Keep the two in sync. pub fn to_principal_string(&self, aws_external_id_suffix: CatalogItemId) -> String { format!( "arn:aws:iam::{}:role/mz_{}_{}", diff --git a/src/environmentd/src/test_util.rs b/src/environmentd/src/test_util.rs index 6706e2623f8b6..578a60c98b19c 100644 --- a/src/environmentd/src/test_util.rs +++ b/src/environmentd/src/test_util.rs @@ -106,6 +106,12 @@ pub struct TestHarness { external_login_password_mz_system: Option, listeners_config: ListenersConfig, unsafe_mode: bool, + /// Whether the connection context carries the AWS external ID prefix and + /// connection role ARN. Default true, matching a cloud deployment. Set false + /// via [`TestHarness::without_aws_connection_context`] to model a deployment + /// that never configured the AWS context, where the context functions fold + /// to NULL. + aws_connection_context: bool, workers: usize, now: NowFn, seed: u32, @@ -190,6 +196,7 @@ impl Default for TestHarness { ], }, unsafe_mode: false, + aws_connection_context: true, workers: 1, now: SYSTEM_TIME.clone(), seed: rand::random(), @@ -306,6 +313,14 @@ impl TestHarness { self } + /// Models a deployment that never configured the AWS context, so the AWS + /// external ID prefix and connection role ARN are absent and the plan-time + /// AWS context functions fold to NULL. + pub fn without_aws_connection_context(mut self) -> Self { + self.aws_connection_context = false; + self + } + pub fn workers(mut self, workers: usize) -> Self { self.workers = workers; self @@ -803,7 +818,11 @@ impl Listeners { let system_dyncfgs = Arc::clone(&persist_clients.cfg().configs); let secrets_controller = Arc::clone(&orchestrator); - let connection_context = ConnectionContext::for_tests(orchestrator.reader()); + let mut connection_context = ConnectionContext::for_tests(orchestrator.reader()); + if !config.aws_connection_context { + connection_context.aws_external_id_prefix = None; + connection_context.aws_connection_role_arn = None; + } let orchestrator = Arc::new(TracingOrchestrator::new( orchestrator, config.orchestrator_tracing_cli_args, diff --git a/src/environmentd/tests/sql.rs b/src/environmentd/tests/sql.rs index ac74a7be27548..93d8c09c9c5b6 100644 --- a/src/environmentd/tests/sql.rs +++ b/src/environmentd/tests/sql.rs @@ -361,6 +361,51 @@ fn test_time() { ); } +/// The builtin `mz_internal.mz_analytics` connection is an assume-role AWS +/// connection present on every environment. Its row in +/// `mz_internal.mz_aws_connections` is reconstructed from the plan-time AWS +/// context functions. Without the AWS context those functions fold to NULL, and +/// the view must drop the row entirely, matching the packer it replaced, which +/// bailed and wrote no row in that case. See MZ_AWS_CONNECTIONS in +/// src/catalog/src/builtin/mz_internal.rs. +#[mz_ore::test] +#[allow(clippy::disallowed_methods)] +fn test_mz_aws_connections_context_dependent_row() { + // With the AWS context configured (the default), the assume-role row is + // present and fully populated. + let server = test_util::TestHarness::default().start_blocking(); + let mut client = server.connect(postgres::NoTls).unwrap(); + let count: i64 = client + .query_one("SELECT count(*) FROM mz_internal.mz_aws_connections", &[]) + .unwrap() + .get(0); + assert_eq!(count, 1, "mz_analytics row present with AWS context"); + + // Without the AWS context, the assume-role row is dropped rather than + // emitted with a malformed example_trust_policy. + let server = test_util::TestHarness::default() + .without_aws_connection_context() + .start_blocking(); + let mut client = server.connect(postgres::NoTls).unwrap(); + let count: i64 = client + .query_one("SELECT count(*) FROM mz_internal.mz_aws_connections", &[]) + .unwrap() + .get(0); + assert_eq!(count, 0, "mz_analytics row dropped without AWS context"); + + // The connection object itself still exists; only its derived view row is + // suppressed. + let count: i64 = client + .query_one( + "SELECT count(*) FROM mz_catalog.mz_connections \ + WHERE name = 'mz_analytics' AND type = 'aws'", + &[], + ) + .unwrap() + .get(0); + assert_eq!(count, 1, "mz_analytics connection object still present"); +} + #[mz_ore::test] #[allow(clippy::disallowed_methods)] fn test_subscribe_consolidation() { diff --git a/src/expr/src/scalar/func/impls/jsonb.rs b/src/expr/src/scalar/func/impls/jsonb.rs index 1788c342e7a4f..6929ea134192f 100644 --- a/src/expr/src/scalar/func/impls/jsonb.rs +++ b/src/expr/src/scalar/func/impls/jsonb.rs @@ -18,9 +18,10 @@ use mz_repr::role_id::RoleId; use mz_repr::{ArrayRustType, Datum, Row, RowPacker, SqlColumnType, SqlScalarType, strconv}; use mz_sql_parser::ast::display::AstDisplay; use mz_sql_parser::ast::{ - AstInfo, AvroSchema, CreateSubsourceOptionName, Format, FormatSpecifier, - KafkaSourceConfigOptionName, PgConfigOptionName, ProtobufSchema, RawClusterName, RawItemName, - SourceEnvelope, SourceErrorPolicy, Value, WithOptionValue, + AstInfo, AvroSchema, ConnectionOption, ConnectionOptionName, CreateConnectionType, + CreateSubsourceOptionName, Format, FormatSpecifier, KafkaSourceConfigOptionName, + PgConfigOptionName, ProtobufSchema, RawClusterName, RawItemName, SourceEnvelope, + SourceErrorPolicy, UnresolvedItemName, Value, WithOptionValue, }; use prost::Message as _; use serde::{Deserialize, Serialize}; @@ -380,6 +381,27 @@ fn parse_catalog_acl_mode<'a>(a: JsonbRef<'a>) -> Result { .map_err(|e| EvalError::InvalidCatalogJson(e.into())) } +/// Extracts the string form of a `WithOptionValue`, matching how the planner +/// coerces option values. The blanket `TryFromValue>` impl in +/// `src/sql/src/plan/with_options.rs` accepts a quoted string, a bare identifier, +/// and a 1-part unresolved item name, all yielding a string. Any other variant +/// yields None. +/// +/// `AstDisplay` does not re-quote bare identifiers or 1-part names, so those +/// forms persist unquoted in `create_sql`. Matching only `Value::String` here +/// would miss them and silently fall back to a default, so the catalog-raw +/// parsers below route their string options through this helper. +fn option_string(value: &WithOptionValue) -> Option { + match value { + WithOptionValue::Value(Value::String(s)) => Some(s.clone()), + WithOptionValue::Ident(ident) => Some(ident.clone().into_string()), + WithOptionValue::UnresolvedItemName(UnresolvedItemName(parts)) if parts.len() == 1 => { + Some(parts[0].clone().into_string()) + } + _ => None, + } +} + /// Parses a catalog `create_sql` string into a JSONB object. /// /// The returned JSONB does not fully reflect the parsed SQL and instead contains only fields @@ -478,14 +500,32 @@ fn parse_catalog_create_sql<'a>(a: &'a str) -> Result { Some(mz_sql_parser::ast::SourceEnvelope::Debezium) ); - if let Some(envelope) = stmt.envelope { - use mz_sql_parser::ast::SourceEnvelope::*; - let envelope_type = match envelope { - None => "none", - Debezium => "debezium", - Upsert { .. } => "upsert", - CdcV2 => "materialize", - }; + // An old-syntax kafka source ingests into its own relation, so an + // omitted ENVELOPE means the default ENVELOPE NONE and the pre-MV + // packer reported 'none'. A new-syntax source (no progress + // subsource, hence no EXPOSE PROGRESS AS in create_sql) ingests + // nothing itself. Its envelopes live on the per-table exports, so + // its own envelope_type stays absent (SQL NULL), matching released + // behavior. `progress_subsource.is_some()` is the planner's own + // old-vs-new discriminator (see `OldSyntaxIngestion` in + // plan_create_source). Non-kafka sources carry no envelope either. + // See the `mz_sources.envelope_type` column. + let envelope_type = match &stmt.envelope { + Some(envelope) => { + use mz_sql_parser::ast::SourceEnvelope::*; + Some(match envelope { + None => "none", + Debezium => "debezium", + Upsert { .. } => "upsert", + CdcV2 => "materialize", + }) + } + None if source_type == "kafka" && stmt.progress_subsource.is_some() => { + Some("none") + } + None => None, + }; + if let Some(envelope_type) = envelope_type { info.insert("envelope_type", json!(envelope_type)); } @@ -676,10 +716,7 @@ fn parse_kafka_source_details<'a>(a: &'a str) -> Result { let mut topic: Option = None; let mut group_id_prefix: Option = None; for opt in options { - let string_value = match opt.value { - Some(WithOptionValue::Value(Value::String(s))) => Some(s), - _ => None, - }; + let string_value = opt.value.as_ref().and_then(option_string); match opt.name { KafkaSourceConfigOptionName::Topic => topic = string_value, KafkaSourceConfigOptionName::GroupIdPrefix => group_id_prefix = string_value, @@ -889,6 +926,163 @@ fn parse_source_export_details<'a>(a: &'a str) -> Result { Ok(jsonb) } +/// Extracts connection-detail metadata from a catalog `create_sql`. +/// +/// Returns a per-connection-type object with the fields that the +/// `mz_kafka_connections`, `mz_ssh_tunnel_connections`, and `mz_aws_connections` +/// builtin views need. For everything else (other connection types, including +/// aws-privatelink whose only detail is context-derived, and non-connection +/// statements) it returns jsonb `null`, so callers filter on `IS NOT NULL` and +/// gate on the connection type separately (via +/// `parse_catalog_create_sql(...)->>'connection_type'`, the way `mz_connections` +/// already does). +/// +/// The shape per type: +/// +/// ```json +/// // kafka +/// { "brokers": ["host:port", ...], "progress_topic": } +/// // ssh-tunnel +/// { "public_key_1": "", "public_key_2": "" } +/// // aws +/// { +/// "auth_kind": "credentials" | "assume-role", +/// "endpoint": , "region": , +/// "access_key_id": , "access_key_id_secret_id": , +/// "secret_access_key_secret_id": , +/// "session_token": , "session_token_secret_id": , +/// "assume_role_arn": , "assume_role_session_name": +/// } +/// ``` +/// +/// `progress_topic` is null when the connection does not set an explicit +/// `PROGRESS TOPIC`. The default (`_materialize-progress--`) is +/// reconstructed by the view, not here, because it needs the environment id and +/// the connection's own id. Values derived only from environment context +/// (AWS principal, external id, trust policy, privatelink principal) are also +/// left to the view. This keeps the helper a pure function of the `create_sql`. +/// +/// For aws, an option is either an inline value or a secret reference. Inline +/// values land in `access_key_id`/`session_token`; a secret reference lands in +/// the matching `*_secret_id` as the referenced secret's catalog item id (the +/// persisted `create_sql` stores resolved references as `[uNNN AS name]`). +/// `auth_kind` is `assume-role` when `ASSUME ROLE ARN` is present, else +/// `credentials`, matching the `AwsAuth` variant the removed packer read. +/// +/// Errors if the statement fails to parse. +#[sqlfunc] +fn parse_connection_details<'a>(a: &'a str) -> Result { + // The persisted `create_sql` stores an inline broker as a single `BROKER` + // option and a broker list as a `BROKERS (...)` sequence. Either way we + // only need the addresses, which are present regardless of the tunnel + // (direct, SSH, or PrivateLink). + fn broker_addresses(values: &[ConnectionOption]) -> Vec { + let mut brokers = Vec::new(); + for opt in values { + match (&opt.name, &opt.value) { + (ConnectionOptionName::Broker, Some(WithOptionValue::ConnectionKafkaBroker(b))) => { + brokers.push(b.address.clone()); + } + (ConnectionOptionName::Brokers, Some(WithOptionValue::Sequence(seq))) => { + for v in seq { + if let WithOptionValue::ConnectionKafkaBroker(b) = v { + brokers.push(b.address.clone()); + } + } + } + _ => {} + } + } + brokers + } + + fn string_option( + values: &[ConnectionOption], + name: ConnectionOptionName, + ) -> Option { + values + .iter() + .find(|o| o.name == name) + .and_then(|o| o.value.as_ref()) + .and_then(option_string) + } + + // The catalog id of the secret a `SECRET ...` option references. Resolved + // references persist as `RawItemName::Id`, so an unresolved name yields + // None (the same treatment `parse_source_export_details` gives item names). + fn secret_id_option>( + values: &[ConnectionOption], + name: ConnectionOptionName, + ) -> Option { + values.iter().find_map(|o| match &o.value { + Some(WithOptionValue::Secret(RawItemName::Id(id, _, _))) if o.name == name => { + Some(id.clone()) + } + _ => None, + }) + } + + let parse = || -> Result { + let mut stmts = mz_sql_parser::parser::parse_statements(a) + .map_err(|e| format!("failed to parse create_sql: {e}"))?; + let stmt = match stmts.len() { + 1 => stmts.remove(0).ast, + n => return Err(format!("expected a single statement, found {n}")), + }; + + use mz_sql_parser::ast::Statement::CreateConnection; + let stmt = match stmt { + CreateConnection(stmt) => stmt, + _ => return Ok(serde_json::Value::Null), + }; + + match stmt.connection_type { + CreateConnectionType::Kafka => Ok(json!({ + "brokers": broker_addresses(&stmt.values), + "progress_topic": string_option(&stmt.values, ConnectionOptionName::ProgressTopic), + })), + CreateConnectionType::Ssh => Ok(json!({ + "public_key_1": string_option(&stmt.values, ConnectionOptionName::PublicKey1), + "public_key_2": string_option(&stmt.values, ConnectionOptionName::PublicKey2), + })), + CreateConnectionType::Aws => { + let assume_role_arn = + string_option(&stmt.values, ConnectionOptionName::AssumeRoleArn); + let auth_kind = if assume_role_arn.is_some() { + "assume-role" + } else { + "credentials" + }; + Ok(json!({ + "auth_kind": auth_kind, + // Planning coerces an empty ENDPOINT to None (see + // `src/sql/src/plan/statement/ddl/connection.rs`), so the + // removed packer wrote NULL for `ENDPOINT = ''`. Match that. + "endpoint": string_option(&stmt.values, ConnectionOptionName::Endpoint) + .filter(|s| !s.is_empty()), + "region": string_option(&stmt.values, ConnectionOptionName::Region), + "access_key_id": string_option(&stmt.values, ConnectionOptionName::AccessKeyId), + "access_key_id_secret_id": + secret_id_option(&stmt.values, ConnectionOptionName::AccessKeyId), + "secret_access_key_secret_id": + secret_id_option(&stmt.values, ConnectionOptionName::SecretAccessKey), + "session_token": string_option(&stmt.values, ConnectionOptionName::SessionToken), + "session_token_secret_id": + secret_id_option(&stmt.values, ConnectionOptionName::SessionToken), + "assume_role_arn": assume_role_arn, + "assume_role_session_name": + string_option(&stmt.values, ConnectionOptionName::AssumeRoleSessionName), + })) + } + _ => Ok(serde_json::Value::Null), + } + }; + + let val = parse().map_err(|e| EvalError::InvalidCatalogJson(e.into()))?; + let jsonb = Jsonb::from_serde_json(val).expect("valid JSONB"); + Ok(jsonb) +} + #[cfg(test)] mod tests { use mz_repr::adt::jsonb::Jsonb; @@ -1064,6 +1258,21 @@ mod tests { ); } + #[mz_ore::test] + fn kafka_unquoted_topic_and_prefix() { + // Planning accepts a bare identifier for TOPIC / GROUP ID PREFIX, and it + // persists unquoted in create_sql. Matching only quoted strings would + // drop TOPIC and error the whole mz_kafka_source_tables view. + let sql = "CREATE SOURCE \"materialize\".\"public\".\"k_src\" \ + IN CLUSTER [u42] \ + FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \ + (TOPIC = my_topic, GROUP ID PREFIX = my_prefix) FORMAT TEXT"; + let out = super::parse_kafka_source_details(sql).expect("ok"); + let out = as_serde(out); + assert_eq!(out["topic"], json!("my_topic")); + assert_eq!(out["group_id_prefix"], json!("my_prefix")); + } + #[mz_ore::test] fn kafka_unresolved_connection_errors() { // A bare-name connection reference never happens after purification, @@ -1124,6 +1333,16 @@ mod tests { ); } + #[mz_ore::test] + fn export_table_kafka_omitted_envelope_is_null() { + // Omitting ENVELOPE persists as absent in create_sql, so this + // source-type-agnostic helper reports null. The mz_kafka_source_tables + // view is responsible for defaulting kafka's null envelope to 'none'. + let sql = table_from_source_sql("\"topic\"", " FORMAT TEXT"); + let out = super::parse_source_export_details(&sql).expect("ok"); + assert_eq!(as_serde(out)["envelope_type"], serde_json::Value::Null); + } + #[mz_ore::test] fn export_table_kafka_key_value_format() { let sql = table_from_source_sql( @@ -1237,4 +1456,245 @@ mod tests { "wrong error variant/message" ); } + + // --- parse_connection_details -------------------------------------------- + + #[mz_ore::test] + fn connection_kafka_single_broker_default_progress() { + // No explicit PROGRESS TOPIC: the helper leaves it null and the view + // reconstructs the default. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \ + (BROKER = 'localhost:9092', SECURITY PROTOCOL = plaintext)"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "brokers": ["localhost:9092"], + "progress_topic": null, + }), + ); + } + + #[mz_ore::test] + fn connection_kafka_explicit_progress_topic() { + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \ + (BROKER = 'localhost:9092', PROGRESS TOPIC = 'override', \ + SECURITY PROTOCOL = plaintext)"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "brokers": ["localhost:9092"], + "progress_topic": "override", + }), + ); + } + + #[mz_ore::test] + fn connection_kafka_broker_list() { + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \ + (BROKERS ('b1:9092', 'b2:9092'), SECURITY PROTOCOL = plaintext)"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "brokers": ["b1:9092", "b2:9092"], + "progress_topic": null, + }), + ); + } + + #[mz_ore::test] + fn connection_ssh_public_keys() { + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO SSH TUNNEL \ + (HOST = 'ssh.example.com', PORT = 22, USER = 'mz', \ + PUBLIC KEY 1 = 'ssh-ed25519 AAAA', PUBLIC KEY 2 = 'ssh-ed25519 BBBB')"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "public_key_1": "ssh-ed25519 AAAA", + "public_key_2": "ssh-ed25519 BBBB", + }), + ); + } + + #[mz_ore::test] + fn connection_aws_credentials_inline_key() { + // Inline ACCESS KEY ID, secret SECRET ACCESS KEY. Assume-role columns + // stay null and auth_kind is credentials. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \ + (ACCESS KEY ID = 'AKIAEXAMPLE', \ + SECRET ACCESS KEY = SECRET [u1 AS \"materialize\".\"public\".\"sk\"])"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "auth_kind": "credentials", + "endpoint": null, + "region": null, + "access_key_id": "AKIAEXAMPLE", + "access_key_id_secret_id": null, + "secret_access_key_secret_id": "u1", + "session_token": null, + "session_token_secret_id": null, + "assume_role_arn": null, + "assume_role_session_name": null, + }), + ); + } + + #[mz_ore::test] + fn connection_aws_credentials_secret_key_and_session_token() { + // Every credential provided as a secret reference lands in the matching + // *_secret_id column as the referenced secret's catalog id. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \ + (ENDPOINT = 'http://localhost', REGION = 'us-east-1', \ + ACCESS KEY ID = SECRET [u1 AS \"materialize\".\"public\".\"ak\"], \ + SECRET ACCESS KEY = SECRET [u2 AS \"materialize\".\"public\".\"sk\"], \ + SESSION TOKEN = SECRET [u3 AS \"materialize\".\"public\".\"st\"])"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "auth_kind": "credentials", + "endpoint": "http://localhost", + "region": "us-east-1", + "access_key_id": null, + "access_key_id_secret_id": "u1", + "secret_access_key_secret_id": "u2", + "session_token": null, + "session_token_secret_id": "u3", + "assume_role_arn": null, + "assume_role_session_name": null, + }), + ); + } + + #[mz_ore::test] + fn connection_aws_assume_role() { + // Assume-role sets auth_kind and the assume-role columns; credential + // columns stay null. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \ + (ASSUME ROLE ARN 'arn:aws:iam::123:role/mz', \ + ASSUME ROLE SESSION NAME 'sess')"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "auth_kind": "assume-role", + "endpoint": null, + "region": null, + "access_key_id": null, + "access_key_id_secret_id": null, + "secret_access_key_secret_id": null, + "session_token": null, + "session_token_secret_id": null, + "assume_role_arn": "arn:aws:iam::123:role/mz", + "assume_role_session_name": "sess", + }), + ); + } + + #[mz_ore::test] + fn connection_kafka_unquoted_progress_topic() { + // A bare identifier PROGRESS TOPIC persists unquoted in create_sql. The + // helper must surface it, else the view substitutes the default topic. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \ + (BROKER = 'localhost:9092', PROGRESS TOPIC = my_topic, \ + SECURITY PROTOCOL = plaintext)"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!(as_serde(out)["progress_topic"], json!("my_topic")); + } + + #[mz_ore::test] + fn connection_aws_unquoted_option_values() { + // Planning accepts bare identifiers for these options and persists them + // unquoted. The helper must surface them, not fall back to NULL. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \ + (ENDPOINT = localhost, REGION = useast1, \ + ASSUME ROLE ARN 'arn:aws:iam::123:role/mz', \ + ASSUME ROLE SESSION NAME = mysession)"; + let out = super::parse_connection_details(sql).expect("ok"); + let out = as_serde(out); + assert_eq!(out["endpoint"], json!("localhost")); + assert_eq!(out["region"], json!("useast1")); + assert_eq!(out["assume_role_session_name"], json!("mysession")); + } + + #[mz_ore::test] + fn connection_aws_empty_endpoint_is_null() { + // Planning coerces ENDPOINT = '' to None, so the packer wrote NULL. The + // view must match, not report an empty string. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \ + (ENDPOINT = '', ASSUME ROLE ARN 'arn:aws:iam::123:role/mz')"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!(as_serde(out)["endpoint"], serde_json::Value::Null); + } + + #[mz_ore::test] + fn connection_other_type_returns_null_jsonb() { + // A connection type without a detail view (postgres) yields null. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO POSTGRES \ + (HOST = 'db', DATABASE = 'postgres', USER = 'mz')"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!(as_serde(out), serde_json::Value::Null); + } + + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` + fn connection_non_connection_returns_null_jsonb() { + let sql = "CREATE VIEW v AS SELECT 1"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!(as_serde(out), serde_json::Value::Null); + } + + // --- parse_catalog_create_sql envelope_type ------------------------------ + + #[mz_ore::test] + fn catalog_kafka_old_syntax_omitted_envelope_defaults_none() { + // An old-syntax kafka source (carries EXPOSE PROGRESS AS) ingests into + // its own relation, so an omitted ENVELOPE means the default NONE, which + // the pre-MV packer reported as 'none'. + let sql = "CREATE SOURCE \"materialize\".\"public\".\"k\" \ + IN CLUSTER [u42] \ + FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \ + (TOPIC 'test') FORMAT TEXT \ + EXPOSE PROGRESS AS [u12 AS \"materialize\".\"public\".\"k_progress\"]"; + let out = super::parse_catalog_create_sql(sql).expect("ok"); + assert_eq!(as_serde(out)["envelope_type"], json!("none")); + } + + #[mz_ore::test] + fn catalog_kafka_old_syntax_explicit_envelope() { + let sql = "CREATE SOURCE \"materialize\".\"public\".\"k\" \ + IN CLUSTER [u42] \ + FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \ + (TOPIC 'test') FORMAT BYTES ENVELOPE UPSERT \ + EXPOSE PROGRESS AS [u12 AS \"materialize\".\"public\".\"k_progress\"]"; + let out = super::parse_catalog_create_sql(sql).expect("ok"); + assert_eq!(as_serde(out)["envelope_type"], json!("upsert")); + } + + #[mz_ore::test] + fn catalog_kafka_new_syntax_source_omits_envelope_type() { + // A new-syntax kafka source has no progress subsource (no EXPOSE PROGRESS + // AS). It ingests nothing itself. Envelopes live on the per-table exports, + // so its own envelope_type stays absent (SQL NULL). + let sql = "CREATE SOURCE \"materialize\".\"public\".\"k\" \ + IN CLUSTER [u42] \ + FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"]"; + let out = super::parse_catalog_create_sql(sql).expect("ok"); + assert_eq!(as_serde(out).get("envelope_type"), None); + } + + #[mz_ore::test] + fn catalog_non_kafka_source_omits_envelope_type() { + // Non-kafka sources carry no envelope, so envelope_type stays absent + // (SQL NULL), not 'none'. + let sql = "CREATE SOURCE \"materialize\".\"public\".\"lg\" \ + IN CLUSTER [u42] FROM LOAD GENERATOR COUNTER"; + let out = super::parse_catalog_create_sql(sql).expect("ok"); + assert_eq!(as_serde(out).get("envelope_type"), None); + } } diff --git a/src/expr/src/scalar/func/unary.rs b/src/expr/src/scalar/func/unary.rs index 9723c257be76a..61913b35bd346 100644 --- a/src/expr/src/scalar/func/unary.rs +++ b/src/expr/src/scalar/func/unary.rs @@ -468,6 +468,7 @@ derive_unary!( ParseCatalogCreateSql, ParseCatalogId, ParseCatalogPrivileges, + ParseConnectionDetails, ParseKafkaSourceDetails, ParsePostgresSourceDetails, ParseSourceExportDetails, diff --git a/src/mz-deploy/src/project/compiler/typecheck/catalog.rs b/src/mz-deploy/src/project/compiler/typecheck/catalog.rs index fd408fd68e5cd..4afd7326864a7 100644 --- a/src/mz-deploy/src/project/compiler/typecheck/catalog.rs +++ b/src/mz-deploy/src/project/compiler/typecheck/catalog.rs @@ -628,6 +628,7 @@ impl CatalogRuntime { connection_context: mz_storage_types::connections::ConnectionContext::for_tests( secrets_reader, ), + aws_account_id: None, helm_chart_version: None, }; let mut catalog = Self { diff --git a/src/pgrepr-consts/src/oid.rs b/src/pgrepr-consts/src/oid.rs index 293bf64e90dc6..9d1c0672e5086 100644 --- a/src/pgrepr-consts/src/oid.rs +++ b/src/pgrepr-consts/src/oid.rs @@ -417,7 +417,7 @@ pub const LOG_MZ_MESSAGE_COUNTS_RECEIVED_RAW_OID: u32 = 16691; pub const LOG_MZ_MESSAGE_COUNTS_SENT_RAW_OID: u32 = 16692; pub const LOG_MZ_DATAFLOW_OPERATOR_REACHABILITY_RAW_OID: u32 = 16693; pub const TABLE_MZ_KAFKA_SINKS_OID: u32 = 16694; -pub const TABLE_MZ_KAFKA_CONNECTIONS_OID: u32 = 16695; +pub const MV_MZ_KAFKA_CONNECTIONS_OID: u32 = 16695; pub const MV_MZ_KAFKA_SOURCES_OID: u32 = 16696; pub const MV_MZ_POSTGRES_SOURCES_OID: u32 = 16697; pub const TABLE_MZ_OBJECT_DEPENDENCIES_OID: u32 = 16698; @@ -431,7 +431,7 @@ pub const MV_MZ_INDEXES_OID: u32 = 16705; pub const TABLE_MZ_INDEX_COLUMNS_OID: u32 = 16706; pub const TABLE_MZ_TABLES_OID: u32 = 16707; pub const MV_MZ_CONNECTIONS_OID: u32 = 16708; -pub const TABLE_MZ_SSH_TUNNEL_CONNECTIONS_OID: u32 = 16709; +pub const MV_MZ_SSH_TUNNEL_CONNECTIONS_OID: u32 = 16709; pub const MV_MZ_SOURCES_OID: u32 = 16710; pub const TABLE_MZ_SINKS_OID: u32 = 16711; pub const TABLE_MZ_VIEWS_OID: u32 = 16712; @@ -477,8 +477,8 @@ pub const SOURCE_MZ_SINK_STATUS_HISTORY_OID: u32 = 16752; pub const VIEW_MZ_SINK_STATUSES_OID: u32 = 16753; pub const TABLE_MZ_STORAGE_USAGE_BY_SHARD_OID: u32 = 16754; pub const TABLE_MZ_EGRESS_IPS_OID: u32 = 16755; -pub const TABLE_MZ_AWS_PRIVATELINK_CONNECTIONS_OID: u32 = 16756; -pub const TABLE_MZ_AWS_CONNECTIONS_OID: u32 = 16757; +pub const MV_MZ_AWS_PRIVATELINK_CONNECTIONS_OID: u32 = 16756; +pub const MV_MZ_AWS_CONNECTIONS_OID: u32 = 16757; pub const VIEW_MZ_CLUSTER_REPLICA_METRICS_OID: u32 = 16758; pub const SOURCE_MZ_CLUSTER_REPLICA_FRONTIERS_OID: u32 = 16759; pub const SOURCE_MZ_FRONTIERS_OID: u32 = 16760; @@ -665,8 +665,8 @@ pub const ROLE_PUBLIC_OID: u32 = 16944; pub const MV_MZ_ROLE_PARAMETERS_OID: u32 = 16945; pub const TABLE_MZ_MATERIALIZED_VIEW_REFRESH_STRATEGIES_OID: u32 = 16946; pub const MV_MZ_CLUSTER_SCHEDULES_OID: u32 = 16947; -pub const TABLE_MZ_POSTGRES_SOURCE_TABLES_OID: u32 = 16948; -pub const TABLE_MZ_MYSQL_SOURCE_TABLES_OID: u32 = 16949; +pub const MV_MZ_POSTGRES_SOURCE_TABLES_OID: u32 = 16948; +pub const MV_MZ_MYSQL_SOURCE_TABLES_OID: u32 = 16949; pub const INDEX_MZ_KAFKA_SOURCES_IND_OID: u32 = 16950; pub const INDEX_MZ_WEBHOOK_SOURCES_IND_OID: u32 = 16951; pub const TABLE_MZ_HISTORY_RETENTION_STRATEGIES_OID: u32 = 16952; @@ -742,7 +742,7 @@ pub const SOURCE_MZ_WALLCLOCK_LAG_HISTORY_OID: u32 = 17021; pub const VIEW_MZ_WALLCLOCK_GLOBAL_LAG_HISTORY_OID: u32 = 17022; pub const VIEW_MZ_WALLCLOCK_GLOBAL_LAG_RECENT_HISTORY_OID: u32 = 17023; pub const INDEX_MZ_WALLCLOCK_GLOBAL_LAG_RECENT_HISTORY_IND_OID: u32 = 17024; -pub const TABLE_MZ_KAFKA_SOURCE_TABLES_OID: u32 = 17025; +pub const MV_MZ_KAFKA_SOURCE_TABLES_OID: u32 = 17025; pub const VIEW_MZ_OBJECTS_ID_NAMESPACE_TYPES_OID: u32 = 17030; pub const VIEW_MZ_OBJECT_HISTORY_OID: u32 = 17031; pub const INDEX_MZ_OBJECT_HISTORY_IND_OID: u32 = 17032; @@ -767,7 +767,7 @@ pub const VIEW_MZ_MAPPABLE_OBJECTS_OID: u32 = 17053; pub const VIEW_MZ_WALLCLOCK_GLOBAL_LAG_OID: u32 = 17054; pub const SOURCE_MZ_WALLCLOCK_GLOBAL_LAG_HISTOGRAM_RAW_OID: u32 = 17055; pub const VIEW_MZ_WALLCLOCK_GLOBAL_LAG_HISTOGRAM_OID: u32 = 17056; -pub const TABLE_MZ_SQL_SERVER_SOURCE_TABLES_OID: u32 = 17057; +pub const MV_MZ_SQL_SERVER_SOURCE_TABLES_OID: u32 = 17057; pub const TABLE_MZ_LICENSE_KEYS_OID: u32 = 17058; pub const TABLE_MZ_ROLE_AUTH_OID: u32 = 17059; pub const TABLE_MZ_ICEBERG_SINKS_OID: u32 = 17060; @@ -817,8 +817,8 @@ pub const MV_MZ_CLUSTER_AUTO_SCALING_STRATEGIES_OID: u32 = 17103; pub const INDEX_MZ_CLUSTER_AUTO_SCALING_STRATEGIES_IND_OID: u32 = 17104; pub const FUNC_PARSE_POSTGRES_SOURCE_DETAILS_OID: u32 = 17105; pub const FUNC_PARSE_KAFKA_SOURCE_DETAILS_OID: u32 = 17106; -pub const MV_MZ_POSTGRES_SOURCE_TABLES_OID: u32 = 17107; -pub const MV_MZ_MYSQL_SOURCE_TABLES_OID: u32 = 17108; -pub const MV_MZ_SQL_SERVER_SOURCE_TABLES_OID: u32 = 17109; -pub const MV_MZ_KAFKA_SOURCE_TABLES_OID: u32 = 17110; pub const FUNC_PARSE_SOURCE_EXPORT_DETAILS_OID: u32 = 17111; +pub const FUNC_PARSE_CONNECTION_DETAILS_OID: u32 = 17112; +pub const FUNC_MZ_AWS_ACCOUNT_ID_OID: u32 = 17113; +pub const FUNC_MZ_AWS_EXTERNAL_ID_PREFIX_OID: u32 = 17114; +pub const FUNC_MZ_AWS_CONNECTION_ROLE_ARN_OID: u32 = 17115; diff --git a/src/sql/src/catalog.rs b/src/sql/src/catalog.rs index 9806b73ccc2fc..068eb6c0751bb 100644 --- a/src/sql/src/catalog.rs +++ b/src/sql/src/catalog.rs @@ -440,6 +440,15 @@ pub struct CatalogConfig { pub now: NowFn, /// Context for source and sink connections. pub connection_context: ConnectionContext, + /// The AWS account ID that Materialize runs under, if configured. + /// + /// This is the `aws_account_id` from the catalog's `AwsPrincipalContext`. + /// It is plumbed through here (rather than the whole context) because the + /// external-id prefix and connection role ARN already live on + /// [`ConnectionContext`], and because `AwsPrincipalContext` is defined in + /// `mz-catalog`, which cannot be a dependency of `mz-sql`. Folded into a + /// literal by `mz_aws_account_id()`; `None` on non-cloud/local envs. + pub aws_account_id: Option, /// Helm chart version pub helm_chart_version: Option, } diff --git a/src/sql/src/func.rs b/src/sql/src/func.rs index 5152a8902c68d..cd535bea785a9 100644 --- a/src/sql/src/func.rs +++ b/src/sql/src/func.rs @@ -4709,6 +4709,56 @@ pub static MZ_CATALOG_BUILTINS: LazyLock> = LazyLoc )) }) => String, oid::FUNC_MZ_ENVIRONMENT_ID_OID; }, + // The three AWS-context functions below are plan-time constants, fixed + // for the lifetime of the envd process, folded to a literal here for the + // same reason as `mz_environment_id`: the mz_aws_connections and + // mz_aws_privatelink_connections materialized views reproduce the AWS + // principal, external id, and trust policy in SQL, and a materialized + // view cannot reference an unmaterializable function. They return NULL + // on environments without the corresponding context (non-cloud/local). + // + // Like `mz_environment_id`, they are not gated by + // `restrict_to_user_objects`. The fold bakes the value into any stored + // view that references it, so a gate here could only catch direct calls, + // not values already materialized into a view, which is more misleading + // than no gate. Restricted sessions are still blocked from the system + // connection views themselves (they are system relations). See + // doc/developer/design/20260508_restrict_to_user_objects.md. + "mz_aws_account_id" => Scalar { + params!() => Operation::nullary(|ecx| { + Ok(match &ecx.catalog().config().aws_account_id { + Some(account_id) => { + HirScalarExpr::literal(Datum::String(account_id), SqlScalarType::String) + } + None => HirScalarExpr::literal_null(SqlScalarType::String), + }) + }) => String, oid::FUNC_MZ_AWS_ACCOUNT_ID_OID; + }, + "mz_aws_external_id_prefix" => Scalar { + params!() => Operation::nullary(|ecx| { + let prefix = ecx + .catalog() + .config() + .connection_context + .aws_external_id_prefix + .as_ref() + .map(|p| p.to_string()); + Ok(match prefix { + Some(prefix) => { + HirScalarExpr::literal(Datum::String(&prefix), SqlScalarType::String) + } + None => HirScalarExpr::literal_null(SqlScalarType::String), + }) + }) => String, oid::FUNC_MZ_AWS_EXTERNAL_ID_PREFIX_OID; + }, + "mz_aws_connection_role_arn" => Scalar { + params!() => Operation::nullary(|ecx| { + Ok(match &ecx.catalog().config().connection_context.aws_connection_role_arn { + Some(arn) => HirScalarExpr::literal(Datum::String(arn), SqlScalarType::String), + None => HirScalarExpr::literal_null(SqlScalarType::String), + }) + }) => String, oid::FUNC_MZ_AWS_CONNECTION_ROLE_ARN_OID; + }, "mz_is_superuser" => Scalar { params!() => UnmaterializableFunc::MzIsSuperuser => SqlScalarType::Bool, oid::FUNC_MZ_IS_SUPERUSER; @@ -5320,6 +5370,11 @@ pub static MZ_INTERNAL_BUILTINS: LazyLock> = LazyLo func::ParseSourceExportDetails, ) => Jsonb, oid::FUNC_PARSE_SOURCE_EXPORT_DETAILS_OID; }, + "parse_connection_details" => Scalar { + params!(String) => UnaryFunc::ParseConnectionDetails( + func::ParseConnectionDetails, + ) => Jsonb, oid::FUNC_PARSE_CONNECTION_DETAILS_OID; + }, "redact_sql" => Scalar { params!(String) => UnaryFunc::RedactSql(func::RedactSql) => String, oid::FUNC_REDACT_SQL_OID; diff --git a/src/storage-types/src/connections.rs b/src/storage-types/src/connections.rs index b915233ddd876..7f76c8ff2b5f4 100644 --- a/src/storage-types/src/connections.rs +++ b/src/storage-types/src/connections.rs @@ -1124,6 +1124,11 @@ impl KafkaConnection { /// /// The caller is responsible for providing the connection ID as it is not /// known to `KafkaConnection`. + /// + /// NOTE: the `mz_catalog.mz_kafka_connections` builtin materialized view + /// reconstructs the default (`_materialize-progress--`) in SQL + /// (see `MZ_KAFKA_CONNECTIONS` in `src/catalog/src/builtin/mz_catalog.rs`). + /// Keep the two in sync. pub fn progress_topic( &self, connection_context: &ConnectionContext, diff --git a/src/storage-types/src/connections/aws.rs b/src/storage-types/src/connections/aws.rs index 19c2016e68198..0e3f15c993bc3 100644 --- a/src/storage-types/src/connections/aws.rs +++ b/src/storage-types/src/connections/aws.rs @@ -283,6 +283,10 @@ impl AwsAssumeRole { Ok(credentials.build_from_provider(jump_credentials).await) } + // NOTE: the `mz_internal.mz_aws_connections` builtin materialized view + // reconstructs this `mz__` format in SQL (see + // MZ_AWS_CONNECTIONS in src/catalog/src/builtin/mz_internal.rs). Keep the + // two in sync. pub fn external_id( &self, connection_context: &ConnectionContext, @@ -294,6 +298,10 @@ impl AwsAssumeRole { Ok(format!("mz_{}_{}", aws_external_id_prefix, connection_id)) } + // NOTE: the `mz_internal.mz_aws_connections` builtin materialized view + // reconstructs this trust-policy JSON with `jsonb_build_object` (see + // MZ_AWS_CONNECTIONS in src/catalog/src/builtin/mz_internal.rs). Keep the + // structure (keys, nesting) in sync. pub fn example_trust_policy( &self, connection_context: &ConnectionContext, diff --git a/src/testdrive/src/action.rs b/src/testdrive/src/action.rs index 74a70be517939..66192b1c2fcaa 100644 --- a/src/testdrive/src/action.rs +++ b/src/testdrive/src/action.rs @@ -22,7 +22,7 @@ use aws_credential_types::provider::ProvideCredentials; use aws_types::SdkConfig; use futures::future::FutureExt; use itertools::Itertools; -use mz_adapter::catalog::{Catalog, ConnCatalog}; +use mz_adapter::catalog::{Catalog, ConnCatalog, DebugAwsContext}; use mz_adapter::session::Session; use mz_build_info::BuildInfo; use mz_catalog::config::ClusterReplicaSizeMap; @@ -218,6 +218,16 @@ pub struct MaterializeState { pgclient: tokio_postgres::Client, environment_id: EnvironmentId, bootstrap_args: BootstrapArgs, + // The AWS environment context, queried once from the running environmentd at + // startup, the same way environment_id is. The builtin AWS connection views + // fold these values into their optimized expressions, so the catalog copy + // opened for the consistency check must resolve them the same way. Queried + // at startup rather than at check time because a check-time query runs after + // arbitrary test state and is less reliable, while startup runs against a + // clean session on the currently connected version. + aws_account_id: Option, + aws_external_id_prefix: Option, + aws_connection_role_arn: Option, } pub struct State { @@ -464,6 +474,11 @@ impl State { &self.persist_clients, ) .await?; + let aws_context = DebugAwsContext { + aws_account_id: self.materialize.aws_account_id.clone(), + aws_external_id_prefix: self.materialize.aws_external_id_prefix.clone(), + aws_connection_role_arn: self.materialize.aws_connection_role_arn.clone(), + }; let catalog = Catalog::open_debug_read_only_persist_catalog_config( persist_client, SYSTEM_TIME.clone(), @@ -472,6 +487,7 @@ impl State { build_info, bootstrap_args, enable_expression_cache_override, + Some(aws_context), ) .await?; let res = f(catalog.for_session(&Session::dummy())); @@ -1349,6 +1365,27 @@ async fn create_materialize_state( .parse() .context("parsing environment ID")?; + // Tolerate the functions not existing: upgrade tests run testdrive against an + // older environmentd whose catalog predates them. A version without them has + // no context-folding views either, so no context is the correct fold. + let (aws_account_id, aws_external_id_prefix, aws_connection_role_arn) = match pg_query_one( + &pgclient, + pg_sql!( + "SELECT mz_aws_account_id(), mz_aws_external_id_prefix(), \ + mz_aws_connection_role_arn()" + ), + &[], + ) + .await + { + Ok(row) => ( + row.get::<_, Option>(0), + row.get::<_, Option>(1), + row.get::<_, Option>(2), + ), + Err(_) => (None, None, None), + }; + let bootstrap_args = BootstrapArgs { cluster_replica_size_map: config.materialize_cluster_replica_sizes.clone(), default_cluster_replica_size: "ABC".to_string(), @@ -1369,6 +1406,9 @@ async fn create_materialize_state( pgclient, environment_id, bootstrap_args, + aws_account_id, + aws_external_id_prefix, + aws_connection_role_arn, }; Ok(materialize_state) diff --git a/test/sqllogictest/autogenerated/mz_catalog.slt b/test/sqllogictest/autogenerated/mz_catalog.slt index b31b8154c4be7..558547b86a68a 100644 --- a/test/sqllogictest/autogenerated/mz_catalog.slt +++ b/test/sqllogictest/autogenerated/mz_catalog.slt @@ -371,7 +371,7 @@ name text The␠name␠of␠the␠source. type text The␠type␠of␠the␠source:␠`kafka`,␠`mysql`,␠`postgres`,␠`load-generator`,␠`progress`,␠or␠`subsource`. connection_id text The␠ID␠of␠the␠connection␠associated␠with␠the␠source,␠if␠any.␠Corresponds␠to␠`mz_connections.id`. size text *Deprecated*␠The␠size␠of␠the␠source. -envelope_type text For␠Kafka␠sources,␠the␠envelope␠type:␠`none`,␠`upsert`,␠or␠`debezium`.␠`NULL`␠for␠other␠source␠types. +envelope_type text For␠old-syntax␠Kafka␠sources,␠the␠envelope␠type:␠`none`,␠`upsert`,␠or␠`debezium`.␠`NULL`␠for␠new-syntax␠Kafka␠sources,␠whose␠envelopes␠are␠defined␠per␠source␠table␠(see␠`mz_kafka_source_tables`),␠and␠for␠other␠source␠types. key_format text For␠Kafka␠sources,␠the␠format␠of␠the␠Kafka␠message␠key:␠`avro`,␠`csv`,␠`regex`,␠`bytes`,␠`json`,␠`text`,␠or␠`NULL`. value_format text For␠Kafka␠sources,␠the␠format␠of␠the␠Kafka␠message␠value:␠`avro`,␠`csv`,␠`regex`,␠`bytes`,␠`json`,␠`text`.␠`NULL`␠for␠other␠source␠types. cluster_id text The␠ID␠of␠the␠cluster␠maintaining␠the␠source.␠Corresponds␠to␠`mz_clusters.id`. diff --git a/test/sqllogictest/autogenerated/mz_internal.slt b/test/sqllogictest/autogenerated/mz_internal.slt index 180ec8924ad4d..d4fc2da6c6547 100644 --- a/test/sqllogictest/autogenerated/mz_internal.slt +++ b/test/sqllogictest/autogenerated/mz_internal.slt @@ -448,7 +448,7 @@ SELECT name, type, comment FROM objects WHERE schema = 'mz_internal' AND object ---- id text The␠ID␠of␠the␠table.␠Corresponds␠to␠`mz_catalog.mz_tables.id`. topic text The␠topic␠being␠ingested. -envelope_type text The␠envelope␠type:␠`none`,␠`upsert`,␠or␠`debezium`.␠`NULL`␠for␠other␠source␠types. +envelope_type text The␠envelope␠type:␠`none`,␠`upsert`,␠or␠`debezium`.␠Defaults␠to␠`none`␠when␠the␠source␠table␠omits␠an␠explicit␠envelope. key_format text The␠format␠of␠the␠Kafka␠message␠key:␠`avro`,␠`csv`,␠`regex`,␠`bytes`,␠`json`,␠`text`,␠or␠`NULL`. value_format text The␠format␠of␠the␠Kafka␠message␠value:␠`avro`,␠`csv`,␠`regex`,␠`bytes`,␠`json`,␠`text`.␠`NULL`␠for␠other␠source␠types. diff --git a/test/sqllogictest/catalog_server_explain.slt b/test/sqllogictest/catalog_server_explain.slt index e11092f9d5c18..300281a804373 100644 --- a/test/sqllogictest/catalog_server_explain.slt +++ b/test/sqllogictest/catalog_server_explain.slt @@ -4497,6 +4497,16 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_aws_privatelink_connections"; +---- +mz_catalog.mz_aws_privatelink_connections: + →Constant (0 rows) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_cluster_replicas"; ---- @@ -4855,6 +4865,70 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_kafka_connections"; +---- +mz_catalog.mz_kafka_connections: + →With + cte l0 = + →Read mz_internal.mz_catalog_raw + cte l1 = + →Distinct GroupAggregate + →Fused with Child Map/Filter/Project + Project: #1 + →Read l0 + cte l2 = + →Non-incremental GroupAggregate + Aggregation: array_agg[order_by=[#0 asc nulls_last]](row(array[#1], #2)) + Key: + Project: #0 + →Table Function jsonb_array_elements_text[with_ordinality]((#0{details} -> "brokers")) + →Arranged l1 + cte l3 = + →Union + →Unarranged Raw Stream + →Arranged l2 + →Map/Filter/Project + Project: #0, #1 + Map: null + →Consolidating Union + →Negate Diffs + →Fused with Child Map/Filter/Project + Project: #0 + →Arranged l2 + Key: (#0) + →Unarranged Raw Stream + →Arranged l1 + →Return + →Differential Join %1[#0] » %0:l0[#1] + after %0: + Project: #3..=#5 + Map: parse_catalog_id(((#2{data} -> "key") -> "gid")), (#1 || {}), coalesce((#0{details} ->> "progress_topic"), ("_materialize-progress-local-az1--" || #3)) + →Arrange (#1) + →Stream l0 + →Arrange (#0) + →Union + →Stream l3 + →Map/Filter/Project + Project: #0, #1 + Map: null + →Consolidating Union + →Negate Diffs + →Fused with Child Map/Filter/Project + Project: #0 + →Read l3 + →Unarranged Raw Stream + →Arranged l1 + +Source mz_internal.mz_catalog_raw + project=(#0, #2) + filter=(("Item" = (#0{data} ->> "kind")) AND ("kafka" = (parse_catalog_create_sql(#1) ->> "connection_type"))) + map=(((((#0{data} -> "value") -> "definition") -> "V1") ->> "create_sql"), parse_connection_details(#1)) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_kafka_sources"; ---- @@ -4907,7 +4981,7 @@ mz_catalog.mz_materialized_views: Project: #4, #0, #5, #1, #6, #2, #9, #8, #3, #7 Map: {=r/s1, s1=r/s1}, "s1" →Arrange (#1{schema_name}, #2{name}) - →Constant (33 rows) + →Constant (37 rows) →Arrange (empty key) (#0{schema_name}) (#0{schema_name}, #1{name}) →Fused with Child Map/Filter/Project Project: #4, #3, #5 @@ -5182,6 +5256,21 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_ssh_tunnel_connections"; +---- +mz_catalog.mz_ssh_tunnel_connections: + →Read mz_internal.mz_catalog_raw + +Source mz_internal.mz_catalog_raw + project=(#3..=#5) + filter=(("Item" = (#0{data} ->> "kind")) AND ("ssh-tunnel" = (parse_catalog_create_sql(#1) ->> "connection_type"))) + map=(((((#0{data} -> "value") -> "definition") -> "V1") ->> "create_sql"), parse_connection_details(#1), parse_catalog_id(((#0{data} -> "key") -> "gid")), (#2{details} ->> "public_key_1"), (#2{details} ->> "public_key_2")) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_system_privileges"; ---- @@ -5198,6 +5287,21 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_aws_connections"; +---- +mz_internal.mz_aws_connections: + →Read mz_internal.mz_catalog_raw + +Source mz_internal.mz_catalog_raw + project=(#3..=#12, #14..=#16) + filter=(("Item" = (#0{data} ->> "kind")) AND ("aws" = (parse_catalog_create_sql(#1) ->> "connection_type"))) + map=(((((#0{data} -> "value") -> "definition") -> "V1") ->> "create_sql"), parse_connection_details(#1), parse_catalog_id(((#0{data} -> "key") -> "gid")), (#2{details} ->> "endpoint"), (#2{details} ->> "region"), (#2{details} ->> "access_key_id"), (#2{details} ->> "access_key_id_secret_id"), (#2{details} ->> "secret_access_key_secret_id"), (#2{details} ->> "session_token"), (#2{details} ->> "session_token_secret_id"), (#2{details} ->> "assume_role_arn"), (#2{details} ->> "assume_role_session_name"), ("assume-role" = (#2{details} ->> "auth_kind")), case when #13 then "arn:aws:iam::123456789000:role/MaterializeConnection" else null end, case when #13 then ("mz_test-aws-external-id-prefix_" || #3) else null end, case when #13 then jsonb_build_object("Version", "2012-10-17", "Statement", jsonbable_to_jsonb(jsonb_build_array(jsonbable_to_jsonb(jsonb_build_object("Effect", "Allow", "Principal", {AWS: "arn:aws:iam::123456789000:role/MaterializeConnection"}, "Action", "sts:AssumeRole", "Condition", jsonbable_to_jsonb(jsonb_build_object("StringEquals", jsonbable_to_jsonb(jsonb_build_object("sts:ExternalId", jsonbable_to_jsonb(("mz_test-aws-external-id-prefix_" || #3))))))))))) else null end) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_cluster_auto_scaling_strategies"; ---- @@ -5311,7 +5415,7 @@ mz_internal.mz_kafka_source_tables: path %0: after %1: Project: #2, #1, #3..=#6 - Map: ((#2{details} -> "external_reference") ->> 0), (#2{details} ->> "envelope_type"), (#2{details} ->> "key_format"), (#2{details} ->> "value_format") + Map: ((#2{details} -> "external_reference") ->> 0), coalesce((#2{details} ->> "envelope_type"), "none"), (#2{details} ->> "key_format"), (#2{details} ->> "value_format") after %2: Project: #2..=#6 Filter: (#10{type} = "kafka") @@ -5325,7 +5429,7 @@ mz_internal.mz_kafka_source_tables: path %2: after %1: Project: #1, #3..=#6 - Map: ((#2{details} -> "external_reference") ->> 0), (#2{details} ->> "envelope_type"), (#2{details} ->> "key_format"), (#2{details} ->> "value_format") + Map: ((#2{details} -> "external_reference") ->> 0), coalesce((#2{details} ->> "envelope_type"), "none"), (#2{details} ->> "key_format"), (#2{details} ->> "value_format") after %0: Project: #5, #1..=#4 Map: parse_catalog_id(((#0{data} -> "key") -> "gid")) @@ -7842,7 +7946,7 @@ query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_materialized_views"; ---- Explained Query (fast path): - →Constant (33 rows) + →Constant (37 rows) Target cluster: mz_catalog_server diff --git a/test/sqllogictest/information_schema_tables.slt b/test/sqllogictest/information_schema_tables.slt index 9faa98ae3cf76..8ea3e4fdb71a7 100644 --- a/test/sqllogictest/information_schema_tables.slt +++ b/test/sqllogictest/information_schema_tables.slt @@ -102,7 +102,7 @@ MATERIALIZED VIEW materialize mz_catalog mz_aws_privatelink_connections -BASE TABLE +MATERIALIZED VIEW materialize mz_catalog mz_base_types @@ -162,7 +162,7 @@ MATERIALIZED VIEW materialize mz_catalog mz_kafka_connections -BASE TABLE +MATERIALIZED VIEW materialize mz_catalog mz_kafka_sinks @@ -238,7 +238,7 @@ MATERIALIZED VIEW materialize mz_catalog mz_ssh_tunnel_connections -BASE TABLE +MATERIALIZED VIEW materialize mz_catalog mz_storage_usage @@ -278,7 +278,7 @@ BASE TABLE materialize mz_internal mz_aws_connections -BASE TABLE +MATERIALIZED VIEW materialize mz_internal mz_aws_privatelink_connection_status_history diff --git a/test/sqllogictest/mz_aws_connections.slt b/test/sqllogictest/mz_aws_connections.slt new file mode 100644 index 0000000000000..96ca58cd70462 --- /dev/null +++ b/test/sqllogictest/mz_aws_connections.slt @@ -0,0 +1,155 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Behavioural tests for mz_internal.mz_aws_connections. The view is a +# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see +# MZ_AWS_CONNECTIONS in src/catalog/src/builtin/mz_internal.rs and the +# parse_connection_details helper in +# src/expr/src/scalar/func/impls/jsonb.rs. The assume-role columns (principal, +# external_id, example_trust_policy) are reconstructed from the AWS context +# functions; the sqllogictest env configures that context (see +# ConnectionContext::for_tests), so the built-in mz_analytics connection +# doubles as the assume-role parity check here. Real credential connections +# with live AWS are exercised by test/aws-localstack/aws-connection/. + +mode cockroach + +reset-server + +# --- Schema ------------------------------------------------------------------- + +# Lock down: 13 columns, canonical order, types, and nullability. +query TTT +SELECT c.name, c.type, c.nullable::text +FROM mz_catalog.mz_columns c +JOIN mz_catalog.mz_objects o ON c.id = o.id +JOIN mz_catalog.mz_schemas s ON o.schema_id = s.id +WHERE o.name = 'mz_aws_connections' AND s.name = 'mz_internal' +ORDER BY c.position +---- +id text false +endpoint text true +region text true +access_key_id text true +access_key_id_secret_id text true +secret_access_key_secret_id text true +session_token text true +session_token_secret_id text true +assume_role_arn text true +assume_role_session_name text true +principal text true +external_id text true +example_trust_policy jsonb true + +# --- Fresh env: only the built-in mz_analytics connection --------------------- + +# Anchor count. mz_analytics is the sole AWS connection on a fresh env. +query I +SELECT count(*) FROM mz_internal.mz_aws_connections +---- +1 + +# mz_analytics is an assume-role connection. With the configured AWS context, +# principal, external_id, and example_trust_policy are reconstructed exactly as +# AwsAssumeRole::external_id / example_trust_policy produced them: principal is +# the connection role ARN, external_id is mz__, and the trust +# policy carries both. This is the SQL<->Rust parity check for the assume-role +# branch. +query TTTT +SELECT + ac.assume_role_arn, + ac.principal, + ac.external_id = 'mz_test-aws-external-id-prefix_' || ac.id, + ac.example_trust_policy = jsonb_build_object( + 'Version', '2012-10-17', + 'Statement', jsonb_build_array(jsonb_build_object( + 'Effect', 'Allow', + 'Principal', jsonb_build_object( + 'AWS', 'arn:aws:iam::123456789000:role/MaterializeConnection'), + 'Action', 'sts:AssumeRole', + 'Condition', jsonb_build_object( + 'StringEquals', jsonb_build_object( + 'sts:ExternalId', 'mz_test-aws-external-id-prefix_' || ac.id)) + )) + ) +FROM mz_internal.mz_aws_connections ac +JOIN mz_connections c USING (id) +WHERE c.name = 'mz_analytics' +---- +(empty) arn:aws:iam::123456789000:role/MaterializeConnection true true + +# --- A credentials connection populates the create_sql-derived columns -------- + +statement ok +CREATE SECRET sak AS 'secret-access-key' + +statement ok +CREATE CONNECTION ac TO AWS ( + ENDPOINT = 'http://localhost:4566', + REGION = 'us-east-1', + ACCESS KEY ID = 'AKIAEXAMPLE', + SECRET ACCESS KEY = SECRET sak +) WITH (VALIDATE = false) + +# The inline access key id is shown; the secret access key is shown as its +# secret id (matching mz_secrets.id). The assume-role and context columns are +# NULL for a credentials connection. +query TTTTTTTTTT +SELECT + endpoint, + region, + access_key_id, + access_key_id_secret_id, + secret_access_key_secret_id = (SELECT id FROM mz_secrets WHERE name = 'sak'), + assume_role_arn, + assume_role_session_name, + principal, + external_id, + example_trust_policy +FROM mz_internal.mz_aws_connections ac +JOIN mz_connections c USING (id) +WHERE c.name = 'ac' +---- +http://localhost:4566 us-east-1 AKIAEXAMPLE NULL true NULL NULL NULL NULL NULL + +# --- Non-aws connections are excluded ----------------------------------------- + +statement ok +CREATE CONNECTION kc TO KAFKA (BROKER 'localhost:9092', SECURITY PROTOCOL PLAINTEXT) WITH (VALIDATE = false) + +query I +SELECT count(*) FROM mz_internal.mz_aws_connections ac +JOIN mz_connections c USING (id) +WHERE c.type != 'aws' +---- +0 + +# --- FK-like invariant: every row joins its parent connection ----------------- + +query I +SELECT count(*) FROM mz_internal.mz_aws_connections ac +LEFT JOIN mz_connections c ON ac.id = c.id +WHERE c.id IS NULL +---- +0 + +# No duplicate ids. +query I +SELECT count(*) - count(DISTINCT id) FROM mz_internal.mz_aws_connections +---- +0 + +statement ok +DROP CONNECTION ac + +statement ok +DROP CONNECTION kc + +statement ok +DROP SECRET sak diff --git a/test/sqllogictest/mz_aws_privatelink_connections.slt b/test/sqllogictest/mz_aws_privatelink_connections.slt new file mode 100644 index 0000000000000..ce9db7b412bef --- /dev/null +++ b/test/sqllogictest/mz_aws_privatelink_connections.slt @@ -0,0 +1,85 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Behavioural tests for mz_catalog.mz_aws_privatelink_connections. The view is a +# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see +# MZ_AWS_PRIVATELINK_CONNECTIONS in src/catalog/src/builtin/mz_catalog.rs. The +# principal is reconstructed from the AWS principal context. A local +# sqllogictest env has no such context, so the context functions fold to NULL +# and the view emits no row, matching the removed packer which skipped the row +# when the context was absent. The populated form is exercised by +# test/cloudtest/test_privatelink_connection.py. + +mode cockroach + +reset-server + +# --- Schema ------------------------------------------------------------------- + +# Lock down: 2 columns, canonical order, types, and nullability. +query TTT +SELECT c.name, c.type, c.nullable::text +FROM mz_catalog.mz_columns c +JOIN mz_catalog.mz_objects o ON c.id = o.id +JOIN mz_catalog.mz_schemas s ON o.schema_id = s.id +WHERE o.name = 'mz_aws_privatelink_connections' AND s.name = 'mz_catalog' +ORDER BY c.position +---- +id text false +principal text false + +# --- Empty on a fresh env ----------------------------------------------------- + +query I +SELECT count(*) FROM mz_aws_privatelink_connections +---- +0 + +# --- Skip on missing AWS context ---------------------------------------------- + +simple conn=mz_system,user=mz_system +ALTER SYSTEM SET max_aws_privatelink_connections = 5 +---- +COMPLETE 0 + +statement ok +CREATE CONNECTION plc TO AWS PRIVATELINK ( + SERVICE NAME 'com.amazonaws.vpce.us-east-1.vpce-svc-0e123abc123198abc', + AVAILABILITY ZONES ('use1-az1', 'use1-az2') +) + +# The connection exists as an aws-privatelink connection ... +query I +SELECT count(*) FROM mz_connections WHERE name = 'plc' AND type = 'aws-privatelink' +---- +1 + +# ... but with no AWS principal context the principal is NULL, so the view skips +# it, exactly like the packer did. +query I +SELECT count(*) FROM mz_aws_privatelink_connections +---- +0 + +# --- FK-like invariant: every row joins its parent connection ----------------- + +query I +SELECT count(*) FROM mz_aws_privatelink_connections plc +LEFT JOIN mz_connections c ON plc.id = c.id +WHERE c.id IS NULL +---- +0 + +statement ok +DROP CONNECTION plc + +simple conn=mz_system,user=mz_system +ALTER SYSTEM RESET max_aws_privatelink_connections +---- +COMPLETE 0 diff --git a/test/sqllogictest/mz_catalog_server_index_accounting.slt b/test/sqllogictest/mz_catalog_server_index_accounting.slt index 53ef49be9947b..3c108d65f8048 100644 --- a/test/sqllogictest/mz_catalog_server_index_accounting.slt +++ b/test/sqllogictest/mz_catalog_server_index_accounting.slt @@ -37,107 +37,107 @@ mz_arrangement_heap_capacity_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangemen mz_arrangement_heap_size_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_heap_size_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_heap_size_raw"␠("operator_id",␠"worker_id") mz_arrangement_records_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_records_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_records_raw"␠("operator_id",␠"worker_id") mz_arrangement_sharing_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_sharing_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_sharing_raw"␠("operator_id",␠"worker_id") -mz_cluster_auto_scaling_strategies_ind CREATE␠INDEX␠"mz_cluster_auto_scaling_strategies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s514␠AS␠"mz_internal"."mz_cluster_auto_scaling_strategies"]␠("cluster_id") -mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s761␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") +mz_cluster_auto_scaling_strategies_ind CREATE␠INDEX␠"mz_cluster_auto_scaling_strategies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s518␠AS␠"mz_internal"."mz_cluster_auto_scaling_strategies"]␠("cluster_id") +mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s765␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") mz_cluster_prometheus_metrics_s2_primary_idx CREATE␠INDEX␠"mz_cluster_prometheus_metrics_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_cluster_prometheus_metrics"␠("process_id",␠"metric_name",␠"labels") -mz_cluster_reconfigurations_ind CREATE␠INDEX␠"mz_cluster_reconfigurations_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s513␠AS␠"mz_internal"."mz_cluster_reconfigurations"]␠("cluster_id") -mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s755␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") -mz_cluster_replica_history_ind CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s611␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at") -mz_cluster_replica_metrics_history_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s519␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id") -mz_cluster_replica_metrics_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s520␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id") -mz_cluster_replica_name_history_ind CREATE␠INDEX␠"mz_cluster_replica_name_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s612␠AS␠"mz_internal"."mz_cluster_replica_name_history"]␠("id") -mz_cluster_replica_size_internal_ind CREATE␠INDEX␠"mz_cluster_replica_size_internal_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s509␠AS␠"mz_internal"."mz_cluster_replica_size_internal"]␠("size") -mz_cluster_replica_sizes_ind CREATE␠INDEX␠"mz_cluster_replica_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s508␠AS␠"mz_catalog"."mz_cluster_replica_sizes"]␠("size") -mz_cluster_replica_status_history_ind CREATE␠INDEX␠"mz_cluster_replica_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s521␠AS␠"mz_internal"."mz_cluster_replica_status_history"]␠("replica_id") -mz_cluster_replica_statuses_ind CREATE␠INDEX␠"mz_cluster_replica_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s522␠AS␠"mz_internal"."mz_cluster_replica_statuses"]␠("replica_id") -mz_cluster_replicas_ind CREATE␠INDEX␠"mz_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s518␠AS␠"mz_catalog"."mz_cluster_replicas"]␠("id") -mz_clusters_ind CREATE␠INDEX␠"mz_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s510␠AS␠"mz_catalog"."mz_clusters"]␠("id") -mz_columns_ind CREATE␠INDEX␠"mz_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s481␠AS␠"mz_catalog"."mz_columns"]␠("name") -mz_comments_ind CREATE␠INDEX␠"mz_comments_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s537␠AS␠"mz_internal"."mz_comments"]␠("id") +mz_cluster_reconfigurations_ind CREATE␠INDEX␠"mz_cluster_reconfigurations_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s517␠AS␠"mz_internal"."mz_cluster_reconfigurations"]␠("cluster_id") +mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s759␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") +mz_cluster_replica_history_ind CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s615␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at") +mz_cluster_replica_metrics_history_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s523␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id") +mz_cluster_replica_metrics_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s524␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id") +mz_cluster_replica_name_history_ind CREATE␠INDEX␠"mz_cluster_replica_name_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s616␠AS␠"mz_internal"."mz_cluster_replica_name_history"]␠("id") +mz_cluster_replica_size_internal_ind CREATE␠INDEX␠"mz_cluster_replica_size_internal_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s513␠AS␠"mz_internal"."mz_cluster_replica_size_internal"]␠("size") +mz_cluster_replica_sizes_ind CREATE␠INDEX␠"mz_cluster_replica_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s512␠AS␠"mz_catalog"."mz_cluster_replica_sizes"]␠("size") +mz_cluster_replica_status_history_ind CREATE␠INDEX␠"mz_cluster_replica_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s525␠AS␠"mz_internal"."mz_cluster_replica_status_history"]␠("replica_id") +mz_cluster_replica_statuses_ind CREATE␠INDEX␠"mz_cluster_replica_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s526␠AS␠"mz_internal"."mz_cluster_replica_statuses"]␠("replica_id") +mz_cluster_replicas_ind CREATE␠INDEX␠"mz_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s522␠AS␠"mz_catalog"."mz_cluster_replicas"]␠("id") +mz_clusters_ind CREATE␠INDEX␠"mz_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s514␠AS␠"mz_catalog"."mz_clusters"]␠("id") +mz_columns_ind CREATE␠INDEX␠"mz_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s485␠AS␠"mz_catalog"."mz_columns"]␠("name") +mz_comments_ind CREATE␠INDEX␠"mz_comments_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s541␠AS␠"mz_internal"."mz_comments"]␠("id") mz_compute_dataflow_global_ids_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_dataflow_global_ids_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_dataflow_global_ids_per_worker"␠("id",␠"worker_id",␠"global_id") -mz_compute_dependencies_ind CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s737␠AS␠"mz_internal"."mz_compute_dependencies"]␠("dependency_id") +mz_compute_dependencies_ind CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s741␠AS␠"mz_internal"."mz_compute_dependencies"]␠("dependency_id") mz_compute_error_counts_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_error_counts_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_error_counts_raw"␠("export_id",␠"worker_id") mz_compute_exports_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_exports_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_exports_per_worker"␠("export_id",␠"worker_id") mz_compute_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_frontiers_per_worker"␠("export_id",␠"worker_id") -mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s746␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") +mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s750␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") mz_compute_hydration_times_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_hydration_times_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_hydration_times_per_worker"␠("export_id",␠"worker_id") mz_compute_import_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_import_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_import_frontiers_per_worker"␠("export_id",␠"import_id",␠"worker_id") mz_compute_lir_mapping_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_lir_mapping_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_lir_mapping_per_worker"␠("global_id",␠"lir_id",␠"worker_id") mz_compute_operator_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_durations_histogram_raw"␠("id",␠"worker_id",␠"duration_ns") mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_hydration_statuses_per_worker"␠("export_id",␠"lir_id",␠"worker_id") -mz_connections_ind CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s516␠AS␠"mz_catalog"."mz_connections"]␠("schema_id") -mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s742␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") -mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s741␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") -mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s740␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") -mz_databases_ind CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s479␠AS␠"mz_catalog"."mz_databases"]␠("name") +mz_connections_ind CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s520␠AS␠"mz_catalog"."mz_connections"]␠("schema_id") +mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s746␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") +mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s745␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") +mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s744␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") +mz_databases_ind CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s483␠AS␠"mz_catalog"."mz_databases"]␠("name") mz_dataflow_addresses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_addresses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_addresses_per_worker"␠("id",␠"worker_id") mz_dataflow_channels_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_channels_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_channels_per_worker"␠("id",␠"worker_id") mz_dataflow_operator_reachability_raw_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operator_reachability_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operator_reachability_raw"␠("id",␠"worker_id",␠"source",␠"port",␠"update_type",␠"time") mz_dataflow_operators_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operators_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operators_per_worker"␠("id",␠"worker_id") -mz_frontiers_ind CREATE␠INDEX␠"mz_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s728␠AS␠"mz_internal"."mz_frontiers"]␠("object_id") -mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s757␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") -mz_indexes_ind CREATE␠INDEX␠"mz_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s482␠AS␠"mz_catalog"."mz_indexes"]␠("id") -mz_kafka_sources_ind CREATE␠INDEX␠"mz_kafka_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s476␠AS␠"mz_catalog"."mz_kafka_sources"]␠("id") -mz_materialized_views_ind CREATE␠INDEX␠"mz_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s540␠AS␠"mz_catalog"."mz_materialized_views"]␠("id") +mz_frontiers_ind CREATE␠INDEX␠"mz_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s732␠AS␠"mz_internal"."mz_frontiers"]␠("object_id") +mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s761␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") +mz_indexes_ind CREATE␠INDEX␠"mz_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s486␠AS␠"mz_catalog"."mz_indexes"]␠("id") +mz_kafka_sources_ind CREATE␠INDEX␠"mz_kafka_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s480␠AS␠"mz_catalog"."mz_kafka_sources"]␠("id") +mz_materialized_views_ind CREATE␠INDEX␠"mz_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s544␠AS␠"mz_catalog"."mz_materialized_views"]␠("id") mz_message_batch_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_batch_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_batch_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_batch_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") -mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s831␠AS␠"mz_internal"."mz_notices"]␠("id") -mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s749␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") -mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s749␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") -mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s747␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") -mz_object_dependencies_ind CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s477␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id") -mz_object_history_ind CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s551␠AS␠"mz_internal"."mz_object_history"]␠("id") -mz_object_lifetimes_ind CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s552␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id") -mz_object_transitive_dependencies_ind CREATE␠INDEX␠"mz_object_transitive_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s568␠AS␠"mz_internal"."mz_object_transitive_dependencies"]␠("object_id") -mz_objects_ind CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s548␠AS␠"mz_catalog"."mz_objects"]␠("schema_id") +mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s835␠AS␠"mz_internal"."mz_notices"]␠("id") +mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") +mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") +mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s751␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") +mz_object_dependencies_ind CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s481␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id") +mz_object_history_ind CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s555␠AS␠"mz_internal"."mz_object_history"]␠("id") +mz_object_lifetimes_ind CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s556␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id") +mz_object_transitive_dependencies_ind CREATE␠INDEX␠"mz_object_transitive_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s572␠AS␠"mz_internal"."mz_object_transitive_dependencies"]␠("object_id") +mz_objects_ind CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s552␠AS␠"mz_catalog"."mz_objects"]␠("schema_id") mz_peek_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_peek_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_peek_durations_histogram_raw"␠("worker_id",␠"type",␠"duration_ns") -mz_recent_activity_log_thinned_ind CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s712␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash") -mz_recent_sql_text_ind CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s708␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash") -mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s824␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") -mz_roles_ind CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s500␠AS␠"mz_catalog"."mz_roles"]␠("id") +mz_recent_activity_log_thinned_ind CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s716␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash") +mz_recent_sql_text_ind CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s712␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash") +mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s828␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") +mz_roles_ind CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s504␠AS␠"mz_catalog"."mz_roles"]␠("id") mz_scheduling_elapsed_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_elapsed_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_elapsed_raw"␠("id",␠"worker_id") mz_scheduling_parks_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_parks_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_parks_histogram_raw"␠("worker_id",␠"slept_for_ns",␠"requested_ns") -mz_schemas_ind CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s480␠AS␠"mz_catalog"."mz_schemas"]␠("database_id") -mz_secrets_ind CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s515␠AS␠"mz_catalog"."mz_secrets"]␠("name") -mz_show_all_objects_ind CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s596␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id") -mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s759␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") -mz_show_clusters_ind CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s598␠AS␠"mz_internal"."mz_show_clusters"]␠("name") -mz_show_columns_ind CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s597␠AS␠"mz_internal"."mz_show_columns"]␠("id") -mz_show_connections_ind CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s606␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id") -mz_show_databases_ind CREATE␠INDEX␠"mz_show_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s600␠AS␠"mz_internal"."mz_show_databases"]␠("name") -mz_show_indexes_ind CREATE␠INDEX␠"mz_show_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s610␠AS␠"mz_internal"."mz_show_indexes"]␠("schema_id") -mz_show_materialized_views_ind CREATE␠INDEX␠"mz_show_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s609␠AS␠"mz_internal"."mz_show_materialized_views"]␠("schema_id") -mz_show_roles_ind CREATE␠INDEX␠"mz_show_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s605␠AS␠"mz_internal"."mz_show_roles"]␠("name") -mz_show_schemas_ind CREATE␠INDEX␠"mz_show_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s601␠AS␠"mz_internal"."mz_show_schemas"]␠("database_id") -mz_show_secrets_ind CREATE␠INDEX␠"mz_show_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s599␠AS␠"mz_internal"."mz_show_secrets"]␠("schema_id") -mz_show_sinks_ind CREATE␠INDEX␠"mz_show_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s608␠AS␠"mz_internal"."mz_show_sinks"]␠("schema_id") -mz_show_sources_ind CREATE␠INDEX␠"mz_show_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s607␠AS␠"mz_internal"."mz_show_sources"]␠("schema_id") -mz_show_tables_ind CREATE␠INDEX␠"mz_show_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s602␠AS␠"mz_internal"."mz_show_tables"]␠("schema_id") -mz_show_types_ind CREATE␠INDEX␠"mz_show_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s604␠AS␠"mz_internal"."mz_show_types"]␠("schema_id") -mz_show_views_ind CREATE␠INDEX␠"mz_show_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s603␠AS␠"mz_internal"."mz_show_views"]␠("schema_id") -mz_sink_statistics_ind CREATE␠INDEX␠"mz_sink_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s725␠AS␠"mz_internal"."mz_sink_statistics"]␠("id",␠"replica_id") -mz_sink_status_history_ind CREATE␠INDEX␠"mz_sink_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s697␠AS␠"mz_internal"."mz_sink_status_history"]␠("sink_id") -mz_sink_statuses_ind CREATE␠INDEX␠"mz_sink_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s698␠AS␠"mz_internal"."mz_sink_statuses"]␠("id") -mz_sinks_ind CREATE␠INDEX␠"mz_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s492␠AS␠"mz_catalog"."mz_sinks"]␠("id") -mz_source_statistics_ind CREATE␠INDEX␠"mz_source_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s723␠AS␠"mz_internal"."mz_source_statistics"]␠("id",␠"replica_id") -mz_source_statistics_with_history_ind CREATE␠INDEX␠"mz_source_statistics_with_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s721␠AS␠"mz_internal"."mz_source_statistics_with_history"]␠("id",␠"replica_id") -mz_source_status_history_ind CREATE␠INDEX␠"mz_source_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s699␠AS␠"mz_internal"."mz_source_status_history"]␠("source_id") -mz_source_statuses_ind CREATE␠INDEX␠"mz_source_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s716␠AS␠"mz_internal"."mz_source_statuses"]␠("id") -mz_sources_ind CREATE␠INDEX␠"mz_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s485␠AS␠"mz_catalog"."mz_sources"]␠("id") -mz_tables_ind CREATE␠INDEX␠"mz_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s484␠AS␠"mz_catalog"."mz_tables"]␠("schema_id") -mz_types_ind CREATE␠INDEX␠"mz_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s494␠AS␠"mz_catalog"."mz_types"]␠("schema_id") -mz_views_ind CREATE␠INDEX␠"mz_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s493␠AS␠"mz_catalog"."mz_views"]␠("schema_id") -mz_wallclock_global_lag_recent_history_ind CREATE␠INDEX␠"mz_wallclock_global_lag_recent_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s732␠AS␠"mz_internal"."mz_wallclock_global_lag_recent_history"]␠("object_id") -mz_webhook_sources_ind CREATE␠INDEX␠"mz_webhook_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s538␠AS␠"mz_internal"."mz_webhook_sources"]␠("id") -pg_attrdef_all_databases_ind CREATE␠INDEX␠"pg_attrdef_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s637␠AS␠"mz_internal"."pg_attrdef_all_databases"]␠("oid",␠"adrelid",␠"adnum",␠"adbin",␠"adsrc") -pg_attribute_all_databases_ind CREATE␠INDEX␠"pg_attribute_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s630␠AS␠"mz_internal"."pg_attribute_all_databases"]␠("attrelid",␠"attname",␠"atttypid",␠"attlen",␠"attnum",␠"atttypmod",␠"attnotnull",␠"atthasdef",␠"attidentity",␠"attgenerated",␠"attisdropped",␠"attcollation",␠"database_name",␠"pg_type_database_name") -pg_authid_core_ind CREATE␠INDEX␠"pg_authid_core_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s647␠AS␠"mz_internal"."pg_authid_core"]␠("rolname") -pg_class_all_databases_ind CREATE␠INDEX␠"pg_class_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s618␠AS␠"mz_internal"."pg_class_all_databases"]␠("relname") -pg_description_all_databases_ind CREATE␠INDEX␠"pg_description_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s627␠AS␠"mz_internal"."pg_description_all_databases"]␠("objoid",␠"classoid",␠"objsubid",␠"description",␠"oid_database_name",␠"class_database_name") -pg_namespace_all_databases_ind CREATE␠INDEX␠"pg_namespace_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s615␠AS␠"mz_internal"."pg_namespace_all_databases"]␠("nspname") -pg_type_all_databases_ind CREATE␠INDEX␠"pg_type_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s624␠AS␠"mz_internal"."pg_type_all_databases"]␠("oid") +mz_schemas_ind CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s484␠AS␠"mz_catalog"."mz_schemas"]␠("database_id") +mz_secrets_ind CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s519␠AS␠"mz_catalog"."mz_secrets"]␠("name") +mz_show_all_objects_ind CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s600␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id") +mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s763␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") +mz_show_clusters_ind CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s602␠AS␠"mz_internal"."mz_show_clusters"]␠("name") +mz_show_columns_ind CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s601␠AS␠"mz_internal"."mz_show_columns"]␠("id") +mz_show_connections_ind CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s610␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id") +mz_show_databases_ind CREATE␠INDEX␠"mz_show_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s604␠AS␠"mz_internal"."mz_show_databases"]␠("name") +mz_show_indexes_ind CREATE␠INDEX␠"mz_show_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s614␠AS␠"mz_internal"."mz_show_indexes"]␠("schema_id") +mz_show_materialized_views_ind CREATE␠INDEX␠"mz_show_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s613␠AS␠"mz_internal"."mz_show_materialized_views"]␠("schema_id") +mz_show_roles_ind CREATE␠INDEX␠"mz_show_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s609␠AS␠"mz_internal"."mz_show_roles"]␠("name") +mz_show_schemas_ind CREATE␠INDEX␠"mz_show_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s605␠AS␠"mz_internal"."mz_show_schemas"]␠("database_id") +mz_show_secrets_ind CREATE␠INDEX␠"mz_show_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s603␠AS␠"mz_internal"."mz_show_secrets"]␠("schema_id") +mz_show_sinks_ind CREATE␠INDEX␠"mz_show_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s612␠AS␠"mz_internal"."mz_show_sinks"]␠("schema_id") +mz_show_sources_ind CREATE␠INDEX␠"mz_show_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s611␠AS␠"mz_internal"."mz_show_sources"]␠("schema_id") +mz_show_tables_ind CREATE␠INDEX␠"mz_show_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s606␠AS␠"mz_internal"."mz_show_tables"]␠("schema_id") +mz_show_types_ind CREATE␠INDEX␠"mz_show_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s608␠AS␠"mz_internal"."mz_show_types"]␠("schema_id") +mz_show_views_ind CREATE␠INDEX␠"mz_show_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s607␠AS␠"mz_internal"."mz_show_views"]␠("schema_id") +mz_sink_statistics_ind CREATE␠INDEX␠"mz_sink_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s729␠AS␠"mz_internal"."mz_sink_statistics"]␠("id",␠"replica_id") +mz_sink_status_history_ind CREATE␠INDEX␠"mz_sink_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s701␠AS␠"mz_internal"."mz_sink_status_history"]␠("sink_id") +mz_sink_statuses_ind CREATE␠INDEX␠"mz_sink_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s702␠AS␠"mz_internal"."mz_sink_statuses"]␠("id") +mz_sinks_ind CREATE␠INDEX␠"mz_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s496␠AS␠"mz_catalog"."mz_sinks"]␠("id") +mz_source_statistics_ind CREATE␠INDEX␠"mz_source_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s727␠AS␠"mz_internal"."mz_source_statistics"]␠("id",␠"replica_id") +mz_source_statistics_with_history_ind CREATE␠INDEX␠"mz_source_statistics_with_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s725␠AS␠"mz_internal"."mz_source_statistics_with_history"]␠("id",␠"replica_id") +mz_source_status_history_ind CREATE␠INDEX␠"mz_source_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s703␠AS␠"mz_internal"."mz_source_status_history"]␠("source_id") +mz_source_statuses_ind CREATE␠INDEX␠"mz_source_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s720␠AS␠"mz_internal"."mz_source_statuses"]␠("id") +mz_sources_ind CREATE␠INDEX␠"mz_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s489␠AS␠"mz_catalog"."mz_sources"]␠("id") +mz_tables_ind CREATE␠INDEX␠"mz_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s488␠AS␠"mz_catalog"."mz_tables"]␠("schema_id") +mz_types_ind CREATE␠INDEX␠"mz_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s498␠AS␠"mz_catalog"."mz_types"]␠("schema_id") +mz_views_ind CREATE␠INDEX␠"mz_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s497␠AS␠"mz_catalog"."mz_views"]␠("schema_id") +mz_wallclock_global_lag_recent_history_ind CREATE␠INDEX␠"mz_wallclock_global_lag_recent_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s736␠AS␠"mz_internal"."mz_wallclock_global_lag_recent_history"]␠("object_id") +mz_webhook_sources_ind CREATE␠INDEX␠"mz_webhook_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s542␠AS␠"mz_internal"."mz_webhook_sources"]␠("id") +pg_attrdef_all_databases_ind CREATE␠INDEX␠"pg_attrdef_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s641␠AS␠"mz_internal"."pg_attrdef_all_databases"]␠("oid",␠"adrelid",␠"adnum",␠"adbin",␠"adsrc") +pg_attribute_all_databases_ind CREATE␠INDEX␠"pg_attribute_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s634␠AS␠"mz_internal"."pg_attribute_all_databases"]␠("attrelid",␠"attname",␠"atttypid",␠"attlen",␠"attnum",␠"atttypmod",␠"attnotnull",␠"atthasdef",␠"attidentity",␠"attgenerated",␠"attisdropped",␠"attcollation",␠"database_name",␠"pg_type_database_name") +pg_authid_core_ind CREATE␠INDEX␠"pg_authid_core_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s651␠AS␠"mz_internal"."pg_authid_core"]␠("rolname") +pg_class_all_databases_ind CREATE␠INDEX␠"pg_class_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s622␠AS␠"mz_internal"."pg_class_all_databases"]␠("relname") +pg_description_all_databases_ind CREATE␠INDEX␠"pg_description_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s631␠AS␠"mz_internal"."pg_description_all_databases"]␠("objoid",␠"classoid",␠"objsubid",␠"description",␠"oid_database_name",␠"class_database_name") +pg_namespace_all_databases_ind CREATE␠INDEX␠"pg_namespace_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s619␠AS␠"mz_internal"."pg_namespace_all_databases"]␠("nspname") +pg_type_all_databases_ind CREATE␠INDEX␠"pg_type_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s628␠AS␠"mz_internal"."pg_type_all_databases"]␠("oid") # Record all transitive dependencies (tables, sources, views, mvs) of indexes on # the mz_catalog_server cluster. diff --git a/test/sqllogictest/mz_kafka_connections.slt b/test/sqllogictest/mz_kafka_connections.slt new file mode 100644 index 0000000000000..18e30b1b71b10 --- /dev/null +++ b/test/sqllogictest/mz_kafka_connections.slt @@ -0,0 +1,116 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Behavioural tests for mz_catalog.mz_kafka_connections. The view is a +# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see +# MZ_KAFKA_CONNECTIONS in src/catalog/src/builtin/mz_catalog.rs and the +# parse_connection_details helper in +# src/expr/src/scalar/func/impls/jsonb.rs. Rows with a live broker are +# exercised by test/testdrive/connection-create-drop.td; this file pins the +# column shape, the progress-topic default, and the parent-type filtering, +# none of which need a live Kafka. + +mode cockroach + +reset-server + +# --- Schema ------------------------------------------------------------------- + +# Lock down: 3 columns, canonical order, types, and nullability. +query TTT +SELECT c.name, c.type, c.nullable::text +FROM mz_catalog.mz_columns c +JOIN mz_catalog.mz_objects o ON c.id = o.id +JOIN mz_catalog.mz_schemas s ON o.schema_id = s.id +WHERE o.name = 'mz_kafka_connections' AND s.name = 'mz_catalog' +ORDER BY c.position +---- +id text false +brokers text[] false +sink_progress_topic text false + +# --- Empty on a fresh env ----------------------------------------------------- + +query I +SELECT count(*) FROM mz_kafka_connections +---- +0 + +# --- A kafka connection populates the view ------------------------------------ + +statement ok +CREATE CONNECTION kc TO KAFKA (BROKER 'localhost:9092', SECURITY PROTOCOL PLAINTEXT) WITH (VALIDATE = false) + +query T +SELECT brokers FROM mz_kafka_connections kc +JOIN mz_connections c USING (id) +WHERE c.name = 'kc' +---- +{localhost:9092} + +# The default progress topic reconstructs KafkaConnection::progress_topic. +query T +SELECT sink_progress_topic = + '_materialize-progress-' || mz_environment_id() || '-' || id +FROM mz_kafka_connections kc +JOIN mz_connections c USING (id) +WHERE c.name = 'kc' +---- +true + +# An explicit PROGRESS TOPIC is passed through verbatim. +statement ok +CREATE CONNECTION kc_progress TO KAFKA ( + BROKER 'localhost:9092', + PROGRESS TOPIC 'my_progress', + SECURITY PROTOCOL PLAINTEXT +) WITH (VALIDATE = false) + +query T +SELECT sink_progress_topic FROM mz_kafka_connections kc +JOIN mz_connections c USING (id) +WHERE c.name = 'kc_progress' +---- +my_progress + +# --- Non-kafka connections are excluded --------------------------------------- + +statement ok +CREATE CONNECTION sc TO SSH TUNNEL (HOST 'ssh.example.com', USER 'mz') + +query I +SELECT count(*) FROM mz_kafka_connections kc +JOIN mz_connections c USING (id) +WHERE c.type != 'kafka' +---- +0 + +# --- FK-like invariant: every row joins its parent connection ----------------- + +query I +SELECT count(*) FROM mz_kafka_connections kc +LEFT JOIN mz_connections c ON kc.id = c.id +WHERE c.id IS NULL +---- +0 + +# No duplicate ids. +query I +SELECT count(*) - count(DISTINCT id) FROM mz_kafka_connections +---- +0 + +statement ok +DROP CONNECTION kc + +statement ok +DROP CONNECTION kc_progress + +statement ok +DROP CONNECTION sc diff --git a/test/sqllogictest/mz_kafka_source_tables.slt b/test/sqllogictest/mz_kafka_source_tables.slt index 2bdc3acd5f3ae..bf2e55335d509 100644 --- a/test/sqllogictest/mz_kafka_source_tables.slt +++ b/test/sqllogictest/mz_kafka_source_tables.slt @@ -33,7 +33,7 @@ ORDER BY c.position ---- id text false topic text false -envelope_type text true +envelope_type text false key_format text true value_format text true diff --git a/test/sqllogictest/mz_ssh_tunnel_connections.slt b/test/sqllogictest/mz_ssh_tunnel_connections.slt new file mode 100644 index 0000000000000..b555697bb352e --- /dev/null +++ b/test/sqllogictest/mz_ssh_tunnel_connections.slt @@ -0,0 +1,88 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Behavioural tests for mz_catalog.mz_ssh_tunnel_connections. The view is a +# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see +# MZ_SSH_TUNNEL_CONNECTIONS in src/catalog/src/builtin/mz_catalog.rs and the +# parse_connection_details helper in +# src/expr/src/scalar/func/impls/jsonb.rs. The end-to-end SSH tunnel behaviour +# is exercised by test/ssh-connection/*; this file pins the column shape and +# the parent-type filtering. + +mode cockroach + +reset-server + +# --- Schema ------------------------------------------------------------------- + +# Lock down: 3 columns, canonical order, types, and nullability. +query TTT +SELECT c.name, c.type, c.nullable::text +FROM mz_catalog.mz_columns c +JOIN mz_catalog.mz_objects o ON c.id = o.id +JOIN mz_catalog.mz_schemas s ON o.schema_id = s.id +WHERE o.name = 'mz_ssh_tunnel_connections' AND s.name = 'mz_catalog' +ORDER BY c.position +---- +id text false +public_key_1 text false +public_key_2 text false + +# --- Empty on a fresh env ----------------------------------------------------- + +query I +SELECT count(*) FROM mz_ssh_tunnel_connections +---- +0 + +# --- An ssh connection populates the view with generated public keys ---------- + +statement ok +CREATE CONNECTION sc TO SSH TUNNEL (HOST 'ssh.example.com', USER 'mz') + +query TT +SELECT public_key_1 LIKE 'ssh-ed25519%', public_key_2 LIKE 'ssh-ed25519%' +FROM mz_ssh_tunnel_connections stc +JOIN mz_connections c USING (id) +WHERE c.name = 'sc' +---- +true true + +# --- Non-ssh connections are excluded ----------------------------------------- + +statement ok +CREATE CONNECTION kc TO KAFKA (BROKER 'localhost:9092', SECURITY PROTOCOL PLAINTEXT) WITH (VALIDATE = false) + +query I +SELECT count(*) FROM mz_ssh_tunnel_connections stc +JOIN mz_connections c USING (id) +WHERE c.type != 'ssh-tunnel' +---- +0 + +# --- FK-like invariant: every row joins its parent connection ----------------- + +query I +SELECT count(*) FROM mz_ssh_tunnel_connections stc +LEFT JOIN mz_connections c ON stc.id = c.id +WHERE c.id IS NULL +---- +0 + +# No duplicate ids. +query I +SELECT count(*) - count(DISTINCT id) FROM mz_ssh_tunnel_connections +---- +0 + +statement ok +DROP CONNECTION sc + +statement ok +DROP CONNECTION kc diff --git a/test/sqllogictest/oid.slt b/test/sqllogictest/oid.slt index 8b4f2ce860925..a563054773926 100644 --- a/test/sqllogictest/oid.slt +++ b/test/sqllogictest/oid.slt @@ -1096,6 +1096,8 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 16945 mz_role_parameters 16946 mz_materialized_view_refresh_strategies 16947 mz_cluster_schedules +16948 mz_postgres_source_tables +16949 mz_mysql_source_tables 16950 mz_kafka_sources_ind 16951 mz_webhook_sources_ind 16952 mz_history_retention_strategies @@ -1165,6 +1167,7 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17022 mz_wallclock_global_lag_history 17023 mz_wallclock_global_lag_recent_history 17024 mz_wallclock_global_lag_recent_history_ind +17025 mz_kafka_source_tables 17030 mz_objects_id_namespace_types 17031 mz_object_history 17032 mz_object_history_ind @@ -1188,6 +1191,7 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17054 mz_wallclock_global_lag 17055 mz_wallclock_global_lag_histogram_raw 17056 mz_wallclock_global_lag_histogram +17057 mz_sql_server_source_tables 17058 mz_license_keys 17059 mz_role_auth 17060 mz_iceberg_sinks @@ -1236,8 +1240,8 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17104 mz_cluster_auto_scaling_strategies_ind 17105 parse_postgres_source_details 17106 parse_kafka_source_details -17107 mz_postgres_source_tables -17108 mz_mysql_source_tables -17109 mz_sql_server_source_tables -17110 mz_kafka_source_tables 17111 parse_source_export_details +17112 parse_connection_details +17113 mz_aws_account_id +17114 mz_aws_external_id_prefix +17115 mz_aws_connection_role_arn diff --git a/test/testdrive/catalog.td b/test/testdrive/catalog.td index ab904914157af..eef5713d4fc51 100644 --- a/test/testdrive/catalog.td +++ b/test/testdrive/catalog.td @@ -511,7 +511,6 @@ mz_cluster_replica_frontiers source "" name comment -------------------------------------- mz_array_types "" -mz_aws_privatelink_connections "" mz_base_types "" mz_cluster_replica_sizes "" mz_columns "" @@ -519,7 +518,6 @@ mz_egress_ips "" mz_functions "" mz_iceberg_sinks "" mz_index_columns "" -mz_kafka_connections "" mz_kafka_sinks "" mz_list_types "" mz_map_types "" @@ -527,7 +525,6 @@ mz_operators "" mz_pseudo_types "" mz_role_auth "" mz_sinks "" -mz_ssh_tunnel_connections "" mz_tables "" mz_types "" mz_views "" @@ -546,12 +543,14 @@ mz_timezone_names "" name cluster comment -------------------------------------------------- mz_audit_events mz_catalog_server "" +mz_aws_privatelink_connections mz_catalog_server "" mz_clusters mz_catalog_server "" mz_cluster_replicas mz_catalog_server "" mz_connections mz_catalog_server "" mz_databases mz_catalog_server "" mz_default_privileges mz_catalog_server "" mz_indexes mz_catalog_server "" +mz_kafka_connections mz_catalog_server "" mz_kafka_sources mz_catalog_server "" mz_role_members mz_catalog_server "" mz_role_parameters mz_catalog_server "" @@ -559,6 +558,7 @@ mz_roles mz_catalog_server "" mz_schemas mz_catalog_server "" mz_secrets mz_catalog_server "" mz_sources mz_catalog_server "" +mz_ssh_tunnel_connections mz_catalog_server "" mz_system_privileges mz_catalog_server "" mz_materialized_views mz_catalog_server "" @@ -603,7 +603,6 @@ mz_wallclock_lag_history source "" name comment ------------------------------------------------ mz_aggregates "" -mz_aws_connections "" mz_cluster_replica_size_internal "" mz_history_retention_strategies "" mz_license_keys "" @@ -717,6 +716,7 @@ pg_authid_core "" > SHOW MATERIALIZED VIEWS FROM mz_internal name cluster comment ---------------------------------------------------- +mz_aws_connections mz_catalog_server "" mz_cluster_auto_scaling_strategies mz_catalog_server "" mz_cluster_reconfigurations mz_catalog_server "" mz_cluster_schedules mz_catalog_server "" @@ -830,7 +830,7 @@ test_table "" # `SHOW TABLES` and `mz_tables` should agree. > SELECT COUNT(*) FROM mz_tables WHERE id LIKE 's%' -38 +34 # There is one entry in mz_indexes for each field_number/expression of the index. > SELECT COUNT(id) FROM mz_indexes WHERE id LIKE 's%' diff --git a/test/testdrive/mz-sources.td b/test/testdrive/mz-sources.td index f2b1fe2fb5f34..49bd3db771ce7 100644 --- a/test/testdrive/mz-sources.td +++ b/test/testdrive/mz-sources.td @@ -178,3 +178,21 @@ testdrive-upsert-topic-${testdrive.seed} none text testdrive-none-topic-${testdrive.seed} none text text testdrive-dbzupsert-${testdrive.seed} debezium avro avro testdrive-upsert-topic-${testdrive.seed} upsert avro avro + +# A kafka source table that omits the ENVELOPE clause defaults to ENVELOPE NONE, +# so envelope_type must report 'none', not NULL. The omitted clause persists as +# absent in create_sql, and the view supplies the default. +> CREATE TABLE table_omit_envelope FROM SOURCE upsert_source (REFERENCE "testdrive-upsert-topic-${testdrive.seed}") + FORMAT TEXT + +> SELECT kst.envelope_type + FROM mz_internal.mz_kafka_source_tables kst + JOIN mz_tables t ON t.id = kst.id + WHERE t.name = 'table_omit_envelope' +none + +# A new-syntax kafka source ingests nothing into its own relation (envelopes are +# defined per table), so mz_sources.envelope_type is NULL for it, even though its +# tables report their own envelopes above. +> SELECT envelope_type FROM mz_sources WHERE name = 'upsert_source' + From 3de6a5f911cf2b8b0caf8ea6830e3bd7099fd600 Mon Sep 17 00:00:00 2001 From: alex-hunt-materialize <86254996+alex-hunt-materialize@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:02:07 +0100 Subject: [PATCH 3/6] Bump k8s-openapi feature to Kubernetes 1.34 (#38059) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bump k8s-openapi feature to Kubernetes 1.34. Bump kind to v0.32.0 and kubectl to v1.34.10 in the CI builder image and dev tooling. Bump all kind cluster configs to `kindest/node:v1.34.8` nodes (the pre-built 1.34 image for kind v0.32.0). Fix a race in our minio setup, which shows up more consistently in Kubernetes 1.34, where the PVC could end up on a different node and minio pods would never become ready. [DB-143](https://linear.app/materializeinc/issue/DB-143) Kubernetes 1.34 is the oldest currently supported version. Newer Kubernetes versions may not support legacy APIs over time, so better to stay current. This will require a similar bump in the cloud repo when bumping the submodule. ### Motivation Keep up to date, on supported platforms. ### Verification Nightly pipelines that exercise the bumped kind/kubectl/node images and should be run on this PR: - `cloudtest`, `cloudtest-slow`, `cloudtest-upgrade`: run kind via the rebuilt ci-builder image. - `orchestratord-*`: run in the `test/orchestratord` kind cluster, now on 1.34.8 nodes. - `k8s-node-recovery-*`: use `misc/kind/cluster-node-recovery-test.yaml`, now on 1.34.8 nodes. These are the steps the minio fix targets. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Fable 5 Co-authored-by: Dennis Felsing --- Cargo.toml | 2 +- ci/builder/Dockerfile | 26 ++++++------ console/README.md | 2 +- doc/developer/cloudtest.md | 4 +- misc/kind/cluster-node-recovery-test.yaml | 22 +++++----- misc/kind/cluster.yaml | 14 +++---- .../cloudtest/app/materialize_application.py | 14 +++++-- .../python/materialize/cloudtest/k8s/minio.py | 40 ++++++++++++------- misc/scratch/provision.bash | 8 ++-- test/orchestratord/cluster.yaml.tmpl | 6 +-- test/orchestratord/mzcompose.py | 4 +- 11 files changed, 78 insertions(+), 64 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a581d55e26722..e72662ef8b42b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -399,7 +399,7 @@ jemalloc_pprof = "0.8.2" jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] } junit-report = "0.8.3" k8s-controller = "0.12.0" -k8s-openapi = { version = "0.27.0", features = ["schemars", "v1_32"] } +k8s-openapi = { version = "0.27.0", features = ["schemars", "v1_34"] } kube = { version = "3.1.0", default-features = false, features = ["client", "derive", "openssl-tls", "runtime", "ws"] } launchdarkly-server-sdk = { version = "3.1.1", default-features = false, features = ["hyper-rustls-native-roots", "crypto-aws-lc-rs"] } launchdarkly-sdk-transport = "0.1.4" diff --git a/ci/builder/Dockerfile b/ci/builder/Dockerfile index 1df05693d3d1e..bb61c9e987bd2 100644 --- a/ci/builder/Dockerfile +++ b/ci/builder/Dockerfile @@ -325,16 +325,15 @@ RUN if [ $ARCH_GCC = x86_64 ]; then \ # Install KinD, kubectl, helm & helm-docs -# TODO(def-) Upgrading kind/kubectl seems to cause Cloudtest failures -RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \ +RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \ && chmod +x /usr/local/bin/kind \ - && if [ $ARCH_GO = amd64 ]; then echo 'c72eda46430f065fb45c5f70e7c957cc9209402ef309294821978677c8fb3284 /usr/local/bin/kind' | sha256sum --check; fi \ - && if [ $ARCH_GO = arm64 ]; then echo '03d45095dbd9cc1689f179a3e5e5da24b77c2d1b257d7645abf1b4174bebcf2a /usr/local/bin/kind' | sha256sum --check; fi + && if [ $ARCH_GO = amd64 ]; then echo '50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 /usr/local/bin/kind' | sha256sum --check; fi \ + && if [ $ARCH_GO = arm64 ]; then echo 'b92cd615e97585de8ddade28ed5cd7feb4248d717c233eea5b03c37298900f5d /usr/local/bin/kind' | sha256sum --check; fi -RUN curl -fsSL https://dl.k8s.io/release/v1.24.3/bin/linux/$ARCH_GO/kubectl > /usr/local/bin/kubectl \ +RUN curl -fsSL https://dl.k8s.io/release/v1.34.10/bin/linux/$ARCH_GO/kubectl > /usr/local/bin/kubectl \ && chmod +x /usr/local/bin/kubectl \ - && if [ $ARCH_GO = amd64 ]; then echo '8a45348bdaf81d46caf1706c8bf95b3f431150554f47d444ffde89e8cdd712c1 /usr/local/bin/kubectl' | sha256sum --check; fi \ - && if [ $ARCH_GO = arm64 ]; then echo 'bdad4d3063ddb7bfa5ecf17fb8b029d5d81d7d4ea1650e4369aafa13ed97149a /usr/local/bin/kubectl' | sha256sum --check; fi + && if [ $ARCH_GO = amd64 ]; then echo '95bd70842bd11a524d24acd5b68726899e3488e153e45e2b4ae846545beda050 /usr/local/bin/kubectl' | sha256sum --check; fi \ + && if [ $ARCH_GO = arm64 ]; then echo '52d3aeefea32fdfa3671ccd636be5da463ddfd0a2fc09d7bcbaedcff4c76cad5 /usr/local/bin/kubectl' | sha256sum --check; fi RUN curl -fsSL https://get.helm.sh/helm-v4.0.0-linux-$ARCH_GO.tar.gz > helm.tar.gz \ && if [ $ARCH_GO = amd64 ]; then echo 'c77e9e7c1cc96e066bd240d190d1beed9a6b08060b2043ef0862c4f865eca08f helm.tar.gz' | sha256sum --check; fi \ @@ -517,16 +516,15 @@ RUN ln -s ../lib/node_modules/corepack/dist/corepack.js /usr/local/bin/corepack && ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx \ && corepack enable -# TODO(def-) Upgrading kind/kubectl seems to cause Cloudtest failures -RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \ +RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \ && chmod +x /usr/local/bin/kind \ - && if [ $ARCH_GO = amd64 ]; then echo 'c72eda46430f065fb45c5f70e7c957cc9209402ef309294821978677c8fb3284 /usr/local/bin/kind' | sha256sum --check; fi \ - && if [ $ARCH_GO = arm64 ]; then echo '03d45095dbd9cc1689f179a3e5e5da24b77c2d1b257d7645abf1b4174bebcf2a /usr/local/bin/kind' | sha256sum --check; fi + && if [ $ARCH_GO = amd64 ]; then echo '50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 /usr/local/bin/kind' | sha256sum --check; fi \ + && if [ $ARCH_GO = arm64 ]; then echo 'b92cd615e97585de8ddade28ed5cd7feb4248d717c233eea5b03c37298900f5d /usr/local/bin/kind' | sha256sum --check; fi -RUN curl -fsSL https://dl.k8s.io/release/v1.24.3/bin/linux/$ARCH_GO/kubectl > /usr/local/bin/kubectl \ +RUN curl -fsSL https://dl.k8s.io/release/v1.34.10/bin/linux/$ARCH_GO/kubectl > /usr/local/bin/kubectl \ && chmod +x /usr/local/bin/kubectl \ - && if [ $ARCH_GO = amd64 ]; then echo '8a45348bdaf81d46caf1706c8bf95b3f431150554f47d444ffde89e8cdd712c1 /usr/local/bin/kubectl' | sha256sum --check; fi \ - && if [ $ARCH_GO = arm64 ]; then echo 'bdad4d3063ddb7bfa5ecf17fb8b029d5d81d7d4ea1650e4369aafa13ed97149a /usr/local/bin/kubectl' | sha256sum --check; fi + && if [ $ARCH_GO = amd64 ]; then echo '95bd70842bd11a524d24acd5b68726899e3488e153e45e2b4ae846545beda050 /usr/local/bin/kubectl' | sha256sum --check; fi \ + && if [ $ARCH_GO = arm64 ]; then echo '52d3aeefea32fdfa3671ccd636be5da463ddfd0a2fc09d7bcbaedcff4c76cad5 /usr/local/bin/kubectl' | sha256sum --check; fi # Use Helm 3 (not 4) because the cloud repo's bin/kind-create uses relative # file paths for vendored charts, which Helm 4 rejects as invalid URLs. diff --git a/console/README.md b/console/README.md index 2a514e627ad40..e5f00ce227e90 100644 --- a/console/README.md +++ b/console/README.md @@ -67,7 +67,7 @@ Install the [aws-cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-s Install some k8s utils: ```shell -brew install kubectl@1.24 k9s kind +brew install kubernetes-cli@1.34 k9s kind ``` Run the commands below to configure your aws account and k8s contexts diff --git a/doc/developer/cloudtest.md b/doc/developer/cloudtest.md index d518700f971a8..35963e5a21d45 100644 --- a/doc/developer/cloudtest.md +++ b/doc/developer/cloudtest.md @@ -29,7 +29,7 @@ official [`kubernetes`] Python library to control the Kubernetes cluster. On Linux, use: ``` - curl -fL https://dl.k8s.io/release/v1.34.5/bin/linux/amd64/kubectl > kubectl + curl -fL https://dl.k8s.io/release/v1.34.10/bin/linux/amd64/kubectl > kubectl chmod +x kubectl sudo mv kubectl /usr/local/bin ``` @@ -48,7 +48,7 @@ official [`kubernetes`] Python library to control the Kubernetes cluster. On Linux, use: ``` - curl -fL https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-amd64 > kind + curl -fL https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-amd64 > kind chmod +x kind sudo mv kind /usr/local/bin ``` diff --git a/misc/kind/cluster-node-recovery-test.yaml b/misc/kind/cluster-node-recovery-test.yaml index f7b4ffe386b0d..726083343ab00 100644 --- a/misc/kind/cluster-node-recovery-test.yaml +++ b/misc/kind/cluster-node-recovery-test.yaml @@ -22,7 +22,7 @@ kubeadmConfigPatches: "service-node-port-range": "32000-32063" nodes: - role: control-plane - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 extraPortMappings: - containerPort: 32000 hostPort: 32000 @@ -154,37 +154,37 @@ nodes: hostPort: 32063 - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "1" topology.kubernetes.io/zone: "1" - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "1" topology.kubernetes.io/zone: "1" - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "2" topology.kubernetes.io/zone: "2" - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "2" topology.kubernetes.io/zone: "2" - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "3" topology.kubernetes.io/zone: "3" - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "3" @@ -192,7 +192,7 @@ nodes: # node for the `quickstart` cluster replica - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "quickstart" @@ -200,14 +200,14 @@ nodes: # only envd (nodes will be tainted in the setup) - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true environmentd: true materialize.cloud/availability-zone: "3" topology.kubernetes.io/zone: "3" - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true environmentd: true @@ -216,7 +216,7 @@ nodes: # for supporting services - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: supporting-services: true materialize.cloud/availability-zone: "3" diff --git a/misc/kind/cluster.yaml b/misc/kind/cluster.yaml index bb6a0cada8bbe..a991a18a33fe3 100644 --- a/misc/kind/cluster.yaml +++ b/misc/kind/cluster.yaml @@ -22,7 +22,7 @@ kubeadmConfigPatches: "service-node-port-range": "32000-32063" nodes: - role: control-plane - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 extraPortMappings: - containerPort: 32000 hostPort: 32000 @@ -154,21 +154,21 @@ nodes: hostPort: 32063 - role: worker - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/swap: true materialize.cloud/availability-zone: "1" topology.kubernetes.io/zone: "1" - role: worker - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/swap: true materialize.cloud/availability-zone: "2" topology.kubernetes.io/zone: "2" - role: worker - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/swap: true @@ -177,7 +177,7 @@ nodes: # no-disk - role: worker - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: false materialize.cloud/swap: false @@ -186,14 +186,14 @@ nodes: # others - role: worker - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/swap: true materialize.cloud/availability-zone: "3" topology.kubernetes.io/zone: "3" - role: worker - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/swap: true diff --git a/misc/python/materialize/cloudtest/app/materialize_application.py b/misc/python/materialize/cloudtest/app/materialize_application.py index 9122eb354e77e..bfcc5047f8a7b 100644 --- a/misc/python/materialize/cloudtest/app/materialize_application.py +++ b/misc/python/materialize/cloudtest/app/materialize_application.py @@ -152,12 +152,20 @@ def wait_for_sql(self) -> None: wait(condition="condition=Ready", resource="pod/environmentd-0") start = datetime.now() - while datetime.now() - start < timedelta(seconds=300): + while True: try: self.environmentd.sql("SELECT 1") break - except InterfaceError as e: - # Since we crash environmentd, we expect some errors that we swallow. + except (InterfaceError, OSError) as e: + if datetime.now() - start > timedelta(seconds=300): + raise + # Since we crash environmentd, we expect some errors that we + # swallow. pg8000 wraps most connection failures in + # InterfaceError, but raw socket errors from its SSL + # negotiation (e.g. ConnectionResetError when the connection + # is accepted by the port-forwarding proxy and then reset + # because environmentd is not listening yet) leak through + # unwrapped. LOGGER.info(f"SQL interface not ready, {e} while SELECT 1. Waiting...") time.sleep(2) diff --git a/misc/python/materialize/cloudtest/k8s/minio.py b/misc/python/materialize/cloudtest/k8s/minio.py index efc06bb4695c5..6f9da83e00a96 100644 --- a/misc/python/materialize/cloudtest/k8s/minio.py +++ b/misc/python/materialize/cloudtest/k8s/minio.py @@ -7,6 +7,8 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. +import yaml + from materialize import MZ_ROOT from materialize.cloudtest import DEFAULT_K8S_NAMESPACE from materialize.cloudtest.k8s.api.k8s_resource import K8sResource @@ -35,9 +37,8 @@ def create(self) -> None: "true", ) - # the PVC will be created afterwards for yaml_file in [ - "minio-standalone-deployment", + "minio-standalone-pvc", "minio-standalone-service", ]: self.kubectl( @@ -46,22 +47,20 @@ def create(self) -> None: str(MINIO_YAML_DIRECTORY / f"{yaml_file}.yaml"), ) - if self.apply_node_selectors: - self.kubectl( - "patch", - "deployment", - "minio-deployment", - "--type", - "json", - "-p", - '[{"op": "add", "path": "/spec/template/spec/nodeSelector", "value": {"supporting-services": "true"} }]', - ) - - # the PVC needs to be created after patching the deployment + # NOTE: The deployment must carry its final nodeSelector before it is + # created, so it is injected here rather than patched in afterwards. + # The claim's storage class binds with WaitForFirstConsumer, so the + # provisioner pins the volume to whichever node the scheduler picks for + # the first pod that consumes the claim. A pod created without the + # nodeSelector can drive that decision even if it is replaced moments + # later, pinning the volume to a node the final pod may not run on. The + # pod then stays Pending forever, because nothing can satisfy both the + # volume's node affinity and the pod's node selector. self.kubectl( "create", "-f", - str(MINIO_YAML_DIRECTORY / "minio-standalone-pvc.yaml"), + "-", + input=self.deployment_manifest(), ) self.wait( @@ -72,6 +71,17 @@ def create(self) -> None: self.create_buckets(["persist", "copytos3", "copyfroms3"]) + def deployment_manifest(self) -> str: + with open(MINIO_YAML_DIRECTORY / "minio-standalone-deployment.yaml") as f: + deployment = yaml.safe_load(f) + + if self.apply_node_selectors: + deployment["spec"]["template"]["spec"]["nodeSelector"] = { + "supporting-services": "true" + } + + return yaml.dump(deployment) + def create_buckets(self, buckets: list[str]) -> None: cmds = [ f"mc config host add myminio http://minio-service.{self.namespace()}:9000 minio minio123" diff --git a/misc/scratch/provision.bash b/misc/scratch/provision.bash index 8b17b13267dd9..3d062569f0bfc 100644 --- a/misc/scratch/provision.bash +++ b/misc/scratch/provision.bash @@ -79,12 +79,10 @@ rm -rf /tmp/awscli.zip /tmp/aws) & # uv (curl -fsSL https://astral.sh/uv/install.sh | sudo -u ubuntu sh >/dev/null 2>&1) & -# kubectl + kind + k9s. kind and kubectl mirror ci/builder/Dockerfile, which -# deliberately holds them back: newer versions break Cloudtest (minio fails to -# come up on the kind cluster). -(curl -fsSL "https://dl.k8s.io/release/v1.24.3/bin/linux/$ARCH_GO/kubectl" -o /usr/local/bin/kubectl +# kubectl + kind + k9s. kind and kubectl mirror ci/builder/Dockerfile. +(curl -fsSL "https://dl.k8s.io/release/v1.34.10/bin/linux/$ARCH_GO/kubectl" -o /usr/local/bin/kubectl chmod +x /usr/local/bin/kubectl) & -(curl -fsSL "https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-$ARCH_GO" -o /usr/local/bin/kind +(curl -fsSL "https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-$ARCH_GO" -o /usr/local/bin/kind chmod +x /usr/local/bin/kind) & (curl -fsSL "https://github.com/derailed/k9s/releases/download/v0.50.18/k9s_Linux_$ARCH_GO.tar.gz" \ | tar xzf - -C /usr/local/bin k9s diff --git a/test/orchestratord/cluster.yaml.tmpl b/test/orchestratord/cluster.yaml.tmpl index ded66e7011634..97e69103dfdc8 100644 --- a/test/orchestratord/cluster.yaml.tmpl +++ b/test/orchestratord/cluster.yaml.tmpl @@ -30,7 +30,7 @@ kubeadmConfigPatches: "service-node-port-range": "32000-32063" nodes: - role: control-plane - image: kindest/node:v1.32.5 + image: kindest/node:v1.34.8 extraMounts: - containerPath: /var/lib/kubelet/config.json hostPath: "$DOCKER_CONFIG/config.json" @@ -165,7 +165,7 @@ nodes: hostPort: 32063 - role: worker - image: kindest/node:v1.32.5 + image: kindest/node:v1.34.8 labels: materialize.cloud/swap: "true" materialize.cloud/availability-zone: "1" @@ -175,7 +175,7 @@ nodes: - containerPath: /var/lib/kubelet/config.json hostPath: "$DOCKER_CONFIG/config.json" - role: worker - image: kindest/node:v1.32.5 + image: kindest/node:v1.34.8 labels: materialize.cloud/scratch-fs: "true" materialize.cloud/disk: "true" diff --git a/test/orchestratord/mzcompose.py b/test/orchestratord/mzcompose.py index b1eb4266332cc..aa716df8bcc65 100644 --- a/test/orchestratord/mzcompose.py +++ b/test/orchestratord/mzcompose.py @@ -293,8 +293,8 @@ def get_console_app_config(namespace="materialize-environment") -> dict[str, Any def ensure_kind_version() -> None: kind_version = Version.parse(spawn.capture(["kind", "version"]).split(" ")[1][1:]) assert kind_version >= Version.parse( - "0.29.0" - ), f"kind >= v0.29.0 required, while you are on {kind_version}" + "0.32.0" + ), f"kind >= v0.32.0 required, while you are on {kind_version}" def stop_and_remove_container(container_name: str) -> None: From 533cabe97e0f06440c3409fa2a907b8181b93cc1 Mon Sep 17 00:00:00 2001 From: Qindeel Ishtiaq Date: Thu, 6 Aug 2026 10:31:16 -0400 Subject: [PATCH 4/6] workload-replay: add console-scalability nightly workflow (#38048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a nightly CI step that measures console performance against replayed production workloads. For each captured workload (sharded across 3 parallel jobs, `--skip-large`, 1% initial data), the `console-scalability` workflow: 1. Boots the workload and waits for hydration. 2. Keeps the captured queries and ingestions running as background load. 3. Runs the Playwright scalability suite (page loads, per-query timing audits, multi-tab stress) against the console bundled in the materialized image. 4. Replays the freshly dumped console query set with k6, ramping 0→100 virtual users against the SQL HTTP endpoint. 5. Uploads per-workload timing reports, query dumps, and k6 summaries as build artifacts. The runs are metrics-only while thresholds are determined from nightly trends: test failures and k6 threshold breaches are logged but do not fail the step. A red step means infrastructure breakage. Supporting changes: * The workload-replay executor's `test()` gains an optional `during_continuous` callback, so the continuous load phase can last exactly as long as the tests. * mzcompose allows the `ipc` service config key (Chromium needs more than the 64MB `/dev/shm` docker default). * The k6 script's labels are refreshed for the current console query set (`clusters.list` and `replicaUtilizationHistory` are SUBSCRIBE-fed now and no longer appear as HTTP queries). Validated end-to-end in [nightly build 17658](https://buildkite.com/materialize/nightly/builds/17658) (single-workload version): 13/14 Playwright tests passed under load, k6 completed the full ramp with 0% request failures, and all artifacts uploaded. The two k6 p95 outliers it found (clusterFreshness 3.8s, materializationLag 4.9s at 100 VUs) match the known production bottleneck ranking. ### Motivation Gives us recurring visibility into console performance under load. Part of [CNS-122](https://linear.app/materializeinc/issue/CNS-122/add-scalable-testing-in-the-nightly-builds). 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Fable 5 --- ci/nightly/pipeline.template.yml | 19 ++ console/.gitignore | 3 +- console/e2e-tests/k6/cluster-detail.ts | 16 +- .../e2e-tests/scalability/page-load.spec.ts | 4 +- misc/python/materialize/mzcompose/service.py | 3 + .../materialize/workload_replay/executor.py | 15 +- test/workload-replay/mzcompose.py | 263 +++++++++++++++++- 7 files changed, 311 insertions(+), 12 deletions(-) diff --git a/ci/nightly/pipeline.template.yml b/ci/nightly/pipeline.template.yml index a65b8e902e864..b1111ba2231ba 100644 --- a/ci/nightly/pipeline.template.yml +++ b/ci/nightly/pipeline.template.yml @@ -311,6 +311,25 @@ steps: agents: queue: hetzner-x86-64-dedi-8cpu-32gb + - id: console-scalability + topics: [iceberg, kafka, mysql, postgres, sql-server, ssh-tunnel] + label: "Console scalability (workload replay)" + depends_on: build-x86_64 + timeout_in_minutes: 120 + parallelism: 3 + artifact_paths: + - console/test-results/**/* + - console/e2e-tests/k6/queries-*.json + - console/e2e-tests/k6/summary-*.json + plugins: + - ./ci/plugins/mzcompose: + composition: workload-replay + run: console-scalability + args: [--skip-large] + agents: + # Larger instance is more stable in performance + queue: hetzner-x86-64-dedi-16cpu-64gb + - group: Kafka key: kafka steps: diff --git a/console/.gitignore b/console/.gitignore index afaff1403c5da..98dce77a4328a 100644 --- a/console/.gitignore +++ b/console/.gitignore @@ -12,7 +12,8 @@ yarn-error.log # Playwrite E2E tests /e2e-tests/state*.json -/e2e-tests/k6/queries.json +/e2e-tests/k6/queries*.json +/e2e-tests/k6/summary*.json /test-results # Vite diff --git a/console/e2e-tests/k6/cluster-detail.ts b/console/e2e-tests/k6/cluster-detail.ts index dd681dfa0ad69..455aeb781c72e 100644 --- a/console/e2e-tests/k6/cluster-detail.ts +++ b/console/e2e-tests/k6/cluster-detail.ts @@ -67,9 +67,10 @@ const POLL_INTERVAL_MS: Record = { healthCheck: 30_000, "clusters.largestClusterReplica": 60_000, "clusters.largestMaintainedQueries": 60_000, - "clusters.owners": 300_000, + // Cached name lookup, refetched only when the largest-queries set changes. + "clusters.maintainedObjectNames": Infinity, "clusters.replicaOfflineEvents": 20_000, - "clusters.replicaUtilizationHistory": 20_000, + "roles.owners": 300_000, // No refetchInterval in the console: fetched on mount and window refocus // only, so replay them once per VU rather than at the 5s default. "clusters.clusterFreshness": Infinity, @@ -116,15 +117,18 @@ export const options = { gracefulRampDown: "30s", }, }, + // clusters.list and clusters.replicaUtilizationHistory carry no + // thresholds: both are SUBSCRIBE-fed in the console now, so they never + // appear in queries.json and an empty metric would render as a vacuous + // pass. thresholds: { http_req_failed: ["rate<0.01"], "http_req_duration{label:healthCheck}": ["p(95)<500"], - "http_req_duration{label:clusters.list}": ["p(95)<2000"], "http_req_duration{label:clusters.materializationLag}": ["p(95)<2000"], "http_req_duration{label:clusters.clusterFreshness}": ["p(95)<2000"], - "http_req_duration{label:clusters.replicaUtilizationHistory}": [ - "p(95)<2000", - ], + "http_req_duration{label:clusters.maintainedObjectNames}": ["p(95)<2000"], + "http_req_duration{label:clusters.replicaOfflineEvents}": ["p(95)<2000"], + "http_req_duration{label:roles.owners}": ["p(95)<2000"], "http_req_duration{label:clusters.largestClusterReplica}": ["p(95)<5000"], "http_req_duration{label:clusters.largestMaintainedQueries}": [ "p(95)<5000", diff --git a/console/e2e-tests/scalability/page-load.spec.ts b/console/e2e-tests/scalability/page-load.spec.ts index 4540a829113e7..14a372b059a64 100644 --- a/console/e2e-tests/scalability/page-load.spec.ts +++ b/console/e2e-tests/scalability/page-load.spec.ts @@ -30,10 +30,12 @@ import { sourcesListUrl, } from "./helpers"; +// Generous ceiling: this suite measures load times rather than asserting +// them, and 30s renders were observed under CI workload-replay load. async function measureUntilVisible( page: Page, selectors: string | string[], - timeoutMs = 30_000, + timeoutMs = 120_000, ): Promise { const start = performance.now(); const selectorList = Array.isArray(selectors) ? selectors : [selectors]; diff --git a/misc/python/materialize/mzcompose/service.py b/misc/python/materialize/mzcompose/service.py index 56aa5a4dea75a..d6c1b1d3f8ca4 100644 --- a/misc/python/materialize/mzcompose/service.py +++ b/misc/python/materialize/mzcompose/service.py @@ -115,6 +115,9 @@ class ServiceConfig(TypedDict, total=False): init: bool """Whether to run an init process in the container.""" + ipc: str + """The IPC isolation mode for the container, e.g. `host`.""" + ports: Sequence[int | str] """Service ports to expose to the host.""" diff --git a/misc/python/materialize/workload_replay/executor.py b/misc/python/materialize/workload_replay/executor.py index c28f05f0f6507..c5fe4d8075c07 100644 --- a/misc/python/materialize/workload_replay/executor.py +++ b/misc/python/materialize/workload_replay/executor.py @@ -19,6 +19,7 @@ import threading import time from collections import defaultdict +from collections.abc import Callable from typing import Any from materialize.docker import image_registry @@ -93,8 +94,13 @@ def test( run_ingestions: bool, run_queries: bool, max_concurrent_queries: int, + during_continuous: Callable[[], None] | None = None, ) -> dict[str, Any]: - """Run a single workload test.""" + """Run a single workload test. + + When `during_continuous` is set, the continuous phase runs until the + callback returns instead of for `runtime` seconds. + """ print(f"--- {posixpath.relpath(file, LOCATION)}") services = set() @@ -272,12 +278,17 @@ def test( thread.start() try: - stop_event.wait(timeout=runtime) + if during_continuous is not None: + during_continuous() + else: + stop_event.wait(timeout=runtime) finally: stop_event.set() for thread in threads: thread.join() print_replay_stats(stats) + elif during_continuous is not None: + during_continuous() else: print("No continuous ingestions or queries defined, skipping phase") diff --git a/test/workload-replay/mzcompose.py b/test/workload-replay/mzcompose.py index a6377495e83b1..23dd1adc067b5 100644 --- a/test/workload-replay/mzcompose.py +++ b/test/workload-replay/mzcompose.py @@ -12,14 +12,20 @@ """ import argparse +import os import pathlib import random import time import yaml -from materialize import buildkite -from materialize.mzcompose.composition import Composition, WorkflowArgumentParser +from materialize import MZ_ROOT, buildkite +from materialize.mzcompose.composition import ( + Composition, + Service, + WorkflowArgumentParser, +) +from materialize.mzcompose.service import Service as ServiceDefinition from materialize.mzcompose.services.azurite import Azurite from materialize.mzcompose.services.kafka import Kafka from materialize.mzcompose.services.materialized import Materialized @@ -87,8 +93,40 @@ ], ), SqlServer(), + # Runs the console Playwright scalability suite against the console + # bundled in the materialized image. The image tag must match the + # @playwright/test version in console/package.json so the preinstalled + # browsers are compatible. + ServiceDefinition( + name="console-scalability-runner", + config={ + "image": "mcr.microsoft.com/playwright:v1.61.0-noble", + "volumes": [f"{MZ_ROOT}:/workdir"], + "working_dir": "/workdir/console", + "init": True, + # Chromium crashes with the 64MB docker default for /dev/shm. + "ipc": "host", + }, + ), + # Replays the console's captured /api/sql traffic against the SQL HTTP + # endpoint at increasing virtual-user counts. Reads the queries.json + # that dump-queries produces during the Playwright suite. + ServiceDefinition( + name="console-k6", + config={ + "image": "grafana/k6:1.6.0", + "volumes": [f"{MZ_ROOT}:/workdir"], + "working_dir": "/workdir/console/e2e-tests/k6", + # The summary export is written into the bind mount. + "propagate_uid_gid": True, + }, + ), ] +# UID/GID of the host user, used to run yarn commands so they don't create +# root-owned files in the bind-mounted volume. +_HOST_UID_GID = f"{os.getuid()}:{os.getgid()}" + def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: parser.add_argument( @@ -338,6 +376,227 @@ def workflow_benchmark(c: Composition, parser: WorkflowArgumentParser) -> None: ) +def workflow_console_scalability( + c: Composition, parser: WorkflowArgumentParser +) -> None: + """ + Run the console Playwright scalability suite against each captured + workload. + + Per workload: boot (objects, initial data, hydration), then run the + suite against the console bundled in the materialized image while + continuous ingestions and queries provide background load. The suite + only reports timings. Suite failures do not fail the workflow, the runs + exist to collect data for determining thresholds. + """ + parser.add_argument( + "--factor-initial-data", + type=float, + default=0.01, + help="scale factor for initial data generation", + ) + parser.add_argument( + "--factor-ingestions", + type=float, + default=1, + help="scale factor for runtime data ingestion rate", + ) + parser.add_argument( + "--factor-queries", + type=float, + default=1, + help="scale factor for runtime queries", + ) + parser.add_argument( + "--max-concurrent-queries", + type=int, + default=1000, + help="max. number of concurrent queries during continuous phase", + ) + parser.add_argument( + "--seed", + metavar="SEED", + type=str, + default=str(int(time.time())), + help="random seed", + ) + parser.add_argument("--verbose", action=argparse.BooleanOptionalAction) + parser.add_argument( + "--skip-large", + action="store_true", + help="skip workloads tagged `settings.large: true`", + ) + parser.add_argument( + "files", + nargs="*", + default=["*.yml"], + help="run against the specified files", + ) + args = parser.parse_args() + + print(f"-- Random seed is {args.seed}") + random.seed(args.seed) + update_captured_workloads_repo() + + all_paths = get_paths(args.files) + workloads: dict[pathlib.Path, dict] = {} + for path in all_paths: + workload = load_workload(path) + if args.skip_large and workload.get("settings", {}).get("large", False): + print(f"-- Skipping {path} (settings.large: true and --skip-large)") + continue + workloads[path] = workload + files: list[pathlib.Path] = buildkite.shard_list( + list(workloads.keys()), + lambda file: str(file), + ) + + k6_dir = MZ_ROOT / "console" / "e2e-tests" / "k6" + + def run(file: pathlib.Path) -> None: + workload = workloads[file] + settings = workload.get("settings", {}) + # When scale_data is false, use 100% initial data, like + # workflow_default. + factor_initial_data = args.factor_initial_data + if not settings.get("scale_data", True): + factor_initial_data = 1.0 + else: + factor_initial_data *= settings.get("factor_initial_data_multiplier", 1.0) + print_workload_stats(file, workload) + + # Reset any state left over from the previous workload. + service_names = [s.name for s in SERVICES] + try: + c.kill(*service_names) + except Exception: + pass + c.rm(*service_names, destroy_volumes=True) + c.rm_volumes("mzdata", force=True) + + # Prepare the Playwright runner up front so dependency installation + # does not eat into the measurement window. Cheap after the first + # workload, node_modules lives in the bind-mounted checkout. + c.up(Service("console-scalability-runner", idle=True)) + c.exec( + "console-scalability-runner", + "sh", + "-c", + "corepack enable", + env_extra={"COREPACK_ENABLE_DOWNLOAD_PROMPT": "0"}, + ) + _console_runner_sh( + c, "yarn install --immutable --network-timeout 30000", user=_HOST_UID_GID + ) + # Fetches matching browsers in case @playwright/test drifted from the + # image tag. Runs as root because it writes to /ms-playwright. + _console_runner_sh(c, "node_modules/.bin/playwright install chromium") + + # k6 must only ever replay queries captured from this workload's run, + # so remove a stale dump. + queries_json = k6_dir / "queries.json" + queries_json.unlink(missing_ok=True) + + def run_suite() -> None: + print(f"+++ Running console scalability suite ({file.stem})") + try: + _console_runner_sh( + c, + "yarn test:e2e:scalability --workers=1 --trace=off " + f"--output=test-results/{file.stem}", + user=_HOST_UID_GID, + env={"BASE_URL": "http://materialized:6874"}, + ) + except Exception as e: + print( + "Console scalability suite failed. Not failing the workflow, " + f"the suite only collects timing data: {e}" + ) + # k6 replays the query set that dump-queries captured above (it + # runs first in the suite), isolating server-side concurrency + # from browser behavior. + if queries_json.exists(): + print(f"+++ Running console k6 load test ({file.stem})") + try: + c.run( + "console-k6", + "run", + "--summary-export=summary.json", + "cluster-detail.ts", + rm=True, + env_extra={"MZ_HTTP_URL": "http://materialized:6876"}, + ) + except Exception as e: + print( + "Console k6 load test failed. Not failing the workflow, " + f"it only collects timing data: {e}" + ) + else: + print( + "dump-queries produced no queries.json, skipping the k6 load test" + ) + # Keep per-workload copies, the next workload overwrites the + # originals. + for name in ("queries", "summary"): + src = k6_dir / f"{name}.json" + if src.exists(): + src.rename(k6_dir / f"{name}-{file.stem}.json") + + try: + test( + c, + workload, + file, + factor_initial_data, + args.factor_ingestions, + args.factor_queries, + 0, # runtime is unused, the suite governs the continuous phase + args.verbose, + True, # create_objects + True, # initial_data + True, # early_initial_data + True, # run_ingestions + True, # run_queries + args.max_concurrent_queries, + during_continuous=run_suite, + ) + finally: + # The runner idles on sleep infinity, don't let it outlive the + # workflow into CI teardown. + try: + c.kill("console-scalability-runner") + c.rm("console-scalability-runner") + except Exception: + pass + + c.test_parts(files, run) + + +def _console_runner_sh( + c: Composition, + command: str, + user: str | None = None, + env: dict[str, str] | None = None, +) -> None: + """Run a shell command in the console-scalability-runner service.""" + full_env = { + "COREPACK_ENABLE_DOWNLOAD_PROMPT": "0", + # yarn needs a writable HOME when running as the host user. + "HOME": "/tmp", + **(env or {}), + } + c.invoke( + "exec", + *(["--user", user] if user else []), + *(f"-e{k}={v}" for k, v in full_env.items()), + "-T", + "console-scalability-runner", + "sh", + "-c", + command, + ) + + def workflow_stats(c: Composition, parser: WorkflowArgumentParser) -> None: with c.override(Materialized(sanity_restart=False)): parser.add_argument( From d92a868ee96e926230729b088a4694b671f5a3e9 Mon Sep 17 00:00:00 2001 From: Sang Jun Bak Date: Thu, 6 Aug 2026 10:56:10 -0400 Subject: [PATCH 5/6] CS-681: Track build version in segment (#37969) - Adds build version to segment - Extracts json building function such that we can test ### Motivation Followup to retro for incident-1165 --------- Co-authored-by: Claude Fable 5 --- src/environmentd/src/lib.rs | 2 + src/environmentd/src/telemetry.rs | 168 +++++++++++++++++++++++++----- 2 files changed, 144 insertions(+), 26 deletions(-) diff --git a/src/environmentd/src/lib.rs b/src/environmentd/src/lib.rs index bef2349b25938..3d5cb5b85c354 100644 --- a/src/environmentd/src/lib.rs +++ b/src/environmentd/src/lib.rs @@ -844,6 +844,7 @@ impl Listeners { adapter_client: adapter_client.clone(), environment_id: config.environment_id, license_key: license_key.clone(), + helm_chart_version: config.helm_chart_version.clone(), report_interval: Duration::from_secs(3600), }); } else if config.test_only_dummy_segment_client { @@ -859,6 +860,7 @@ impl Listeners { adapter_client: adapter_client.clone(), environment_id: config.environment_id, license_key: license_key.clone(), + helm_chart_version: config.helm_chart_version.clone(), report_interval: Duration::from_secs(180), }); } diff --git a/src/environmentd/src/telemetry.rs b/src/environmentd/src/telemetry.rs index 6acdd38a309ec..5740a2cba82b2 100644 --- a/src/environmentd/src/telemetry.rs +++ b/src/environmentd/src/telemetry.rs @@ -82,6 +82,8 @@ use chrono::Utc; use futures::StreamExt; use mz_adapter::PeekResponseUnary; use mz_adapter::telemetry::{EventDetails, SegmentClientExt}; +use mz_build_info::BuildInfo; +use mz_license_keys::ValidatedLicenseKey; use mz_ore::collections::CollectionExt; use mz_ore::retry::Retry; use mz_ore::{soft_panic_or_log, task}; @@ -90,6 +92,8 @@ use mz_sql::catalog::EnvironmentId; use serde_json::json; use tokio::time::{self, Duration}; +use crate::BUILD_INFO; + /// Telemetry configuration. #[derive(Clone)] pub struct Config { @@ -102,7 +106,10 @@ pub struct Config { /// The validated license key for this environment. Reported so downstream /// analytics can tell which entitlements are active and whether the key has /// expired. - pub license_key: mz_license_keys::ValidatedLicenseKey, + pub license_key: ValidatedLicenseKey, + /// The version of the helm chart that deployed this environment, if + /// self-managed. Folded into the reported `mz_version` trait. + pub helm_chart_version: Option, /// How frequently to send a summary to Segment. pub report_interval: Duration, } @@ -118,6 +125,7 @@ async fn report_loop( adapter_client, environment_id, license_key, + helm_chart_version, report_interval, }: Config, ) { @@ -212,32 +220,12 @@ async fn report_loop( } }; - // Merge in the license key's organization and environment IDs, plus its - // current state. The license key is the authoritative source for these - // IDs. `EnvironmentId::organization_id` only matches the real - // organization in cloud SaaS, so we report the license key's `sub` - // (organization) and `aud` (environment) instead. - // - // Expiry is recomputed against the wall clock each interval rather than - // reusing the flag computed once at startup, so a key that lapses while - // environmentd keeps running is reported as expired. The - // `expiration == 0` guard exempts the sentinel disabled/emulator key, - // whose zero expiration is not a real timestamp. - let now_secs = u64::try_from(Utc::now().timestamp()).unwrap_or(0); - let license_expired = license_key.expired - || (license_key.expiration != 0 && now_secs >= license_key.expiration); if let Some(traits) = traits.as_object_mut() { - traits.insert("organization_id".into(), json!(license_key.organization)); - traits.insert("environment_id".into(), json!(license_key.environment_id)); - traits.insert("license_key_id".into(), json!(license_key.id)); - traits.insert( - "license_expiration_timestamp".into(), - json!(license_key.expiration), - ); - traits.insert("license_expired".into(), json!(license_expired)); - traits.insert( - "license_expiration_behavior".into(), - json!(license_key.expiration_behavior), + build_segment_traits( + traits, + &license_key, + &BUILD_INFO, + helm_chart_version.clone(), ); } @@ -285,3 +273,131 @@ async fn report_loop( last_stats = Some(current_stats); } } + +/// Merges the build version and the license key's identity and state into the +/// collected traits. +/// +/// The license key is the authoritative source for the organization and +/// environment IDs. `EnvironmentId::organization_id` only matches the real +/// organization in cloud SaaS, so we report the license key's `sub` +/// (organization) and `aud` (environment) instead. +/// +/// Expiry is recomputed against the wall clock on every call rather than +/// reusing the flag computed once at startup, so a key that lapses while +/// environmentd keeps running is reported as expired. The `expiration == 0` +/// guard exempts the sentinel disabled/emulator key, whose zero expiration is +/// not a real timestamp. +fn build_segment_traits( + traits: &mut serde_json::Map, + license_key: &ValidatedLicenseKey, + build_info: &BuildInfo, + helm_chart_version: Option, +) { + let now_secs = u64::try_from(Utc::now().timestamp()).unwrap_or(0); + let license_expired = + license_key.expired || (license_key.expiration != 0 && now_secs >= license_key.expiration); + traits.insert("organization_id".into(), json!(license_key.organization)); + traits.insert("environment_id".into(), json!(license_key.environment_id)); + traits.insert("license_key_id".into(), json!(license_key.id)); + traits.insert( + "license_expiration_timestamp".into(), + json!(license_key.expiration), + ); + traits.insert("license_expired".into(), json!(license_expired)); + traits.insert( + "license_expiration_behavior".into(), + json!(license_key.expiration_behavior), + ); + traits.insert( + "mz_version".into(), + json!(build_info.human_version(helm_chart_version)), + ); +} + +#[cfg(test)] +mod tests { + use mz_build_info::DUMMY_BUILD_INFO; + use mz_license_keys::ValidatedLicenseKey; + use serde_json::json; + + use super::build_segment_traits; + + fn license_key(expiration: u64, expired: bool) -> ValidatedLicenseKey { + ValidatedLicenseKey { + id: "test-license-key-id".into(), + organization: "test-organization-id".into(), + environment_id: "test-environment-id".into(), + expiration, + expired, + ..ValidatedLicenseKey::for_tests() + } + } + + // Expiry is computed against the real wall clock, so these tests use + // expirations in the distant past (1, i.e. just after the Unix epoch) or + // distant future (`u64::MAX`) to stay deterministic. + + #[mz_ore::test] + fn reports_license_and_version_traits() { + let mut traits = json!({"active_clusters": 1}); + build_segment_traits( + traits.as_object_mut().unwrap(), + &license_key(u64::MAX, false), + &DUMMY_BUILD_INFO, + Some("25.1.0".into()), + ); + assert_eq!( + traits, + json!({ + "active_clusters": 1, + "organization_id": "test-organization-id", + "environment_id": "test-environment-id", + "license_key_id": "test-license-key-id", + "license_expiration_timestamp": u64::MAX, + "license_expired": false, + "license_expiration_behavior": "Warn", + "mz_version": DUMMY_BUILD_INFO.human_version(Some("25.1.0".into())) + }) + ); + } + + #[mz_ore::test] + fn expiry_is_recomputed_from_wall_clock() { + // The key was valid at startup (`expired: false`) but the clock has + // since passed its expiration. + let mut traits = json!({}); + build_segment_traits( + traits.as_object_mut().unwrap(), + &license_key(1, false), + &DUMMY_BUILD_INFO, + None, + ); + assert_eq!(&traits["license_expired"], &json!(true)); + } + + #[mz_ore::test] + fn startup_expired_flag_is_preserved() { + // A key marked expired at startup stays expired even if the clock + // reads before its expiration. + let mut traits = json!({}); + build_segment_traits( + traits.as_object_mut().unwrap(), + &license_key(u64::MAX, true), + &DUMMY_BUILD_INFO, + None, + ); + assert_eq!(&traits["license_expired"], &json!(true)); + } + + #[mz_ore::test] + fn zero_expiration_sentinel_never_expires() { + let mut traits = json!({}); + build_segment_traits( + traits.as_object_mut().unwrap(), + &license_key(0, false), + &DUMMY_BUILD_INFO, + None, + ); + assert_eq!(&traits["license_expired"], &json!(false)); + } +} From 486844218349ab01181cd5e620504521d698985f Mon Sep 17 00:00:00 2001 From: Justin Bradfield Date: Thu, 6 Aug 2026 11:09:23 -0500 Subject: [PATCH 6/6] CLO-143: fix network policy name resolution for quoted identifiers (#38027) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Fix `plan_alter_network_policy` to use `normalize::ident_ref()` instead of `name.to_string()` - Fix `fold_network_policy_name` to extract raw identifier string from `RawNetworkPolicyName` - Fix `resolve_network_policy` helper (used by ALTER OWNER) with the same pattern ## Test plan - Added regression tests for: - `ALTER NETWORK POLICY "hyphenated-name"` - `GRANT/REVOKE USAGE ON NETWORK POLICY "hyphenated-name"` - `ALTER NETWORK POLICY "hyphenated-name" OWNER TO role` Fixes https://linear.app/materializeinc/issue/CLO-143/alter-network-policy-cannot-resolve-quoted-hyphenated-name 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.5 --- src/sql/src/names.rs | 6 ++- src/sql/src/plan/statement/ddl.rs | 9 +++- test/sqllogictest/network_policy.slt | 72 ++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/src/sql/src/names.rs b/src/sql/src/names.rs index 3d9bddbeb3aaa..dc2f3ade3caf3 100644 --- a/src/sql/src/names.rs +++ b/src/sql/src/names.rs @@ -2104,7 +2104,11 @@ impl<'a> Fold for NameResolver<'a> { &mut self, name: ::NetworkPolicyName, ) -> ::NetworkPolicyName { - match self.catalog.resolve_network_policy(&name.to_string()) { + let name_str = match &name { + RawNetworkPolicyName::Unresolved(ident) => ident.as_str(), + RawNetworkPolicyName::Resolved(s) => s.as_str(), + }; + match self.catalog.resolve_network_policy(name_str) { Ok(policy) => ResolvedNetworkPolicyName { id: policy.id(), name: policy.name().to_string(), diff --git a/src/sql/src/plan/statement/ddl.rs b/src/sql/src/plan/statement/ddl.rs index 19f7073711fdc..cbfbd9de9c976 100644 --- a/src/sql/src/plan/statement/ddl.rs +++ b/src/sql/src/plan/statement/ddl.rs @@ -4778,7 +4778,9 @@ pub fn plan_alter_network_policy( ctx.require_feature_flag(&vars::ENABLE_NETWORK_POLICIES)?; let policy_options: NetworkPolicyOptionExtracted = options.try_into()?; - let policy = ctx.catalog.resolve_network_policy(&name.to_string())?; + let policy = ctx + .catalog + .resolve_network_policy(normalize::ident_ref(&name))?; let Some(rule_defs) = policy_options.rules else { sql_bail!("RULES must be specified when creating network policies."); @@ -8345,7 +8347,10 @@ pub(crate) fn resolve_network_policy<'a>( name: Ident, if_exists: bool, ) -> Result, PlanError> { - match scx.catalog.resolve_network_policy(&name.to_string()) { + match scx + .catalog + .resolve_network_policy(normalize::ident_ref(&name)) + { Ok(policy) => Ok(Some(ResolvedNetworkPolicyName { id: policy.id(), name: policy.name().to_string(), diff --git a/test/sqllogictest/network_policy.slt b/test/sqllogictest/network_policy.slt index 07a4550dbb3b6..a3b2daeb7d246 100644 --- a/test/sqllogictest/network_policy.slt +++ b/test/sqllogictest/network_policy.slt @@ -271,3 +271,75 @@ simple conn=mz_system,user=mz_system DROP ROLE np_owner ---- COMPLETE 0 + +# Regression test: ALTER NETWORK POLICY must correctly resolve quoted +# (hyphenated) names. Previously, plan_alter_network_policy passed the +# display-quoted form (via Ident::to_string) to resolve_network_policy, +# causing "np-x" to be looked up as "\"np-x\"" and fail. + +simple conn=mz_system,user=mz_system +CREATE NETWORK POLICY "np-x" (RULES (r0 (action='allow', direction='ingress', address='0.0.0.0/0'))) +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +ALTER NETWORK POLICY "np-x" SET (RULES (r1 (action='allow', direction='ingress', address='1.1.1.1/32'))) +---- +COMPLETE 0 + +query TTTT colnames +SELECT name, action, address, direction FROM mz_internal.mz_network_policy_rules WHERE policy_id = (SELECT id FROM mz_internal.mz_network_policies WHERE name = 'np-x') +---- +name action address direction +r1 +allow +1.1.1.1/32 +ingress + +simple conn=mz_system,user=mz_system +DROP NETWORK POLICY "np-x" +---- +COMPLETE 0 + +# Regression test: GRANT/REVOKE on a hyphenated network policy name. The +# fold_network_policy_name function had the same bug with quoted identifiers. + +simple conn=mz_system,user=mz_system +CREATE ROLE np_hyphen_grantee +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +CREATE NETWORK POLICY "np-hyphen" (RULES (r0 (action='allow', direction='ingress', address='0.0.0.0/0'))) +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +GRANT USAGE ON NETWORK POLICY "np-hyphen" TO np_hyphen_grantee +---- +COMPLETE 0 + +query TT rowsort +SELECT np.name, mz_internal.mz_aclitem_privileges(priv) +FROM mz_internal.mz_network_policies np, + unnest(np.privileges) AS priv, + mz_roles r +WHERE mz_internal.mz_aclitem_grantee(priv) = r.id AND r.name = 'np_hyphen_grantee' +---- +np-hyphen +U + +simple conn=mz_system,user=mz_system +REVOKE USAGE ON NETWORK POLICY "np-hyphen" FROM np_hyphen_grantee +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +DROP NETWORK POLICY "np-hyphen" +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +DROP ROLE np_hyphen_grantee +---- +COMPLETE 0