[pull] main from MaterializeInc:main

Cargo.toml

@@ -399,7 +399,7 @@ jemalloc_pprof = "0.8.2"
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
 junit-report = "0.8.3"
 k8s-controller = "0.12.0"
-k8s-openapi = { version = "0.27.0", features = ["schemars", "v1_32"] }
+k8s-openapi = { version = "0.27.0", features = ["schemars", "v1_34"] }
 kube = { version = "3.1.0", default-features = false, features = ["client", "derive", "openssl-tls", "runtime", "ws"] }
 launchdarkly-server-sdk = { version = "3.1.1", default-features = false, features = ["hyper-rustls-native-roots", "crypto-aws-lc-rs"] }
 launchdarkly-sdk-transport = "0.1.4"

ci/builder/Dockerfile

@@ -325,16 +325,15 @@ RUN if [ $ARCH_GCC = x86_64 ]; then \
 
 # Install KinD, kubectl, helm & helm-docs
 
-# TODO(def-) Upgrading kind/kubectl seems to cause Cloudtest failures
-RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \
+RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \
     && chmod +x /usr/local/bin/kind \
-    && if [ $ARCH_GO = amd64 ]; then echo 'c72eda46430f065fb45c5f70e7c957cc9209402ef309294821978677c8fb3284 /usr/local/bin/kind' | sha256sum --check; fi \
-    && if [ $ARCH_GO = arm64 ]; then echo '03d45095dbd9cc1689f179a3e5e5da24b77c2d1b257d7645abf1b4174bebcf2a /usr/local/bin/kind' | sha256sum --check; fi
+    && if [ $ARCH_GO = amd64 ]; then echo '50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 /usr/local/bin/kind' | sha256sum --check; fi \
+    && if [ $ARCH_GO = arm64 ]; then echo 'b92cd615e97585de8ddade28ed5cd7feb4248d717c233eea5b03c37298900f5d /usr/local/bin/kind' | sha256sum --check; fi
 
-RUN curl -fsSL https://dl.k8s.io/release/v1.24.3/bin/linux/$ARCH_GO/kubectl > /usr/local/bin/kubectl \
+RUN curl -fsSL https://dl.k8s.io/release/v1.34.10/bin/linux/$ARCH_GO/kubectl > /usr/local/bin/kubectl \
     && chmod +x /usr/local/bin/kubectl \
-    && if [ $ARCH_GO = amd64 ]; then echo '8a45348bdaf81d46caf1706c8bf95b3f431150554f47d444ffde89e8cdd712c1 /usr/local/bin/kubectl' | sha256sum --check; fi \
-    && if [ $ARCH_GO = arm64 ]; then echo 'bdad4d3063ddb7bfa5ecf17fb8b029d5d81d7d4ea1650e4369aafa13ed97149a /usr/local/bin/kubectl' | sha256sum --check; fi
+    && if [ $ARCH_GO = amd64 ]; then echo '95bd70842bd11a524d24acd5b68726899e3488e153e45e2b4ae846545beda050 /usr/local/bin/kubectl' | sha256sum --check; fi \
+    && if [ $ARCH_GO = arm64 ]; then echo '52d3aeefea32fdfa3671ccd636be5da463ddfd0a2fc09d7bcbaedcff4c76cad5 /usr/local/bin/kubectl' | sha256sum --check; fi
 
 RUN curl -fsSL https://get.helm.sh/helm-v4.0.0-linux-$ARCH_GO.tar.gz > helm.tar.gz \
     && if [ $ARCH_GO = amd64 ]; then echo 'c77e9e7c1cc96e066bd240d190d1beed9a6b08060b2043ef0862c4f865eca08f helm.tar.gz' | sha256sum --check; fi \
@@ -517,16 +516,15 @@ RUN ln -s ../lib/node_modules/corepack/dist/corepack.js /usr/local/bin/corepack
     && ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx \
     && corepack enable
 
-# TODO(def-) Upgrading kind/kubectl seems to cause Cloudtest failures
-RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \
+RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \
     && chmod +x /usr/local/bin/kind \
-    && if [ $ARCH_GO = amd64 ]; then echo 'c72eda46430f065fb45c5f70e7c957cc9209402ef309294821978677c8fb3284 /usr/local/bin/kind' | sha256sum --check; fi \
-    && if [ $ARCH_GO = arm64 ]; then echo '03d45095dbd9cc1689f179a3e5e5da24b77c2d1b257d7645abf1b4174bebcf2a /usr/local/bin/kind' | sha256sum --check; fi
+    && if [ $ARCH_GO = amd64 ]; then echo '50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 /usr/local/bin/kind' | sha256sum --check; fi \
+    && if [ $ARCH_GO = arm64 ]; then echo 'b92cd615e97585de8ddade28ed5cd7feb4248d717c233eea5b03c37298900f5d /usr/local/bin/kind' | sha256sum --check; fi
 
-RUN curl -fsSL https://dl.k8s.io/release/v1.24.3/bin/linux/$ARCH_GO/kubectl > /usr/local/bin/kubectl \
+RUN curl -fsSL https://dl.k8s.io/release/v1.34.10/bin/linux/$ARCH_GO/kubectl > /usr/local/bin/kubectl \
     && chmod +x /usr/local/bin/kubectl \
-    && if [ $ARCH_GO = amd64 ]; then echo '8a45348bdaf81d46caf1706c8bf95b3f431150554f47d444ffde89e8cdd712c1 /usr/local/bin/kubectl' | sha256sum --check; fi \
-    && if [ $ARCH_GO = arm64 ]; then echo 'bdad4d3063ddb7bfa5ecf17fb8b029d5d81d7d4ea1650e4369aafa13ed97149a /usr/local/bin/kubectl' | sha256sum --check; fi
+    && if [ $ARCH_GO = amd64 ]; then echo '95bd70842bd11a524d24acd5b68726899e3488e153e45e2b4ae846545beda050 /usr/local/bin/kubectl' | sha256sum --check; fi \
+    && if [ $ARCH_GO = arm64 ]; then echo '52d3aeefea32fdfa3671ccd636be5da463ddfd0a2fc09d7bcbaedcff4c76cad5 /usr/local/bin/kubectl' | sha256sum --check; fi
 
 # Use Helm 3 (not 4) because the cloud repo's bin/kind-create uses relative
 # file paths for vendored charts, which Helm 4 rejects as invalid URLs.

ci/nightly/pipeline.template.yml

@@ -311,6 +311,25 @@ steps:
         agents:
           queue: hetzner-x86-64-dedi-8cpu-32gb
 
+      - id: console-scalability
+        topics: [iceberg, kafka, mysql, postgres, sql-server, ssh-tunnel]
+        label: "Console scalability (workload replay)"
+        depends_on: build-x86_64
+        timeout_in_minutes: 120
+        parallelism: 3
+        artifact_paths:
+          - console/test-results/**/*
+          - console/e2e-tests/k6/queries-*.json
+          - console/e2e-tests/k6/summary-*.json
+        plugins:
+          - ./ci/plugins/mzcompose:
+              composition: workload-replay
+              run: console-scalability
+              args: [--skip-large]
+        agents:
+          # Larger instance is more stable in performance
+          queue: hetzner-x86-64-dedi-16cpu-64gb
+
   - group: Kafka
     key: kafka
     steps:

ci/release-qualification/pipeline.template.yml

@@ -189,7 +189,7 @@ steps:
       plugins:
         - ./ci/plugins/mzcompose:
             composition: zippy
-            # TODO(def-) Increase number of actions when database-issues#7257 is fixed
+            # TODO(def-) Increase number of actions when SS-422 is fixed
             args: [--scenario=KafkaParallelInsert, --transaction-isolation=serializable, --actions=50000, --max-execution-time=4h]
 
   - group: Benchmarks

console/.gitignore

@@ -12,7 +12,8 @@ yarn-error.log
 
 # Playwrite E2E tests
 /e2e-tests/state*.json
-/e2e-tests/k6/queries.json
+/e2e-tests/k6/queries*.json
+/e2e-tests/k6/summary*.json
 /test-results
 
 # Vite

console/README.md

@@ -67,7 +67,7 @@ Install the [aws-cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-s
 Install some k8s utils:
 
 ```shell
-brew install kubectl@1.24 k9s kind
+brew install kubernetes-cli@1.34 k9s kind
 ```
 
 Run the commands below to configure your aws account and k8s contexts

console/e2e-tests/k6/cluster-detail.ts

@@ -67,9 +67,10 @@ const POLL_INTERVAL_MS: Record<string, number> = {
   healthCheck: 30_000,
   "clusters.largestClusterReplica": 60_000,
   "clusters.largestMaintainedQueries": 60_000,
-  "clusters.owners": 300_000,
+  // Cached name lookup, refetched only when the largest-queries set changes.
+  "clusters.maintainedObjectNames": Infinity,
   "clusters.replicaOfflineEvents": 20_000,
-  "clusters.replicaUtilizationHistory": 20_000,
+  "roles.owners": 300_000,
   // No refetchInterval in the console: fetched on mount and window refocus
   // only, so replay them once per VU rather than at the 5s default.
   "clusters.clusterFreshness": Infinity,
@@ -116,15 +117,18 @@ export const options = {
       gracefulRampDown: "30s",
     },
   },
+  // clusters.list and clusters.replicaUtilizationHistory carry no
+  // thresholds: both are SUBSCRIBE-fed in the console now, so they never
+  // appear in queries.json and an empty metric would render as a vacuous
+  // pass.
   thresholds: {
     http_req_failed: ["rate<0.01"],
     "http_req_duration{label:healthCheck}": ["p(95)<500"],
-    "http_req_duration{label:clusters.list}": ["p(95)<2000"],
     "http_req_duration{label:clusters.materializationLag}": ["p(95)<2000"],
     "http_req_duration{label:clusters.clusterFreshness}": ["p(95)<2000"],
-    "http_req_duration{label:clusters.replicaUtilizationHistory}": [
-      "p(95)<2000",
-    ],
+    "http_req_duration{label:clusters.maintainedObjectNames}": ["p(95)<2000"],
+    "http_req_duration{label:clusters.replicaOfflineEvents}": ["p(95)<2000"],
+    "http_req_duration{label:roles.owners}": ["p(95)<2000"],
     "http_req_duration{label:clusters.largestClusterReplica}": ["p(95)<5000"],
     "http_req_duration{label:clusters.largestMaintainedQueries}": [
       "p(95)<5000",

console/e2e-tests/scalability/page-load.spec.ts

@@ -30,10 +30,12 @@ import {
   sourcesListUrl,
 } from "./helpers";
 
+// Generous ceiling: this suite measures load times rather than asserting
+// them, and 30s renders were observed under CI workload-replay load.
 async function measureUntilVisible(
   page: Page,
   selectors: string | string[],
-  timeoutMs = 30_000,
+  timeoutMs = 120_000,
 ): Promise<number> {
   const start = performance.now();
   const selectorList = Array.isArray(selectors) ? selectors : [selectors];

doc/developer/cloudtest.md

@@ -29,7 +29,7 @@ official [`kubernetes`] Python library to control the Kubernetes cluster.
     On Linux, use:
 
     ```
-    curl -fL https://dl.k8s.io/release/v1.34.5/bin/linux/amd64/kubectl > kubectl
+    curl -fL https://dl.k8s.io/release/v1.34.10/bin/linux/amd64/kubectl > kubectl
     chmod +x kubectl
     sudo mv kubectl /usr/local/bin
     ```
@@ -48,7 +48,7 @@ official [`kubernetes`] Python library to control the Kubernetes cluster.
     On Linux, use:
 
     ```
-    curl -fL https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-amd64 > kind
+    curl -fL https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-amd64 > kind
     chmod +x kind
     sudo mv kind /usr/local/bin
     ```

doc/user/content/reference/system-catalog/mz_catalog.md

@@ -562,7 +562,7 @@ Field            | Type                 | Meaning
 `type`           | [`text`]             | The type of the source: `kafka`, `mysql`, `postgres`, `load-generator`, `progress`, or `subsource`.
 `connection_id`  | [`text`]             | The ID of the connection associated with the source, if any. Corresponds to [`mz_connections.id`](/reference/system-catalog/mz_catalog/#mz_connections).
 `size`           | [`text`]             | *Deprecated* The [size](/sql/create-source/#sizing-a-source) of the source.
-`envelope_type`  | [`text`]             | For Kafka sources, the [envelope](/sql/create-source/kafka/#envelopes) type: `none`, `upsert`, or `debezium`. `NULL` for other source types.
+`envelope_type`  | [`text`]             | For old-syntax Kafka sources, the [envelope](/sql/create-source/kafka/#envelopes) type: `none`, `upsert`, or `debezium`. `NULL` for new-syntax Kafka sources, whose envelopes are defined per source table (see `mz_kafka_source_tables`), and for other source types.
 `key_format`     | [`text`]             | For Kafka sources, the [format](/sql/create-source/kafka#syntax) of the Kafka message key: `avro`, `csv`, `regex`, `bytes`, `json`, `text`, or `NULL`.
 `value_format`     | [`text`]           | For Kafka sources, the [format](/sql/create-source/kafka#syntax) of the Kafka message value: `avro`, `csv`, `regex`, `bytes`, `json`, `text`. `NULL` for other source types.
 `cluster_id`     | [`text`]             | The ID of the cluster maintaining the source. Corresponds to [`mz_clusters.id`](/reference/system-catalog/mz_catalog/#mz_clusters).

doc/user/content/reference/system-catalog/mz_internal.md

@@ -806,7 +806,7 @@ table and the corresponding upstream Kafka topic being ingested.
 | ------------------- | ---------------- | --------                                                                                                       |
 | `id`                | [`text`]         | The ID of the table. Corresponds to [`mz_catalog.mz_tables.id`](../mz_catalog#mz_tables).                   |
 | `topic`             | [`text`]         | The topic being ingested. |
-| `envelope_type`     | [`text`]         | The [envelope](/sql/create-source/kafka/#envelopes) type: `none`, `upsert`, or `debezium`. `NULL` for other source types. |
+| `envelope_type`     | [`text`]         | The [envelope](/sql/create-source/kafka/#envelopes) type: `none`, `upsert`, or `debezium`. Defaults to `none` when the source table omits an explicit envelope. |
 | `key_format`        | [`text`]         | The [format](/sql/create-source/kafka/#syntax) of the Kafka message key: `avro`, `csv`, `regex`, `bytes`, `json`, `text`, or `NULL`. |
 | `value_format`      | [`text`]         | The [format](/sql/create-source/kafka/#syntax) of the Kafka message value: `avro`, `csv`, `regex`, `bytes`, `json`, `text`. `NULL` for other source types. |
 

misc/kind/cluster-node-recovery-test.yaml

@@ -22,7 +22,7 @@ kubeadmConfigPatches:
         "service-node-port-range": "32000-32063"
 nodes:
   - role: control-plane
-    image: kindest/node:v1.33.1
+    image: kindest/node:v1.34.8
     extraPortMappings:
       - containerPort: 32000
         hostPort: 32000
@@ -154,60 +154,60 @@ nodes:
         hostPort: 32063
 
   - role: worker
-    image: kindest/node:v1.33.1
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       materialize.cloud/availability-zone: "1"
       topology.kubernetes.io/zone: "1"
   - role: worker
-    image: kindest/node:v1.33.1
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       materialize.cloud/availability-zone: "1"
       topology.kubernetes.io/zone: "1"
   - role: worker
-    image: kindest/node:v1.33.1
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       materialize.cloud/availability-zone: "2"
       topology.kubernetes.io/zone: "2"
   - role: worker
-    image: kindest/node:v1.33.1
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       materialize.cloud/availability-zone: "2"
       topology.kubernetes.io/zone: "2"
   - role: worker
-    image: kindest/node:v1.33.1
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       materialize.cloud/availability-zone: "3"
       topology.kubernetes.io/zone: "3"
   - role: worker
-    image: kindest/node:v1.33.1
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       materialize.cloud/availability-zone: "3"
       topology.kubernetes.io/zone: "3"
 
   # node for the `quickstart` cluster replica
   - role: worker
-    image: kindest/node:v1.33.1
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       materialize.cloud/availability-zone: "quickstart"
       topology.kubernetes.io/zone: "quickstart"
 
   # only envd (nodes will be tainted in the setup)
   - role: worker
-    image: kindest/node:v1.33.1
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       environmentd: true
       materialize.cloud/availability-zone: "3"
       topology.kubernetes.io/zone: "3"
   - role: worker
-    image: kindest/node:v1.33.1
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       environmentd: true
@@ -216,7 +216,7 @@ nodes:
 
   # for supporting services
   - role: worker
-    image: kindest/node:v1.33.1
+    image: kindest/node:v1.34.8
     labels:
       supporting-services: true
       materialize.cloud/availability-zone: "3"

misc/kind/cluster.yaml

@@ -22,7 +22,7 @@ kubeadmConfigPatches:
         "service-node-port-range": "32000-32063"
 nodes:
   - role: control-plane
-    image: kindest/node:v1.31.6
+    image: kindest/node:v1.34.8
     extraPortMappings:
       - containerPort: 32000
         hostPort: 32000
@@ -154,21 +154,21 @@ nodes:
         hostPort: 32063
 
   - role: worker
-    image: kindest/node:v1.31.6
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       materialize.cloud/swap: true
       materialize.cloud/availability-zone: "1"
       topology.kubernetes.io/zone: "1"
   - role: worker
-    image: kindest/node:v1.31.6
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       materialize.cloud/swap: true
       materialize.cloud/availability-zone: "2"
       topology.kubernetes.io/zone: "2"
   - role: worker
-    image: kindest/node:v1.31.6
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       materialize.cloud/swap: true
@@ -177,7 +177,7 @@ nodes:
 
   # no-disk
   - role: worker
-    image: kindest/node:v1.31.6
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: false
       materialize.cloud/swap: false
@@ -186,14 +186,14 @@ nodes:
 
   # others
   - role: worker
-    image: kindest/node:v1.31.6
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       materialize.cloud/swap: true
       materialize.cloud/availability-zone: "3"
       topology.kubernetes.io/zone: "3"
   - role: worker
-    image: kindest/node:v1.31.6
+    image: kindest/node:v1.34.8
     labels:
       materialize.cloud/disk: true
       materialize.cloud/swap: true

misc/python/materialize/cloudtest/app/materialize_application.py

@@ -152,12 +152,20 @@ def wait_for_sql(self) -> None:
         wait(condition="condition=Ready", resource="pod/environmentd-0")
 
         start = datetime.now()
-        while datetime.now() - start < timedelta(seconds=300):
+        while True:
             try:
                 self.environmentd.sql("SELECT 1")
                 break
-            except InterfaceError as e:
-                # Since we crash environmentd, we expect some errors that we swallow.
+            except (InterfaceError, OSError) as e:
+                if datetime.now() - start > timedelta(seconds=300):
+                    raise
+                # Since we crash environmentd, we expect some errors that we
+                # swallow. pg8000 wraps most connection failures in
+                # InterfaceError, but raw socket errors from its SSL
+                # negotiation (e.g. ConnectionResetError when the connection
+                # is accepted by the port-forwarding proxy and then reset
+                # because environmentd is not listening yet) leak through
+                # unwrapped.
                 LOGGER.info(f"SQL interface not ready, {e} while SELECT 1. Waiting...")
                 time.sleep(2)