diff --git a/Cargo.toml b/Cargo.toml index a581d55e26722..e72662ef8b42b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -399,7 +399,7 @@ jemalloc_pprof = "0.8.2" jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] } junit-report = "0.8.3" k8s-controller = "0.12.0" -k8s-openapi = { version = "0.27.0", features = ["schemars", "v1_32"] } +k8s-openapi = { version = "0.27.0", features = ["schemars", "v1_34"] } kube = { version = "3.1.0", default-features = false, features = ["client", "derive", "openssl-tls", "runtime", "ws"] } launchdarkly-server-sdk = { version = "3.1.1", default-features = false, features = ["hyper-rustls-native-roots", "crypto-aws-lc-rs"] } launchdarkly-sdk-transport = "0.1.4" diff --git a/ci/builder/Dockerfile b/ci/builder/Dockerfile index 1df05693d3d1e..bb61c9e987bd2 100644 --- a/ci/builder/Dockerfile +++ b/ci/builder/Dockerfile @@ -325,16 +325,15 @@ RUN if [ $ARCH_GCC = x86_64 ]; then \ # Install KinD, kubectl, helm & helm-docs -# TODO(def-) Upgrading kind/kubectl seems to cause Cloudtest failures -RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \ +RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \ && chmod +x /usr/local/bin/kind \ - && if [ $ARCH_GO = amd64 ]; then echo 'c72eda46430f065fb45c5f70e7c957cc9209402ef309294821978677c8fb3284 /usr/local/bin/kind' | sha256sum --check; fi \ - && if [ $ARCH_GO = arm64 ]; then echo '03d45095dbd9cc1689f179a3e5e5da24b77c2d1b257d7645abf1b4174bebcf2a /usr/local/bin/kind' | sha256sum --check; fi + && if [ $ARCH_GO = amd64 ]; then echo '50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 /usr/local/bin/kind' | sha256sum --check; fi \ + && if [ $ARCH_GO = arm64 ]; then echo 'b92cd615e97585de8ddade28ed5cd7feb4248d717c233eea5b03c37298900f5d /usr/local/bin/kind' | sha256sum --check; fi -RUN curl -fsSL https://dl.k8s.io/release/v1.24.3/bin/linux/$ARCH_GO/kubectl > /usr/local/bin/kubectl \ +RUN curl -fsSL https://dl.k8s.io/release/v1.34.10/bin/linux/$ARCH_GO/kubectl > /usr/local/bin/kubectl \ && chmod +x /usr/local/bin/kubectl \ - && if [ $ARCH_GO = amd64 ]; then echo '8a45348bdaf81d46caf1706c8bf95b3f431150554f47d444ffde89e8cdd712c1 /usr/local/bin/kubectl' | sha256sum --check; fi \ - && if [ $ARCH_GO = arm64 ]; then echo 'bdad4d3063ddb7bfa5ecf17fb8b029d5d81d7d4ea1650e4369aafa13ed97149a /usr/local/bin/kubectl' | sha256sum --check; fi + && if [ $ARCH_GO = amd64 ]; then echo '95bd70842bd11a524d24acd5b68726899e3488e153e45e2b4ae846545beda050 /usr/local/bin/kubectl' | sha256sum --check; fi \ + && if [ $ARCH_GO = arm64 ]; then echo '52d3aeefea32fdfa3671ccd636be5da463ddfd0a2fc09d7bcbaedcff4c76cad5 /usr/local/bin/kubectl' | sha256sum --check; fi RUN curl -fsSL https://get.helm.sh/helm-v4.0.0-linux-$ARCH_GO.tar.gz > helm.tar.gz \ && if [ $ARCH_GO = amd64 ]; then echo 'c77e9e7c1cc96e066bd240d190d1beed9a6b08060b2043ef0862c4f865eca08f helm.tar.gz' | sha256sum --check; fi \ @@ -517,16 +516,15 @@ RUN ln -s ../lib/node_modules/corepack/dist/corepack.js /usr/local/bin/corepack && ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx \ && corepack enable -# TODO(def-) Upgrading kind/kubectl seems to cause Cloudtest failures -RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \ +RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \ && chmod +x /usr/local/bin/kind \ - && if [ $ARCH_GO = amd64 ]; then echo 'c72eda46430f065fb45c5f70e7c957cc9209402ef309294821978677c8fb3284 /usr/local/bin/kind' | sha256sum --check; fi \ - && if [ $ARCH_GO = arm64 ]; then echo '03d45095dbd9cc1689f179a3e5e5da24b77c2d1b257d7645abf1b4174bebcf2a /usr/local/bin/kind' | sha256sum --check; fi + && if [ $ARCH_GO = amd64 ]; then echo '50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 /usr/local/bin/kind' | sha256sum --check; fi \ + && if [ $ARCH_GO = arm64 ]; then echo 'b92cd615e97585de8ddade28ed5cd7feb4248d717c233eea5b03c37298900f5d /usr/local/bin/kind' | sha256sum --check; fi -RUN curl -fsSL https://dl.k8s.io/release/v1.24.3/bin/linux/$ARCH_GO/kubectl > /usr/local/bin/kubectl \ +RUN curl -fsSL https://dl.k8s.io/release/v1.34.10/bin/linux/$ARCH_GO/kubectl > /usr/local/bin/kubectl \ && chmod +x /usr/local/bin/kubectl \ - && if [ $ARCH_GO = amd64 ]; then echo '8a45348bdaf81d46caf1706c8bf95b3f431150554f47d444ffde89e8cdd712c1 /usr/local/bin/kubectl' | sha256sum --check; fi \ - && if [ $ARCH_GO = arm64 ]; then echo 'bdad4d3063ddb7bfa5ecf17fb8b029d5d81d7d4ea1650e4369aafa13ed97149a /usr/local/bin/kubectl' | sha256sum --check; fi + && if [ $ARCH_GO = amd64 ]; then echo '95bd70842bd11a524d24acd5b68726899e3488e153e45e2b4ae846545beda050 /usr/local/bin/kubectl' | sha256sum --check; fi \ + && if [ $ARCH_GO = arm64 ]; then echo '52d3aeefea32fdfa3671ccd636be5da463ddfd0a2fc09d7bcbaedcff4c76cad5 /usr/local/bin/kubectl' | sha256sum --check; fi # Use Helm 3 (not 4) because the cloud repo's bin/kind-create uses relative # file paths for vendored charts, which Helm 4 rejects as invalid URLs. diff --git a/ci/nightly/pipeline.template.yml b/ci/nightly/pipeline.template.yml index a65b8e902e864..b1111ba2231ba 100644 --- a/ci/nightly/pipeline.template.yml +++ b/ci/nightly/pipeline.template.yml @@ -311,6 +311,25 @@ steps: agents: queue: hetzner-x86-64-dedi-8cpu-32gb + - id: console-scalability + topics: [iceberg, kafka, mysql, postgres, sql-server, ssh-tunnel] + label: "Console scalability (workload replay)" + depends_on: build-x86_64 + timeout_in_minutes: 120 + parallelism: 3 + artifact_paths: + - console/test-results/**/* + - console/e2e-tests/k6/queries-*.json + - console/e2e-tests/k6/summary-*.json + plugins: + - ./ci/plugins/mzcompose: + composition: workload-replay + run: console-scalability + args: [--skip-large] + agents: + # Larger instance is more stable in performance + queue: hetzner-x86-64-dedi-16cpu-64gb + - group: Kafka key: kafka steps: diff --git a/ci/release-qualification/pipeline.template.yml b/ci/release-qualification/pipeline.template.yml index 99960bdd7575c..594ff61162afb 100644 --- a/ci/release-qualification/pipeline.template.yml +++ b/ci/release-qualification/pipeline.template.yml @@ -189,7 +189,7 @@ steps: plugins: - ./ci/plugins/mzcompose: composition: zippy - # TODO(def-) Increase number of actions when database-issues#7257 is fixed + # TODO(def-) Increase number of actions when SS-422 is fixed args: [--scenario=KafkaParallelInsert, --transaction-isolation=serializable, --actions=50000, --max-execution-time=4h] - group: Benchmarks diff --git a/console/.gitignore b/console/.gitignore index afaff1403c5da..98dce77a4328a 100644 --- a/console/.gitignore +++ b/console/.gitignore @@ -12,7 +12,8 @@ yarn-error.log # Playwrite E2E tests /e2e-tests/state*.json -/e2e-tests/k6/queries.json +/e2e-tests/k6/queries*.json +/e2e-tests/k6/summary*.json /test-results # Vite diff --git a/console/README.md b/console/README.md index 2a514e627ad40..e5f00ce227e90 100644 --- a/console/README.md +++ b/console/README.md @@ -67,7 +67,7 @@ Install the [aws-cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-s Install some k8s utils: ```shell -brew install kubectl@1.24 k9s kind +brew install kubernetes-cli@1.34 k9s kind ``` Run the commands below to configure your aws account and k8s contexts diff --git a/console/e2e-tests/k6/cluster-detail.ts b/console/e2e-tests/k6/cluster-detail.ts index dd681dfa0ad69..455aeb781c72e 100644 --- a/console/e2e-tests/k6/cluster-detail.ts +++ b/console/e2e-tests/k6/cluster-detail.ts @@ -67,9 +67,10 @@ const POLL_INTERVAL_MS: Record = { healthCheck: 30_000, "clusters.largestClusterReplica": 60_000, "clusters.largestMaintainedQueries": 60_000, - "clusters.owners": 300_000, + // Cached name lookup, refetched only when the largest-queries set changes. + "clusters.maintainedObjectNames": Infinity, "clusters.replicaOfflineEvents": 20_000, - "clusters.replicaUtilizationHistory": 20_000, + "roles.owners": 300_000, // No refetchInterval in the console: fetched on mount and window refocus // only, so replay them once per VU rather than at the 5s default. "clusters.clusterFreshness": Infinity, @@ -116,15 +117,18 @@ export const options = { gracefulRampDown: "30s", }, }, + // clusters.list and clusters.replicaUtilizationHistory carry no + // thresholds: both are SUBSCRIBE-fed in the console now, so they never + // appear in queries.json and an empty metric would render as a vacuous + // pass. thresholds: { http_req_failed: ["rate<0.01"], "http_req_duration{label:healthCheck}": ["p(95)<500"], - "http_req_duration{label:clusters.list}": ["p(95)<2000"], "http_req_duration{label:clusters.materializationLag}": ["p(95)<2000"], "http_req_duration{label:clusters.clusterFreshness}": ["p(95)<2000"], - "http_req_duration{label:clusters.replicaUtilizationHistory}": [ - "p(95)<2000", - ], + "http_req_duration{label:clusters.maintainedObjectNames}": ["p(95)<2000"], + "http_req_duration{label:clusters.replicaOfflineEvents}": ["p(95)<2000"], + "http_req_duration{label:roles.owners}": ["p(95)<2000"], "http_req_duration{label:clusters.largestClusterReplica}": ["p(95)<5000"], "http_req_duration{label:clusters.largestMaintainedQueries}": [ "p(95)<5000", diff --git a/console/e2e-tests/scalability/page-load.spec.ts b/console/e2e-tests/scalability/page-load.spec.ts index 4540a829113e7..14a372b059a64 100644 --- a/console/e2e-tests/scalability/page-load.spec.ts +++ b/console/e2e-tests/scalability/page-load.spec.ts @@ -30,10 +30,12 @@ import { sourcesListUrl, } from "./helpers"; +// Generous ceiling: this suite measures load times rather than asserting +// them, and 30s renders were observed under CI workload-replay load. async function measureUntilVisible( page: Page, selectors: string | string[], - timeoutMs = 30_000, + timeoutMs = 120_000, ): Promise { const start = performance.now(); const selectorList = Array.isArray(selectors) ? selectors : [selectors]; diff --git a/doc/developer/cloudtest.md b/doc/developer/cloudtest.md index d518700f971a8..35963e5a21d45 100644 --- a/doc/developer/cloudtest.md +++ b/doc/developer/cloudtest.md @@ -29,7 +29,7 @@ official [`kubernetes`] Python library to control the Kubernetes cluster. On Linux, use: ``` - curl -fL https://dl.k8s.io/release/v1.34.5/bin/linux/amd64/kubectl > kubectl + curl -fL https://dl.k8s.io/release/v1.34.10/bin/linux/amd64/kubectl > kubectl chmod +x kubectl sudo mv kubectl /usr/local/bin ``` @@ -48,7 +48,7 @@ official [`kubernetes`] Python library to control the Kubernetes cluster. On Linux, use: ``` - curl -fL https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-amd64 > kind + curl -fL https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-amd64 > kind chmod +x kind sudo mv kind /usr/local/bin ``` diff --git a/doc/user/content/reference/system-catalog/mz_catalog.md b/doc/user/content/reference/system-catalog/mz_catalog.md index fc49cbcde6296..c10f9100761e3 100644 --- a/doc/user/content/reference/system-catalog/mz_catalog.md +++ b/doc/user/content/reference/system-catalog/mz_catalog.md @@ -562,7 +562,7 @@ Field | Type | Meaning `type` | [`text`] | The type of the source: `kafka`, `mysql`, `postgres`, `load-generator`, `progress`, or `subsource`. `connection_id` | [`text`] | The ID of the connection associated with the source, if any. Corresponds to [`mz_connections.id`](/reference/system-catalog/mz_catalog/#mz_connections). `size` | [`text`] | *Deprecated* The [size](/sql/create-source/#sizing-a-source) of the source. -`envelope_type` | [`text`] | For Kafka sources, the [envelope](/sql/create-source/kafka/#envelopes) type: `none`, `upsert`, or `debezium`. `NULL` for other source types. +`envelope_type` | [`text`] | For old-syntax Kafka sources, the [envelope](/sql/create-source/kafka/#envelopes) type: `none`, `upsert`, or `debezium`. `NULL` for new-syntax Kafka sources, whose envelopes are defined per source table (see `mz_kafka_source_tables`), and for other source types. `key_format` | [`text`] | For Kafka sources, the [format](/sql/create-source/kafka#syntax) of the Kafka message key: `avro`, `csv`, `regex`, `bytes`, `json`, `text`, or `NULL`. `value_format` | [`text`] | For Kafka sources, the [format](/sql/create-source/kafka#syntax) of the Kafka message value: `avro`, `csv`, `regex`, `bytes`, `json`, `text`. `NULL` for other source types. `cluster_id` | [`text`] | The ID of the cluster maintaining the source. Corresponds to [`mz_clusters.id`](/reference/system-catalog/mz_catalog/#mz_clusters). diff --git a/doc/user/content/reference/system-catalog/mz_internal.md b/doc/user/content/reference/system-catalog/mz_internal.md index 75812a14efcf5..bc2694d5c903f 100644 --- a/doc/user/content/reference/system-catalog/mz_internal.md +++ b/doc/user/content/reference/system-catalog/mz_internal.md @@ -806,7 +806,7 @@ table and the corresponding upstream Kafka topic being ingested. | ------------------- | ---------------- | -------- | | `id` | [`text`] | The ID of the table. Corresponds to [`mz_catalog.mz_tables.id`](../mz_catalog#mz_tables). | | `topic` | [`text`] | The topic being ingested. | -| `envelope_type` | [`text`] | The [envelope](/sql/create-source/kafka/#envelopes) type: `none`, `upsert`, or `debezium`. `NULL` for other source types. | +| `envelope_type` | [`text`] | The [envelope](/sql/create-source/kafka/#envelopes) type: `none`, `upsert`, or `debezium`. Defaults to `none` when the source table omits an explicit envelope. | | `key_format` | [`text`] | The [format](/sql/create-source/kafka/#syntax) of the Kafka message key: `avro`, `csv`, `regex`, `bytes`, `json`, `text`, or `NULL`. | | `value_format` | [`text`] | The [format](/sql/create-source/kafka/#syntax) of the Kafka message value: `avro`, `csv`, `regex`, `bytes`, `json`, `text`. `NULL` for other source types. | diff --git a/misc/kind/cluster-node-recovery-test.yaml b/misc/kind/cluster-node-recovery-test.yaml index f7b4ffe386b0d..726083343ab00 100644 --- a/misc/kind/cluster-node-recovery-test.yaml +++ b/misc/kind/cluster-node-recovery-test.yaml @@ -22,7 +22,7 @@ kubeadmConfigPatches: "service-node-port-range": "32000-32063" nodes: - role: control-plane - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 extraPortMappings: - containerPort: 32000 hostPort: 32000 @@ -154,37 +154,37 @@ nodes: hostPort: 32063 - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "1" topology.kubernetes.io/zone: "1" - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "1" topology.kubernetes.io/zone: "1" - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "2" topology.kubernetes.io/zone: "2" - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "2" topology.kubernetes.io/zone: "2" - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "3" topology.kubernetes.io/zone: "3" - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "3" @@ -192,7 +192,7 @@ nodes: # node for the `quickstart` cluster replica - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/availability-zone: "quickstart" @@ -200,14 +200,14 @@ nodes: # only envd (nodes will be tainted in the setup) - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true environmentd: true materialize.cloud/availability-zone: "3" topology.kubernetes.io/zone: "3" - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true environmentd: true @@ -216,7 +216,7 @@ nodes: # for supporting services - role: worker - image: kindest/node:v1.33.1 + image: kindest/node:v1.34.8 labels: supporting-services: true materialize.cloud/availability-zone: "3" diff --git a/misc/kind/cluster.yaml b/misc/kind/cluster.yaml index bb6a0cada8bbe..a991a18a33fe3 100644 --- a/misc/kind/cluster.yaml +++ b/misc/kind/cluster.yaml @@ -22,7 +22,7 @@ kubeadmConfigPatches: "service-node-port-range": "32000-32063" nodes: - role: control-plane - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 extraPortMappings: - containerPort: 32000 hostPort: 32000 @@ -154,21 +154,21 @@ nodes: hostPort: 32063 - role: worker - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/swap: true materialize.cloud/availability-zone: "1" topology.kubernetes.io/zone: "1" - role: worker - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/swap: true materialize.cloud/availability-zone: "2" topology.kubernetes.io/zone: "2" - role: worker - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/swap: true @@ -177,7 +177,7 @@ nodes: # no-disk - role: worker - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: false materialize.cloud/swap: false @@ -186,14 +186,14 @@ nodes: # others - role: worker - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/swap: true materialize.cloud/availability-zone: "3" topology.kubernetes.io/zone: "3" - role: worker - image: kindest/node:v1.31.6 + image: kindest/node:v1.34.8 labels: materialize.cloud/disk: true materialize.cloud/swap: true diff --git a/misc/python/materialize/cloudtest/app/materialize_application.py b/misc/python/materialize/cloudtest/app/materialize_application.py index 9122eb354e77e..bfcc5047f8a7b 100644 --- a/misc/python/materialize/cloudtest/app/materialize_application.py +++ b/misc/python/materialize/cloudtest/app/materialize_application.py @@ -152,12 +152,20 @@ def wait_for_sql(self) -> None: wait(condition="condition=Ready", resource="pod/environmentd-0") start = datetime.now() - while datetime.now() - start < timedelta(seconds=300): + while True: try: self.environmentd.sql("SELECT 1") break - except InterfaceError as e: - # Since we crash environmentd, we expect some errors that we swallow. + except (InterfaceError, OSError) as e: + if datetime.now() - start > timedelta(seconds=300): + raise + # Since we crash environmentd, we expect some errors that we + # swallow. pg8000 wraps most connection failures in + # InterfaceError, but raw socket errors from its SSL + # negotiation (e.g. ConnectionResetError when the connection + # is accepted by the port-forwarding proxy and then reset + # because environmentd is not listening yet) leak through + # unwrapped. LOGGER.info(f"SQL interface not ready, {e} while SELECT 1. Waiting...") time.sleep(2) diff --git a/misc/python/materialize/cloudtest/k8s/minio.py b/misc/python/materialize/cloudtest/k8s/minio.py index efc06bb4695c5..6f9da83e00a96 100644 --- a/misc/python/materialize/cloudtest/k8s/minio.py +++ b/misc/python/materialize/cloudtest/k8s/minio.py @@ -7,6 +7,8 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. +import yaml + from materialize import MZ_ROOT from materialize.cloudtest import DEFAULT_K8S_NAMESPACE from materialize.cloudtest.k8s.api.k8s_resource import K8sResource @@ -35,9 +37,8 @@ def create(self) -> None: "true", ) - # the PVC will be created afterwards for yaml_file in [ - "minio-standalone-deployment", + "minio-standalone-pvc", "minio-standalone-service", ]: self.kubectl( @@ -46,22 +47,20 @@ def create(self) -> None: str(MINIO_YAML_DIRECTORY / f"{yaml_file}.yaml"), ) - if self.apply_node_selectors: - self.kubectl( - "patch", - "deployment", - "minio-deployment", - "--type", - "json", - "-p", - '[{"op": "add", "path": "/spec/template/spec/nodeSelector", "value": {"supporting-services": "true"} }]', - ) - - # the PVC needs to be created after patching the deployment + # NOTE: The deployment must carry its final nodeSelector before it is + # created, so it is injected here rather than patched in afterwards. + # The claim's storage class binds with WaitForFirstConsumer, so the + # provisioner pins the volume to whichever node the scheduler picks for + # the first pod that consumes the claim. A pod created without the + # nodeSelector can drive that decision even if it is replaced moments + # later, pinning the volume to a node the final pod may not run on. The + # pod then stays Pending forever, because nothing can satisfy both the + # volume's node affinity and the pod's node selector. self.kubectl( "create", "-f", - str(MINIO_YAML_DIRECTORY / "minio-standalone-pvc.yaml"), + "-", + input=self.deployment_manifest(), ) self.wait( @@ -72,6 +71,17 @@ def create(self) -> None: self.create_buckets(["persist", "copytos3", "copyfroms3"]) + def deployment_manifest(self) -> str: + with open(MINIO_YAML_DIRECTORY / "minio-standalone-deployment.yaml") as f: + deployment = yaml.safe_load(f) + + if self.apply_node_selectors: + deployment["spec"]["template"]["spec"]["nodeSelector"] = { + "supporting-services": "true" + } + + return yaml.dump(deployment) + def create_buckets(self, buckets: list[str]) -> None: cmds = [ f"mc config host add myminio http://minio-service.{self.namespace()}:9000 minio minio123" diff --git a/misc/python/materialize/data_ingest/data_type.py b/misc/python/materialize/data_ingest/data_type.py index b9a246b41562e..762f96db54e22 100644 --- a/misc/python/materialize/data_ingest/data_type.py +++ b/misc/python/materialize/data_ingest/data_type.py @@ -1030,6 +1030,16 @@ def name(backend: Backend = Backend.MATERIALIZE) -> str: raise ValueError("Unsupported") elif backend == Backend.JSON: raise ValueError("Unsupported") + elif backend == Backend.MYSQL: + # Fractional-second precision spelled out, because a bare MySQL TIME + # has fsp 0 and *rounds* what it is given. The 23:59:59.999999 edge + # value above then lands as 24:00:00, which MySQL accepts (its TIME + # range is +/-838:59:59) but Materialize cannot represent: the source + # decodes TIME into a chrono::NaiveTime, so ingestion fails with + # "Couldn't convert the value Time(..) to a desired type" and the + # source has to be dropped and recreated. At fsp 6 nothing rounds and + # the value survives the round trip. + return "time(6)" else: return "time" diff --git a/misc/python/materialize/data_ingest/executor.py b/misc/python/materialize/data_ingest/executor.py index 41dca129ff0bb..b37f64aecc2ee 100644 --- a/misc/python/materialize/data_ingest/executor.py +++ b/misc/python/materialize/data_ingest/executor.py @@ -546,7 +546,12 @@ def create(self, logging_exe: Any | None = None) -> None: ) self.execute( cur, - f"""CREATE TABLE {identifier(self.table)} + # Qualified like the source above. Unqualified, the table lands + # in the connection's default schema (`public`) rather than + # `self.schema`, and a caller addressing it as + # `..`, as parallel-workload does, never + # resolves it. + f"""CREATE TABLE {identifier(self.database)}.{identifier(self.schema)}.{identifier(self.table)} FROM SOURCE {identifier(self.database)}.{identifier(self.schema)}.{identifier(self.source)} (REFERENCE {identifier(self.table)})""", ) @@ -696,7 +701,8 @@ def create(self, logging_exe: Any | None = None, prepopulate_rows: int = 0) -> N ) self.execute( cur, - f"""CREATE TABLE {identifier(self.table)} + # Qualified like the source above, see SqlServerExecutor.create. + f"""CREATE TABLE {identifier(self.database)}.{identifier(self.schema)}.{identifier(self.table)} FROM SOURCE {identifier(self.database)}.{identifier(self.schema)}.{identifier(self.source)} (REFERENCE mysql.{identifier(self.table)})""", ) @@ -826,7 +832,8 @@ def create(self, logging_exe: Any | None = None) -> None: ) self.execute( cur, - f"""CREATE TABLE {identifier(self.table)} + # Qualified like the source above, see SqlServerExecutor.create. + f"""CREATE TABLE {identifier(self.database)}.{identifier(self.schema)}.{identifier(self.table)} FROM SOURCE {identifier(self.database)}.{identifier(self.schema)}.{identifier(self.source)} (REFERENCE {identifier(self.table)})""", ) diff --git a/misc/python/materialize/mzcompose/service.py b/misc/python/materialize/mzcompose/service.py index 56aa5a4dea75a..d6c1b1d3f8ca4 100644 --- a/misc/python/materialize/mzcompose/service.py +++ b/misc/python/materialize/mzcompose/service.py @@ -115,6 +115,9 @@ class ServiceConfig(TypedDict, total=False): init: bool """Whether to run an init process in the container.""" + ipc: str + """The IPC isolation mode for the container, e.g. `host`.""" + ports: Sequence[int | str] """Service ports to expose to the host.""" diff --git a/misc/python/materialize/output_consistency/ignore_filter/internal_output_inconsistency_ignore_filter.py b/misc/python/materialize/output_consistency/ignore_filter/internal_output_inconsistency_ignore_filter.py index 59eae8d957017..3b1f24ad959ba 100644 --- a/misc/python/materialize/output_consistency/ignore_filter/internal_output_inconsistency_ignore_filter.py +++ b/misc/python/materialize/output_consistency/ignore_filter/internal_output_inconsistency_ignore_filter.py @@ -84,7 +84,7 @@ def _matches_problematic_operation_or_function_invocation( isinstance(arg_type_spec, NumericReturnTypeSpec) and not arg_type_spec.only_integer ): - return YesIgnore("database-issues#4341") + return YesIgnore("DB-191") return NoIgnore() @@ -104,13 +104,13 @@ def _matches_problematic_function_invocation( "var_pop", }: if ExpressionCharacteristics.MAX_VALUE in all_involved_characteristics: - return YesIgnore("database-issues#4341") + return YesIgnore("DB-191") if ( ExpressionCharacteristics.DECIMAL in all_involved_characteristics and ExpressionCharacteristics.TINY_VALUE in all_involved_characteristics ): - return YesIgnore("database-issues#4341") + return YesIgnore("DB-191") return NoIgnore() diff --git a/misc/python/materialize/parallel_workload/action.py b/misc/python/materialize/parallel_workload/action.py index d314cc10db9c9..c585e9bccb09a 100644 --- a/misc/python/materialize/parallel_workload/action.py +++ b/misc/python/materialize/parallel_workload/action.py @@ -46,7 +46,6 @@ VarChar, ) from materialize.data_ingest.query_error import QueryError -from materialize.data_ingest.row import Operation from materialize.mzcompose import get_default_system_parameters from materialize.mzcompose.composition import Composition from materialize.mzcompose.services.materialized import ( @@ -243,6 +242,25 @@ def applicable(self, exe: Executor) -> bool: coverage check meaningful.""" return True + def insert_batch_size(self, exe: Executor, table: Table) -> int: + """How many rows the next insert into `table` may add, 0 once the table + sits at MAX_ROWS.""" + available = MAX_ROWS - table.num_rows + if available < 1: + return 0 + # A per-worker share of the budget, not the whole remainder: `num_rows` + # is read here but only bumped once the statement returns, so every + # worker would otherwise size a full-budget batch off the same stale + # counter and the table would overshoot MAX_ROWS by roughly a + # worker-count factor, which is what kept the cap from binding at all. + # A share bounds the overshoot at one extra round of concurrent batches. + # Charging the budget up front instead would bind exactly, but a claim + # is not given back when the statement's transaction rolls back, so the + # insert actions would stall against tables that read as full while + # holding far fewer rows. + share = max(1, MAX_ROWS // exe.db.num_threads) + return self.rng.randint(1, min(available, share)) + def create_system_connection( self, exe: Executor, num_attempts: int = 10 ) -> Connection: @@ -1066,10 +1084,13 @@ def run(self, exe: Executor) -> bool: return False table = self.rng.choice(tables) + num_rows = self.insert_batch_size(exe, table) + if not num_rows: + return False + column_names = ", ".join(column.name(True) for column in table.columns) column_values = [] - max_rows = min(100, MAX_ROWS - table.num_rows) - for i in range(self.rng.randrange(1, max_rows + 1)): + for i in range(num_rows): column_values.append( ", ".join(column.value(self.rng, True) for column in table.columns) ) @@ -1081,7 +1102,7 @@ def run(self, exe: Executor) -> bool: self.exe_prepared(query, f"insert{self.stmt_id}", exe) else: exe.execute(query, http=Http.RANDOM) - table.num_rows += len(column_values) + table.num_rows += num_rows exe.insert_table = table.table_id return True @@ -1137,10 +1158,11 @@ def run(self, exe: Executor) -> bool: f"({expression(column.data_type, source.columns, self.rng, kind=ExprKind.WRITE)})::{column.data_type.name()}" for column in table.columns ) - # `num_rows` can be stale, so clamp instead of trusting the filter - # above. The LIMIT keeps a self-insert from doubling the table on every + # The LIMIT keeps a self-insert from doubling the table on every # attempt. - limit = self.rng.randint(1, max(1, min(100, MAX_ROWS - table.num_rows))) + limit = self.insert_batch_size(exe, table) + if not limit: + return False query = ( f"INSERT INTO {table} ({column_names}) SELECT {expressions} FROM {source}" f" WHERE {expression(Boolean, source.columns, self.rng, kind=ExprKind.WRITE)}" @@ -1199,13 +1221,16 @@ def run(self, exe: Executor) -> bool: return False table = self.rng.choice(tables) + num_rows = self.insert_batch_size(exe, table) + if not num_rows: + return False + values = [] - max_rows = min(100, MAX_ROWS - table.num_rows) - for i in range(self.rng.randrange(1, max_rows + 1)): + for i in range(num_rows): values.append([column.value(self.rng, False) for column in table.columns]) query = f"COPY INTO {table} FROM STDIN" exe.copy(query, values) - table.num_rows += len(values) + table.num_rows += num_rows exe.insert_table = table.table_id return True @@ -1250,10 +1275,13 @@ def run(self, exe: Executor) -> bool: return False table = self.rng.choice(tables) + num_rows = self.insert_batch_size(exe, table) + if not num_rows: + return False + column_names = ", ".join(column.name(True) for column in table.columns) column_values = [] - max_rows = min(100, MAX_ROWS - table.num_rows) - for i in range(self.rng.randrange(1, max_rows + 1)): + for i in range(num_rows): column_values.append( ", ".join(column.value(self.rng, True) for column in table.columns) ) @@ -1280,7 +1308,7 @@ def run(self, exe: Executor) -> bool: self.exe_prepared(query, f"insert_returning{self.stmt_id}", exe) else: exe.execute(query, http=Http.RANDOM) - table.num_rows += len(column_values) + table.num_rows += num_rows exe.insert_table = table.table_id return True @@ -1319,16 +1347,28 @@ def run(self, exe: Executor) -> bool: class SourceInsertAction(Action): + """Feed one workload transaction to a random CDC source's upstream system. + + Unlike the table DML actions this carries no MAX_ROWS gate, because a + `data_ingest` workload bounds its own upstream table: the upsert workloads + only ever write key 0, and the delete-at-end-of-day workloads insert + `Records.SOME` keys and then delete exactly those keys again, so the table + oscillates and every cycle nets to zero. A gate could only ever misfire. + `next()` has already consumed a transaction by the time a row count could be + checked, so refusing to run it would drop one half of an insert/delete pair + and leave the count drifting away from the upstream table for the rest of the + run. A workload that grows without bound would need a different mechanism, + not a row count, since `MySqlSource.prepopulate_rows` already puts up to + 30,000 rows upstream before the source is even created.""" + def run(self, exe: Executor) -> bool: with exe.db.lock: - sources = [ - source - for source in exe.db.kafka_sources + sources = ( + exe.db.kafka_sources + exe.db.postgres_sources + exe.db.mysql_sources + exe.db.sql_server_sources - if source.num_rows < MAX_ROWS - ] + ) if not sources: return False source = self.rng.choice(sources) @@ -1341,14 +1381,7 @@ def run(self, exe: Executor) -> bool: ]: return False - transaction = next(source.generator) - for row_list in transaction.row_lists: - for row in row_list.rows: - if row.operation == Operation.INSERT: - source.num_rows += 1 - elif row.operation == Operation.DELETE: - source.num_rows -= 1 - source.executor.run(transaction, logging_exe=exe) + source.executor.run(next(source.generator), logging_exe=exe) return True @@ -1469,7 +1502,7 @@ def run(self, exe: Executor) -> bool: for t in exe.db.tables if t != table and (not t.temp or t in exe.temp_objects) ] - # TODO: Drop the RepeatRow gate once database-issues#9308 is fixed. + # TODO: Drop the RepeatRow gate once STG-36 is fixed. # DELETE .. USING lowers to a semijoin whose DistinctBy can leave the # target table with a net-negative row, and every later reader of that # table then surfaces the corruption. Tolerating that class outside @@ -2768,6 +2801,18 @@ def run(self, exe: Executor) -> bool: class FlipFlagsAction(Action): + # Shortest gap between two flips, fleet-wide. Every worker's action list + # carries this action, so weights alone put it at ~30 flips/s, which drove + # ~43k dyncfg applications per replica and most of a 132 MB services.log. + # What this action is after is a flag changing under a running dataflow, and + # one change per second is plenty for that. + MIN_INTERVAL_SEC = 1.0 + + # Guards `last_flip`, which is shared across workers since each holds its + # own instance of this action. + interval_lock = threading.Lock() + last_flip = 0.0 + def __init__( self, rng: random.Random, @@ -2927,10 +2972,10 @@ def __init__( BOOLEAN_FLAG_VALUES ) self.flags_with_values["cluster"] = ["quickstart", "dont_exist"] - # NOTE: enable_frontend_peek_sequencing is pinned off in - # ADDITIONAL_SYSTEM_PARAMETER_DEFAULTS (frontend-peek read-hold vs - # compaction race, https://linear.app/materializeinc/issue/SQL-520), so - # it is not flipped here. + self.flags_with_values["enable_frontend_peek_sequencing"] = [ + "true", + "false", + ] self.flags_with_values["enable_frontend_subscribes"] = [ "true", "false", @@ -3251,6 +3296,12 @@ def errors_to_ignore(self, exe: Executor) -> list[str]: ] + super().errors_to_ignore(exe) def run(self, exe: Executor) -> bool: + with FlipFlagsAction.interval_lock: + now = time.time() + if now - FlipFlagsAction.last_flip < FlipFlagsAction.MIN_INTERVAL_SEC: + return False + FlipFlagsAction.last_flip = now + # A tenth of the time set a random cluster's arrangement dictionary # compression instead of flipping a global flag. The per-cluster option # and the global `enable_arrangement_dictionary_compression_alpha` flag @@ -3895,6 +3946,20 @@ def run(self, exe: Executor) -> bool: f"{new_size} sat in-progress {read_ts_ms - int(record_deadline)}ms " f"past its deadline {record_deadline}" ) + # A settled record is retained with its terminal status, so a + # NULL from the LEFT JOIN means the record this ALTER wrote is + # gone, which no code path is allowed to do. Every writer of the + # durable field keeps the record and only moves its status: + # `reshape_alter_cluster_managed` writes InProgress or, for an + # ALTER back to the realized shape, Cancelled; + # `cancel_carried_reconfiguration` mutates the status in place; + # and the controller's three writes are Finalized, TimedOut and + # ResourceExhausted. `ReconfigurationWrite.record` is an Option + # documented as "None to clear it", but nothing constructs that. + # The relation also drops a cluster that stops being managed, and + # environmentd's bootstrap is the one place that resets the field + # outright, hence the scenario carve-outs in `applicable` and no + # `SET (MANAGED ...)` anywhere in the workload. if read_ts_ms > our_deadline_ms and status is None: raise ValueError( f"Reconfiguration record of cluster {cluster} is gone as of " @@ -5624,10 +5689,16 @@ def run(self, exe: Executor) -> bool: log = f"POST {url} Headers: {', '.join(headers_strs)} Body: {payload.encode('utf-8')}" exe.log(log) try: - source.num_rows += 1 result = requests.post(url, data=payload.encode(), headers=headers) if result.status_code != 200: raise QueryError(f"{result.status_code}: {result.text}", log) + # Count after the POST landed. A webhook source is append-only, + # so its row budget is never released, and counting rejected + # POSTs retires the source without a single row ever reaching + # it. A source whose POSTs all 404 (a concurrent cascading drop, + # a rename) then silently stops being posted to for the rest of + # the run. + source.num_rows += 1 except requests.exceptions.ConnectionError: # Expected when Mz is killed if exe.db.scenario not in ( @@ -6177,13 +6248,7 @@ def __init__( (CopyToStdoutAction, 20), (ShowAction, 10), (SystemCatalogReadAction, 10), - # TODO: Reenable once EXPLAIN FILTER PUSHDOWN can no longer panic the - # coordinator when a referenced compute collection is concurrently - # dropped. sequence_explain_pushdown -> acquire_read_holds().expect( - # "missing compute collection") at read_policy.rs:389 (normal peeks and - # EXPLAIN ANALYZE handle the drop gracefully). - # See https://linear.app/materializeinc/issue/SQL-519 - # (ExplainFilterPushdownAction, 5), + (ExplainFilterPushdownAction, 5), # PREPARED BUT DISABLED (see class docstrings): enabling these now just # re-detects known-unfixed coordinator bugs rather than finding new ones. # (DependencyConsistencyAction, 5), # TODO: enable once SQL-521 fixed @@ -6279,9 +6344,8 @@ def __init__( (DropLoadGeneratorSourceAction, 4), (CreateMultiLoadGeneratorSourceAction, 2), (DropMultiLoadGeneratorSourceAction, 2), - # TODO: Reenable when https://linear.app/materializeinc/issue/SS-307 is fixed - # (CreateMySqlSourceAction, 4), - # (DropMySqlSourceAction, 4), + (CreateMySqlSourceAction, 4), + (DropMySqlSourceAction, 4), (CreatePostgresSourceAction, 4), (DropPostgresSourceAction, 4), # TODO: Reenable when https://linear.app/materializeinc/issue/SS-290 is fixed @@ -6343,10 +6407,7 @@ def __init__( (DDLTransactionAction, 2), (SystemCatalogReadAction, 4), (ExplainAnalyzeAction, 4), - # TODO: Reenable with EXPLAIN FILTER PUSHDOWN's coordinator panic on a - # concurrently-dropped compute collection (read_policy.rs:389). - # See https://linear.app/materializeinc/issue/SQL-519 - # (ExplainFilterPushdownAction, 2), + (ExplainFilterPushdownAction, 2), (FlipFlagsAction, 2), # TODO: Reenable when https://linear.app/materializeinc/issue/SQL-405 is fixed. # (AlterTableAddColumnAction, 10), diff --git a/misc/python/materialize/parallel_workload/database.py b/misc/python/materialize/parallel_workload/database.py index b3d72d2fbb281..49ccfbabfe48e 100644 --- a/misc/python/materialize/parallel_workload/database.py +++ b/misc/python/materialize/parallel_workload/database.py @@ -61,17 +61,15 @@ MAX_COLUMNS = 50 MAX_INCLUDE_HEADERS = 5 -# The row count a view's join squares. A view over two tables joins them on a -# random boolean predicate, so the join is a cross product and every read of -# that view builds MAX_ROWS**2 wide intermediate rows, whatever LIMIT the -# reading query carries. Measured at 500 with only four columns, one -# `COPY (SELECT * FROM view WHERE .. LIMIT 100) TO 's3://..'` cost ~450 MiB of -# replica RSS, and pw tables carry up to MAX_COLUMNS of them. A handful of such -# reads at once is what grew a clusterd to 19 GiB and had the kernel OOM-killer -# take it, plus environmentd and the Kafka broker, out of the one cgroup all of -# a run's containers share (nightlies 17660-17701). 100 keeps enough rows for -# the DML, persist and index paths to be interesting while cutting the -# intermediate 25-fold. +# The row count a table is held at. Measured with a view whose join was a cross +# product, one `COPY (SELECT * FROM view WHERE .. LIMIT 100) TO 's3://..'` over +# 500-row bases cost ~450 MiB of replica RSS with only four columns, and pw +# tables carry up to MAX_COLUMNS of them. A handful of such reads at once is +# what grew a clusterd to 19 GiB and had the kernel OOM-killer take it, plus +# environmentd and the Kafka broker, out of the one cgroup all of a run's +# containers share (nightlies 17660-17701). 100 keeps enough rows for the DML, +# persist and index paths to be interesting. See also JOIN_KEY_TYPES, which +# keeps a view's join from squaring this in the first place. MAX_ROWS = 100 MAX_CLUSTERS = 4 MAX_CLUSTER_REPLICAS = 2 @@ -94,7 +92,6 @@ MAX_INITIAL_DBS = 1 MAX_INITIAL_SCHEMAS = 1 -MAX_INITIAL_CLUSTERS = 2 MAX_INITIAL_TABLES = 2 MAX_INITIAL_VIEWS = 2 MAX_INITIAL_ROLES = 1 @@ -105,6 +102,16 @@ MAX_INITIAL_POSTGRES_SOURCES = 1 MAX_INITIAL_LOADGEN_SOURCES = 1 +# Types eligible as the equality anchor of a two-base view's join, see +# `View.join_key_expr`. A type qualifies when Materialize has `=` for it (map +# has none) and, less obviously, when its random values collide often enough +# that the equality still lets rows through: every type here draws one of a +# handful of sentinels (the type's min or max, 0, 1, one of five fixed strings) +# about a fifth of the time, so on the order of a percent of the cross product +# survives. An equality on uuid or timestamp columns would bound the join just +# as well but always produce an empty view, which tests nothing. +JOIN_KEY_TYPES = NUMBER_TYPES + [Text] + class BodyFormat(Enum): TEXT = 1 @@ -333,9 +340,17 @@ def __init__( ) if base_object2: - self.on_expr = expression( + # The random boolean alone references an arbitrary subset of the + # columns, so it usually names only one side or neither, and the + # join plans as a cross product whose output squares MAX_ROWS. The + # equality anchor makes the output linear in the input instead. + # Cross joins stay covered by the ad-hoc SELECTs, which carry a + # LIMIT the optimizer can push into the join. + join_key = self.join_key_expr(base_object, base_object2, rng) + predicate = expression( Boolean, all_columns, rng, kind=ExprKind.MATERIALIZABLE ) + self.on_expr = f"{join_key} AND ({predicate})" self.union = rng.random() < 0.1 if self.union: @@ -354,6 +369,32 @@ def __init__( self.columns = [Column(rng, 0, Long, self)] self.union = False + @staticmethod + def join_key_expr(left: DBObject, right: DBObject, rng: random.Random) -> str: + """An equality between one column of each side, so that the join has a + key and its output stays linear in the input. + + Matches the join shape `SelectAction.generate_select_query` emits, with + two additions. A `JOIN_KEY_TYPES` pair wins over an equally valid pair of + some other type, whose values would never collide and leave the view + permanently empty. And a pair of objects sharing no column type at all, + which a query can answer by not joining but a two-base view cannot, falls + back to comparing the columns as text. That bounds the join just as well, + only with a near-empty result.""" + candidates = [ + (left_column, right_column) + for left_column in left.columns + for right_column in right.columns + # Materialize has no `=` for map. + if left_column.data_type is right_column.data_type + and left_column.data_type is not TextTextMap + ] + colliding = [pair for pair in candidates if pair[0].data_type in JOIN_KEY_TYPES] + if candidates: + left_column, right_column = rng.choice(colliding or candidates) + return f"{left_column} = {right_column}" + return f"{rng.choice(left.columns)}::text = {rng.choice(right.columns)}::text" + def name(self) -> str: if self.rename: return naughtify(f"v-{self.view_id}-{self.rename}") @@ -528,7 +569,6 @@ class KafkaSource(DBObject): lock: threading.Lock columns: list[KafkaColumn] schema: Schema - num_rows: int def __init__( self, @@ -542,7 +582,6 @@ def __init__( self.source_id = source_id self.cluster = cluster self.schema = schema - self.num_rows = 0 fields = [] for i in range(rng.randint(1, 10)): fields.append( @@ -759,7 +798,6 @@ class MySqlSource(DBObject): lock: threading.Lock columns: list[MySqlColumn] schema: Schema - num_rows: int def __init__( self, @@ -773,7 +811,6 @@ def __init__( self.source_id = source_id self.cluster = cluster self.schema = schema - self.num_rows = 0 # Bias toward a single-column key: only tables with a single-column # integer primary key take the parallel PK-range snapshot path, the # rest fall back to a whole-table read by one worker. @@ -823,7 +860,6 @@ class PostgresSource(DBObject): lock: threading.Lock columns: list[PostgresColumn] schema: Schema - num_rows: int def __init__( self, @@ -837,7 +873,6 @@ def __init__( self.source_id = source_id self.cluster = cluster self.schema = schema - self.num_rows = 0 fields = [] for i in range(rng.randint(1, 10)): fields.append( @@ -878,7 +913,6 @@ class SqlServerSource(DBObject): lock: threading.Lock columns: list[SqlServerColumn] schema: Schema - num_rows: int def __init__( self, @@ -893,7 +927,6 @@ def __init__( self.source_id = source_id self.cluster = cluster self.schema = schema - self.num_rows = 0 fields = [] for i in range(rng.randint(1, 10)): fields.append(Field(f"key{i}", rng.choice(DATA_TYPES_FOR_KEY), True)) @@ -1448,6 +1481,7 @@ class Database: seed: str sqlsmith_state: str flags: dict[str, str] + num_threads: int def __init__( self, @@ -1458,12 +1492,14 @@ def __init__( complexity: Complexity, scenario: Scenario, naughty_identifiers: bool, + num_threads: int, ): self.host = host self.ports = ports self.complexity = complexity self.scenario = scenario self.seed = seed + self.num_threads = num_threads set_naughty_identifiers(naughty_identifiers) self.s3_path = 0 @@ -1499,16 +1535,35 @@ def __init__( self.view_id = len(self.views) self.roles = [Role(i) for i in range(rng.randint(0, MAX_INITIAL_ROLES))] self.role_id = len(self.roles) - # At least one storage cluster required for WebhookSources + # Cluster 0 is the one cluster whose shape nothing churns: every cluster + # action skips it and operates on `clusters[1:]`, so it stays available + # for sources and sinks. Which cluster a given source or sink actually + # lands on is still random, both here and in the Create*Action paths. + # + # Each of those actions needs a cluster of a specific kind past cluster + # 0: managed for AlterClusterSetAction and FlipFlagsAction's per-cluster + # arrangement compression, unmanaged for Create/DropClusterReplicaAction. + # Drawing the count and the kinds the way the other initial objects are + # drawn covers both in one run out of eight, and the rest spend the whole + # run skipping. Nightly 17774 reported 369 AlterClusterSet, 129 + # ReconfigureCluster and 117 CreateClusterReplica attempts with not one + # success between them. So the kinds are fixed and only cluster 0's is + # left to the seed. self.clusters = [ Cluster( - i, - managed=rng.choice([True, False]), + cluster_id, + managed=managed, size=rng.choice(["scale=1,workers=1", "scale=1,workers=2"]), + # One replica each, which is what lets Create and + # DropClusterReplicaAction take turns on the unmanaged cluster: + # Create needs room below MAX_CLUSTER_REPLICAS, Drop needs more + # than one replica so the cluster keeps serving requests. replication_factor=1, introspection_interval="1s", ) - for i in range(rng.randint(1, MAX_INITIAL_CLUSTERS)) + for cluster_id, managed in enumerate( + [rng.choice([True, False]), True, False] + ) ] self.cluster_id = len(self.clusters) self.indexes = set() @@ -1517,6 +1572,9 @@ def __init__( for i in range(rng.randint(0, MAX_INITIAL_WEBHOOK_SOURCES)) ] self.webhook_source_id = len(self.webhook_sources) + # The CDC source lists are populated by `create_initial_sources`, not + # here: such a source's executor connects to the source's database on + # construction, and that database only exists once `create` has made it. self.kafka_sources = [] self.mysql_sources = [] self.postgres_sources = [] @@ -1621,6 +1679,70 @@ def __iter__(self): self.schemas + self.clusters + self.roles + self.db_objects() ).__iter__() + def create_initial_sources(self, exe: Executor) -> None: + """Populate the CDC source lists, once the databases they connect to + exist. + + SourceInsertAction is the only writer of a CDC source, and DML + complexity gives the DDL action list weight 0, so nothing there ever + runs a Create*SourceAction. Without a source from the start that action + cannot succeed once in a whole run, which is why the Kafka source is + unconditional rather than randomized down to zero like the other initial + objects. + + Skipped for a 0dt deploy. A source executor connects on construction and + picks one of the two environmentds at random every time it connects, and + the second one only comes up mid-run when ZeroDowntimeDeployAction + deploys it, so constructing a source here fails outright half the time. + That scenario runs at DDL complexity, so its actions create the sources + instead.""" + if self.scenario == Scenario.ZeroDowntimeDeploy: + return + self.kafka_sources = [ + KafkaSource( + i, + exe.rng.choice(self.clusters), + exe.rng.choice(self.schemas), + self.ports, + exe.rng, + ) + for i in range(MAX_INITIAL_KAFKA_SOURCES) + ] + self.kafka_source_id = len(self.kafka_sources) + self.postgres_sources = [ + PostgresSource( + i, + exe.rng.choice(self.clusters), + exe.rng.choice(self.schemas), + self.ports, + exe.rng, + ) + # A Postgres source is not expected to survive a backup & restore, + # see database-issues#6881 and + # CreatePostgresSourceAction.applicable. + for i in range( + 0 + if self.scenario == Scenario.BackupRestore + else exe.rng.randint(0, MAX_INITIAL_POSTGRES_SOURCES) + ) + ] + self.postgres_source_id = len(self.postgres_sources) + self.mysql_sources = [ + MySqlSource( + i, + exe.rng.choice(self.clusters), + exe.rng.choice(self.schemas), + self.ports, + exe.rng, + ) + for i in range(exe.rng.randint(0, MAX_INITIAL_MYSQL_SOURCES)) + ] + self.mysql_source_id = len(self.mysql_sources) + # MAX_INITIAL_SQL_SERVER_SOURCES stays unused: a SQL Server source + # drives CPU far above the other source kinds (SS-290), enough to have + # starved whole runs, and one in every run's initial database would put + # that back for good. + def create(self, exe: Executor, composition: Composition) -> None: for db in self.dbs: db.drop(exe) @@ -1706,6 +1828,8 @@ def create(self, exe: Executor, composition: Composition) -> None: print("Creating relations") + self.create_initial_sources(exe) + for relation in self: relation.create(exe) diff --git a/misc/python/materialize/parallel_workload/negative_accumulation_errors.py b/misc/python/materialize/parallel_workload/negative_accumulation_errors.py index 6f60be9596676..52eeb8a868315 100644 --- a/misc/python/materialize/parallel_workload/negative_accumulation_errors.py +++ b/misc/python/materialize/parallel_workload/negative_accumulation_errors.py @@ -12,12 +12,12 @@ Parallel Workload intentionally generates queries (via `repeat_row`) that can trigger these errors; this module is the single place to update when the -messages change. Source of truth: database-issues#9308 (section "For reference, +messages change. Source of truth: STG-36 (section "For reference, here is a list of places in the code / error msgs where we detect negative accumulations or similar issues"). """ -# Keep this list in sync with database-issues#9308. Each entry is a substring +# Keep this list in sync with STG-36. Each entry is a substring # that may appear in an error surfaced to the client. When the underlying # error messages are reworded, update this list in a single commit so the # rest of Parallel Workload picks up the new strings automatically. diff --git a/misc/python/materialize/parallel_workload/parallel_workload.py b/misc/python/materialize/parallel_workload/parallel_workload.py index 4a2740fc70842..addeca00435fe 100644 --- a/misc/python/materialize/parallel_workload/parallel_workload.py +++ b/misc/python/materialize/parallel_workload/parallel_workload.py @@ -119,7 +119,7 @@ def run( ).timestamp() database = Database( - rng, seed, host, ports, complexity, scenario, naughty_identifiers + rng, seed, host, ports, complexity, scenario, naughty_identifiers, num_threads ) system_conn = psycopg.connect( @@ -210,31 +210,48 @@ def run( database.create(Executor(rng, cur, None, database), composition) conn.close() + weights: list[float] + if complexity == Complexity.DDL: + weights = [60, 30, 30, 30, 100] + elif complexity == Complexity.DML: + weights = [60, 30, 30, 30, 0] + elif complexity == Complexity.Read: + weights = [60, 30, 0, 0, 0] + elif complexity == Complexity.DDLOnly: + weights = [0, 0, 0, 0, 100] + else: + raise ValueError(f"Unknown complexity {complexity}") + candidate_lists = [ + read_action_list, + fetch_action_list, + write_action_list, + dml_nontrans_action_list, + ddl_action_list, + ] + # Cover every list the complexity enables before sampling the rest. Drawing + # each worker independently leaves a list unmanned surprisingly often: at 8 + # threads the fetch, write and dml lists each go empty in ~36% of runs and + # the DDL list in ~2%, and an unmanned list is silent, the run just reports + # zero attempts for its actions. That is how nightly 17774's + # `--scenario=rename` run ended up with no DDL worker, so it renamed + # nothing, created no source (leaving HttpPostAction and SourceInsertAction + # with nothing to write to for 100k attempts each), and never dropped the + # initial cross-product view that then OOM-killed its replica. + # Heaviest-first so a thread count below the number of enabled lists still + # covers the ones the complexity cares about most, with the seed breaking + # ties among equal weights. + covered = [lst for lst, weight in zip(candidate_lists, weights) if weight] + rng.shuffle(covered) + covered.sort(key=lambda lst: -weights[candidate_lists.index(lst)]) + workers = [] threads = [] for i in range(num_threads): - weights: list[float] - if complexity == Complexity.DDL: - weights = [60, 30, 30, 30, 100] - elif complexity == Complexity.DML: - weights = [60, 30, 30, 30, 0] - elif complexity == Complexity.Read: - weights = [60, 30, 0, 0, 0] - elif complexity == Complexity.DDLOnly: - weights = [0, 0, 0, 0, 100] - else: - raise ValueError(f"Unknown complexity {complexity}") worker_rng = random.Random(rng.randrange(SEED_RANGE)) - action_list = worker_rng.choices( - [ - read_action_list, - fetch_action_list, - write_action_list, - dml_nontrans_action_list, - ddl_action_list, - ], - weights, - )[0] + if i < len(covered): + action_list = covered[i] + else: + action_list = worker_rng.choices(candidate_lists, weights)[0] actions = [ action_class(worker_rng, composition) for action_class in action_list.action_classes @@ -301,6 +318,7 @@ def run( autocommit=False, system=False, composition=composition, + off_session_work=True, ) workers.append(worker) thread = threading.Thread( @@ -329,6 +347,7 @@ def run( autocommit=False, system=False, composition=composition, + off_session_work=True, ) workers.append(worker) thread = threading.Thread( @@ -350,10 +369,11 @@ def run( autocommit=False, system=False, composition=composition, + off_session_work=True, ) workers.append(worker) thread = threading.Thread( - name="kill", + name="backup-restore", target=worker.run, args=(host, ports["materialized"], ports["http"], "materialize", database), daemon=True, @@ -435,11 +455,21 @@ def run( # loop (exe is None), which can only be hanging inside a connect call # to the server. One that never waits on the server is stuck in the # workload's own code, e.g. deadlocked on two object locks taken in - # opposite orders, which no timeout ever clears. Nothing legitimate - # pauses without a round trip for the 300s above (the longest such - # pause is BackupRestoreAction's 240s sleep), but a single sample can - # catch a worker between round trips, so only call a worker wedged if - # it stays off the server for a whole confirmation window. + # opposite orders, which no timeout ever clears. A single sample can + # catch a worker between round trips, so only call a worker wedged if it + # stays off the server for a whole confirmation window. + # + # Workers doing off-session work are left out entirely, in both + # directions. `last_status` tracks session round trips, and a scenario + # worker driving the composition makes none for as long as its action + # takes: BackupRestoreAction sleeps up to 240s and then runs a CRDB + # backup, a restore and a blocking `up("materialized")` that waits on + # environmentd's health check, which together outlast the 300s join + # window. Reading its stale status would report a healthy backup as a + # workload deadlock. Reporting it as on the server instead would be just + # as wrong the other way, because one worker on the server clears the + # verdict for all of them, so a real deadlock elsewhere would go + # unreported for the whole scenario. # # A tolerated server-side hang also explains every other worker: the # hung statement holds its actions' object locks (an `ALTER SINK ... @@ -451,19 +481,22 @@ def run( # diagnosed when no worker is on the server at all. Waiters and # deadlocks both resolve when the statement does, and calling a waiter # deadlocked costs the run its final checks. + def session_status(worker: Worker, thread: threading.Thread) -> str | None: + """This worker's last session status, or None when it has none worth + reading: already exited, not connected yet, or off-session work.""" + if not thread.is_alive() or worker.exe is None or worker.off_session_work: + return None + return worker.exe.last_status + def not_on_server() -> set[str]: return { t.name for w, t in alive - if t.is_alive() and w.exe is not None and w.exe.last_status != "running" + if (status := session_status(w, t)) is not None and status != "running" } def on_server() -> set[str]: - return { - t.name - for w, t in alive - if t.is_alive() and w.exe is not None and w.exe.last_status == "running" - } + return {t.name for w, t in alive if session_status(w, t) == "running"} wedged = not_on_server() confirm_until = time.time() + 60 diff --git a/misc/python/materialize/parallel_workload/settings.py b/misc/python/materialize/parallel_workload/settings.py index 4efdd32a3876e..8b1d306501944 100644 --- a/misc/python/materialize/parallel_workload/settings.py +++ b/misc/python/materialize/parallel_workload/settings.py @@ -58,14 +58,6 @@ def _missing_(cls, value): # it on outside that scenario is harmless: no Parallel Workload codegen # emits `repeat_row` unless the scenario is active. "enable_repeat_row": "true", - # TODO: Reenable once the frontend-peek path stops acquiring a read hold - # whose `since` has compacted past the chosen `as_of`. The soft-assert - # assert_read_holds_correct (frontend_peek.rs:1759) fires for peeks in - # multi-statement transactions ("... read hold at .. is not enough for - # as_of .."), panicking the coordinator. Peeks still work via the classic - # coordinator path with this off. - # See https://linear.app/materializeinc/issue/SQL-520 - "enable_frontend_peek_sequencing": "false", # 64 MiB, down from the 1 GiB default. A peek's result is materialized in # memory on the replica serving it before this bound errors it # (`to_error_if_exceeds`, compute_state.rs), and measured on a workload-like diff --git a/misc/python/materialize/parallel_workload/worker.py b/misc/python/materialize/parallel_workload/worker.py index f4eb99f0737a1..312394c226ee2 100644 --- a/misc/python/materialize/parallel_workload/worker.py +++ b/misc/python/materialize/parallel_workload/worker.py @@ -42,6 +42,13 @@ class Worker: ignored_errors: defaultdict[str, Counter[type[Action]]] composition: Composition | None occurred_exception: Exception | None + # Set for a worker whose action spends long stretches driving the mzcompose + # composition rather than its own session: sleeping, killing and restarting + # environmentd, backing up and restoring. `Executor.last_status` only tracks + # session round trips, so for such a worker it says nothing at all, and the + # end-of-run wedge check has to leave it out rather than read its stale + # status as evidence of a deadlock. + off_session_work: bool def __init__( self, @@ -53,12 +60,14 @@ def __init__( system: bool, composition: Composition | None, action_list: ActionList | None = None, + off_session_work: bool = False, ): self.rng = rng self.action_list = action_list self.actions = actions self.weights = weights self.end_time = end_time + self.off_session_work = off_session_work self.num_queries = Counter() # Unlike num_queries, these are never cleared: they feed the # end-of-run action coverage check. diff --git a/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py b/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py index 1f9688ed59558..9a06efe6a791c 100644 --- a/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py +++ b/misc/python/materialize/postgres_consistency/ignore_filter/pg_inconsistency_ignore_filter.py @@ -35,7 +35,6 @@ matches_op_by_any_pattern, matches_op_by_pattern, matches_x_and_y, - matches_x_or_y, ) from materialize.output_consistency.ignore_filter.ignore_verdict import ( IgnoreVerdict, @@ -267,7 +266,7 @@ def _matches_problematic_function_invocation( if db_function.function_name_in_lower_case == "replace": # replace is not working properly with empty text; however, it is not possible to reliably determine if an # expression is an empty text, we therefore need to exclude the function completely - return YesIgnore("database-issues#6619: replace") + return YesIgnore("DB-189: replace") if db_function.function_name_in_lower_case == "regexp_replace": if expression.args[2].has_any_characteristic( @@ -992,17 +991,6 @@ def matches_nullif(expression: Expression) -> bool: ): return YesIgnore("database-issues#6623: mod") - if query_template.matches_specific_select_or_filter_expression( - col_index, - partial( - matches_x_or_y, - x=partial(matches_fun_by_name, function_name_in_lower_case="date_part"), - y=partial(matches_op_by_pattern, pattern="EXTRACT($ FROM $)"), - ), - True, - ): - return YesIgnore("database-issues#7089") - if query_template.matches_specific_select_or_filter_expression( col_index, matches_math_op_with_large_or_tiny_val, True ): diff --git a/misc/python/materialize/workload_replay/executor.py b/misc/python/materialize/workload_replay/executor.py index c28f05f0f6507..c5fe4d8075c07 100644 --- a/misc/python/materialize/workload_replay/executor.py +++ b/misc/python/materialize/workload_replay/executor.py @@ -19,6 +19,7 @@ import threading import time from collections import defaultdict +from collections.abc import Callable from typing import Any from materialize.docker import image_registry @@ -93,8 +94,13 @@ def test( run_ingestions: bool, run_queries: bool, max_concurrent_queries: int, + during_continuous: Callable[[], None] | None = None, ) -> dict[str, Any]: - """Run a single workload test.""" + """Run a single workload test. + + When `during_continuous` is set, the continuous phase runs until the + callback returns instead of for `runtime` seconds. + """ print(f"--- {posixpath.relpath(file, LOCATION)}") services = set() @@ -272,12 +278,17 @@ def test( thread.start() try: - stop_event.wait(timeout=runtime) + if during_continuous is not None: + during_continuous() + else: + stop_event.wait(timeout=runtime) finally: stop_event.set() for thread in threads: thread.join() print_replay_stats(stats) + elif during_continuous is not None: + during_continuous() else: print("No continuous ingestions or queries defined, skipping phase") diff --git a/misc/scratch/provision.bash b/misc/scratch/provision.bash index 8b17b13267dd9..3d062569f0bfc 100644 --- a/misc/scratch/provision.bash +++ b/misc/scratch/provision.bash @@ -79,12 +79,10 @@ rm -rf /tmp/awscli.zip /tmp/aws) & # uv (curl -fsSL https://astral.sh/uv/install.sh | sudo -u ubuntu sh >/dev/null 2>&1) & -# kubectl + kind + k9s. kind and kubectl mirror ci/builder/Dockerfile, which -# deliberately holds them back: newer versions break Cloudtest (minio fails to -# come up on the kind cluster). -(curl -fsSL "https://dl.k8s.io/release/v1.24.3/bin/linux/$ARCH_GO/kubectl" -o /usr/local/bin/kubectl +# kubectl + kind + k9s. kind and kubectl mirror ci/builder/Dockerfile. +(curl -fsSL "https://dl.k8s.io/release/v1.34.10/bin/linux/$ARCH_GO/kubectl" -o /usr/local/bin/kubectl chmod +x /usr/local/bin/kubectl) & -(curl -fsSL "https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-$ARCH_GO" -o /usr/local/bin/kind +(curl -fsSL "https://kind.sigs.k8s.io/dl/v0.32.0/kind-linux-$ARCH_GO" -o /usr/local/bin/kind chmod +x /usr/local/bin/kind) & (curl -fsSL "https://github.com/derailed/k9s/releases/download/v0.50.18/k9s_Linux_$ARCH_GO.tar.gz" \ | tar xzf - -C /usr/local/bin k9s diff --git a/src/adapter/src/catalog.rs b/src/adapter/src/catalog.rs index 8b3a1f919da38..c66df7dd95262 100644 --- a/src/adapter/src/catalog.rs +++ b/src/adapter/src/catalog.rs @@ -32,7 +32,9 @@ use mz_catalog::builtin::{ BUILTIN_PREFIXES, BuiltinCluster, BuiltinLog, BuiltinSource, BuiltinTable, MZ_CATALOG_SERVER_CLUSTER, }; -use mz_catalog::config::{BuiltinItemMigrationConfig, ClusterReplicaSizeMap, Config, StateConfig}; +use mz_catalog::config::{ + AwsPrincipalContext, BuiltinItemMigrationConfig, ClusterReplicaSizeMap, Config, StateConfig, +}; #[cfg(test)] use mz_catalog::durable::CatalogError; use mz_catalog::durable::{ @@ -45,6 +47,7 @@ use mz_catalog::memory::objects::{ CatalogCollectionEntry, CatalogEntry, CatalogItem, Cluster, ClusterReplica, Database, NetworkPolicy, Role, RoleAuth, Schema, }; +use mz_cloud_resources::AwsExternalIdPrefix; use mz_compute_types::dataflows::DataflowDescription; use mz_controller::clusters::ReplicaLocation; use mz_controller_types::{ClusterId, ReplicaId}; @@ -333,6 +336,23 @@ impl ConnectionResolver for ConnCatalog<'_> { } } +/// AWS environment context for a debug catalog. +/// +/// The `mz_aws_connections` and `mz_aws_privatelink_connections` builtin +/// materialized views fold `mz_aws_account_id()`, `mz_aws_external_id_prefix()`, +/// and `mz_aws_connection_role_arn()` into their optimized expressions. A debug +/// catalog opened to be compared against a live environment (the testdrive +/// consistency check) must fold those functions to the same values the live +/// environment used, otherwise the optimized expressions diverge. This mirrors +/// how `environment_id` is threaded into the catalog copy. Fields left `None` +/// fold to SQL NULL, matching an environment without that context. +#[derive(Debug, Clone, Default)] +pub struct DebugAwsContext { + pub aws_account_id: Option, + pub aws_external_id_prefix: Option, + pub aws_connection_role_arn: Option, +} + impl Catalog { /// Returns the catalog's transient revision, which starts at 1 and is /// incremented on every change. This is not persisted to disk, and will @@ -378,6 +398,28 @@ impl Catalog { f(catalog).await } + /// Like [`Catalog::with_debug`], but folds the given AWS context into the + /// catalog. Used to exercise builtin materialized views that reproduce AWS + /// environment context in SQL (see [`DebugAwsContext`]). + pub async fn with_debug_aws_context(aws_context: DebugAwsContext, f: F) -> T + where + F: FnOnce(Catalog) -> Fut, + Fut: Future, + { + let persist_client = PersistClient::new_for_tests().await; + let organization_id = Uuid::new_v4(); + let bootstrap_args = test_bootstrap_args(); + let catalog = Self::open_debug_catalog_with_aws_context( + persist_client, + organization_id, + &bootstrap_args, + Some(aws_context), + ) + .await + .expect("can open debug catalog"); + f(catalog).await + } + /// Like [`Catalog::with_debug`], but the catalog created believes that bootstrap is still /// in progress. pub async fn with_debug_in_bootstrap(f: F) -> T @@ -441,6 +483,44 @@ impl Catalog { system_parameter_defaults, bootstrap_args, None, + None, + ) + .await + } + + /// Like [`Catalog::open_debug_catalog`], but folds `aws_context` into the + /// catalog (see [`DebugAwsContext`]). + /// + /// Calls `open_debug_catalog_inner` directly rather than delegating through + /// [`Catalog::open_debug_catalog`], to avoid adding an `async fn` layer to + /// the returned future. The catalog-open future is close to the crate + /// `recursion_limit`, and the extra nesting overflows it when computing the + /// layout of callers such as the `catalog` benchmark. + pub async fn open_debug_catalog_with_aws_context( + persist_client: PersistClient, + organization_id: Uuid, + bootstrap_args: &BootstrapArgs, + aws_context: Option, + ) -> Result { + let now = SYSTEM_TIME.clone(); + let environment_id = None; + let openable_storage = TestCatalogStateBuilder::new(persist_client.clone()) + .with_organization_id(organization_id) + .with_default_deploy_generation() + .build() + .await?; + let storage = openable_storage.open(now().into(), bootstrap_args).await?; + let system_parameter_defaults = BTreeMap::default(); + Self::open_debug_catalog_inner( + persist_client, + storage, + now, + environment_id, + &DUMMY_BUILD_INFO, + system_parameter_defaults, + bootstrap_args, + None, + aws_context, ) .await } @@ -473,6 +553,7 @@ impl Catalog { system_parameter_defaults, bootstrap_args, None, + None, ) .await } @@ -489,6 +570,7 @@ impl Catalog { build_info: &'static BuildInfo, bootstrap_args: &BootstrapArgs, enable_expression_cache_override: Option, + aws_context: Option, ) -> Result { let openable_storage = TestCatalogStateBuilder::new(persist_client.clone()) .with_organization_id(environment_id.organization_id()) @@ -510,6 +592,7 @@ impl Catalog { system_parameter_defaults, bootstrap_args, enable_expression_cache_override, + aws_context, ) .await } @@ -523,6 +606,7 @@ impl Catalog { system_parameter_defaults: BTreeMap, bootstrap_args: &BootstrapArgs, enable_expression_cache_override: Option, + aws_context: Option, ) -> Result { let metrics_registry = &MetricsRegistry::new(); let secrets_reader = Arc::new(InMemorySecretsController::new()); @@ -532,6 +616,37 @@ impl Catalog { let replica_size = &bootstrap_args.default_cluster_replica_size; let read_only = false; + // Fold the requested AWS context into the connection context and + // principal context. The builtin AWS connection views reproduce these + // values in SQL, so a catalog compared against a live environment must + // resolve them identically. When `aws_context` is `None` the default + // `for_tests` context is used unchanged. + let mut connection_context = ConnectionContext::for_tests(secrets_reader); + let aws_principal_context = match aws_context { + None => None, + Some(aws_context) => { + connection_context.aws_external_id_prefix = + aws_context.aws_external_id_prefix.as_deref().map(|prefix| { + AwsExternalIdPrefix::new_from_cli_argument_or_environment_variable(prefix) + .expect("infallible") + }); + connection_context.aws_connection_role_arn = aws_context.aws_connection_role_arn; + aws_context.aws_account_id.map(|aws_account_id| { + AwsPrincipalContext { + aws_account_id, + // Only `aws_account_id` reaches the `mz_aws_account_id()` + // fold. The external id prefix is read from the connection + // context above, so any placeholder works here. + aws_external_id_prefix: + AwsExternalIdPrefix::new_from_cli_argument_or_environment_variable( + "debug", + ) + .expect("infallible"), + } + }) + } + }; + let OpenCatalogResult { catalog, migrated_storage_collections_0dt: _, @@ -576,10 +691,10 @@ impl Catalog { remote_system_parameters: None, availability_zones: vec![], egress_addresses: vec![], - aws_principal_context: None, + aws_principal_context, aws_privatelink_availability_zones: None, http_host_name: None, - connection_context: ConnectionContext::for_tests(secrets_reader), + connection_context, builtin_item_migration_config: BuiltinItemMigrationConfig { persist_client: persist_client.clone(), read_only, @@ -2416,7 +2531,7 @@ mod tests { use mz_sql::session::vars::{SystemVars, VarInput}; use crate::catalog::state::LocalExpressionCache; - use crate::catalog::{Catalog, Op}; + use crate::catalog::{Catalog, DebugAwsContext, Op}; use crate::optimize::dataflows::{EvalTime, ExprPrep, ExprPrepOneShot}; use crate::session::Session; @@ -2845,7 +2960,20 @@ mod tests { #[mz_ore::test(tokio::test)] #[cfg_attr(miri, ignore)] // unsupported operation: can't call foreign function `TLS_client_method` on OS `linux` async fn verify_builtin_descs() { - Catalog::with_debug(|catalog| async move { + // Provide a full AWS context so the builtin AWS connection views resolve + // as they do in a cloud environment. Without an account id, + // `mz_aws_privatelink_connections.principal` folds to NULL, its + // `WHERE principal IS NOT NULL` filter is statically unsatisfiable, and + // the optimizer infers an extra empty key that the declared desc, which + // describes the cloud shape, does not carry. + let aws_context = DebugAwsContext { + aws_account_id: Some("123456789000".to_string()), + aws_external_id_prefix: Some("eb5cb59b-e2fe-41f3-87ca-d2176a495345".to_string()), + aws_connection_role_arn: Some( + "arn:aws:iam::123456789000:role/MaterializeConnection".to_string(), + ), + }; + Catalog::with_debug_aws_context(aws_context, |catalog| async move { let conn_catalog = catalog.for_system_session(); for builtin in BUILTINS::iter() { diff --git a/src/adapter/src/catalog/builtin_table_updates.rs b/src/adapter/src/catalog/builtin_table_updates.rs index 7c100ef46ef30..24ea3082ccf43 100644 --- a/src/adapter/src/catalog/builtin_table_updates.rs +++ b/src/adapter/src/catalog/builtin_table_updates.rs @@ -15,22 +15,19 @@ use mz_adapter_types::compaction::CompactionWindow; use mz_audit_log::VersionedStorageUsage; use mz_catalog::SYSTEM_CONN_ID; use mz_catalog::builtin::{ - BuiltinTable, MZ_AGGREGATES, MZ_ARRAY_TYPES, MZ_AWS_CONNECTIONS, - MZ_AWS_PRIVATELINK_CONNECTIONS, MZ_BASE_TYPES, MZ_CLUSTER_REPLICA_SIZE_INTERNAL, + BuiltinTable, MZ_AGGREGATES, MZ_ARRAY_TYPES, MZ_BASE_TYPES, MZ_CLUSTER_REPLICA_SIZE_INTERNAL, MZ_CLUSTER_REPLICA_SIZES, MZ_COLUMNS, MZ_EGRESS_IPS, MZ_FUNCTIONS, - MZ_HISTORY_RETENTION_STRATEGIES, MZ_ICEBERG_SINKS, MZ_INDEX_COLUMNS, MZ_KAFKA_CONNECTIONS, - MZ_KAFKA_SINKS, MZ_LICENSE_KEYS, MZ_LIST_TYPES, MZ_MAP_TYPES, - MZ_MATERIALIZED_VIEW_REFRESH_STRATEGIES, MZ_OBJECT_DEPENDENCIES, MZ_OBJECT_GLOBAL_IDS, - MZ_OPERATORS, MZ_PSEUDO_TYPES, MZ_REPLACEMENTS, MZ_ROLE_AUTH, MZ_SESSIONS, MZ_SINKS, - MZ_SOURCE_REFERENCES, MZ_SSH_TUNNEL_CONNECTIONS, MZ_STORAGE_USAGE_BY_SHARD, MZ_SUBSCRIPTIONS, - MZ_TABLES, MZ_TYPE_PG_METADATA, MZ_TYPES, MZ_VIEWS, MZ_WEBHOOKS_SOURCES, + MZ_HISTORY_RETENTION_STRATEGIES, MZ_ICEBERG_SINKS, MZ_INDEX_COLUMNS, MZ_KAFKA_SINKS, + MZ_LICENSE_KEYS, MZ_LIST_TYPES, MZ_MAP_TYPES, MZ_MATERIALIZED_VIEW_REFRESH_STRATEGIES, + MZ_OBJECT_DEPENDENCIES, MZ_OBJECT_GLOBAL_IDS, MZ_OPERATORS, MZ_PSEUDO_TYPES, MZ_REPLACEMENTS, + MZ_ROLE_AUTH, MZ_SESSIONS, MZ_SINKS, MZ_SOURCE_REFERENCES, MZ_STORAGE_USAGE_BY_SHARD, + MZ_SUBSCRIPTIONS, MZ_TABLES, MZ_TYPE_PG_METADATA, MZ_TYPES, MZ_VIEWS, MZ_WEBHOOKS_SOURCES, }; -use mz_catalog::config::AwsPrincipalContext; use mz_catalog::durable::SourceReferences; use mz_catalog::memory::error::Error; use mz_catalog::memory::objects::{ - CatalogEntry, CatalogItem, Connection, DataSourceDesc, Func, Index, MaterializedView, Sink, - Table, TableDataSource, Type, View, + CatalogEntry, CatalogItem, DataSourceDesc, Func, Index, MaterializedView, Sink, Table, + TableDataSource, Type, View, }; use mz_expr::MirScalarExpr; use mz_license_keys::ValidatedLicenseKey; @@ -51,13 +48,8 @@ use mz_sql::ast::{CreateIndexStatement, Statement}; use mz_sql::catalog::{CatalogType, TypeCategory}; use mz_sql::func::FuncImplCatalogDetails; use mz_sql::names::SchemaSpecifier; -use mz_sql::plan::{ConnectionDetails, SshKey}; use mz_sql_parser::ast::display::AstDisplay; use mz_storage_client::client::TableData; -use mz_storage_types::connections::KafkaConnection; -use mz_storage_types::connections::aws::{AwsAuth, AwsConnection}; -use mz_storage_types::connections::inline::ReferencedConnection; -use mz_storage_types::connections::string_or_secret::StringOrSecret; use mz_storage_types::sinks::{IcebergSinkConnection, KafkaSinkConnection, StorageSinkConnection}; use smallvec::smallvec; @@ -210,9 +202,11 @@ impl CatalogState { self.pack_func_update(id, schema_id, name, owner_id, func, diff) } CatalogItem::Log(_) | CatalogItem::Secret(_) => vec![], - CatalogItem::Connection(connection) => { - self.pack_connection_update(id, connection, diff) - } + // Connection details (mz_kafka_connections, mz_ssh_tunnel_connections, + // mz_aws_connections, mz_aws_privatelink_connections) are now derived + // from the persisted create_sql by materialized views over + // mz_catalog_raw, so connections need no special packing here. + CatalogItem::Connection(_) => vec![], }; if !entry.item().is_temporary() { @@ -393,191 +387,6 @@ impl CatalogState { )] } - fn pack_connection_update( - &self, - id: CatalogItemId, - connection: &Connection, - diff: Diff, - ) -> Vec> { - let mut updates = vec![]; - match connection.details { - ConnectionDetails::Kafka(ref kafka) => { - updates.extend(self.pack_kafka_connection_update(id, kafka, diff)); - } - ConnectionDetails::Aws(ref aws_config) => { - match self.pack_aws_connection_update(id, aws_config, diff) { - Ok(update) => { - updates.push(update); - } - Err(e) => { - tracing::error!(%id, %e, "failed writing row to mz_aws_connections table"); - } - } - } - ConnectionDetails::AwsPrivatelink(_) => { - if let Some(aws_principal_context) = self.aws_principal_context.as_ref() { - updates.push(self.pack_aws_privatelink_connection_update( - id, - aws_principal_context, - diff, - )); - } else { - tracing::error!(%id, "missing AWS principal context; cannot write row to mz_aws_privatelink_connections table"); - } - } - ConnectionDetails::Ssh { - ref key_1, - ref key_2, - .. - } => { - updates.push(self.pack_ssh_tunnel_connection_update(id, key_1, key_2, diff)); - } - ConnectionDetails::Csr(_) - | ConnectionDetails::GlueSchemaRegistry(_) - | ConnectionDetails::Gcp(_) - | ConnectionDetails::Postgres(_) - | ConnectionDetails::MySql(_) - | ConnectionDetails::SqlServer(_) - | ConnectionDetails::IcebergCatalog(_) => (), - }; - updates - } - - pub(crate) fn pack_ssh_tunnel_connection_update( - &self, - id: CatalogItemId, - key_1: &SshKey, - key_2: &SshKey, - diff: Diff, - ) -> BuiltinTableUpdate<&'static BuiltinTable> { - BuiltinTableUpdate::row( - &*MZ_SSH_TUNNEL_CONNECTIONS, - Row::pack_slice(&[ - Datum::String(&id.to_string()), - Datum::String(key_1.public_key().as_str()), - Datum::String(key_2.public_key().as_str()), - ]), - diff, - ) - } - - fn pack_kafka_connection_update( - &self, - id: CatalogItemId, - kafka: &KafkaConnection, - diff: Diff, - ) -> Vec> { - let progress_topic = kafka.progress_topic(&self.config.connection_context, id); - let mut row = Row::default(); - row.packer() - .try_push_array( - &[ArrayDimension { - lower_bound: 1, - length: kafka.brokers.len(), - }], - kafka - .brokers - .iter() - .map(|broker| Datum::String(&broker.address)), - ) - .expect("kafka.brokers is 1 dimensional, and its length is used for the array length"); - let brokers = row.unpack_first(); - vec![BuiltinTableUpdate::row( - &*MZ_KAFKA_CONNECTIONS, - Row::pack_slice(&[ - Datum::String(&id.to_string()), - brokers, - Datum::String(&progress_topic), - ]), - diff, - )] - } - - pub fn pack_aws_privatelink_connection_update( - &self, - connection_id: CatalogItemId, - aws_principal_context: &AwsPrincipalContext, - diff: Diff, - ) -> BuiltinTableUpdate<&'static BuiltinTable> { - let id = &MZ_AWS_PRIVATELINK_CONNECTIONS; - let row = Row::pack_slice(&[ - Datum::String(&connection_id.to_string()), - Datum::String(&aws_principal_context.to_principal_string(connection_id)), - ]); - BuiltinTableUpdate::row(id, row, diff) - } - - pub fn pack_aws_connection_update( - &self, - connection_id: CatalogItemId, - aws_config: &AwsConnection, - diff: Diff, - ) -> Result, anyhow::Error> { - let id = &MZ_AWS_CONNECTIONS; - - let mut access_key_id = None; - let mut access_key_id_secret_id = None; - let mut secret_access_key_secret_id = None; - let mut session_token = None; - let mut session_token_secret_id = None; - let mut assume_role_arn = None; - let mut assume_role_session_name = None; - let mut principal = None; - let mut external_id = None; - let mut example_trust_policy = None; - match &aws_config.auth { - AwsAuth::Credentials(credentials) => { - match &credentials.access_key_id { - StringOrSecret::String(s) => access_key_id = Some(s.as_str()), - StringOrSecret::Secret(s) => access_key_id_secret_id = Some(s.to_string()), - } - secret_access_key_secret_id = Some(credentials.secret_access_key.to_string()); - match credentials.session_token.as_ref() { - None => (), - Some(StringOrSecret::String(s)) => session_token = Some(s.as_str()), - Some(StringOrSecret::Secret(s)) => { - session_token_secret_id = Some(s.to_string()) - } - } - } - AwsAuth::AssumeRole(assume_role) => { - assume_role_arn = Some(assume_role.arn.as_str()); - assume_role_session_name = assume_role.session_name.as_deref(); - principal = self - .config - .connection_context - .aws_connection_role_arn - .as_deref(); - external_id = - Some(assume_role.external_id(&self.config.connection_context, connection_id)?); - example_trust_policy = { - let policy = assume_role - .example_trust_policy(&self.config.connection_context, connection_id)?; - let policy = Jsonb::from_serde_json(policy).expect("valid json"); - Some(policy.into_row()) - }; - } - } - - let row = Row::pack_slice(&[ - Datum::String(&connection_id.to_string()), - Datum::from(aws_config.endpoint.as_deref()), - Datum::from(aws_config.region.as_deref()), - Datum::from(access_key_id), - Datum::from(access_key_id_secret_id.as_deref()), - Datum::from(secret_access_key_secret_id.as_deref()), - Datum::from(session_token), - Datum::from(session_token_secret_id.as_deref()), - Datum::from(assume_role_arn), - Datum::from(assume_role_session_name), - Datum::from(principal), - Datum::from(external_id.as_deref()), - Datum::from(example_trust_policy.as_ref().map(|p| p.into_element())), - ]); - - Ok(BuiltinTableUpdate::row(id, row, diff)) - } - fn pack_view_update( &self, id: CatalogItemId, diff --git a/src/adapter/src/catalog/open.rs b/src/adapter/src/catalog/open.rs index 57e99d313dda8..afe4f8f1729c7 100644 --- a/src/adapter/src/catalog/open.rs +++ b/src/adapter/src/catalog/open.rs @@ -166,6 +166,10 @@ impl Catalog { build_info: config.build_info, now: config.now.clone(), connection_context: config.connection_context, + aws_account_id: config + .aws_principal_context + .as_ref() + .map(|c| c.aws_account_id.clone()), helm_chart_version: config.helm_chart_version, }, cluster_replica_sizes: config.cluster_replica_sizes, diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs index 77da28cefcd23..58b4ec995a88a 100644 --- a/src/adapter/src/catalog/open/builtin_schema_migration.rs +++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs @@ -328,6 +328,33 @@ static MIGRATIONS: LazyLock> = LazyLock::new(|| { MZ_INTERNAL_SCHEMA, "mz_kafka_source_tables", ), + // Converting the connection-detail builtin tables to materialized views + // changes their catalog fingerprint, so each needs an explicit + // replacement step. + MigrationStep::replacement( + "26.37.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_kafka_connections", + ), + MigrationStep::replacement( + "26.37.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_ssh_tunnel_connections", + ), + MigrationStep::replacement( + "26.37.0-dev.0", + CatalogItemType::MaterializedView, + MZ_INTERNAL_SCHEMA, + "mz_aws_connections", + ), + MigrationStep::replacement( + "26.37.0-dev.0", + CatalogItemType::MaterializedView, + MZ_CATALOG_SCHEMA, + "mz_aws_privatelink_connections", + ), ] }); diff --git a/src/adapter/src/catalog/state.rs b/src/adapter/src/catalog/state.rs index d4932ebe883be..63a704a30e6da 100644 --- a/src/adapter/src/catalog/state.rs +++ b/src/adapter/src/catalog/state.rs @@ -323,6 +323,7 @@ impl CatalogState { connection_context: ConnectionContext::for_tests(Arc::new( InMemorySecretsController::new(), )), + aws_account_id: None, helm_chart_version: None, }, cluster_replica_sizes: ClusterReplicaSizeMap::for_tests(), diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs index 2ec708348e946..3dfb9e2ae8774 100644 --- a/src/catalog/src/builtin.rs +++ b/src/catalog/src/builtin.rs @@ -1108,7 +1108,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::Log(&MZ_COMPUTE_HYDRATION_TIMES_PER_WORKER), Builtin::Log(&MZ_COMPUTE_OPERATOR_HYDRATION_STATUSES_PER_WORKER), Builtin::Table(&MZ_KAFKA_SINKS), - Builtin::Table(&MZ_KAFKA_CONNECTIONS), + Builtin::MaterializedView(&MZ_KAFKA_CONNECTIONS), Builtin::MaterializedView(&MZ_KAFKA_SOURCES), Builtin::Table(&MZ_OBJECT_DEPENDENCIES), Builtin::Table(&MZ_ICEBERG_SINKS), @@ -1150,7 +1150,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::MaterializedView(&MZ_CLUSTER_AUTO_SCALING_STRATEGIES), Builtin::MaterializedView(&MZ_SECRETS), Builtin::MaterializedView(&MZ_CONNECTIONS), - Builtin::Table(&MZ_SSH_TUNNEL_CONNECTIONS), + Builtin::MaterializedView(&MZ_SSH_TUNNEL_CONNECTIONS), Builtin::MaterializedView(&MZ_CLUSTER_REPLICAS), Builtin::Source(&MZ_CLUSTER_REPLICA_METRICS_HISTORY), Builtin::View(&MZ_CLUSTER_REPLICA_METRICS), @@ -1161,8 +1161,8 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne Builtin::MaterializedView(&MZ_AUDIT_EVENTS), Builtin::Table(&MZ_STORAGE_USAGE_BY_SHARD), Builtin::Table(&MZ_EGRESS_IPS), - Builtin::Table(&MZ_AWS_PRIVATELINK_CONNECTIONS), - Builtin::Table(&MZ_AWS_CONNECTIONS), + Builtin::MaterializedView(&MZ_AWS_PRIVATELINK_CONNECTIONS), + Builtin::MaterializedView(&MZ_AWS_CONNECTIONS), Builtin::Table(&MZ_SUBSCRIPTIONS), Builtin::Table(&MZ_SESSIONS), Builtin::MaterializedView(&MZ_OVERRIDDEN_SYSTEM_PARAMETERS), diff --git a/src/catalog/src/builtin/builtin.rs b/src/catalog/src/builtin/builtin.rs index 51834a5f581f5..e62910c4ab1fe 100644 --- a/src/catalog/src/builtin/builtin.rs +++ b/src/catalog/src/builtin/builtin.rs @@ -338,7 +338,7 @@ SELECT * FROM builtin_sources"); ("size", "*Deprecated* The size of the source."), ( "envelope_type", - "For Kafka sources, the envelope type: `none`, `upsert`, or `debezium`. `NULL` for other source types.", + "For old-syntax Kafka sources, the envelope type: `none`, `upsert`, or `debezium`. `NULL` for new-syntax Kafka sources, whose envelopes are defined per source table (see `mz_kafka_source_tables`), and for other source types.", ), ( "key_format", diff --git a/src/catalog/src/builtin/mz_catalog.rs b/src/catalog/src/builtin/mz_catalog.rs index 3de16050b1f21..c2fb6d2a75be2 100644 --- a/src/catalog/src/builtin/mz_catalog.rs +++ b/src/catalog/src/builtin/mz_catalog.rs @@ -262,43 +262,88 @@ pub static MZ_KAFKA_SINKS: LazyLock = LazyLock::new(|| BuiltinTabl column_semantic_types: &[("id", SemanticType::CatalogItemId)], }), }); -pub static MZ_KAFKA_CONNECTIONS: LazyLock = LazyLock::new(|| BuiltinTable { - name: "mz_kafka_connections", - schema: MZ_CATALOG_SCHEMA, - oid: oid::TABLE_MZ_KAFKA_CONNECTIONS_OID, - desc: RelationDesc::builder() - .with_column("id", SqlScalarType::String.nullable(false)) - .with_column( - "brokers", - SqlScalarType::Array(Box::new(SqlScalarType::String)).nullable(false), - ) - .with_column("sink_progress_topic", SqlScalarType::String.nullable(false)) - .finish(), - column_comments: BTreeMap::from_iter([ - ("id", "The ID of the connection."), - ( - "brokers", - "The addresses of the Kafka brokers to connect to.", - ), - ( - "sink_progress_topic", - "The name of the Kafka topic where any sinks associated with this connection will track their progress information and other metadata. The contents of this topic are unspecified.", - ), - ]), - is_retained_metrics_object: false, - access: vec![PUBLIC_SELECT], - ontology: Some(Ontology { - entity_name: "kafka_connection", - description: "Kafka-specific connection configuration (brokers, progress topic)", - links: &const { - [OntologyLink { - name: "details_of", - target: "connection", - properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), - }] - }, - column_semantic_types: &[("id", SemanticType::CatalogItemId)], - }), +// Reads Item rows from `mz_catalog_raw`, pulls the broker addresses and any +// explicit progress topic out of the persisted `create_sql` via +// `parse_connection_details`, and keeps only kafka connections (the connection +// type is derived the same way `mz_connections` derives it). The default +// progress topic is reconstructed here rather than in the helper because it +// needs the environment id and the connection's own id. +pub static MZ_KAFKA_CONNECTIONS: LazyLock = LazyLock::new(|| { + BuiltinMaterializedView { + name: "mz_kafka_connections", + schema: MZ_CATALOG_SCHEMA, + oid: oid::MV_MZ_KAFKA_CONNECTIONS_OID, + desc: RelationDesc::builder() + .with_column("id", SqlScalarType::String.nullable(false)) + .with_column( + "brokers", + SqlScalarType::Array(Box::new(SqlScalarType::String)).nullable(false), + ) + .with_column("sink_progress_topic", SqlScalarType::String.nullable(false)) + .with_key(vec![0]) + .finish(), + column_comments: BTreeMap::from_iter([ + ("id", "The ID of the connection."), + ( + "brokers", + "The addresses of the Kafka brokers to connect to.", + ), + ( + "sink_progress_topic", + "The name of the Kafka topic where any sinks associated with this connection will track their progress information and other metadata. The contents of this topic are unspecified.", + ), + ]), + // The `sink_progress_topic` default must match + // `KafkaConnection::progress_topic` in + // src/storage-types/src/connections.rs. Keep the two in sync. + sql: " +IN CLUSTER mz_catalog_server +WITH ( + ASSERT NOT NULL id, + ASSERT NOT NULL brokers, + ASSERT NOT NULL sink_progress_topic +) AS +SELECT + mz_internal.parse_catalog_id(r.data->'key'->'gid') AS id, + ARRAY( + SELECT b.value + FROM jsonb_array_elements_text(details->'brokers') + WITH ORDINALITY AS b(value, ord) + ORDER BY b.ord + ) AS brokers, + COALESCE( + details->>'progress_topic', + '_materialize-progress-' || mz_environment_id() || '-' + || mz_internal.parse_catalog_id(r.data->'key'->'gid') + ) AS sink_progress_topic +FROM + mz_internal.mz_catalog_raw r, + LATERAL ( + SELECT mz_internal.parse_connection_details( + r.data->'value'->'definition'->'V1'->>'create_sql') + ) AS d(details) +WHERE + r.data->>'kind' = 'Item' AND + -- The connection_type filter selects the kind. A non-matching row yields a + -- NULL connection_type and is dropped here, so no `details IS NOT NULL` is + -- needed (parse_connection_details returns jsonb null, which passes it). + mz_internal.parse_catalog_create_sql( + r.data->'value'->'definition'->'V1'->>'create_sql')->>'connection_type' = 'kafka'", + is_retained_metrics_object: false, + access: vec![PUBLIC_SELECT], + ontology: Some(Ontology { + entity_name: "kafka_connection", + description: "Kafka-specific connection configuration (brokers, progress topic)", + links: &const { + [OntologyLink { + name: "details_of", + target: "connection", + properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), + }] + }, + column_semantic_types: &[("id", SemanticType::CatalogItemId)], + }), + } }); pub static MZ_KAFKA_SOURCES: LazyLock = LazyLock::new(|| { BuiltinMaterializedView { @@ -1112,41 +1157,71 @@ WHERE } }); -pub static MZ_SSH_TUNNEL_CONNECTIONS: LazyLock = LazyLock::new(|| BuiltinTable { - name: "mz_ssh_tunnel_connections", - schema: MZ_CATALOG_SCHEMA, - oid: oid::TABLE_MZ_SSH_TUNNEL_CONNECTIONS_OID, - desc: RelationDesc::builder() - .with_column("id", SqlScalarType::String.nullable(false)) - .with_column("public_key_1", SqlScalarType::String.nullable(false)) - .with_column("public_key_2", SqlScalarType::String.nullable(false)) - .finish(), - column_comments: BTreeMap::from_iter([ - ("id", "The ID of the connection."), - ( - "public_key_1", - "The first public key associated with the SSH tunnel.", - ), - ( - "public_key_2", - "The second public key associated with the SSH tunnel.", - ), - ]), - is_retained_metrics_object: false, - access: vec![PUBLIC_SELECT], - ontology: Some(Ontology { - entity_name: "ssh_tunnel_connection", - description: "SSH tunnel connection with public keys", - links: &const { - [OntologyLink { - name: "details_of", - target: "connection", - properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), - }] - }, - column_semantic_types: &[("id", SemanticType::CatalogItemId)], - }), -}); +// The two SSH public keys are stored verbatim in the persisted `create_sql` +// (they are generated, not user-supplied, and explicitly not redacted), so the +// view reads them straight out via `parse_connection_details` and keeps only +// ssh-tunnel connections. +pub static MZ_SSH_TUNNEL_CONNECTIONS: LazyLock = + LazyLock::new(|| BuiltinMaterializedView { + name: "mz_ssh_tunnel_connections", + schema: MZ_CATALOG_SCHEMA, + oid: oid::MV_MZ_SSH_TUNNEL_CONNECTIONS_OID, + desc: RelationDesc::builder() + .with_column("id", SqlScalarType::String.nullable(false)) + .with_column("public_key_1", SqlScalarType::String.nullable(false)) + .with_column("public_key_2", SqlScalarType::String.nullable(false)) + .with_key(vec![0]) + .finish(), + column_comments: BTreeMap::from_iter([ + ("id", "The ID of the connection."), + ( + "public_key_1", + "The first public key associated with the SSH tunnel.", + ), + ( + "public_key_2", + "The second public key associated with the SSH tunnel.", + ), + ]), + sql: " +IN CLUSTER mz_catalog_server +WITH ( + ASSERT NOT NULL id, + ASSERT NOT NULL public_key_1, + ASSERT NOT NULL public_key_2 +) AS +SELECT + mz_internal.parse_catalog_id(r.data->'key'->'gid') AS id, + details->>'public_key_1' AS public_key_1, + details->>'public_key_2' AS public_key_2 +FROM + mz_internal.mz_catalog_raw r, + LATERAL ( + SELECT mz_internal.parse_connection_details( + r.data->'value'->'definition'->'V1'->>'create_sql') + ) AS d(details) +WHERE + r.data->>'kind' = 'Item' AND + -- The connection_type filter selects the kind. A non-matching row yields a + -- NULL connection_type and is dropped here, so no `details IS NOT NULL` is + -- needed (parse_connection_details returns jsonb null, which passes it). + mz_internal.parse_catalog_create_sql( + r.data->'value'->'definition'->'V1'->>'create_sql')->>'connection_type' = 'ssh-tunnel'", + is_retained_metrics_object: false, + access: vec![PUBLIC_SELECT], + ontology: Some(Ontology { + entity_name: "ssh_tunnel_connection", + description: "SSH tunnel connection with public keys", + links: &const { + [OntologyLink { + name: "details_of", + target: "connection", + properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), + }] + }, + column_semantic_types: &[("id", SemanticType::CatalogItemId)], + }), + }); // mz_sources is generated dynamically in BUILTINS_STATIC via builtin::make_mz_sources() // with builtin source/log entries inlined as VALUES. See builtin/builtin.rs. pub static MZ_SINKS: LazyLock = LazyLock::new(|| { @@ -2706,36 +2781,68 @@ pub static MZ_EGRESS_IPS: LazyLock = LazyLock::new(|| BuiltinTable }), }); -pub static MZ_AWS_PRIVATELINK_CONNECTIONS: LazyLock = - LazyLock::new(|| BuiltinTable { - name: "mz_aws_privatelink_connections", - schema: MZ_CATALOG_SCHEMA, - oid: oid::TABLE_MZ_AWS_PRIVATELINK_CONNECTIONS_OID, - desc: RelationDesc::builder() - .with_column("id", SqlScalarType::String.nullable(false)) - .with_column("principal", SqlScalarType::String.nullable(false)) - .finish(), - column_comments: BTreeMap::from_iter([ - ("id", "The ID of the connection."), - ( - "principal", - "The AWS Principal that Materialize will use to connect to the VPC endpoint.", - ), - ]), - is_retained_metrics_object: false, - access: vec![PUBLIC_SELECT], - ontology: Some(Ontology { - entity_name: "aws_privatelink_connection", - description: "AWS PrivateLink connection configuration", - links: &const { - [OntologyLink { - name: "details_of", - target: "connection", - properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), - }] - }, - column_semantic_types: &[("id", SemanticType::CatalogItemId)], - }), +// The privatelink principal is entirely context-derived (there is nothing to +// parse out of create_sql), so this view selects the aws-privatelink +// connections and reconstructs the principal ARN from the plan-time AWS context +// functions. `principal` is NOT NULL and the removed packer emitted no row when +// the AWS principal context was absent. A view cannot skip a row, but the +// context functions fold to NULL without the context, so `WHERE principal IS +// NOT NULL` reproduces that skip (zero rows on envs without the context). +pub static MZ_AWS_PRIVATELINK_CONNECTIONS: LazyLock = + LazyLock::new(|| { + BuiltinMaterializedView { + name: "mz_aws_privatelink_connections", + schema: MZ_CATALOG_SCHEMA, + oid: oid::MV_MZ_AWS_PRIVATELINK_CONNECTIONS_OID, + desc: RelationDesc::builder() + .with_column("id", SqlScalarType::String.nullable(false)) + .with_column("principal", SqlScalarType::String.nullable(false)) + .with_key(vec![0]) + .finish(), + column_comments: BTreeMap::from_iter([ + ("id", "The ID of the connection."), + ( + "principal", + "The AWS Principal that Materialize will use to connect to the VPC endpoint.", + ), + ]), + // `principal` reproduces `AwsPrincipalContext::to_principal_string` in + // src/catalog/src/config.rs. Keep the two in sync. + sql: " +IN CLUSTER mz_catalog_server +WITH ( + ASSERT NOT NULL id, + ASSERT NOT NULL principal +) AS +SELECT id, principal FROM ( + SELECT + mz_internal.parse_catalog_id(r.data->'key'->'gid') AS id, + 'arn:aws:iam::' || mz_aws_account_id() || ':role/mz_' + || mz_aws_external_id_prefix() || '_' + || mz_internal.parse_catalog_id(r.data->'key'->'gid') AS principal + FROM mz_internal.mz_catalog_raw r + WHERE + r.data->>'kind' = 'Item' AND + mz_internal.parse_catalog_create_sql( + r.data->'value'->'definition'->'V1'->>'create_sql')->>'connection_type' + = 'aws-privatelink' +) +WHERE principal IS NOT NULL", + is_retained_metrics_object: false, + access: vec![PUBLIC_SELECT], + ontology: Some(Ontology { + entity_name: "aws_privatelink_connection", + description: "AWS PrivateLink connection configuration", + links: &const { + [OntologyLink { + name: "details_of", + target: "connection", + properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), + }] + }, + column_semantic_types: &[("id", SemanticType::CatalogItemId)], + }), + } }); pub static MZ_CLUSTER_REPLICA_FRONTIERS: LazyLock = diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs index 9ad99a5e417b1..c84850ec10ed5 100644 --- a/src/catalog/src/builtin/mz_internal.rs +++ b/src/catalog/src/builtin/mz_internal.rs @@ -321,7 +321,7 @@ pub static MZ_KAFKA_SOURCE_TABLES: LazyLock = LazyLock: desc: RelationDesc::builder() .with_column("id", SqlScalarType::String.nullable(false)) .with_column("topic", SqlScalarType::String.nullable(false)) - .with_column("envelope_type", SqlScalarType::String.nullable(true)) + .with_column("envelope_type", SqlScalarType::String.nullable(false)) .with_column("key_format", SqlScalarType::String.nullable(true)) .with_column("value_format", SqlScalarType::String.nullable(true)) .with_key(vec![0]) @@ -334,7 +334,7 @@ pub static MZ_KAFKA_SOURCE_TABLES: LazyLock = LazyLock: ("topic", "The topic being ingested."), ( "envelope_type", - "The envelope type: `none`, `upsert`, or `debezium`. `NULL` for other source types.", + "The envelope type: `none`, `upsert`, or `debezium`. Defaults to `none` when the source table omits an explicit envelope.", ), ( "key_format", @@ -361,7 +361,10 @@ WITH ( SELECT mz_internal.parse_catalog_id(r.data->'key'->'gid') AS id, details->'external_reference'->>0 AS topic, - details->>'envelope_type' AS envelope_type, + -- Kafka defaults to ENVELOPE NONE when the clause is omitted. The parser + -- helper is source-type agnostic and reports NULL for that case, so default + -- to 'none' here, where the join has already scoped rows to kafka. + COALESCE(details->>'envelope_type', 'none') AS envelope_type, details->>'key_format' AS key_format, details->>'value_format' AS value_format FROM @@ -2582,96 +2585,179 @@ pub static MZ_STORAGE_USAGE_BY_SHARD: LazyLock = LazyLock::new(|| }), }); -pub static MZ_AWS_CONNECTIONS: LazyLock = LazyLock::new(|| BuiltinTable { - name: "mz_aws_connections", - schema: MZ_INTERNAL_SCHEMA, - oid: oid::TABLE_MZ_AWS_CONNECTIONS_OID, - desc: RelationDesc::builder() - .with_column("id", SqlScalarType::String.nullable(false)) - .with_column("endpoint", SqlScalarType::String.nullable(true)) - .with_column("region", SqlScalarType::String.nullable(true)) - .with_column("access_key_id", SqlScalarType::String.nullable(true)) - .with_column( - "access_key_id_secret_id", - SqlScalarType::String.nullable(true), - ) - .with_column( - "secret_access_key_secret_id", - SqlScalarType::String.nullable(true), - ) - .with_column("session_token", SqlScalarType::String.nullable(true)) - .with_column( - "session_token_secret_id", - SqlScalarType::String.nullable(true), +// Reads Item rows from `mz_catalog_raw`, pulls the create_sql-derived fields +// (endpoint, region, credentials, assume-role options) out via +// `parse_connection_details`, and keeps only aws connections. The three +// context-derived columns (principal, external_id, example_trust_policy) apply +// to assume-role connections only and are reconstructed here from the plan-time +// AWS context functions, because they depend on the environment, not on +// create_sql. On an environment without the AWS context an assume-role +// connection is dropped entirely (see the WHERE guard below), matching the +// packer this view replaces, which bailed and wrote no row in that case. This +// is not the same as folding the columns to NULL: `example_trust_policy` uses +// `jsonb_build_object`, which keeps null values, so a context-less row would +// carry a non-NULL policy object full of nulls rather than SQL NULL. +pub static MZ_AWS_CONNECTIONS: LazyLock = LazyLock::new(|| { + BuiltinMaterializedView { + name: "mz_aws_connections", + schema: MZ_INTERNAL_SCHEMA, + oid: oid::MV_MZ_AWS_CONNECTIONS_OID, + desc: RelationDesc::builder() + .with_column("id", SqlScalarType::String.nullable(false)) + .with_column("endpoint", SqlScalarType::String.nullable(true)) + .with_column("region", SqlScalarType::String.nullable(true)) + .with_column("access_key_id", SqlScalarType::String.nullable(true)) + .with_column( + "access_key_id_secret_id", + SqlScalarType::String.nullable(true), + ) + .with_column( + "secret_access_key_secret_id", + SqlScalarType::String.nullable(true), + ) + .with_column("session_token", SqlScalarType::String.nullable(true)) + .with_column( + "session_token_secret_id", + SqlScalarType::String.nullable(true), + ) + .with_column("assume_role_arn", SqlScalarType::String.nullable(true)) + .with_column( + "assume_role_session_name", + SqlScalarType::String.nullable(true), + ) + .with_column("principal", SqlScalarType::String.nullable(true)) + .with_column("external_id", SqlScalarType::String.nullable(true)) + .with_column("example_trust_policy", SqlScalarType::Jsonb.nullable(true)) + .with_key(vec![0]) + .finish(), + column_comments: BTreeMap::from_iter([ + ("id", "The ID of the connection."), + ("endpoint", "The value of the `ENDPOINT` option, if set."), + ("region", "The value of the `REGION` option, if set."), + ( + "access_key_id", + "The value of the `ACCESS KEY ID` option, if provided in line.", + ), + ( + "access_key_id_secret_id", + "The ID of the secret referenced by the `ACCESS KEY ID` option, if provided via a secret.", + ), + ( + "secret_access_key_secret_id", + "The ID of the secret referenced by the `SECRET ACCESS KEY` option, if set.", + ), + ( + "session_token", + "The value of the `SESSION TOKEN` option, if provided in line.", + ), + ( + "session_token_secret_id", + "The ID of the secret referenced by the `SESSION TOKEN` option, if provided via a secret.", + ), + ( + "assume_role_arn", + "The value of the `ASSUME ROLE ARN` option, if set.", + ), + ( + "assume_role_session_name", + "The value of the `ASSUME ROLE SESSION NAME` option, if set.", + ), + ( + "principal", + "The ARN of the AWS principal Materialize will use when assuming the provided role, if the connection is configured to use role assumption.", + ), + ( + "external_id", + "The external ID Materialize will use when assuming the provided role, if the connection is configured to use role assumption.", + ), + ( + "example_trust_policy", + "An example of an IAM role trust policy that allows this connection's principal and external ID to assume the role.", + ), + ]), + // `external_id` reproduces `AwsAssumeRole::external_id` and + // `example_trust_policy` reproduces `AwsAssumeRole::example_trust_policy` + // (both in src/storage-types/src/connections/aws.rs). Keep them in sync. + sql: " +IN CLUSTER mz_catalog_server +WITH ( + ASSERT NOT NULL id +) AS +SELECT + mz_internal.parse_catalog_id(r.data->'key'->'gid') AS id, + details->>'endpoint' AS endpoint, + details->>'region' AS region, + details->>'access_key_id' AS access_key_id, + details->>'access_key_id_secret_id' AS access_key_id_secret_id, + details->>'secret_access_key_secret_id' AS secret_access_key_secret_id, + details->>'session_token' AS session_token, + details->>'session_token_secret_id' AS session_token_secret_id, + details->>'assume_role_arn' AS assume_role_arn, + details->>'assume_role_session_name' AS assume_role_session_name, + CASE WHEN details->>'auth_kind' = 'assume-role' + THEN mz_aws_connection_role_arn() + END AS principal, + CASE WHEN details->>'auth_kind' = 'assume-role' + THEN 'mz_' || mz_aws_external_id_prefix() || '_' + || mz_internal.parse_catalog_id(r.data->'key'->'gid') + END AS external_id, + CASE WHEN details->>'auth_kind' = 'assume-role' + THEN jsonb_build_object( + 'Version', '2012-10-17', + 'Statement', jsonb_build_array(jsonb_build_object( + 'Effect', 'Allow', + 'Principal', jsonb_build_object('AWS', mz_aws_connection_role_arn()), + 'Action', 'sts:AssumeRole', + 'Condition', jsonb_build_object( + 'StringEquals', jsonb_build_object( + 'sts:ExternalId', + 'mz_' || mz_aws_external_id_prefix() || '_' + || mz_internal.parse_catalog_id(r.data->'key'->'gid') + ) + ) + )) ) - .with_column("assume_role_arn", SqlScalarType::String.nullable(true)) - .with_column( - "assume_role_session_name", - SqlScalarType::String.nullable(true), + END AS example_trust_policy +FROM + mz_internal.mz_catalog_raw r, + LATERAL ( + SELECT mz_internal.parse_connection_details( + r.data->'value'->'definition'->'V1'->>'create_sql') + ) AS d(details) +WHERE + r.data->>'kind' = 'Item' AND + -- The connection_type filter selects the kind. A non-matching row yields a + -- NULL connection_type and is dropped here, so no `details IS NOT NULL` is + -- needed (parse_connection_details returns jsonb null, which passes it). + mz_internal.parse_catalog_create_sql( + r.data->'value'->'definition'->'V1'->>'create_sql')->>'connection_type' = 'aws' AND + -- Drop assume-role connections when the AWS context is absent, matching the + -- packer this view replaces. `AwsAssumeRole::external_id` needs the external + -- ID prefix and `example_trust_policy` needs the connection role ARN. Either + -- one missing made the packer bail and write no row. Without this guard the + -- view would instead emit a row whose `example_trust_policy` is a non-NULL + -- JSON object full of nulls, because `jsonb_build_object` keeps null values. + ( + details->>'auth_kind' IS DISTINCT FROM 'assume-role' OR ( + mz_aws_external_id_prefix() IS NOT NULL AND + mz_aws_connection_role_arn() IS NOT NULL ) - .with_column("principal", SqlScalarType::String.nullable(true)) - .with_column("external_id", SqlScalarType::String.nullable(true)) - .with_column("example_trust_policy", SqlScalarType::Jsonb.nullable(true)) - .finish(), - column_comments: BTreeMap::from_iter([ - ("id", "The ID of the connection."), - ("endpoint", "The value of the `ENDPOINT` option, if set."), - ("region", "The value of the `REGION` option, if set."), - ( - "access_key_id", - "The value of the `ACCESS KEY ID` option, if provided in line.", - ), - ( - "access_key_id_secret_id", - "The ID of the secret referenced by the `ACCESS KEY ID` option, if provided via a secret.", - ), - ( - "secret_access_key_secret_id", - "The ID of the secret referenced by the `SECRET ACCESS KEY` option, if set.", - ), - ( - "session_token", - "The value of the `SESSION TOKEN` option, if provided in line.", - ), - ( - "session_token_secret_id", - "The ID of the secret referenced by the `SESSION TOKEN` option, if provided via a secret.", - ), - ( - "assume_role_arn", - "The value of the `ASSUME ROLE ARN` option, if set.", - ), - ( - "assume_role_session_name", - "The value of the `ASSUME ROLE SESSION NAME` option, if set.", - ), - ( - "principal", - "The ARN of the AWS principal Materialize will use when assuming the provided role, if the connection is configured to use role assumption.", - ), - ( - "external_id", - "The external ID Materialize will use when assuming the provided role, if the connection is configured to use role assumption.", - ), - ( - "example_trust_policy", - "An example of an IAM role trust policy that allows this connection's principal and external ID to assume the role.", - ), - ]), - is_retained_metrics_object: false, - access: vec![PUBLIC_SELECT], - ontology: Some(Ontology { - entity_name: "aws_connection", - description: "AWS connection configuration details", - links: &const { - [OntologyLink { - name: "details_of", - target: "connection", - properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), - }] - }, - column_semantic_types: &[], - }), + )", + is_retained_metrics_object: false, + access: vec![PUBLIC_SELECT], + ontology: Some(Ontology { + entity_name: "aws_connection", + description: "AWS connection configuration details", + links: &const { + [OntologyLink { + name: "details_of", + target: "connection", + properties: LinkProperties::fk("id", "id", Cardinality::OneToOne), + }] + }, + column_semantic_types: &[], + }), + } }); pub static MZ_CLUSTER_REPLICA_METRICS_HISTORY: LazyLock = diff --git a/src/catalog/src/config.rs b/src/catalog/src/config.rs index f96ff2a2b0405..b11bba066073e 100644 --- a/src/catalog/src/config.rs +++ b/src/catalog/src/config.rs @@ -312,6 +312,10 @@ pub struct AwsPrincipalContext { } impl AwsPrincipalContext { + // NOTE: the `mz_catalog.mz_aws_privatelink_connections` builtin materialized + // view reconstructs this ARN format in SQL from the mz_aws_account_id() and + // mz_aws_external_id_prefix() functions (see MZ_AWS_PRIVATELINK_CONNECTIONS + // in src/catalog/src/builtin/mz_catalog.rs). Keep the two in sync. pub fn to_principal_string(&self, aws_external_id_suffix: CatalogItemId) -> String { format!( "arn:aws:iam::{}:role/mz_{}_{}", diff --git a/src/environmentd/src/lib.rs b/src/environmentd/src/lib.rs index bef2349b25938..3d5cb5b85c354 100644 --- a/src/environmentd/src/lib.rs +++ b/src/environmentd/src/lib.rs @@ -844,6 +844,7 @@ impl Listeners { adapter_client: adapter_client.clone(), environment_id: config.environment_id, license_key: license_key.clone(), + helm_chart_version: config.helm_chart_version.clone(), report_interval: Duration::from_secs(3600), }); } else if config.test_only_dummy_segment_client { @@ -859,6 +860,7 @@ impl Listeners { adapter_client: adapter_client.clone(), environment_id: config.environment_id, license_key: license_key.clone(), + helm_chart_version: config.helm_chart_version.clone(), report_interval: Duration::from_secs(180), }); } diff --git a/src/environmentd/src/telemetry.rs b/src/environmentd/src/telemetry.rs index 6acdd38a309ec..5740a2cba82b2 100644 --- a/src/environmentd/src/telemetry.rs +++ b/src/environmentd/src/telemetry.rs @@ -82,6 +82,8 @@ use chrono::Utc; use futures::StreamExt; use mz_adapter::PeekResponseUnary; use mz_adapter::telemetry::{EventDetails, SegmentClientExt}; +use mz_build_info::BuildInfo; +use mz_license_keys::ValidatedLicenseKey; use mz_ore::collections::CollectionExt; use mz_ore::retry::Retry; use mz_ore::{soft_panic_or_log, task}; @@ -90,6 +92,8 @@ use mz_sql::catalog::EnvironmentId; use serde_json::json; use tokio::time::{self, Duration}; +use crate::BUILD_INFO; + /// Telemetry configuration. #[derive(Clone)] pub struct Config { @@ -102,7 +106,10 @@ pub struct Config { /// The validated license key for this environment. Reported so downstream /// analytics can tell which entitlements are active and whether the key has /// expired. - pub license_key: mz_license_keys::ValidatedLicenseKey, + pub license_key: ValidatedLicenseKey, + /// The version of the helm chart that deployed this environment, if + /// self-managed. Folded into the reported `mz_version` trait. + pub helm_chart_version: Option, /// How frequently to send a summary to Segment. pub report_interval: Duration, } @@ -118,6 +125,7 @@ async fn report_loop( adapter_client, environment_id, license_key, + helm_chart_version, report_interval, }: Config, ) { @@ -212,32 +220,12 @@ async fn report_loop( } }; - // Merge in the license key's organization and environment IDs, plus its - // current state. The license key is the authoritative source for these - // IDs. `EnvironmentId::organization_id` only matches the real - // organization in cloud SaaS, so we report the license key's `sub` - // (organization) and `aud` (environment) instead. - // - // Expiry is recomputed against the wall clock each interval rather than - // reusing the flag computed once at startup, so a key that lapses while - // environmentd keeps running is reported as expired. The - // `expiration == 0` guard exempts the sentinel disabled/emulator key, - // whose zero expiration is not a real timestamp. - let now_secs = u64::try_from(Utc::now().timestamp()).unwrap_or(0); - let license_expired = license_key.expired - || (license_key.expiration != 0 && now_secs >= license_key.expiration); if let Some(traits) = traits.as_object_mut() { - traits.insert("organization_id".into(), json!(license_key.organization)); - traits.insert("environment_id".into(), json!(license_key.environment_id)); - traits.insert("license_key_id".into(), json!(license_key.id)); - traits.insert( - "license_expiration_timestamp".into(), - json!(license_key.expiration), - ); - traits.insert("license_expired".into(), json!(license_expired)); - traits.insert( - "license_expiration_behavior".into(), - json!(license_key.expiration_behavior), + build_segment_traits( + traits, + &license_key, + &BUILD_INFO, + helm_chart_version.clone(), ); } @@ -285,3 +273,131 @@ async fn report_loop( last_stats = Some(current_stats); } } + +/// Merges the build version and the license key's identity and state into the +/// collected traits. +/// +/// The license key is the authoritative source for the organization and +/// environment IDs. `EnvironmentId::organization_id` only matches the real +/// organization in cloud SaaS, so we report the license key's `sub` +/// (organization) and `aud` (environment) instead. +/// +/// Expiry is recomputed against the wall clock on every call rather than +/// reusing the flag computed once at startup, so a key that lapses while +/// environmentd keeps running is reported as expired. The `expiration == 0` +/// guard exempts the sentinel disabled/emulator key, whose zero expiration is +/// not a real timestamp. +fn build_segment_traits( + traits: &mut serde_json::Map, + license_key: &ValidatedLicenseKey, + build_info: &BuildInfo, + helm_chart_version: Option, +) { + let now_secs = u64::try_from(Utc::now().timestamp()).unwrap_or(0); + let license_expired = + license_key.expired || (license_key.expiration != 0 && now_secs >= license_key.expiration); + traits.insert("organization_id".into(), json!(license_key.organization)); + traits.insert("environment_id".into(), json!(license_key.environment_id)); + traits.insert("license_key_id".into(), json!(license_key.id)); + traits.insert( + "license_expiration_timestamp".into(), + json!(license_key.expiration), + ); + traits.insert("license_expired".into(), json!(license_expired)); + traits.insert( + "license_expiration_behavior".into(), + json!(license_key.expiration_behavior), + ); + traits.insert( + "mz_version".into(), + json!(build_info.human_version(helm_chart_version)), + ); +} + +#[cfg(test)] +mod tests { + use mz_build_info::DUMMY_BUILD_INFO; + use mz_license_keys::ValidatedLicenseKey; + use serde_json::json; + + use super::build_segment_traits; + + fn license_key(expiration: u64, expired: bool) -> ValidatedLicenseKey { + ValidatedLicenseKey { + id: "test-license-key-id".into(), + organization: "test-organization-id".into(), + environment_id: "test-environment-id".into(), + expiration, + expired, + ..ValidatedLicenseKey::for_tests() + } + } + + // Expiry is computed against the real wall clock, so these tests use + // expirations in the distant past (1, i.e. just after the Unix epoch) or + // distant future (`u64::MAX`) to stay deterministic. + + #[mz_ore::test] + fn reports_license_and_version_traits() { + let mut traits = json!({"active_clusters": 1}); + build_segment_traits( + traits.as_object_mut().unwrap(), + &license_key(u64::MAX, false), + &DUMMY_BUILD_INFO, + Some("25.1.0".into()), + ); + assert_eq!( + traits, + json!({ + "active_clusters": 1, + "organization_id": "test-organization-id", + "environment_id": "test-environment-id", + "license_key_id": "test-license-key-id", + "license_expiration_timestamp": u64::MAX, + "license_expired": false, + "license_expiration_behavior": "Warn", + "mz_version": DUMMY_BUILD_INFO.human_version(Some("25.1.0".into())) + }) + ); + } + + #[mz_ore::test] + fn expiry_is_recomputed_from_wall_clock() { + // The key was valid at startup (`expired: false`) but the clock has + // since passed its expiration. + let mut traits = json!({}); + build_segment_traits( + traits.as_object_mut().unwrap(), + &license_key(1, false), + &DUMMY_BUILD_INFO, + None, + ); + assert_eq!(&traits["license_expired"], &json!(true)); + } + + #[mz_ore::test] + fn startup_expired_flag_is_preserved() { + // A key marked expired at startup stays expired even if the clock + // reads before its expiration. + let mut traits = json!({}); + build_segment_traits( + traits.as_object_mut().unwrap(), + &license_key(u64::MAX, true), + &DUMMY_BUILD_INFO, + None, + ); + assert_eq!(&traits["license_expired"], &json!(true)); + } + + #[mz_ore::test] + fn zero_expiration_sentinel_never_expires() { + let mut traits = json!({}); + build_segment_traits( + traits.as_object_mut().unwrap(), + &license_key(0, false), + &DUMMY_BUILD_INFO, + None, + ); + assert_eq!(&traits["license_expired"], &json!(false)); + } +} diff --git a/src/environmentd/src/test_util.rs b/src/environmentd/src/test_util.rs index 6706e2623f8b6..578a60c98b19c 100644 --- a/src/environmentd/src/test_util.rs +++ b/src/environmentd/src/test_util.rs @@ -106,6 +106,12 @@ pub struct TestHarness { external_login_password_mz_system: Option, listeners_config: ListenersConfig, unsafe_mode: bool, + /// Whether the connection context carries the AWS external ID prefix and + /// connection role ARN. Default true, matching a cloud deployment. Set false + /// via [`TestHarness::without_aws_connection_context`] to model a deployment + /// that never configured the AWS context, where the context functions fold + /// to NULL. + aws_connection_context: bool, workers: usize, now: NowFn, seed: u32, @@ -190,6 +196,7 @@ impl Default for TestHarness { ], }, unsafe_mode: false, + aws_connection_context: true, workers: 1, now: SYSTEM_TIME.clone(), seed: rand::random(), @@ -306,6 +313,14 @@ impl TestHarness { self } + /// Models a deployment that never configured the AWS context, so the AWS + /// external ID prefix and connection role ARN are absent and the plan-time + /// AWS context functions fold to NULL. + pub fn without_aws_connection_context(mut self) -> Self { + self.aws_connection_context = false; + self + } + pub fn workers(mut self, workers: usize) -> Self { self.workers = workers; self @@ -803,7 +818,11 @@ impl Listeners { let system_dyncfgs = Arc::clone(&persist_clients.cfg().configs); let secrets_controller = Arc::clone(&orchestrator); - let connection_context = ConnectionContext::for_tests(orchestrator.reader()); + let mut connection_context = ConnectionContext::for_tests(orchestrator.reader()); + if !config.aws_connection_context { + connection_context.aws_external_id_prefix = None; + connection_context.aws_connection_role_arn = None; + } let orchestrator = Arc::new(TracingOrchestrator::new( orchestrator, config.orchestrator_tracing_cli_args, diff --git a/src/environmentd/tests/sql.rs b/src/environmentd/tests/sql.rs index ac74a7be27548..93d8c09c9c5b6 100644 --- a/src/environmentd/tests/sql.rs +++ b/src/environmentd/tests/sql.rs @@ -361,6 +361,51 @@ fn test_time() { ); } +/// The builtin `mz_internal.mz_analytics` connection is an assume-role AWS +/// connection present on every environment. Its row in +/// `mz_internal.mz_aws_connections` is reconstructed from the plan-time AWS +/// context functions. Without the AWS context those functions fold to NULL, and +/// the view must drop the row entirely, matching the packer it replaced, which +/// bailed and wrote no row in that case. See MZ_AWS_CONNECTIONS in +/// src/catalog/src/builtin/mz_internal.rs. +#[mz_ore::test] +#[allow(clippy::disallowed_methods)] +fn test_mz_aws_connections_context_dependent_row() { + // With the AWS context configured (the default), the assume-role row is + // present and fully populated. + let server = test_util::TestHarness::default().start_blocking(); + let mut client = server.connect(postgres::NoTls).unwrap(); + let count: i64 = client + .query_one("SELECT count(*) FROM mz_internal.mz_aws_connections", &[]) + .unwrap() + .get(0); + assert_eq!(count, 1, "mz_analytics row present with AWS context"); + + // Without the AWS context, the assume-role row is dropped rather than + // emitted with a malformed example_trust_policy. + let server = test_util::TestHarness::default() + .without_aws_connection_context() + .start_blocking(); + let mut client = server.connect(postgres::NoTls).unwrap(); + let count: i64 = client + .query_one("SELECT count(*) FROM mz_internal.mz_aws_connections", &[]) + .unwrap() + .get(0); + assert_eq!(count, 0, "mz_analytics row dropped without AWS context"); + + // The connection object itself still exists; only its derived view row is + // suppressed. + let count: i64 = client + .query_one( + "SELECT count(*) FROM mz_catalog.mz_connections \ + WHERE name = 'mz_analytics' AND type = 'aws'", + &[], + ) + .unwrap() + .get(0); + assert_eq!(count, 1, "mz_analytics connection object still present"); +} + #[mz_ore::test] #[allow(clippy::disallowed_methods)] fn test_subscribe_consolidation() { diff --git a/src/expr/src/scalar/func/impls/jsonb.rs b/src/expr/src/scalar/func/impls/jsonb.rs index 1788c342e7a4f..6929ea134192f 100644 --- a/src/expr/src/scalar/func/impls/jsonb.rs +++ b/src/expr/src/scalar/func/impls/jsonb.rs @@ -18,9 +18,10 @@ use mz_repr::role_id::RoleId; use mz_repr::{ArrayRustType, Datum, Row, RowPacker, SqlColumnType, SqlScalarType, strconv}; use mz_sql_parser::ast::display::AstDisplay; use mz_sql_parser::ast::{ - AstInfo, AvroSchema, CreateSubsourceOptionName, Format, FormatSpecifier, - KafkaSourceConfigOptionName, PgConfigOptionName, ProtobufSchema, RawClusterName, RawItemName, - SourceEnvelope, SourceErrorPolicy, Value, WithOptionValue, + AstInfo, AvroSchema, ConnectionOption, ConnectionOptionName, CreateConnectionType, + CreateSubsourceOptionName, Format, FormatSpecifier, KafkaSourceConfigOptionName, + PgConfigOptionName, ProtobufSchema, RawClusterName, RawItemName, SourceEnvelope, + SourceErrorPolicy, UnresolvedItemName, Value, WithOptionValue, }; use prost::Message as _; use serde::{Deserialize, Serialize}; @@ -380,6 +381,27 @@ fn parse_catalog_acl_mode<'a>(a: JsonbRef<'a>) -> Result { .map_err(|e| EvalError::InvalidCatalogJson(e.into())) } +/// Extracts the string form of a `WithOptionValue`, matching how the planner +/// coerces option values. The blanket `TryFromValue>` impl in +/// `src/sql/src/plan/with_options.rs` accepts a quoted string, a bare identifier, +/// and a 1-part unresolved item name, all yielding a string. Any other variant +/// yields None. +/// +/// `AstDisplay` does not re-quote bare identifiers or 1-part names, so those +/// forms persist unquoted in `create_sql`. Matching only `Value::String` here +/// would miss them and silently fall back to a default, so the catalog-raw +/// parsers below route their string options through this helper. +fn option_string(value: &WithOptionValue) -> Option { + match value { + WithOptionValue::Value(Value::String(s)) => Some(s.clone()), + WithOptionValue::Ident(ident) => Some(ident.clone().into_string()), + WithOptionValue::UnresolvedItemName(UnresolvedItemName(parts)) if parts.len() == 1 => { + Some(parts[0].clone().into_string()) + } + _ => None, + } +} + /// Parses a catalog `create_sql` string into a JSONB object. /// /// The returned JSONB does not fully reflect the parsed SQL and instead contains only fields @@ -478,14 +500,32 @@ fn parse_catalog_create_sql<'a>(a: &'a str) -> Result { Some(mz_sql_parser::ast::SourceEnvelope::Debezium) ); - if let Some(envelope) = stmt.envelope { - use mz_sql_parser::ast::SourceEnvelope::*; - let envelope_type = match envelope { - None => "none", - Debezium => "debezium", - Upsert { .. } => "upsert", - CdcV2 => "materialize", - }; + // An old-syntax kafka source ingests into its own relation, so an + // omitted ENVELOPE means the default ENVELOPE NONE and the pre-MV + // packer reported 'none'. A new-syntax source (no progress + // subsource, hence no EXPOSE PROGRESS AS in create_sql) ingests + // nothing itself. Its envelopes live on the per-table exports, so + // its own envelope_type stays absent (SQL NULL), matching released + // behavior. `progress_subsource.is_some()` is the planner's own + // old-vs-new discriminator (see `OldSyntaxIngestion` in + // plan_create_source). Non-kafka sources carry no envelope either. + // See the `mz_sources.envelope_type` column. + let envelope_type = match &stmt.envelope { + Some(envelope) => { + use mz_sql_parser::ast::SourceEnvelope::*; + Some(match envelope { + None => "none", + Debezium => "debezium", + Upsert { .. } => "upsert", + CdcV2 => "materialize", + }) + } + None if source_type == "kafka" && stmt.progress_subsource.is_some() => { + Some("none") + } + None => None, + }; + if let Some(envelope_type) = envelope_type { info.insert("envelope_type", json!(envelope_type)); } @@ -676,10 +716,7 @@ fn parse_kafka_source_details<'a>(a: &'a str) -> Result { let mut topic: Option = None; let mut group_id_prefix: Option = None; for opt in options { - let string_value = match opt.value { - Some(WithOptionValue::Value(Value::String(s))) => Some(s), - _ => None, - }; + let string_value = opt.value.as_ref().and_then(option_string); match opt.name { KafkaSourceConfigOptionName::Topic => topic = string_value, KafkaSourceConfigOptionName::GroupIdPrefix => group_id_prefix = string_value, @@ -889,6 +926,163 @@ fn parse_source_export_details<'a>(a: &'a str) -> Result { Ok(jsonb) } +/// Extracts connection-detail metadata from a catalog `create_sql`. +/// +/// Returns a per-connection-type object with the fields that the +/// `mz_kafka_connections`, `mz_ssh_tunnel_connections`, and `mz_aws_connections` +/// builtin views need. For everything else (other connection types, including +/// aws-privatelink whose only detail is context-derived, and non-connection +/// statements) it returns jsonb `null`, so callers filter on `IS NOT NULL` and +/// gate on the connection type separately (via +/// `parse_catalog_create_sql(...)->>'connection_type'`, the way `mz_connections` +/// already does). +/// +/// The shape per type: +/// +/// ```json +/// // kafka +/// { "brokers": ["host:port", ...], "progress_topic": } +/// // ssh-tunnel +/// { "public_key_1": "", "public_key_2": "" } +/// // aws +/// { +/// "auth_kind": "credentials" | "assume-role", +/// "endpoint": , "region": , +/// "access_key_id": , "access_key_id_secret_id": , +/// "secret_access_key_secret_id": , +/// "session_token": , "session_token_secret_id": , +/// "assume_role_arn": , "assume_role_session_name": +/// } +/// ``` +/// +/// `progress_topic` is null when the connection does not set an explicit +/// `PROGRESS TOPIC`. The default (`_materialize-progress--`) is +/// reconstructed by the view, not here, because it needs the environment id and +/// the connection's own id. Values derived only from environment context +/// (AWS principal, external id, trust policy, privatelink principal) are also +/// left to the view. This keeps the helper a pure function of the `create_sql`. +/// +/// For aws, an option is either an inline value or a secret reference. Inline +/// values land in `access_key_id`/`session_token`; a secret reference lands in +/// the matching `*_secret_id` as the referenced secret's catalog item id (the +/// persisted `create_sql` stores resolved references as `[uNNN AS name]`). +/// `auth_kind` is `assume-role` when `ASSUME ROLE ARN` is present, else +/// `credentials`, matching the `AwsAuth` variant the removed packer read. +/// +/// Errors if the statement fails to parse. +#[sqlfunc] +fn parse_connection_details<'a>(a: &'a str) -> Result { + // The persisted `create_sql` stores an inline broker as a single `BROKER` + // option and a broker list as a `BROKERS (...)` sequence. Either way we + // only need the addresses, which are present regardless of the tunnel + // (direct, SSH, or PrivateLink). + fn broker_addresses(values: &[ConnectionOption]) -> Vec { + let mut brokers = Vec::new(); + for opt in values { + match (&opt.name, &opt.value) { + (ConnectionOptionName::Broker, Some(WithOptionValue::ConnectionKafkaBroker(b))) => { + brokers.push(b.address.clone()); + } + (ConnectionOptionName::Brokers, Some(WithOptionValue::Sequence(seq))) => { + for v in seq { + if let WithOptionValue::ConnectionKafkaBroker(b) = v { + brokers.push(b.address.clone()); + } + } + } + _ => {} + } + } + brokers + } + + fn string_option( + values: &[ConnectionOption], + name: ConnectionOptionName, + ) -> Option { + values + .iter() + .find(|o| o.name == name) + .and_then(|o| o.value.as_ref()) + .and_then(option_string) + } + + // The catalog id of the secret a `SECRET ...` option references. Resolved + // references persist as `RawItemName::Id`, so an unresolved name yields + // None (the same treatment `parse_source_export_details` gives item names). + fn secret_id_option>( + values: &[ConnectionOption], + name: ConnectionOptionName, + ) -> Option { + values.iter().find_map(|o| match &o.value { + Some(WithOptionValue::Secret(RawItemName::Id(id, _, _))) if o.name == name => { + Some(id.clone()) + } + _ => None, + }) + } + + let parse = || -> Result { + let mut stmts = mz_sql_parser::parser::parse_statements(a) + .map_err(|e| format!("failed to parse create_sql: {e}"))?; + let stmt = match stmts.len() { + 1 => stmts.remove(0).ast, + n => return Err(format!("expected a single statement, found {n}")), + }; + + use mz_sql_parser::ast::Statement::CreateConnection; + let stmt = match stmt { + CreateConnection(stmt) => stmt, + _ => return Ok(serde_json::Value::Null), + }; + + match stmt.connection_type { + CreateConnectionType::Kafka => Ok(json!({ + "brokers": broker_addresses(&stmt.values), + "progress_topic": string_option(&stmt.values, ConnectionOptionName::ProgressTopic), + })), + CreateConnectionType::Ssh => Ok(json!({ + "public_key_1": string_option(&stmt.values, ConnectionOptionName::PublicKey1), + "public_key_2": string_option(&stmt.values, ConnectionOptionName::PublicKey2), + })), + CreateConnectionType::Aws => { + let assume_role_arn = + string_option(&stmt.values, ConnectionOptionName::AssumeRoleArn); + let auth_kind = if assume_role_arn.is_some() { + "assume-role" + } else { + "credentials" + }; + Ok(json!({ + "auth_kind": auth_kind, + // Planning coerces an empty ENDPOINT to None (see + // `src/sql/src/plan/statement/ddl/connection.rs`), so the + // removed packer wrote NULL for `ENDPOINT = ''`. Match that. + "endpoint": string_option(&stmt.values, ConnectionOptionName::Endpoint) + .filter(|s| !s.is_empty()), + "region": string_option(&stmt.values, ConnectionOptionName::Region), + "access_key_id": string_option(&stmt.values, ConnectionOptionName::AccessKeyId), + "access_key_id_secret_id": + secret_id_option(&stmt.values, ConnectionOptionName::AccessKeyId), + "secret_access_key_secret_id": + secret_id_option(&stmt.values, ConnectionOptionName::SecretAccessKey), + "session_token": string_option(&stmt.values, ConnectionOptionName::SessionToken), + "session_token_secret_id": + secret_id_option(&stmt.values, ConnectionOptionName::SessionToken), + "assume_role_arn": assume_role_arn, + "assume_role_session_name": + string_option(&stmt.values, ConnectionOptionName::AssumeRoleSessionName), + })) + } + _ => Ok(serde_json::Value::Null), + } + }; + + let val = parse().map_err(|e| EvalError::InvalidCatalogJson(e.into()))?; + let jsonb = Jsonb::from_serde_json(val).expect("valid JSONB"); + Ok(jsonb) +} + #[cfg(test)] mod tests { use mz_repr::adt::jsonb::Jsonb; @@ -1064,6 +1258,21 @@ mod tests { ); } + #[mz_ore::test] + fn kafka_unquoted_topic_and_prefix() { + // Planning accepts a bare identifier for TOPIC / GROUP ID PREFIX, and it + // persists unquoted in create_sql. Matching only quoted strings would + // drop TOPIC and error the whole mz_kafka_source_tables view. + let sql = "CREATE SOURCE \"materialize\".\"public\".\"k_src\" \ + IN CLUSTER [u42] \ + FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \ + (TOPIC = my_topic, GROUP ID PREFIX = my_prefix) FORMAT TEXT"; + let out = super::parse_kafka_source_details(sql).expect("ok"); + let out = as_serde(out); + assert_eq!(out["topic"], json!("my_topic")); + assert_eq!(out["group_id_prefix"], json!("my_prefix")); + } + #[mz_ore::test] fn kafka_unresolved_connection_errors() { // A bare-name connection reference never happens after purification, @@ -1124,6 +1333,16 @@ mod tests { ); } + #[mz_ore::test] + fn export_table_kafka_omitted_envelope_is_null() { + // Omitting ENVELOPE persists as absent in create_sql, so this + // source-type-agnostic helper reports null. The mz_kafka_source_tables + // view is responsible for defaulting kafka's null envelope to 'none'. + let sql = table_from_source_sql("\"topic\"", " FORMAT TEXT"); + let out = super::parse_source_export_details(&sql).expect("ok"); + assert_eq!(as_serde(out)["envelope_type"], serde_json::Value::Null); + } + #[mz_ore::test] fn export_table_kafka_key_value_format() { let sql = table_from_source_sql( @@ -1237,4 +1456,245 @@ mod tests { "wrong error variant/message" ); } + + // --- parse_connection_details -------------------------------------------- + + #[mz_ore::test] + fn connection_kafka_single_broker_default_progress() { + // No explicit PROGRESS TOPIC: the helper leaves it null and the view + // reconstructs the default. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \ + (BROKER = 'localhost:9092', SECURITY PROTOCOL = plaintext)"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "brokers": ["localhost:9092"], + "progress_topic": null, + }), + ); + } + + #[mz_ore::test] + fn connection_kafka_explicit_progress_topic() { + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \ + (BROKER = 'localhost:9092', PROGRESS TOPIC = 'override', \ + SECURITY PROTOCOL = plaintext)"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "brokers": ["localhost:9092"], + "progress_topic": "override", + }), + ); + } + + #[mz_ore::test] + fn connection_kafka_broker_list() { + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \ + (BROKERS ('b1:9092', 'b2:9092'), SECURITY PROTOCOL = plaintext)"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "brokers": ["b1:9092", "b2:9092"], + "progress_topic": null, + }), + ); + } + + #[mz_ore::test] + fn connection_ssh_public_keys() { + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO SSH TUNNEL \ + (HOST = 'ssh.example.com', PORT = 22, USER = 'mz', \ + PUBLIC KEY 1 = 'ssh-ed25519 AAAA', PUBLIC KEY 2 = 'ssh-ed25519 BBBB')"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "public_key_1": "ssh-ed25519 AAAA", + "public_key_2": "ssh-ed25519 BBBB", + }), + ); + } + + #[mz_ore::test] + fn connection_aws_credentials_inline_key() { + // Inline ACCESS KEY ID, secret SECRET ACCESS KEY. Assume-role columns + // stay null and auth_kind is credentials. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \ + (ACCESS KEY ID = 'AKIAEXAMPLE', \ + SECRET ACCESS KEY = SECRET [u1 AS \"materialize\".\"public\".\"sk\"])"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "auth_kind": "credentials", + "endpoint": null, + "region": null, + "access_key_id": "AKIAEXAMPLE", + "access_key_id_secret_id": null, + "secret_access_key_secret_id": "u1", + "session_token": null, + "session_token_secret_id": null, + "assume_role_arn": null, + "assume_role_session_name": null, + }), + ); + } + + #[mz_ore::test] + fn connection_aws_credentials_secret_key_and_session_token() { + // Every credential provided as a secret reference lands in the matching + // *_secret_id column as the referenced secret's catalog id. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \ + (ENDPOINT = 'http://localhost', REGION = 'us-east-1', \ + ACCESS KEY ID = SECRET [u1 AS \"materialize\".\"public\".\"ak\"], \ + SECRET ACCESS KEY = SECRET [u2 AS \"materialize\".\"public\".\"sk\"], \ + SESSION TOKEN = SECRET [u3 AS \"materialize\".\"public\".\"st\"])"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "auth_kind": "credentials", + "endpoint": "http://localhost", + "region": "us-east-1", + "access_key_id": null, + "access_key_id_secret_id": "u1", + "secret_access_key_secret_id": "u2", + "session_token": null, + "session_token_secret_id": "u3", + "assume_role_arn": null, + "assume_role_session_name": null, + }), + ); + } + + #[mz_ore::test] + fn connection_aws_assume_role() { + // Assume-role sets auth_kind and the assume-role columns; credential + // columns stay null. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \ + (ASSUME ROLE ARN 'arn:aws:iam::123:role/mz', \ + ASSUME ROLE SESSION NAME 'sess')"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!( + as_serde(out), + json!({ + "auth_kind": "assume-role", + "endpoint": null, + "region": null, + "access_key_id": null, + "access_key_id_secret_id": null, + "secret_access_key_secret_id": null, + "session_token": null, + "session_token_secret_id": null, + "assume_role_arn": "arn:aws:iam::123:role/mz", + "assume_role_session_name": "sess", + }), + ); + } + + #[mz_ore::test] + fn connection_kafka_unquoted_progress_topic() { + // A bare identifier PROGRESS TOPIC persists unquoted in create_sql. The + // helper must surface it, else the view substitutes the default topic. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \ + (BROKER = 'localhost:9092', PROGRESS TOPIC = my_topic, \ + SECURITY PROTOCOL = plaintext)"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!(as_serde(out)["progress_topic"], json!("my_topic")); + } + + #[mz_ore::test] + fn connection_aws_unquoted_option_values() { + // Planning accepts bare identifiers for these options and persists them + // unquoted. The helper must surface them, not fall back to NULL. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \ + (ENDPOINT = localhost, REGION = useast1, \ + ASSUME ROLE ARN 'arn:aws:iam::123:role/mz', \ + ASSUME ROLE SESSION NAME = mysession)"; + let out = super::parse_connection_details(sql).expect("ok"); + let out = as_serde(out); + assert_eq!(out["endpoint"], json!("localhost")); + assert_eq!(out["region"], json!("useast1")); + assert_eq!(out["assume_role_session_name"], json!("mysession")); + } + + #[mz_ore::test] + fn connection_aws_empty_endpoint_is_null() { + // Planning coerces ENDPOINT = '' to None, so the packer wrote NULL. The + // view must match, not report an empty string. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \ + (ENDPOINT = '', ASSUME ROLE ARN 'arn:aws:iam::123:role/mz')"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!(as_serde(out)["endpoint"], serde_json::Value::Null); + } + + #[mz_ore::test] + fn connection_other_type_returns_null_jsonb() { + // A connection type without a detail view (postgres) yields null. + let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO POSTGRES \ + (HOST = 'db', DATABASE = 'postgres', USER = 'mz')"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!(as_serde(out), serde_json::Value::Null); + } + + #[mz_ore::test] + #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux` + fn connection_non_connection_returns_null_jsonb() { + let sql = "CREATE VIEW v AS SELECT 1"; + let out = super::parse_connection_details(sql).expect("ok"); + assert_eq!(as_serde(out), serde_json::Value::Null); + } + + // --- parse_catalog_create_sql envelope_type ------------------------------ + + #[mz_ore::test] + fn catalog_kafka_old_syntax_omitted_envelope_defaults_none() { + // An old-syntax kafka source (carries EXPOSE PROGRESS AS) ingests into + // its own relation, so an omitted ENVELOPE means the default NONE, which + // the pre-MV packer reported as 'none'. + let sql = "CREATE SOURCE \"materialize\".\"public\".\"k\" \ + IN CLUSTER [u42] \ + FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \ + (TOPIC 'test') FORMAT TEXT \ + EXPOSE PROGRESS AS [u12 AS \"materialize\".\"public\".\"k_progress\"]"; + let out = super::parse_catalog_create_sql(sql).expect("ok"); + assert_eq!(as_serde(out)["envelope_type"], json!("none")); + } + + #[mz_ore::test] + fn catalog_kafka_old_syntax_explicit_envelope() { + let sql = "CREATE SOURCE \"materialize\".\"public\".\"k\" \ + IN CLUSTER [u42] \ + FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \ + (TOPIC 'test') FORMAT BYTES ENVELOPE UPSERT \ + EXPOSE PROGRESS AS [u12 AS \"materialize\".\"public\".\"k_progress\"]"; + let out = super::parse_catalog_create_sql(sql).expect("ok"); + assert_eq!(as_serde(out)["envelope_type"], json!("upsert")); + } + + #[mz_ore::test] + fn catalog_kafka_new_syntax_source_omits_envelope_type() { + // A new-syntax kafka source has no progress subsource (no EXPOSE PROGRESS + // AS). It ingests nothing itself. Envelopes live on the per-table exports, + // so its own envelope_type stays absent (SQL NULL). + let sql = "CREATE SOURCE \"materialize\".\"public\".\"k\" \ + IN CLUSTER [u42] \ + FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"]"; + let out = super::parse_catalog_create_sql(sql).expect("ok"); + assert_eq!(as_serde(out).get("envelope_type"), None); + } + + #[mz_ore::test] + fn catalog_non_kafka_source_omits_envelope_type() { + // Non-kafka sources carry no envelope, so envelope_type stays absent + // (SQL NULL), not 'none'. + let sql = "CREATE SOURCE \"materialize\".\"public\".\"lg\" \ + IN CLUSTER [u42] FROM LOAD GENERATOR COUNTER"; + let out = super::parse_catalog_create_sql(sql).expect("ok"); + assert_eq!(as_serde(out).get("envelope_type"), None); + } } diff --git a/src/expr/src/scalar/func/unary.rs b/src/expr/src/scalar/func/unary.rs index 9723c257be76a..61913b35bd346 100644 --- a/src/expr/src/scalar/func/unary.rs +++ b/src/expr/src/scalar/func/unary.rs @@ -468,6 +468,7 @@ derive_unary!( ParseCatalogCreateSql, ParseCatalogId, ParseCatalogPrivileges, + ParseConnectionDetails, ParseKafkaSourceDetails, ParsePostgresSourceDetails, ParseSourceExportDetails, diff --git a/src/mz-deploy/src/project/compiler/typecheck/catalog.rs b/src/mz-deploy/src/project/compiler/typecheck/catalog.rs index fd408fd68e5cd..4afd7326864a7 100644 --- a/src/mz-deploy/src/project/compiler/typecheck/catalog.rs +++ b/src/mz-deploy/src/project/compiler/typecheck/catalog.rs @@ -628,6 +628,7 @@ impl CatalogRuntime { connection_context: mz_storage_types::connections::ConnectionContext::for_tests( secrets_reader, ), + aws_account_id: None, helm_chart_version: None, }; let mut catalog = Self { diff --git a/src/pgrepr-consts/src/oid.rs b/src/pgrepr-consts/src/oid.rs index 293bf64e90dc6..9d1c0672e5086 100644 --- a/src/pgrepr-consts/src/oid.rs +++ b/src/pgrepr-consts/src/oid.rs @@ -417,7 +417,7 @@ pub const LOG_MZ_MESSAGE_COUNTS_RECEIVED_RAW_OID: u32 = 16691; pub const LOG_MZ_MESSAGE_COUNTS_SENT_RAW_OID: u32 = 16692; pub const LOG_MZ_DATAFLOW_OPERATOR_REACHABILITY_RAW_OID: u32 = 16693; pub const TABLE_MZ_KAFKA_SINKS_OID: u32 = 16694; -pub const TABLE_MZ_KAFKA_CONNECTIONS_OID: u32 = 16695; +pub const MV_MZ_KAFKA_CONNECTIONS_OID: u32 = 16695; pub const MV_MZ_KAFKA_SOURCES_OID: u32 = 16696; pub const MV_MZ_POSTGRES_SOURCES_OID: u32 = 16697; pub const TABLE_MZ_OBJECT_DEPENDENCIES_OID: u32 = 16698; @@ -431,7 +431,7 @@ pub const MV_MZ_INDEXES_OID: u32 = 16705; pub const TABLE_MZ_INDEX_COLUMNS_OID: u32 = 16706; pub const TABLE_MZ_TABLES_OID: u32 = 16707; pub const MV_MZ_CONNECTIONS_OID: u32 = 16708; -pub const TABLE_MZ_SSH_TUNNEL_CONNECTIONS_OID: u32 = 16709; +pub const MV_MZ_SSH_TUNNEL_CONNECTIONS_OID: u32 = 16709; pub const MV_MZ_SOURCES_OID: u32 = 16710; pub const TABLE_MZ_SINKS_OID: u32 = 16711; pub const TABLE_MZ_VIEWS_OID: u32 = 16712; @@ -477,8 +477,8 @@ pub const SOURCE_MZ_SINK_STATUS_HISTORY_OID: u32 = 16752; pub const VIEW_MZ_SINK_STATUSES_OID: u32 = 16753; pub const TABLE_MZ_STORAGE_USAGE_BY_SHARD_OID: u32 = 16754; pub const TABLE_MZ_EGRESS_IPS_OID: u32 = 16755; -pub const TABLE_MZ_AWS_PRIVATELINK_CONNECTIONS_OID: u32 = 16756; -pub const TABLE_MZ_AWS_CONNECTIONS_OID: u32 = 16757; +pub const MV_MZ_AWS_PRIVATELINK_CONNECTIONS_OID: u32 = 16756; +pub const MV_MZ_AWS_CONNECTIONS_OID: u32 = 16757; pub const VIEW_MZ_CLUSTER_REPLICA_METRICS_OID: u32 = 16758; pub const SOURCE_MZ_CLUSTER_REPLICA_FRONTIERS_OID: u32 = 16759; pub const SOURCE_MZ_FRONTIERS_OID: u32 = 16760; @@ -665,8 +665,8 @@ pub const ROLE_PUBLIC_OID: u32 = 16944; pub const MV_MZ_ROLE_PARAMETERS_OID: u32 = 16945; pub const TABLE_MZ_MATERIALIZED_VIEW_REFRESH_STRATEGIES_OID: u32 = 16946; pub const MV_MZ_CLUSTER_SCHEDULES_OID: u32 = 16947; -pub const TABLE_MZ_POSTGRES_SOURCE_TABLES_OID: u32 = 16948; -pub const TABLE_MZ_MYSQL_SOURCE_TABLES_OID: u32 = 16949; +pub const MV_MZ_POSTGRES_SOURCE_TABLES_OID: u32 = 16948; +pub const MV_MZ_MYSQL_SOURCE_TABLES_OID: u32 = 16949; pub const INDEX_MZ_KAFKA_SOURCES_IND_OID: u32 = 16950; pub const INDEX_MZ_WEBHOOK_SOURCES_IND_OID: u32 = 16951; pub const TABLE_MZ_HISTORY_RETENTION_STRATEGIES_OID: u32 = 16952; @@ -742,7 +742,7 @@ pub const SOURCE_MZ_WALLCLOCK_LAG_HISTORY_OID: u32 = 17021; pub const VIEW_MZ_WALLCLOCK_GLOBAL_LAG_HISTORY_OID: u32 = 17022; pub const VIEW_MZ_WALLCLOCK_GLOBAL_LAG_RECENT_HISTORY_OID: u32 = 17023; pub const INDEX_MZ_WALLCLOCK_GLOBAL_LAG_RECENT_HISTORY_IND_OID: u32 = 17024; -pub const TABLE_MZ_KAFKA_SOURCE_TABLES_OID: u32 = 17025; +pub const MV_MZ_KAFKA_SOURCE_TABLES_OID: u32 = 17025; pub const VIEW_MZ_OBJECTS_ID_NAMESPACE_TYPES_OID: u32 = 17030; pub const VIEW_MZ_OBJECT_HISTORY_OID: u32 = 17031; pub const INDEX_MZ_OBJECT_HISTORY_IND_OID: u32 = 17032; @@ -767,7 +767,7 @@ pub const VIEW_MZ_MAPPABLE_OBJECTS_OID: u32 = 17053; pub const VIEW_MZ_WALLCLOCK_GLOBAL_LAG_OID: u32 = 17054; pub const SOURCE_MZ_WALLCLOCK_GLOBAL_LAG_HISTOGRAM_RAW_OID: u32 = 17055; pub const VIEW_MZ_WALLCLOCK_GLOBAL_LAG_HISTOGRAM_OID: u32 = 17056; -pub const TABLE_MZ_SQL_SERVER_SOURCE_TABLES_OID: u32 = 17057; +pub const MV_MZ_SQL_SERVER_SOURCE_TABLES_OID: u32 = 17057; pub const TABLE_MZ_LICENSE_KEYS_OID: u32 = 17058; pub const TABLE_MZ_ROLE_AUTH_OID: u32 = 17059; pub const TABLE_MZ_ICEBERG_SINKS_OID: u32 = 17060; @@ -817,8 +817,8 @@ pub const MV_MZ_CLUSTER_AUTO_SCALING_STRATEGIES_OID: u32 = 17103; pub const INDEX_MZ_CLUSTER_AUTO_SCALING_STRATEGIES_IND_OID: u32 = 17104; pub const FUNC_PARSE_POSTGRES_SOURCE_DETAILS_OID: u32 = 17105; pub const FUNC_PARSE_KAFKA_SOURCE_DETAILS_OID: u32 = 17106; -pub const MV_MZ_POSTGRES_SOURCE_TABLES_OID: u32 = 17107; -pub const MV_MZ_MYSQL_SOURCE_TABLES_OID: u32 = 17108; -pub const MV_MZ_SQL_SERVER_SOURCE_TABLES_OID: u32 = 17109; -pub const MV_MZ_KAFKA_SOURCE_TABLES_OID: u32 = 17110; pub const FUNC_PARSE_SOURCE_EXPORT_DETAILS_OID: u32 = 17111; +pub const FUNC_PARSE_CONNECTION_DETAILS_OID: u32 = 17112; +pub const FUNC_MZ_AWS_ACCOUNT_ID_OID: u32 = 17113; +pub const FUNC_MZ_AWS_EXTERNAL_ID_PREFIX_OID: u32 = 17114; +pub const FUNC_MZ_AWS_CONNECTION_ROLE_ARN_OID: u32 = 17115; diff --git a/src/sql/src/catalog.rs b/src/sql/src/catalog.rs index 9806b73ccc2fc..068eb6c0751bb 100644 --- a/src/sql/src/catalog.rs +++ b/src/sql/src/catalog.rs @@ -440,6 +440,15 @@ pub struct CatalogConfig { pub now: NowFn, /// Context for source and sink connections. pub connection_context: ConnectionContext, + /// The AWS account ID that Materialize runs under, if configured. + /// + /// This is the `aws_account_id` from the catalog's `AwsPrincipalContext`. + /// It is plumbed through here (rather than the whole context) because the + /// external-id prefix and connection role ARN already live on + /// [`ConnectionContext`], and because `AwsPrincipalContext` is defined in + /// `mz-catalog`, which cannot be a dependency of `mz-sql`. Folded into a + /// literal by `mz_aws_account_id()`; `None` on non-cloud/local envs. + pub aws_account_id: Option, /// Helm chart version pub helm_chart_version: Option, } diff --git a/src/sql/src/func.rs b/src/sql/src/func.rs index 5152a8902c68d..cd535bea785a9 100644 --- a/src/sql/src/func.rs +++ b/src/sql/src/func.rs @@ -4709,6 +4709,56 @@ pub static MZ_CATALOG_BUILTINS: LazyLock> = LazyLoc )) }) => String, oid::FUNC_MZ_ENVIRONMENT_ID_OID; }, + // The three AWS-context functions below are plan-time constants, fixed + // for the lifetime of the envd process, folded to a literal here for the + // same reason as `mz_environment_id`: the mz_aws_connections and + // mz_aws_privatelink_connections materialized views reproduce the AWS + // principal, external id, and trust policy in SQL, and a materialized + // view cannot reference an unmaterializable function. They return NULL + // on environments without the corresponding context (non-cloud/local). + // + // Like `mz_environment_id`, they are not gated by + // `restrict_to_user_objects`. The fold bakes the value into any stored + // view that references it, so a gate here could only catch direct calls, + // not values already materialized into a view, which is more misleading + // than no gate. Restricted sessions are still blocked from the system + // connection views themselves (they are system relations). See + // doc/developer/design/20260508_restrict_to_user_objects.md. + "mz_aws_account_id" => Scalar { + params!() => Operation::nullary(|ecx| { + Ok(match &ecx.catalog().config().aws_account_id { + Some(account_id) => { + HirScalarExpr::literal(Datum::String(account_id), SqlScalarType::String) + } + None => HirScalarExpr::literal_null(SqlScalarType::String), + }) + }) => String, oid::FUNC_MZ_AWS_ACCOUNT_ID_OID; + }, + "mz_aws_external_id_prefix" => Scalar { + params!() => Operation::nullary(|ecx| { + let prefix = ecx + .catalog() + .config() + .connection_context + .aws_external_id_prefix + .as_ref() + .map(|p| p.to_string()); + Ok(match prefix { + Some(prefix) => { + HirScalarExpr::literal(Datum::String(&prefix), SqlScalarType::String) + } + None => HirScalarExpr::literal_null(SqlScalarType::String), + }) + }) => String, oid::FUNC_MZ_AWS_EXTERNAL_ID_PREFIX_OID; + }, + "mz_aws_connection_role_arn" => Scalar { + params!() => Operation::nullary(|ecx| { + Ok(match &ecx.catalog().config().connection_context.aws_connection_role_arn { + Some(arn) => HirScalarExpr::literal(Datum::String(arn), SqlScalarType::String), + None => HirScalarExpr::literal_null(SqlScalarType::String), + }) + }) => String, oid::FUNC_MZ_AWS_CONNECTION_ROLE_ARN_OID; + }, "mz_is_superuser" => Scalar { params!() => UnmaterializableFunc::MzIsSuperuser => SqlScalarType::Bool, oid::FUNC_MZ_IS_SUPERUSER; @@ -5320,6 +5370,11 @@ pub static MZ_INTERNAL_BUILTINS: LazyLock> = LazyLo func::ParseSourceExportDetails, ) => Jsonb, oid::FUNC_PARSE_SOURCE_EXPORT_DETAILS_OID; }, + "parse_connection_details" => Scalar { + params!(String) => UnaryFunc::ParseConnectionDetails( + func::ParseConnectionDetails, + ) => Jsonb, oid::FUNC_PARSE_CONNECTION_DETAILS_OID; + }, "redact_sql" => Scalar { params!(String) => UnaryFunc::RedactSql(func::RedactSql) => String, oid::FUNC_REDACT_SQL_OID; diff --git a/src/sql/src/names.rs b/src/sql/src/names.rs index 3d9bddbeb3aaa..dc2f3ade3caf3 100644 --- a/src/sql/src/names.rs +++ b/src/sql/src/names.rs @@ -2104,7 +2104,11 @@ impl<'a> Fold for NameResolver<'a> { &mut self, name: ::NetworkPolicyName, ) -> ::NetworkPolicyName { - match self.catalog.resolve_network_policy(&name.to_string()) { + let name_str = match &name { + RawNetworkPolicyName::Unresolved(ident) => ident.as_str(), + RawNetworkPolicyName::Resolved(s) => s.as_str(), + }; + match self.catalog.resolve_network_policy(name_str) { Ok(policy) => ResolvedNetworkPolicyName { id: policy.id(), name: policy.name().to_string(), diff --git a/src/sql/src/plan/statement/ddl.rs b/src/sql/src/plan/statement/ddl.rs index 19f7073711fdc..cbfbd9de9c976 100644 --- a/src/sql/src/plan/statement/ddl.rs +++ b/src/sql/src/plan/statement/ddl.rs @@ -4778,7 +4778,9 @@ pub fn plan_alter_network_policy( ctx.require_feature_flag(&vars::ENABLE_NETWORK_POLICIES)?; let policy_options: NetworkPolicyOptionExtracted = options.try_into()?; - let policy = ctx.catalog.resolve_network_policy(&name.to_string())?; + let policy = ctx + .catalog + .resolve_network_policy(normalize::ident_ref(&name))?; let Some(rule_defs) = policy_options.rules else { sql_bail!("RULES must be specified when creating network policies."); @@ -8345,7 +8347,10 @@ pub(crate) fn resolve_network_policy<'a>( name: Ident, if_exists: bool, ) -> Result, PlanError> { - match scx.catalog.resolve_network_policy(&name.to_string()) { + match scx + .catalog + .resolve_network_policy(normalize::ident_ref(&name)) + { Ok(policy) => Ok(Some(ResolvedNetworkPolicyName { id: policy.id(), name: policy.name().to_string(), diff --git a/src/storage-types/src/connections.rs b/src/storage-types/src/connections.rs index b915233ddd876..7f76c8ff2b5f4 100644 --- a/src/storage-types/src/connections.rs +++ b/src/storage-types/src/connections.rs @@ -1124,6 +1124,11 @@ impl KafkaConnection { /// /// The caller is responsible for providing the connection ID as it is not /// known to `KafkaConnection`. + /// + /// NOTE: the `mz_catalog.mz_kafka_connections` builtin materialized view + /// reconstructs the default (`_materialize-progress--`) in SQL + /// (see `MZ_KAFKA_CONNECTIONS` in `src/catalog/src/builtin/mz_catalog.rs`). + /// Keep the two in sync. pub fn progress_topic( &self, connection_context: &ConnectionContext, diff --git a/src/storage-types/src/connections/aws.rs b/src/storage-types/src/connections/aws.rs index 19c2016e68198..0e3f15c993bc3 100644 --- a/src/storage-types/src/connections/aws.rs +++ b/src/storage-types/src/connections/aws.rs @@ -283,6 +283,10 @@ impl AwsAssumeRole { Ok(credentials.build_from_provider(jump_credentials).await) } + // NOTE: the `mz_internal.mz_aws_connections` builtin materialized view + // reconstructs this `mz__` format in SQL (see + // MZ_AWS_CONNECTIONS in src/catalog/src/builtin/mz_internal.rs). Keep the + // two in sync. pub fn external_id( &self, connection_context: &ConnectionContext, @@ -294,6 +298,10 @@ impl AwsAssumeRole { Ok(format!("mz_{}_{}", aws_external_id_prefix, connection_id)) } + // NOTE: the `mz_internal.mz_aws_connections` builtin materialized view + // reconstructs this trust-policy JSON with `jsonb_build_object` (see + // MZ_AWS_CONNECTIONS in src/catalog/src/builtin/mz_internal.rs). Keep the + // structure (keys, nesting) in sync. pub fn example_trust_policy( &self, connection_context: &ConnectionContext, diff --git a/src/testdrive/src/action.rs b/src/testdrive/src/action.rs index 74a70be517939..66192b1c2fcaa 100644 --- a/src/testdrive/src/action.rs +++ b/src/testdrive/src/action.rs @@ -22,7 +22,7 @@ use aws_credential_types::provider::ProvideCredentials; use aws_types::SdkConfig; use futures::future::FutureExt; use itertools::Itertools; -use mz_adapter::catalog::{Catalog, ConnCatalog}; +use mz_adapter::catalog::{Catalog, ConnCatalog, DebugAwsContext}; use mz_adapter::session::Session; use mz_build_info::BuildInfo; use mz_catalog::config::ClusterReplicaSizeMap; @@ -218,6 +218,16 @@ pub struct MaterializeState { pgclient: tokio_postgres::Client, environment_id: EnvironmentId, bootstrap_args: BootstrapArgs, + // The AWS environment context, queried once from the running environmentd at + // startup, the same way environment_id is. The builtin AWS connection views + // fold these values into their optimized expressions, so the catalog copy + // opened for the consistency check must resolve them the same way. Queried + // at startup rather than at check time because a check-time query runs after + // arbitrary test state and is less reliable, while startup runs against a + // clean session on the currently connected version. + aws_account_id: Option, + aws_external_id_prefix: Option, + aws_connection_role_arn: Option, } pub struct State { @@ -464,6 +474,11 @@ impl State { &self.persist_clients, ) .await?; + let aws_context = DebugAwsContext { + aws_account_id: self.materialize.aws_account_id.clone(), + aws_external_id_prefix: self.materialize.aws_external_id_prefix.clone(), + aws_connection_role_arn: self.materialize.aws_connection_role_arn.clone(), + }; let catalog = Catalog::open_debug_read_only_persist_catalog_config( persist_client, SYSTEM_TIME.clone(), @@ -472,6 +487,7 @@ impl State { build_info, bootstrap_args, enable_expression_cache_override, + Some(aws_context), ) .await?; let res = f(catalog.for_session(&Session::dummy())); @@ -1349,6 +1365,27 @@ async fn create_materialize_state( .parse() .context("parsing environment ID")?; + // Tolerate the functions not existing: upgrade tests run testdrive against an + // older environmentd whose catalog predates them. A version without them has + // no context-folding views either, so no context is the correct fold. + let (aws_account_id, aws_external_id_prefix, aws_connection_role_arn) = match pg_query_one( + &pgclient, + pg_sql!( + "SELECT mz_aws_account_id(), mz_aws_external_id_prefix(), \ + mz_aws_connection_role_arn()" + ), + &[], + ) + .await + { + Ok(row) => ( + row.get::<_, Option>(0), + row.get::<_, Option>(1), + row.get::<_, Option>(2), + ), + Err(_) => (None, None, None), + }; + let bootstrap_args = BootstrapArgs { cluster_replica_size_map: config.materialize_cluster_replica_sizes.clone(), default_cluster_replica_size: "ABC".to_string(), @@ -1369,6 +1406,9 @@ async fn create_materialize_state( pgclient, environment_id, bootstrap_args, + aws_account_id, + aws_external_id_prefix, + aws_connection_role_arn, }; Ok(materialize_state) diff --git a/test/orchestratord/cluster.yaml.tmpl b/test/orchestratord/cluster.yaml.tmpl index ded66e7011634..97e69103dfdc8 100644 --- a/test/orchestratord/cluster.yaml.tmpl +++ b/test/orchestratord/cluster.yaml.tmpl @@ -30,7 +30,7 @@ kubeadmConfigPatches: "service-node-port-range": "32000-32063" nodes: - role: control-plane - image: kindest/node:v1.32.5 + image: kindest/node:v1.34.8 extraMounts: - containerPath: /var/lib/kubelet/config.json hostPath: "$DOCKER_CONFIG/config.json" @@ -165,7 +165,7 @@ nodes: hostPort: 32063 - role: worker - image: kindest/node:v1.32.5 + image: kindest/node:v1.34.8 labels: materialize.cloud/swap: "true" materialize.cloud/availability-zone: "1" @@ -175,7 +175,7 @@ nodes: - containerPath: /var/lib/kubelet/config.json hostPath: "$DOCKER_CONFIG/config.json" - role: worker - image: kindest/node:v1.32.5 + image: kindest/node:v1.34.8 labels: materialize.cloud/scratch-fs: "true" materialize.cloud/disk: "true" diff --git a/test/orchestratord/mzcompose.py b/test/orchestratord/mzcompose.py index b1eb4266332cc..aa716df8bcc65 100644 --- a/test/orchestratord/mzcompose.py +++ b/test/orchestratord/mzcompose.py @@ -293,8 +293,8 @@ def get_console_app_config(namespace="materialize-environment") -> dict[str, Any def ensure_kind_version() -> None: kind_version = Version.parse(spawn.capture(["kind", "version"]).split(" ")[1][1:]) assert kind_version >= Version.parse( - "0.29.0" - ), f"kind >= v0.29.0 required, while you are on {kind_version}" + "0.32.0" + ), f"kind >= v0.32.0 required, while you are on {kind_version}" def stop_and_remove_container(container_name: str) -> None: diff --git a/test/sqllogictest/autogenerated/mz_catalog.slt b/test/sqllogictest/autogenerated/mz_catalog.slt index b31b8154c4be7..558547b86a68a 100644 --- a/test/sqllogictest/autogenerated/mz_catalog.slt +++ b/test/sqllogictest/autogenerated/mz_catalog.slt @@ -371,7 +371,7 @@ name text The␠name␠of␠the␠source. type text The␠type␠of␠the␠source:␠`kafka`,␠`mysql`,␠`postgres`,␠`load-generator`,␠`progress`,␠or␠`subsource`. connection_id text The␠ID␠of␠the␠connection␠associated␠with␠the␠source,␠if␠any.␠Corresponds␠to␠`mz_connections.id`. size text *Deprecated*␠The␠size␠of␠the␠source. -envelope_type text For␠Kafka␠sources,␠the␠envelope␠type:␠`none`,␠`upsert`,␠or␠`debezium`.␠`NULL`␠for␠other␠source␠types. +envelope_type text For␠old-syntax␠Kafka␠sources,␠the␠envelope␠type:␠`none`,␠`upsert`,␠or␠`debezium`.␠`NULL`␠for␠new-syntax␠Kafka␠sources,␠whose␠envelopes␠are␠defined␠per␠source␠table␠(see␠`mz_kafka_source_tables`),␠and␠for␠other␠source␠types. key_format text For␠Kafka␠sources,␠the␠format␠of␠the␠Kafka␠message␠key:␠`avro`,␠`csv`,␠`regex`,␠`bytes`,␠`json`,␠`text`,␠or␠`NULL`. value_format text For␠Kafka␠sources,␠the␠format␠of␠the␠Kafka␠message␠value:␠`avro`,␠`csv`,␠`regex`,␠`bytes`,␠`json`,␠`text`.␠`NULL`␠for␠other␠source␠types. cluster_id text The␠ID␠of␠the␠cluster␠maintaining␠the␠source.␠Corresponds␠to␠`mz_clusters.id`. diff --git a/test/sqllogictest/autogenerated/mz_internal.slt b/test/sqllogictest/autogenerated/mz_internal.slt index 180ec8924ad4d..d4fc2da6c6547 100644 --- a/test/sqllogictest/autogenerated/mz_internal.slt +++ b/test/sqllogictest/autogenerated/mz_internal.slt @@ -448,7 +448,7 @@ SELECT name, type, comment FROM objects WHERE schema = 'mz_internal' AND object ---- id text The␠ID␠of␠the␠table.␠Corresponds␠to␠`mz_catalog.mz_tables.id`. topic text The␠topic␠being␠ingested. -envelope_type text The␠envelope␠type:␠`none`,␠`upsert`,␠or␠`debezium`.␠`NULL`␠for␠other␠source␠types. +envelope_type text The␠envelope␠type:␠`none`,␠`upsert`,␠or␠`debezium`.␠Defaults␠to␠`none`␠when␠the␠source␠table␠omits␠an␠explicit␠envelope. key_format text The␠format␠of␠the␠Kafka␠message␠key:␠`avro`,␠`csv`,␠`regex`,␠`bytes`,␠`json`,␠`text`,␠or␠`NULL`. value_format text The␠format␠of␠the␠Kafka␠message␠value:␠`avro`,␠`csv`,␠`regex`,␠`bytes`,␠`json`,␠`text`.␠`NULL`␠for␠other␠source␠types. diff --git a/test/sqllogictest/boolean.slt b/test/sqllogictest/boolean.slt index 32ab8874205a4..be1df8129b277 100644 --- a/test/sqllogictest/boolean.slt +++ b/test/sqllogictest/boolean.slt @@ -411,8 +411,7 @@ NULL 2 1 NULL 1 2 -# TODO: Reenable when database-issues#8807 is fixed -# query T -# SELECT CASE WHEN FALSE THEN 'short_string'::char(20) ELSE 'long_string_long_string'::char(30) END -# ---- -# long_string_long_string +query T +SELECT CASE WHEN FALSE THEN 'short_string'::char(20) ELSE 'long_string_long_string'::char(30) END +---- +long_string_long_string diff --git a/test/sqllogictest/catalog_server_explain.slt b/test/sqllogictest/catalog_server_explain.slt index e11092f9d5c18..300281a804373 100644 --- a/test/sqllogictest/catalog_server_explain.slt +++ b/test/sqllogictest/catalog_server_explain.slt @@ -4497,6 +4497,16 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_aws_privatelink_connections"; +---- +mz_catalog.mz_aws_privatelink_connections: + →Constant (0 rows) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_cluster_replicas"; ---- @@ -4855,6 +4865,70 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_kafka_connections"; +---- +mz_catalog.mz_kafka_connections: + →With + cte l0 = + →Read mz_internal.mz_catalog_raw + cte l1 = + →Distinct GroupAggregate + →Fused with Child Map/Filter/Project + Project: #1 + →Read l0 + cte l2 = + →Non-incremental GroupAggregate + Aggregation: array_agg[order_by=[#0 asc nulls_last]](row(array[#1], #2)) + Key: + Project: #0 + →Table Function jsonb_array_elements_text[with_ordinality]((#0{details} -> "brokers")) + →Arranged l1 + cte l3 = + →Union + →Unarranged Raw Stream + →Arranged l2 + →Map/Filter/Project + Project: #0, #1 + Map: null + →Consolidating Union + →Negate Diffs + →Fused with Child Map/Filter/Project + Project: #0 + →Arranged l2 + Key: (#0) + →Unarranged Raw Stream + →Arranged l1 + →Return + →Differential Join %1[#0] » %0:l0[#1] + after %0: + Project: #3..=#5 + Map: parse_catalog_id(((#2{data} -> "key") -> "gid")), (#1 || {}), coalesce((#0{details} ->> "progress_topic"), ("_materialize-progress-local-az1--" || #3)) + →Arrange (#1) + →Stream l0 + →Arrange (#0) + →Union + →Stream l3 + →Map/Filter/Project + Project: #0, #1 + Map: null + →Consolidating Union + →Negate Diffs + →Fused with Child Map/Filter/Project + Project: #0 + →Read l3 + →Unarranged Raw Stream + →Arranged l1 + +Source mz_internal.mz_catalog_raw + project=(#0, #2) + filter=(("Item" = (#0{data} ->> "kind")) AND ("kafka" = (parse_catalog_create_sql(#1) ->> "connection_type"))) + map=(((((#0{data} -> "value") -> "definition") -> "V1") ->> "create_sql"), parse_connection_details(#1)) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_kafka_sources"; ---- @@ -4907,7 +4981,7 @@ mz_catalog.mz_materialized_views: Project: #4, #0, #5, #1, #6, #2, #9, #8, #3, #7 Map: {=r/s1, s1=r/s1}, "s1" →Arrange (#1{schema_name}, #2{name}) - →Constant (33 rows) + →Constant (37 rows) →Arrange (empty key) (#0{schema_name}) (#0{schema_name}, #1{name}) →Fused with Child Map/Filter/Project Project: #4, #3, #5 @@ -5182,6 +5256,21 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_ssh_tunnel_connections"; +---- +mz_catalog.mz_ssh_tunnel_connections: + →Read mz_internal.mz_catalog_raw + +Source mz_internal.mz_catalog_raw + project=(#3..=#5) + filter=(("Item" = (#0{data} ->> "kind")) AND ("ssh-tunnel" = (parse_catalog_create_sql(#1) ->> "connection_type"))) + map=(((((#0{data} -> "value") -> "definition") -> "V1") ->> "create_sql"), parse_connection_details(#1), parse_catalog_id(((#0{data} -> "key") -> "gid")), (#2{details} ->> "public_key_1"), (#2{details} ->> "public_key_2")) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN MATERIALIZED VIEW "mz_catalog"."mz_system_privileges"; ---- @@ -5198,6 +5287,21 @@ Target cluster: mz_catalog_server EOF +query T multiline +EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_aws_connections"; +---- +mz_internal.mz_aws_connections: + →Read mz_internal.mz_catalog_raw + +Source mz_internal.mz_catalog_raw + project=(#3..=#12, #14..=#16) + filter=(("Item" = (#0{data} ->> "kind")) AND ("aws" = (parse_catalog_create_sql(#1) ->> "connection_type"))) + map=(((((#0{data} -> "value") -> "definition") -> "V1") ->> "create_sql"), parse_connection_details(#1), parse_catalog_id(((#0{data} -> "key") -> "gid")), (#2{details} ->> "endpoint"), (#2{details} ->> "region"), (#2{details} ->> "access_key_id"), (#2{details} ->> "access_key_id_secret_id"), (#2{details} ->> "secret_access_key_secret_id"), (#2{details} ->> "session_token"), (#2{details} ->> "session_token_secret_id"), (#2{details} ->> "assume_role_arn"), (#2{details} ->> "assume_role_session_name"), ("assume-role" = (#2{details} ->> "auth_kind")), case when #13 then "arn:aws:iam::123456789000:role/MaterializeConnection" else null end, case when #13 then ("mz_test-aws-external-id-prefix_" || #3) else null end, case when #13 then jsonb_build_object("Version", "2012-10-17", "Statement", jsonbable_to_jsonb(jsonb_build_array(jsonbable_to_jsonb(jsonb_build_object("Effect", "Allow", "Principal", {AWS: "arn:aws:iam::123456789000:role/MaterializeConnection"}, "Action", "sts:AssumeRole", "Condition", jsonbable_to_jsonb(jsonb_build_object("StringEquals", jsonbable_to_jsonb(jsonb_build_object("sts:ExternalId", jsonbable_to_jsonb(("mz_test-aws-external-id-prefix_" || #3))))))))))) else null end) + +Target cluster: mz_catalog_server + +EOF + query T multiline EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_cluster_auto_scaling_strategies"; ---- @@ -5311,7 +5415,7 @@ mz_internal.mz_kafka_source_tables: path %0: after %1: Project: #2, #1, #3..=#6 - Map: ((#2{details} -> "external_reference") ->> 0), (#2{details} ->> "envelope_type"), (#2{details} ->> "key_format"), (#2{details} ->> "value_format") + Map: ((#2{details} -> "external_reference") ->> 0), coalesce((#2{details} ->> "envelope_type"), "none"), (#2{details} ->> "key_format"), (#2{details} ->> "value_format") after %2: Project: #2..=#6 Filter: (#10{type} = "kafka") @@ -5325,7 +5429,7 @@ mz_internal.mz_kafka_source_tables: path %2: after %1: Project: #1, #3..=#6 - Map: ((#2{details} -> "external_reference") ->> 0), (#2{details} ->> "envelope_type"), (#2{details} ->> "key_format"), (#2{details} ->> "value_format") + Map: ((#2{details} -> "external_reference") ->> 0), coalesce((#2{details} ->> "envelope_type"), "none"), (#2{details} ->> "key_format"), (#2{details} ->> "value_format") after %0: Project: #5, #1..=#4 Map: parse_catalog_id(((#0{data} -> "key") -> "gid")) @@ -7842,7 +7946,7 @@ query T multiline EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_materialized_views"; ---- Explained Query (fast path): - →Constant (33 rows) + →Constant (37 rows) Target cluster: mz_catalog_server diff --git a/test/sqllogictest/float.slt b/test/sqllogictest/float.slt index f1b1dd21cf9d4..99954250f35d8 100644 --- a/test/sqllogictest/float.slt +++ b/test/sqllogictest/float.slt @@ -152,7 +152,7 @@ SELECT SUM(f1) FROM t1 ---- -282409603651671152154661355520.000 -# This *should* be zero, known issue https://github.com/MaterializeInc/database-issues/issues/4341 +# This *should* be zero, known issue DB-191 query T SELECT MIN(f1+f2)-SUM(f1) FROM t1 ---- diff --git a/test/sqllogictest/information_schema_tables.slt b/test/sqllogictest/information_schema_tables.slt index 9faa98ae3cf76..8ea3e4fdb71a7 100644 --- a/test/sqllogictest/information_schema_tables.slt +++ b/test/sqllogictest/information_schema_tables.slt @@ -102,7 +102,7 @@ MATERIALIZED VIEW materialize mz_catalog mz_aws_privatelink_connections -BASE TABLE +MATERIALIZED VIEW materialize mz_catalog mz_base_types @@ -162,7 +162,7 @@ MATERIALIZED VIEW materialize mz_catalog mz_kafka_connections -BASE TABLE +MATERIALIZED VIEW materialize mz_catalog mz_kafka_sinks @@ -238,7 +238,7 @@ MATERIALIZED VIEW materialize mz_catalog mz_ssh_tunnel_connections -BASE TABLE +MATERIALIZED VIEW materialize mz_catalog mz_storage_usage @@ -278,7 +278,7 @@ BASE TABLE materialize mz_internal mz_aws_connections -BASE TABLE +MATERIALIZED VIEW materialize mz_internal mz_aws_privatelink_connection_status_history diff --git a/test/sqllogictest/mz_aws_connections.slt b/test/sqllogictest/mz_aws_connections.slt new file mode 100644 index 0000000000000..96ca58cd70462 --- /dev/null +++ b/test/sqllogictest/mz_aws_connections.slt @@ -0,0 +1,155 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Behavioural tests for mz_internal.mz_aws_connections. The view is a +# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see +# MZ_AWS_CONNECTIONS in src/catalog/src/builtin/mz_internal.rs and the +# parse_connection_details helper in +# src/expr/src/scalar/func/impls/jsonb.rs. The assume-role columns (principal, +# external_id, example_trust_policy) are reconstructed from the AWS context +# functions; the sqllogictest env configures that context (see +# ConnectionContext::for_tests), so the built-in mz_analytics connection +# doubles as the assume-role parity check here. Real credential connections +# with live AWS are exercised by test/aws-localstack/aws-connection/. + +mode cockroach + +reset-server + +# --- Schema ------------------------------------------------------------------- + +# Lock down: 13 columns, canonical order, types, and nullability. +query TTT +SELECT c.name, c.type, c.nullable::text +FROM mz_catalog.mz_columns c +JOIN mz_catalog.mz_objects o ON c.id = o.id +JOIN mz_catalog.mz_schemas s ON o.schema_id = s.id +WHERE o.name = 'mz_aws_connections' AND s.name = 'mz_internal' +ORDER BY c.position +---- +id text false +endpoint text true +region text true +access_key_id text true +access_key_id_secret_id text true +secret_access_key_secret_id text true +session_token text true +session_token_secret_id text true +assume_role_arn text true +assume_role_session_name text true +principal text true +external_id text true +example_trust_policy jsonb true + +# --- Fresh env: only the built-in mz_analytics connection --------------------- + +# Anchor count. mz_analytics is the sole AWS connection on a fresh env. +query I +SELECT count(*) FROM mz_internal.mz_aws_connections +---- +1 + +# mz_analytics is an assume-role connection. With the configured AWS context, +# principal, external_id, and example_trust_policy are reconstructed exactly as +# AwsAssumeRole::external_id / example_trust_policy produced them: principal is +# the connection role ARN, external_id is mz__, and the trust +# policy carries both. This is the SQL<->Rust parity check for the assume-role +# branch. +query TTTT +SELECT + ac.assume_role_arn, + ac.principal, + ac.external_id = 'mz_test-aws-external-id-prefix_' || ac.id, + ac.example_trust_policy = jsonb_build_object( + 'Version', '2012-10-17', + 'Statement', jsonb_build_array(jsonb_build_object( + 'Effect', 'Allow', + 'Principal', jsonb_build_object( + 'AWS', 'arn:aws:iam::123456789000:role/MaterializeConnection'), + 'Action', 'sts:AssumeRole', + 'Condition', jsonb_build_object( + 'StringEquals', jsonb_build_object( + 'sts:ExternalId', 'mz_test-aws-external-id-prefix_' || ac.id)) + )) + ) +FROM mz_internal.mz_aws_connections ac +JOIN mz_connections c USING (id) +WHERE c.name = 'mz_analytics' +---- +(empty) arn:aws:iam::123456789000:role/MaterializeConnection true true + +# --- A credentials connection populates the create_sql-derived columns -------- + +statement ok +CREATE SECRET sak AS 'secret-access-key' + +statement ok +CREATE CONNECTION ac TO AWS ( + ENDPOINT = 'http://localhost:4566', + REGION = 'us-east-1', + ACCESS KEY ID = 'AKIAEXAMPLE', + SECRET ACCESS KEY = SECRET sak +) WITH (VALIDATE = false) + +# The inline access key id is shown; the secret access key is shown as its +# secret id (matching mz_secrets.id). The assume-role and context columns are +# NULL for a credentials connection. +query TTTTTTTTTT +SELECT + endpoint, + region, + access_key_id, + access_key_id_secret_id, + secret_access_key_secret_id = (SELECT id FROM mz_secrets WHERE name = 'sak'), + assume_role_arn, + assume_role_session_name, + principal, + external_id, + example_trust_policy +FROM mz_internal.mz_aws_connections ac +JOIN mz_connections c USING (id) +WHERE c.name = 'ac' +---- +http://localhost:4566 us-east-1 AKIAEXAMPLE NULL true NULL NULL NULL NULL NULL + +# --- Non-aws connections are excluded ----------------------------------------- + +statement ok +CREATE CONNECTION kc TO KAFKA (BROKER 'localhost:9092', SECURITY PROTOCOL PLAINTEXT) WITH (VALIDATE = false) + +query I +SELECT count(*) FROM mz_internal.mz_aws_connections ac +JOIN mz_connections c USING (id) +WHERE c.type != 'aws' +---- +0 + +# --- FK-like invariant: every row joins its parent connection ----------------- + +query I +SELECT count(*) FROM mz_internal.mz_aws_connections ac +LEFT JOIN mz_connections c ON ac.id = c.id +WHERE c.id IS NULL +---- +0 + +# No duplicate ids. +query I +SELECT count(*) - count(DISTINCT id) FROM mz_internal.mz_aws_connections +---- +0 + +statement ok +DROP CONNECTION ac + +statement ok +DROP CONNECTION kc + +statement ok +DROP SECRET sak diff --git a/test/sqllogictest/mz_aws_privatelink_connections.slt b/test/sqllogictest/mz_aws_privatelink_connections.slt new file mode 100644 index 0000000000000..ce9db7b412bef --- /dev/null +++ b/test/sqllogictest/mz_aws_privatelink_connections.slt @@ -0,0 +1,85 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Behavioural tests for mz_catalog.mz_aws_privatelink_connections. The view is a +# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see +# MZ_AWS_PRIVATELINK_CONNECTIONS in src/catalog/src/builtin/mz_catalog.rs. The +# principal is reconstructed from the AWS principal context. A local +# sqllogictest env has no such context, so the context functions fold to NULL +# and the view emits no row, matching the removed packer which skipped the row +# when the context was absent. The populated form is exercised by +# test/cloudtest/test_privatelink_connection.py. + +mode cockroach + +reset-server + +# --- Schema ------------------------------------------------------------------- + +# Lock down: 2 columns, canonical order, types, and nullability. +query TTT +SELECT c.name, c.type, c.nullable::text +FROM mz_catalog.mz_columns c +JOIN mz_catalog.mz_objects o ON c.id = o.id +JOIN mz_catalog.mz_schemas s ON o.schema_id = s.id +WHERE o.name = 'mz_aws_privatelink_connections' AND s.name = 'mz_catalog' +ORDER BY c.position +---- +id text false +principal text false + +# --- Empty on a fresh env ----------------------------------------------------- + +query I +SELECT count(*) FROM mz_aws_privatelink_connections +---- +0 + +# --- Skip on missing AWS context ---------------------------------------------- + +simple conn=mz_system,user=mz_system +ALTER SYSTEM SET max_aws_privatelink_connections = 5 +---- +COMPLETE 0 + +statement ok +CREATE CONNECTION plc TO AWS PRIVATELINK ( + SERVICE NAME 'com.amazonaws.vpce.us-east-1.vpce-svc-0e123abc123198abc', + AVAILABILITY ZONES ('use1-az1', 'use1-az2') +) + +# The connection exists as an aws-privatelink connection ... +query I +SELECT count(*) FROM mz_connections WHERE name = 'plc' AND type = 'aws-privatelink' +---- +1 + +# ... but with no AWS principal context the principal is NULL, so the view skips +# it, exactly like the packer did. +query I +SELECT count(*) FROM mz_aws_privatelink_connections +---- +0 + +# --- FK-like invariant: every row joins its parent connection ----------------- + +query I +SELECT count(*) FROM mz_aws_privatelink_connections plc +LEFT JOIN mz_connections c ON plc.id = c.id +WHERE c.id IS NULL +---- +0 + +statement ok +DROP CONNECTION plc + +simple conn=mz_system,user=mz_system +ALTER SYSTEM RESET max_aws_privatelink_connections +---- +COMPLETE 0 diff --git a/test/sqllogictest/mz_catalog_server_index_accounting.slt b/test/sqllogictest/mz_catalog_server_index_accounting.slt index 53ef49be9947b..3c108d65f8048 100644 --- a/test/sqllogictest/mz_catalog_server_index_accounting.slt +++ b/test/sqllogictest/mz_catalog_server_index_accounting.slt @@ -37,107 +37,107 @@ mz_arrangement_heap_capacity_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangemen mz_arrangement_heap_size_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_heap_size_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_heap_size_raw"␠("operator_id",␠"worker_id") mz_arrangement_records_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_records_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_records_raw"␠("operator_id",␠"worker_id") mz_arrangement_sharing_raw_s2_primary_idx CREATE␠INDEX␠"mz_arrangement_sharing_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_sharing_raw"␠("operator_id",␠"worker_id") -mz_cluster_auto_scaling_strategies_ind CREATE␠INDEX␠"mz_cluster_auto_scaling_strategies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s514␠AS␠"mz_internal"."mz_cluster_auto_scaling_strategies"]␠("cluster_id") -mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s761␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") +mz_cluster_auto_scaling_strategies_ind CREATE␠INDEX␠"mz_cluster_auto_scaling_strategies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s518␠AS␠"mz_internal"."mz_cluster_auto_scaling_strategies"]␠("cluster_id") +mz_cluster_deployment_lineage_ind CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s765␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id") mz_cluster_prometheus_metrics_s2_primary_idx CREATE␠INDEX␠"mz_cluster_prometheus_metrics_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_cluster_prometheus_metrics"␠("process_id",␠"metric_name",␠"labels") -mz_cluster_reconfigurations_ind CREATE␠INDEX␠"mz_cluster_reconfigurations_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s513␠AS␠"mz_internal"."mz_cluster_reconfigurations"]␠("cluster_id") -mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s755␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") -mz_cluster_replica_history_ind CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s611␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at") -mz_cluster_replica_metrics_history_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s519␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id") -mz_cluster_replica_metrics_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s520␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id") -mz_cluster_replica_name_history_ind CREATE␠INDEX␠"mz_cluster_replica_name_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s612␠AS␠"mz_internal"."mz_cluster_replica_name_history"]␠("id") -mz_cluster_replica_size_internal_ind CREATE␠INDEX␠"mz_cluster_replica_size_internal_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s509␠AS␠"mz_internal"."mz_cluster_replica_size_internal"]␠("size") -mz_cluster_replica_sizes_ind CREATE␠INDEX␠"mz_cluster_replica_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s508␠AS␠"mz_catalog"."mz_cluster_replica_sizes"]␠("size") -mz_cluster_replica_status_history_ind CREATE␠INDEX␠"mz_cluster_replica_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s521␠AS␠"mz_internal"."mz_cluster_replica_status_history"]␠("replica_id") -mz_cluster_replica_statuses_ind CREATE␠INDEX␠"mz_cluster_replica_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s522␠AS␠"mz_internal"."mz_cluster_replica_statuses"]␠("replica_id") -mz_cluster_replicas_ind CREATE␠INDEX␠"mz_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s518␠AS␠"mz_catalog"."mz_cluster_replicas"]␠("id") -mz_clusters_ind CREATE␠INDEX␠"mz_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s510␠AS␠"mz_catalog"."mz_clusters"]␠("id") -mz_columns_ind CREATE␠INDEX␠"mz_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s481␠AS␠"mz_catalog"."mz_columns"]␠("name") -mz_comments_ind CREATE␠INDEX␠"mz_comments_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s537␠AS␠"mz_internal"."mz_comments"]␠("id") +mz_cluster_reconfigurations_ind CREATE␠INDEX␠"mz_cluster_reconfigurations_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s517␠AS␠"mz_internal"."mz_cluster_reconfigurations"]␠("cluster_id") +mz_cluster_replica_frontiers_ind CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s759␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id") +mz_cluster_replica_history_ind CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s615␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at") +mz_cluster_replica_metrics_history_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s523␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id") +mz_cluster_replica_metrics_ind CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s524␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id") +mz_cluster_replica_name_history_ind CREATE␠INDEX␠"mz_cluster_replica_name_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s616␠AS␠"mz_internal"."mz_cluster_replica_name_history"]␠("id") +mz_cluster_replica_size_internal_ind CREATE␠INDEX␠"mz_cluster_replica_size_internal_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s513␠AS␠"mz_internal"."mz_cluster_replica_size_internal"]␠("size") +mz_cluster_replica_sizes_ind CREATE␠INDEX␠"mz_cluster_replica_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s512␠AS␠"mz_catalog"."mz_cluster_replica_sizes"]␠("size") +mz_cluster_replica_status_history_ind CREATE␠INDEX␠"mz_cluster_replica_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s525␠AS␠"mz_internal"."mz_cluster_replica_status_history"]␠("replica_id") +mz_cluster_replica_statuses_ind CREATE␠INDEX␠"mz_cluster_replica_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s526␠AS␠"mz_internal"."mz_cluster_replica_statuses"]␠("replica_id") +mz_cluster_replicas_ind CREATE␠INDEX␠"mz_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s522␠AS␠"mz_catalog"."mz_cluster_replicas"]␠("id") +mz_clusters_ind CREATE␠INDEX␠"mz_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s514␠AS␠"mz_catalog"."mz_clusters"]␠("id") +mz_columns_ind CREATE␠INDEX␠"mz_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s485␠AS␠"mz_catalog"."mz_columns"]␠("name") +mz_comments_ind CREATE␠INDEX␠"mz_comments_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s541␠AS␠"mz_internal"."mz_comments"]␠("id") mz_compute_dataflow_global_ids_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_dataflow_global_ids_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_dataflow_global_ids_per_worker"␠("id",␠"worker_id",␠"global_id") -mz_compute_dependencies_ind CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s737␠AS␠"mz_internal"."mz_compute_dependencies"]␠("dependency_id") +mz_compute_dependencies_ind CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s741␠AS␠"mz_internal"."mz_compute_dependencies"]␠("dependency_id") mz_compute_error_counts_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_error_counts_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_error_counts_raw"␠("export_id",␠"worker_id") mz_compute_exports_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_exports_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_exports_per_worker"␠("export_id",␠"worker_id") mz_compute_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_frontiers_per_worker"␠("export_id",␠"worker_id") -mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s746␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") +mz_compute_hydration_times_ind CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s750␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id") mz_compute_hydration_times_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_hydration_times_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_hydration_times_per_worker"␠("export_id",␠"worker_id") mz_compute_import_frontiers_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_import_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_import_frontiers_per_worker"␠("export_id",␠"import_id",␠"worker_id") mz_compute_lir_mapping_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_lir_mapping_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_lir_mapping_per_worker"␠("global_id",␠"lir_id",␠"worker_id") mz_compute_operator_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_durations_histogram_raw"␠("id",␠"worker_id",␠"duration_ns") mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_compute_operator_hydration_statuses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_hydration_statuses_per_worker"␠("export_id",␠"lir_id",␠"worker_id") -mz_connections_ind CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s516␠AS␠"mz_catalog"."mz_connections"]␠("schema_id") -mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s742␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") -mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s741␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") -mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s740␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") -mz_databases_ind CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s479␠AS␠"mz_catalog"."mz_databases"]␠("name") +mz_connections_ind CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s520␠AS␠"mz_catalog"."mz_connections"]␠("schema_id") +mz_console_cluster_utilization_overview_24h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_24h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s746␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_24h"]␠("cluster_id") +mz_console_cluster_utilization_overview_3h_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_3h_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s745␠AS␠"mz_internal"."mz_console_cluster_utilization_overview_3h"]␠("cluster_id") +mz_console_cluster_utilization_overview_ind CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s744␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id") +mz_databases_ind CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s483␠AS␠"mz_catalog"."mz_databases"]␠("name") mz_dataflow_addresses_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_addresses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_addresses_per_worker"␠("id",␠"worker_id") mz_dataflow_channels_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_channels_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_channels_per_worker"␠("id",␠"worker_id") mz_dataflow_operator_reachability_raw_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operator_reachability_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operator_reachability_raw"␠("id",␠"worker_id",␠"source",␠"port",␠"update_type",␠"time") mz_dataflow_operators_per_worker_s2_primary_idx CREATE␠INDEX␠"mz_dataflow_operators_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_operators_per_worker"␠("id",␠"worker_id") -mz_frontiers_ind CREATE␠INDEX␠"mz_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s728␠AS␠"mz_internal"."mz_frontiers"]␠("object_id") -mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s757␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") -mz_indexes_ind CREATE␠INDEX␠"mz_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s482␠AS␠"mz_catalog"."mz_indexes"]␠("id") -mz_kafka_sources_ind CREATE␠INDEX␠"mz_kafka_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s476␠AS␠"mz_catalog"."mz_kafka_sources"]␠("id") -mz_materialized_views_ind CREATE␠INDEX␠"mz_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s540␠AS␠"mz_catalog"."mz_materialized_views"]␠("id") +mz_frontiers_ind CREATE␠INDEX␠"mz_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s732␠AS␠"mz_internal"."mz_frontiers"]␠("object_id") +mz_hydration_statuses_ind CREATE␠INDEX␠"mz_hydration_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s761␠AS␠"mz_internal"."mz_hydration_statuses"]␠("object_id",␠"replica_id") +mz_indexes_ind CREATE␠INDEX␠"mz_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s486␠AS␠"mz_catalog"."mz_indexes"]␠("id") +mz_kafka_sources_ind CREATE␠INDEX␠"mz_kafka_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s480␠AS␠"mz_catalog"."mz_kafka_sources"]␠("id") +mz_materialized_views_ind CREATE␠INDEX␠"mz_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s544␠AS␠"mz_catalog"."mz_materialized_views"]␠("id") mz_message_batch_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_batch_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_batch_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_batch_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_received_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") mz_message_counts_sent_raw_s2_primary_idx CREATE␠INDEX␠"mz_message_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id") -mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s831␠AS␠"mz_internal"."mz_notices"]␠("id") -mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s749␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") -mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s749␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") -mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s747␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") -mz_object_dependencies_ind CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s477␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id") -mz_object_history_ind CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s551␠AS␠"mz_internal"."mz_object_history"]␠("id") -mz_object_lifetimes_ind CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s552␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id") -mz_object_transitive_dependencies_ind CREATE␠INDEX␠"mz_object_transitive_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s568␠AS␠"mz_internal"."mz_object_transitive_dependencies"]␠("object_id") -mz_objects_ind CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s548␠AS␠"mz_catalog"."mz_objects"]␠("schema_id") +mz_notices_ind CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s835␠AS␠"mz_internal"."mz_notices"]␠("id") +mz_object_arrangement_size_history_object_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_object_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("object_id") +mz_object_arrangement_size_history_ts_ind CREATE␠INDEX␠"mz_object_arrangement_size_history_ts_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s753␠AS␠"mz_internal"."mz_object_arrangement_size_history"]␠("collection_timestamp") +mz_object_arrangement_sizes_ind CREATE␠INDEX␠"mz_object_arrangement_sizes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s751␠AS␠"mz_internal"."mz_object_arrangement_sizes"]␠("replica_id") +mz_object_dependencies_ind CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s481␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id") +mz_object_history_ind CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s555␠AS␠"mz_internal"."mz_object_history"]␠("id") +mz_object_lifetimes_ind CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s556␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id") +mz_object_transitive_dependencies_ind CREATE␠INDEX␠"mz_object_transitive_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s572␠AS␠"mz_internal"."mz_object_transitive_dependencies"]␠("object_id") +mz_objects_ind CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s552␠AS␠"mz_catalog"."mz_objects"]␠("schema_id") mz_peek_durations_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_peek_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_peek_durations_histogram_raw"␠("worker_id",␠"type",␠"duration_ns") -mz_recent_activity_log_thinned_ind CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s712␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash") -mz_recent_sql_text_ind CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s708␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash") -mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s824␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") -mz_roles_ind CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s500␠AS␠"mz_catalog"."mz_roles"]␠("id") +mz_recent_activity_log_thinned_ind CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s716␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash") +mz_recent_sql_text_ind CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s712␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash") +mz_recent_storage_usage_ind CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s828␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id") +mz_roles_ind CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s504␠AS␠"mz_catalog"."mz_roles"]␠("id") mz_scheduling_elapsed_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_elapsed_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_elapsed_raw"␠("id",␠"worker_id") mz_scheduling_parks_histogram_raw_s2_primary_idx CREATE␠INDEX␠"mz_scheduling_parks_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_parks_histogram_raw"␠("worker_id",␠"slept_for_ns",␠"requested_ns") -mz_schemas_ind CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s480␠AS␠"mz_catalog"."mz_schemas"]␠("database_id") -mz_secrets_ind CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s515␠AS␠"mz_catalog"."mz_secrets"]␠("name") -mz_show_all_objects_ind CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s596␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id") -mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s759␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") -mz_show_clusters_ind CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s598␠AS␠"mz_internal"."mz_show_clusters"]␠("name") -mz_show_columns_ind CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s597␠AS␠"mz_internal"."mz_show_columns"]␠("id") -mz_show_connections_ind CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s606␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id") -mz_show_databases_ind CREATE␠INDEX␠"mz_show_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s600␠AS␠"mz_internal"."mz_show_databases"]␠("name") -mz_show_indexes_ind CREATE␠INDEX␠"mz_show_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s610␠AS␠"mz_internal"."mz_show_indexes"]␠("schema_id") -mz_show_materialized_views_ind CREATE␠INDEX␠"mz_show_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s609␠AS␠"mz_internal"."mz_show_materialized_views"]␠("schema_id") -mz_show_roles_ind CREATE␠INDEX␠"mz_show_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s605␠AS␠"mz_internal"."mz_show_roles"]␠("name") -mz_show_schemas_ind CREATE␠INDEX␠"mz_show_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s601␠AS␠"mz_internal"."mz_show_schemas"]␠("database_id") -mz_show_secrets_ind CREATE␠INDEX␠"mz_show_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s599␠AS␠"mz_internal"."mz_show_secrets"]␠("schema_id") -mz_show_sinks_ind CREATE␠INDEX␠"mz_show_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s608␠AS␠"mz_internal"."mz_show_sinks"]␠("schema_id") -mz_show_sources_ind CREATE␠INDEX␠"mz_show_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s607␠AS␠"mz_internal"."mz_show_sources"]␠("schema_id") -mz_show_tables_ind CREATE␠INDEX␠"mz_show_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s602␠AS␠"mz_internal"."mz_show_tables"]␠("schema_id") -mz_show_types_ind CREATE␠INDEX␠"mz_show_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s604␠AS␠"mz_internal"."mz_show_types"]␠("schema_id") -mz_show_views_ind CREATE␠INDEX␠"mz_show_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s603␠AS␠"mz_internal"."mz_show_views"]␠("schema_id") -mz_sink_statistics_ind CREATE␠INDEX␠"mz_sink_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s725␠AS␠"mz_internal"."mz_sink_statistics"]␠("id",␠"replica_id") -mz_sink_status_history_ind CREATE␠INDEX␠"mz_sink_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s697␠AS␠"mz_internal"."mz_sink_status_history"]␠("sink_id") -mz_sink_statuses_ind CREATE␠INDEX␠"mz_sink_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s698␠AS␠"mz_internal"."mz_sink_statuses"]␠("id") -mz_sinks_ind CREATE␠INDEX␠"mz_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s492␠AS␠"mz_catalog"."mz_sinks"]␠("id") -mz_source_statistics_ind CREATE␠INDEX␠"mz_source_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s723␠AS␠"mz_internal"."mz_source_statistics"]␠("id",␠"replica_id") -mz_source_statistics_with_history_ind CREATE␠INDEX␠"mz_source_statistics_with_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s721␠AS␠"mz_internal"."mz_source_statistics_with_history"]␠("id",␠"replica_id") -mz_source_status_history_ind CREATE␠INDEX␠"mz_source_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s699␠AS␠"mz_internal"."mz_source_status_history"]␠("source_id") -mz_source_statuses_ind CREATE␠INDEX␠"mz_source_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s716␠AS␠"mz_internal"."mz_source_statuses"]␠("id") -mz_sources_ind CREATE␠INDEX␠"mz_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s485␠AS␠"mz_catalog"."mz_sources"]␠("id") -mz_tables_ind CREATE␠INDEX␠"mz_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s484␠AS␠"mz_catalog"."mz_tables"]␠("schema_id") -mz_types_ind CREATE␠INDEX␠"mz_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s494␠AS␠"mz_catalog"."mz_types"]␠("schema_id") -mz_views_ind CREATE␠INDEX␠"mz_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s493␠AS␠"mz_catalog"."mz_views"]␠("schema_id") -mz_wallclock_global_lag_recent_history_ind CREATE␠INDEX␠"mz_wallclock_global_lag_recent_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s732␠AS␠"mz_internal"."mz_wallclock_global_lag_recent_history"]␠("object_id") -mz_webhook_sources_ind CREATE␠INDEX␠"mz_webhook_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s538␠AS␠"mz_internal"."mz_webhook_sources"]␠("id") -pg_attrdef_all_databases_ind CREATE␠INDEX␠"pg_attrdef_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s637␠AS␠"mz_internal"."pg_attrdef_all_databases"]␠("oid",␠"adrelid",␠"adnum",␠"adbin",␠"adsrc") -pg_attribute_all_databases_ind CREATE␠INDEX␠"pg_attribute_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s630␠AS␠"mz_internal"."pg_attribute_all_databases"]␠("attrelid",␠"attname",␠"atttypid",␠"attlen",␠"attnum",␠"atttypmod",␠"attnotnull",␠"atthasdef",␠"attidentity",␠"attgenerated",␠"attisdropped",␠"attcollation",␠"database_name",␠"pg_type_database_name") -pg_authid_core_ind CREATE␠INDEX␠"pg_authid_core_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s647␠AS␠"mz_internal"."pg_authid_core"]␠("rolname") -pg_class_all_databases_ind CREATE␠INDEX␠"pg_class_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s618␠AS␠"mz_internal"."pg_class_all_databases"]␠("relname") -pg_description_all_databases_ind CREATE␠INDEX␠"pg_description_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s627␠AS␠"mz_internal"."pg_description_all_databases"]␠("objoid",␠"classoid",␠"objsubid",␠"description",␠"oid_database_name",␠"class_database_name") -pg_namespace_all_databases_ind CREATE␠INDEX␠"pg_namespace_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s615␠AS␠"mz_internal"."pg_namespace_all_databases"]␠("nspname") -pg_type_all_databases_ind CREATE␠INDEX␠"pg_type_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s624␠AS␠"mz_internal"."pg_type_all_databases"]␠("oid") +mz_schemas_ind CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s484␠AS␠"mz_catalog"."mz_schemas"]␠("database_id") +mz_secrets_ind CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s519␠AS␠"mz_catalog"."mz_secrets"]␠("name") +mz_show_all_objects_ind CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s600␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id") +mz_show_cluster_replicas_ind CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s763␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster") +mz_show_clusters_ind CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s602␠AS␠"mz_internal"."mz_show_clusters"]␠("name") +mz_show_columns_ind CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s601␠AS␠"mz_internal"."mz_show_columns"]␠("id") +mz_show_connections_ind CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s610␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id") +mz_show_databases_ind CREATE␠INDEX␠"mz_show_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s604␠AS␠"mz_internal"."mz_show_databases"]␠("name") +mz_show_indexes_ind CREATE␠INDEX␠"mz_show_indexes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s614␠AS␠"mz_internal"."mz_show_indexes"]␠("schema_id") +mz_show_materialized_views_ind CREATE␠INDEX␠"mz_show_materialized_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s613␠AS␠"mz_internal"."mz_show_materialized_views"]␠("schema_id") +mz_show_roles_ind CREATE␠INDEX␠"mz_show_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s609␠AS␠"mz_internal"."mz_show_roles"]␠("name") +mz_show_schemas_ind CREATE␠INDEX␠"mz_show_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s605␠AS␠"mz_internal"."mz_show_schemas"]␠("database_id") +mz_show_secrets_ind CREATE␠INDEX␠"mz_show_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s603␠AS␠"mz_internal"."mz_show_secrets"]␠("schema_id") +mz_show_sinks_ind CREATE␠INDEX␠"mz_show_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s612␠AS␠"mz_internal"."mz_show_sinks"]␠("schema_id") +mz_show_sources_ind CREATE␠INDEX␠"mz_show_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s611␠AS␠"mz_internal"."mz_show_sources"]␠("schema_id") +mz_show_tables_ind CREATE␠INDEX␠"mz_show_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s606␠AS␠"mz_internal"."mz_show_tables"]␠("schema_id") +mz_show_types_ind CREATE␠INDEX␠"mz_show_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s608␠AS␠"mz_internal"."mz_show_types"]␠("schema_id") +mz_show_views_ind CREATE␠INDEX␠"mz_show_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s607␠AS␠"mz_internal"."mz_show_views"]␠("schema_id") +mz_sink_statistics_ind CREATE␠INDEX␠"mz_sink_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s729␠AS␠"mz_internal"."mz_sink_statistics"]␠("id",␠"replica_id") +mz_sink_status_history_ind CREATE␠INDEX␠"mz_sink_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s701␠AS␠"mz_internal"."mz_sink_status_history"]␠("sink_id") +mz_sink_statuses_ind CREATE␠INDEX␠"mz_sink_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s702␠AS␠"mz_internal"."mz_sink_statuses"]␠("id") +mz_sinks_ind CREATE␠INDEX␠"mz_sinks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s496␠AS␠"mz_catalog"."mz_sinks"]␠("id") +mz_source_statistics_ind CREATE␠INDEX␠"mz_source_statistics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s727␠AS␠"mz_internal"."mz_source_statistics"]␠("id",␠"replica_id") +mz_source_statistics_with_history_ind CREATE␠INDEX␠"mz_source_statistics_with_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s725␠AS␠"mz_internal"."mz_source_statistics_with_history"]␠("id",␠"replica_id") +mz_source_status_history_ind CREATE␠INDEX␠"mz_source_status_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s703␠AS␠"mz_internal"."mz_source_status_history"]␠("source_id") +mz_source_statuses_ind CREATE␠INDEX␠"mz_source_statuses_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s720␠AS␠"mz_internal"."mz_source_statuses"]␠("id") +mz_sources_ind CREATE␠INDEX␠"mz_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s489␠AS␠"mz_catalog"."mz_sources"]␠("id") +mz_tables_ind CREATE␠INDEX␠"mz_tables_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s488␠AS␠"mz_catalog"."mz_tables"]␠("schema_id") +mz_types_ind CREATE␠INDEX␠"mz_types_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s498␠AS␠"mz_catalog"."mz_types"]␠("schema_id") +mz_views_ind CREATE␠INDEX␠"mz_views_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s497␠AS␠"mz_catalog"."mz_views"]␠("schema_id") +mz_wallclock_global_lag_recent_history_ind CREATE␠INDEX␠"mz_wallclock_global_lag_recent_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s736␠AS␠"mz_internal"."mz_wallclock_global_lag_recent_history"]␠("object_id") +mz_webhook_sources_ind CREATE␠INDEX␠"mz_webhook_sources_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s542␠AS␠"mz_internal"."mz_webhook_sources"]␠("id") +pg_attrdef_all_databases_ind CREATE␠INDEX␠"pg_attrdef_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s641␠AS␠"mz_internal"."pg_attrdef_all_databases"]␠("oid",␠"adrelid",␠"adnum",␠"adbin",␠"adsrc") +pg_attribute_all_databases_ind CREATE␠INDEX␠"pg_attribute_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s634␠AS␠"mz_internal"."pg_attribute_all_databases"]␠("attrelid",␠"attname",␠"atttypid",␠"attlen",␠"attnum",␠"atttypmod",␠"attnotnull",␠"atthasdef",␠"attidentity",␠"attgenerated",␠"attisdropped",␠"attcollation",␠"database_name",␠"pg_type_database_name") +pg_authid_core_ind CREATE␠INDEX␠"pg_authid_core_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s651␠AS␠"mz_internal"."pg_authid_core"]␠("rolname") +pg_class_all_databases_ind CREATE␠INDEX␠"pg_class_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s622␠AS␠"mz_internal"."pg_class_all_databases"]␠("relname") +pg_description_all_databases_ind CREATE␠INDEX␠"pg_description_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s631␠AS␠"mz_internal"."pg_description_all_databases"]␠("objoid",␠"classoid",␠"objsubid",␠"description",␠"oid_database_name",␠"class_database_name") +pg_namespace_all_databases_ind CREATE␠INDEX␠"pg_namespace_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s619␠AS␠"mz_internal"."pg_namespace_all_databases"]␠("nspname") +pg_type_all_databases_ind CREATE␠INDEX␠"pg_type_all_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s628␠AS␠"mz_internal"."pg_type_all_databases"]␠("oid") # Record all transitive dependencies (tables, sources, views, mvs) of indexes on # the mz_catalog_server cluster. diff --git a/test/sqllogictest/mz_kafka_connections.slt b/test/sqllogictest/mz_kafka_connections.slt new file mode 100644 index 0000000000000..18e30b1b71b10 --- /dev/null +++ b/test/sqllogictest/mz_kafka_connections.slt @@ -0,0 +1,116 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Behavioural tests for mz_catalog.mz_kafka_connections. The view is a +# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see +# MZ_KAFKA_CONNECTIONS in src/catalog/src/builtin/mz_catalog.rs and the +# parse_connection_details helper in +# src/expr/src/scalar/func/impls/jsonb.rs. Rows with a live broker are +# exercised by test/testdrive/connection-create-drop.td; this file pins the +# column shape, the progress-topic default, and the parent-type filtering, +# none of which need a live Kafka. + +mode cockroach + +reset-server + +# --- Schema ------------------------------------------------------------------- + +# Lock down: 3 columns, canonical order, types, and nullability. +query TTT +SELECT c.name, c.type, c.nullable::text +FROM mz_catalog.mz_columns c +JOIN mz_catalog.mz_objects o ON c.id = o.id +JOIN mz_catalog.mz_schemas s ON o.schema_id = s.id +WHERE o.name = 'mz_kafka_connections' AND s.name = 'mz_catalog' +ORDER BY c.position +---- +id text false +brokers text[] false +sink_progress_topic text false + +# --- Empty on a fresh env ----------------------------------------------------- + +query I +SELECT count(*) FROM mz_kafka_connections +---- +0 + +# --- A kafka connection populates the view ------------------------------------ + +statement ok +CREATE CONNECTION kc TO KAFKA (BROKER 'localhost:9092', SECURITY PROTOCOL PLAINTEXT) WITH (VALIDATE = false) + +query T +SELECT brokers FROM mz_kafka_connections kc +JOIN mz_connections c USING (id) +WHERE c.name = 'kc' +---- +{localhost:9092} + +# The default progress topic reconstructs KafkaConnection::progress_topic. +query T +SELECT sink_progress_topic = + '_materialize-progress-' || mz_environment_id() || '-' || id +FROM mz_kafka_connections kc +JOIN mz_connections c USING (id) +WHERE c.name = 'kc' +---- +true + +# An explicit PROGRESS TOPIC is passed through verbatim. +statement ok +CREATE CONNECTION kc_progress TO KAFKA ( + BROKER 'localhost:9092', + PROGRESS TOPIC 'my_progress', + SECURITY PROTOCOL PLAINTEXT +) WITH (VALIDATE = false) + +query T +SELECT sink_progress_topic FROM mz_kafka_connections kc +JOIN mz_connections c USING (id) +WHERE c.name = 'kc_progress' +---- +my_progress + +# --- Non-kafka connections are excluded --------------------------------------- + +statement ok +CREATE CONNECTION sc TO SSH TUNNEL (HOST 'ssh.example.com', USER 'mz') + +query I +SELECT count(*) FROM mz_kafka_connections kc +JOIN mz_connections c USING (id) +WHERE c.type != 'kafka' +---- +0 + +# --- FK-like invariant: every row joins its parent connection ----------------- + +query I +SELECT count(*) FROM mz_kafka_connections kc +LEFT JOIN mz_connections c ON kc.id = c.id +WHERE c.id IS NULL +---- +0 + +# No duplicate ids. +query I +SELECT count(*) - count(DISTINCT id) FROM mz_kafka_connections +---- +0 + +statement ok +DROP CONNECTION kc + +statement ok +DROP CONNECTION kc_progress + +statement ok +DROP CONNECTION sc diff --git a/test/sqllogictest/mz_kafka_source_tables.slt b/test/sqllogictest/mz_kafka_source_tables.slt index 2bdc3acd5f3ae..bf2e55335d509 100644 --- a/test/sqllogictest/mz_kafka_source_tables.slt +++ b/test/sqllogictest/mz_kafka_source_tables.slt @@ -33,7 +33,7 @@ ORDER BY c.position ---- id text false topic text false -envelope_type text true +envelope_type text false key_format text true value_format text true diff --git a/test/sqllogictest/mz_ssh_tunnel_connections.slt b/test/sqllogictest/mz_ssh_tunnel_connections.slt new file mode 100644 index 0000000000000..b555697bb352e --- /dev/null +++ b/test/sqllogictest/mz_ssh_tunnel_connections.slt @@ -0,0 +1,88 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Behavioural tests for mz_catalog.mz_ssh_tunnel_connections. The view is a +# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see +# MZ_SSH_TUNNEL_CONNECTIONS in src/catalog/src/builtin/mz_catalog.rs and the +# parse_connection_details helper in +# src/expr/src/scalar/func/impls/jsonb.rs. The end-to-end SSH tunnel behaviour +# is exercised by test/ssh-connection/*; this file pins the column shape and +# the parent-type filtering. + +mode cockroach + +reset-server + +# --- Schema ------------------------------------------------------------------- + +# Lock down: 3 columns, canonical order, types, and nullability. +query TTT +SELECT c.name, c.type, c.nullable::text +FROM mz_catalog.mz_columns c +JOIN mz_catalog.mz_objects o ON c.id = o.id +JOIN mz_catalog.mz_schemas s ON o.schema_id = s.id +WHERE o.name = 'mz_ssh_tunnel_connections' AND s.name = 'mz_catalog' +ORDER BY c.position +---- +id text false +public_key_1 text false +public_key_2 text false + +# --- Empty on a fresh env ----------------------------------------------------- + +query I +SELECT count(*) FROM mz_ssh_tunnel_connections +---- +0 + +# --- An ssh connection populates the view with generated public keys ---------- + +statement ok +CREATE CONNECTION sc TO SSH TUNNEL (HOST 'ssh.example.com', USER 'mz') + +query TT +SELECT public_key_1 LIKE 'ssh-ed25519%', public_key_2 LIKE 'ssh-ed25519%' +FROM mz_ssh_tunnel_connections stc +JOIN mz_connections c USING (id) +WHERE c.name = 'sc' +---- +true true + +# --- Non-ssh connections are excluded ----------------------------------------- + +statement ok +CREATE CONNECTION kc TO KAFKA (BROKER 'localhost:9092', SECURITY PROTOCOL PLAINTEXT) WITH (VALIDATE = false) + +query I +SELECT count(*) FROM mz_ssh_tunnel_connections stc +JOIN mz_connections c USING (id) +WHERE c.type != 'ssh-tunnel' +---- +0 + +# --- FK-like invariant: every row joins its parent connection ----------------- + +query I +SELECT count(*) FROM mz_ssh_tunnel_connections stc +LEFT JOIN mz_connections c ON stc.id = c.id +WHERE c.id IS NULL +---- +0 + +# No duplicate ids. +query I +SELECT count(*) - count(DISTINCT id) FROM mz_ssh_tunnel_connections +---- +0 + +statement ok +DROP CONNECTION sc + +statement ok +DROP CONNECTION kc diff --git a/test/sqllogictest/network_policy.slt b/test/sqllogictest/network_policy.slt index 07a4550dbb3b6..a3b2daeb7d246 100644 --- a/test/sqllogictest/network_policy.slt +++ b/test/sqllogictest/network_policy.slt @@ -271,3 +271,75 @@ simple conn=mz_system,user=mz_system DROP ROLE np_owner ---- COMPLETE 0 + +# Regression test: ALTER NETWORK POLICY must correctly resolve quoted +# (hyphenated) names. Previously, plan_alter_network_policy passed the +# display-quoted form (via Ident::to_string) to resolve_network_policy, +# causing "np-x" to be looked up as "\"np-x\"" and fail. + +simple conn=mz_system,user=mz_system +CREATE NETWORK POLICY "np-x" (RULES (r0 (action='allow', direction='ingress', address='0.0.0.0/0'))) +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +ALTER NETWORK POLICY "np-x" SET (RULES (r1 (action='allow', direction='ingress', address='1.1.1.1/32'))) +---- +COMPLETE 0 + +query TTTT colnames +SELECT name, action, address, direction FROM mz_internal.mz_network_policy_rules WHERE policy_id = (SELECT id FROM mz_internal.mz_network_policies WHERE name = 'np-x') +---- +name action address direction +r1 +allow +1.1.1.1/32 +ingress + +simple conn=mz_system,user=mz_system +DROP NETWORK POLICY "np-x" +---- +COMPLETE 0 + +# Regression test: GRANT/REVOKE on a hyphenated network policy name. The +# fold_network_policy_name function had the same bug with quoted identifiers. + +simple conn=mz_system,user=mz_system +CREATE ROLE np_hyphen_grantee +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +CREATE NETWORK POLICY "np-hyphen" (RULES (r0 (action='allow', direction='ingress', address='0.0.0.0/0'))) +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +GRANT USAGE ON NETWORK POLICY "np-hyphen" TO np_hyphen_grantee +---- +COMPLETE 0 + +query TT rowsort +SELECT np.name, mz_internal.mz_aclitem_privileges(priv) +FROM mz_internal.mz_network_policies np, + unnest(np.privileges) AS priv, + mz_roles r +WHERE mz_internal.mz_aclitem_grantee(priv) = r.id AND r.name = 'np_hyphen_grantee' +---- +np-hyphen +U + +simple conn=mz_system,user=mz_system +REVOKE USAGE ON NETWORK POLICY "np-hyphen" FROM np_hyphen_grantee +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +DROP NETWORK POLICY "np-hyphen" +---- +COMPLETE 0 + +simple conn=mz_system,user=mz_system +DROP ROLE np_hyphen_grantee +---- +COMPLETE 0 diff --git a/test/sqllogictest/oid.slt b/test/sqllogictest/oid.slt index 8b4f2ce860925..a563054773926 100644 --- a/test/sqllogictest/oid.slt +++ b/test/sqllogictest/oid.slt @@ -1096,6 +1096,8 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 16945 mz_role_parameters 16946 mz_materialized_view_refresh_strategies 16947 mz_cluster_schedules +16948 mz_postgres_source_tables +16949 mz_mysql_source_tables 16950 mz_kafka_sources_ind 16951 mz_webhook_sources_ind 16952 mz_history_retention_strategies @@ -1165,6 +1167,7 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17022 mz_wallclock_global_lag_history 17023 mz_wallclock_global_lag_recent_history 17024 mz_wallclock_global_lag_recent_history_ind +17025 mz_kafka_source_tables 17030 mz_objects_id_namespace_types 17031 mz_object_history 17032 mz_object_history_ind @@ -1188,6 +1191,7 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17054 mz_wallclock_global_lag 17055 mz_wallclock_global_lag_histogram_raw 17056 mz_wallclock_global_lag_histogram +17057 mz_sql_server_source_tables 17058 mz_license_keys 17059 mz_role_auth 17060 mz_iceberg_sinks @@ -1236,8 +1240,8 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid 17104 mz_cluster_auto_scaling_strategies_ind 17105 parse_postgres_source_details 17106 parse_kafka_source_details -17107 mz_postgres_source_tables -17108 mz_mysql_source_tables -17109 mz_sql_server_source_tables -17110 mz_kafka_source_tables 17111 parse_source_export_details +17112 parse_connection_details +17113 mz_aws_account_id +17114 mz_aws_external_id_prefix +17115 mz_aws_connection_role_arn diff --git a/test/sqllogictest/transform/fold_vs_dataflow/1_numbers_dataflow.slt b/test/sqllogictest/transform/fold_vs_dataflow/1_numbers_dataflow.slt index 41d9944eab7e4..790665ccd9131 100644 --- a/test/sqllogictest/transform/fold_vs_dataflow/1_numbers_dataflow.slt +++ b/test/sqllogictest/transform/fold_vs_dataflow/1_numbers_dataflow.slt @@ -22,7 +22,7 @@ statement ok INSERT INTO t_using_dataflow_rendering VALUES (1e-39::real, 1e-39::double, 1e-39::numeric); # DIFF TO CONSTANT FOLDING (SELECT on types [REAL, DOUBLE])! -# to be addressed with https://github.com/MaterializeInc/database-issues/issues/4341 +# to be addressed with DB-191 query RRR SELECT * FROM t_using_dataflow_rendering UNION ALL @@ -55,7 +55,7 @@ statement ok INSERT INTO t_using_dataflow_rendering VALUES (1e38::real, 1e38::double, 1e38::numeric); # DIFF TO CONSTANT FOLDING (SELECT on types [REAL, DOUBLE])! -# to be addressed with https://github.com/MaterializeInc/database-issues/issues/4341 +# to be addressed with DB-191 # The dataflow float sum accumulates in a fixed-point domain that cannot # represent values this large (|x| > ~1.01e31). Such values wrap modulo 2^128, # which is what makes cancelling sums correct (database-issues#11265) at the cost diff --git a/test/sqllogictest/transform/fold_vs_dataflow/3_number_aggfns_dataflow.slt b/test/sqllogictest/transform/fold_vs_dataflow/3_number_aggfns_dataflow.slt index cb61d9cf64caa..2334c5fc0d0d7 100644 --- a/test/sqllogictest/transform/fold_vs_dataflow/3_number_aggfns_dataflow.slt +++ b/test/sqllogictest/transform/fold_vs_dataflow/3_number_aggfns_dataflow.slt @@ -24,7 +24,7 @@ INSERT INTO t_using_dataflow_rendering VALUES (2, 2, 2, 32767::INT2, 1::INT2); # DIFF TO CONSTANT FOLDING ([SUM, AVG] with types [REAL, DOUBLE])! -# to be addressed with https://github.com/MaterializeInc/database-issues/issues/4341 +# to be addressed with DB-191 # The dataflow float sum accumulates in a fixed-point domain that cannot # represent values this large (|x| > ~1.01e31). Such values wrap modulo 2^128 # (database-issues#11265), so SUM/AVG over REAL/DOUBLE here are nonsensical; the diff --git a/test/sqllogictest/unsigned_int.slt b/test/sqllogictest/unsigned_int.slt index 93b083f3809ab..c7293decad5b3 100644 --- a/test/sqllogictest/unsigned_int.slt +++ b/test/sqllogictest/unsigned_int.slt @@ -1353,7 +1353,7 @@ SELECT STDDEV(a) FROM t8 ---- 4611686017711549096.944 -# Avoid overflow (known issue https://github.com/MaterializeInc/database-issues/issues/4341) +# Avoid overflow (known issue DB-191) statement ok DROP TABLE t8 diff --git a/test/testdrive/catalog.td b/test/testdrive/catalog.td index ab904914157af..eef5713d4fc51 100644 --- a/test/testdrive/catalog.td +++ b/test/testdrive/catalog.td @@ -511,7 +511,6 @@ mz_cluster_replica_frontiers source "" name comment -------------------------------------- mz_array_types "" -mz_aws_privatelink_connections "" mz_base_types "" mz_cluster_replica_sizes "" mz_columns "" @@ -519,7 +518,6 @@ mz_egress_ips "" mz_functions "" mz_iceberg_sinks "" mz_index_columns "" -mz_kafka_connections "" mz_kafka_sinks "" mz_list_types "" mz_map_types "" @@ -527,7 +525,6 @@ mz_operators "" mz_pseudo_types "" mz_role_auth "" mz_sinks "" -mz_ssh_tunnel_connections "" mz_tables "" mz_types "" mz_views "" @@ -546,12 +543,14 @@ mz_timezone_names "" name cluster comment -------------------------------------------------- mz_audit_events mz_catalog_server "" +mz_aws_privatelink_connections mz_catalog_server "" mz_clusters mz_catalog_server "" mz_cluster_replicas mz_catalog_server "" mz_connections mz_catalog_server "" mz_databases mz_catalog_server "" mz_default_privileges mz_catalog_server "" mz_indexes mz_catalog_server "" +mz_kafka_connections mz_catalog_server "" mz_kafka_sources mz_catalog_server "" mz_role_members mz_catalog_server "" mz_role_parameters mz_catalog_server "" @@ -559,6 +558,7 @@ mz_roles mz_catalog_server "" mz_schemas mz_catalog_server "" mz_secrets mz_catalog_server "" mz_sources mz_catalog_server "" +mz_ssh_tunnel_connections mz_catalog_server "" mz_system_privileges mz_catalog_server "" mz_materialized_views mz_catalog_server "" @@ -603,7 +603,6 @@ mz_wallclock_lag_history source "" name comment ------------------------------------------------ mz_aggregates "" -mz_aws_connections "" mz_cluster_replica_size_internal "" mz_history_retention_strategies "" mz_license_keys "" @@ -717,6 +716,7 @@ pg_authid_core "" > SHOW MATERIALIZED VIEWS FROM mz_internal name cluster comment ---------------------------------------------------- +mz_aws_connections mz_catalog_server "" mz_cluster_auto_scaling_strategies mz_catalog_server "" mz_cluster_reconfigurations mz_catalog_server "" mz_cluster_schedules mz_catalog_server "" @@ -830,7 +830,7 @@ test_table "" # `SHOW TABLES` and `mz_tables` should agree. > SELECT COUNT(*) FROM mz_tables WHERE id LIKE 's%' -38 +34 # There is one entry in mz_indexes for each field_number/expression of the index. > SELECT COUNT(id) FROM mz_indexes WHERE id LIKE 's%' diff --git a/test/testdrive/mz-sources.td b/test/testdrive/mz-sources.td index f2b1fe2fb5f34..49bd3db771ce7 100644 --- a/test/testdrive/mz-sources.td +++ b/test/testdrive/mz-sources.td @@ -178,3 +178,21 @@ testdrive-upsert-topic-${testdrive.seed} none text testdrive-none-topic-${testdrive.seed} none text text testdrive-dbzupsert-${testdrive.seed} debezium avro avro testdrive-upsert-topic-${testdrive.seed} upsert avro avro + +# A kafka source table that omits the ENVELOPE clause defaults to ENVELOPE NONE, +# so envelope_type must report 'none', not NULL. The omitted clause persists as +# absent in create_sql, and the view supplies the default. +> CREATE TABLE table_omit_envelope FROM SOURCE upsert_source (REFERENCE "testdrive-upsert-topic-${testdrive.seed}") + FORMAT TEXT + +> SELECT kst.envelope_type + FROM mz_internal.mz_kafka_source_tables kst + JOIN mz_tables t ON t.id = kst.id + WHERE t.name = 'table_omit_envelope' +none + +# A new-syntax kafka source ingests nothing into its own relation (envelopes are +# defined per table), so mz_sources.envelope_type is NULL for it, even though its +# tables report their own envelopes above. +> SELECT envelope_type FROM mz_sources WHERE name = 'upsert_source' + diff --git a/test/workload-replay/mzcompose.py b/test/workload-replay/mzcompose.py index a6377495e83b1..23dd1adc067b5 100644 --- a/test/workload-replay/mzcompose.py +++ b/test/workload-replay/mzcompose.py @@ -12,14 +12,20 @@ """ import argparse +import os import pathlib import random import time import yaml -from materialize import buildkite -from materialize.mzcompose.composition import Composition, WorkflowArgumentParser +from materialize import MZ_ROOT, buildkite +from materialize.mzcompose.composition import ( + Composition, + Service, + WorkflowArgumentParser, +) +from materialize.mzcompose.service import Service as ServiceDefinition from materialize.mzcompose.services.azurite import Azurite from materialize.mzcompose.services.kafka import Kafka from materialize.mzcompose.services.materialized import Materialized @@ -87,8 +93,40 @@ ], ), SqlServer(), + # Runs the console Playwright scalability suite against the console + # bundled in the materialized image. The image tag must match the + # @playwright/test version in console/package.json so the preinstalled + # browsers are compatible. + ServiceDefinition( + name="console-scalability-runner", + config={ + "image": "mcr.microsoft.com/playwright:v1.61.0-noble", + "volumes": [f"{MZ_ROOT}:/workdir"], + "working_dir": "/workdir/console", + "init": True, + # Chromium crashes with the 64MB docker default for /dev/shm. + "ipc": "host", + }, + ), + # Replays the console's captured /api/sql traffic against the SQL HTTP + # endpoint at increasing virtual-user counts. Reads the queries.json + # that dump-queries produces during the Playwright suite. + ServiceDefinition( + name="console-k6", + config={ + "image": "grafana/k6:1.6.0", + "volumes": [f"{MZ_ROOT}:/workdir"], + "working_dir": "/workdir/console/e2e-tests/k6", + # The summary export is written into the bind mount. + "propagate_uid_gid": True, + }, + ), ] +# UID/GID of the host user, used to run yarn commands so they don't create +# root-owned files in the bind-mounted volume. +_HOST_UID_GID = f"{os.getuid()}:{os.getgid()}" + def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: parser.add_argument( @@ -338,6 +376,227 @@ def workflow_benchmark(c: Composition, parser: WorkflowArgumentParser) -> None: ) +def workflow_console_scalability( + c: Composition, parser: WorkflowArgumentParser +) -> None: + """ + Run the console Playwright scalability suite against each captured + workload. + + Per workload: boot (objects, initial data, hydration), then run the + suite against the console bundled in the materialized image while + continuous ingestions and queries provide background load. The suite + only reports timings. Suite failures do not fail the workflow, the runs + exist to collect data for determining thresholds. + """ + parser.add_argument( + "--factor-initial-data", + type=float, + default=0.01, + help="scale factor for initial data generation", + ) + parser.add_argument( + "--factor-ingestions", + type=float, + default=1, + help="scale factor for runtime data ingestion rate", + ) + parser.add_argument( + "--factor-queries", + type=float, + default=1, + help="scale factor for runtime queries", + ) + parser.add_argument( + "--max-concurrent-queries", + type=int, + default=1000, + help="max. number of concurrent queries during continuous phase", + ) + parser.add_argument( + "--seed", + metavar="SEED", + type=str, + default=str(int(time.time())), + help="random seed", + ) + parser.add_argument("--verbose", action=argparse.BooleanOptionalAction) + parser.add_argument( + "--skip-large", + action="store_true", + help="skip workloads tagged `settings.large: true`", + ) + parser.add_argument( + "files", + nargs="*", + default=["*.yml"], + help="run against the specified files", + ) + args = parser.parse_args() + + print(f"-- Random seed is {args.seed}") + random.seed(args.seed) + update_captured_workloads_repo() + + all_paths = get_paths(args.files) + workloads: dict[pathlib.Path, dict] = {} + for path in all_paths: + workload = load_workload(path) + if args.skip_large and workload.get("settings", {}).get("large", False): + print(f"-- Skipping {path} (settings.large: true and --skip-large)") + continue + workloads[path] = workload + files: list[pathlib.Path] = buildkite.shard_list( + list(workloads.keys()), + lambda file: str(file), + ) + + k6_dir = MZ_ROOT / "console" / "e2e-tests" / "k6" + + def run(file: pathlib.Path) -> None: + workload = workloads[file] + settings = workload.get("settings", {}) + # When scale_data is false, use 100% initial data, like + # workflow_default. + factor_initial_data = args.factor_initial_data + if not settings.get("scale_data", True): + factor_initial_data = 1.0 + else: + factor_initial_data *= settings.get("factor_initial_data_multiplier", 1.0) + print_workload_stats(file, workload) + + # Reset any state left over from the previous workload. + service_names = [s.name for s in SERVICES] + try: + c.kill(*service_names) + except Exception: + pass + c.rm(*service_names, destroy_volumes=True) + c.rm_volumes("mzdata", force=True) + + # Prepare the Playwright runner up front so dependency installation + # does not eat into the measurement window. Cheap after the first + # workload, node_modules lives in the bind-mounted checkout. + c.up(Service("console-scalability-runner", idle=True)) + c.exec( + "console-scalability-runner", + "sh", + "-c", + "corepack enable", + env_extra={"COREPACK_ENABLE_DOWNLOAD_PROMPT": "0"}, + ) + _console_runner_sh( + c, "yarn install --immutable --network-timeout 30000", user=_HOST_UID_GID + ) + # Fetches matching browsers in case @playwright/test drifted from the + # image tag. Runs as root because it writes to /ms-playwright. + _console_runner_sh(c, "node_modules/.bin/playwright install chromium") + + # k6 must only ever replay queries captured from this workload's run, + # so remove a stale dump. + queries_json = k6_dir / "queries.json" + queries_json.unlink(missing_ok=True) + + def run_suite() -> None: + print(f"+++ Running console scalability suite ({file.stem})") + try: + _console_runner_sh( + c, + "yarn test:e2e:scalability --workers=1 --trace=off " + f"--output=test-results/{file.stem}", + user=_HOST_UID_GID, + env={"BASE_URL": "http://materialized:6874"}, + ) + except Exception as e: + print( + "Console scalability suite failed. Not failing the workflow, " + f"the suite only collects timing data: {e}" + ) + # k6 replays the query set that dump-queries captured above (it + # runs first in the suite), isolating server-side concurrency + # from browser behavior. + if queries_json.exists(): + print(f"+++ Running console k6 load test ({file.stem})") + try: + c.run( + "console-k6", + "run", + "--summary-export=summary.json", + "cluster-detail.ts", + rm=True, + env_extra={"MZ_HTTP_URL": "http://materialized:6876"}, + ) + except Exception as e: + print( + "Console k6 load test failed. Not failing the workflow, " + f"it only collects timing data: {e}" + ) + else: + print( + "dump-queries produced no queries.json, skipping the k6 load test" + ) + # Keep per-workload copies, the next workload overwrites the + # originals. + for name in ("queries", "summary"): + src = k6_dir / f"{name}.json" + if src.exists(): + src.rename(k6_dir / f"{name}-{file.stem}.json") + + try: + test( + c, + workload, + file, + factor_initial_data, + args.factor_ingestions, + args.factor_queries, + 0, # runtime is unused, the suite governs the continuous phase + args.verbose, + True, # create_objects + True, # initial_data + True, # early_initial_data + True, # run_ingestions + True, # run_queries + args.max_concurrent_queries, + during_continuous=run_suite, + ) + finally: + # The runner idles on sleep infinity, don't let it outlive the + # workflow into CI teardown. + try: + c.kill("console-scalability-runner") + c.rm("console-scalability-runner") + except Exception: + pass + + c.test_parts(files, run) + + +def _console_runner_sh( + c: Composition, + command: str, + user: str | None = None, + env: dict[str, str] | None = None, +) -> None: + """Run a shell command in the console-scalability-runner service.""" + full_env = { + "COREPACK_ENABLE_DOWNLOAD_PROMPT": "0", + # yarn needs a writable HOME when running as the host user. + "HOME": "/tmp", + **(env or {}), + } + c.invoke( + "exec", + *(["--user", user] if user else []), + *(f"-e{k}={v}" for k, v in full_env.items()), + "-T", + "console-scalability-runner", + "sh", + "-c", + command, + ) + + def workflow_stats(c: Composition, parser: WorkflowArgumentParser) -> None: with c.override(Materialized(sanity_restart=False)): parser.add_argument(