diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 8eedc3a4504e8..ddc82382e03b7 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -54,6 +54,7 @@
/src/build-info @MaterializeInc/testing @SangJunBak @DAlperin
/src/catalog @MaterializeInc/adapter
/src/catalog-debug @MaterializeInc/adapter
+/src/catalog-decode @MaterializeInc/adapter
/src/ccsr @MaterializeInc/adapter @MaterializeInc/cluster
/src/cloud-resources @MaterializeInc/cloud
/src/cluster @MaterializeInc/cluster
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 402ebb9e45f63..d47050bb6efc6 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -92,6 +92,15 @@ updates:
default-days: 3
open-pull-requests-limit: 50
labels: [A-dependencies]
+ # Each `module` block is its own dependency, so without a group every module
+ # sourced from the same repository gets a separate PR. The modules of one
+ # release must move together, so group all updates, including major ones,
+ # across all roots into a single PR.
+ groups:
+ terraform:
+ applies-to: version-updates
+ patterns:
+ - "*"
- package-ecosystem: pip
directory: /misc/dbt-materialize
schedule:
diff --git a/Cargo.lock b/Cargo.lock
index 19a6fad634970..444d79c204057 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -229,7 +229,7 @@ dependencies = [
"crc32fast",
"digest 0.10.7",
"log",
- "miniz_oxide",
+ "miniz_oxide 0.8.9",
"num-bigint",
"quad-rand",
"rand 0.9.4",
@@ -240,7 +240,7 @@ dependencies = [
"snap",
"strum",
"strum_macros",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"uuid",
"zstd",
]
@@ -864,13 +864,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"
[[package]]
name = "async-trait"
-version = "0.1.89"
+version = "0.1.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
+checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.119",
+ "syn 3.0.6",
]
[[package]]
@@ -930,9 +930,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "aws-config"
-version = "1.8.15"
+version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc"
+checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -944,6 +944,7 @@ dependencies = [
"aws-smithy-json",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
+ "aws-smithy-schema",
"aws-smithy-types",
"aws-types",
"bytes",
@@ -960,9 +961,9 @@ dependencies = [
[[package]]
name = "aws-credential-types"
-version = "1.2.14"
+version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7"
+checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api",
@@ -996,9 +997,9 @@ dependencies = [
[[package]]
name = "aws-runtime"
-version = "1.7.2"
+version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17"
+checksum = "2b8a9911551b4ea6ca13805ef52ed96f7d2bbb43cc3b4a14cb0776a71f33cfaa"
dependencies = [
"aws-credential-types",
"aws-sigv4",
@@ -1012,9 +1013,7 @@ dependencies = [
"bytes",
"bytes-utils",
"fastrand 2.3.0",
- "http 0.2.12",
"http 1.4.2",
- "http-body 0.4.6",
"http-body 1.0.1",
"percent-encoding",
"pin-project-lite",
@@ -1024,10 +1023,11 @@ dependencies = [
[[package]]
name = "aws-sdk-glue"
-version = "1.142.1"
+version = "1.169.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc77647907307c70ffd751db85653804552e4a3c27f054d3af7a0874ef4dfe22"
+checksum = "4356134342723fcfdbc9a90d1ecc70e82c7bb63fcbe948a9d8d392335901c6d1"
dependencies = [
+ "arc-swap",
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
@@ -1036,11 +1036,11 @@ dependencies = [
"aws-smithy-observability",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
+ "aws-smithy-schema",
"aws-smithy-types",
"aws-types",
"bytes",
"fastrand 2.3.0",
- "http 0.2.12",
"http 1.4.2",
"regex-lite",
"tracing",
@@ -1048,10 +1048,11 @@ dependencies = [
[[package]]
name = "aws-sdk-kms"
-version = "1.104.0"
+version = "1.122.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c41ae6a33da941457e89075ef8ca5b4870c8009fe4dceeba82fce2f30f313ac6"
+checksum = "ae92186214a60f1316f6e25df2b5bef8f67b5b08b45f8e3eb6e1278b9ddd7b29"
dependencies = [
+ "arc-swap",
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
@@ -1060,11 +1061,11 @@ dependencies = [
"aws-smithy-observability",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
+ "aws-smithy-schema",
"aws-smithy-types",
"aws-types",
"bytes",
"fastrand 2.3.0",
- "http 0.2.12",
"http 1.4.2",
"regex-lite",
"tracing",
@@ -1072,10 +1073,11 @@ dependencies = [
[[package]]
name = "aws-sdk-s3"
-version = "1.129.0"
+version = "1.150.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d4e8410fadbc0ee453145dd77a4958227b18b05bf67c2795d0a8b8596c9aa0f"
+checksum = "2fecaa77c862fa1e9d636d54fa06e0049a19a4f071b0d72ea1b9086887d09359"
dependencies = [
+ "arc-swap",
"aws-credential-types",
"aws-runtime",
"aws-sigv4",
@@ -1087,30 +1089,31 @@ dependencies = [
"aws-smithy-observability",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
+ "aws-smithy-schema",
"aws-smithy-types",
"aws-smithy-xml",
"aws-types",
"bytes",
"fastrand 2.3.0",
"hex",
- "hmac 0.12.1",
- "http 0.2.12",
+ "hmac 0.13.0",
"http 1.4.2",
"http-body 1.0.1",
- "lru",
+ "lru 0.18.5",
"percent-encoding",
"regex-lite",
- "sha2 0.10.9",
+ "sha2 0.11.0",
"tracing",
"url",
]
[[package]]
name = "aws-sdk-secretsmanager"
-version = "1.103.0"
+version = "1.119.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae64963d3d16d8070aaa2fb79c11cd3b13f44d2f13bba3fe8f49dcd2c42f2987"
+checksum = "f28259db7d655b96a6b5748132a9a780361e908ab2d692f767729a4581033a4b"
dependencies = [
+ "arc-swap",
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
@@ -1119,11 +1122,11 @@ dependencies = [
"aws-smithy-observability",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
+ "aws-smithy-schema",
"aws-smithy-types",
"aws-types",
"bytes",
"fastrand 2.3.0",
- "http 0.2.12",
"http 1.4.2",
"regex-lite",
"tracing",
@@ -1131,10 +1134,11 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
-version = "1.96.0"
+version = "1.113.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f64a6eded248c6b453966e915d32aeddb48ea63ad17932682774eb026fbef5b1"
+checksum = "87184f36ade012f46cdc773c7be10031b4545a1aea70a9a22a044e8f36f1e5c5"
dependencies = [
+ "arc-swap",
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
@@ -1143,11 +1147,11 @@ dependencies = [
"aws-smithy-observability",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
+ "aws-smithy-schema",
"aws-smithy-types",
"aws-types",
"bytes",
"fastrand 2.3.0",
- "http 0.2.12",
"http 1.4.2",
"regex-lite",
"tracing",
@@ -1155,10 +1159,11 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
-version = "1.98.0"
+version = "1.115.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db96d720d3c622fcbe08bae1c4b04a72ce6257d8b0584cb5418da00ae20a344f"
+checksum = "84edc7495d56e006dff0031cbc9782784902456f7024651c2da902a3d2336262"
dependencies = [
+ "arc-swap",
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
@@ -1167,11 +1172,11 @@ dependencies = [
"aws-smithy-observability",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
+ "aws-smithy-schema",
"aws-smithy-types",
"aws-types",
"bytes",
"fastrand 2.3.0",
- "http 0.2.12",
"http 1.4.2",
"regex-lite",
"tracing",
@@ -1179,10 +1184,11 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
-version = "1.101.0"
+version = "1.118.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab41ad64e4051ecabeea802d6a17845a91e83287e1dd249e6963ea1ba78c428a"
+checksum = "df7572569d018ca72923073494eac3c4671a0daea90b9e3923c2d27a3a02a2c1"
dependencies = [
+ "arc-swap",
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
@@ -1192,11 +1198,11 @@ dependencies = [
"aws-smithy-query",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
+ "aws-smithy-schema",
"aws-smithy-types",
"aws-smithy-xml",
"aws-types",
"fastrand 2.3.0",
- "http 0.2.12",
"http 1.4.2",
"regex-lite",
"tracing",
@@ -1204,9 +1210,9 @@ dependencies = [
[[package]]
name = "aws-sigv4"
-version = "1.4.2"
+version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4"
+checksum = "2312577f088c9fbf4206dfdb884cf1de9407b43e1a923cbed5237775116fc24b"
dependencies = [
"aws-credential-types",
"aws-smithy-eventstream",
@@ -1216,20 +1222,19 @@ dependencies = [
"bytes",
"form_urlencoded",
"hex",
- "hmac 0.12.1",
- "http 0.2.12",
+ "hmac 0.13.0",
"http 1.4.2",
"percent-encoding",
- "sha2 0.10.9",
+ "sha2 0.11.0",
"time",
"tracing",
]
[[package]]
name = "aws-smithy-async"
-version = "1.2.14"
+version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc"
+checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316"
dependencies = [
"futures-util",
"pin-project-lite",
@@ -1238,9 +1243,9 @@ dependencies = [
[[package]]
name = "aws-smithy-checksums"
-version = "0.64.6"
+version = "0.65.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6750f3dd509b0694a4377f0293ed2f9630d710b1cebe281fa8bac8f099f88bc6"
+checksum = "b67ecd999972b58e67cab052f5129906c08c25883bd0788ceefc55ef97d61307"
dependencies = [
"aws-smithy-http",
"aws-smithy-types",
@@ -1250,18 +1255,18 @@ dependencies = [
"http 1.4.2",
"http-body 1.0.1",
"http-body-util",
- "md-5 0.10.6",
+ "md-5 0.11.0",
"pin-project-lite",
- "sha1 0.10.6",
- "sha2 0.10.9",
+ "sha1 0.11.0",
+ "sha2 0.11.0",
"tracing",
]
[[package]]
name = "aws-smithy-eventstream"
-version = "0.60.20"
+version = "0.61.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "faf09d74e5e32f76b8762da505a3cd59303e367a664ca67295387baa8c1d7548"
+checksum = "80c2051c2f1016fb8e6548dd07b8bc2ac9c3fe583721444b92f515e856d31609"
dependencies = [
"aws-smithy-types",
"bytes",
@@ -1270,9 +1275,9 @@ dependencies = [
[[package]]
name = "aws-smithy-http"
-version = "0.63.6"
+version = "0.64.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231"
+checksum = "639b4d8f8555f24a9be649811c3eb0b4d4616f4d61daf0c32e28873bc1ea9af1"
dependencies = [
"aws-smithy-eventstream",
"aws-smithy-runtime-api",
@@ -1292,9 +1297,9 @@ dependencies = [
[[package]]
name = "aws-smithy-http-client"
-version = "1.1.12"
+version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a2f165a7feee6f263028b899d0a181987f4fa7179a6411a32a439fba7c5f769"
+checksum = "7bd25384a4e437aa8d8f339afad4b69e786b936a7cb10db668a7aaf66717b1a8"
dependencies = [
"aws-smithy-async",
"aws-smithy-protocol-test",
@@ -1318,9 +1323,9 @@ dependencies = [
[[package]]
name = "aws-smithy-json"
-version = "0.62.6"
+version = "0.63.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "517089205f18ab4adc5a3e02888cb139bbbbb2e168eac9f396216925d1fbeaf5"
+checksum = "3385d469edbe8b60cc72002784652b5efca39178192aa9cc4b44c9875c6bdc18"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-schema",
@@ -1329,9 +1334,9 @@ dependencies = [
[[package]]
name = "aws-smithy-mocks"
-version = "0.2.6"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d9a9490933f8faa0aeb1eb9fbb8bf4f1c214b3149c61b3a4074b68030b21bd5"
+checksum = "d0922f8ee1ce1f90f0baf9c9036f566161547a7ab9f09571d61d56455ba35598"
dependencies = [
"aws-smithy-http-client",
"aws-smithy-runtime-api",
@@ -1341,18 +1346,18 @@ dependencies = [
[[package]]
name = "aws-smithy-observability"
-version = "0.2.6"
+version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c"
+checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c"
dependencies = [
"aws-smithy-runtime-api",
]
[[package]]
name = "aws-smithy-protocol-test"
-version = "0.63.14"
+version = "0.64.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b227aa94af99a8e5ee52551cc7e3ee30a217019ef99207b6f0b7a1527685941"
+checksum = "f76511a0e223ce78deb6a78b8afebda99cb737cfbc8a58d96dcb190f012dd40a"
dependencies = [
"assert-json-diff",
"aws-smithy-runtime-api",
@@ -1364,24 +1369,27 @@ dependencies = [
"regex-lite",
"roxmltree",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
]
[[package]]
name = "aws-smithy-query"
-version = "0.60.15"
+version = "0.62.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd"
+checksum = "f1d1d71f6562be974caa85442ecd90194c40fdb5df045f182a6c2e872ce95056"
dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
"aws-smithy-types",
+ "aws-smithy-xml",
"urlencoding",
]
[[package]]
name = "aws-smithy-runtime"
-version = "1.11.3"
+version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182"
+checksum = "591e0024cdf2a8de8711860f7e1d5cfbca26b800ce13793a9aa2f4f5918e6d95"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
@@ -1392,23 +1400,20 @@ dependencies = [
"aws-smithy-types",
"bytes",
"fastrand 2.3.0",
- "http 0.2.12",
"http 1.4.2",
- "http-body 0.4.6",
"http-body 1.0.1",
"http-body-util",
"pin-project-lite",
"pin-utils",
"tokio",
"tracing",
- "tracing-subscriber",
]
[[package]]
name = "aws-smithy-runtime-api"
-version = "1.12.1"
+version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc117c179ecf39a62a0a3f49f600e9ac26a7ad7dd172177999f83933af776c32"
+checksum = "9bb0deb40a69de59f7809251cfe1ab16de135f42c49d222d1202b4d0ce19720f"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api-macros",
@@ -1424,9 +1429,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api-macros"
-version = "1.0.0"
+version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7"
+checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3"
dependencies = [
"proc-macro2",
"quote",
@@ -1435,9 +1440,9 @@ dependencies = [
[[package]]
name = "aws-smithy-schema"
-version = "0.1.0"
+version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5"
+checksum = "e8f395d93304280b64b7632fea798d177e74897fe7f063416ce627cd6fa24829"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
@@ -1446,9 +1451,9 @@ dependencies = [
[[package]]
name = "aws-smithy-types"
-version = "1.4.8"
+version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "056b66dbce2f81cc0c1e2b05bb402eb58f8a3530479d650efadd5bbae9a4050b"
+checksum = "69bb407740a197147da48238ecc94498493c9e85445732360cec180296ca45f1"
dependencies = [
"base64-simd",
"bytes",
@@ -1472,18 +1477,21 @@ dependencies = [
[[package]]
name = "aws-smithy-xml"
-version = "0.60.15"
+version = "0.62.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3"
+checksum = "0b932c8d6dc127fc980eecd78f8694ae9b9551b69a93a7def2a199c1c0033daf"
dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
+ "aws-smithy-types",
"xmlparser",
]
[[package]]
name = "aws-types"
-version = "1.3.16"
+version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531"
+checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc"
dependencies = [
"aws-credential-types",
"aws-smithy-async",
@@ -1720,7 +1728,7 @@ dependencies = [
"addr2line",
"cfg-if",
"libc",
- "miniz_oxide",
+ "miniz_oxide 0.8.9",
"object",
"rustc-demangle",
"windows-link 0.2.1",
@@ -2020,9 +2028,9 @@ dependencies = [
[[package]]
name = "buildid"
-version = "1.0.4"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "834a85d2478807f41c43251c7ad226e1185bc34feb2133872f796a11b57155ae"
+checksum = "4b8e278b5bdc31756d01d65c17662e23692e0dbaa19e65adae9b58fc4683bc09"
dependencies = [
"cfg-if",
"libc",
@@ -2201,9 +2209,9 @@ dependencies = [
[[package]]
name = "cfg-if"
-version = "1.0.4"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
[[package]]
name = "cfg_aliases"
@@ -2315,9 +2323,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.6.3"
+version = "4.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fb99565819980999fb7b4a1796046a5c949e6d4ff132cf5fadf5a641e20d776"
+checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946"
dependencies = [
"clap_builder",
"clap_derive",
@@ -2325,9 +2333,9 @@ dependencies = [
[[package]]
name = "clap_builder"
-version = "4.6.2"
+version = "4.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
+checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d"
dependencies = [
"anstream",
"anstyle",
@@ -2338,23 +2346,23 @@ dependencies = [
[[package]]
name = "clap_complete"
-version = "4.6.9"
+version = "4.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3be2ad0423bdbbb0e25bc89add796f3559706d4a95e1bc98e4d9662a957b6a19"
+checksum = "037e2a1a92236d0aff7e845093f64661d6df4c02c9fcc61a60e9e1d736fa392f"
dependencies = [
"clap",
]
[[package]]
name = "clap_derive"
-version = "4.6.3"
+version = "4.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32f2392eae7f16557a3d727ef3a12e57b2b2ca6f98566a5f4fb41ffe305df077"
+checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0"
dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn 2.0.119",
+ "syn 3.0.6",
]
[[package]]
@@ -2396,9 +2404,9 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
name = "columnar"
-version = "0.13.1"
+version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94063ab5cdb61ed8e5a8294c3c7cc8466c1a9eafc7b9ca02b2a521ea06e52d92"
+checksum = "283ec011cb03e19e71ff8c57e8ae4e736687e16b9d7ad43c6c24bd4ac1da6252"
dependencies = [
"bytemuck",
"columnar_derive",
@@ -2407,9 +2415,9 @@ dependencies = [
[[package]]
name = "columnar_derive"
-version = "0.13.0"
+version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72c9c88441fb60d0d5d9e939b5765abdff87d1a4d4cc80a90e0f945a6711c438"
+checksum = "54a1d493d903c04df5d9b8fc9f5d1051dfad92664a9d56c39a2b08b68a276ea7"
dependencies = [
"proc-macro2",
"quote",
@@ -2694,9 +2702,9 @@ dependencies = [
[[package]]
name = "crc32fast"
-version = "1.5.0"
+version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
+checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
dependencies = [
"cfg-if",
]
@@ -2745,9 +2753,9 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b"
[[package]]
name = "crossbeam"
-version = "0.8.4"
+version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
+checksum = "e71406cd8807725f7ac2f999a4cdd32e98f829fdf65f528343cebf945e41df1e"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
@@ -2758,9 +2766,9 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
-version = "0.5.16"
+version = "0.5.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e"
+checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1"
dependencies = [
"crossbeam-utils",
]
@@ -3257,7 +3265,7 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e"
dependencies = [
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
]
[[package]]
@@ -3445,7 +3453,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
- "windows-sys 0.61.1",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -3998,7 +4006,7 @@ dependencies = [
"serde_json",
"sha2 0.10.9",
"socket2 0.6.5",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tokio-postgres",
"tokio-stream",
@@ -4028,13 +4036,13 @@ dependencies = [
[[package]]
name = "flate2"
-version = "1.1.9"
+version = "1.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
+checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
dependencies = [
"crc32fast",
"libz-sys",
- "miniz_oxide",
+ "miniz_oxide 0.9.1",
"zlib-rs",
]
@@ -4113,9 +4121,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
[[package]]
name = "futures"
-version = "0.3.33"
+version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218"
+checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3"
dependencies = [
"futures-channel",
"futures-core",
@@ -4144,9 +4152,9 @@ checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
[[package]]
name = "futures-executor"
-version = "0.3.33"
+version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458"
+checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432"
dependencies = [
"futures-core",
"futures-task",
@@ -4189,13 +4197,13 @@ dependencies = [
[[package]]
name = "futures-macro"
-version = "0.3.33"
+version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b"
+checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.119",
+ "syn 3.0.6",
]
[[package]]
@@ -4218,9 +4226,9 @@ checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
[[package]]
name = "futures-util"
-version = "0.3.33"
+version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa"
+checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
dependencies = [
"futures-channel",
"futures-core",
@@ -4253,7 +4261,7 @@ dependencies = [
"rustls-pki-types",
"serde",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tracing",
"tracing-futures",
@@ -4490,9 +4498,6 @@ name = "hashbrown"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
-dependencies = [
- "ahash 0.8.12",
-]
[[package]]
name = "hashbrown"
@@ -4519,23 +4524,26 @@ name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+dependencies = [
+ "foldhash 0.2.0",
+]
[[package]]
name = "hashlink"
-version = "0.9.1"
+version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
+checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
dependencies = [
- "hashbrown 0.14.5",
+ "hashbrown 0.15.3",
]
[[package]]
name = "hashlink"
-version = "0.10.0"
+version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
+checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb"
dependencies = [
- "hashbrown 0.15.3",
+ "hashbrown 0.17.1",
]
[[package]]
@@ -4553,9 +4561,9 @@ dependencies = [
[[package]]
name = "headers"
-version = "0.4.1"
+version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb"
+checksum = "fc2700e3fe555c3310aa7286cac6167449f2c87e0eb58769c9208a1c58a1d106"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -4610,9 +4618,9 @@ dependencies = [
[[package]]
name = "hickory-net"
-version = "0.26.1"
+version = "0.26.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183"
+checksum = "c480823ed7c2c5d0f09c41020cb6b7c28029ce60ec42dc942158dcf22f8e0a4d"
dependencies = [
"async-trait",
"cfg-if",
@@ -4625,7 +4633,7 @@ dependencies = [
"ipnet",
"jni",
"rand 0.10.2",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tinyvec",
"tokio",
"tracing",
@@ -4634,9 +4642,9 @@ dependencies = [
[[package]]
name = "hickory-proto"
-version = "0.26.1"
+version = "0.26.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643"
+checksum = "12b92608f679a6fa515dd1d15c1ff89443026e391200a2c840c7afcba482893d"
dependencies = [
"data-encoding",
"idna",
@@ -4646,7 +4654,7 @@ dependencies = [
"prefix-trie",
"rand 0.10.2",
"ring",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tinyvec",
"tracing",
"url",
@@ -4654,9 +4662,9 @@ dependencies = [
[[package]]
name = "hickory-resolver"
-version = "0.26.1"
+version = "0.26.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c"
+checksum = "3f3da5255c95d5a716857d54b5b8f4e8d67c3484d3beaaaae2ce25063b3ba981"
dependencies = [
"cfg-if",
"futures-util",
@@ -4673,7 +4681,7 @@ dependencies = [
"resolv-conf",
"smallvec",
"system-configuration",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tracing",
]
@@ -4790,9 +4798,9 @@ checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
[[package]]
name = "httpdate"
-version = "1.0.2"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "humantime"
@@ -4955,16 +4963,17 @@ dependencies = [
[[package]]
name = "hyper-util"
-version = "0.1.20"
+version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
+checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff"
dependencies = [
- "base64 0.22.1",
+ "base64 0.23.1",
"bytes",
"futures-channel",
"futures-util",
"http 1.4.2",
"http-body 1.0.1",
+ "httparse",
"hyper 1.9.0",
"ipnet",
"libc",
@@ -5191,7 +5200,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5"
dependencies = [
"equivalent",
- "hashbrown 0.16.1",
+ "hashbrown 0.15.3",
"serde",
"serde_core",
]
@@ -5276,9 +5285,9 @@ dependencies = [
[[package]]
name = "ipnet"
-version = "2.12.1"
+version = "2.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78"
+checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0"
dependencies = [
"serde",
]
@@ -5300,7 +5309,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
- "windows-sys 0.61.1",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -5443,7 +5452,7 @@ dependencies = [
"jni-sys",
"log",
"simd_cesu8",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"walkdir",
"windows-link 0.2.1",
]
@@ -5523,7 +5532,7 @@ dependencies = [
"pest_derive",
"regex",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
]
[[package]]
@@ -5579,7 +5588,7 @@ dependencies = [
"rand 0.9.4",
"serde",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tracing",
]
@@ -5646,11 +5655,11 @@ dependencies = [
"serde",
"serde_json",
"serde_yaml",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tokio-util",
"tower 0.5.3",
- "tower-http",
+ "tower-http 0.6.11",
"tracing",
]
@@ -5670,7 +5679,7 @@ dependencies = [
"serde",
"serde-value",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
]
[[package]]
@@ -5708,7 +5717,7 @@ dependencies = [
"pin-project",
"serde",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tokio-util",
"tracing",
@@ -5752,13 +5761,13 @@ dependencies = [
"launchdarkly-sdk-transport",
"launchdarkly-server-sdk-evaluation",
"log",
- "lru",
+ "lru 0.16.3",
"moka",
"parking_lot",
"rand 0.9.4",
"serde",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tokio-stream",
"uuid",
@@ -5894,7 +5903,7 @@ dependencies = [
"serde",
"serde_json",
"tar",
- "zip",
+ "zip 6.0.0",
]
[[package]]
@@ -5945,9 +5954,9 @@ dependencies = [
[[package]]
name = "librocksdb-sys"
-version = "0.17.3+10.4.2"
+version = "0.19.0+11.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cef2a00ee60fe526157c9023edab23943fae1ce2ab6f4abb2a807c1746835de9"
+checksum = "4f45e86edad8e88efe97dbf384b4e48e1ff0f111eabf154c7b09d7a1e5fb573c"
dependencies = [
"bindgen",
"bzip2-sys",
@@ -5955,14 +5964,15 @@ dependencies = [
"libc",
"libz-sys",
"lz4-sys",
+ "rustflags",
"zstd-sys",
]
[[package]]
name = "libsqlite3-sys"
-version = "0.30.1"
+version = "0.38.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149"
+checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8"
dependencies = [
"cc",
"pkg-config",
@@ -6072,6 +6082,15 @@ dependencies = [
"hashbrown 0.16.1",
]
+[[package]]
+name = "lru"
+version = "0.18.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5"
+dependencies = [
+ "hashbrown 0.17.1",
+]
+
[[package]]
name = "lru-slab"
version = "0.1.3"
@@ -6234,6 +6253,15 @@ name = "miniz_oxide"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
+dependencies = [
+ "adler2",
+]
+
+[[package]]
+name = "miniz_oxide"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
dependencies = [
"adler2",
"simd-adler32",
@@ -6301,7 +6329,7 @@ dependencies = [
"futures-sink",
"futures-util",
"keyed_priority_queue",
- "lru",
+ "lru 0.16.3",
"mysql_common",
"native-tls",
"pem 3.0.6",
@@ -6310,7 +6338,7 @@ dependencies = [
"serde",
"serde_json",
"socket2 0.5.10",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tokio-native-tls",
"tokio-util",
@@ -6343,7 +6371,7 @@ dependencies = [
"serde_json",
"sha1 0.10.6",
"sha2 0.10.9",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"uuid",
"zstd",
]
@@ -6379,7 +6407,7 @@ dependencies = [
"tabled",
"tempfile",
"termcolor",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"time",
"tokio",
"toml",
@@ -6475,7 +6503,7 @@ dependencies = [
"sha2 0.10.9",
"smallvec",
"static_assertions",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"timely",
"tokio",
"tokio-postgres",
@@ -6617,7 +6645,7 @@ dependencies = [
"aws-smithy-mocks",
"aws-types",
"mz-ore",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"uuid",
]
@@ -6655,7 +6683,7 @@ dependencies = [
"hyper-tls 0.5.0",
"mz-ore",
"pin-project",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tower-service",
"uuid",
@@ -6703,7 +6731,7 @@ dependencies = [
"rustls",
"semver",
"tempfile",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tokio-metrics",
"tokio-openssl",
@@ -6790,7 +6818,7 @@ dependencies = [
"serde_plain",
"sha2 0.10.9",
"static_assertions",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"timely",
"tokio",
"tracing",
@@ -6824,6 +6852,19 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "mz-catalog-decode"
+version = "0.0.0"
+dependencies = [
+ "hex",
+ "mz-ore",
+ "mz-repr",
+ "mz-sql-parser",
+ "prost",
+ "serde_json",
+ "tracing",
+]
+
[[package]]
name = "mz-catalog-protos"
version = "0.0.0"
@@ -6882,7 +6923,7 @@ dependencies = [
"mz-frontegg-client",
"reqwest 0.12.28",
"serde",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"url",
]
@@ -7081,7 +7122,7 @@ dependencies = [
"scopeguard",
"serde",
"smallvec",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"timely",
"tokio",
"tracing",
@@ -7118,7 +7159,7 @@ dependencies = [
"prometheus",
"serde",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"timely",
"tokio",
"tracing",
@@ -7220,7 +7261,7 @@ dependencies = [
"tracing-subscriber",
"url",
"walkdir",
- "zip",
+ "zip 8.6.0",
]
[[package]]
@@ -7267,7 +7308,7 @@ dependencies = [
"strsim",
"supports-color 3.0.2",
"tempfile",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"time",
"tokio",
"tokio-postgres",
@@ -7457,20 +7498,20 @@ dependencies = [
"similar-asserts",
"sysctl",
"tempfile",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"timely",
"tokio",
"tokio-metrics",
"tokio-postgres",
"tokio-stream",
"tower 0.5.3",
- "tower-http",
+ "tower-http 0.7.1",
"tower-sessions",
"tracing",
"tracing-capture",
"tracing-opentelemetry",
"tracing-subscriber",
- "tungstenite",
+ "tungstenite 0.30.0",
"url",
"uuid",
]
@@ -7503,6 +7544,7 @@ dependencies = [
"md-5 0.10.6",
"murmur2",
"mz-build-tools",
+ "mz-catalog-decode",
"mz-expr",
"mz-expr-derive",
"mz-expr-derive-impl",
@@ -7582,7 +7624,7 @@ dependencies = [
"derivative",
"futures",
"jsonwebtoken",
- "lru",
+ "lru 0.16.3",
"mz-auth",
"mz-ore",
"mz-repr",
@@ -7592,7 +7634,7 @@ dependencies = [
"reqwest-retry",
"serde",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tracing",
"uuid",
@@ -7606,7 +7648,7 @@ dependencies = [
"mz-frontegg-auth",
"reqwest 0.12.28",
"serde",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"url",
"uuid",
@@ -7654,7 +7696,7 @@ dependencies = [
"serde_json",
"tokio",
"tower 0.5.3",
- "tower-http",
+ "tower-http 0.7.1",
"tracing",
"tracing-subscriber",
]
@@ -7718,7 +7760,7 @@ dependencies = [
"rdkafka",
"serde",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tracing",
"url",
@@ -7786,7 +7828,7 @@ dependencies = [
"nix 0.31.3",
"paste",
"prometheus",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tracing",
]
@@ -7834,7 +7876,7 @@ dependencies = [
"regex",
"serde",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tracing",
"url",
@@ -7993,9 +8035,9 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.9",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
- "tower-http",
+ "tower-http 0.7.1",
"tracing",
"uuid",
]
@@ -8054,7 +8096,7 @@ dependencies = [
"smallvec",
"stacker",
"tempfile",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tokio-native-tls",
"tokio-openssl",
@@ -8381,7 +8423,7 @@ dependencies = [
"prost",
"prost-build",
"serde",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tokio-postgres",
"tracing",
@@ -8523,7 +8565,7 @@ dependencies = [
"smallvec",
"static_assertions",
"strsim",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"timely",
"tokio-postgres",
"tracing",
@@ -8547,7 +8589,7 @@ dependencies = [
"rocksdb",
"serde",
"tempfile",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tracing",
]
@@ -8738,7 +8780,7 @@ dependencies = [
"serde",
"serde_json",
"static_assertions",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tokio-postgres",
"tracing",
@@ -8787,7 +8829,7 @@ dependencies = [
"mz-walkabout",
"serde",
"smallvec",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tracing",
"unicode-width 0.2.2",
]
@@ -8802,7 +8844,7 @@ dependencies = [
"mz-sql-lexer",
"mz-sql-parser",
"pretty",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
]
[[package]]
@@ -8834,7 +8876,7 @@ dependencies = [
"serde",
"smallvec",
"static_assertions",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tiberius",
"timely",
"tokio",
@@ -8890,7 +8932,7 @@ dependencies = [
"tokio",
"tokio-postgres",
"tokio-stream",
- "tower-http",
+ "tower-http 0.7.1",
"tracing",
"uuid",
"walkdir",
@@ -8990,7 +9032,7 @@ dependencies = [
"sha2 0.10.9",
"sqlparser",
"tempfile",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tiberius",
"timely",
"tokio",
@@ -9114,7 +9156,7 @@ dependencies = [
"sentry",
"serde",
"smallvec",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"timely",
"tokio",
"tokio-util",
@@ -9193,7 +9235,7 @@ dependencies = [
"reqwest 0.12.28",
"serde",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tiberius",
"timely",
"tokio",
@@ -9352,7 +9394,7 @@ dependencies = [
"mz-ore",
"openssl",
"postgres-openssl",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio-postgres",
]
@@ -10079,7 +10121,7 @@ dependencies = [
"openssh-mux-client",
"shell-escape",
"tempfile",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
]
@@ -10108,7 +10150,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9879168afb48a235200e30d93df320b7191568cda8621df02f145c1c0f1af95a"
dependencies = [
"ssh_format_error",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
]
[[package]]
@@ -10166,23 +10208,23 @@ dependencies = [
[[package]]
name = "opentelemetry"
-version = "0.32.0"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682"
+checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5"
dependencies = [
"futures-core",
"futures-sink",
"js-sys",
"pin-project-lite",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tracing",
]
[[package]]
name = "opentelemetry-http"
-version = "0.32.0"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331"
+checksum = "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300"
dependencies = [
"async-trait",
"bytes",
@@ -10193,18 +10235,19 @@ dependencies = [
[[package]]
name = "opentelemetry-otlp"
-version = "0.32.0"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35"
+checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344"
dependencies = [
"http 1.4.2",
+ "httpdate",
"opentelemetry",
"opentelemetry-http",
"opentelemetry-proto",
"opentelemetry_sdk",
"prost",
"reqwest 0.13.5",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tonic",
"tonic-types",
@@ -10212,9 +10255,9 @@ dependencies = [
[[package]]
name = "opentelemetry-proto"
-version = "0.32.0"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638"
+checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d"
dependencies = [
"opentelemetry",
"opentelemetry_sdk",
@@ -10225,9 +10268,9 @@ dependencies = [
[[package]]
name = "opentelemetry_sdk"
-version = "0.32.1"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9"
+checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520"
dependencies = [
"futures-channel",
"futures-executor",
@@ -10236,7 +10279,7 @@ dependencies = [
"percent-encoding",
"portable-atomic",
"rand 0.9.4",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tokio-stream",
]
@@ -10310,9 +10353,9 @@ checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a"
[[package]]
name = "owo-colors"
-version = "4.3.0"
+version = "4.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d"
+checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8"
dependencies = [
"supports-color 2.1.0",
"supports-color 3.0.2",
@@ -10957,7 +11000,7 @@ dependencies = [
"spin 0.10.1",
"symbolic-demangle",
"tempfile",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
]
[[package]]
@@ -11108,28 +11151,6 @@ dependencies = [
"toml_edit 0.22.26",
]
-[[package]]
-name = "proc-macro-error-attr2"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5"
-dependencies = [
- "proc-macro2",
- "quote",
-]
-
-[[package]]
-name = "proc-macro-error2"
-version = "2.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802"
-dependencies = [
- "proc-macro-error-attr2",
- "proc-macro2",
- "quote",
- "syn 2.0.119",
-]
-
[[package]]
name = "proc-macro2"
version = "1.0.107"
@@ -11151,7 +11172,7 @@ dependencies = [
"memchr",
"parking_lot",
"protobuf",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
]
[[package]]
@@ -11289,9 +11310,9 @@ dependencies = [
[[package]]
name = "proxy-header"
-version = "0.1.2"
+version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc1493f63ddddfba840c3169e997c2905d09538ace72d64e84af6324c6e0e065"
+checksum = "accf7ba1ae5e4f5b6acfb2d88d3cf68d03c333fca0260b06fb9aafcfcf22e2f5"
[[package]]
name = "psl-types"
@@ -11432,7 +11453,7 @@ dependencies = [
"rustc-hash",
"rustls",
"socket2 0.6.5",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tracing",
"web-time",
@@ -11455,7 +11476,7 @@ dependencies = [
"rustls",
"rustls-pki-types",
"slab",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tinyvec",
"tracing",
"web-time",
@@ -11472,7 +11493,7 @@ dependencies = [
"once_cell",
"socket2 0.6.5",
"tracing",
- "windows-sys 0.61.1",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -11773,7 +11794,7 @@ checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b"
dependencies = [
"getrandom 0.2.16",
"libredox",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
]
[[package]]
@@ -11989,7 +12010,7 @@ dependencies = [
"tokio-native-tls",
"tokio-util",
"tower 0.5.3",
- "tower-http",
+ "tower-http 0.6.11",
"tower-service",
"url",
"wasm-bindgen",
@@ -12033,7 +12054,7 @@ dependencies = [
"tokio-rustls",
"tokio-util",
"tower 0.5.3",
- "tower-http",
+ "tower-http 0.6.11",
"tower-service",
"url",
"wasm-bindgen",
@@ -12072,7 +12093,7 @@ dependencies = [
"reqwest 0.12.28",
"reqwest-middleware",
"retry-policies",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tracing",
"wasmtimer",
@@ -12173,9 +12194,9 @@ dependencies = [
[[package]]
name = "rocksdb"
-version = "0.24.0"
+version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddb7af00d2b17dbd07d82c0063e25411959748ff03e8d4f96134c2ff41fce34f"
+checksum = "d8d90add70d1d420ee487bce4a1449880a8d147451c6051b2ee5f8354553dcbf"
dependencies = [
"libc",
"librocksdb-sys",
@@ -12232,6 +12253,16 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "rsqlite-vfs"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c"
+dependencies = [
+ "hashbrown 0.16.1",
+ "thiserror 2.0.21",
+]
+
[[package]]
name = "rtoolbox"
version = "0.0.1"
@@ -12244,16 +12275,17 @@ dependencies = [
[[package]]
name = "rusqlite"
-version = "0.32.1"
+version = "0.40.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e"
+checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3"
dependencies = [
"bitflags 2.11.0",
"fallible-iterator 0.3.0",
"fallible-streaming-iterator",
- "hashlink 0.9.1",
+ "hashlink 0.12.2",
"libsqlite3-sys",
"smallvec",
+ "sqlite-wasm-rs",
]
[[package]]
@@ -12304,6 +12336,12 @@ dependencies = [
"semver",
]
+[[package]]
+name = "rustflags"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a39e0e9135d7a7208ee80aa4e3e4b88f0f5ad7be92153ed70686c38a03db2e63"
+
[[package]]
name = "rustix"
version = "0.38.44"
@@ -12327,7 +12365,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.12.1",
- "windows-sys 0.61.1",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -12386,7 +12424,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
- "windows-sys 0.61.1",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -12590,7 +12628,7 @@ dependencies = [
"reqwest 0.12.28",
"serde",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"time",
]
@@ -12616,9 +12654,9 @@ dependencies = [
[[package]]
name = "sentry"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76a88b65feb368d9dde5d531a2b59b25016e36fd6e4978432371a3ee77746ca2"
+checksum = "9fa951f4e644464ebfd2347a7ab03b0828c512d2448a76262cad2607b343dff4"
dependencies = [
"cfg_aliases",
"httpdate",
@@ -12635,9 +12673,9 @@ dependencies = [
[[package]]
name = "sentry-backtrace"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c51511d67ed670266a93afeaa26a08019b04ad1420cb9191da30f2338d22c48"
+checksum = "f4f27cf048aa63ccd5bc94c29989133eabb82e9c6bf7441b58e9d4a95837a105"
dependencies = [
"backtrace",
"regex",
@@ -12646,9 +12684,9 @@ dependencies = [
[[package]]
name = "sentry-contexts"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b757ac1f335856e8d7f5062a1fd9bc07a05a7eb3cc48247db79bf2618a490bd"
+checksum = "c5ed680585dbdacdf9859c2b4dc53358ff02bd256766882e22e343836db36b22"
dependencies = [
"hostname",
"libc",
@@ -12660,11 +12698,11 @@ dependencies = [
[[package]]
name = "sentry-core"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d889520a375e5b93efb0a66def1630d21d168b0251eb55b286b03fa940ccfc84"
+checksum = "c3ed398494bb8fc412a11a182c1883a10ae147587b8f6314abd222d1d626f31b"
dependencies = [
- "rand 0.9.4",
+ "rand 0.10.2",
"sentry-types",
"serde",
"serde_json",
@@ -12673,9 +12711,9 @@ dependencies = [
[[package]]
name = "sentry-debug-images"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "656487fd5f7b7c0105b2e82171982aac64489e0cb679436038febf070f2f76d6"
+checksum = "55854bb40681e84ffd2a69578ad4ef9bfaf5bf2bf1380a745578b591fa24af22"
dependencies = [
"findshlibs",
"sentry-core",
@@ -12683,9 +12721,9 @@ dependencies = [
[[package]]
name = "sentry-panic"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e0fe456a7380e9df875a1ef4df1e468d35b19b9051599845fab9d8f0c71c4ae"
+checksum = "b7153e68642b761ed3aa62c022795c11b2facc728b2db7484084492373c944f9"
dependencies = [
"sentry-backtrace",
"sentry-core",
@@ -12693,9 +12731,9 @@ dependencies = [
[[package]]
name = "sentry-tracing"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77796d14eedbef22c2fe78e9c69fb09f14c7ad607e624d48dce92eedc17a136e"
+checksum = "e7e7db1aefdf623f4e48e21ec9e4af535024187aeafb2a4cc7e29d035def5a13"
dependencies = [
"bitflags 2.11.0",
"sentry-backtrace",
@@ -12706,16 +12744,16 @@ dependencies = [
[[package]]
name = "sentry-types"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc71f5ca55942d9b2901af95d5df5c5d65c164134c22a83682cac3d0b6c7ef2d"
+checksum = "1c18bf7b159bdfc7c8f97816f476b4947a1937fb6bea9d38d3479e58631614ac"
dependencies = [
"debugid",
"hex",
- "rand 0.9.4",
+ "rand 0.10.2",
"serde",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"time",
"url",
"uuid",
@@ -12766,9 +12804,9 @@ dependencies = [
[[package]]
name = "serde-reflection"
-version = "0.5.2"
+version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68fb2363ca88876b3e16442b02dde5305646fd50df297c79a4b54fc5f5cf51d4"
+checksum = "af0fdf9c543af37502581e3d6cf4e6cd1628d6616d6a1f8f4ca88aae3a405f5c"
dependencies = [
"erased-discriminant",
"once_cell",
@@ -13195,6 +13233,18 @@ dependencies = [
"der",
]
+[[package]]
+name = "sqlite-wasm-rs"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75"
+dependencies = [
+ "cc",
+ "js-sys",
+ "rsqlite-vfs",
+ "wasm-bindgen",
+]
+
[[package]]
name = "sqlparser"
version = "0.63.0"
@@ -13442,7 +13492,7 @@ dependencies = [
"byteorder",
"enum-as-inner",
"libc",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"walkdir",
]
@@ -13484,9 +13534,9 @@ dependencies = [
[[package]]
name = "tabled"
-version = "0.21.0"
+version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5dc662e6da844ad6e428ad16b57967c9d33c82e16bb1c258326c0c078605dff"
+checksum = "2d2596a104db1900b943f97d793a6c99addca918c4525c999f751a0f17d472eb"
dependencies = [
"papergrid",
"tabled_derive",
@@ -13495,12 +13545,11 @@ dependencies = [
[[package]]
name = "tabled_derive"
-version = "0.11.0"
+version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ea5d1b13ca6cff1f9231ffd62f15eefd72543dab5e468735f1a456728a02846"
+checksum = "7dca1937322a1e892b1a65f6a6736183bb0a29d3b4234d1d53bc436dff9beb76"
dependencies = [
"heck",
- "proc-macro-error2",
"proc-macro2",
"quote",
"syn 2.0.119",
@@ -13539,7 +13588,7 @@ dependencies = [
"getrandom 0.4.3",
"once_cell",
"rustix 1.1.4",
- "windows-sys 0.61.1",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -13558,7 +13607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874"
dependencies = [
"rustix 1.1.4",
- "windows-sys 0.61.1",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -13587,11 +13636,11 @@ dependencies = [
[[package]]
name = "thiserror"
-version = "2.0.20"
+version = "2.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
+checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e"
dependencies = [
- "thiserror-impl 2.0.20",
+ "thiserror-impl 2.0.21",
]
[[package]]
@@ -13607,9 +13656,9 @@ dependencies = [
[[package]]
name = "thiserror-impl"
-version = "2.0.20"
+version = "2.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
+checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
dependencies = [
"proc-macro2",
"quote",
@@ -13956,7 +14005,7 @@ dependencies = [
"futures-util",
"log",
"tokio",
- "tungstenite",
+ "tungstenite 0.29.0",
]
[[package]]
@@ -14162,17 +14211,38 @@ name = "tower-http"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
+dependencies = [
+ "base64 0.22.1",
+ "bitflags 2.11.0",
+ "bytes",
+ "futures-util",
+ "http 1.4.2",
+ "http-body 1.0.1",
+ "mime",
+ "pin-project-lite",
+ "tower 0.5.3",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "tower-http"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c"
dependencies = [
"async-compression",
"base64 0.22.1",
"bitflags 2.11.0",
"bytes",
"futures-core",
- "futures-util",
"http 1.4.2",
"http-body 1.0.1",
"http-body-util",
"mime",
+ "percent-encoding",
"pin-project-lite",
"tokio",
"tokio-util",
@@ -14180,7 +14250,6 @@ dependencies = [
"tower-layer",
"tower-service",
"tracing",
- "url",
]
[[package]]
@@ -14262,7 +14331,7 @@ dependencies = [
"rand 0.9.4",
"serde",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"time",
"tokio",
"tracing",
@@ -14349,9 +14418,9 @@ dependencies = [
[[package]]
name = "tracing-opentelemetry"
-version = "0.33.0"
+version = "0.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26"
+checksum = "0a904802a1b902f43638b677ff2a650847e3b4404101b6c586d648e8c1e3e8fe"
dependencies = [
"js-sys",
"opentelemetry",
@@ -14429,7 +14498,23 @@ dependencies = [
"log",
"rand 0.9.4",
"sha1 0.10.6",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
+]
+
+[[package]]
+name = "tungstenite"
+version = "0.30.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e48ac77174b19c110a50ab2128b24215ac9cb40e0e12e093fb602d175c569d22"
+dependencies = [
+ "bytes",
+ "data-encoding",
+ "http 1.4.2",
+ "httparse",
+ "log",
+ "rand 0.10.2",
+ "sha1 0.11.0",
+ "thiserror 2.0.21",
]
[[package]]
@@ -14499,6 +14584,12 @@ dependencies = [
"syn 2.0.119",
]
+[[package]]
+name = "typed-path"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
+
[[package]]
name = "typeid"
version = "1.0.3"
@@ -14730,9 +14821,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "utoipa"
-version = "5.5.0"
+version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bde15df68e80b16c7d16b9616e80770ad158988daa56a27dccd1e55558b0160"
+checksum = "8765fe27aeff71012a3f90fa474cf1df863a7d0dd81ed25de4e63fff2544994f"
dependencies = [
"indexmap 2.11.4",
"serde",
@@ -14742,20 +14833,20 @@ dependencies = [
[[package]]
name = "utoipa-gen"
-version = "5.5.0"
+version = "6.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ba0b99ee52df3028635d93840c797102da61f8a7bb3cf751032455895b52ef8"
+checksum = "d935f1c83fdf8b88f09bbe050d0cea78ea4afee6d7b0317403228c1200b2c8ab"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.119",
+ "syn 3.0.6",
]
[[package]]
name = "uuid"
-version = "1.24.1"
+version = "1.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9"
+checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce"
dependencies = [
"getrandom 0.4.3",
"js-sys",
@@ -15389,6 +15480,19 @@ dependencies = [
"zopfli",
]
+[[package]]
+name = "zip"
+version = "8.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
+dependencies = [
+ "crc32fast",
+ "flate2",
+ "indexmap 2.11.4",
+ "memchr",
+ "typed-path",
+]
+
[[package]]
name = "zlib-rs"
version = "0.6.0"
diff --git a/Cargo.toml b/Cargo.toml
index b29e8efb7ed1d..26e6aa42c64d9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -16,6 +16,7 @@ members = [
"src/build-info",
"src/build-tools",
"src/catalog",
+ "src/catalog-decode",
"src/catalog-debug",
"src/catalog-protos",
"src/ccsr",
@@ -143,6 +144,7 @@ default-members = [
"src/build-info",
"src/build-tools",
"src/catalog",
+ "src/catalog-decode",
"src/catalog-debug",
"src/catalog-protos",
"src/ccsr",
@@ -283,20 +285,20 @@ assert_cmd = "2.2.2"
async-compression = { version = "0.4.27", features = ["bzip2-sys", "gzip", "tokio", "xz", "zstd"] }
async-stream = "0.3.6"
async-trait = "0.1.89"
-aws-config = { version = "1.8.15", default-features = false, features = ["sso"] }
-aws-credential-types = { version = "1.2.10", features = ["hardcoded-credentials"] }
+aws-config = { version = "1.12.0", default-features = false, features = ["sso"] }
+aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] }
aws-lc-rs = "1"
-aws-sdk-glue = { version = "1.104.0", default-features = false, features = ["rt-tokio"] }
-aws-sdk-kms = { version = "1.104.0", default-features = false, features = ["rt-tokio"] }
-aws-sdk-s3 = { version = "1.129.0", default-features = false, features = ["rt-tokio"] }
-aws-sdk-secretsmanager = { version = "1.103.0", default-features = false, features = ["rt-tokio"] }
-aws-sdk-sts = { version = "1.41.0", default-features = false, features = ["rt-tokio"] }
-aws-sigv4 = "1.3.6"
-aws-smithy-mocks = "0.2.4"
-aws-smithy-runtime = { version = "1.9.8", features = ["connector-hyper-0-14-x"] }
-aws-smithy-runtime-api = "1.10.0"
-aws-smithy-types = { version = "1.1.8", features = ["byte-stream-poll-next"] }
-aws-types = "1.3.9"
+aws-sdk-glue = { version = "1.169.0", default-features = false, features = ["rt-tokio"] }
+aws-sdk-kms = { version = "1.122.0", default-features = false, features = ["rt-tokio"] }
+aws-sdk-s3 = { version = "1.150.0", default-features = false, features = ["rt-tokio"] }
+aws-sdk-secretsmanager = { version = "1.119.0", default-features = false, features = ["rt-tokio"] }
+aws-sdk-sts = { version = "1.118.0", default-features = false, features = ["rt-tokio"] }
+aws-sigv4 = "1.6.0"
+aws-smithy-mocks = "0.3.0"
+aws-smithy-runtime = { version = "1.15.0", features = ["connector-hyper-0-14-x"] }
+aws-smithy-runtime-api = "1.18.0"
+aws-smithy-types = { version = "1.8.1", features = ["byte-stream-poll-next"] }
+aws-types = "1.6.0"
axum = { version = "0.8.9", features = ["ws"] }
axum-extra = { version = "0.12.5", features = ["typed-header"] }
axum-server = { version = "0.8.0", features = ["tls-rustls"] }
@@ -425,9 +427,9 @@ openssh-mux-client = "0.17.9"
openssl = { version = "0.10.80", features = ["vendored"] }
openssl-probe = "0.2.1"
openssl-sys = { version = "0.9.108", features = ["vendored"] }
-opentelemetry = { version = "0.32.0", features = ["trace"] }
-opentelemetry-otlp = { version = "0.32.0", features = ["grpc-tonic"] }
-opentelemetry_sdk = { version = "0.32.1", features = ["experimental_trace_batch_span_processor_with_async_runtime", "rt-tokio"] }
+opentelemetry = { version = "0.33.0", features = ["trace"] }
+opentelemetry-otlp = { version = "0.33.0", features = ["grpc-tonic"] }
+opentelemetry_sdk = { version = "0.33.0", features = ["experimental_trace_batch_span_processor_with_async_runtime", "rt-tokio"] }
ordered-float = { version = "5.1.0", features = ["serde"] }
os_info = "3.11.0"
owo-colors = "4.3.0"
@@ -475,11 +477,11 @@ reqwest = { version = "0.12.28", features = ["blocking", "charset", "cookies", "
reqwest-middleware = { version = "0.4.2", features = ["json"] }
reqwest-retry = "0.8.0"
rlimit = "0.11.0"
-rocksdb = { version = "0.24.0", default-features = false, features = ["lz4", "snappy", "zstd"] }
+rocksdb = { version = "0.25.0", default-features = false, features = ["lz4", "snappy", "zstd"] }
ropey = "1.6.1"
rustls = "0.23.38"
rpassword = "7.5.1"
-rusqlite = { version = "0.32.1", features = ["bundled"] }
+rusqlite = { version = "0.40.2", features = ["bundled"] }
ryu = "1.0.23"
schemars = { version = "1.2.1", features = ["uuid1"] }
scopeguard = "1.2.0"
@@ -491,7 +493,7 @@ sentry-panic = "0.49.1"
sentry-tracing = "0.49.1"
serde = { version = "1.0.219", features = ["derive"] }
serde-aux = "4.7.0"
-serde-reflection = "0.5.0"
+serde-reflection = "0.6.0"
serde_bytes = "0.11.19"
serde_json = { version = "1.0.150", features = ["raw_value"] }
serde_plain = "1.0.2"
@@ -516,7 +518,7 @@ supports-color = "3.0.2"
syn = { version = "3.0.5", features = ["extra-traits", "full", "parsing", "printing"] }
sysctl = "0.7.1"
sysinfo = "0.29.11"
-tabled = "0.21.0"
+tabled = "0.22.0"
tar = "0.4.46"
tempfile = "3.27.0"
termcolor = "1.4.1"
@@ -540,35 +542,35 @@ tonic = { version = "0.14.6", features = ["gzip", "transport"] }
tonic-prost = "0.14.6"
tonic-prost-build = "0.14.6"
tower = { version = "0.5.3", features = ["balance", "buffer", "filter", "limit", "load-shed", "retry", "timeout", "util"] }
-tower-http = { version = "0.6.11", features = ["auth", "cors", "decompression-br", "decompression-deflate", "decompression-gzip", "decompression-zstd", "map-response-body", "trace", "util"] }
+tower-http = { version = "0.7.1", features = ["auth", "cors", "decompression-br", "decompression-deflate", "decompression-gzip", "decompression-zstd", "map-response-body", "trace", "util"] }
tower-lsp = { version = "0.20.0", features = ["proposed"] }
tower-sessions = "0.15.0"
tower-sessions-memory-store = "0.14.0"
tracing = "0.1.44"
tracing-capture = "0.1.0"
tracing-core = "0.1.35"
-tracing-opentelemetry = "0.33.0"
+tracing-opentelemetry = "0.34.0"
tracing-subscriber = { version = "0.3.23", features = ["env-filter", "fmt", "json", "tracing-log"] }
# Held back: must match the `tungstenite` that `axum`'s `ws` feature pulls in
# via `tokio-tungstenite`, else the two versions coexist in the dependency
# graph. `axum` 0.8.9 is on `tokio-tungstenite` 0.29, which pins this to 0.29.
# 0.30 additionally moves to `sha1` 0.11, duplicating the whole RustCrypto
# hashing stack.
-tungstenite = "0.29.0"
+tungstenite = "0.30.0"
turmoil = "0.7.2"
uncased = "0.9.10"
unicode-normalization = "0.1.25"
unicode-width = "0.2.2"
url = "2.5.8"
urlencoding = "2.1.3"
-utoipa = "5.4.0"
+utoipa = "6.0.0"
uuid = "1.19.0"
version-compare = "0.2.1"
walkdir = "2.5.0"
which = "8"
yansi = "1.0.1"
zeroize = { version = "1.8.2", features = ["derive", "serde"] }
-zip = { version = "6.0.0", default-features = false, features = ["deflate-flate2"] }
+zip = { version = "8.6.0", default-features = false, features = ["deflate-flate2"] }
[profile.dev]
split-debuginfo = "unpacked"
diff --git a/bin/claude-builder b/bin/claude-builder
index eed6d0934a17d..864602db9cd38 100755
--- a/bin/claude-builder
+++ b/bin/claude-builder
@@ -20,8 +20,8 @@ set -euo pipefail
"$(dirname "$0")"/ci-builder run stable --no-secrets bash -lc '
export HOME=/tmp
export NPM_CONFIG_UPDATE_NOTIFIER=false npm_config_update_notifier=false NO_UPDATE_NOTIFIER=1
- if [ ! -e "/tmp/.claude/skills/materialize-docs" ]; then
- (npx -q -y skills add MaterializeInc/agent-skills -g -a claude-code --copy -y --skill materialize-docs) >/dev/null 2>&1 || true
+ if [ ! -e "/tmp/.claude/skills/mz-docs" ]; then
+ (npx -q -y skills add MaterializeInc/agent-skills -g -a claude-code --copy -y --skill mz-docs) >/dev/null 2>&1 || true
fi
exec npx -q -y @anthropic-ai/claude-code@latest --dangerously-skip-permissions "$@"
' -- "$@"
diff --git a/bin/codex-builder b/bin/codex-builder
index 414903b33dc99..5ee002961f80e 100755
--- a/bin/codex-builder
+++ b/bin/codex-builder
@@ -20,8 +20,8 @@ set -euo pipefail
"$(dirname "$0")"/ci-builder run stable --no-secrets bash -lc '
export HOME=/tmp
export NPM_CONFIG_UPDATE_NOTIFIER=false npm_config_update_notifier=false NO_UPDATE_NOTIFIER=1
- if [ ! -e "/tmp/.codex/skills/materialize-docs" ]; then
- (npx -q -y skills add MaterializeInc/agent-skills -g -a codex --copy -y --skill materialize-docs) >/dev/null 2>&1 || true
+ if [ ! -e "/tmp/.codex/skills/mz-docs" ]; then
+ (npx -q -y skills add MaterializeInc/agent-skills -g -a codex --copy -y --skill mz-docs) >/dev/null 2>&1 || true
fi
exec npx -q -y @openai/codex@latest --dangerously-bypass-approvals-and-sandbox "$@"
' -- "$@"
diff --git a/bin/gen-claude-skill b/bin/gen-claude-skill
index e133413276376..4625d4e745ccb 100755
--- a/bin/gen-claude-skill
+++ b/bin/gen-claude-skill
@@ -16,7 +16,7 @@
#
# Arguments: OUTPUT_DIR Directory to output the skill files. The OUTPUT_DIR
# directory is cleaned before generation.
-# Default: .agents/skills/materialize-docs.
+# Default: .agents/skills/mz-docs.
# For publishing to the website,
# we will output to doc/user/public/markdown-docs directory.
#
@@ -30,7 +30,7 @@ cd "$(dirname "$0")/.."
# For publishing to the website, we will output to doc/user/public/markdown-docs
# directory. The OUTPUT_DIR directory is cleaned before generation.
-OUTPUT_DIR="${1:-.agents/skills/materialize-docs}"
+OUTPUT_DIR="${1:-.agents/skills/mz-docs}"
# Ensure output directory exists
mkdir -p "$OUTPUT_DIR"
diff --git a/ci/builder/requirements.txt b/ci/builder/requirements.txt
index 3785e40802fb1..71bdba8494798 100644
--- a/ci/builder/requirements.txt
+++ b/ci/builder/requirements.txt
@@ -15,7 +15,7 @@ aiohttp==3.14.3
black==26.3.1
# Pulls in dependencies that make launchdarkly/cloud tests fail, keep for now
boto3-stubs[ec2,iam,kinesis,s3,sqs,ssm,sts,s3tables]==1.41.5
-boto3==1.43.88
+boto3==1.43.102
click==8.1.3
cryptography==50.0.1
colored==2.3.2
@@ -27,14 +27,14 @@ ipympl==0.10.0
ipywidgets==8.1.9
junit-xml==1.9
junitparser==5.0.3
-jupyterlab==4.6.3
+jupyterlab==4.6.4
jupyter-black==0.4.0
kubernetes==25.3.0
kubernetes-stubs==22.6.0.post1
launchdarkly-api==17.2.0
# NOTE matplotlib 3.10.9 is required for Python 3.10 compatibility
matplotlib==3.10.9; python_version < "3.11"
-matplotlib==3.11.1; python_version >= "3.11"
+matplotlib==3.11.2; python_version >= "3.11"
matplotlib-stubs==0.3.4
# Keep on 3.4.2 for Python 3.10 compatibility
networkx==3.4.2
@@ -54,8 +54,8 @@ psycopg==3.3.3
psycopg-binary==3.3.3
pydantic==2.12.5
pyelftools==0.33
-pyjwt==2.13.0
-PyMySQL==1.2.0
+pyjwt==2.15.0
+PyMySQL==1.2.3
pytest==9.0.3
pytest-split==0.11.0
pyyaml==6.0.3
@@ -66,17 +66,17 @@ ruff==0.0.292
scipy==1.15.3; python_version < "3.11"
scipy==1.17.1; python_version >= "3.11"
scipy-stubs==1.15.3.0
-semver==3.0.4
+semver==3.1.0
shtab==1.12.1
sqlparse==0.6.0
toml==0.10.2
twine==7.0.0
types-Markdown==3.10.2.20260712
types-prettytable==3.4.2.6
-types-psutil==7.2.2.20260827
-types-PyMYSQL==1.2.0.20260807
-types-PyYAML==6.0.12.20260815
-types-requests==2.33.0.20260712
+types-psutil==7.2.2.20260906
+types-PyMYSQL==1.2.0.20260923
+types-PyYAML==6.0.12.20260906
+types-requests==2.33.0.20260906
types-setuptools==84.0.0.20260812
types-toml==0.10.8.20260518
typing-extensions==4.16.0
@@ -91,5 +91,5 @@ pyarrow-stubs==20.0.0.20251215
pyarrow==23.0.1
minio==7.2.15
zstandard==0.25.0
-build==1.6.0
-hatchling==1.32.0
+build==1.6.1
+hatchling==1.32.4
diff --git a/console/misc/docker/Dockerfile b/console/misc/docker/Dockerfile
index 1d62bb0d7ae17..539ce9f4a351c 100644
--- a/console/misc/docker/Dockerfile
+++ b/console/misc/docker/Dockerfile
@@ -38,7 +38,7 @@ RUN yarn build
# Pinned by rolling tag + multi-arch index digest so dependabot refreshes the
# digest for security fixes.
-FROM nginxinc/nginx-unprivileged:alpine@sha256:c3fed6436b61d2bf2201ec032c35c000871f7ed062dea5d586bc6bf4d0fdd140
+FROM nginxinc/nginx-unprivileged:alpine@sha256:6a23acdfca2b9cfbcec61419e3f1426bcbedb91362f2f19306a8567423bb4612
# Add labels
LABEL maintainer="support@materialize.com" \
diff --git a/deny.toml b/deny.toml
index 962b38741f960..fe99f84c46bd2 100644
--- a/deny.toml
+++ b/deny.toml
@@ -106,6 +106,16 @@ skip = [
{ name = "nix", version = "0.26.4" },
# Held back by mysql_async `v0.36.1`
{ name = "socket2", version = "0.5.10" },
+ # kube-client and reqwest still pull 0.6.
+ { name = "tower-http", version = "0.6.11" },
+ # tokio-tungstenite still pulls 0.29.
+ { name = "tungstenite", version = "0.29.0" },
+ # libduckdb-sys still pulls 6.
+ { name = "zip", version = "6.0.0" },
+ # aws-sdk-s3 is on 0.18, while launchdarkly-server-sdk, mysql_async, and mz-frontegg-auth still use 0.16.
+ { name = "lru", version = "0.16.3" },
+ # flate2 is on 0.9, while apache-avro and backtrace still pull 0.8.
+ { name = "miniz_oxide", version = "0.8.9" },
# Used by azure_core
{ name = "quick-xml", version = "0.31.0" },
# reqsign-aws (via iceberg) is ahead of opendal, which pulls 0.39.
@@ -179,8 +189,8 @@ skip = [
# held back by tower-lsp 0.20 (mz-deploy LSP server)
{ name = "dashmap", version = "5.5.3" },
{ name = "tower", version = "0.4.13" },
- # held back by rusqlite 0.32 (mz-deploy build cache)
- { name = "hashlink", version = "0.9.1" },
+ # held back by duckdb 1.4 (rusqlite in mz-deploy's build cache is on 0.12)
+ { name = "hashlink", version = "0.10.0" },
# held back by owo-colors 4.3 (mz-deploy terminal styling)
{ name = "supports-color", version = "2.1.0" },
diff --git a/doc/user/content/developer-tools/mcp-server/coding-agent-skills.md b/doc/user/content/developer-tools/mcp-server/coding-agent-skills.md
index 9e45cb25c6822..bde6a4ad3b7a9 100644
--- a/doc/user/content/developer-tools/mcp-server/coding-agent-skills.md
+++ b/doc/user/content/developer-tools/mcp-server/coding-agent-skills.md
@@ -40,11 +40,12 @@ When you install the skills as a plugin, they are namespaced under the plugin
name, for example `materialize:mz-dbt`.
If you already installed the skills with `npx skills`, remove them before you
-install the plugin. Run this in each project where you installed them, or add
-`-g` if you installed them globally:
+install the plugin. The command lists both the current and the previous skill
+names, and skips any you don't have. Run it in each project where you installed
+them, or add `-g` if you installed them globally:
```bash
-npx skills remove materialize-docs mz-dbt mz-debug-freshness mz-deploy mz-health-check mz-ontology-design mz-optimize-memory mz-terraform-provider mz-terraform-self-managed
+npx skills remove mz-docs mz-dbt mz-debug-freshness mz-deploy mz-health-check mz-ontology-design mz-optimize-memory mz-terraform-provider mz-terraform-self-managed materialize-docs materialize-dbt materialize-debug-freshness materialize-terraform-provider materialize-terraform-self-managed mcp-developer-analysis
```
#### Claude Code
@@ -114,15 +115,15 @@ npx skills update
#### Migrate from the previous skill names
-Most skills now use the `mz-` prefix, and `mcp-developer-analysis` is now
-`mz-health-check`. `materialize-docs` keeps its name. If you installed the
+All skills now use the `mz-` prefix: `materialize-docs` is now `mz-docs`, and
+`mcp-developer-analysis` is now `mz-health-check`. If you installed the
skills before the rename, remove the old copies and install the skills again,
so each skill appears once under its new name. Run these in each project where
you installed them, or add `-g` to both commands if you installed them
globally:
```bash
-npx skills remove materialize-dbt materialize-debug-freshness materialize-terraform-provider materialize-terraform-self-managed mcp-developer-analysis
+npx skills remove materialize-docs materialize-dbt materialize-debug-freshness materialize-terraform-provider materialize-terraform-self-managed mcp-developer-analysis
npx skills add MaterializeInc/agent-skills
```
@@ -143,7 +144,7 @@ installation and configuration.
## Reduce permission prompts (Claude Code)
Claude Code prompts before reading files outside your project, so it may ask
-to approve reads each time the `materialize-docs` skill opens a new
+to approve reads each time the `mz-docs` skill opens a new
documentation subdirectory. To stop these prompts, grant read access to the
directory where the skill is installed in `~/.claude/settings.json`.
@@ -160,13 +161,13 @@ update, so grant the parent:
```
If you installed the skills globally [with `npx skills`](#install-with-npx),
-they live under `~/.claude/skills/`. Grant the `materialize-docs` skill's
+they live under `~/.claude/skills/`. Grant the `mz-docs` skill's
directory:
```json
{
"permissions": {
- "additionalDirectories": ["~/.claude/skills/materialize-docs"]
+ "additionalDirectories": ["~/.claude/skills/mz-docs"]
}
}
```
diff --git a/doc/user/content/headless/agent-skills-table.md b/doc/user/content/headless/agent-skills-table.md
index 897a417fc8a76..4af3ecaf40235 100644
--- a/doc/user/content/headless/agent-skills-table.md
+++ b/doc/user/content/headless/agent-skills-table.md
@@ -6,7 +6,7 @@ headless: true
| `mz-health-check` | Use for operational introspection and troubleshooting via the `materialize-developer` server. Covers exact catalog schemas, diagnostic workflows, remediation runbooks, and guardrails for known pitfalls (cluster-scoped queries, uint8 ID mismatches, etc.).
Examples: *"why is my materialized view stale?"*, *"what can I optimize to save costs?"*, *"is my source healthy?"* |
| `mz-debug-freshness` | Use for diagnosing why an object is behind wall-clock time, from the lag ranking down to the operator and the SQL responsible; pairs with the `materialize-developer` server. Covers reading `mz_wallclock_global_lag` as a distribution, the status sweep that catches a stalled source whose frontier still looks current, lag history, hop-by-hop attribution through `mz_materialization_lag`, hydration and replica selection before `EXPLAIN ANALYZE`, per-operator cost and worker skew, and mapping an operator back to the clause that produced it. Diagnosis only, not remediation.
Examples: *"which object is causing my freshness alert?"*, *"is this delay ingestion or compute?"*, *"why is this dataflow falling behind its input?"* |
| `mz-optimize-memory` | Use for reducing the memory footprint and cost of a compute cluster; works over SQL or the `materialize-developer` MCP server. Covers finding which objects hold the memory, choosing and sizing the fix (indexes to add or drop, `GROUP SIZE` hints, slimmer views, subquery rewrites), verifying each change by measurement on an experiment cluster, and sizing the replica afterwards.
Examples: *"why is this cluster using so much memory?"*, *"can I downsize this replica?"*, *"which of these indexes should I drop?"* |
-| `materialize-docs` | Use for authoring view definitions, learning concepts, and looking up patterns; useful with either MCP server. Covers comprehensive Materialize documentation, including SQL syntax, idiomatic patterns, data ingestion, concepts, and best practices (400+ reference files).
Examples: *"show me how to deduplicate a stream"*, *"what's the idiomatic top-K pattern?"*, *"how do I create a Kafka source?"* |
+| `mz-docs` | Use for authoring view definitions, learning concepts, and looking up patterns; useful with either MCP server. Covers comprehensive Materialize documentation, including SQL syntax, idiomatic patterns, data ingestion, concepts, and best practices (400+ reference files).
Examples: *"show me how to deduplicate a stream"*, *"what's the idiomatic top-K pattern?"*, *"how do I create a Kafka source?"* |
| `mz-dbt` | Use for managing Materialize pipelines with dbt. Covers dbt-materialize adapter usage: materializations, profile configuration, index creation, blue/green deployments, and testing.
Examples: *"write a dbt model for a materialized view"*, *"how do I do a blue/green deployment with dbt?"* |
| `mz-terraform-provider` | Use for managing Materialize resources declaratively with Terraform. Covers provider configuration for Cloud and self-managed, navigation into the provider's auto-generated resource reference, cross-resource patterns, import workflows, and gotchas.
Examples: *"create a Kafka source with Terraform"*, *"import my existing clusters into Terraform state"*, *"set up RBAC grants in Terraform"* |
| `mz-terraform-self-managed` | Use for deploying or operating self-managed Materialize infrastructure with Terraform. Covers module layout and variables for deploying on AWS, Azure, and GCP: networking, Kubernetes, backend URL formats, instance sizing, upgrades, and gotchas.
Examples: *"deploy Materialize on EKS"*, *"what instance types should Materialize nodes use?"*, *"upgrade my self-managed Materialize"* |
diff --git a/doc/user/layouts/index.skill.md b/doc/user/layouts/index.skill.md
index 097e604180c05..1b14ee05b16e8 100644
--- a/doc/user/layouts/index.skill.md
+++ b/doc/user/layouts/index.skill.md
@@ -1,6 +1,6 @@
{{- /* Home page template - generates SKILL.md for Claude */ -}}
---
-name: materialize-docs
+name: mz-docs
description: Materialize documentation for SQL syntax, data ingestion, concepts, and best practices. Use when users ask about Materialize queries, sources, sinks, views, or clusters.
---
diff --git a/misc/python/materialize/scratch.py b/misc/python/materialize/scratch.py
index 3e824165e1083..94b23caa5dbef 100644
--- a/misc/python/materialize/scratch.py
+++ b/misc/python/materialize/scratch.py
@@ -274,7 +274,7 @@ def msftp(
def setup_ai_tools(instance: Instance) -> None:
"""Transfer local Claude Code and Codex configs to a scratch instance
- and install the materialize-docs skill."""
+ and install the mz-docs skill."""
import io
import pathlib
import tarfile
@@ -323,7 +323,7 @@ def setup_ai_tools(instance: Instance) -> None:
instance,
"cd materialize && "
"NPM_CONFIG_UPDATE_NOTIFIER=false "
- "npx -q -y skills add MaterializeInc/agent-skills -g -a claude-code --copy -y --skill materialize-docs 2>/dev/null || true",
+ "npx -q -y skills add MaterializeInc/agent-skills -g -a claude-code --copy -y --skill mz-docs 2>/dev/null || true",
quiet=True,
)
@@ -331,7 +331,7 @@ def setup_ai_tools(instance: Instance) -> None:
instance,
"cd materialize && "
"NPM_CONFIG_UPDATE_NOTIFIER=false "
- "npx -q -y skills add MaterializeInc/agent-skills -g -a codex --copy -y --skill materialize-docs 2>/dev/null || true",
+ "npx -q -y skills add MaterializeInc/agent-skills -g -a codex --copy -y --skill mz-docs 2>/dev/null || true",
quiet=True,
)
diff --git a/misc/wasm/Cargo.lock b/misc/wasm/Cargo.lock
index 9d7018742b32f..d1d58bea8c7e9 100644
--- a/misc/wasm/Cargo.lock
+++ b/misc/wasm/Cargo.lock
@@ -134,6 +134,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+[[package]]
+name = "base64"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
+
[[package]]
name = "base64ct"
version = "1.8.3"
@@ -186,6 +192,17 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+[[package]]
+name = "chacha20"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "rand_core 0.10.1",
+]
+
[[package]]
name = "core-foundation"
version = "0.10.1"
@@ -202,6 +219,15 @@ version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+[[package]]
+name = "cpufeatures"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "crc32fast"
version = "1.5.2"
@@ -447,10 +473,22 @@ checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
dependencies = [
"cfg-if",
"libc",
- "r-efi",
+ "r-efi 5.2.0",
"wasi 0.14.2+wasi-0.2.4",
]
+[[package]]
+name = "getrandom"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi 6.0.0",
+ "rand_core 0.10.1",
+]
+
[[package]]
name = "gimli"
version = "0.31.1"
@@ -459,9 +497,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
[[package]]
name = "h2"
-version = "0.4.10"
+version = "0.4.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5"
+checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16"
dependencies = [
"atomic-waker",
"bytes",
@@ -546,9 +584,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "hyper"
-version = "1.8.1"
+version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"
+checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43"
dependencies = [
"atomic-waker",
"bytes",
@@ -561,7 +599,6 @@ dependencies = [
"httpdate",
"itoa",
"pin-project-lite",
- "pin-utils",
"smallvec",
"tokio",
"want",
@@ -598,16 +635,17 @@ dependencies = [
[[package]]
name = "hyper-util"
-version = "0.1.20"
+version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
+checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff"
dependencies = [
- "base64",
+ "base64 0.23.1",
"bytes",
"futures-channel",
"futures-util",
"http",
"http-body",
+ "httparse",
"hyper",
"ipnet",
"libc",
@@ -946,7 +984,7 @@ dependencies = [
"sentry-tracing",
"serde",
"stacker",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio-native-tls",
"tonic",
"tracing",
@@ -1007,7 +1045,7 @@ dependencies = [
"mz-walkabout",
"serde",
"smallvec",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tracing",
]
@@ -1028,7 +1066,7 @@ dependencies = [
"itertools",
"mz-sql-parser",
"pretty",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
]
[[package]]
@@ -1225,23 +1263,23 @@ dependencies = [
[[package]]
name = "opentelemetry"
-version = "0.32.0"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682"
+checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5"
dependencies = [
"futures-core",
"futures-sink",
"js-sys",
"pin-project-lite",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tracing",
]
[[package]]
name = "opentelemetry-http"
-version = "0.32.0"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331"
+checksum = "ee2c3625b8aa04209f7e01e513bc8044da9687fa38a9eacf59628ca5f3b87300"
dependencies = [
"async-trait",
"bytes",
@@ -1252,18 +1290,19 @@ dependencies = [
[[package]]
name = "opentelemetry-otlp"
-version = "0.32.0"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35"
+checksum = "699a67345e21962a231955b9059a157e428b219fa5df8efe11f08346fb23a344"
dependencies = [
"http",
+ "httpdate",
"opentelemetry",
"opentelemetry-http",
"opentelemetry-proto",
"opentelemetry_sdk",
"prost",
"reqwest",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"tokio",
"tonic",
"tonic-types",
@@ -1271,9 +1310,9 @@ dependencies = [
[[package]]
name = "opentelemetry-proto"
-version = "0.32.0"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638"
+checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d"
dependencies = [
"opentelemetry",
"opentelemetry_sdk",
@@ -1284,9 +1323,9 @@ dependencies = [
[[package]]
name = "opentelemetry_sdk"
-version = "0.32.1"
+version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9"
+checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520"
dependencies = [
"futures-channel",
"futures-executor",
@@ -1294,8 +1333,8 @@ dependencies = [
"opentelemetry",
"percent-encoding",
"portable-atomic",
- "rand",
- "thiserror 2.0.20",
+ "rand 0.9.5",
+ "thiserror 2.0.21",
"tokio",
"tokio-stream",
]
@@ -1436,12 +1475,6 @@ version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
-[[package]]
-name = "pin-utils"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
-
[[package]]
name = "pkg-config"
version = "0.3.32"
@@ -1501,7 +1534,7 @@ dependencies = [
"memchr",
"parking_lot",
"protobuf",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
]
[[package]]
@@ -1581,6 +1614,12 @@ version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
+[[package]]
+name = "r-efi"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+
[[package]]
name = "rand"
version = "0.9.5"
@@ -1588,7 +1627,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
dependencies = [
"rand_chacha",
- "rand_core",
+ "rand_core 0.9.3",
+]
+
+[[package]]
+name = "rand"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af"
+dependencies = [
+ "chacha20",
+ "getrandom 0.4.3",
+ "rand_core 0.10.1",
]
[[package]]
@@ -1598,7 +1648,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
- "rand_core",
+ "rand_core 0.9.3",
]
[[package]]
@@ -1607,9 +1657,15 @@ version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
dependencies = [
- "getrandom",
+ "getrandom 0.3.3",
]
+[[package]]
+name = "rand_core"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
+
[[package]]
name = "redox_syscall"
version = "0.5.12"
@@ -1654,7 +1710,7 @@ version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"
dependencies = [
- "base64",
+ "base64 0.22.1",
"bytes",
"futures-channel",
"futures-core",
@@ -1780,9 +1836,9 @@ checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0"
[[package]]
name = "sentry"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76a88b65feb368d9dde5d531a2b59b25016e36fd6e4978432371a3ee77746ca2"
+checksum = "9fa951f4e644464ebfd2347a7ab03b0828c512d2448a76262cad2607b343dff4"
dependencies = [
"cfg_aliases",
"httpdate",
@@ -1799,9 +1855,9 @@ dependencies = [
[[package]]
name = "sentry-backtrace"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c51511d67ed670266a93afeaa26a08019b04ad1420cb9191da30f2338d22c48"
+checksum = "f4f27cf048aa63ccd5bc94c29989133eabb82e9c6bf7441b58e9d4a95837a105"
dependencies = [
"backtrace",
"regex",
@@ -1810,9 +1866,9 @@ dependencies = [
[[package]]
name = "sentry-contexts"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b757ac1f335856e8d7f5062a1fd9bc07a05a7eb3cc48247db79bf2618a490bd"
+checksum = "c5ed680585dbdacdf9859c2b4dc53358ff02bd256766882e22e343836db36b22"
dependencies = [
"hostname",
"libc",
@@ -1824,11 +1880,11 @@ dependencies = [
[[package]]
name = "sentry-core"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d889520a375e5b93efb0a66def1630d21d168b0251eb55b286b03fa940ccfc84"
+checksum = "c3ed398494bb8fc412a11a182c1883a10ae147587b8f6314abd222d1d626f31b"
dependencies = [
- "rand",
+ "rand 0.10.3",
"sentry-types",
"serde",
"serde_json",
@@ -1837,9 +1893,9 @@ dependencies = [
[[package]]
name = "sentry-debug-images"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "656487fd5f7b7c0105b2e82171982aac64489e0cb679436038febf070f2f76d6"
+checksum = "55854bb40681e84ffd2a69578ad4ef9bfaf5bf2bf1380a745578b591fa24af22"
dependencies = [
"findshlibs",
"sentry-core",
@@ -1847,9 +1903,9 @@ dependencies = [
[[package]]
name = "sentry-tracing"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77796d14eedbef22c2fe78e9c69fb09f14c7ad607e624d48dce92eedc17a136e"
+checksum = "e7e7db1aefdf623f4e48e21ec9e4af535024187aeafb2a4cc7e29d035def5a13"
dependencies = [
"bitflags",
"sentry-backtrace",
@@ -1860,16 +1916,16 @@ dependencies = [
[[package]]
name = "sentry-types"
-version = "0.49.2"
+version = "0.49.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc71f5ca55942d9b2901af95d5df5c5d65c164134c22a83682cac3d0b6c7ef2d"
+checksum = "1c18bf7b159bdfc7c8f97816f476b4947a1937fb6bea9d38d3479e58631614ac"
dependencies = [
"debugid",
"hex",
- "rand",
+ "rand 0.10.3",
"serde",
"serde_json",
- "thiserror 2.0.20",
+ "thiserror 2.0.21",
"time",
"url",
"uuid",
@@ -1954,9 +2010,9 @@ dependencies = [
[[package]]
name = "smallvec"
-version = "1.16.1"
+version = "1.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891"
+checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e"
[[package]]
name = "socket2"
@@ -2082,11 +2138,11 @@ dependencies = [
[[package]]
name = "thiserror"
-version = "2.0.20"
+version = "2.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
+checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e"
dependencies = [
- "thiserror-impl 2.0.20",
+ "thiserror-impl 2.0.21",
]
[[package]]
@@ -2102,9 +2158,9 @@ dependencies = [
[[package]]
name = "thiserror-impl"
-version = "2.0.20"
+version = "2.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
+checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
dependencies = [
"proc-macro2",
"quote",
@@ -2230,7 +2286,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
dependencies = [
"async-trait",
"axum",
- "base64",
+ "base64 0.22.1",
"bytes",
"flate2",
"h2",
@@ -2368,9 +2424,9 @@ dependencies = [
[[package]]
name = "tracing-opentelemetry"
-version = "0.33.0"
+version = "0.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26"
+checksum = "0a904802a1b902f43638b677ff2a650847e3b4404101b6c586d648e8c1e3e8fe"
dependencies = [
"js-sys",
"opentelemetry",
@@ -2461,7 +2517,7 @@ version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdc97a28575b85cfedf2a7e7d3cc64b3e11bd8ac766666318003abbacc7a21fc"
dependencies = [
- "base64",
+ "base64 0.22.1",
"der",
"log",
"native-tls",
@@ -2478,7 +2534,7 @@ version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f"
dependencies = [
- "base64",
+ "base64 0.22.1",
"http",
"httparse",
"log",
diff --git a/src/adapter/src/catalog/apply.rs b/src/adapter/src/catalog/apply.rs
index a7c3721f3b7e8..ed0c6e5e06aea 100644
--- a/src/adapter/src/catalog/apply.rs
+++ b/src/adapter/src/catalog/apply.rs
@@ -1482,9 +1482,7 @@ impl CatalogState {
}
StateUpdateKind::Item(item) => self.pack_item_update(item.id, diff),
StateUpdateKind::Comment(_) => Vec::new(),
- StateUpdateKind::SourceReferences(source_references) => {
- self.pack_source_references_update(&source_references, diff)
- }
+ StateUpdateKind::SourceReferences(_) => Vec::new(),
// mz_audit_events is a MaterializedView backed by
// mz_internal.mz_catalog_raw, so audit log rows do not produce
// builtin table updates here.
diff --git a/src/adapter/src/catalog/builtin_table_updates.rs b/src/adapter/src/catalog/builtin_table_updates.rs
index b6653b1305ad2..6b92124b586f9 100644
--- a/src/adapter/src/catalog/builtin_table_updates.rs
+++ b/src/adapter/src/catalog/builtin_table_updates.rs
@@ -19,10 +19,9 @@ use mz_catalog::builtin::{
MZ_CLUSTER_REPLICA_SIZES, MZ_COLUMNS, MZ_EGRESS_IPS, MZ_FUNCTIONS,
MZ_HISTORY_RETENTION_STRATEGIES, MZ_INDEX_COLUMNS, MZ_LICENSE_KEYS, MZ_LIST_TYPES,
MZ_MAP_TYPES, MZ_MATERIALIZED_VIEW_REFRESH_STRATEGIES, MZ_OPERATORS, MZ_PSEUDO_TYPES,
- MZ_REPLACEMENTS, MZ_ROLE_AUTH, MZ_SESSIONS, MZ_SOURCE_REFERENCES, MZ_STORAGE_USAGE_BY_SHARD,
- MZ_SUBSCRIPTIONS, MZ_TYPE_PG_METADATA, MZ_TYPES, MZ_WEBHOOKS_SOURCES,
+ MZ_REPLACEMENTS, MZ_ROLE_AUTH, MZ_SESSIONS, MZ_STORAGE_USAGE_BY_SHARD, MZ_SUBSCRIPTIONS,
+ MZ_TYPE_PG_METADATA, MZ_TYPES, MZ_WEBHOOKS_SOURCES,
};
-use mz_catalog::durable::SourceReferences;
use mz_catalog::memory::error::Error;
use mz_catalog::memory::objects::{
CatalogItem, DataSourceDesc, Func, Index, MaterializedView, Table, TableDataSource, Type,
@@ -853,52 +852,4 @@ impl CatalogState {
diff,
)
}
-
- pub fn pack_source_references_update(
- &self,
- source_references: &SourceReferences,
- diff: Diff,
- ) -> Vec> {
- let source_id = source_references.source_id.to_string();
- let updated_at = &source_references.updated_at;
- source_references
- .references
- .iter()
- .map(|reference| {
- let mut row = Row::default();
- let mut packer = row.packer();
- packer.extend([
- Datum::String(&source_id),
- reference
- .namespace
- .as_ref()
- .map(|s| Datum::String(s))
- .unwrap_or(Datum::Null),
- Datum::String(&reference.name),
- Datum::TimestampTz(
- mz_ore::now::to_datetime(*updated_at)
- .try_into()
- .expect("must fit"),
- ),
- ]);
- if reference.columns.len() > 0 {
- packer
- .try_push_array(
- &[ArrayDimension {
- lower_bound: 1,
- length: reference.columns.len(),
- }],
- reference.columns.iter().map(|col| Datum::String(col)),
- )
- .expect(
- "columns is 1 dimensional, and its length is used for the array length",
- );
- } else {
- packer.push(Datum::Null);
- }
-
- BuiltinTableUpdate::row(&*MZ_SOURCE_REFERENCES, row, diff)
- })
- .collect()
- }
}
diff --git a/src/adapter/src/catalog/open/builtin_schema_migration.rs b/src/adapter/src/catalog/open/builtin_schema_migration.rs
index 48e295d21bf2a..707be62f46b59 100644
--- a/src/adapter/src/catalog/open/builtin_schema_migration.rs
+++ b/src/adapter/src/catalog/open/builtin_schema_migration.rs
@@ -487,6 +487,15 @@ static MIGRATIONS: LazyLock> = LazyLock::new(|| {
MZ_INTERNAL_SCHEMA,
"mz_object_global_ids",
),
+ // Converting mz_source_references from a builtin table to a
+ // materialized view over mz_catalog_raw changes its catalog
+ // fingerprint.
+ MigrationStep::replacement(
+ "26.45.0-dev.0",
+ CatalogItemType::MaterializedView,
+ MZ_INTERNAL_SCHEMA,
+ "mz_source_references",
+ ),
]
});
diff --git a/src/audit-log/src/lib.rs b/src/audit-log/src/lib.rs
index 0a0b202704a53..1d63b647128ea 100644
--- a/src/audit-log/src/lib.rs
+++ b/src/audit-log/src/lib.rs
@@ -25,7 +25,7 @@
//! `mz_catalog.mz_audit_events.details` exposes. The durable catalog stores
//! the proto twin from `mz_catalog_protos::objects::audit_log_event_v1`,
//! which `parse_catalog_audit_log_details` (in
-//! `src/expr/src/scalar/func/impls/jsonb.rs`) reshapes back into
+//! `src/catalog-decode/src/audit_log.rs`) reshapes back into
//! `as_json`'s output. Changes here that shift that output (new variants,
//! renamed fields, added `skip_serializing_if`, field-name diffs against
//! the proto) need matching updates there. The round-trip is covered by
diff --git a/src/catalog-decode/Cargo.toml b/src/catalog-decode/Cargo.toml
new file mode 100644
index 0000000000000..118c25f517b4e
--- /dev/null
+++ b/src/catalog-decode/Cargo.toml
@@ -0,0 +1,25 @@
+[package]
+name = "mz-catalog-decode"
+description = "Decoders for durable catalog data, backing the catalog-raw SQL functions."
+version = "0.0.0"
+edition.workspace = true
+rust-version.workspace = true
+publish = false
+
+[lints]
+workspace = true
+
+[dependencies]
+hex.workspace = true
+mz-ore = { path = "../ore", features = ["assert"] }
+mz-repr = { path = "../repr" }
+mz-sql-parser = { path = "../sql-parser" }
+prost.workspace = true
+serde_json.workspace = true
+tracing.workspace = true
+
+[dev-dependencies]
+mz-ore = { path = "../ore", features = ["test"] }
+
+[features]
+default = []
diff --git a/src/expr/src/scalar/func/impls/audit_log_details.rs b/src/catalog-decode/src/audit_log.rs
similarity index 95%
rename from src/expr/src/scalar/func/impls/audit_log_details.rs
rename to src/catalog-decode/src/audit_log.rs
index 1692460f730b0..54ae6c002c6e7 100644
--- a/src/expr/src/scalar/func/impls/audit_log_details.rs
+++ b/src/catalog-decode/src/audit_log.rs
@@ -26,12 +26,9 @@
//! variant and catches drift when either side gains a new variant, field,
//! or serde attribute.
-use mz_expr_derive::sqlfunc;
use mz_repr::adt::jsonb::{Jsonb, JsonbRef};
use mz_repr::{Datum, Row, RowPacker};
-use crate::EvalError;
-
/// Reshapes proto `audit_log_event_v1::Details` JSON from `mz_catalog_raw`
/// (e.g. `{"IdFullNameV1": {"id": "u1", "name": {...}}}`) into the shape
/// `mz_audit_log::EventDetails::as_json` produces: the format the prior
@@ -40,8 +37,7 @@ use crate::EvalError;
/// See the module docstring for the mechanism and the reciprocal side. Rule
/// changes must be paired with tests; the property test at
/// `src/catalog/tests/audit_log_details.rs` is the safety net.
-#[sqlfunc]
-fn parse_catalog_audit_log_details<'a>(a: JsonbRef<'a>) -> Result {
+pub fn details(a: JsonbRef<'_>) -> Result {
/// `(variant, path, field, sub_variant)`: `#[serde(flatten)]` sites.
/// `sub_variant` sets the context for rules on the hoisted content
/// when the flattened struct itself has a `#[serde(flatten)]`.
@@ -472,43 +468,39 @@ fn parse_catalog_audit_log_details<'a>(a: JsonbRef<'a>) -> Result Result {
- let Datum::Map(dict) = a.into_datum() else {
- return Err("expected object".into());
- };
- let mut iter = dict.iter();
- let (variant, inner) = iter
- .next()
- .ok_or_else(|| "empty details enum".to_string())?;
- if iter.next().is_some() {
- return Err("details enum had multiple keys".into());
- }
- let mut row = Row::default();
- // `ResetAllV1` is the only variant `as_json` maps to null (the
- // proto `Empty` payload serializes to `{}`).
- if variant == "ResetAllV1" {
- row.packer().push(Datum::JsonNull);
- return Ok(Jsonb::from_row(row));
- }
- let Datum::Map(_) = inner else {
- return Err(format!("expected inner object for variant {variant}"));
- };
- rewrite(inner, variant, "", &mut row.packer())?;
- Ok(Jsonb::from_row(row))
+ let Datum::Map(dict) = a.into_datum() else {
+ return Err("expected object".into());
};
-
- parse().map_err(|e| EvalError::InvalidCatalogJson(e.into()))
+ let mut iter = dict.iter();
+ let (variant, inner) = iter
+ .next()
+ .ok_or_else(|| "empty details enum".to_string())?;
+ if iter.next().is_some() {
+ return Err("details enum had multiple keys".into());
+ }
+ let mut row = Row::default();
+ // `ResetAllV1` is the only variant `as_json` maps to null (the
+ // proto `Empty` payload serializes to `{}`).
+ if variant == "ResetAllV1" {
+ row.packer().push(Datum::JsonNull);
+ return Ok(Jsonb::from_row(row));
+ }
+ let Datum::Map(_) = inner else {
+ return Err(format!("expected inner object for variant {variant}"));
+ };
+ rewrite(inner, variant, "", &mut row.packer())?;
+ Ok(Jsonb::from_row(row))
}
#[cfg(test)]
mod tests {
use super::*;
- /// Round-trip a JSON `input` through `parse_catalog_audit_log_details` and
+ /// Round-trip a JSON `input` through `details` and
/// assert the resulting JSON parses equal to `expected`.
fn check(input: &str, expected: &str) {
let input: Jsonb = input.parse().expect("valid input JSONB");
- let actual = parse_catalog_audit_log_details(input.as_ref())
+ let actual = details(input.as_ref())
.expect("helper succeeded")
.to_string();
let actual_value: serde_json::Value =
@@ -518,11 +510,11 @@ mod tests {
assert_eq!(actual_value, expected_value);
}
- /// Run `parse_catalog_audit_log_details` on `input` and assert it returns
- /// a `InvalidCatalogJson` error containing `expected_substr`.
+ /// Run `details` on `input` and assert it returns an error containing
+ /// `expected_substr`.
fn check_err(input: &str, expected_substr: &str) {
let input: Jsonb = input.parse().expect("valid input JSONB");
- let err = parse_catalog_audit_log_details(input.as_ref()).expect_err("helper should error");
+ let err = details(input.as_ref()).expect_err("helper should error");
let msg = format!("{err:?}");
assert!(
msg.contains(expected_substr),
diff --git a/src/catalog-decode/src/create_sql.rs b/src/catalog-decode/src/create_sql.rs
new file mode 100644
index 0000000000000..e79c46cc20a64
--- /dev/null
+++ b/src/catalog-decode/src/create_sql.rs
@@ -0,0 +1,1960 @@
+// Copyright Materialize, Inc. and contributors. All rights reserved.
+//
+// Use of this software is governed by the Business Source License
+// included in the LICENSE file.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0.
+
+//! Decoders for catalog `create_sql` strings.
+//!
+//! Each function parses a persisted `create_sql` and returns a JSON value with
+//! the fields a builtin catalog view projects. Errors are human-readable
+//! strings.
+
+use std::collections::BTreeMap;
+
+use mz_sql_parser::ast::display::AstDisplay;
+use mz_sql_parser::ast::item_refs::collect_item_references;
+use mz_sql_parser::ast::{
+ AstInfo, AvroSchema, ConnectionOption, ConnectionOptionName, CreateConnectionType,
+ CreateSinkConnection, CreateSubsourceOptionName, Format, FormatSpecifier,
+ IcebergSinkConfigOptionName, IcebergSinkMode, KafkaSinkConfigOptionName,
+ KafkaSourceConfigOptionName, PgConfigOptionName, ProtobufSchema, Raw, RawClusterName,
+ RawItemName, SinkEnvelope, SourceEnvelope, SourceErrorPolicy, Statement, UnresolvedItemName,
+ Value, WithOptionValue,
+};
+use prost::Message as _;
+use serde_json::json;
+
+/// Parses `sql`, which must contain exactly one statement.
+fn parse_single_statement(sql: &str) -> Result, String> {
+ let mut stmts = mz_sql_parser::parser::parse_statements(sql)
+ .map_err(|e| format!("failed to parse create_sql: {e}"))?;
+ match stmts.len() {
+ 1 => Ok(stmts.remove(0).ast),
+ n => Err(format!("expected a single statement, found {n}")),
+ }
+}
+
+fn item_id(item: RawItemName) -> Result {
+ match item {
+ RawItemName::Id(id, _, _) => Ok(id),
+ RawItemName::Name(_) => Err("unresolved item name"),
+ }
+}
+
+fn cluster_id(in_cluster: RawClusterName) -> Result {
+ match in_cluster {
+ RawClusterName::Resolved(s) => Ok(s),
+ RawClusterName::Unresolved(_) => Err("unresolved cluster name"),
+ }
+}
+
+fn format_name(fmt: &Format) -> &'static str {
+ match fmt {
+ Format::Bytes => "bytes",
+ Format::Avro(_) => "avro",
+ Format::Protobuf(_) => "protobuf",
+ Format::Regex(_) => "regex",
+ Format::Csv { .. } => "csv",
+ Format::Json { .. } => "json",
+ Format::Text => "text",
+ }
+}
+
+/// Extracts the string form of a `WithOptionValue`, matching how the planner
+/// coerces option values. The blanket `TryFromValue>` impl in
+/// `src/sql/src/plan/with_options.rs` accepts a quoted string, a bare identifier,
+/// and a 1-part unresolved item name, all yielding a string. Any other variant
+/// yields None.
+///
+/// `AstDisplay` does not re-quote bare identifiers or 1-part names, so those
+/// forms persist unquoted in `create_sql`. Matching only `Value::String` here
+/// would miss them and silently fall back to a default, so the catalog-raw
+/// parsers below route their string options through this helper.
+fn option_string(value: &WithOptionValue) -> Option {
+ match value {
+ WithOptionValue::Value(Value::String(s)) => Some(s.clone()),
+ WithOptionValue::Ident(ident) => Some(ident.clone().into_string()),
+ WithOptionValue::UnresolvedItemName(UnresolvedItemName(parts)) if parts.len() == 1 => {
+ Some(parts[0].clone().into_string())
+ }
+ _ => None,
+ }
+}
+
+/// Parses a catalog `create_sql` string into a JSON object.
+///
+/// The returned JSON does not fully reflect the parsed SQL and instead contains only fields
+/// required by current callers.
+pub fn item_details(a: &str) -> Result {
+ let stmt = parse_single_statement(a)?;
+
+ let mut info = BTreeMap::<&str, serde_json::Value>::new();
+
+ use mz_sql_parser::ast::Statement::*;
+ let item_type = match stmt {
+ CreateSecret(_) => "secret",
+ CreateConnection(stmt) => {
+ let connection_type = stmt.connection_type.as_str();
+ info.insert("connection_type", json!(connection_type));
+
+ "connection"
+ }
+ CreateView(stmt) => {
+ let mut definition = stmt.definition.query.to_ast_string_stable();
+ // PostgreSQL appends a semicolon in `pg_views.definition`, we
+ // do the same for compatibility's sake.
+ definition.push(';');
+ info.insert("definition", json!(definition));
+
+ "view"
+ }
+ CreateMaterializedView(stmt) => {
+ let Some(in_cluster) = stmt.in_cluster else {
+ return Err("missing IN CLUSTER".into());
+ };
+ let cluster_id = match in_cluster {
+ RawClusterName::Unresolved(ident) => ident.into_string(),
+ RawClusterName::Resolved(s) => s,
+ };
+ info.insert("cluster_id", json!(cluster_id));
+
+ let mut definition = stmt.query.to_ast_string_stable();
+ definition.push(';');
+ info.insert("definition", json!(definition));
+
+ "materialized-view"
+ }
+ CreateTable(_) => "table",
+ CreateTableFromSource(stmt) => {
+ let source_id = item_id(stmt.source)?;
+ info.insert("source_id", json!(source_id));
+
+ "table"
+ }
+ CreateSource(stmt) => {
+ let Some(in_cluster) = stmt.in_cluster else {
+ return Err("missing IN CLUSTER".into());
+ };
+ let cluster_id = cluster_id(in_cluster)?;
+ info.insert("cluster_id", json!(cluster_id));
+
+ use mz_sql_parser::ast::CreateSourceConnection::*;
+ let (source_type, connection) = match stmt.connection {
+ Kafka { connection, .. } => ("kafka", Some(connection)),
+ Postgres { connection, .. } => ("postgres", Some(connection)),
+ MySql { connection, .. } => ("mysql", Some(connection)),
+ SqlServer { connection, .. } => ("sql-server", Some(connection)),
+ LoadGenerator { .. } => ("load-generator", None),
+ };
+ info.insert("source_type", json!(source_type));
+ if let Some(conn) = connection {
+ let conn_id = item_id(conn)?;
+ info.insert("connection_id", json!(conn_id));
+ }
+
+ let is_debezium = matches!(
+ stmt.envelope,
+ Some(mz_sql_parser::ast::SourceEnvelope::Debezium)
+ );
+
+ // An old-syntax kafka source ingests into its own relation, so an
+ // omitted ENVELOPE means the default ENVELOPE NONE and the pre-MV
+ // packer reported 'none'. A new-syntax source (no progress
+ // subsource, hence no EXPOSE PROGRESS AS in create_sql) ingests
+ // nothing itself. Its envelopes live on the per-table exports, so
+ // its own envelope_type stays absent (SQL NULL), matching released
+ // behavior. `progress_subsource.is_some()` is the planner's own
+ // old-vs-new discriminator (see `OldSyntaxIngestion` in
+ // plan_create_source). Non-kafka sources carry no envelope either.
+ // See the `mz_sources.envelope_type` column.
+ let envelope_type = match &stmt.envelope {
+ Some(envelope) => {
+ use mz_sql_parser::ast::SourceEnvelope::*;
+ Some(match envelope {
+ None => "none",
+ Debezium => "debezium",
+ Upsert { .. } => "upsert",
+ CdcV2 => "materialize",
+ })
+ }
+ None if source_type == "kafka" && stmt.progress_subsource.is_some() => Some("none"),
+ None => None,
+ };
+ if let Some(envelope_type) = envelope_type {
+ info.insert("envelope_type", json!(envelope_type));
+ }
+
+ if let Some(format_spec) = stmt.format {
+ match &format_spec {
+ FormatSpecifier::Bare(fmt) => {
+ // Debezium sources with a single format spec implicitly use
+ // the same format for both key and value.
+ if is_debezium {
+ info.insert("key_format", json!(format_name(fmt)));
+ }
+ info.insert("value_format", json!(format_name(fmt)));
+ }
+ FormatSpecifier::KeyValue { key, value } => {
+ info.insert("key_format", json!(format_name(key)));
+ info.insert("value_format", json!(format_name(value)));
+ }
+ }
+ }
+
+ "source"
+ }
+ CreateWebhookSource(stmt) => {
+ if stmt.is_table {
+ "table"
+ } else {
+ info.insert("source_type", json!("webhook"));
+ if let Some(in_cluster) = stmt.in_cluster {
+ let cluster_id = cluster_id(in_cluster)?;
+ info.insert("cluster_id", json!(cluster_id));
+ }
+ "source"
+ }
+ }
+ CreateSubsource(stmt) => {
+ use mz_sql_parser::ast::CreateSubsourceOptionName;
+ let is_progress = stmt
+ .with_options
+ .iter()
+ .any(|o| matches!(o.name, CreateSubsourceOptionName::Progress));
+ let source_type = if is_progress { "progress" } else { "subsource" };
+ info.insert("source_type", json!(source_type));
+
+ if let Some(of_source) = stmt.of_source {
+ let of_source_id = item_id(of_source)?;
+ info.insert("of_source_id", json!(of_source_id));
+ }
+
+ "subsource"
+ }
+ // Everything the mz_sinks, mz_kafka_sinks and mz_iceberg_sinks
+ // views read. The Rust side of each value lives in `Sink` and
+ // `StorageSinkConnection`, so those and this have to move together.
+ //
+ // NOTE: we bail below if a sink has no resolved `IN CLUSTER`, no
+ // `TOPIC` on kafka, or no `NAMESPACE`/`TABLE` on iceberg. Planning
+ // guarantees all four. But if one ever slipped through it would
+ // take down every view built on this function, not just the sink
+ // ones.
+ CreateSink(stmt) => {
+ let Some(in_cluster) = stmt.in_cluster else {
+ return Err("missing IN CLUSTER".into());
+ };
+ info.insert("cluster_id", json!(cluster_id(in_cluster)?));
+
+ match stmt.connection {
+ CreateSinkConnection::Kafka {
+ connection,
+ options,
+ key: sink_key,
+ ..
+ } => {
+ info.insert("sink_type", json!("kafka"));
+ info.insert("connection_id", json!(item_id(connection)?));
+
+ let topic = options
+ .into_iter()
+ .find(|o| o.name == KafkaSinkConfigOptionName::Topic)
+ .and_then(|o| o.value.as_ref().and_then(option_string))
+ .ok_or("kafka sink missing TOPIC")?;
+ info.insert("topic", json!(topic));
+
+ if let Some(envelope) = stmt.envelope {
+ let envelope_type = match envelope {
+ SinkEnvelope::Upsert => "upsert",
+ SinkEnvelope::Debezium => "debezium",
+ };
+ info.insert("envelope_type", json!(envelope_type));
+ }
+
+ if let Some(format_spec) = stmt.format {
+ // A key format only survives if the sink has a
+ // `KEY`. Without one `kafka_sink_builder` throws
+ // away the key half of a key/value spec, and does
+ // not copy a bare spec over to the key either.
+ let (key_format, value_format) = match (&format_spec, sink_key.is_some()) {
+ (FormatSpecifier::Bare(fmt), false) => (None, format_name(fmt)),
+ (FormatSpecifier::Bare(fmt), true) => {
+ (Some(format_name(fmt)), format_name(fmt))
+ }
+ (FormatSpecifier::KeyValue { value, .. }, false) => {
+ (None, format_name(value))
+ }
+ (FormatSpecifier::KeyValue { key, value }, true) => {
+ (Some(format_name(key)), format_name(value))
+ }
+ };
+ if let Some(key_format) = key_format {
+ info.insert("key_format", json!(key_format));
+ }
+ info.insert("value_format", json!(value_format));
+
+ // The deprecated combined `format`. Only avro/avro
+ // and json/json collapse to a single name.
+ // Everything else, text/text and bytes/bytes
+ // included, gets the composite form.
+ let combined = match key_format {
+ None => value_format.to_string(),
+ Some(key_format)
+ if key_format == value_format
+ && matches!(value_format, "avro" | "json") =>
+ {
+ value_format.to_string()
+ }
+ Some(key_format) => {
+ format!("key-{key_format}-value-{value_format}")
+ }
+ };
+ info.insert("format", json!(combined));
+ }
+ }
+ CreateSinkConnection::Iceberg {
+ catalog_connection,
+ options,
+ ..
+ } => {
+ info.insert("sink_type", json!("iceberg"));
+ // The catalog connection, not the optional AWS one.
+ info.insert("connection_id", json!(item_id(catalog_connection)?));
+
+ let mut namespace = None;
+ let mut table = None;
+ for option in options {
+ match option.name {
+ IcebergSinkConfigOptionName::Namespace => {
+ namespace = option.value.as_ref().and_then(option_string)
+ }
+ IcebergSinkConfigOptionName::Table => {
+ table = option.value.as_ref().and_then(option_string)
+ }
+ }
+ }
+ info.insert(
+ "namespace",
+ json!(namespace.ok_or("iceberg sink missing NAMESPACE")?),
+ );
+ info.insert("table", json!(table.ok_or("iceberg sink missing TABLE")?));
+
+ // Iceberg spells the envelope `MODE`, and has no format
+ // columns at all.
+ if let Some(mode) = stmt.mode {
+ let envelope_type = match mode {
+ IcebergSinkMode::Upsert => "upsert",
+ IcebergSinkMode::Append => "append",
+ };
+ info.insert("envelope_type", json!(envelope_type));
+ }
+ }
+ }
+
+ "sink"
+ }
+ CreateMetricSink(stmt) => {
+ let Some(in_cluster) = stmt.in_cluster else {
+ return Err("missing IN CLUSTER".into());
+ };
+ let cluster_id = cluster_id(in_cluster)?;
+ info.insert("cluster_id", json!(cluster_id));
+ let from_id = item_id(stmt.from)?;
+ info.insert("from_id", json!(from_id));
+ "metric-sink"
+ }
+ CreateIndex(stmt) => {
+ let Some(in_cluster) = stmt.in_cluster else {
+ return Err("missing IN CLUSTER".into());
+ };
+ let cluster_id = cluster_id(in_cluster)?;
+ info.insert("cluster_id", json!(cluster_id));
+ let on_id = item_id(stmt.on_name)?;
+ info.insert("on_id", json!(on_id));
+ "index"
+ }
+ CreateType(_) => "type",
+ // NOTE: every statement that creates a catalog item needs an arm above. These
+ // catalog views run this over every item row before their type filter drops the
+ // unwanted rows, so one unclassified `create_sql` takes out `mz_objects`,
+ // `mz_indexes`, and every sibling view at once. The match is exhaustive to make
+ // that a compile error here, not a runtime failure.
+ Select(_)
+ | Insert(_)
+ | Copy(_)
+ | Update(_)
+ | Delete(_)
+ | CreateDatabase(_)
+ | CreateSchema(_)
+ | CreateRole(_)
+ | CreateCluster(_)
+ | CreateClusterReplica(_)
+ | CreateNetworkPolicy(_)
+ | AlterCluster(_)
+ | AlterOwner(_)
+ | AlterObjectRename(_)
+ | AlterObjectSwap(_)
+ | AlterRetainHistory(_)
+ | AlterIndex(_)
+ | AlterSecret(_)
+ | AlterSetCluster(_)
+ | AlterSink(_)
+ | AlterSource(_)
+ | AlterSystemSet(_)
+ | AlterSystemReset(_)
+ | AlterSystemResetAll(_)
+ | AlterConnection(_)
+ | AlterNetworkPolicy(_)
+ | AlterRole(_)
+ | AlterTableAddColumn(_)
+ | AlterMaterializedViewApplyReplacement(_)
+ | Discard(_)
+ | DropObjects(_)
+ | DropOwned(_)
+ | SetVariable(_)
+ | ResetVariable(_)
+ | Show(_)
+ | StartTransaction(_)
+ | SetTransaction(_)
+ | Commit(_)
+ | Rollback(_)
+ | Subscribe(_)
+ | ExplainPlan(_)
+ | ExplainPushdown(_)
+ | ExplainTimestamp(_)
+ | ExplainSinkSchema(_)
+ | ExplainAnalyzeObject(_)
+ | ExplainAnalyzeCluster(_)
+ | Declare(_)
+ | Fetch(_)
+ | Close(_)
+ | Prepare(_)
+ | Execute(_)
+ | ExecuteUnitTest(_)
+ | Deallocate(_)
+ | Raise(_)
+ | GrantRole(_)
+ | RevokeRole(_)
+ | GrantPrivileges(_)
+ | RevokePrivileges(_)
+ | AlterDefaultPrivileges(_)
+ | ReassignOwned(_)
+ | ValidateConnection(_)
+ | Comment(_) => return Err("not a CREATE item statement".into()),
+ };
+ info.insert("type", json!(item_type));
+
+ let info = info.into_iter().map(|(k, v)| (k.to_string(), v)).collect();
+ Ok(info)
+}
+
+/// Extracts the catalog item references from a catalog `create_sql` string as a JSONB object.
+///
+/// - `ids`: array of catalog id strings from `[ AS ]` references.
+/// - `named_funcs`: array of function references such as "pg_catalog"."max".
+/// - `named_types`: array of type references such as "pg_catalog"."int4", exclusive from the ones
+/// in `ids`.
+/// - `named_relations`: array of relation references, exclusive from the ones in `ids`.
+pub fn item_references(a: &str) -> Result {
+ fn qualified(name: &UnresolvedItemName) -> serde_json::Value {
+ match &name.0[..] {
+ [.., schema, item] => json!({"schema": schema.as_str(), "name": item.as_str()}),
+ [item] => json!({"schema": serde_json::Value::Null, "name": item.as_str()}),
+ [] => json!({"schema": serde_json::Value::Null, "name": ""}),
+ }
+ }
+
+ let stmt = parse_single_statement(a)?;
+
+ let refs = collect_item_references(&stmt);
+ mz_ore::soft_assert_or_log!(
+ refs.named_array_elements.is_empty(),
+ "persisted create_sql should never carry an array type T[] \
+ and resolve its array type in `ids`: {:?}",
+ refs.named_array_elements
+ );
+ Ok(json!({
+ "ids": refs.ids.iter().collect::>(),
+ "named_funcs": refs.named_funcs.iter().map(qualified).collect::>(),
+ "named_types": refs.named_types.iter().map(qualified).collect::>(),
+ "named_relations": refs.named_relations.iter().map(qualified).collect::>(),
+ }))
+}
+
+/// Minimal decoder for `ProtoPostgresSourcePublicationDetails`. The
+/// canonical proto lives in `mz-storage-types`, which depends on this
+/// crate via `mz-expr`, so we redeclare the two tags we read here. Upstream tag
+/// renumbers slip past silently. The `mz_postgres_sources` lockdown
+/// SLTs catch them.
+#[derive(Clone, PartialEq, ::prost::Message)]
+struct PostgresPublicationDetailsSubset {
+ #[prost(string, tag = "2")]
+ slot: String,
+ #[prost(uint64, optional, tag = "3")]
+ timeline_id: Option,
+}
+
+/// Extracts postgres source publication details (slot, timeline_id) from a
+/// catalog `create_sql`. Returns:
+///
+/// - jsonb `{"slot": , "timeline_id": }` for `CREATE SOURCE ... FROM POSTGRES
+/// CONNECTION ... (DETAILS = ...)` statements.
+/// - jsonb `null` for any other statement.
+///
+/// Errors if the statement fails to parse, is a postgres source without
+/// a `DETAILS` option, or if the `DETAILS` value can't be hex- and
+/// proto-decoded.
+pub fn postgres_source_details(a: &str) -> Result {
+ let stmt = parse_single_statement(a)?;
+
+ use mz_sql_parser::ast::CreateSourceConnection;
+ use mz_sql_parser::ast::Statement::CreateSource;
+ let options = match stmt {
+ CreateSource(stmt) => match stmt.connection {
+ CreateSourceConnection::Postgres { options, .. } => options,
+ _ => return Ok(serde_json::Value::Null),
+ },
+ _ => return Ok(serde_json::Value::Null),
+ };
+
+ let details_hex = options
+ .into_iter()
+ .find(|opt| opt.name == PgConfigOptionName::Details)
+ .and_then(|opt| match opt.value {
+ Some(WithOptionValue::Value(Value::String(s))) => Some(s),
+ _ => None,
+ })
+ .ok_or("missing DETAILS option on postgres source")?;
+
+ let details_bytes =
+ hex::decode(&details_hex).map_err(|e| format!("DETAILS is not valid hex: {e}"))?;
+
+ let details = PostgresPublicationDetailsSubset::decode(&*details_bytes)
+ .map_err(|e| format!("DETAILS is not a valid publication-details proto: {e}"))?;
+
+ Ok(json!({
+ "slot": details.slot,
+ "timeline_id": details.timeline_id,
+ }))
+}
+
+/// Extracts kafka source configuration (topic, group id prefix, connection
+/// id) from a catalog `create_sql`. Returns:
+///
+/// - jsonb `{"topic": , "group_id_prefix": , "connection_id": }` for
+/// `CREATE SOURCE ... FROM KAFKA CONNECTION ... (TOPIC = ..., [GROUP ID PREFIX = ...])`
+/// statements.
+/// - jsonb `null` for any other statement.
+///
+/// Errors if the statement fails to parse, is a kafka source without a
+/// `TOPIC` option, or references an unresolved connection name (i.e. one
+/// that hasn't been through purification).
+pub fn kafka_source_details(a: &str) -> Result {
+ let stmt = parse_single_statement(a)?;
+
+ use mz_sql_parser::ast::CreateSourceConnection;
+ use mz_sql_parser::ast::Statement::CreateSource;
+ let (connection, options) = match stmt {
+ CreateSource(stmt) => match stmt.connection {
+ CreateSourceConnection::Kafka {
+ connection,
+ options,
+ } => (connection, options),
+ _ => return Ok(serde_json::Value::Null),
+ },
+ _ => return Ok(serde_json::Value::Null),
+ };
+
+ let connection_id = item_id(connection)?;
+
+ let mut topic: Option = None;
+ let mut group_id_prefix: Option = None;
+ for opt in options {
+ let string_value = opt.value.as_ref().and_then(option_string);
+ match opt.name {
+ KafkaSourceConfigOptionName::Topic => topic = string_value,
+ KafkaSourceConfigOptionName::GroupIdPrefix => group_id_prefix = string_value,
+ _ => {}
+ }
+ }
+
+ let topic = topic.ok_or("missing TOPIC option on kafka source")?;
+
+ Ok(json!({
+ "topic": topic,
+ "group_id_prefix": group_id_prefix,
+ "connection_id": connection_id,
+ }))
+}
+
+/// Extracts source-export (source table) metadata from a catalog `create_sql`.
+///
+/// Returns, for a `CREATE TABLE ... FROM SOURCE` or a non-progress
+/// `CREATE SUBSOURCE ... OF SOURCE ...` statement:
+///
+/// ```json
+/// {
+/// "source_id": "",
+/// "external_reference": ["part1", "part2", ...],
+/// "envelope_type": ,
+/// "key_format": ,
+/// "value_format":
+/// }
+/// ```
+///
+/// `envelope_type`, `key_format`, and `value_format` are always null for a
+/// `CREATE SUBSOURCE` (the postgres/mysql/sql-server exports that use the old
+/// subsource syntax carry neither format nor envelope). They may also be null
+/// for a `CREATE TABLE ... FROM SOURCE` that omits FORMAT/ENVELOPE.
+///
+/// Returns jsonb `null` for progress subsources and for any statement that is
+/// not a source export. The caller distinguishes the four source-table views
+/// by joining `source_id` against `mz_sources` and filtering on the parent's
+/// type, so this helper stays connection-type agnostic.
+///
+/// Errors if the statement fails to parse, references an unresolved item name,
+/// or is a non-progress subsource missing its OF SOURCE or EXTERNAL REFERENCE.
+///
+/// The `key_format`/`value_format` derivation mirrors the runtime
+/// `DataSourceDesc::formats()` that the removed `pack_kafka_source_tables_update`
+/// packer read. A bare FORMAT only carries a key when it resolves to an
+/// encoding that has one, which among bare formats is only Avro or Protobuf
+/// read from a Confluent Schema Registry whose purified seed carries a key
+/// schema. A KEY FORMAT ... VALUE FORMAT ... spec always carries both.
+pub fn source_export_details(a: &str) -> Result {
+ // A bare FORMAT resolves to an encoding with a key only for Avro or
+ // Protobuf read from a schema registry whose purified seed carries a key
+ // schema. Every other bare format is value-only.
+ fn bare_format_has_key(fmt: &Format) -> bool {
+ match fmt {
+ Format::Avro(AvroSchema::Csr { csr_connection }) => csr_connection
+ .seed
+ .as_ref()
+ .is_some_and(|seed| seed.key_schema.is_some()),
+ Format::Protobuf(ProtobufSchema::Csr { csr_connection }) => csr_connection
+ .seed
+ .as_ref()
+ .is_some_and(|seed| seed.key.is_some()),
+ _ => false,
+ }
+ }
+
+ fn key_value_formats(
+ spec: &FormatSpecifier,
+ ) -> (Option<&'static str>, Option<&'static str>) {
+ match spec {
+ FormatSpecifier::KeyValue { key, value } => {
+ (Some(format_name(key)), Some(format_name(value)))
+ }
+ FormatSpecifier::Bare(fmt) => {
+ let value = Some(format_name(fmt));
+ let key = bare_format_has_key(fmt).then(|| format_name(fmt));
+ (key, value)
+ }
+ }
+ }
+
+ fn envelope_name(envelope: &SourceEnvelope) -> &'static str {
+ match envelope {
+ SourceEnvelope::None => "none",
+ SourceEnvelope::Debezium => "debezium",
+ SourceEnvelope::Upsert {
+ value_decode_err_policy,
+ } => {
+ if value_decode_err_policy
+ .iter()
+ .any(|p| matches!(p, SourceErrorPolicy::Inline { .. }))
+ {
+ "upsert-value-err-inline"
+ } else {
+ "upsert"
+ }
+ }
+ SourceEnvelope::CdcV2 => "materialize",
+ }
+ }
+
+ let stmt = parse_single_statement(a)?;
+
+ use mz_sql_parser::ast::Statement::{CreateSubsource, CreateTableFromSource};
+ match stmt {
+ CreateTableFromSource(stmt) => {
+ let source_id = item_id(stmt.source)?;
+ let external_reference = stmt
+ .external_reference
+ .ok_or("missing external reference on CREATE TABLE FROM SOURCE")?
+ .0
+ .into_iter()
+ .map(|ident| ident.into_string())
+ .collect::>();
+
+ let envelope_type = stmt.envelope.as_ref().map(envelope_name);
+ let (key_format, value_format) = match &stmt.format {
+ Some(spec) => key_value_formats(spec),
+ None => (None, None),
+ };
+
+ Ok(json!({
+ "source_id": source_id,
+ "external_reference": external_reference,
+ "envelope_type": envelope_type,
+ "key_format": key_format,
+ "value_format": value_format,
+ }))
+ }
+ CreateSubsource(stmt) => {
+ // Progress subsources track ingestion progress and are not
+ // source tables. They have no external reference.
+ let is_progress = stmt
+ .with_options
+ .iter()
+ .any(|o| matches!(o.name, CreateSubsourceOptionName::Progress));
+ if is_progress {
+ return Ok(serde_json::Value::Null);
+ }
+
+ let source_id = stmt
+ .of_source
+ .ok_or("non-progress CREATE SUBSOURCE without OF SOURCE")
+ .and_then(item_id)?;
+
+ let external_reference = stmt
+ .with_options
+ .into_iter()
+ .find(|o| matches!(o.name, CreateSubsourceOptionName::ExternalReference))
+ .and_then(|o| match o.value {
+ Some(WithOptionValue::UnresolvedItemName(name)) => Some(name),
+ _ => None,
+ })
+ .ok_or("CREATE SUBSOURCE missing EXTERNAL REFERENCE option")?
+ .0
+ .into_iter()
+ .map(|ident| ident.into_string())
+ .collect::>();
+
+ Ok(json!({
+ "source_id": source_id,
+ "external_reference": external_reference,
+ "envelope_type": serde_json::Value::Null,
+ "key_format": serde_json::Value::Null,
+ "value_format": serde_json::Value::Null,
+ }))
+ }
+ _ => Ok(serde_json::Value::Null),
+ }
+}
+
+/// Extracts connection-detail metadata from a catalog `create_sql`.
+///
+/// Returns a per-connection-type object with the fields that the
+/// `mz_kafka_connections`, `mz_ssh_tunnel_connections`, and `mz_aws_connections`
+/// builtin views need. For everything else (other connection types, including
+/// aws-privatelink whose only detail is context-derived, and non-connection
+/// statements) it returns jsonb `null`, so callers filter on `IS NOT NULL` and
+/// gate on the connection type separately (via
+/// `parse_catalog_create_sql(...)->>'connection_type'`, the way `mz_connections`
+/// already does).
+///
+/// The shape per type:
+///
+/// ```json
+/// // kafka
+/// { "brokers": ["host:port", ...], "progress_topic": }
+/// // ssh-tunnel
+/// { "public_key_1": "", "public_key_2": "" }
+/// // aws
+/// {
+/// "auth_kind": "credentials" | "assume-role",
+/// "endpoint": , "region": ,
+/// "access_key_id": , "access_key_id_secret_id": ,
+/// "secret_access_key_secret_id": ,
+/// "session_token": , "session_token_secret_id": ,
+/// "assume_role_arn": , "assume_role_session_name":
+/// }
+/// ```
+///
+/// `progress_topic` is null when the connection does not set an explicit
+/// `PROGRESS TOPIC`. The default (`_materialize-progress--`) is
+/// reconstructed by the view, not here, because it needs the environment id and
+/// the connection's own id. Values derived only from environment context
+/// (AWS principal, external id, trust policy, privatelink principal) are also
+/// left to the view. This keeps the helper a pure function of the `create_sql`.
+///
+/// For aws, an option is either an inline value or a secret reference. Inline
+/// values land in `access_key_id`/`session_token`; a secret reference lands in
+/// the matching `*_secret_id` as the referenced secret's catalog item id (the
+/// persisted `create_sql` stores resolved references as `[uNNN AS name]`).
+/// `auth_kind` is `assume-role` when `ASSUME ROLE ARN` is present, else
+/// `credentials`, matching the `AwsAuth` variant the removed packer read.
+///
+/// Errors if the statement fails to parse.
+pub fn connection_details(a: &str) -> Result {
+ // The persisted `create_sql` stores an inline broker as a single `BROKER`
+ // option and a broker list as a `BROKERS (...)` sequence. Either way we
+ // only need the addresses, which are present regardless of the tunnel
+ // (direct, SSH, or PrivateLink).
+ fn broker_addresses(values: &[ConnectionOption]) -> Vec {
+ let mut brokers = Vec::new();
+ for opt in values {
+ match (&opt.name, &opt.value) {
+ (ConnectionOptionName::Broker, Some(WithOptionValue::ConnectionKafkaBroker(b))) => {
+ brokers.push(b.address.clone());
+ }
+ (ConnectionOptionName::Brokers, Some(WithOptionValue::Sequence(seq))) => {
+ for v in seq {
+ if let WithOptionValue::ConnectionKafkaBroker(b) = v {
+ brokers.push(b.address.clone());
+ }
+ }
+ }
+ _ => {}
+ }
+ }
+ brokers
+ }
+
+ fn string_option(
+ values: &[ConnectionOption],
+ name: ConnectionOptionName,
+ ) -> Option {
+ values
+ .iter()
+ .find(|o| o.name == name)
+ .and_then(|o| o.value.as_ref())
+ .and_then(option_string)
+ }
+
+ // The catalog id of the secret a `SECRET ...` option references. Resolved
+ // references persist as `RawItemName::Id`, so an unresolved name yields
+ // None (the same treatment [`item_id`] gives item names).
+ fn secret_id_option>(
+ values: &[ConnectionOption],
+ name: ConnectionOptionName,
+ ) -> Option {
+ values.iter().find_map(|o| match &o.value {
+ Some(WithOptionValue::Secret(RawItemName::Id(id, _, _))) if o.name == name => {
+ Some(id.clone())
+ }
+ _ => None,
+ })
+ }
+
+ let stmt = parse_single_statement(a)?;
+
+ use mz_sql_parser::ast::Statement::CreateConnection;
+ let stmt = match stmt {
+ CreateConnection(stmt) => stmt,
+ _ => return Ok(serde_json::Value::Null),
+ };
+
+ match stmt.connection_type {
+ CreateConnectionType::Kafka => Ok(json!({
+ "brokers": broker_addresses(&stmt.values),
+ "progress_topic": string_option(&stmt.values, ConnectionOptionName::ProgressTopic),
+ })),
+ CreateConnectionType::Ssh => Ok(json!({
+ "public_key_1": string_option(&stmt.values, ConnectionOptionName::PublicKey1),
+ "public_key_2": string_option(&stmt.values, ConnectionOptionName::PublicKey2),
+ })),
+ CreateConnectionType::Aws => {
+ let assume_role_arn = string_option(&stmt.values, ConnectionOptionName::AssumeRoleArn);
+ let auth_kind = if assume_role_arn.is_some() {
+ "assume-role"
+ } else {
+ "credentials"
+ };
+ Ok(json!({
+ "auth_kind": auth_kind,
+ // Planning coerces an empty ENDPOINT to None (see
+ // `src/sql/src/plan/statement/ddl/connection.rs`), so the
+ // removed packer wrote NULL for `ENDPOINT = ''`. Match that.
+ "endpoint": string_option(&stmt.values, ConnectionOptionName::Endpoint)
+ .filter(|s| !s.is_empty()),
+ "region": string_option(&stmt.values, ConnectionOptionName::Region),
+ "access_key_id": string_option(&stmt.values, ConnectionOptionName::AccessKeyId),
+ "access_key_id_secret_id":
+ secret_id_option(&stmt.values, ConnectionOptionName::AccessKeyId),
+ "secret_access_key_secret_id":
+ secret_id_option(&stmt.values, ConnectionOptionName::SecretAccessKey),
+ "session_token": string_option(&stmt.values, ConnectionOptionName::SessionToken),
+ "session_token_secret_id":
+ secret_id_option(&stmt.values, ConnectionOptionName::SessionToken),
+ "assume_role_arn": assume_role_arn,
+ "assume_role_session_name":
+ string_option(&stmt.values, ConnectionOptionName::AssumeRoleSessionName),
+ }))
+ }
+ _ => Ok(serde_json::Value::Null),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use prost::Message as _;
+ use serde_json::json;
+
+ /// Encode the two proto fields our decoder cares about, using the same
+ /// tag numbering as the canonical proto.
+ fn encode_pg_details(slot: &str, timeline_id: Option) -> String {
+ let details = super::PostgresPublicationDetailsSubset {
+ slot: slot.to_string(),
+ timeline_id,
+ };
+ hex::encode(details.encode_to_vec())
+ }
+
+ fn pg_source_sql(details_hex: &str) -> String {
+ format!(
+ "CREATE SOURCE \"materialize\".\"public\".\"pg_src\" \
+ IN CLUSTER [u42] \
+ FROM POSTGRES CONNECTION [u10 AS \"materialize\".\"public\".\"pg_conn\"] \
+ (DETAILS = '{details_hex}', PUBLICATION = 'mz_source') \
+ FOR ALL TABLES"
+ )
+ }
+
+ fn kafka_source_sql(with_prefix: bool) -> String {
+ let prefix_opt = if with_prefix {
+ ", GROUP ID PREFIX 'my-prefix-'"
+ } else {
+ ""
+ };
+ format!(
+ "CREATE SOURCE \"materialize\".\"public\".\"k_src\" \
+ IN CLUSTER [u42] \
+ FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \
+ (TOPIC 'test'{prefix_opt}) FORMAT TEXT"
+ )
+ }
+
+ // --- parse_postgres_source_details ---------------------------------------
+
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decContextDefault` on OS `linux`
+ fn pg_happy_path_with_timeline() {
+ let hex = encode_pg_details("materialize_abc", Some(42));
+ let sql = pg_source_sql(&hex);
+ let out = super::postgres_source_details(&sql).expect("ok");
+ assert_eq!(out, json!({ "slot": "materialize_abc", "timeline_id": 42 }));
+ }
+
+ #[mz_ore::test]
+ fn pg_happy_path_null_timeline() {
+ // Pre-2024 sources have no timeline_id field. The decoder must
+ // surface that as JSON null, not error.
+ let hex = encode_pg_details("materialize_legacy", None);
+ let sql = pg_source_sql(&hex);
+ let out = super::postgres_source_details(&sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({ "slot": "materialize_legacy", "timeline_id": null }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn pg_non_postgres_source_returns_null_jsonb() {
+ let sql = "CREATE SOURCE \"materialize\".\"public\".\"lg\" \
+ IN CLUSTER [u42] FROM LOAD GENERATOR COUNTER";
+ let out = super::postgres_source_details(sql).expect("ok");
+ assert_eq!(out, serde_json::Value::Null);
+ }
+
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn pg_non_create_source_returns_null_jsonb() {
+ let sql = "CREATE VIEW v AS SELECT 1";
+ let out = super::postgres_source_details(sql).expect("ok");
+ assert_eq!(out, serde_json::Value::Null);
+ }
+
+ #[mz_ore::test]
+ fn pg_missing_details_option_errors() {
+ let sql = "CREATE SOURCE \"materialize\".\"public\".\"pg_src\" \
+ IN CLUSTER [u42] \
+ FROM POSTGRES CONNECTION [u10 AS \"materialize\".\"public\".\"pg_conn\"] \
+ (PUBLICATION = 'mz_source') FOR ALL TABLES";
+ let err = super::postgres_source_details(sql).unwrap_err();
+ assert!(
+ err.contains("missing DETAILS"),
+ "wrong error message: {err}"
+ );
+ }
+
+ #[mz_ore::test]
+ fn pg_malformed_hex_errors() {
+ let sql = pg_source_sql("not-hex!!");
+ let err = super::postgres_source_details(&sql).unwrap_err();
+ assert!(err.contains("valid hex"), "wrong error message: {err}");
+ }
+
+ #[mz_ore::test]
+ fn pg_malformed_proto_errors() {
+ // Valid hex, garbage bytes. Prost decoding fails on unexpected wire
+ // format.
+ let sql = pg_source_sql("ffff");
+ let err = super::postgres_source_details(&sql).unwrap_err();
+ assert!(
+ err.contains("publication-details proto"),
+ "wrong error message: {err}"
+ );
+ }
+
+ // --- parse_kafka_source_details ------------------------------------------
+
+ #[mz_ore::test]
+ fn kafka_happy_path_with_prefix() {
+ let sql = kafka_source_sql(true);
+ let out = super::kafka_source_details(&sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "topic": "test",
+ "group_id_prefix": "my-prefix-",
+ "connection_id": "u11",
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn kafka_happy_path_without_prefix() {
+ let sql = kafka_source_sql(false);
+ let out = super::kafka_source_details(&sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "topic": "test",
+ "group_id_prefix": null,
+ "connection_id": "u11",
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn kafka_non_kafka_source_returns_null_jsonb() {
+ let sql = "CREATE SOURCE \"materialize\".\"public\".\"lg\" \
+ IN CLUSTER [u42] FROM LOAD GENERATOR COUNTER";
+ let out = super::kafka_source_details(sql).expect("ok");
+ assert_eq!(out, serde_json::Value::Null);
+ }
+
+ #[mz_ore::test]
+ fn kafka_missing_topic_errors() {
+ let sql = "CREATE SOURCE \"materialize\".\"public\".\"k_src\" \
+ IN CLUSTER [u42] \
+ FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \
+ FORMAT TEXT";
+ let err = super::kafka_source_details(sql).unwrap_err();
+ assert!(err.contains("missing TOPIC"), "wrong error message: {err}");
+ }
+
+ #[mz_ore::test]
+ fn kafka_unquoted_topic_and_prefix() {
+ // Planning accepts a bare identifier for TOPIC / GROUP ID PREFIX, and it
+ // persists unquoted in create_sql. Matching only quoted strings would
+ // drop TOPIC and error the whole mz_kafka_source_tables view.
+ let sql = "CREATE SOURCE \"materialize\".\"public\".\"k_src\" \
+ IN CLUSTER [u42] \
+ FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \
+ (TOPIC = my_topic, GROUP ID PREFIX = my_prefix) FORMAT TEXT";
+ let out = super::kafka_source_details(sql).expect("ok");
+ assert_eq!(out["topic"], json!("my_topic"));
+ assert_eq!(out["group_id_prefix"], json!("my_prefix"));
+ }
+
+ #[mz_ore::test]
+ fn kafka_unresolved_connection_errors() {
+ // A bare-name connection reference never happens after purification,
+ // but the decoder must reject it explicitly rather than silently
+ // dropping the connection_id.
+ let sql = "CREATE SOURCE \"materialize\".\"public\".\"k_src\" \
+ IN CLUSTER [u42] \
+ FROM KAFKA CONNECTION k_conn (TOPIC 'test') FORMAT TEXT";
+ let err = super::kafka_source_details(sql).unwrap_err();
+ assert!(
+ err.contains("unresolved item name"),
+ "wrong error message: {err}"
+ );
+ }
+
+ // --- parse_source_export_details -----------------------------------------
+
+ fn table_from_source_sql(reference: &str, suffix: &str) -> String {
+ format!(
+ "CREATE TABLE \"materialize\".\"public\".\"tbl\" \
+ FROM SOURCE [u1 AS \"materialize\".\"public\".\"src\"] \
+ (REFERENCE = {reference}){suffix}"
+ )
+ }
+
+ #[mz_ore::test]
+ fn export_table_postgres_style_no_format() {
+ // Postgres/mysql/sql-server tables carry a multi-part external
+ // reference and no format or envelope.
+ let sql = table_from_source_sql("\"db\".\"public\".\"t\"", "");
+ let out = super::source_export_details(&sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "source_id": "u1",
+ "external_reference": ["db", "public", "t"],
+ "envelope_type": null,
+ "key_format": null,
+ "value_format": null,
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn export_table_kafka_bare_value_only() {
+ // A bare non-registry FORMAT is value-only: no key format.
+ let sql = table_from_source_sql("\"topic\"", " FORMAT TEXT ENVELOPE NONE");
+ let out = super::source_export_details(&sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "source_id": "u1",
+ "external_reference": ["topic"],
+ "envelope_type": "none",
+ "key_format": null,
+ "value_format": "text",
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn export_table_kafka_omitted_envelope_is_null() {
+ // Omitting ENVELOPE persists as absent in create_sql, so this
+ // source-type-agnostic helper reports null. The mz_kafka_source_tables
+ // view is responsible for defaulting kafka's null envelope to 'none'.
+ let sql = table_from_source_sql("\"topic\"", " FORMAT TEXT");
+ let out = super::source_export_details(&sql).expect("ok");
+ assert_eq!(out["envelope_type"], serde_json::Value::Null);
+ }
+
+ #[mz_ore::test]
+ fn export_table_kafka_key_value_format() {
+ let sql = table_from_source_sql(
+ "\"topic\"",
+ " KEY FORMAT TEXT VALUE FORMAT TEXT ENVELOPE NONE",
+ );
+ let out = super::source_export_details(&sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "source_id": "u1",
+ "external_reference": ["topic"],
+ "envelope_type": "none",
+ "key_format": "text",
+ "value_format": "text",
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn export_table_kafka_bare_avro_seed_with_key() {
+ // A bare Avro CSR format whose seed carries a key schema resolves to
+ // an encoding with a key, so key_format mirrors value_format. This is
+ // the upsert/debezium path.
+ let sql = table_from_source_sql(
+ "\"topic\"",
+ " FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY \
+ CONNECTION [u5 AS \"materialize\".\"public\".\"csr\"] \
+ SEED KEY SCHEMA 'k' VALUE SCHEMA 'v' ENVELOPE UPSERT",
+ );
+ let out = super::source_export_details(&sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "source_id": "u1",
+ "external_reference": ["topic"],
+ "envelope_type": "upsert",
+ "key_format": "avro",
+ "value_format": "avro",
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn export_table_kafka_bare_avro_seed_without_key() {
+ // A bare Avro CSR seed with only a value schema is value-only.
+ let sql = table_from_source_sql(
+ "\"topic\"",
+ " FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY \
+ CONNECTION [u5 AS \"materialize\".\"public\".\"csr\"] \
+ SEED VALUE SCHEMA 'v' ENVELOPE NONE",
+ );
+ let out = super::source_export_details(&sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "source_id": "u1",
+ "external_reference": ["topic"],
+ "envelope_type": "none",
+ "key_format": null,
+ "value_format": "avro",
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn export_subsource_non_progress() {
+ // Old-syntax subsource: external reference lives in a WITH option, and
+ // there is never a format or envelope.
+ let sql = "CREATE SUBSOURCE \"materialize\".\"public\".\"sub\" (id int4) \
+ OF SOURCE [u1 AS \"materialize\".\"public\".\"src\"] \
+ WITH (EXTERNAL REFERENCE = \"db\".\"public\".\"t\")";
+ let out = super::source_export_details(sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "source_id": "u1",
+ "external_reference": ["db", "public", "t"],
+ "envelope_type": null,
+ "key_format": null,
+ "value_format": null,
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn export_progress_subsource_returns_null_jsonb() {
+ let sql = "CREATE SUBSOURCE \"materialize\".\"public\".\"progress\" (id int4) \
+ WITH (PROGRESS)";
+ let out = super::source_export_details(sql).expect("ok");
+ assert_eq!(out, serde_json::Value::Null);
+ }
+
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn export_non_source_export_returns_null_jsonb() {
+ let sql = "CREATE VIEW v AS SELECT 1";
+ let out = super::source_export_details(sql).expect("ok");
+ assert_eq!(out, serde_json::Value::Null);
+ }
+
+ #[mz_ore::test]
+ fn export_unresolved_source_name_errors() {
+ let sql = "CREATE TABLE \"materialize\".\"public\".\"tbl\" \
+ FROM SOURCE src (REFERENCE = \"topic\")";
+ let err = super::source_export_details(sql).unwrap_err();
+ assert!(
+ err.contains("unresolved item name"),
+ "wrong error message: {err}"
+ );
+ }
+
+ // --- parse_connection_details --------------------------------------------
+
+ #[mz_ore::test]
+ fn connection_kafka_single_broker_default_progress() {
+ // No explicit PROGRESS TOPIC: the helper leaves it null and the view
+ // reconstructs the default.
+ let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \
+ (BROKER = 'localhost:9092', SECURITY PROTOCOL = plaintext)";
+ let out = super::connection_details(sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "brokers": ["localhost:9092"],
+ "progress_topic": null,
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn connection_kafka_explicit_progress_topic() {
+ let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \
+ (BROKER = 'localhost:9092', PROGRESS TOPIC = 'override', \
+ SECURITY PROTOCOL = plaintext)";
+ let out = super::connection_details(sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "brokers": ["localhost:9092"],
+ "progress_topic": "override",
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn connection_kafka_broker_list() {
+ let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \
+ (BROKERS ('b1:9092', 'b2:9092'), SECURITY PROTOCOL = plaintext)";
+ let out = super::connection_details(sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "brokers": ["b1:9092", "b2:9092"],
+ "progress_topic": null,
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn connection_ssh_public_keys() {
+ let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO SSH TUNNEL \
+ (HOST = 'ssh.example.com', PORT = 22, USER = 'mz', \
+ PUBLIC KEY 1 = 'ssh-ed25519 AAAA', PUBLIC KEY 2 = 'ssh-ed25519 BBBB')";
+ let out = super::connection_details(sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "public_key_1": "ssh-ed25519 AAAA",
+ "public_key_2": "ssh-ed25519 BBBB",
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn connection_aws_credentials_inline_key() {
+ // Inline ACCESS KEY ID, secret SECRET ACCESS KEY. Assume-role columns
+ // stay null and auth_kind is credentials.
+ let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \
+ (ACCESS KEY ID = 'AKIAEXAMPLE', \
+ SECRET ACCESS KEY = SECRET [u1 AS \"materialize\".\"public\".\"sk\"])";
+ let out = super::connection_details(sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "auth_kind": "credentials",
+ "endpoint": null,
+ "region": null,
+ "access_key_id": "AKIAEXAMPLE",
+ "access_key_id_secret_id": null,
+ "secret_access_key_secret_id": "u1",
+ "session_token": null,
+ "session_token_secret_id": null,
+ "assume_role_arn": null,
+ "assume_role_session_name": null,
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn connection_aws_credentials_secret_key_and_session_token() {
+ // Every credential provided as a secret reference lands in the matching
+ // *_secret_id column as the referenced secret's catalog id.
+ let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \
+ (ENDPOINT = 'http://localhost', REGION = 'us-east-1', \
+ ACCESS KEY ID = SECRET [u1 AS \"materialize\".\"public\".\"ak\"], \
+ SECRET ACCESS KEY = SECRET [u2 AS \"materialize\".\"public\".\"sk\"], \
+ SESSION TOKEN = SECRET [u3 AS \"materialize\".\"public\".\"st\"])";
+ let out = super::connection_details(sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "auth_kind": "credentials",
+ "endpoint": "http://localhost",
+ "region": "us-east-1",
+ "access_key_id": null,
+ "access_key_id_secret_id": "u1",
+ "secret_access_key_secret_id": "u2",
+ "session_token": null,
+ "session_token_secret_id": "u3",
+ "assume_role_arn": null,
+ "assume_role_session_name": null,
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn connection_aws_assume_role() {
+ // Assume-role sets auth_kind and the assume-role columns; credential
+ // columns stay null.
+ let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \
+ (ASSUME ROLE ARN 'arn:aws:iam::123:role/mz', \
+ ASSUME ROLE SESSION NAME 'sess')";
+ let out = super::connection_details(sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "auth_kind": "assume-role",
+ "endpoint": null,
+ "region": null,
+ "access_key_id": null,
+ "access_key_id_secret_id": null,
+ "secret_access_key_secret_id": null,
+ "session_token": null,
+ "session_token_secret_id": null,
+ "assume_role_arn": "arn:aws:iam::123:role/mz",
+ "assume_role_session_name": "sess",
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn connection_kafka_unquoted_progress_topic() {
+ // A bare identifier PROGRESS TOPIC persists unquoted in create_sql. The
+ // helper must surface it, else the view substitutes the default topic.
+ let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \
+ (BROKER = 'localhost:9092', PROGRESS TOPIC = my_topic, \
+ SECURITY PROTOCOL = plaintext)";
+ let out = super::connection_details(sql).expect("ok");
+ assert_eq!(out["progress_topic"], json!("my_topic"));
+ }
+
+ #[mz_ore::test]
+ fn connection_aws_unquoted_option_values() {
+ // Planning accepts bare identifiers for these options and persists them
+ // unquoted. The helper must surface them, not fall back to NULL.
+ let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \
+ (ENDPOINT = localhost, REGION = useast1, \
+ ASSUME ROLE ARN 'arn:aws:iam::123:role/mz', \
+ ASSUME ROLE SESSION NAME = mysession)";
+ let out = super::connection_details(sql).expect("ok");
+ assert_eq!(out["endpoint"], json!("localhost"));
+ assert_eq!(out["region"], json!("useast1"));
+ assert_eq!(out["assume_role_session_name"], json!("mysession"));
+ }
+
+ #[mz_ore::test]
+ fn connection_aws_empty_endpoint_is_null() {
+ // Planning coerces ENDPOINT = '' to None, so the packer wrote NULL. The
+ // view must match, not report an empty string.
+ let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \
+ (ENDPOINT = '', ASSUME ROLE ARN 'arn:aws:iam::123:role/mz')";
+ let out = super::connection_details(sql).expect("ok");
+ assert_eq!(out["endpoint"], serde_json::Value::Null);
+ }
+
+ #[mz_ore::test]
+ fn connection_other_type_returns_null_jsonb() {
+ // A connection type without a detail view (postgres) yields null.
+ let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO POSTGRES \
+ (HOST = 'db', DATABASE = 'postgres', USER = 'mz')";
+ let out = super::connection_details(sql).expect("ok");
+ assert_eq!(out, serde_json::Value::Null);
+ }
+
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn connection_non_connection_returns_null_jsonb() {
+ let sql = "CREATE VIEW v AS SELECT 1";
+ let out = super::connection_details(sql).expect("ok");
+ assert_eq!(out, serde_json::Value::Null);
+ }
+
+ // --- parse_catalog_create_sql envelope_type ------------------------------
+
+ #[mz_ore::test]
+ fn catalog_kafka_old_syntax_omitted_envelope_defaults_none() {
+ // An old-syntax kafka source (carries EXPOSE PROGRESS AS) ingests into
+ // its own relation, so an omitted ENVELOPE means the default NONE, which
+ // the pre-MV packer reported as 'none'.
+ let sql = "CREATE SOURCE \"materialize\".\"public\".\"k\" \
+ IN CLUSTER [u42] \
+ FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \
+ (TOPIC 'test') FORMAT TEXT \
+ EXPOSE PROGRESS AS [u12 AS \"materialize\".\"public\".\"k_progress\"]";
+ let out = super::item_details(sql).expect("ok");
+ assert_eq!(out["envelope_type"], json!("none"));
+ }
+
+ #[mz_ore::test]
+ fn catalog_kafka_old_syntax_explicit_envelope() {
+ let sql = "CREATE SOURCE \"materialize\".\"public\".\"k\" \
+ IN CLUSTER [u42] \
+ FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \
+ (TOPIC 'test') FORMAT BYTES ENVELOPE UPSERT \
+ EXPOSE PROGRESS AS [u12 AS \"materialize\".\"public\".\"k_progress\"]";
+ let out = super::item_details(sql).expect("ok");
+ assert_eq!(out["envelope_type"], json!("upsert"));
+ }
+
+ #[mz_ore::test]
+ fn catalog_kafka_new_syntax_source_omits_envelope_type() {
+ // A new-syntax kafka source has no progress subsource (no EXPOSE PROGRESS
+ // AS). It ingests nothing itself. Envelopes live on the per-table exports,
+ // so its own envelope_type stays absent (SQL NULL).
+ let sql = "CREATE SOURCE \"materialize\".\"public\".\"k\" \
+ IN CLUSTER [u42] \
+ FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"]";
+ let out = super::item_details(sql).expect("ok");
+ assert_eq!(out.get("envelope_type"), None);
+ }
+
+ #[mz_ore::test]
+ fn catalog_non_kafka_source_omits_envelope_type() {
+ // Non-kafka sources carry no envelope, so envelope_type stays absent
+ // (SQL NULL), not 'none'.
+ let sql = "CREATE SOURCE \"materialize\".\"public\".\"lg\" \
+ IN CLUSTER [u42] FROM LOAD GENERATOR COUNTER";
+ let out = super::item_details(sql).expect("ok");
+ assert_eq!(out.get("envelope_type"), None);
+ }
+
+ // --- parse_catalog_create_sql, CreateSink arm ----------------------------
+
+ const AVRO_FORMAT: &str = "FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION \
+ [u12 AS \"materialize\".\"public\".\"csr_conn\"]";
+
+ /// A persisted kafka-sink `create_sql`: resolved names, and a `TOPIC` that
+ /// planning guarantees.
+ fn kafka_sink_sql(key: Option<&str>, format: &str, envelope: &str) -> String {
+ let key_clause = key.map(|k| format!(" KEY ({k})")).unwrap_or_default();
+ format!(
+ "CREATE SINK \"materialize\".\"public\".\"snk\" \
+ IN CLUSTER [u42] \
+ FROM [u1 AS \"materialize\".\"public\".\"t\"] \
+ INTO KAFKA CONNECTION [u10 AS \"materialize\".\"public\".\"k_conn\"] \
+ (TOPIC 'sink-topic'){key_clause} {format} ENVELOPE {envelope}"
+ )
+ }
+
+ fn iceberg_sink_sql(mode: &str) -> String {
+ format!(
+ "CREATE SINK \"materialize\".\"public\".\"ice\" \
+ IN CLUSTER [u42] \
+ FROM [u1 AS \"materialize\".\"public\".\"t\"] \
+ INTO ICEBERG CATALOG CONNECTION [u20 AS \"materialize\".\"public\".\"cat_conn\"] \
+ (NAMESPACE 'ns', TABLE 'tbl') \
+ USING AWS CONNECTION [u21 AS \"materialize\".\"public\".\"aws_conn\"] \
+ MODE {mode}"
+ )
+ }
+
+ #[mz_ore::test]
+ fn sink_kafka_bare_format_without_key() {
+ let sql = kafka_sink_sql(None, "FORMAT JSON", "DEBEZIUM");
+ let out = super::item_details(&sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "type": "sink",
+ "sink_type": "kafka",
+ "cluster_id": "u42",
+ "connection_id": "u10",
+ "topic": "sink-topic",
+ "envelope_type": "debezium",
+ "format": "json",
+ "value_format": "json",
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn sink_kafka_bare_format_with_key_derives_key_format() {
+ // A bare format applies to the key too once the sink has a KEY, which
+ // is what makes the deprecated `format` column collapse to `avro`.
+ let sql = kafka_sink_sql(Some("a"), AVRO_FORMAT, "UPSERT");
+ let out = super::item_details(&sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "type": "sink",
+ "sink_type": "kafka",
+ "cluster_id": "u42",
+ "connection_id": "u10",
+ "topic": "sink-topic",
+ "envelope_type": "upsert",
+ "format": "avro",
+ "key_format": "avro",
+ "value_format": "avro",
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn sink_kafka_bare_text_format_with_key_does_not_collapse() {
+ // Only avro/avro and json/json collapse, so a keyed text sink reports
+ // the composite form even though both halves are `text`.
+ let sql = kafka_sink_sql(Some("a"), "FORMAT TEXT", "UPSERT");
+ let out = super::item_details(&sql).expect("ok");
+ assert_eq!(out["format"], json!("key-text-value-text"));
+ assert_eq!(out["key_format"], json!("text"));
+ assert_eq!(out["value_format"], json!("text"));
+ }
+
+ #[mz_ore::test]
+ fn sink_kafka_key_value_json_collapses() {
+ let sql = kafka_sink_sql(Some("a"), "KEY FORMAT JSON VALUE FORMAT JSON", "UPSERT");
+ let out = super::item_details(&sql).expect("ok");
+ assert_eq!(out["format"], json!("json"));
+ assert_eq!(out["key_format"], json!("json"));
+ assert_eq!(out["value_format"], json!("json"));
+ }
+
+ #[mz_ore::test]
+ fn sink_kafka_key_value_mixed_is_composite() {
+ let sql = kafka_sink_sql(Some("a"), "KEY FORMAT TEXT VALUE FORMAT BYTES", "UPSERT");
+ let out = super::item_details(&sql).expect("ok");
+ assert_eq!(out["format"], json!("key-text-value-bytes"));
+ assert_eq!(out["key_format"], json!("text"));
+ assert_eq!(out["value_format"], json!("bytes"));
+ }
+
+ #[mz_ore::test]
+ fn sink_kafka_key_format_without_key_is_dropped() {
+ // `kafka_sink_builder` ignores the key half of the format spec when the
+ // sink has no KEY, so neither `key_format` nor the composite `format`
+ // may reflect it.
+ let sql = kafka_sink_sql(None, "KEY FORMAT JSON VALUE FORMAT TEXT", "DEBEZIUM");
+ let out = super::item_details(&sql).expect("ok");
+ assert_eq!(out["format"], json!("text"));
+ assert_eq!(out["key_format"], serde_json::Value::Null);
+ assert_eq!(out["value_format"], json!("text"));
+ }
+
+ #[mz_ore::test]
+ fn sink_kafka_missing_topic_errors() {
+ let sql = "CREATE SINK \"materialize\".\"public\".\"snk\" \
+ IN CLUSTER [u42] \
+ FROM [u1 AS \"materialize\".\"public\".\"t\"] \
+ INTO KAFKA CONNECTION [u10 AS \"materialize\".\"public\".\"k_conn\"] \
+ FORMAT JSON ENVELOPE DEBEZIUM";
+ let err = super::item_details(sql).unwrap_err();
+ assert!(err.contains("missing TOPIC"), "wrong error message: {err}");
+ }
+
+ #[mz_ore::test]
+ fn sink_iceberg_upsert_mode() {
+ let sql = iceberg_sink_sql("UPSERT");
+ let out = super::item_details(&sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "type": "sink",
+ "sink_type": "iceberg",
+ "cluster_id": "u42",
+ // The catalog connection, never the AWS connection (u21).
+ "connection_id": "u20",
+ "namespace": "ns",
+ "table": "tbl",
+ "envelope_type": "upsert",
+ }),
+ );
+ }
+
+ #[mz_ore::test]
+ fn sink_iceberg_append_mode() {
+ let sql = iceberg_sink_sql("APPEND");
+ let out = super::item_details(&sql).expect("ok");
+ // `append` is reachable only through an iceberg sink's MODE.
+ assert_eq!(out["envelope_type"], json!("append"));
+ }
+
+ #[mz_ore::test]
+ fn sink_iceberg_missing_table_errors() {
+ let sql = "CREATE SINK \"materialize\".\"public\".\"ice\" \
+ IN CLUSTER [u42] \
+ FROM [u1 AS \"materialize\".\"public\".\"t\"] \
+ INTO ICEBERG CATALOG CONNECTION [u20 AS \"materialize\".\"public\".\"cat_conn\"] \
+ (NAMESPACE 'ns') MODE UPSERT";
+ let err = super::item_details(sql).unwrap_err();
+ assert!(err.contains("missing TABLE"), "wrong error message: {err}");
+ }
+
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn sink_arm_leaves_other_item_types_alone() {
+ let sql = "CREATE VIEW \"materialize\".\"public\".\"v\" AS SELECT 1";
+ let out = super::item_details(sql).expect("ok");
+ assert_eq!(out, json!({ "type": "view", "definition": "SELECT 1;" }));
+ }
+
+ // --- parse_catalog_create_sql --------------------------------------------
+
+ /// `type` for a `create_sql`, or the error message if parsing failed.
+ fn item_type(sql: &str) -> Result {
+ match super::item_details(sql) {
+ Ok(out) => match out {
+ serde_json::Value::Object(mut m) => match m.remove("type") {
+ Some(serde_json::Value::String(s)) => Ok(s),
+ other => panic!("no string `type` key: {other:?}"),
+ },
+ other => panic!("not a JSON object: {other:?}"),
+ },
+ Err(msg) => Err(msg),
+ }
+ }
+
+ fn view_sql(query: &str) -> String {
+ format!("CREATE VIEW \"materialize\".\"public\".\"v\" AS {query}")
+ }
+
+ /// `definition` for a `CREATE VIEW` whose query is `query`.
+ fn view_definition(query: &str) -> String {
+ match super::item_details(&view_sql(query)).expect("ok") {
+ serde_json::Value::Object(mut m) => match m.remove("definition") {
+ Some(serde_json::Value::String(s)) => s,
+ other => panic!("no string `definition` key: {other:?}"),
+ },
+ other => panic!("not a JSON object: {other:?}"),
+ }
+ }
+
+ /// `mz_tables` and `mz_views` select rows by
+ /// `parse_catalog_create_sql(...)->>'type'`, and the function runs over
+ /// every `Item` row in the catalog, so a statement kind that changes its
+ /// reported type silently gains or loses rows in those relations. Pin the
+ /// type of every kind the catalog can hold.
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn catalog_item_type_per_statement_kind() {
+ let cases = [
+ (
+ "CREATE TABLE \"materialize\".\"public\".\"t\" (a int4)",
+ "table",
+ ),
+ // A table created from a source, and a webhook table, are both
+ // `table`, so both land in mz_tables.
+ (
+ "CREATE TABLE \"materialize\".\"public\".\"tbl\" \
+ FROM SOURCE [u1 AS \"materialize\".\"public\".\"src\"] \
+ (REFERENCE = \"topic\") FORMAT TEXT",
+ "table",
+ ),
+ (
+ "CREATE TABLE \"materialize\".\"public\".\"wht\" FROM WEBHOOK BODY FORMAT JSON",
+ "table",
+ ),
+ (
+ "CREATE VIEW \"materialize\".\"public\".\"v\" AS SELECT 1",
+ "view",
+ ),
+ (
+ "CREATE MATERIALIZED VIEW \"materialize\".\"public\".\"mv\" \
+ IN CLUSTER [u1] AS SELECT 1",
+ "materialized-view",
+ ),
+ (
+ "CREATE SOURCE \"materialize\".\"public\".\"lg\" \
+ IN CLUSTER [u1] FROM LOAD GENERATOR COUNTER",
+ "source",
+ ),
+ (
+ "CREATE SOURCE \"materialize\".\"public\".\"wh\" \
+ IN CLUSTER [u1] FROM WEBHOOK BODY FORMAT JSON",
+ "source",
+ ),
+ (
+ "CREATE SUBSOURCE \"materialize\".\"public\".\"sub\" (id int4) \
+ OF SOURCE [u1 AS \"materialize\".\"public\".\"src\"]",
+ "subsource",
+ ),
+ (
+ "CREATE SUBSOURCE \"materialize\".\"public\".\"progress\" (id int4) \
+ WITH (PROGRESS)",
+ "subsource",
+ ),
+ (
+ "CREATE SINK \"materialize\".\"public\".\"snk\" IN CLUSTER [u1] \
+ FROM [u1 AS \"materialize\".\"public\".\"t\"] \
+ INTO KAFKA CONNECTION [u2 AS \"materialize\".\"public\".\"c\"] \
+ (TOPIC 'tp') FORMAT JSON ENVELOPE DEBEZIUM",
+ "sink",
+ ),
+ (
+ "CREATE INDEX \"i\" IN CLUSTER [u1] \
+ ON [u1 AS \"materialize\".\"public\".\"t\"] (\"a\")",
+ "index",
+ ),
+ (
+ "CREATE TYPE \"materialize\".\"public\".\"ty\" AS LIST (ELEMENT TYPE = int4)",
+ "type",
+ ),
+ (
+ "CREATE SECRET \"materialize\".\"public\".\"s\" AS 'x'",
+ "secret",
+ ),
+ (
+ "CREATE CONNECTION \"materialize\".\"public\".\"c\" \
+ TO KAFKA (BROKER 'b', SECURITY PROTOCOL PLAINTEXT)",
+ "connection",
+ ),
+ ];
+ for (sql, expected) in cases {
+ assert_eq!(item_type(sql).as_deref(), Ok(expected), "for {sql}");
+ }
+ }
+
+ /// `mz_views.definition` is produced here. It used to be produced by
+ /// `pack_view_update` in the adapter, so the exact rendering is a
+ /// compatibility surface: `pg_views.definition` reads it.
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn catalog_view_definition() {
+ // Identifiers and function names come back fully quoted, literals
+ // untouched, and PostgreSQL's trailing semicolon is appended.
+ assert_eq!(view_definition("SELECT 1"), "SELECT 1;");
+ assert_eq!(
+ view_definition("WITH c AS (SELECT 1 AS a) SELECT a FROM c"),
+ "WITH \"c\" AS (SELECT 1 AS \"a\") SELECT \"a\" FROM \"c\";"
+ );
+ assert_eq!(
+ view_definition("SELECT 1 UNION ALL SELECT 2"),
+ "SELECT 1 UNION ALL SELECT 2;"
+ );
+ assert_eq!(
+ view_definition("SELECT (SELECT max(a) FROM [u1 AS \"materialize\".\"public\".\"t\"])"),
+ "SELECT (SELECT \"max\"(\"a\") FROM [u1 AS \"materialize\".\"public\".\"t\"]);"
+ );
+ // Identifiers needing quotes, an embedded double quote, non-ASCII, an
+ // embedded single quote in a literal, and ORDER BY all survive.
+ assert_eq!(
+ view_definition(
+ "SELECT \"a b\", \"héllo\", \"q\"\"x\" \
+ FROM [u1 AS \"materialize\".\"public\".\"t\"] \
+ WHERE s = 'lit''eral' AND n = 42 ORDER BY 1"
+ ),
+ "SELECT \"a b\", \"héllo\", \"q\"\"x\" \
+ FROM [u1 AS \"materialize\".\"public\".\"t\"] \
+ WHERE \"s\" = 'lit''eral' AND \"n\" = 42 ORDER BY 1;"
+ );
+ }
+
+ /// The rendering must be a fixed point: `pg_views` consumers re-issue
+ /// `definition` as the body of a new view, so a second pass through the
+ /// parser has to produce the identical string. The trailing `;` is part of
+ /// what gets re-parsed.
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn catalog_view_definition_is_idempotent() {
+ for query in [
+ "SELECT 1",
+ "WITH c AS (SELECT 1 AS a) SELECT a FROM c",
+ "SELECT 1 UNION ALL SELECT 2",
+ "SELECT \"a b\", \"q\"\"x\" FROM [u1 AS \"materialize\".\"public\".\"t\"] \
+ WHERE s = 'lit''eral' ORDER BY 1",
+ ] {
+ let once = view_definition(query);
+ assert_eq!(
+ view_definition(&once),
+ once,
+ "not a fixed point for {query}"
+ );
+ }
+ }
+
+ /// `mz_tables.source_id` comes from this key. A table with no source must
+ /// omit it entirely, so the MV's `->>'source_id'` yields SQL NULL.
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn catalog_table_source_id() {
+ let from_source = super::item_details(
+ "CREATE TABLE \"materialize\".\"public\".\"tbl\" \
+ FROM SOURCE [u1 AS \"materialize\".\"public\".\"src\"] \
+ (REFERENCE = \"topic\") FORMAT TEXT",
+ )
+ .expect("ok");
+ assert_eq!(from_source, json!({ "type": "table", "source_id": "u1" }));
+
+ for sql in [
+ "CREATE TABLE \"materialize\".\"public\".\"t\" (a int4)",
+ "CREATE TABLE \"materialize\".\"public\".\"wht\" FROM WEBHOOK BODY FORMAT JSON",
+ ] {
+ assert_eq!(
+ super::item_details(sql).expect("ok"),
+ json!({ "type": "table" }),
+ "for {sql}"
+ );
+ }
+ }
+
+ /// Every error here is fatal to the whole of `mz_tables`/`mz_views`, not to
+ /// one row: the MVs call this function inside their `WHERE` clause, so an
+ /// item the parser rejects makes the relation unreadable for everyone.
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn catalog_create_sql_errors() {
+ assert_eq!(
+ item_type("this is not sql"),
+ Err(
+ "failed to parse create_sql: Expected a keyword at the beginning of a statement, \
+ found identifier \"this\""
+ .to_string()
+ )
+ );
+ assert_eq!(
+ item_type("CREATE TABLE t (a int4); CREATE TABLE u (b int4)"),
+ Err("expected a single statement, found 2".to_string())
+ );
+ // A statement that is not a CREATE of a catalog item, e.g. if a future
+ // change persists something else in an Item record.
+ assert_eq!(
+ item_type("SELECT 1"),
+ Err("not a CREATE item statement".to_string())
+ );
+ // Catalog `create_sql` always names items by id. An unresolved name
+ // means the record was written wrong.
+ assert_eq!(
+ item_type(
+ "CREATE TABLE \"materialize\".\"public\".\"tbl\" \
+ FROM SOURCE src (REFERENCE = \"topic\")"
+ ),
+ Err("unresolved item name".to_string())
+ );
+ }
+
+ // --- parse_catalog_item_references ----------------------------------------
+
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn item_references_view() {
+ let sql = "CREATE VIEW \"materialize\".\"public\".\"v\" AS \
+ SELECT \"pg_catalog\".\"abs\"(\"a\"::[s20 AS \"pg_catalog\".\"int4\"]) \
+ FROM [u1 AS \"materialize\".\"public\".\"t\"]";
+ let out = super::item_references(sql).expect("ok");
+ assert_eq!(
+ out,
+ json!({
+ "ids": ["s20", "u1"],
+ "named_funcs": [{"schema": "pg_catalog", "name": "abs"}],
+ "named_types": [],
+ "named_relations": [],
+ })
+ );
+ }
+
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn item_references_connection() {
+ let sql = "CREATE CONNECTION \"materialize\".\"public\".\"kc\" TO KAFKA (\
+ BROKER 'kafka:9092', \
+ SSH TUNNEL [u5 AS \"materialize\".\"public\".\"ssh\"], \
+ SASL PASSWORD = SECRET [u7 AS \"materialize\".\"public\".\"pw\"])";
+ let out = super::item_references(sql).expect("ok");
+ assert_eq!(out["ids"], json!(["u5", "u7"]));
+ assert_eq!(out["named_funcs"], json!([]));
+ }
+
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn item_references_cte_shadowing() {
+ let sql = "CREATE VIEW \"materialize\".\"public\".\"v\" AS \
+ WITH \"c\" AS (SELECT * FROM [u1 AS \"materialize\".\"public\".\"t\"]) \
+ SELECT * FROM \"c\"";
+ let out = super::item_references(sql).expect("ok");
+ assert_eq!(out["ids"], json!(["u1"]));
+ assert_eq!(out["named_relations"], json!([]));
+ }
+
+ #[mz_ore::test]
+ #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
+ fn item_references_name_based_relation_surfaces() {
+ // Stored SQL always prints relations as id references. A name-based
+ // relation must surface in "named_relations" rather than vanish.
+ let sql = "CREATE VIEW \"materialize\".\"public\".\"v\" AS \
+ SELECT * FROM \"mz_catalog\".\"mz_tables\"";
+ let out = super::item_references(sql).expect("ok");
+ assert_eq!(
+ out["named_relations"],
+ json!([{"schema": "mz_catalog", "name": "mz_tables"}])
+ );
+ }
+
+ #[mz_ore::test]
+ fn item_references_parse_error() {
+ let err = super::item_references("NOT SQL").unwrap_err();
+ assert!(
+ err.contains("failed to parse"),
+ "wrong error message: {err}"
+ );
+ }
+}
diff --git a/src/catalog-decode/src/durable.rs b/src/catalog-decode/src/durable.rs
new file mode 100644
index 0000000000000..6fdd592bbc9a6
--- /dev/null
+++ b/src/catalog-decode/src/durable.rs
@@ -0,0 +1,146 @@
+// Copyright Materialize, Inc. and contributors. All rights reserved.
+//
+// Use of this software is governed by the Business Source License
+// included in the LICENSE file.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0.
+
+//! Decoders for values in their durable catalog JSON encoding, the serde shape
+//! of the `mz-catalog-protos` types.
+
+use mz_repr::Datum;
+use mz_repr::adt::jsonb::JsonbRef;
+use mz_repr::adt::mz_acl_item::{AclMode, MzAclItem};
+use mz_repr::adt::numeric;
+use mz_repr::role_id::RoleId;
+
+/// Converts a JSONB `Datum` into a `u64`.
+fn jsonb_datum_to_u64<'a>(d: Datum<'a>) -> Result {
+ let Datum::Numeric(n) = d else {
+ return Err("expected numeric value".into());
+ };
+
+ let mut cx = numeric::cx_datum();
+ cx.try_into_u64(n.0)
+ .map_err(|_| format!("number out of u64 range: {n}"))
+}
+
+/// Decodes a JSONB object of shape `{"bitflags": }` into an `AclMode`.
+///
+/// Shared decoder for [`privileges`] (which embeds the object as the
+/// `acl_mode` field of each privilege) and [`acl_mode`] (which receives the
+/// object at the top level).
+fn jsonb_datum_to_acl_mode(d: Datum) -> Result {
+ let Datum::Map(dict) = d else {
+ return Err(format!("unexpected acl_mode: {d}"));
+ };
+ let mut bits = None;
+ for (key, val) in dict.iter() {
+ match key {
+ "bitflags" => bits = Some(jsonb_datum_to_u64(val)?),
+ other => return Err(format!("unexpected acl_mode field: {other}")),
+ }
+ }
+ let bits = bits.ok_or_else(|| "missing acl_mode bitflags".to_string())?;
+ AclMode::from_bits(bits).ok_or_else(|| format!("invalid acl_mode bitflags: {bits}"))
+}
+
+/// Converts a JSONB `Datum` into a `RoleId`.
+fn jsonb_datum_to_role_id(d: Datum) -> Result {
+ match d {
+ Datum::String("Public") => Ok(RoleId::Public),
+ Datum::String(other) => Err(format!("unexpected role ID variant: {other}")),
+ Datum::Map(dict) => {
+ let (key, val) = dict.iter().next().ok_or_else(|| "empty".to_string())?;
+ let n = jsonb_datum_to_u64(val)?;
+ match key {
+ "User" => Ok(RoleId::User(n)),
+ "System" => Ok(RoleId::System(n)),
+ "Predefined" => Ok(RoleId::Predefined(n)),
+ other => Err(format!("unexpected role ID variant: {other}")),
+ }
+ }
+ _ => Err("expected string or object".into()),
+ }
+}
+
+/// Converts a catalog JSON-serialized ID value into the appropriate string format.
+///
+/// Supports all of Materialize's various ID types of the form ``.
+pub fn catalog_id(a: JsonbRef<'_>) -> Result {
+ match a.into_datum() {
+ // Unit variant, e.g. "Public"
+ Datum::String(variant) => match variant {
+ "Explain" => Ok("e".to_string()),
+ "Public" => Ok("p".to_string()),
+ other => Err(format!("unexpected ID variant: {other}")),
+ },
+ // Newtype variant, e.g. {"User": 1}
+ Datum::Map(dict) => {
+ let (key, val) = dict.iter().next().ok_or_else(|| "empty".to_string())?;
+ let prefix = match key {
+ "IntrospectionSourceIndex" => "si",
+ "Predefined" => "g",
+ "System" => "s",
+ "Transient" => "t",
+ "User" => "u",
+ other => return Err(format!("unexpected ID variant: {other}")),
+ };
+ let n = jsonb_datum_to_u64(val)?;
+ Ok(format!("{prefix}{n}"))
+ }
+ _ => Err("expected string or object".into()),
+ }
+}
+
+/// Converts a catalog JSON-serialized privilege array into a list of `MzAclItem`s.
+pub fn privileges(a: JsonbRef<'_>) -> Result, String> {
+ let parse_one = |datum| match datum {
+ Datum::Map(dict) => {
+ let mut grantee = None;
+ let mut grantor = None;
+ let mut acl_mode = None;
+ for (key, val) in dict.iter() {
+ match key {
+ "grantee" => {
+ let id = jsonb_datum_to_role_id(val)?;
+ grantee = Some(id);
+ }
+ "grantor" => {
+ let id = jsonb_datum_to_role_id(val)?;
+ grantor = Some(id);
+ }
+ "acl_mode" => {
+ acl_mode = Some(jsonb_datum_to_acl_mode(val)?);
+ }
+ other => return Err(format!("unexpected privilege field: {other}")),
+ }
+ }
+ Ok(MzAclItem {
+ grantee: grantee.ok_or_else(|| format!("missing grantee: {dict:?}"))?,
+ grantor: grantor.ok_or_else(|| "missing grantor in privilege".to_string())?,
+ acl_mode: acl_mode.ok_or_else(|| "missing acl_mode in privilege".to_string())?,
+ })
+ }
+ other => Err(format!("expected object in array, found: {other}")),
+ };
+
+ match a.into_datum() {
+ Datum::List(list) => {
+ let mut result = Vec::new();
+ for item in list.iter() {
+ result.push(parse_one(item)?);
+ }
+ Ok(result)
+ }
+ _ => Err("expected array".to_string()),
+ }
+}
+
+/// Converts a catalog JSON-serialized `AclMode` bitflags object
+/// (e.g. `{"bitflags": 514}`) into an `AclMode`.
+pub fn acl_mode(a: JsonbRef<'_>) -> Result {
+ jsonb_datum_to_acl_mode(a.into_datum())
+}
diff --git a/src/catalog-decode/src/lib.rs b/src/catalog-decode/src/lib.rs
new file mode 100644
index 0000000000000..321a21c7610f9
--- /dev/null
+++ b/src/catalog-decode/src/lib.rs
@@ -0,0 +1,22 @@
+// Copyright Materialize, Inc. and contributors. All rights reserved.
+//
+// Use of this software is governed by the Business Source License
+// included in the LICENSE file.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0.
+
+//! Decoders for durable catalog data.
+//!
+//! The builtin catalog views derive their columns from `mz_catalog_raw`, which
+//! exposes catalog objects in their durable encoding. The decoding logic lives
+//! here, and `mz-expr` exposes each decoder as an `mz_internal.parse_*` scalar
+//! function. It sits below `mz-expr` in the dependency graph, so it cannot use
+//! the crates that own the decoded formats (`mz-catalog-protos`, `mz-sql`,
+//! `mz-storage-types`, `mz-audit-log`) and mirrors their shapes instead.
+//! Tests in those crates and the catalog sqllogictests catch drift.
+
+pub mod audit_log;
+pub mod create_sql;
+pub mod durable;
diff --git a/src/catalog/src/builtin.rs b/src/catalog/src/builtin.rs
index 7d10a3a92531a..bb659d6d8a1fe 100644
--- a/src/catalog/src/builtin.rs
+++ b/src/catalog/src/builtin.rs
@@ -1159,7 +1159,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne
Builtin::Table(&MZ_INDEX_COLUMNS),
Builtin::MaterializedView(&MZ_TABLES),
// mz_sources is generated dynamically below with inlined builtin VALUES.
- Builtin::Table(&MZ_SOURCE_REFERENCES),
+ Builtin::MaterializedView(&MZ_SOURCE_REFERENCES),
Builtin::MaterializedView(&MZ_POSTGRES_SOURCES),
Builtin::MaterializedView(&MZ_POSTGRES_SOURCE_TABLES),
Builtin::MaterializedView(&MZ_MYSQL_SOURCE_TABLES),
@@ -1536,7 +1536,7 @@ pub static BUILTINS_STATIC: LazyLock>> = LazyLock::ne
// all items that follow it in the list.
let insert_pos = builtin_items
.iter()
- .position(|b| matches!(b, Builtin::Table(t) if t.name == "mz_source_references"))
+ .position(|b| b.name() == "mz_source_references")
.expect("mz_source_references must be present in builtin_items");
builtin_items.insert(insert_pos, Builtin::MaterializedView(mz_sources_ref));
}
diff --git a/src/catalog/src/builtin/mz_internal.rs b/src/catalog/src/builtin/mz_internal.rs
index abd0bda3efd28..4b155ca94720a 100644
--- a/src/catalog/src/builtin/mz_internal.rs
+++ b/src/catalog/src/builtin/mz_internal.rs
@@ -3572,39 +3572,72 @@ FROM commented",
}
});
-pub static MZ_SOURCE_REFERENCES: LazyLock = LazyLock::new(|| BuiltinTable {
- name: "mz_source_references",
- schema: MZ_INTERNAL_SCHEMA,
- oid: oid::TABLE_MZ_SOURCE_REFERENCES_OID,
- desc: RelationDesc::builder()
- .with_column("source_id", SqlScalarType::String.nullable(false))
- .with_column("namespace", SqlScalarType::String.nullable(true))
- .with_column("name", SqlScalarType::String.nullable(false))
- .with_column(
- "updated_at",
- SqlScalarType::TimestampTz { precision: None }.nullable(false),
- )
- .with_column(
- "columns",
- SqlScalarType::Array(Box::new(SqlScalarType::String)).nullable(true),
- )
- .finish(),
- column_comments: BTreeMap::new(),
- is_retained_metrics_object: false,
- access: vec![PUBLIC_SELECT],
- ontology: Some(Ontology {
- entity_name: "source_reference",
- description: "External references tracked by sources",
- links: &const {
- [OntologyLink {
- name: "references_source",
- target: "source",
- properties: LinkProperties::fk("source_id", "id", Cardinality::ManyToOne),
- }]
- },
- column_semantic_types: &[("source_id", SemanticType::CatalogItemId)],
- }),
-});
+pub static MZ_SOURCE_REFERENCES: LazyLock =
+ LazyLock::new(|| BuiltinMaterializedView {
+ name: "mz_source_references",
+ schema: MZ_INTERNAL_SCHEMA,
+ oid: oid::MV_MZ_SOURCE_REFERENCES_OID,
+ desc: RelationDesc::builder()
+ .with_column("source_id", SqlScalarType::String.nullable(false))
+ .with_column("namespace", SqlScalarType::String.nullable(true))
+ .with_column("name", SqlScalarType::String.nullable(false))
+ .with_column(
+ "updated_at",
+ SqlScalarType::TimestampTz { precision: None }.nullable(false),
+ )
+ .with_column(
+ "columns",
+ SqlScalarType::Array(Box::new(SqlScalarType::String)).nullable(true),
+ )
+ .finish(),
+ column_comments: BTreeMap::new(),
+ sql: "
+IN CLUSTER mz_catalog_server
+WITH (
+ ASSERT NOT NULL source_id,
+ ASSERT NOT NULL name,
+ ASSERT NOT NULL updated_at
+) AS
+-- `refs` projects away usage of the `data` column before the `columns` subquery.
+-- Without the CTE, each row would need a copy of `data` which contains all source
+-- references, leading to memory growing quadratically.
+WITH refs AS (
+ SELECT
+ mz_internal.parse_catalog_id(data->'key'->'source') AS source_id,
+ -- `updated_at` is an `EpochMillis`, serialized as `{\"millis\": }`
+ to_timestamp(((data->'value'->'updated_at'->>'millis')::double) / 1000.0) AS updated_at,
+ reference
+ FROM
+ mz_internal.mz_catalog_raw,
+ jsonb_array_elements(data->'value'->'references') AS reference
+ WHERE data->>'kind' = 'SourceReferences'
+)
+SELECT
+ source_id,
+ reference->>'namespace' AS namespace,
+ reference->>'name' AS name,
+ updated_at,
+ (
+ SELECT array_agg(c.value ORDER BY c.ord)
+ FROM jsonb_array_elements_text(reference->'columns')
+ WITH ORDINALITY AS c(value, ord)
+ ) AS columns
+FROM refs",
+ is_retained_metrics_object: false,
+ access: vec![PUBLIC_SELECT],
+ ontology: Some(Ontology {
+ entity_name: "source_reference",
+ description: "External references tracked by sources",
+ links: &const {
+ [OntologyLink {
+ name: "references_source",
+ target: "source",
+ properties: LinkProperties::fk("source_id", "id", Cardinality::ManyToOne),
+ }]
+ },
+ column_semantic_types: &[("source_id", SemanticType::CatalogItemId)],
+ }),
+ });
pub static MZ_WEBHOOKS_SOURCES: LazyLock = LazyLock::new(|| BuiltinTable {
name: "mz_webhook_sources",
diff --git a/src/compute-types/tests/snapshots/func_registry_source.json b/src/compute-types/tests/snapshots/func_registry_source.json
index 1a141a9baee86..f5aaf0b008968 100644
--- a/src/compute-types/tests/snapshots/func_registry_source.json
+++ b/src/compute-types/tests/snapshots/func_registry_source.json
@@ -527,16 +527,16 @@
"pad_char": {"display":"padchar","sqlfunc_decl":null,"body_fingerprint":null},
"pad_char[unbounded]": {"display":"padchar","sqlfunc_decl":null,"body_fingerprint":null},
"panic": {"display":"mz_panic","sqlfunc_decl":"#[sqlfunc(sqlname = \"mz_panic\")] fn panic<'a>(a: &'a str) -> String","body_fingerprint":"95469a722cc261a6"},
- "parse_catalog_acl_mode": {"display":"parse_catalog_acl_mode","sqlfunc_decl":"#[sqlfunc] fn parse_catalog_acl_mode<'a>(a: JsonbRef<'a>) -> Result","body_fingerprint":"9f9ba9300791097b"},
- "parse_catalog_audit_log_details": {"display":"parse_catalog_audit_log_details","sqlfunc_decl":"#[sqlfunc] fn parse_catalog_audit_log_details<'a>(a: JsonbRef<'a>) -> Result","body_fingerprint":"4d767508de48e9e0"},
- "parse_catalog_create_sql": {"display":"parse_catalog_create_sql","sqlfunc_decl":"#[sqlfunc] fn parse_catalog_create_sql<'a>(a: &'a str) -> Result","body_fingerprint":"525420f3abbc957f"},
- "parse_catalog_id": {"display":"parse_catalog_id","sqlfunc_decl":"#[sqlfunc] fn parse_catalog_id<'a>(a: JsonbRef<'a>) -> Result","body_fingerprint":"cf39474c1e9e1bb9"},
- "parse_catalog_item_references": {"display":"parse_catalog_item_references","sqlfunc_decl":"#[sqlfunc] fn parse_catalog_item_references<'a>(a: &'a str) -> Result","body_fingerprint":"3296877cab7a9c6b"},
- "parse_catalog_privileges": {"display":"parse_catalog_privileges","sqlfunc_decl":"#[sqlfunc] fn parse_catalog_privileges<'a>(a: JsonbRef<'a>) -> Result, EvalError>","body_fingerprint":"203b2915643066b2"},
- "parse_connection_details": {"display":"parse_connection_details","sqlfunc_decl":"#[sqlfunc] fn parse_connection_details<'a>(a: &'a str) -> Result","body_fingerprint":"4b1754595ee6cc23"},
- "parse_kafka_source_details": {"display":"parse_kafka_source_details","sqlfunc_decl":"#[sqlfunc] fn parse_kafka_source_details<'a>(a: &'a str) -> Result","body_fingerprint":"da33c684761dac6e"},
- "parse_postgres_source_details": {"display":"parse_postgres_source_details","sqlfunc_decl":"#[sqlfunc] fn parse_postgres_source_details<'a>(a: &'a str) -> Result","body_fingerprint":"c61ad0624dcc7138"},
- "parse_source_export_details": {"display":"parse_source_export_details","sqlfunc_decl":"#[sqlfunc] fn parse_source_export_details<'a>(a: &'a str) -> Result","body_fingerprint":"b73b420b90bf9bae"},
+ "parse_catalog_acl_mode": {"display":"parse_catalog_acl_mode","sqlfunc_decl":"#[sqlfunc] fn parse_catalog_acl_mode<'a>(a: JsonbRef<'a>) -> Result","body_fingerprint":"416a57d9f77c1f57"},
+ "parse_catalog_audit_log_details": {"display":"parse_catalog_audit_log_details","sqlfunc_decl":"#[sqlfunc] fn parse_catalog_audit_log_details<'a>(a: JsonbRef<'a>) -> Result","body_fingerprint":"d71918c119d2ecf5"},
+ "parse_catalog_create_sql": {"display":"parse_catalog_create_sql","sqlfunc_decl":"#[sqlfunc] fn parse_catalog_create_sql<'a>(a: &'a str) -> Result","body_fingerprint":"826a24b886478cf7"},
+ "parse_catalog_id": {"display":"parse_catalog_id","sqlfunc_decl":"#[sqlfunc] fn parse_catalog_id<'a>(a: JsonbRef<'a>) -> Result","body_fingerprint":"3df74fa5b95ad2a3"},
+ "parse_catalog_item_references": {"display":"parse_catalog_item_references","sqlfunc_decl":"#[sqlfunc] fn parse_catalog_item_references<'a>(a: &'a str) -> Result","body_fingerprint":"afac69531d801621"},
+ "parse_catalog_privileges": {"display":"parse_catalog_privileges","sqlfunc_decl":"#[sqlfunc] fn parse_catalog_privileges<'a>(a: JsonbRef<'a>) -> Result, EvalError>","body_fingerprint":"0189155db8d73f28"},
+ "parse_connection_details": {"display":"parse_connection_details","sqlfunc_decl":"#[sqlfunc] fn parse_connection_details<'a>(a: &'a str) -> Result","body_fingerprint":"479b90a7cbe943f6"},
+ "parse_kafka_source_details": {"display":"parse_kafka_source_details","sqlfunc_decl":"#[sqlfunc] fn parse_kafka_source_details<'a>(a: &'a str) -> Result","body_fingerprint":"8717ae9a730e89a0"},
+ "parse_postgres_source_details": {"display":"parse_postgres_source_details","sqlfunc_decl":"#[sqlfunc] fn parse_postgres_source_details<'a>(a: &'a str) -> Result","body_fingerprint":"00a79828148946a7"},
+ "parse_source_export_details": {"display":"parse_source_export_details","sqlfunc_decl":"#[sqlfunc] fn parse_source_export_details<'a>(a: &'a str) -> Result","body_fingerprint":"a8e38e11bf5b92ac"},
"pg_column_size": {"display":"pg_column_size","sqlfunc_decl":"#[sqlfunc] fn pg_column_size<'a>(a: Datum<'a>) -> Result, EvalError>","body_fingerprint":"b7ff6ebfef49b70e"},
"pg_size_pretty": {"display":"pg_size_pretty","sqlfunc_decl":"#[sqlfunc(sqlname = \"pg_size_pretty\", preserves_uniqueness = false)] fn pg_size_pretty(mut a: Numeric) -> Result","body_fingerprint":"c6b922cd4760b991"},
"quote_ident": {"display":"quote_ident","sqlfunc_decl":"#[sqlfunc(sqlname = \"quote_ident\", preserves_uniqueness = true)] fn quote_ident<'a>(a: &'a str) -> Result","body_fingerprint":"bbf06e94673c774c"},
diff --git a/src/expr/Cargo.toml b/src/expr/Cargo.toml
index a914d891dc1e1..ba29ae746a882 100644
--- a/src/expr/Cargo.toml
+++ b/src/expr/Cargo.toml
@@ -40,6 +40,7 @@ hmac.workspace = true
itertools.workspace = true
md-5.workspace = true
murmur2.workspace = true
+mz-catalog-decode = { path = "../catalog-decode" }
mz-expr-derive = { path = "../expr-derive", features = ["test"] }
mz-ore = { path = "../ore", features = ["network", "proptest", "id_gen"] }
mz-persist-types = { path = "../persist-types" }
diff --git a/src/expr/src/scalar/func/impls.rs b/src/expr/src/scalar/func/impls.rs
index a1bbe477d3190..ae00788120800 100644
--- a/src/expr/src/scalar/func/impls.rs
+++ b/src/expr/src/scalar/func/impls.rs
@@ -8,10 +8,10 @@
// by the Apache License, Version 2.0.
mod array;
-mod audit_log_details;
mod boolean;
mod byte;
mod case_literal;
+mod catalog;
mod char;
mod date;
mod datum;
@@ -43,10 +43,10 @@ mod uuid;
mod varchar;
pub use crate::scalar::func::impls::array::*;
-pub use crate::scalar::func::impls::audit_log_details::*;
pub use crate::scalar::func::impls::boolean::*;
pub use crate::scalar::func::impls::byte::*;
pub use crate::scalar::func::impls::case_literal::*;
+pub use crate::scalar::func::impls::catalog::*;
pub use crate::scalar::func::impls::char::*;
pub use crate::scalar::func::impls::date::*;
pub use crate::scalar::func::impls::datum::*;
diff --git a/src/expr/src/scalar/func/impls/catalog.rs b/src/expr/src/scalar/func/impls/catalog.rs
new file mode 100644
index 0000000000000..a639394dfe195
--- /dev/null
+++ b/src/expr/src/scalar/func/impls/catalog.rs
@@ -0,0 +1,95 @@
+// Copyright Materialize, Inc. and contributors. All rights reserved.
+//
+// Use of this software is governed by the Business Source License
+// included in the LICENSE file.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0.
+
+//! Scalar functions that decode durable catalog data for the builtin catalog
+//! views. The decoders live in `mz_catalog_decode`, documented there.
+
+use mz_catalog_decode::{audit_log, create_sql, durable};
+use mz_expr_derive::sqlfunc;
+use mz_repr::ArrayRustType;
+use mz_repr::adt::jsonb::{Jsonb, JsonbRef};
+use mz_repr::adt::mz_acl_item::MzAclItem;
+
+use crate::EvalError;
+
+fn catalog_err(e: String) -> EvalError {
+ EvalError::InvalidCatalogJson(e.into())
+}
+
+fn to_jsonb(val: serde_json::Value) -> Jsonb {
+ Jsonb::from_serde_json(val).expect("valid JSONB")
+}
+
+#[sqlfunc]
+fn parse_catalog_id<'a>(a: JsonbRef<'a>) -> Result {
+ durable::catalog_id(a).map_err(catalog_err)
+}
+
+#[sqlfunc]
+fn parse_catalog_privileges<'a>(a: JsonbRef<'a>) -> Result, EvalError> {
+ durable::privileges(a)
+ .map(ArrayRustType)
+ .map_err(catalog_err)
+}
+
+/// Returns the PostgreSQL ACL char-code string (e.g. `"ar"`) of a catalog
+/// `AclMode`.
+#[sqlfunc]
+fn parse_catalog_acl_mode<'a>(a: JsonbRef<'a>) -> Result {
+ durable::acl_mode(a)
+ .map(|mode| mode.to_string())
+ .map_err(catalog_err)
+}
+
+#[sqlfunc]
+fn parse_catalog_audit_log_details<'a>(a: JsonbRef<'a>) -> Result {
+ audit_log::details(a).map_err(catalog_err)
+}
+
+#[sqlfunc]
+fn parse_catalog_create_sql<'a>(a: &'a str) -> Result {
+ create_sql::item_details(a)
+ .map(to_jsonb)
+ .map_err(catalog_err)
+}
+
+#[sqlfunc]
+fn parse_catalog_item_references<'a>(a: &'a str) -> Result {
+ create_sql::item_references(a)
+ .map(to_jsonb)
+ .map_err(catalog_err)
+}
+
+#[sqlfunc]
+fn parse_postgres_source_details<'a>(a: &'a str) -> Result {
+ create_sql::postgres_source_details(a)
+ .map(to_jsonb)
+ .map_err(catalog_err)
+}
+
+#[sqlfunc]
+fn parse_kafka_source_details<'a>(a: &'a str) -> Result {
+ create_sql::kafka_source_details(a)
+ .map(to_jsonb)
+ .map_err(catalog_err)
+}
+
+#[sqlfunc]
+fn parse_source_export_details<'a>(a: &'a str) -> Result {
+ create_sql::source_export_details(a)
+ .map(to_jsonb)
+ .map_err(catalog_err)
+}
+
+#[sqlfunc]
+fn parse_connection_details<'a>(a: &'a str) -> Result {
+ create_sql::connection_details(a)
+ .map(to_jsonb)
+ .map_err(catalog_err)
+}
diff --git a/src/expr/src/scalar/func/impls/jsonb.rs b/src/expr/src/scalar/func/impls/jsonb.rs
index f21b7109b1b00..92fadbe54f8bd 100644
--- a/src/expr/src/scalar/func/impls/jsonb.rs
+++ b/src/expr/src/scalar/func/impls/jsonb.rs
@@ -7,27 +7,13 @@
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0.
-use std::collections::BTreeMap;
use std::fmt;
use mz_expr_derive::sqlfunc;
use mz_repr::adt::jsonb::{Jsonb, JsonbRef};
-use mz_repr::adt::mz_acl_item::{AclMode, MzAclItem};
use mz_repr::adt::numeric::{self, Numeric, NumericMaxScale};
-use mz_repr::role_id::RoleId;
-use mz_repr::{ArrayRustType, Datum, Row, RowPacker, SqlColumnType, SqlScalarType, strconv};
-use mz_sql_parser::ast::display::AstDisplay;
-use mz_sql_parser::ast::item_refs::collect_item_references;
-use mz_sql_parser::ast::{
- AstInfo, AvroSchema, ConnectionOption, ConnectionOptionName, CreateConnectionType,
- CreateSinkConnection, CreateSubsourceOptionName, Format, FormatSpecifier,
- IcebergSinkConfigOptionName, IcebergSinkMode, KafkaSinkConfigOptionName,
- KafkaSourceConfigOptionName, PgConfigOptionName, ProtobufSchema, RawClusterName, RawItemName,
- SinkEnvelope, SourceEnvelope, SourceErrorPolicy, UnresolvedItemName, Value, WithOptionValue,
-};
-use prost::Message as _;
+use mz_repr::{Datum, Row, RowPacker, SqlColumnType, SqlScalarType, strconv};
use serde::{Deserialize, Serialize};
-use serde_json::json;
use crate::EvalError;
use crate::scalar::func::EagerUnaryFunc;
@@ -243,2199 +229,3 @@ fn jsonb_pretty<'a>(a: JsonbRef<'a>) -> String {
strconv::format_jsonb_pretty(&mut buf, a);
buf
}
-
-/// Converts a JSONB `Datum` into a `u64`.
-fn jsonb_datum_to_u64<'a>(d: Datum<'a>) -> Result {
- let Datum::Numeric(n) = d else {
- return Err("expected numeric value".into());
- };
-
- let mut cx = numeric::cx_datum();
- cx.try_into_u64(n.0)
- .map_err(|_| format!("number out of u64 range: {n}"))
-}
-
-/// Decodes a JSONB object of shape `{"bitflags": }` into an `AclMode`.
-///
-/// Shared decoder for `parse_catalog_privileges` (which embeds the object as
-/// the `acl_mode` field of each privilege) and `parse_catalog_acl_mode` (which
-/// receives the object at the top level).
-fn jsonb_datum_to_acl_mode(d: Datum) -> Result {
- let Datum::Map(dict) = d else {
- return Err(format!("unexpected acl_mode: {d}"));
- };
- let mut bits = None;
- for (key, val) in dict.iter() {
- match key {
- "bitflags" => bits = Some(jsonb_datum_to_u64(val)?),
- other => return Err(format!("unexpected acl_mode field: {other}")),
- }
- }
- let bits = bits.ok_or_else(|| "missing acl_mode bitflags".to_string())?;
- AclMode::from_bits(bits).ok_or_else(|| format!("invalid acl_mode bitflags: {bits}"))
-}
-
-/// Converts a JSONB `Datum` into a `RoleId`.
-fn jsonb_datum_to_role_id(d: Datum) -> Result {
- match d {
- Datum::String("Public") => Ok(RoleId::Public),
- Datum::String(other) => Err(format!("unexpected role ID variant: {other}")),
- Datum::Map(dict) => {
- let (key, val) = dict.iter().next().ok_or_else(|| "empty".to_string())?;
- let n = jsonb_datum_to_u64(val)?;
- match key {
- "User" => Ok(RoleId::User(n)),
- "System" => Ok(RoleId::System(n)),
- "Predefined" => Ok(RoleId::Predefined(n)),
- other => Err(format!("unexpected role ID variant: {other}")),
- }
- }
- _ => Err("expected string or object".into()),
- }
-}
-
-/// Converts a catalog JSON-serialized ID value into the appropriate string format.
-///
-/// Supports all of Materialize's various ID types of the form ``.
-#[sqlfunc]
-fn parse_catalog_id<'a>(a: JsonbRef<'a>) -> Result {
- let parse = || match a.into_datum() {
- // Unit variant, e.g. "Public"
- Datum::String(variant) => match variant {
- "Explain" => Ok("e".to_string()),
- "Public" => Ok("p".to_string()),
- other => Err(format!("unexpected ID variant: {other}")),
- },
- // Newtype variant, e.g. {"User": 1}
- Datum::Map(dict) => {
- let (key, val) = dict.iter().next().ok_or_else(|| "empty".to_string())?;
- let prefix = match key {
- "IntrospectionSourceIndex" => "si",
- "Predefined" => "g",
- "System" => "s",
- "Transient" => "t",
- "User" => "u",
- other => return Err(format!("unexpected ID variant: {other}")),
- };
- let n = jsonb_datum_to_u64(val)?;
- Ok(format!("{prefix}{n}"))
- }
- _ => Err("expected string or object".into()),
- };
-
- parse().map_err(|e| EvalError::InvalidCatalogJson(e.into()))
-}
-
-/// Converts a catalog JSON-serialized privilege array into an `mz_aclitem[]`.
-#[sqlfunc]
-fn parse_catalog_privileges<'a>(a: JsonbRef<'a>) -> Result, EvalError> {
- let parse_one = |datum| match datum {
- Datum::Map(dict) => {
- let mut grantee = None;
- let mut grantor = None;
- let mut acl_mode = None;
- for (key, val) in dict.iter() {
- match key {
- "grantee" => {
- let id = jsonb_datum_to_role_id(val)?;
- grantee = Some(id);
- }
- "grantor" => {
- let id = jsonb_datum_to_role_id(val)?;
- grantor = Some(id);
- }
- "acl_mode" => {
- acl_mode = Some(jsonb_datum_to_acl_mode(val)?);
- }
- other => return Err(format!("unexpected privilege field: {other}")),
- }
- }
- Ok(MzAclItem {
- grantee: grantee.ok_or_else(|| format!("missing grantee: {dict:?}"))?,
- grantor: grantor.ok_or_else(|| "missing grantor in privilege".to_string())?,
- acl_mode: acl_mode.ok_or_else(|| "missing acl_mode in privilege".to_string())?,
- })
- }
- other => Err(format!("expected object in array, found: {other}")),
- };
-
- let parse = || match a.into_datum() {
- Datum::List(list) => {
- let mut result = Vec::new();
- for item in list.iter() {
- result.push(parse_one(item)?);
- }
- Ok(result)
- }
- _ => Err("expected array".to_string()),
- };
-
- parse()
- .map(ArrayRustType)
- .map_err(|e| EvalError::InvalidCatalogJson(e.into()))
-}
-
-/// Converts a catalog JSON-serialized `AclMode` bitflags object into a
-/// PostgreSQL ACL char-code string (e.g. `{"bitflags": 514}` → `"ar"`).
-#[sqlfunc]
-fn parse_catalog_acl_mode<'a>(a: JsonbRef<'a>) -> Result {
- jsonb_datum_to_acl_mode(a.into_datum())
- .map(|mode| mode.to_string())
- .map_err(|e| EvalError::InvalidCatalogJson(e.into()))
-}
-
-/// Extracts the string form of a `WithOptionValue`, matching how the planner
-/// coerces option values. The blanket `TryFromValue>` impl in
-/// `src/sql/src/plan/with_options.rs` accepts a quoted string, a bare identifier,
-/// and a 1-part unresolved item name, all yielding a string. Any other variant
-/// yields None.
-///
-/// `AstDisplay` does not re-quote bare identifiers or 1-part names, so those
-/// forms persist unquoted in `create_sql`. Matching only `Value::String` here
-/// would miss them and silently fall back to a default, so the catalog-raw
-/// parsers below route their string options through this helper.
-fn option_string(value: &WithOptionValue) -> Option {
- match value {
- WithOptionValue::Value(Value::String(s)) => Some(s.clone()),
- WithOptionValue::Ident(ident) => Some(ident.clone().into_string()),
- WithOptionValue::UnresolvedItemName(UnresolvedItemName(parts)) if parts.len() == 1 => {
- Some(parts[0].clone().into_string())
- }
- _ => None,
- }
-}
-
-/// Parses a catalog `create_sql` string into a JSONB object.
-///
-/// The returned JSONB does not fully reflect the parsed SQL and instead contains only fields
-/// required by current callers.
-// TODO: This function isn't parsing JSONB and therefore shouldn't live in the `jsonb` module.
-// Consider moving all the `parse_catalog_*` functions into their own module.
-#[sqlfunc]
-fn parse_catalog_create_sql<'a>(a: &'a str) -> Result {
- fn get_cluster_id(in_cluster: RawClusterName) -> Result {
- match in_cluster {
- RawClusterName::Resolved(s) => Ok(s),
- RawClusterName::Unresolved(_) => Err("unresolved cluster name"),
- }
- }
-
- fn get_item_id(item: RawItemName) -> Result {
- match item {
- RawItemName::Id(id, _, _) => Ok(id),
- RawItemName::Name(_) => Err("unresolved item name"),
- }
- }
-
- fn format_name(fmt: &Format) -> &'static str {
- match fmt {
- Format::Bytes => "bytes",
- Format::Avro(_) => "avro",
- Format::Protobuf(_) => "protobuf",
- Format::Regex(_) => "regex",
- Format::Csv { .. } => "csv",
- Format::Json { .. } => "json",
- Format::Text => "text",
- }
- }
-
- let parse = || -> Result {
- let mut stmts = mz_sql_parser::parser::parse_statements(a)
- .map_err(|e| format!("failed to parse create_sql: {e}"))?;
- let stmt = match stmts.len() {
- 1 => stmts.remove(0).ast,
- n => return Err(format!("expected a single statement, found {n}")),
- };
-
- let mut info = BTreeMap::<&str, serde_json::Value>::new();
-
- use mz_sql_parser::ast::Statement::*;
- let item_type = match stmt {
- CreateSecret(_) => "secret",
- CreateConnection(stmt) => {
- let connection_type = stmt.connection_type.as_str();
- info.insert("connection_type", json!(connection_type));
-
- "connection"
- }
- CreateView(stmt) => {
- let mut definition = stmt.definition.query.to_ast_string_stable();
- // PostgreSQL appends a semicolon in `pg_views.definition`, we
- // do the same for compatibility's sake.
- definition.push(';');
- info.insert("definition", json!(definition));
-
- "view"
- }
- CreateMaterializedView(stmt) => {
- let Some(in_cluster) = stmt.in_cluster else {
- return Err("missing IN CLUSTER".into());
- };
- let cluster_id = match in_cluster {
- RawClusterName::Unresolved(ident) => ident.into_string(),
- RawClusterName::Resolved(s) => s,
- };
- info.insert("cluster_id", json!(cluster_id));
-
- let mut definition = stmt.query.to_ast_string_stable();
- definition.push(';');
- info.insert("definition", json!(definition));
-
- "materialized-view"
- }
- CreateTable(_) => "table",
- CreateTableFromSource(stmt) => {
- let source_id = get_item_id(stmt.source)?;
- info.insert("source_id", json!(source_id));
-
- "table"
- }
- CreateSource(stmt) => {
- let Some(in_cluster) = stmt.in_cluster else {
- return Err("missing IN CLUSTER".into());
- };
- let cluster_id = get_cluster_id(in_cluster)?;
- info.insert("cluster_id", json!(cluster_id));
-
- use mz_sql_parser::ast::CreateSourceConnection::*;
- let (source_type, connection) = match stmt.connection {
- Kafka { connection, .. } => ("kafka", Some(connection)),
- Postgres { connection, .. } => ("postgres", Some(connection)),
- MySql { connection, .. } => ("mysql", Some(connection)),
- SqlServer { connection, .. } => ("sql-server", Some(connection)),
- LoadGenerator { .. } => ("load-generator", None),
- };
- info.insert("source_type", json!(source_type));
- if let Some(conn) = connection {
- let conn_id = get_item_id(conn)?;
- info.insert("connection_id", json!(conn_id));
- }
-
- let is_debezium = matches!(
- stmt.envelope,
- Some(mz_sql_parser::ast::SourceEnvelope::Debezium)
- );
-
- // An old-syntax kafka source ingests into its own relation, so an
- // omitted ENVELOPE means the default ENVELOPE NONE and the pre-MV
- // packer reported 'none'. A new-syntax source (no progress
- // subsource, hence no EXPOSE PROGRESS AS in create_sql) ingests
- // nothing itself. Its envelopes live on the per-table exports, so
- // its own envelope_type stays absent (SQL NULL), matching released
- // behavior. `progress_subsource.is_some()` is the planner's own
- // old-vs-new discriminator (see `OldSyntaxIngestion` in
- // plan_create_source). Non-kafka sources carry no envelope either.
- // See the `mz_sources.envelope_type` column.
- let envelope_type = match &stmt.envelope {
- Some(envelope) => {
- use mz_sql_parser::ast::SourceEnvelope::*;
- Some(match envelope {
- None => "none",
- Debezium => "debezium",
- Upsert { .. } => "upsert",
- CdcV2 => "materialize",
- })
- }
- None if source_type == "kafka" && stmt.progress_subsource.is_some() => {
- Some("none")
- }
- None => None,
- };
- if let Some(envelope_type) = envelope_type {
- info.insert("envelope_type", json!(envelope_type));
- }
-
- if let Some(format_spec) = stmt.format {
- match &format_spec {
- FormatSpecifier::Bare(fmt) => {
- // Debezium sources with a single format spec implicitly use
- // the same format for both key and value.
- if is_debezium {
- info.insert("key_format", json!(format_name(fmt)));
- }
- info.insert("value_format", json!(format_name(fmt)));
- }
- FormatSpecifier::KeyValue { key, value } => {
- info.insert("key_format", json!(format_name(key)));
- info.insert("value_format", json!(format_name(value)));
- }
- }
- }
-
- "source"
- }
- CreateWebhookSource(stmt) => {
- if stmt.is_table {
- "table"
- } else {
- info.insert("source_type", json!("webhook"));
- if let Some(in_cluster) = stmt.in_cluster {
- let cluster_id = get_cluster_id(in_cluster)?;
- info.insert("cluster_id", json!(cluster_id));
- }
- "source"
- }
- }
- CreateSubsource(stmt) => {
- use mz_sql_parser::ast::CreateSubsourceOptionName;
- let is_progress = stmt
- .with_options
- .iter()
- .any(|o| matches!(o.name, CreateSubsourceOptionName::Progress));
- let source_type = if is_progress { "progress" } else { "subsource" };
- info.insert("source_type", json!(source_type));
-
- if let Some(of_source) = stmt.of_source {
- let of_source_id = get_item_id(of_source)?;
- info.insert("of_source_id", json!(of_source_id));
- }
-
- "subsource"
- }
- // Everything the mz_sinks, mz_kafka_sinks and mz_iceberg_sinks
- // views read. The Rust side of each value lives in `Sink` and
- // `StorageSinkConnection`, so those and this have to move together.
- //
- // NOTE: we bail below if a sink has no resolved `IN CLUSTER`, no
- // `TOPIC` on kafka, or no `NAMESPACE`/`TABLE` on iceberg. Planning
- // guarantees all four. But if one ever slipped through it would
- // take down every view built on this function, not just the sink
- // ones.
- CreateSink(stmt) => {
- let Some(in_cluster) = stmt.in_cluster else {
- return Err("missing IN CLUSTER".into());
- };
- info.insert("cluster_id", json!(get_cluster_id(in_cluster)?));
-
- match stmt.connection {
- CreateSinkConnection::Kafka {
- connection,
- options,
- key: sink_key,
- ..
- } => {
- info.insert("sink_type", json!("kafka"));
- info.insert("connection_id", json!(get_item_id(connection)?));
-
- let topic = options
- .into_iter()
- .find(|o| o.name == KafkaSinkConfigOptionName::Topic)
- .and_then(|o| o.value.as_ref().and_then(option_string))
- .ok_or("kafka sink missing TOPIC")?;
- info.insert("topic", json!(topic));
-
- if let Some(envelope) = stmt.envelope {
- let envelope_type = match envelope {
- SinkEnvelope::Upsert => "upsert",
- SinkEnvelope::Debezium => "debezium",
- };
- info.insert("envelope_type", json!(envelope_type));
- }
-
- if let Some(format_spec) = stmt.format {
- // A key format only survives if the sink has a
- // `KEY`. Without one `kafka_sink_builder` throws
- // away the key half of a key/value spec, and does
- // not copy a bare spec over to the key either.
- let (key_format, value_format) =
- match (&format_spec, sink_key.is_some()) {
- (FormatSpecifier::Bare(fmt), false) => (None, format_name(fmt)),
- (FormatSpecifier::Bare(fmt), true) => {
- (Some(format_name(fmt)), format_name(fmt))
- }
- (FormatSpecifier::KeyValue { value, .. }, false) => {
- (None, format_name(value))
- }
- (FormatSpecifier::KeyValue { key, value }, true) => {
- (Some(format_name(key)), format_name(value))
- }
- };
- if let Some(key_format) = key_format {
- info.insert("key_format", json!(key_format));
- }
- info.insert("value_format", json!(value_format));
-
- // The deprecated combined `format`. Only avro/avro
- // and json/json collapse to a single name.
- // Everything else, text/text and bytes/bytes
- // included, gets the composite form.
- let combined = match key_format {
- None => value_format.to_string(),
- Some(key_format)
- if key_format == value_format
- && matches!(value_format, "avro" | "json") =>
- {
- value_format.to_string()
- }
- Some(key_format) => {
- format!("key-{key_format}-value-{value_format}")
- }
- };
- info.insert("format", json!(combined));
- }
- }
- CreateSinkConnection::Iceberg {
- catalog_connection,
- options,
- ..
- } => {
- info.insert("sink_type", json!("iceberg"));
- // The catalog connection, not the optional AWS one.
- info.insert("connection_id", json!(get_item_id(catalog_connection)?));
-
- let mut namespace = None;
- let mut table = None;
- for option in options {
- match option.name {
- IcebergSinkConfigOptionName::Namespace => {
- namespace = option.value.as_ref().and_then(option_string)
- }
- IcebergSinkConfigOptionName::Table => {
- table = option.value.as_ref().and_then(option_string)
- }
- }
- }
- info.insert(
- "namespace",
- json!(namespace.ok_or("iceberg sink missing NAMESPACE")?),
- );
- info.insert("table", json!(table.ok_or("iceberg sink missing TABLE")?));
-
- // Iceberg spells the envelope `MODE`, and has no format
- // columns at all.
- if let Some(mode) = stmt.mode {
- let envelope_type = match mode {
- IcebergSinkMode::Upsert => "upsert",
- IcebergSinkMode::Append => "append",
- };
- info.insert("envelope_type", json!(envelope_type));
- }
- }
- }
-
- "sink"
- }
- CreateMetricSink(stmt) => {
- let Some(in_cluster) = stmt.in_cluster else {
- return Err("missing IN CLUSTER".into());
- };
- let cluster_id = get_cluster_id(in_cluster)?;
- info.insert("cluster_id", json!(cluster_id));
- let from_id = get_item_id(stmt.from)?;
- info.insert("from_id", json!(from_id));
- "metric-sink"
- }
- CreateIndex(stmt) => {
- let Some(in_cluster) = stmt.in_cluster else {
- return Err("missing IN CLUSTER".into());
- };
- let cluster_id = get_cluster_id(in_cluster)?;
- info.insert("cluster_id", json!(cluster_id));
- let on_id = get_item_id(stmt.on_name)?;
- info.insert("on_id", json!(on_id));
- "index"
- }
- CreateType(_) => "type",
- // NOTE: every statement that creates a catalog item needs an arm above. These
- // catalog views run this over every item row before their type filter drops the
- // unwanted rows, so one unclassified `create_sql` takes out `mz_objects`,
- // `mz_indexes`, and every sibling view at once. The match is exhaustive to make
- // that a compile error here, not a runtime failure.
- Select(_)
- | Insert(_)
- | Copy(_)
- | Update(_)
- | Delete(_)
- | CreateDatabase(_)
- | CreateSchema(_)
- | CreateRole(_)
- | CreateCluster(_)
- | CreateClusterReplica(_)
- | CreateNetworkPolicy(_)
- | AlterCluster(_)
- | AlterOwner(_)
- | AlterObjectRename(_)
- | AlterObjectSwap(_)
- | AlterRetainHistory(_)
- | AlterIndex(_)
- | AlterSecret(_)
- | AlterSetCluster(_)
- | AlterSink(_)
- | AlterSource(_)
- | AlterSystemSet(_)
- | AlterSystemReset(_)
- | AlterSystemResetAll(_)
- | AlterConnection(_)
- | AlterNetworkPolicy(_)
- | AlterRole(_)
- | AlterTableAddColumn(_)
- | AlterMaterializedViewApplyReplacement(_)
- | Discard(_)
- | DropObjects(_)
- | DropOwned(_)
- | SetVariable(_)
- | ResetVariable(_)
- | Show(_)
- | StartTransaction(_)
- | SetTransaction(_)
- | Commit(_)
- | Rollback(_)
- | Subscribe(_)
- | ExplainPlan(_)
- | ExplainPushdown(_)
- | ExplainTimestamp(_)
- | ExplainSinkSchema(_)
- | ExplainAnalyzeObject(_)
- | ExplainAnalyzeCluster(_)
- | Declare(_)
- | Fetch(_)
- | Close(_)
- | Prepare(_)
- | Execute(_)
- | ExecuteUnitTest(_)
- | Deallocate(_)
- | Raise(_)
- | GrantRole(_)
- | RevokeRole(_)
- | GrantPrivileges(_)
- | RevokePrivileges(_)
- | AlterDefaultPrivileges(_)
- | ReassignOwned(_)
- | ValidateConnection(_)
- | Comment(_) => return Err("not a CREATE item statement".into()),
- };
- info.insert("type", json!(item_type));
-
- let info = info.into_iter().map(|(k, v)| (k.to_string(), v)).collect();
- Ok(info)
- };
-
- let val = parse().map_err(|e| EvalError::InvalidCatalogJson(e.into()))?;
- let jsonb = Jsonb::from_serde_json(val).expect("valid JSONB");
- Ok(jsonb)
-}
-
-/// Extracts the catalog item references from a catalog `create_sql` string as a JSONB object.
-///
-/// - `ids`: array of catalog id strings from `[ AS ]` references.
-/// - `named_funcs`: array of function references such as "pg_catalog"."max".
-/// - `named_types`: array of type references such as "pg_catalog"."int4", exclusive from the ones
-/// in `ids`.
-/// - `named_relations`: array of relation references, exclusive from the ones in `ids`.
-#[sqlfunc]
-fn parse_catalog_item_references<'a>(a: &'a str) -> Result {
- fn qualified(name: &UnresolvedItemName) -> serde_json::Value {
- match &name.0[..] {
- [.., schema, item] => json!({"schema": schema.as_str(), "name": item.as_str()}),
- [item] => json!({"schema": serde_json::Value::Null, "name": item.as_str()}),
- [] => json!({"schema": serde_json::Value::Null, "name": ""}),
- }
- }
-
- let parse = || -> Result {
- let mut stmts = mz_sql_parser::parser::parse_statements(a)
- .map_err(|e| format!("failed to parse create_sql: {e}"))?;
- let stmt = match stmts.len() {
- 1 => stmts.remove(0).ast,
- n => return Err(format!("expected a single statement, found {n}")),
- };
-
- let refs = collect_item_references(&stmt);
- mz_ore::soft_assert_or_log!(
- refs.named_array_elements.is_empty(),
- "persisted create_sql should never carry an array type T[] \
- and resolve its array type in `ids`: {:?}",
- refs.named_array_elements
- );
- Ok(json!({
- "ids": refs.ids.iter().collect::>(),
- "named_funcs": refs.named_funcs.iter().map(qualified).collect::>(),
- "named_types": refs.named_types.iter().map(qualified).collect::>(),
- "named_relations": refs.named_relations.iter().map(qualified).collect::>(),
- }))
- };
-
- let val = parse().map_err(|e| EvalError::InvalidCatalogJson(e.into()))?;
- let jsonb = Jsonb::from_serde_json(val).expect("valid JSONB");
- Ok(jsonb)
-}
-
-/// Minimal decoder for `ProtoPostgresSourcePublicationDetails`. The
-/// canonical proto lives in `mz-storage-types`, which depends on
-/// `mz-expr`, so we redeclare the two tags we read here. Upstream tag
-/// renumbers slip past silently. The `mz_postgres_sources` lockdown
-/// SLTs catch them.
-#[derive(Clone, PartialEq, ::prost::Message)]
-struct PostgresPublicationDetailsSubset {
- #[prost(string, tag = "2")]
- slot: String,
- #[prost(uint64, optional, tag = "3")]
- timeline_id: Option,
-}
-
-/// Extracts postgres source publication details (slot, timeline_id) from a
-/// catalog `create_sql`. Returns:
-///
-/// - jsonb `{"slot": , "timeline_id": }` for `CREATE SOURCE ... FROM POSTGRES
-/// CONNECTION ... (DETAILS = ...)` statements.
-/// - jsonb `null` for any other statement.
-///
-/// Errors if the statement fails to parse, is a postgres source without
-/// a `DETAILS` option, or if the `DETAILS` value can't be hex- and
-/// proto-decoded.
-#[sqlfunc]
-fn parse_postgres_source_details<'a>(a: &'a str) -> Result {
- let parse = || -> Result {
- let mut stmts = mz_sql_parser::parser::parse_statements(a)
- .map_err(|e| format!("failed to parse create_sql: {e}"))?;
- let stmt = match stmts.len() {
- 1 => stmts.remove(0).ast,
- n => return Err(format!("expected a single statement, found {n}")),
- };
-
- use mz_sql_parser::ast::CreateSourceConnection;
- use mz_sql_parser::ast::Statement::CreateSource;
- let options = match stmt {
- CreateSource(stmt) => match stmt.connection {
- CreateSourceConnection::Postgres { options, .. } => options,
- _ => return Ok(serde_json::Value::Null),
- },
- _ => return Ok(serde_json::Value::Null),
- };
-
- let details_hex = options
- .into_iter()
- .find(|opt| opt.name == PgConfigOptionName::Details)
- .and_then(|opt| match opt.value {
- Some(WithOptionValue::Value(Value::String(s))) => Some(s),
- _ => None,
- })
- .ok_or("missing DETAILS option on postgres source")?;
-
- let details_bytes =
- hex::decode(&details_hex).map_err(|e| format!("DETAILS is not valid hex: {e}"))?;
-
- let details = PostgresPublicationDetailsSubset::decode(&*details_bytes)
- .map_err(|e| format!("DETAILS is not a valid publication-details proto: {e}"))?;
-
- Ok(json!({
- "slot": details.slot,
- "timeline_id": details.timeline_id,
- }))
- };
-
- let val = parse().map_err(|e| EvalError::InvalidCatalogJson(e.into()))?;
- let jsonb = Jsonb::from_serde_json(val).expect("valid JSONB");
- Ok(jsonb)
-}
-
-/// Extracts kafka source configuration (topic, group id prefix, connection
-/// id) from a catalog `create_sql`. Returns:
-///
-/// - jsonb `{"topic": , "group_id_prefix": , "connection_id": }` for
-/// `CREATE SOURCE ... FROM KAFKA CONNECTION ... (TOPIC = ..., [GROUP ID PREFIX = ...])`
-/// statements.
-/// - jsonb `null` for any other statement.
-///
-/// Errors if the statement fails to parse, is a kafka source without a
-/// `TOPIC` option, or references an unresolved connection name (i.e. one
-/// that hasn't been through purification).
-#[sqlfunc]
-fn parse_kafka_source_details<'a>(a: &'a str) -> Result {
- fn get_item_id(item: RawItemName) -> Result {
- match item {
- RawItemName::Id(id, _, _) => Ok(id),
- RawItemName::Name(_) => Err("unresolved item name"),
- }
- }
-
- let parse = || -> Result {
- let mut stmts = mz_sql_parser::parser::parse_statements(a)
- .map_err(|e| format!("failed to parse create_sql: {e}"))?;
- let stmt = match stmts.len() {
- 1 => stmts.remove(0).ast,
- n => return Err(format!("expected a single statement, found {n}")),
- };
-
- use mz_sql_parser::ast::CreateSourceConnection;
- use mz_sql_parser::ast::Statement::CreateSource;
- let (connection, options) = match stmt {
- CreateSource(stmt) => match stmt.connection {
- CreateSourceConnection::Kafka {
- connection,
- options,
- } => (connection, options),
- _ => return Ok(serde_json::Value::Null),
- },
- _ => return Ok(serde_json::Value::Null),
- };
-
- let connection_id = get_item_id(connection)?;
-
- let mut topic: Option = None;
- let mut group_id_prefix: Option = None;
- for opt in options {
- let string_value = opt.value.as_ref().and_then(option_string);
- match opt.name {
- KafkaSourceConfigOptionName::Topic => topic = string_value,
- KafkaSourceConfigOptionName::GroupIdPrefix => group_id_prefix = string_value,
- _ => {}
- }
- }
-
- let topic = topic.ok_or("missing TOPIC option on kafka source")?;
-
- Ok(json!({
- "topic": topic,
- "group_id_prefix": group_id_prefix,
- "connection_id": connection_id,
- }))
- };
-
- let val = parse().map_err(|e| EvalError::InvalidCatalogJson(e.into()))?;
- let jsonb = Jsonb::from_serde_json(val).expect("valid JSONB");
- Ok(jsonb)
-}
-
-/// Extracts source-export (source table) metadata from a catalog `create_sql`.
-///
-/// Returns, for a `CREATE TABLE ... FROM SOURCE` or a non-progress
-/// `CREATE SUBSOURCE ... OF SOURCE ...` statement:
-///
-/// ```json
-/// {
-/// "source_id": "",
-/// "external_reference": ["part1", "part2", ...],
-/// "envelope_type": ,
-/// "key_format": ,
-/// "value_format":
-/// }
-/// ```
-///
-/// `envelope_type`, `key_format`, and `value_format` are always null for a
-/// `CREATE SUBSOURCE` (the postgres/mysql/sql-server exports that use the old
-/// subsource syntax carry neither format nor envelope). They may also be null
-/// for a `CREATE TABLE ... FROM SOURCE` that omits FORMAT/ENVELOPE.
-///
-/// Returns jsonb `null` for progress subsources and for any statement that is
-/// not a source export. The caller distinguishes the four source-table views
-/// by joining `source_id` against `mz_sources` and filtering on the parent's
-/// type, so this helper stays connection-type agnostic.
-///
-/// Errors if the statement fails to parse, references an unresolved item name,
-/// or is a non-progress subsource missing its OF SOURCE or EXTERNAL REFERENCE.
-///
-/// The `key_format`/`value_format` derivation mirrors the runtime
-/// `DataSourceDesc::formats()` that the removed `pack_kafka_source_tables_update`
-/// packer read. A bare FORMAT only carries a key when it resolves to an
-/// encoding that has one, which among bare formats is only Avro or Protobuf
-/// read from a Confluent Schema Registry whose purified seed carries a key
-/// schema. A KEY FORMAT ... VALUE FORMAT ... spec always carries both.
-#[sqlfunc]
-fn parse_source_export_details<'a>(a: &'a str) -> Result {
- fn get_item_id(item: RawItemName) -> Result {
- match item {
- RawItemName::Id(id, _, _) => Ok(id),
- RawItemName::Name(_) => Err("unresolved item name"),
- }
- }
-
- fn format_name(fmt: &Format) -> &'static str {
- match fmt {
- Format::Bytes => "bytes",
- Format::Avro(_) => "avro",
- Format::Protobuf(_) => "protobuf",
- Format::Regex(_) => "regex",
- Format::Csv { .. } => "csv",
- Format::Json { .. } => "json",
- Format::Text => "text",
- }
- }
-
- // A bare FORMAT resolves to an encoding with a key only for Avro or
- // Protobuf read from a schema registry whose purified seed carries a key
- // schema. Every other bare format is value-only.
- fn bare_format_has_key(fmt: &Format) -> bool {
- match fmt {
- Format::Avro(AvroSchema::Csr { csr_connection }) => csr_connection
- .seed
- .as_ref()
- .is_some_and(|seed| seed.key_schema.is_some()),
- Format::Protobuf(ProtobufSchema::Csr { csr_connection }) => csr_connection
- .seed
- .as_ref()
- .is_some_and(|seed| seed.key.is_some()),
- _ => false,
- }
- }
-
- fn key_value_formats(
- spec: &FormatSpecifier,
- ) -> (Option<&'static str>, Option<&'static str>) {
- match spec {
- FormatSpecifier::KeyValue { key, value } => {
- (Some(format_name(key)), Some(format_name(value)))
- }
- FormatSpecifier::Bare(fmt) => {
- let value = Some(format_name(fmt));
- let key = bare_format_has_key(fmt).then(|| format_name(fmt));
- (key, value)
- }
- }
- }
-
- fn envelope_name(envelope: &SourceEnvelope) -> &'static str {
- match envelope {
- SourceEnvelope::None => "none",
- SourceEnvelope::Debezium => "debezium",
- SourceEnvelope::Upsert {
- value_decode_err_policy,
- } => {
- if value_decode_err_policy
- .iter()
- .any(|p| matches!(p, SourceErrorPolicy::Inline { .. }))
- {
- "upsert-value-err-inline"
- } else {
- "upsert"
- }
- }
- SourceEnvelope::CdcV2 => "materialize",
- }
- }
-
- let parse = || -> Result {
- let mut stmts = mz_sql_parser::parser::parse_statements(a)
- .map_err(|e| format!("failed to parse create_sql: {e}"))?;
- let stmt = match stmts.len() {
- 1 => stmts.remove(0).ast,
- n => return Err(format!("expected a single statement, found {n}")),
- };
-
- use mz_sql_parser::ast::Statement::{CreateSubsource, CreateTableFromSource};
- match stmt {
- CreateTableFromSource(stmt) => {
- let source_id = get_item_id(stmt.source)?;
- let external_reference = stmt
- .external_reference
- .ok_or("missing external reference on CREATE TABLE FROM SOURCE")?
- .0
- .into_iter()
- .map(|ident| ident.into_string())
- .collect::>();
-
- let envelope_type = stmt.envelope.as_ref().map(envelope_name);
- let (key_format, value_format) = match &stmt.format {
- Some(spec) => key_value_formats(spec),
- None => (None, None),
- };
-
- Ok(json!({
- "source_id": source_id,
- "external_reference": external_reference,
- "envelope_type": envelope_type,
- "key_format": key_format,
- "value_format": value_format,
- }))
- }
- CreateSubsource(stmt) => {
- // Progress subsources track ingestion progress and are not
- // source tables. They have no external reference.
- let is_progress = stmt
- .with_options
- .iter()
- .any(|o| matches!(o.name, CreateSubsourceOptionName::Progress));
- if is_progress {
- return Ok(serde_json::Value::Null);
- }
-
- let source_id = stmt
- .of_source
- .ok_or("non-progress CREATE SUBSOURCE without OF SOURCE")
- .and_then(get_item_id)?;
-
- let external_reference = stmt
- .with_options
- .into_iter()
- .find(|o| matches!(o.name, CreateSubsourceOptionName::ExternalReference))
- .and_then(|o| match o.value {
- Some(WithOptionValue::UnresolvedItemName(name)) => Some(name),
- _ => None,
- })
- .ok_or("CREATE SUBSOURCE missing EXTERNAL REFERENCE option")?
- .0
- .into_iter()
- .map(|ident| ident.into_string())
- .collect::>();
-
- Ok(json!({
- "source_id": source_id,
- "external_reference": external_reference,
- "envelope_type": serde_json::Value::Null,
- "key_format": serde_json::Value::Null,
- "value_format": serde_json::Value::Null,
- }))
- }
- _ => Ok(serde_json::Value::Null),
- }
- };
-
- let val = parse().map_err(|e| EvalError::InvalidCatalogJson(e.into()))?;
- let jsonb = Jsonb::from_serde_json(val).expect("valid JSONB");
- Ok(jsonb)
-}
-
-/// Extracts connection-detail metadata from a catalog `create_sql`.
-///
-/// Returns a per-connection-type object with the fields that the
-/// `mz_kafka_connections`, `mz_ssh_tunnel_connections`, and `mz_aws_connections`
-/// builtin views need. For everything else (other connection types, including
-/// aws-privatelink whose only detail is context-derived, and non-connection
-/// statements) it returns jsonb `null`, so callers filter on `IS NOT NULL` and
-/// gate on the connection type separately (via
-/// `parse_catalog_create_sql(...)->>'connection_type'`, the way `mz_connections`
-/// already does).
-///
-/// The shape per type:
-///
-/// ```json
-/// // kafka
-/// { "brokers": ["host:port", ...], "progress_topic": }
-/// // ssh-tunnel
-/// { "public_key_1": "", "public_key_2": "" }
-/// // aws
-/// {
-/// "auth_kind": "credentials" | "assume-role",
-/// "endpoint": , "region": ,
-/// "access_key_id": , "access_key_id_secret_id": ,
-/// "secret_access_key_secret_id": ,
-/// "session_token": , "session_token_secret_id": ,
-/// "assume_role_arn": , "assume_role_session_name":
-/// }
-/// ```
-///
-/// `progress_topic` is null when the connection does not set an explicit
-/// `PROGRESS TOPIC`. The default (`_materialize-progress--`) is
-/// reconstructed by the view, not here, because it needs the environment id and
-/// the connection's own id. Values derived only from environment context
-/// (AWS principal, external id, trust policy, privatelink principal) are also
-/// left to the view. This keeps the helper a pure function of the `create_sql`.
-///
-/// For aws, an option is either an inline value or a secret reference. Inline
-/// values land in `access_key_id`/`session_token`; a secret reference lands in
-/// the matching `*_secret_id` as the referenced secret's catalog item id (the
-/// persisted `create_sql` stores resolved references as `[uNNN AS name]`).
-/// `auth_kind` is `assume-role` when `ASSUME ROLE ARN` is present, else
-/// `credentials`, matching the `AwsAuth` variant the removed packer read.
-///
-/// Errors if the statement fails to parse.
-#[sqlfunc]
-fn parse_connection_details<'a>(a: &'a str) -> Result {
- // The persisted `create_sql` stores an inline broker as a single `BROKER`
- // option and a broker list as a `BROKERS (...)` sequence. Either way we
- // only need the addresses, which are present regardless of the tunnel
- // (direct, SSH, or PrivateLink).
- fn broker_addresses(values: &[ConnectionOption]) -> Vec {
- let mut brokers = Vec::new();
- for opt in values {
- match (&opt.name, &opt.value) {
- (ConnectionOptionName::Broker, Some(WithOptionValue::ConnectionKafkaBroker(b))) => {
- brokers.push(b.address.clone());
- }
- (ConnectionOptionName::Brokers, Some(WithOptionValue::Sequence(seq))) => {
- for v in seq {
- if let WithOptionValue::ConnectionKafkaBroker(b) = v {
- brokers.push(b.address.clone());
- }
- }
- }
- _ => {}
- }
- }
- brokers
- }
-
- fn string_option(
- values: &[ConnectionOption],
- name: ConnectionOptionName,
- ) -> Option {
- values
- .iter()
- .find(|o| o.name == name)
- .and_then(|o| o.value.as_ref())
- .and_then(option_string)
- }
-
- // The catalog id of the secret a `SECRET ...` option references. Resolved
- // references persist as `RawItemName::Id`, so an unresolved name yields
- // None (the same treatment `parse_source_export_details` gives item names).
- fn secret_id_option>(
- values: &[ConnectionOption],
- name: ConnectionOptionName,
- ) -> Option {
- values.iter().find_map(|o| match &o.value {
- Some(WithOptionValue::Secret(RawItemName::Id(id, _, _))) if o.name == name => {
- Some(id.clone())
- }
- _ => None,
- })
- }
-
- let parse = || -> Result {
- let mut stmts = mz_sql_parser::parser::parse_statements(a)
- .map_err(|e| format!("failed to parse create_sql: {e}"))?;
- let stmt = match stmts.len() {
- 1 => stmts.remove(0).ast,
- n => return Err(format!("expected a single statement, found {n}")),
- };
-
- use mz_sql_parser::ast::Statement::CreateConnection;
- let stmt = match stmt {
- CreateConnection(stmt) => stmt,
- _ => return Ok(serde_json::Value::Null),
- };
-
- match stmt.connection_type {
- CreateConnectionType::Kafka => Ok(json!({
- "brokers": broker_addresses(&stmt.values),
- "progress_topic": string_option(&stmt.values, ConnectionOptionName::ProgressTopic),
- })),
- CreateConnectionType::Ssh => Ok(json!({
- "public_key_1": string_option(&stmt.values, ConnectionOptionName::PublicKey1),
- "public_key_2": string_option(&stmt.values, ConnectionOptionName::PublicKey2),
- })),
- CreateConnectionType::Aws => {
- let assume_role_arn =
- string_option(&stmt.values, ConnectionOptionName::AssumeRoleArn);
- let auth_kind = if assume_role_arn.is_some() {
- "assume-role"
- } else {
- "credentials"
- };
- Ok(json!({
- "auth_kind": auth_kind,
- // Planning coerces an empty ENDPOINT to None (see
- // `src/sql/src/plan/statement/ddl/connection.rs`), so the
- // removed packer wrote NULL for `ENDPOINT = ''`. Match that.
- "endpoint": string_option(&stmt.values, ConnectionOptionName::Endpoint)
- .filter(|s| !s.is_empty()),
- "region": string_option(&stmt.values, ConnectionOptionName::Region),
- "access_key_id": string_option(&stmt.values, ConnectionOptionName::AccessKeyId),
- "access_key_id_secret_id":
- secret_id_option(&stmt.values, ConnectionOptionName::AccessKeyId),
- "secret_access_key_secret_id":
- secret_id_option(&stmt.values, ConnectionOptionName::SecretAccessKey),
- "session_token": string_option(&stmt.values, ConnectionOptionName::SessionToken),
- "session_token_secret_id":
- secret_id_option(&stmt.values, ConnectionOptionName::SessionToken),
- "assume_role_arn": assume_role_arn,
- "assume_role_session_name":
- string_option(&stmt.values, ConnectionOptionName::AssumeRoleSessionName),
- }))
- }
- _ => Ok(serde_json::Value::Null),
- }
- };
-
- let val = parse().map_err(|e| EvalError::InvalidCatalogJson(e.into()))?;
- let jsonb = Jsonb::from_serde_json(val).expect("valid JSONB");
- Ok(jsonb)
-}
-
-#[cfg(test)]
-mod tests {
- use mz_repr::adt::jsonb::Jsonb;
- use prost::Message as _;
- use serde_json::json;
-
- use crate::EvalError;
-
- /// Encode the two proto fields our decoder cares about, using the same
- /// tag numbering as the canonical proto.
- fn encode_pg_details(slot: &str, timeline_id: Option) -> String {
- let details = super::PostgresPublicationDetailsSubset {
- slot: slot.to_string(),
- timeline_id,
- };
- hex::encode(details.encode_to_vec())
- }
-
- fn pg_source_sql(details_hex: &str) -> String {
- format!(
- "CREATE SOURCE \"materialize\".\"public\".\"pg_src\" \
- IN CLUSTER [u42] \
- FROM POSTGRES CONNECTION [u10 AS \"materialize\".\"public\".\"pg_conn\"] \
- (DETAILS = '{details_hex}', PUBLICATION = 'mz_source') \
- FOR ALL TABLES"
- )
- }
-
- fn kafka_source_sql(with_prefix: bool) -> String {
- let prefix_opt = if with_prefix {
- ", GROUP ID PREFIX 'my-prefix-'"
- } else {
- ""
- };
- format!(
- "CREATE SOURCE \"materialize\".\"public\".\"k_src\" \
- IN CLUSTER [u42] \
- FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \
- (TOPIC 'test'{prefix_opt}) FORMAT TEXT"
- )
- }
-
- fn as_serde(jsonb: Jsonb) -> serde_json::Value {
- jsonb.as_ref().to_serde_json()
- }
-
- // --- parse_postgres_source_details ---------------------------------------
-
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `decContextDefault` on OS `linux`
- fn pg_happy_path_with_timeline() {
- let hex = encode_pg_details("materialize_abc", Some(42));
- let sql = pg_source_sql(&hex);
- let out = super::parse_postgres_source_details(&sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({ "slot": "materialize_abc", "timeline_id": 42 }),
- );
- }
-
- #[mz_ore::test]
- fn pg_happy_path_null_timeline() {
- // Pre-2024 sources have no timeline_id field. The decoder must
- // surface that as JSON null, not error.
- let hex = encode_pg_details("materialize_legacy", None);
- let sql = pg_source_sql(&hex);
- let out = super::parse_postgres_source_details(&sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({ "slot": "materialize_legacy", "timeline_id": null }),
- );
- }
-
- #[mz_ore::test]
- fn pg_non_postgres_source_returns_null_jsonb() {
- let sql = "CREATE SOURCE \"materialize\".\"public\".\"lg\" \
- IN CLUSTER [u42] FROM LOAD GENERATOR COUNTER";
- let out = super::parse_postgres_source_details(sql).expect("ok");
- assert_eq!(as_serde(out), serde_json::Value::Null);
- }
-
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn pg_non_create_source_returns_null_jsonb() {
- let sql = "CREATE VIEW v AS SELECT 1";
- let out = super::parse_postgres_source_details(sql).expect("ok");
- assert_eq!(as_serde(out), serde_json::Value::Null);
- }
-
- #[mz_ore::test]
- fn pg_missing_details_option_errors() {
- let sql = "CREATE SOURCE \"materialize\".\"public\".\"pg_src\" \
- IN CLUSTER [u42] \
- FROM POSTGRES CONNECTION [u10 AS \"materialize\".\"public\".\"pg_conn\"] \
- (PUBLICATION = 'mz_source') FOR ALL TABLES";
- let err = super::parse_postgres_source_details(sql).unwrap_err();
- assert!(
- matches!(err, EvalError::InvalidCatalogJson(msg) if msg.contains("missing DETAILS")),
- "wrong error variant/message"
- );
- }
-
- #[mz_ore::test]
- fn pg_malformed_hex_errors() {
- let sql = pg_source_sql("not-hex!!");
- let err = super::parse_postgres_source_details(&sql).unwrap_err();
- assert!(
- matches!(err, EvalError::InvalidCatalogJson(msg) if msg.contains("valid hex")),
- "wrong error variant/message"
- );
- }
-
- #[mz_ore::test]
- fn pg_malformed_proto_errors() {
- // Valid hex, garbage bytes. Prost decoding fails on unexpected wire
- // format.
- let sql = pg_source_sql("ffff");
- let err = super::parse_postgres_source_details(&sql).unwrap_err();
- assert!(
- matches!(err, EvalError::InvalidCatalogJson(msg) if msg.contains("publication-details proto")),
- "wrong error variant/message"
- );
- }
-
- // --- parse_kafka_source_details ------------------------------------------
-
- #[mz_ore::test]
- fn kafka_happy_path_with_prefix() {
- let sql = kafka_source_sql(true);
- let out = super::parse_kafka_source_details(&sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "topic": "test",
- "group_id_prefix": "my-prefix-",
- "connection_id": "u11",
- }),
- );
- }
-
- #[mz_ore::test]
- fn kafka_happy_path_without_prefix() {
- let sql = kafka_source_sql(false);
- let out = super::parse_kafka_source_details(&sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "topic": "test",
- "group_id_prefix": null,
- "connection_id": "u11",
- }),
- );
- }
-
- #[mz_ore::test]
- fn kafka_non_kafka_source_returns_null_jsonb() {
- let sql = "CREATE SOURCE \"materialize\".\"public\".\"lg\" \
- IN CLUSTER [u42] FROM LOAD GENERATOR COUNTER";
- let out = super::parse_kafka_source_details(sql).expect("ok");
- assert_eq!(as_serde(out), serde_json::Value::Null);
- }
-
- #[mz_ore::test]
- fn kafka_missing_topic_errors() {
- let sql = "CREATE SOURCE \"materialize\".\"public\".\"k_src\" \
- IN CLUSTER [u42] \
- FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \
- FORMAT TEXT";
- let err = super::parse_kafka_source_details(sql).unwrap_err();
- assert!(
- matches!(err, EvalError::InvalidCatalogJson(msg) if msg.contains("missing TOPIC")),
- "wrong error variant/message"
- );
- }
-
- #[mz_ore::test]
- fn kafka_unquoted_topic_and_prefix() {
- // Planning accepts a bare identifier for TOPIC / GROUP ID PREFIX, and it
- // persists unquoted in create_sql. Matching only quoted strings would
- // drop TOPIC and error the whole mz_kafka_source_tables view.
- let sql = "CREATE SOURCE \"materialize\".\"public\".\"k_src\" \
- IN CLUSTER [u42] \
- FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \
- (TOPIC = my_topic, GROUP ID PREFIX = my_prefix) FORMAT TEXT";
- let out = super::parse_kafka_source_details(sql).expect("ok");
- let out = as_serde(out);
- assert_eq!(out["topic"], json!("my_topic"));
- assert_eq!(out["group_id_prefix"], json!("my_prefix"));
- }
-
- #[mz_ore::test]
- fn kafka_unresolved_connection_errors() {
- // A bare-name connection reference never happens after purification,
- // but the decoder must reject it explicitly rather than silently
- // dropping the connection_id.
- let sql = "CREATE SOURCE \"materialize\".\"public\".\"k_src\" \
- IN CLUSTER [u42] \
- FROM KAFKA CONNECTION k_conn (TOPIC 'test') FORMAT TEXT";
- let err = super::parse_kafka_source_details(sql).unwrap_err();
- assert!(
- matches!(err, EvalError::InvalidCatalogJson(msg) if msg.contains("unresolved item name")),
- "wrong error variant/message"
- );
- }
-
- // --- parse_source_export_details -----------------------------------------
-
- fn table_from_source_sql(reference: &str, suffix: &str) -> String {
- format!(
- "CREATE TABLE \"materialize\".\"public\".\"tbl\" \
- FROM SOURCE [u1 AS \"materialize\".\"public\".\"src\"] \
- (REFERENCE = {reference}){suffix}"
- )
- }
-
- #[mz_ore::test]
- fn export_table_postgres_style_no_format() {
- // Postgres/mysql/sql-server tables carry a multi-part external
- // reference and no format or envelope.
- let sql = table_from_source_sql("\"db\".\"public\".\"t\"", "");
- let out = super::parse_source_export_details(&sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "source_id": "u1",
- "external_reference": ["db", "public", "t"],
- "envelope_type": null,
- "key_format": null,
- "value_format": null,
- }),
- );
- }
-
- #[mz_ore::test]
- fn export_table_kafka_bare_value_only() {
- // A bare non-registry FORMAT is value-only: no key format.
- let sql = table_from_source_sql("\"topic\"", " FORMAT TEXT ENVELOPE NONE");
- let out = super::parse_source_export_details(&sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "source_id": "u1",
- "external_reference": ["topic"],
- "envelope_type": "none",
- "key_format": null,
- "value_format": "text",
- }),
- );
- }
-
- #[mz_ore::test]
- fn export_table_kafka_omitted_envelope_is_null() {
- // Omitting ENVELOPE persists as absent in create_sql, so this
- // source-type-agnostic helper reports null. The mz_kafka_source_tables
- // view is responsible for defaulting kafka's null envelope to 'none'.
- let sql = table_from_source_sql("\"topic\"", " FORMAT TEXT");
- let out = super::parse_source_export_details(&sql).expect("ok");
- assert_eq!(as_serde(out)["envelope_type"], serde_json::Value::Null);
- }
-
- #[mz_ore::test]
- fn export_table_kafka_key_value_format() {
- let sql = table_from_source_sql(
- "\"topic\"",
- " KEY FORMAT TEXT VALUE FORMAT TEXT ENVELOPE NONE",
- );
- let out = super::parse_source_export_details(&sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "source_id": "u1",
- "external_reference": ["topic"],
- "envelope_type": "none",
- "key_format": "text",
- "value_format": "text",
- }),
- );
- }
-
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn export_table_kafka_bare_avro_seed_with_key() {
- // A bare Avro CSR format whose seed carries a key schema resolves to
- // an encoding with a key, so key_format mirrors value_format. This is
- // the upsert/debezium path.
- let sql = table_from_source_sql(
- "\"topic\"",
- " FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY \
- CONNECTION [u5 AS \"materialize\".\"public\".\"csr\"] \
- SEED KEY SCHEMA 'k' VALUE SCHEMA 'v' ENVELOPE UPSERT",
- );
- let out = super::parse_source_export_details(&sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "source_id": "u1",
- "external_reference": ["topic"],
- "envelope_type": "upsert",
- "key_format": "avro",
- "value_format": "avro",
- }),
- );
- }
-
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn export_table_kafka_bare_avro_seed_without_key() {
- // A bare Avro CSR seed with only a value schema is value-only.
- let sql = table_from_source_sql(
- "\"topic\"",
- " FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY \
- CONNECTION [u5 AS \"materialize\".\"public\".\"csr\"] \
- SEED VALUE SCHEMA 'v' ENVELOPE NONE",
- );
- let out = super::parse_source_export_details(&sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "source_id": "u1",
- "external_reference": ["topic"],
- "envelope_type": "none",
- "key_format": null,
- "value_format": "avro",
- }),
- );
- }
-
- #[mz_ore::test]
- fn export_subsource_non_progress() {
- // Old-syntax subsource: external reference lives in a WITH option, and
- // there is never a format or envelope.
- let sql = "CREATE SUBSOURCE \"materialize\".\"public\".\"sub\" (id int4) \
- OF SOURCE [u1 AS \"materialize\".\"public\".\"src\"] \
- WITH (EXTERNAL REFERENCE = \"db\".\"public\".\"t\")";
- let out = super::parse_source_export_details(sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "source_id": "u1",
- "external_reference": ["db", "public", "t"],
- "envelope_type": null,
- "key_format": null,
- "value_format": null,
- }),
- );
- }
-
- #[mz_ore::test]
- fn export_progress_subsource_returns_null_jsonb() {
- let sql = "CREATE SUBSOURCE \"materialize\".\"public\".\"progress\" (id int4) \
- WITH (PROGRESS)";
- let out = super::parse_source_export_details(sql).expect("ok");
- assert_eq!(as_serde(out), serde_json::Value::Null);
- }
-
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn export_non_source_export_returns_null_jsonb() {
- let sql = "CREATE VIEW v AS SELECT 1";
- let out = super::parse_source_export_details(sql).expect("ok");
- assert_eq!(as_serde(out), serde_json::Value::Null);
- }
-
- #[mz_ore::test]
- fn export_unresolved_source_name_errors() {
- let sql = "CREATE TABLE \"materialize\".\"public\".\"tbl\" \
- FROM SOURCE src (REFERENCE = \"topic\")";
- let err = super::parse_source_export_details(sql).unwrap_err();
- assert!(
- matches!(err, EvalError::InvalidCatalogJson(msg) if msg.contains("unresolved item name")),
- "wrong error variant/message"
- );
- }
-
- // --- parse_connection_details --------------------------------------------
-
- #[mz_ore::test]
- fn connection_kafka_single_broker_default_progress() {
- // No explicit PROGRESS TOPIC: the helper leaves it null and the view
- // reconstructs the default.
- let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \
- (BROKER = 'localhost:9092', SECURITY PROTOCOL = plaintext)";
- let out = super::parse_connection_details(sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "brokers": ["localhost:9092"],
- "progress_topic": null,
- }),
- );
- }
-
- #[mz_ore::test]
- fn connection_kafka_explicit_progress_topic() {
- let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \
- (BROKER = 'localhost:9092', PROGRESS TOPIC = 'override', \
- SECURITY PROTOCOL = plaintext)";
- let out = super::parse_connection_details(sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "brokers": ["localhost:9092"],
- "progress_topic": "override",
- }),
- );
- }
-
- #[mz_ore::test]
- fn connection_kafka_broker_list() {
- let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \
- (BROKERS ('b1:9092', 'b2:9092'), SECURITY PROTOCOL = plaintext)";
- let out = super::parse_connection_details(sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "brokers": ["b1:9092", "b2:9092"],
- "progress_topic": null,
- }),
- );
- }
-
- #[mz_ore::test]
- fn connection_ssh_public_keys() {
- let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO SSH TUNNEL \
- (HOST = 'ssh.example.com', PORT = 22, USER = 'mz', \
- PUBLIC KEY 1 = 'ssh-ed25519 AAAA', PUBLIC KEY 2 = 'ssh-ed25519 BBBB')";
- let out = super::parse_connection_details(sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "public_key_1": "ssh-ed25519 AAAA",
- "public_key_2": "ssh-ed25519 BBBB",
- }),
- );
- }
-
- #[mz_ore::test]
- fn connection_aws_credentials_inline_key() {
- // Inline ACCESS KEY ID, secret SECRET ACCESS KEY. Assume-role columns
- // stay null and auth_kind is credentials.
- let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \
- (ACCESS KEY ID = 'AKIAEXAMPLE', \
- SECRET ACCESS KEY = SECRET [u1 AS \"materialize\".\"public\".\"sk\"])";
- let out = super::parse_connection_details(sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "auth_kind": "credentials",
- "endpoint": null,
- "region": null,
- "access_key_id": "AKIAEXAMPLE",
- "access_key_id_secret_id": null,
- "secret_access_key_secret_id": "u1",
- "session_token": null,
- "session_token_secret_id": null,
- "assume_role_arn": null,
- "assume_role_session_name": null,
- }),
- );
- }
-
- #[mz_ore::test]
- fn connection_aws_credentials_secret_key_and_session_token() {
- // Every credential provided as a secret reference lands in the matching
- // *_secret_id column as the referenced secret's catalog id.
- let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \
- (ENDPOINT = 'http://localhost', REGION = 'us-east-1', \
- ACCESS KEY ID = SECRET [u1 AS \"materialize\".\"public\".\"ak\"], \
- SECRET ACCESS KEY = SECRET [u2 AS \"materialize\".\"public\".\"sk\"], \
- SESSION TOKEN = SECRET [u3 AS \"materialize\".\"public\".\"st\"])";
- let out = super::parse_connection_details(sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "auth_kind": "credentials",
- "endpoint": "http://localhost",
- "region": "us-east-1",
- "access_key_id": null,
- "access_key_id_secret_id": "u1",
- "secret_access_key_secret_id": "u2",
- "session_token": null,
- "session_token_secret_id": "u3",
- "assume_role_arn": null,
- "assume_role_session_name": null,
- }),
- );
- }
-
- #[mz_ore::test]
- fn connection_aws_assume_role() {
- // Assume-role sets auth_kind and the assume-role columns; credential
- // columns stay null.
- let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \
- (ASSUME ROLE ARN 'arn:aws:iam::123:role/mz', \
- ASSUME ROLE SESSION NAME 'sess')";
- let out = super::parse_connection_details(sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "auth_kind": "assume-role",
- "endpoint": null,
- "region": null,
- "access_key_id": null,
- "access_key_id_secret_id": null,
- "secret_access_key_secret_id": null,
- "session_token": null,
- "session_token_secret_id": null,
- "assume_role_arn": "arn:aws:iam::123:role/mz",
- "assume_role_session_name": "sess",
- }),
- );
- }
-
- #[mz_ore::test]
- fn connection_kafka_unquoted_progress_topic() {
- // A bare identifier PROGRESS TOPIC persists unquoted in create_sql. The
- // helper must surface it, else the view substitutes the default topic.
- let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO KAFKA \
- (BROKER = 'localhost:9092', PROGRESS TOPIC = my_topic, \
- SECURITY PROTOCOL = plaintext)";
- let out = super::parse_connection_details(sql).expect("ok");
- assert_eq!(as_serde(out)["progress_topic"], json!("my_topic"));
- }
-
- #[mz_ore::test]
- fn connection_aws_unquoted_option_values() {
- // Planning accepts bare identifiers for these options and persists them
- // unquoted. The helper must surface them, not fall back to NULL.
- let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \
- (ENDPOINT = localhost, REGION = useast1, \
- ASSUME ROLE ARN 'arn:aws:iam::123:role/mz', \
- ASSUME ROLE SESSION NAME = mysession)";
- let out = super::parse_connection_details(sql).expect("ok");
- let out = as_serde(out);
- assert_eq!(out["endpoint"], json!("localhost"));
- assert_eq!(out["region"], json!("useast1"));
- assert_eq!(out["assume_role_session_name"], json!("mysession"));
- }
-
- #[mz_ore::test]
- fn connection_aws_empty_endpoint_is_null() {
- // Planning coerces ENDPOINT = '' to None, so the packer wrote NULL. The
- // view must match, not report an empty string.
- let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO AWS \
- (ENDPOINT = '', ASSUME ROLE ARN 'arn:aws:iam::123:role/mz')";
- let out = super::parse_connection_details(sql).expect("ok");
- assert_eq!(as_serde(out)["endpoint"], serde_json::Value::Null);
- }
-
- #[mz_ore::test]
- fn connection_other_type_returns_null_jsonb() {
- // A connection type without a detail view (postgres) yields null.
- let sql = "CREATE CONNECTION \"materialize\".\"public\".\"c\" TO POSTGRES \
- (HOST = 'db', DATABASE = 'postgres', USER = 'mz')";
- let out = super::parse_connection_details(sql).expect("ok");
- assert_eq!(as_serde(out), serde_json::Value::Null);
- }
-
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn connection_non_connection_returns_null_jsonb() {
- let sql = "CREATE VIEW v AS SELECT 1";
- let out = super::parse_connection_details(sql).expect("ok");
- assert_eq!(as_serde(out), serde_json::Value::Null);
- }
-
- // --- parse_catalog_create_sql envelope_type ------------------------------
-
- #[mz_ore::test]
- fn catalog_kafka_old_syntax_omitted_envelope_defaults_none() {
- // An old-syntax kafka source (carries EXPOSE PROGRESS AS) ingests into
- // its own relation, so an omitted ENVELOPE means the default NONE, which
- // the pre-MV packer reported as 'none'.
- let sql = "CREATE SOURCE \"materialize\".\"public\".\"k\" \
- IN CLUSTER [u42] \
- FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \
- (TOPIC 'test') FORMAT TEXT \
- EXPOSE PROGRESS AS [u12 AS \"materialize\".\"public\".\"k_progress\"]";
- let out = super::parse_catalog_create_sql(sql).expect("ok");
- assert_eq!(as_serde(out)["envelope_type"], json!("none"));
- }
-
- #[mz_ore::test]
- fn catalog_kafka_old_syntax_explicit_envelope() {
- let sql = "CREATE SOURCE \"materialize\".\"public\".\"k\" \
- IN CLUSTER [u42] \
- FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"] \
- (TOPIC 'test') FORMAT BYTES ENVELOPE UPSERT \
- EXPOSE PROGRESS AS [u12 AS \"materialize\".\"public\".\"k_progress\"]";
- let out = super::parse_catalog_create_sql(sql).expect("ok");
- assert_eq!(as_serde(out)["envelope_type"], json!("upsert"));
- }
-
- #[mz_ore::test]
- fn catalog_kafka_new_syntax_source_omits_envelope_type() {
- // A new-syntax kafka source has no progress subsource (no EXPOSE PROGRESS
- // AS). It ingests nothing itself. Envelopes live on the per-table exports,
- // so its own envelope_type stays absent (SQL NULL).
- let sql = "CREATE SOURCE \"materialize\".\"public\".\"k\" \
- IN CLUSTER [u42] \
- FROM KAFKA CONNECTION [u11 AS \"materialize\".\"public\".\"k_conn\"]";
- let out = super::parse_catalog_create_sql(sql).expect("ok");
- assert_eq!(as_serde(out).get("envelope_type"), None);
- }
-
- #[mz_ore::test]
- fn catalog_non_kafka_source_omits_envelope_type() {
- // Non-kafka sources carry no envelope, so envelope_type stays absent
- // (SQL NULL), not 'none'.
- let sql = "CREATE SOURCE \"materialize\".\"public\".\"lg\" \
- IN CLUSTER [u42] FROM LOAD GENERATOR COUNTER";
- let out = super::parse_catalog_create_sql(sql).expect("ok");
- assert_eq!(as_serde(out).get("envelope_type"), None);
- }
-
- // --- parse_catalog_create_sql, CreateSink arm ----------------------------
-
- const AVRO_FORMAT: &str = "FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION \
- [u12 AS \"materialize\".\"public\".\"csr_conn\"]";
-
- /// A persisted kafka-sink `create_sql`: resolved names, and a `TOPIC` that
- /// planning guarantees.
- fn kafka_sink_sql(key: Option<&str>, format: &str, envelope: &str) -> String {
- let key_clause = key.map(|k| format!(" KEY ({k})")).unwrap_or_default();
- format!(
- "CREATE SINK \"materialize\".\"public\".\"snk\" \
- IN CLUSTER [u42] \
- FROM [u1 AS \"materialize\".\"public\".\"t\"] \
- INTO KAFKA CONNECTION [u10 AS \"materialize\".\"public\".\"k_conn\"] \
- (TOPIC 'sink-topic'){key_clause} {format} ENVELOPE {envelope}"
- )
- }
-
- fn iceberg_sink_sql(mode: &str) -> String {
- format!(
- "CREATE SINK \"materialize\".\"public\".\"ice\" \
- IN CLUSTER [u42] \
- FROM [u1 AS \"materialize\".\"public\".\"t\"] \
- INTO ICEBERG CATALOG CONNECTION [u20 AS \"materialize\".\"public\".\"cat_conn\"] \
- (NAMESPACE 'ns', TABLE 'tbl') \
- USING AWS CONNECTION [u21 AS \"materialize\".\"public\".\"aws_conn\"] \
- MODE {mode}"
- )
- }
-
- #[mz_ore::test]
- fn sink_kafka_bare_format_without_key() {
- let sql = kafka_sink_sql(None, "FORMAT JSON", "DEBEZIUM");
- let out = super::parse_catalog_create_sql(&sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "type": "sink",
- "sink_type": "kafka",
- "cluster_id": "u42",
- "connection_id": "u10",
- "topic": "sink-topic",
- "envelope_type": "debezium",
- "format": "json",
- "value_format": "json",
- }),
- );
- }
-
- #[mz_ore::test]
- fn sink_kafka_bare_format_with_key_derives_key_format() {
- // A bare format applies to the key too once the sink has a KEY, which
- // is what makes the deprecated `format` column collapse to `avro`.
- let sql = kafka_sink_sql(Some("a"), AVRO_FORMAT, "UPSERT");
- let out = super::parse_catalog_create_sql(&sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "type": "sink",
- "sink_type": "kafka",
- "cluster_id": "u42",
- "connection_id": "u10",
- "topic": "sink-topic",
- "envelope_type": "upsert",
- "format": "avro",
- "key_format": "avro",
- "value_format": "avro",
- }),
- );
- }
-
- #[mz_ore::test]
- fn sink_kafka_bare_text_format_with_key_does_not_collapse() {
- // Only avro/avro and json/json collapse, so a keyed text sink reports
- // the composite form even though both halves are `text`.
- let sql = kafka_sink_sql(Some("a"), "FORMAT TEXT", "UPSERT");
- let out = super::parse_catalog_create_sql(&sql).expect("ok");
- let out = as_serde(out);
- assert_eq!(out["format"], json!("key-text-value-text"));
- assert_eq!(out["key_format"], json!("text"));
- assert_eq!(out["value_format"], json!("text"));
- }
-
- #[mz_ore::test]
- fn sink_kafka_key_value_json_collapses() {
- let sql = kafka_sink_sql(Some("a"), "KEY FORMAT JSON VALUE FORMAT JSON", "UPSERT");
- let out = as_serde(super::parse_catalog_create_sql(&sql).expect("ok"));
- assert_eq!(out["format"], json!("json"));
- assert_eq!(out["key_format"], json!("json"));
- assert_eq!(out["value_format"], json!("json"));
- }
-
- #[mz_ore::test]
- fn sink_kafka_key_value_mixed_is_composite() {
- let sql = kafka_sink_sql(Some("a"), "KEY FORMAT TEXT VALUE FORMAT BYTES", "UPSERT");
- let out = as_serde(super::parse_catalog_create_sql(&sql).expect("ok"));
- assert_eq!(out["format"], json!("key-text-value-bytes"));
- assert_eq!(out["key_format"], json!("text"));
- assert_eq!(out["value_format"], json!("bytes"));
- }
-
- #[mz_ore::test]
- fn sink_kafka_key_format_without_key_is_dropped() {
- // `kafka_sink_builder` ignores the key half of the format spec when the
- // sink has no KEY, so neither `key_format` nor the composite `format`
- // may reflect it.
- let sql = kafka_sink_sql(None, "KEY FORMAT JSON VALUE FORMAT TEXT", "DEBEZIUM");
- let out = as_serde(super::parse_catalog_create_sql(&sql).expect("ok"));
- assert_eq!(out["format"], json!("text"));
- assert_eq!(out["key_format"], serde_json::Value::Null);
- assert_eq!(out["value_format"], json!("text"));
- }
-
- #[mz_ore::test]
- fn sink_kafka_missing_topic_errors() {
- let sql = "CREATE SINK \"materialize\".\"public\".\"snk\" \
- IN CLUSTER [u42] \
- FROM [u1 AS \"materialize\".\"public\".\"t\"] \
- INTO KAFKA CONNECTION [u10 AS \"materialize\".\"public\".\"k_conn\"] \
- FORMAT JSON ENVELOPE DEBEZIUM";
- let err = super::parse_catalog_create_sql(sql).unwrap_err();
- assert!(
- matches!(err, EvalError::InvalidCatalogJson(msg) if msg.contains("missing TOPIC")),
- "wrong error variant/message"
- );
- }
-
- #[mz_ore::test]
- fn sink_iceberg_upsert_mode() {
- let sql = iceberg_sink_sql("UPSERT");
- let out = super::parse_catalog_create_sql(&sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({
- "type": "sink",
- "sink_type": "iceberg",
- "cluster_id": "u42",
- // The catalog connection, never the AWS connection (u21).
- "connection_id": "u20",
- "namespace": "ns",
- "table": "tbl",
- "envelope_type": "upsert",
- }),
- );
- }
-
- #[mz_ore::test]
- fn sink_iceberg_append_mode() {
- let sql = iceberg_sink_sql("APPEND");
- let out = as_serde(super::parse_catalog_create_sql(&sql).expect("ok"));
- // `append` is reachable only through an iceberg sink's MODE.
- assert_eq!(out["envelope_type"], json!("append"));
- }
-
- #[mz_ore::test]
- fn sink_iceberg_missing_table_errors() {
- let sql = "CREATE SINK \"materialize\".\"public\".\"ice\" \
- IN CLUSTER [u42] \
- FROM [u1 AS \"materialize\".\"public\".\"t\"] \
- INTO ICEBERG CATALOG CONNECTION [u20 AS \"materialize\".\"public\".\"cat_conn\"] \
- (NAMESPACE 'ns') MODE UPSERT";
- let err = super::parse_catalog_create_sql(sql).unwrap_err();
- assert!(
- matches!(err, EvalError::InvalidCatalogJson(msg) if msg.contains("missing TABLE")),
- "wrong error variant/message"
- );
- }
-
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn sink_arm_leaves_other_item_types_alone() {
- let sql = "CREATE VIEW \"materialize\".\"public\".\"v\" AS SELECT 1";
- let out = super::parse_catalog_create_sql(sql).expect("ok");
- assert_eq!(
- as_serde(out),
- json!({ "type": "view", "definition": "SELECT 1;" })
- );
- }
-
- // --- parse_catalog_create_sql --------------------------------------------
-
- /// `type` for a `create_sql`, or the error message if parsing failed.
- fn item_type(sql: &str) -> Result {
- match super::parse_catalog_create_sql(sql) {
- Ok(out) => match as_serde(out) {
- serde_json::Value::Object(mut m) => match m.remove("type") {
- Some(serde_json::Value::String(s)) => Ok(s),
- other => panic!("no string `type` key: {other:?}"),
- },
- other => panic!("not a JSON object: {other:?}"),
- },
- Err(EvalError::InvalidCatalogJson(msg)) => Err(msg.to_string()),
- Err(e) => panic!("unexpected error variant: {e:?}"),
- }
- }
-
- fn view_sql(query: &str) -> String {
- format!("CREATE VIEW \"materialize\".\"public\".\"v\" AS {query}")
- }
-
- /// `definition` for a `CREATE VIEW` whose query is `query`.
- fn view_definition(query: &str) -> String {
- match as_serde(super::parse_catalog_create_sql(&view_sql(query)).expect("ok")) {
- serde_json::Value::Object(mut m) => match m.remove("definition") {
- Some(serde_json::Value::String(s)) => s,
- other => panic!("no string `definition` key: {other:?}"),
- },
- other => panic!("not a JSON object: {other:?}"),
- }
- }
-
- /// `mz_tables` and `mz_views` select rows by
- /// `parse_catalog_create_sql(...)->>'type'`, and the function runs over
- /// every `Item` row in the catalog, so a statement kind that changes its
- /// reported type silently gains or loses rows in those relations. Pin the
- /// type of every kind the catalog can hold.
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn catalog_item_type_per_statement_kind() {
- let cases = [
- (
- "CREATE TABLE \"materialize\".\"public\".\"t\" (a int4)",
- "table",
- ),
- // A table created from a source, and a webhook table, are both
- // `table`, so both land in mz_tables.
- (
- "CREATE TABLE \"materialize\".\"public\".\"tbl\" \
- FROM SOURCE [u1 AS \"materialize\".\"public\".\"src\"] \
- (REFERENCE = \"topic\") FORMAT TEXT",
- "table",
- ),
- (
- "CREATE TABLE \"materialize\".\"public\".\"wht\" FROM WEBHOOK BODY FORMAT JSON",
- "table",
- ),
- (
- "CREATE VIEW \"materialize\".\"public\".\"v\" AS SELECT 1",
- "view",
- ),
- (
- "CREATE MATERIALIZED VIEW \"materialize\".\"public\".\"mv\" \
- IN CLUSTER [u1] AS SELECT 1",
- "materialized-view",
- ),
- (
- "CREATE SOURCE \"materialize\".\"public\".\"lg\" \
- IN CLUSTER [u1] FROM LOAD GENERATOR COUNTER",
- "source",
- ),
- (
- "CREATE SOURCE \"materialize\".\"public\".\"wh\" \
- IN CLUSTER [u1] FROM WEBHOOK BODY FORMAT JSON",
- "source",
- ),
- (
- "CREATE SUBSOURCE \"materialize\".\"public\".\"sub\" (id int4) \
- OF SOURCE [u1 AS \"materialize\".\"public\".\"src\"]",
- "subsource",
- ),
- (
- "CREATE SUBSOURCE \"materialize\".\"public\".\"progress\" (id int4) \
- WITH (PROGRESS)",
- "subsource",
- ),
- (
- "CREATE SINK \"materialize\".\"public\".\"snk\" IN CLUSTER [u1] \
- FROM [u1 AS \"materialize\".\"public\".\"t\"] \
- INTO KAFKA CONNECTION [u2 AS \"materialize\".\"public\".\"c\"] \
- (TOPIC 'tp') FORMAT JSON ENVELOPE DEBEZIUM",
- "sink",
- ),
- (
- "CREATE INDEX \"i\" IN CLUSTER [u1] \
- ON [u1 AS \"materialize\".\"public\".\"t\"] (\"a\")",
- "index",
- ),
- (
- "CREATE TYPE \"materialize\".\"public\".\"ty\" AS LIST (ELEMENT TYPE = int4)",
- "type",
- ),
- (
- "CREATE SECRET \"materialize\".\"public\".\"s\" AS 'x'",
- "secret",
- ),
- (
- "CREATE CONNECTION \"materialize\".\"public\".\"c\" \
- TO KAFKA (BROKER 'b', SECURITY PROTOCOL PLAINTEXT)",
- "connection",
- ),
- ];
- for (sql, expected) in cases {
- assert_eq!(item_type(sql).as_deref(), Ok(expected), "for {sql}");
- }
- }
-
- /// `mz_views.definition` is produced here. It used to be produced by
- /// `pack_view_update` in the adapter, so the exact rendering is a
- /// compatibility surface: `pg_views.definition` reads it.
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn catalog_view_definition() {
- // Identifiers and function names come back fully quoted, literals
- // untouched, and PostgreSQL's trailing semicolon is appended.
- assert_eq!(view_definition("SELECT 1"), "SELECT 1;");
- assert_eq!(
- view_definition("WITH c AS (SELECT 1 AS a) SELECT a FROM c"),
- "WITH \"c\" AS (SELECT 1 AS \"a\") SELECT \"a\" FROM \"c\";"
- );
- assert_eq!(
- view_definition("SELECT 1 UNION ALL SELECT 2"),
- "SELECT 1 UNION ALL SELECT 2;"
- );
- assert_eq!(
- view_definition("SELECT (SELECT max(a) FROM [u1 AS \"materialize\".\"public\".\"t\"])"),
- "SELECT (SELECT \"max\"(\"a\") FROM [u1 AS \"materialize\".\"public\".\"t\"]);"
- );
- // Identifiers needing quotes, an embedded double quote, non-ASCII, an
- // embedded single quote in a literal, and ORDER BY all survive.
- assert_eq!(
- view_definition(
- "SELECT \"a b\", \"héllo\", \"q\"\"x\" \
- FROM [u1 AS \"materialize\".\"public\".\"t\"] \
- WHERE s = 'lit''eral' AND n = 42 ORDER BY 1"
- ),
- "SELECT \"a b\", \"héllo\", \"q\"\"x\" \
- FROM [u1 AS \"materialize\".\"public\".\"t\"] \
- WHERE \"s\" = 'lit''eral' AND \"n\" = 42 ORDER BY 1;"
- );
- }
-
- /// The rendering must be a fixed point: `pg_views` consumers re-issue
- /// `definition` as the body of a new view, so a second pass through the
- /// parser has to produce the identical string. The trailing `;` is part of
- /// what gets re-parsed.
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn catalog_view_definition_is_idempotent() {
- for query in [
- "SELECT 1",
- "WITH c AS (SELECT 1 AS a) SELECT a FROM c",
- "SELECT 1 UNION ALL SELECT 2",
- "SELECT \"a b\", \"q\"\"x\" FROM [u1 AS \"materialize\".\"public\".\"t\"] \
- WHERE s = 'lit''eral' ORDER BY 1",
- ] {
- let once = view_definition(query);
- assert_eq!(
- view_definition(&once),
- once,
- "not a fixed point for {query}"
- );
- }
- }
-
- /// `mz_tables.source_id` comes from this key. A table with no source must
- /// omit it entirely, so the MV's `->>'source_id'` yields SQL NULL.
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn catalog_table_source_id() {
- let from_source = as_serde(
- super::parse_catalog_create_sql(
- "CREATE TABLE \"materialize\".\"public\".\"tbl\" \
- FROM SOURCE [u1 AS \"materialize\".\"public\".\"src\"] \
- (REFERENCE = \"topic\") FORMAT TEXT",
- )
- .expect("ok"),
- );
- assert_eq!(from_source, json!({ "type": "table", "source_id": "u1" }));
-
- for sql in [
- "CREATE TABLE \"materialize\".\"public\".\"t\" (a int4)",
- "CREATE TABLE \"materialize\".\"public\".\"wht\" FROM WEBHOOK BODY FORMAT JSON",
- ] {
- assert_eq!(
- as_serde(super::parse_catalog_create_sql(sql).expect("ok")),
- json!({ "type": "table" }),
- "for {sql}"
- );
- }
- }
-
- /// Every error here is fatal to the whole of `mz_tables`/`mz_views`, not to
- /// one row: the MVs call this function inside their `WHERE` clause, so an
- /// item the parser rejects makes the relation unreadable for everyone.
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn catalog_create_sql_errors() {
- assert_eq!(
- item_type("this is not sql"),
- Err(
- "failed to parse create_sql: Expected a keyword at the beginning of a statement, \
- found identifier \"this\""
- .to_string()
- )
- );
- assert_eq!(
- item_type("CREATE TABLE t (a int4); CREATE TABLE u (b int4)"),
- Err("expected a single statement, found 2".to_string())
- );
- // A statement that is not a CREATE of a catalog item, e.g. if a future
- // change persists something else in an Item record.
- assert_eq!(
- item_type("SELECT 1"),
- Err("not a CREATE item statement".to_string())
- );
- // Catalog `create_sql` always names items by id. An unresolved name
- // means the record was written wrong.
- assert_eq!(
- item_type(
- "CREATE TABLE \"materialize\".\"public\".\"tbl\" \
- FROM SOURCE src (REFERENCE = \"topic\")"
- ),
- Err("unresolved item name".to_string())
- );
- }
-
- // --- parse_catalog_item_references ----------------------------------------
-
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn item_references_view() {
- let sql = "CREATE VIEW \"materialize\".\"public\".\"v\" AS \
- SELECT \"pg_catalog\".\"abs\"(\"a\"::[s20 AS \"pg_catalog\".\"int4\"]) \
- FROM [u1 AS \"materialize\".\"public\".\"t\"]";
- let out = as_serde(super::parse_catalog_item_references(sql).expect("ok"));
- assert_eq!(
- out,
- json!({
- "ids": ["s20", "u1"],
- "named_funcs": [{"schema": "pg_catalog", "name": "abs"}],
- "named_types": [],
- "named_relations": [],
- })
- );
- }
-
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn item_references_connection() {
- let sql = "CREATE CONNECTION \"materialize\".\"public\".\"kc\" TO KAFKA (\
- BROKER 'kafka:9092', \
- SSH TUNNEL [u5 AS \"materialize\".\"public\".\"ssh\"], \
- SASL PASSWORD = SECRET [u7 AS \"materialize\".\"public\".\"pw\"])";
- let out = as_serde(super::parse_catalog_item_references(sql).expect("ok"));
- assert_eq!(out["ids"], json!(["u5", "u7"]));
- assert_eq!(out["named_funcs"], json!([]));
- }
-
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn item_references_cte_shadowing() {
- let sql = "CREATE VIEW \"materialize\".\"public\".\"v\" AS \
- WITH \"c\" AS (SELECT * FROM [u1 AS \"materialize\".\"public\".\"t\"]) \
- SELECT * FROM \"c\"";
- let out = as_serde(super::parse_catalog_item_references(sql).expect("ok"));
- assert_eq!(out["ids"], json!(["u1"]));
- assert_eq!(out["named_relations"], json!([]));
- }
-
- #[mz_ore::test]
- #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `rust_psm_stack_pointer` on OS `linux`
- fn item_references_name_based_relation_surfaces() {
- // Stored SQL always prints relations as id references. A name-based
- // relation must surface in "named_relations" rather than vanish.
- let sql = "CREATE VIEW \"materialize\".\"public\".\"v\" AS \
- SELECT * FROM \"mz_catalog\".\"mz_tables\"";
- let out = as_serde(super::parse_catalog_item_references(sql).expect("ok"));
- assert_eq!(
- out["named_relations"],
- json!([{"schema": "mz_catalog", "name": "mz_tables"}])
- );
- }
-
- #[mz_ore::test]
- fn item_references_parse_error() {
- let err = super::parse_catalog_item_references("NOT SQL").unwrap_err();
- assert!(
- matches!(err, EvalError::InvalidCatalogJson(msg) if msg.contains("failed to parse")),
- "wrong error variant/message"
- );
- }
-}
diff --git a/src/pgrepr-consts/src/oid.rs b/src/pgrepr-consts/src/oid.rs
index 5d0e360c6459f..3740c57458562 100644
--- a/src/pgrepr-consts/src/oid.rs
+++ b/src/pgrepr-consts/src/oid.rs
@@ -732,7 +732,7 @@ pub const INDEX_MZ_OBJECTS_IND_OID: u32 = 17011;
pub const INDEX_MZ_COLUMNS_IND_OID: u32 = 17012;
pub const INDEX_MZ_SECRETS_IND_OID: u32 = 17013;
pub const INDEX_MZ_VIEWS_IND_OID: u32 = 17014;
-pub const TABLE_MZ_SOURCE_REFERENCES_OID: u32 = 17015;
+pub const MV_MZ_SOURCE_REFERENCES_OID: u32 = 17015;
pub const SOURCE_MZ_CLUSTER_REPLICA_METRICS_HISTORY_OID: u32 = 17016;
pub const INDEX_MZ_CLUSTER_REPLICA_METRICS_HISTORY_IND_OID: u32 = 17017;
pub const VIEW_MZ_CLUSTER_REPLICA_UTILIZATION_HISTORY_OID: u32 = 17018;
diff --git a/test/kafka-matrix/mzcompose.py b/test/kafka-matrix/mzcompose.py
index eb0e54a08d1a0..8547fec8f98cf 100644
--- a/test/kafka-matrix/mzcompose.py
+++ b/test/kafka-matrix/mzcompose.py
@@ -56,7 +56,14 @@
]
SERVICES = [
- Materialized(default_replication_factor=2),
+ Materialized(
+ default_replication_factor=2,
+ # Curated metric sinks make the first catalog status queries after a
+ # restart take minutes on 4-CPU agents, which times out this suite.
+ # The flag defaults off in production.
+ # TODO(SQL-730): remove once the post-restart slowdown is fixed.
+ additional_system_parameter_defaults={"enable_metric_sink": "false"},
+ ),
# Occasional timeouts in CI with 60s timeout
Testdrive(
volumes_extra=["../testdrive:/workdir/testdrive"], default_timeout="120s"
diff --git a/test/lang/python/requirements.txt b/test/lang/python/requirements.txt
index 4f1b86acc565b..f71938206eec1 100644
--- a/test/lang/python/requirements.txt
+++ b/test/lang/python/requirements.txt
@@ -1,4 +1,4 @@
psycopg==3.3.6
psycopg-binary==3.3.6
psycopg2==2.9.13
-SQLAlchemy==2.0.54
+SQLAlchemy==2.1.0
diff --git a/test/sqllogictest/catalog_server_explain.slt b/test/sqllogictest/catalog_server_explain.slt
index 898186c238260..cad9d876f6b93 100644
--- a/test/sqllogictest/catalog_server_explain.slt
+++ b/test/sqllogictest/catalog_server_explain.slt
@@ -5127,7 +5127,7 @@ mz_catalog.mz_materialized_views:
Project: #4, #0, #5, #1, #6, #2, #9, #8, #3, #7
Map: {=r/s1, s1=r/s1}, "s1"
→Arrange (#1{schema_name}, #2{name})
- →Constant (45 rows)
+ →Constant (46 rows)
→Arrange (empty key) (#0{schema_name}) (#0{schema_name}, #1{name})
→Fused with Child Map/Filter/Project
Project: #4, #3, #5
@@ -5472,7 +5472,7 @@ mz_catalog.mz_tables:
Project: #3, #0, #4, #1, #5, #2, #6, #6, #6
Map: "s1", null
→Arrange (#1{schema_name}, #2{name})
- →Constant (29 rows)
+ →Constant (28 rows)
→Arrange (#0{schema_name}) (#0{schema_name}, #1{name})
→Fused with Child Map/Filter/Project
Project: #5, #4, #6
@@ -5889,7 +5889,7 @@ mz_internal.mz_object_dependencies:
Project: #4..=#6, #2
Filter: case when (#7{ref_kind} = "func") then (#3{object_type} = "8") else case when (#7{ref_kind} = "type") then (#3{object_type} = "7") else ((#3{object_type} != "7") AND (#3{object_type} != "8")) end end
→Arrange (#0{object_schema}..=#2{object_type}) (#3{ref_schema}, #4{ref_name})
- →Constant (1027 rows)
+ →Constant (1034 rows)
→Arrange (#1{schema_name}..=#3{object_type})
→Fused with Child Map/Filter/Project
Project: #5, #4, #2, #3
@@ -6071,6 +6071,72 @@ Target cluster: mz_catalog_server
EOF
+query T multiline
+EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_source_references";
+----
+mz_internal.mz_source_references:
+ →With
+ cte l0 =
+ →Fused with Child Map/Filter/Project
+ Project: #1..=#3
+ Map: parse_catalog_id(((#0{data} -> "key") -> "source")), tots((text_to_double((((#0{data} -> "value") -> "updated_at") ->> "millis")) / 1000))
+ →Table Function jsonb_array_elements(((#0{data} -> "value") -> "references"))
+ →Read mz_internal.mz_catalog_raw
+ cte l1 =
+ →Distinct GroupAggregate
+ →Fused with Child Map/Filter/Project
+ Project: #0
+ →Read l0
+ cte l2 =
+ →Non-incremental GroupAggregate
+ Aggregation: array_agg[order_by=[#0 asc nulls_last]](row(array[#1{value}], #2{ord}))
+ Key:
+ Project: #0
+ →Table Function jsonb_array_elements_text[with_ordinality]((#0{reference} -> "columns"))
+ →Arranged l1
+ cte l3 =
+ →Union
+ →Unarranged Raw Stream
+ →Arranged l2
+ →Map/Filter/Project
+ Project: #0, #1
+ Map: null
+ →Consolidating Union
+ →Negate Diffs
+ →Fused with Child Map/Filter/Project
+ Project: #0
+ →Arranged l2
+ Key: (#0)
+ →Unarranged Raw Stream
+ →Arranged l1
+ →Return
+ →Differential Join %1[#0] » %0:l0[#0]
+ after %0:
+ Project: #2, #4, #5, #3, #1
+ Map: (#0{reference} ->> "namespace"), (#0{reference} ->> "name")
+ →Arrange (#0)
+ →Stream l0
+ →Arrange (#0)
+ →Union
+ →Stream l3
+ →Map/Filter/Project
+ Project: #0, #1
+ Map: null
+ →Consolidating Union
+ →Negate Diffs
+ →Fused with Child Map/Filter/Project
+ Project: #0
+ →Read l3
+ →Unarranged Raw Stream
+ →Arranged l1
+
+Source mz_internal.mz_catalog_raw
+ filter=(("SourceReferences" = (#0{data} ->> "kind")))
+
+Target cluster: mz_catalog_server
+
+EOF
+
query T multiline
EXPLAIN MATERIALIZED VIEW "mz_internal"."mz_sql_server_source_tables";
----
@@ -8374,7 +8440,7 @@ query T multiline
EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_materialized_views";
----
Explained Query (fast path):
- →Constant (45 rows)
+ →Constant (46 rows)
Target cluster: mz_catalog_server
@@ -8394,7 +8460,7 @@ query T multiline
EXPLAIN SELECT * FROM "mz_internal"."mz_builtin_tables";
----
Explained Query (fast path):
- →Constant (29 rows)
+ →Constant (28 rows)
Target cluster: mz_catalog_server
@@ -10322,7 +10388,7 @@ Explained Query:
Project: #3, #4
Filter: case when (#2{ref_kind} = "func") then (#5{object_type} = "8") else case when (#2{ref_kind} = "type") then (#5{object_type} = "7") else ((#5{object_type} != "7") AND (#5{object_type} != "8")) end end
→Unarranged Raw Stream
- →Constant (1027 rows)
+ →Constant (1034 rows)
→Arrange (#1{schema_name}..=#3{object_type})
→Fused with Child Map/Filter/Project
Project: #5, #4, #2, #3
diff --git a/test/sqllogictest/information_schema_tables.slt b/test/sqllogictest/information_schema_tables.slt
index eef88c8d3ba9a..a479593c146be 100644
--- a/test/sqllogictest/information_schema_tables.slt
+++ b/test/sqllogictest/information_schema_tables.slt
@@ -766,7 +766,7 @@ VIEW
materialize
mz_internal
mz_source_references
-BASE TABLE
+MATERIALIZED VIEW
materialize
mz_internal
mz_source_statistics
diff --git a/test/sqllogictest/mz_aws_connections.slt b/test/sqllogictest/mz_aws_connections.slt
index 96ca58cd70462..8a2971072524b 100644
--- a/test/sqllogictest/mz_aws_connections.slt
+++ b/test/sqllogictest/mz_aws_connections.slt
@@ -11,7 +11,7 @@
# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see
# MZ_AWS_CONNECTIONS in src/catalog/src/builtin/mz_internal.rs and the
# parse_connection_details helper in
-# src/expr/src/scalar/func/impls/jsonb.rs. The assume-role columns (principal,
+# src/catalog-decode/src/create_sql.rs. The assume-role columns (principal,
# external_id, example_trust_policy) are reconstructed from the AWS context
# functions; the sqllogictest env configures that context (see
# ConnectionContext::for_tests), so the built-in mz_analytics connection
diff --git a/test/sqllogictest/mz_kafka_connections.slt b/test/sqllogictest/mz_kafka_connections.slt
index 18e30b1b71b10..7a1d2d48d0807 100644
--- a/test/sqllogictest/mz_kafka_connections.slt
+++ b/test/sqllogictest/mz_kafka_connections.slt
@@ -11,7 +11,7 @@
# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see
# MZ_KAFKA_CONNECTIONS in src/catalog/src/builtin/mz_catalog.rs and the
# parse_connection_details helper in
-# src/expr/src/scalar/func/impls/jsonb.rs. Rows with a live broker are
+# src/catalog-decode/src/create_sql.rs. Rows with a live broker are
# exercised by test/testdrive/connection-create-drop.td; this file pins the
# column shape, the progress-topic default, and the parent-type filtering,
# none of which need a live Kafka.
diff --git a/test/sqllogictest/mz_kafka_source_tables.slt b/test/sqllogictest/mz_kafka_source_tables.slt
index bf2e55335d509..357be41b1518e 100644
--- a/test/sqllogictest/mz_kafka_source_tables.slt
+++ b/test/sqllogictest/mz_kafka_source_tables.slt
@@ -11,7 +11,7 @@
# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see
# MZ_KAFKA_SOURCE_TABLES in src/catalog/src/builtin/mz_internal.rs and the
# parse_source_export_details helper in
-# src/expr/src/scalar/func/impls/jsonb.rs. Rows with real upstream data, and
+# src/catalog-decode/src/create_sql.rs. Rows with real upstream data, and
# the envelope/format resolution, are exercised by test/testdrive/mz-sources.td;
# this file pins the column shape and the parent-type filtering that need no
# live Kafka.
diff --git a/test/sqllogictest/mz_kafka_sources.slt b/test/sqllogictest/mz_kafka_sources.slt
index 8011caeba01ca..2323d4826a228 100644
--- a/test/sqllogictest/mz_kafka_sources.slt
+++ b/test/sqllogictest/mz_kafka_sources.slt
@@ -11,7 +11,7 @@
# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw:
# see MZ_KAFKA_SOURCES in src/catalog/src/builtin/mz_catalog.rs and the
# parse_kafka_source_details helper in
-# src/expr/src/scalar/func/impls/jsonb.rs. These tests pin down the
+# src/catalog-decode/src/create_sql.rs. These tests pin down the
# column shape, nullability, and cross-catalog invariants that must be
# preserved as the SQL evolves.
#
diff --git a/test/sqllogictest/mz_mysql_source_tables.slt b/test/sqllogictest/mz_mysql_source_tables.slt
index ec599e474b64b..bde6c6ce27a97 100644
--- a/test/sqllogictest/mz_mysql_source_tables.slt
+++ b/test/sqllogictest/mz_mysql_source_tables.slt
@@ -11,7 +11,7 @@
# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see
# MZ_MYSQL_SOURCE_TABLES in src/catalog/src/builtin/mz_internal.rs and the
# parse_source_export_details helper in
-# src/expr/src/scalar/func/impls/jsonb.rs. Rows with real upstream data are
+# src/catalog-decode/src/create_sql.rs. Rows with real upstream data are
# exercised by test/mysql-cdc; this file pins the column shape and the
# parent-type filtering that need no live MySQL.
diff --git a/test/sqllogictest/mz_postgres_source_tables.slt b/test/sqllogictest/mz_postgres_source_tables.slt
index 9655a5e8207b9..95977208151d7 100644
--- a/test/sqllogictest/mz_postgres_source_tables.slt
+++ b/test/sqllogictest/mz_postgres_source_tables.slt
@@ -11,7 +11,7 @@
# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see
# MZ_POSTGRES_SOURCE_TABLES in src/catalog/src/builtin/mz_internal.rs and the
# parse_source_export_details helper in
-# src/expr/src/scalar/func/impls/jsonb.rs. Rows with real upstream data are
+# src/catalog-decode/src/create_sql.rs. Rows with real upstream data are
# exercised by test/pg-cdc; this file pins the column shape and the
# parent-type filtering that need no live Postgres.
diff --git a/test/sqllogictest/mz_postgres_sources.slt b/test/sqllogictest/mz_postgres_sources.slt
index 7771c324ccc7e..aac57baf52807 100644
--- a/test/sqllogictest/mz_postgres_sources.slt
+++ b/test/sqllogictest/mz_postgres_sources.slt
@@ -11,7 +11,7 @@
# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw:
# see MZ_POSTGRES_SOURCES in src/catalog/src/builtin/mz_internal.rs and
# the parse_postgres_source_details helper in
-# src/expr/src/scalar/func/impls/jsonb.rs. These tests pin down the
+# src/catalog-decode/src/create_sql.rs. These tests pin down the
# column shape, nullability, and cross-catalog invariants that must be
# preserved as the SQL evolves.
diff --git a/test/sqllogictest/mz_sinks.slt b/test/sqllogictest/mz_sinks.slt
index 71ffe8123bcaa..b6a97e830e019 100644
--- a/test/sqllogictest/mz_sinks.slt
+++ b/test/sqllogictest/mz_sinks.slt
@@ -11,7 +11,7 @@
# and mz_catalog.mz_iceberg_sinks. All three are materialized views over
# mz_internal.mz_catalog_raw and get every sink detail out of the
# persisted create_sql, in the CreateSink arm of parse_catalog_create_sql
-# (src/expr/src/scalar/func/impls/jsonb.rs).
+# (src/catalog-decode/src/create_sql.rs).
#
# Note we can't create a sink here at all: purification always talks to
# the broker or the iceberg catalog, and there is no VALIDATE = false to
diff --git a/test/sqllogictest/mz_source_references.slt b/test/sqllogictest/mz_source_references.slt
new file mode 100644
index 0000000000000..a1a0a2496c96c
--- /dev/null
+++ b/test/sqllogictest/mz_source_references.slt
@@ -0,0 +1,67 @@
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+# Behavioural tests for mz_internal.mz_source_references.
+
+mode cockroach
+
+reset-server
+
+# The auction house load generator generates multiple source references.
+statement ok
+CREATE SOURCE lg_auction IN CLUSTER quickstart FROM LOAD GENERATOR AUCTION FOR ALL TABLES
+
+query TTT
+SELECT r.namespace, r.name, r.columns::text
+FROM mz_internal.mz_source_references r
+JOIN mz_catalog.mz_sources s ON s.id = r.source_id
+WHERE s.name = 'lg_auction'
+ORDER BY r.name
+----
+auction accounts {id,org_id,balance}
+auction auctions {id,seller,item,end_time}
+auction bids {id,buyer,auction_id,amount,bid_time}
+auction organizations {id,name}
+auction users {id,org_id,name}
+
+# A single-output load generator records the reference but no column list.
+statement ok
+CREATE SOURCE lg_counter IN CLUSTER quickstart FROM LOAD GENERATOR COUNTER
+
+query TTT
+SELECT r.namespace, r.name, (r.columns IS NULL)::text
+FROM mz_internal.mz_source_references r
+JOIN mz_catalog.mz_sources s ON s.id = r.source_id
+WHERE s.name = 'lg_counter'
+----
+counter counter true
+
+# `updated_at` is when the references were last fetched.
+query B
+SELECT bool_and(updated_at BETWEEN now() - INTERVAL '1 hour' AND now() + INTERVAL '1 minute')
+FROM mz_internal.mz_source_references
+----
+true
+
+# Dropping a source retracts its references.
+statement ok
+DROP SOURCE lg_auction CASCADE
+
+query I
+SELECT count(*) FROM mz_internal.mz_source_references
+----
+1
+
+statement ok
+DROP SOURCE lg_counter CASCADE
+
+query I
+SELECT count(*) FROM mz_internal.mz_source_references
+----
+0
diff --git a/test/sqllogictest/mz_sql_server_source_tables.slt b/test/sqllogictest/mz_sql_server_source_tables.slt
index db079af1c322b..6c5a7b3f931e0 100644
--- a/test/sqllogictest/mz_sql_server_source_tables.slt
+++ b/test/sqllogictest/mz_sql_server_source_tables.slt
@@ -11,7 +11,7 @@
# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see
# MZ_SQL_SERVER_SOURCE_TABLES in src/catalog/src/builtin/mz_internal.rs and the
# parse_source_export_details helper in
-# src/expr/src/scalar/func/impls/jsonb.rs. Rows with real upstream data are
+# src/catalog-decode/src/create_sql.rs. Rows with real upstream data are
# exercised by test/sql-server-cdc; this file pins the column shape and the
# parent-type filtering that need no live SQL Server.
diff --git a/test/sqllogictest/mz_ssh_tunnel_connections.slt b/test/sqllogictest/mz_ssh_tunnel_connections.slt
index b555697bb352e..5f6caf54510e0 100644
--- a/test/sqllogictest/mz_ssh_tunnel_connections.slt
+++ b/test/sqllogictest/mz_ssh_tunnel_connections.slt
@@ -11,7 +11,7 @@
# BuiltinMaterializedView backed by SQL over mz_internal.mz_catalog_raw: see
# MZ_SSH_TUNNEL_CONNECTIONS in src/catalog/src/builtin/mz_catalog.rs and the
# parse_connection_details helper in
-# src/expr/src/scalar/func/impls/jsonb.rs. The end-to-end SSH tunnel behaviour
+# src/catalog-decode/src/create_sql.rs. The end-to-end SSH tunnel behaviour
# is exercised by test/ssh-connection/*; this file pins the column shape and
# the parent-type filtering.
diff --git a/test/sqllogictest/mz_tables.slt b/test/sqllogictest/mz_tables.slt
index 487d2ade49020..ca504a2112105 100644
--- a/test/sqllogictest/mz_tables.slt
+++ b/test/sqllogictest/mz_tables.slt
@@ -12,7 +12,7 @@
# src/catalog/src/builtin/mz_catalog.rs) with two union branches, user tables
# and builtin tables, and it derives every column from the durable catalog JSON
# through parse_catalog_id, parse_catalog_privileges, redact_sql and
-# parse_catalog_create_sql (src/expr/src/scalar/func/impls/jsonb.rs).
+# parse_catalog_create_sql (src/catalog-decode/src/create_sql.rs).
#
# Before that it was a BuiltinTable populated by pack_table_update in the
# adapter, so the values pinned here are a compatibility surface:
diff --git a/test/sqllogictest/mz_views.slt b/test/sqllogictest/mz_views.slt
index 5fd82a0c8a6b5..d7c4cb3bb7e20 100644
--- a/test/sqllogictest/mz_views.slt
+++ b/test/sqllogictest/mz_views.slt
@@ -12,7 +12,7 @@
# src/catalog/src/builtin/mz_catalog.rs) with two union branches, user views and
# builtin views, and it derives every column from the durable catalog JSON
# through parse_catalog_id, parse_catalog_privileges, redact_sql and
-# parse_catalog_create_sql (src/expr/src/scalar/func/impls/jsonb.rs).
+# parse_catalog_create_sql (src/catalog-decode/src/create_sql.rs).
#
# Before that it was a BuiltinTable populated by pack_view_update in the
# adapter, so the values pinned here are a compatibility surface. The
diff --git a/test/terraform/aws-persistent/main.tf b/test/terraform/aws-persistent/main.tf
index 28c80ab42459e..9a967f78ff36f 100644
--- a/test/terraform/aws-persistent/main.tf
+++ b/test/terraform/aws-persistent/main.tf
@@ -15,7 +15,7 @@ resource "random_password" "db_password" {
# 1. Create network infrastructure
module "networking" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/networking?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/networking?ref=v13.12.1"
name_prefix = var.name_prefix
vpc_cidr = "10.0.0.0/16"
@@ -28,7 +28,7 @@ module "networking" {
# 2. Create EKS cluster
module "eks" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/eks?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/eks?ref=v13.12.1"
name_prefix = var.name_prefix
cluster_version = "1.32"
@@ -50,7 +50,7 @@ module "eks" {
# nodes cannot become Ready without one, so it must be installed before any
# node group.
module "vpc_cni" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/vpc-cni?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/vpc-cni?ref=v13.12.1"
name_prefix = var.name_prefix
oidc_provider_arn = module.eks.oidc_provider_arn
@@ -74,7 +74,7 @@ module "vpc_cni" {
}
module "base_node_group" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/eks-node-group?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/eks-node-group?ref=v13.12.1"
cluster_name = module.eks.cluster_name
aws_region = var.aws_region
@@ -102,7 +102,7 @@ module "base_node_group" {
# v21 clusters do not bootstrap CoreDNS either, so the deployment, its service
# account and the kube-dns Service are created here.
module "coredns" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/coredns?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/coredns?ref=v13.12.1"
node_selector = local.base_node_labels
disable_default_coredns_autoscaler = false
@@ -121,7 +121,7 @@ module "coredns" {
}
module "karpenter" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter?ref=v13.12.1"
name_prefix = var.name_prefix
cluster_name = module.eks.cluster_name
@@ -140,7 +140,7 @@ module "karpenter" {
# Create a generic nodeclass and nodepool for system workloads
module "ec2nodeclass_generic" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-ec2nodeclass?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-ec2nodeclass?ref=v13.12.1"
name = local.nodeclass_name_generic
ami_selector_terms = local.ami_selector_terms
@@ -158,7 +158,7 @@ module "ec2nodeclass_generic" {
}
module "nodepool_generic" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-nodepool?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-nodepool?ref=v13.12.1"
name = local.nodeclass_name_generic
nodeclass_name = local.nodeclass_name_generic
@@ -175,7 +175,7 @@ module "nodepool_generic" {
# Create a dedicated nodeclass and nodepool for Materialize pods
module "ec2nodeclass_materialize" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-ec2nodeclass?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-ec2nodeclass?ref=v13.12.1"
name = local.nodeclass_name_materialize
ami_selector_terms = local.ami_selector_terms
@@ -192,7 +192,7 @@ module "ec2nodeclass_materialize" {
}
module "nodepool_materialize" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-nodepool?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-nodepool?ref=v13.12.1"
name = local.nodeclass_name_materialize
nodeclass_name = local.nodeclass_name_materialize
@@ -210,7 +210,7 @@ module "nodepool_materialize" {
# 3. Install AWS Load Balancer Controller
module "aws_lbc" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/aws-lbc?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/aws-lbc?ref=v13.12.1"
name_prefix = var.name_prefix
eks_cluster_name = module.eks.cluster_name
@@ -230,7 +230,7 @@ module "aws_lbc" {
# 4. Install Certificate Manager for TLS
module "cert_manager" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/cert-manager?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/cert-manager?ref=v13.12.1"
node_selector = local.generic_node_labels
@@ -243,7 +243,7 @@ module "cert_manager" {
}
module "self_signed_cluster_issuer" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/self-signed-cluster-issuer?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/self-signed-cluster-issuer?ref=v13.12.1"
name_prefix = var.name_prefix
@@ -254,7 +254,7 @@ module "self_signed_cluster_issuer" {
# 5. Setup dedicated database instance for Materialize
module "database" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/database?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/database?ref=v13.12.1"
name_prefix = var.name_prefix
postgres_version = "15"
@@ -280,7 +280,7 @@ module "database" {
# 6. Setup S3 bucket for Materialize
module "storage" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/storage?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/storage?ref=v13.12.1"
name_prefix = var.name_prefix
bucket_lifecycle_rules = []
@@ -305,7 +305,7 @@ module "storage" {
# 7. Install Materialize Operator
module "operator" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/operator?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/operator?ref=v13.12.1"
name_prefix = var.name_prefix
aws_region = var.aws_region
@@ -348,7 +348,7 @@ module "operator" {
# 8. Setup Materialize instance
module "materialize_instance" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/materialize-instance?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/materialize-instance?ref=v13.12.1"
instance_name = local.materialize_instance_name
instance_namespace = local.materialize_instance_namespace
diff --git a/test/terraform/aws-temporary/main.tf b/test/terraform/aws-temporary/main.tf
index a77d11691bce0..b6efd1efd902c 100644
--- a/test/terraform/aws-temporary/main.tf
+++ b/test/terraform/aws-temporary/main.tf
@@ -15,7 +15,7 @@ resource "random_password" "db_password" {
# 1. Create network infrastructure
module "networking" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/networking?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/networking?ref=v13.12.1"
name_prefix = var.name_prefix
vpc_cidr = "10.0.0.0/16"
@@ -28,7 +28,7 @@ module "networking" {
# 2. Create EKS cluster
module "eks" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/eks?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/eks?ref=v13.12.1"
name_prefix = var.name_prefix
cluster_version = "1.34"
@@ -50,7 +50,7 @@ module "eks" {
# nodes cannot become Ready without one, so it must be installed before any
# node group.
module "vpc_cni" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/vpc-cni?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/vpc-cni?ref=v13.12.1"
name_prefix = var.name_prefix
oidc_provider_arn = module.eks.oidc_provider_arn
@@ -74,7 +74,7 @@ module "vpc_cni" {
}
module "base_node_group" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/eks-node-group?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/eks-node-group?ref=v13.12.1"
cluster_name = module.eks.cluster_name
aws_region = var.aws_region
@@ -102,7 +102,7 @@ module "base_node_group" {
# v21 clusters do not bootstrap CoreDNS either, so the deployment, its service
# account and the kube-dns Service are created here.
module "coredns" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/coredns?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/coredns?ref=v13.12.1"
node_selector = local.base_node_labels
disable_default_coredns_autoscaler = false
@@ -121,7 +121,7 @@ module "coredns" {
}
module "karpenter" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter?ref=v13.12.1"
name_prefix = var.name_prefix
cluster_name = module.eks.cluster_name
@@ -140,7 +140,7 @@ module "karpenter" {
# Create a generic nodeclass and nodepool for system workloads
module "ec2nodeclass_generic" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-ec2nodeclass?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-ec2nodeclass?ref=v13.12.1"
name = local.nodeclass_name_generic
ami_selector_terms = local.ami_selector_terms
@@ -158,7 +158,7 @@ module "ec2nodeclass_generic" {
}
module "nodepool_generic" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-nodepool?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-nodepool?ref=v13.12.1"
name = local.nodeclass_name_generic
nodeclass_name = local.nodeclass_name_generic
@@ -175,7 +175,7 @@ module "nodepool_generic" {
# Create a dedicated nodeclass and nodepool for Materialize pods
module "ec2nodeclass_materialize" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-ec2nodeclass?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-ec2nodeclass?ref=v13.12.1"
name = local.nodeclass_name_materialize
ami_selector_terms = local.ami_selector_terms
@@ -192,7 +192,7 @@ module "ec2nodeclass_materialize" {
}
module "nodepool_materialize" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-nodepool?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-nodepool?ref=v13.12.1"
name = local.nodeclass_name_materialize
nodeclass_name = local.nodeclass_name_materialize
@@ -210,7 +210,7 @@ module "nodepool_materialize" {
# 3. Install AWS Load Balancer Controller
module "aws_lbc" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/aws-lbc?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/aws-lbc?ref=v13.12.1"
name_prefix = var.name_prefix
eks_cluster_name = module.eks.cluster_name
@@ -230,7 +230,7 @@ module "aws_lbc" {
# 4. Install Certificate Manager for TLS
module "cert_manager" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/cert-manager?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/cert-manager?ref=v13.12.1"
node_selector = local.generic_node_labels
@@ -243,7 +243,7 @@ module "cert_manager" {
}
module "self_signed_cluster_issuer" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/self-signed-cluster-issuer?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/self-signed-cluster-issuer?ref=v13.12.1"
name_prefix = var.name_prefix
@@ -254,7 +254,7 @@ module "self_signed_cluster_issuer" {
# 5. Setup dedicated database instance for Materialize
module "database" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/database?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/database?ref=v13.12.1"
name_prefix = var.name_prefix
postgres_version = "15"
@@ -286,7 +286,7 @@ module "database" {
# 6. Setup S3 bucket for Materialize
module "storage" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/storage?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/storage?ref=v13.12.1"
name_prefix = var.name_prefix
bucket_lifecycle_rules = []
@@ -311,7 +311,7 @@ module "storage" {
# 7. Install Materialize Operator
module "operator" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/operator?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/operator?ref=v13.12.1"
name_prefix = var.name_prefix
aws_region = var.aws_region
@@ -354,7 +354,7 @@ module "operator" {
# 8. Setup Materialize instance
module "materialize_instance" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/materialize-instance?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/materialize-instance?ref=v13.12.1"
instance_name = local.materialize_instance_name
instance_namespace = local.materialize_instance_namespace
diff --git a/test/terraform/aws-upgrade/main.tf b/test/terraform/aws-upgrade/main.tf
index a77d11691bce0..b6efd1efd902c 100644
--- a/test/terraform/aws-upgrade/main.tf
+++ b/test/terraform/aws-upgrade/main.tf
@@ -15,7 +15,7 @@ resource "random_password" "db_password" {
# 1. Create network infrastructure
module "networking" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/networking?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/networking?ref=v13.12.1"
name_prefix = var.name_prefix
vpc_cidr = "10.0.0.0/16"
@@ -28,7 +28,7 @@ module "networking" {
# 2. Create EKS cluster
module "eks" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/eks?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/eks?ref=v13.12.1"
name_prefix = var.name_prefix
cluster_version = "1.34"
@@ -50,7 +50,7 @@ module "eks" {
# nodes cannot become Ready without one, so it must be installed before any
# node group.
module "vpc_cni" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/vpc-cni?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/vpc-cni?ref=v13.12.1"
name_prefix = var.name_prefix
oidc_provider_arn = module.eks.oidc_provider_arn
@@ -74,7 +74,7 @@ module "vpc_cni" {
}
module "base_node_group" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/eks-node-group?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/eks-node-group?ref=v13.12.1"
cluster_name = module.eks.cluster_name
aws_region = var.aws_region
@@ -102,7 +102,7 @@ module "base_node_group" {
# v21 clusters do not bootstrap CoreDNS either, so the deployment, its service
# account and the kube-dns Service are created here.
module "coredns" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/coredns?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/coredns?ref=v13.12.1"
node_selector = local.base_node_labels
disable_default_coredns_autoscaler = false
@@ -121,7 +121,7 @@ module "coredns" {
}
module "karpenter" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter?ref=v13.12.1"
name_prefix = var.name_prefix
cluster_name = module.eks.cluster_name
@@ -140,7 +140,7 @@ module "karpenter" {
# Create a generic nodeclass and nodepool for system workloads
module "ec2nodeclass_generic" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-ec2nodeclass?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-ec2nodeclass?ref=v13.12.1"
name = local.nodeclass_name_generic
ami_selector_terms = local.ami_selector_terms
@@ -158,7 +158,7 @@ module "ec2nodeclass_generic" {
}
module "nodepool_generic" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-nodepool?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-nodepool?ref=v13.12.1"
name = local.nodeclass_name_generic
nodeclass_name = local.nodeclass_name_generic
@@ -175,7 +175,7 @@ module "nodepool_generic" {
# Create a dedicated nodeclass and nodepool for Materialize pods
module "ec2nodeclass_materialize" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-ec2nodeclass?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-ec2nodeclass?ref=v13.12.1"
name = local.nodeclass_name_materialize
ami_selector_terms = local.ami_selector_terms
@@ -192,7 +192,7 @@ module "ec2nodeclass_materialize" {
}
module "nodepool_materialize" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-nodepool?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/karpenter-nodepool?ref=v13.12.1"
name = local.nodeclass_name_materialize
nodeclass_name = local.nodeclass_name_materialize
@@ -210,7 +210,7 @@ module "nodepool_materialize" {
# 3. Install AWS Load Balancer Controller
module "aws_lbc" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/aws-lbc?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/aws-lbc?ref=v13.12.1"
name_prefix = var.name_prefix
eks_cluster_name = module.eks.cluster_name
@@ -230,7 +230,7 @@ module "aws_lbc" {
# 4. Install Certificate Manager for TLS
module "cert_manager" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/cert-manager?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/cert-manager?ref=v13.12.1"
node_selector = local.generic_node_labels
@@ -243,7 +243,7 @@ module "cert_manager" {
}
module "self_signed_cluster_issuer" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/self-signed-cluster-issuer?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/self-signed-cluster-issuer?ref=v13.12.1"
name_prefix = var.name_prefix
@@ -254,7 +254,7 @@ module "self_signed_cluster_issuer" {
# 5. Setup dedicated database instance for Materialize
module "database" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/database?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/database?ref=v13.12.1"
name_prefix = var.name_prefix
postgres_version = "15"
@@ -286,7 +286,7 @@ module "database" {
# 6. Setup S3 bucket for Materialize
module "storage" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/storage?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/storage?ref=v13.12.1"
name_prefix = var.name_prefix
bucket_lifecycle_rules = []
@@ -311,7 +311,7 @@ module "storage" {
# 7. Install Materialize Operator
module "operator" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/operator?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//aws/modules/operator?ref=v13.12.1"
name_prefix = var.name_prefix
aws_region = var.aws_region
@@ -354,7 +354,7 @@ module "operator" {
# 8. Setup Materialize instance
module "materialize_instance" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/materialize-instance?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/materialize-instance?ref=v13.12.1"
instance_name = local.materialize_instance_name
instance_namespace = local.materialize_instance_namespace
diff --git a/test/terraform/azure-temporary/main.tf b/test/terraform/azure-temporary/main.tf
index 8ec8654779585..e609d02380115 100644
--- a/test/terraform/azure-temporary/main.tf
+++ b/test/terraform/azure-temporary/main.tf
@@ -102,7 +102,7 @@ resource "azurerm_resource_group" "materialize" {
# 2. Create networking infrastructure
module "networking" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//azure/modules/networking?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//azure/modules/networking?ref=v13.12.1"
resource_group_name = azurerm_resource_group.materialize.name
location = var.location
@@ -120,7 +120,7 @@ module "networking" {
# 3. Create AKS cluster with default node pool
module "aks" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//azure/modules/aks?ref=v13.2.1"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//azure/modules/aks?ref=v13.12.1"
resource_group_name = azurerm_resource_group.materialize.name
kubernetes_version = local.aks_config.kubernetes_version
@@ -153,7 +153,7 @@ module "aks" {
# 3.1 Create Materialize-dedicated node pool with taints
module "materialize_nodepool" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//azure/modules/nodepool?ref=v13.2.1"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//azure/modules/nodepool?ref=v13.12.1"
prefix = var.name_prefix
cluster_id = module.aks.cluster_id
@@ -183,7 +183,7 @@ module "materialize_nodepool" {
# 4. Create PostgreSQL database
module "database" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//azure/modules/database?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//azure/modules/database?ref=v13.12.1"
depends_on = [module.networking]
@@ -218,7 +218,7 @@ module "database" {
# 5. Create Azure Blob Storage
module "storage" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//azure/modules/storage?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//azure/modules/storage?ref=v13.12.1"
resource_group_name = azurerm_resource_group.materialize.name
location = var.location
@@ -240,7 +240,7 @@ module "storage" {
# 6. Install cert-manager for TLS
module "cert_manager" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/cert-manager?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/cert-manager?ref=v13.12.1"
node_selector = local.generic_node_labels
@@ -250,7 +250,7 @@ module "cert_manager" {
}
module "self_signed_cluster_issuer" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/self-signed-cluster-issuer?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/self-signed-cluster-issuer?ref=v13.12.1"
name_prefix = var.name_prefix
@@ -261,7 +261,7 @@ module "self_signed_cluster_issuer" {
# 7. Install Materialize Operator
module "operator" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//azure/modules/operator?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//azure/modules/operator?ref=v13.12.1"
name_prefix = var.name_prefix
location = var.location
@@ -301,7 +301,7 @@ module "operator" {
# 8. Deploy Materialize instance
module "materialize_instance" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/materialize-instance?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/materialize-instance?ref=v13.12.1"
instance_name = local.materialize_instance_name
instance_namespace = local.materialize_instance_namespace
diff --git a/test/terraform/azure-temporary/providers.tf b/test/terraform/azure-temporary/providers.tf
index 5d1f099c771f2..9d3ebf22191b7 100644
--- a/test/terraform/azure-temporary/providers.tf
+++ b/test/terraform/azure-temporary/providers.tf
@@ -13,7 +13,7 @@ terraform {
required_providers {
azurerm = {
source = "hashicorp/azurerm"
- version = "5.6.0"
+ version = "5.7.0"
}
kubernetes = {
source = "hashicorp/kubernetes"
diff --git a/test/terraform/gcp-temporary/main.tf b/test/terraform/gcp-temporary/main.tf
index d1b28d73807c5..f35e22ceefe94 100644
--- a/test/terraform/gcp-temporary/main.tf
+++ b/test/terraform/gcp-temporary/main.tf
@@ -99,7 +99,7 @@ locals {
# 1. Configure networking infrastructure including VPC, subnets, and CIDR blocks
module "networking" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/networking?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/networking?ref=v13.12.1"
project_id = var.project_id
region = var.region
@@ -110,7 +110,7 @@ module "networking" {
# 2. Set up Google Kubernetes Engine (GKE) cluster
module "gke" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/gke?ref=v13.2.1"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/gke?ref=v13.12.1"
depends_on = [module.networking]
@@ -127,7 +127,7 @@ module "gke" {
# 2.1 Create generic node pool for system workloads
module "generic_nodepool" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/nodepool?ref=v13.2.1"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/nodepool?ref=v13.12.1"
depends_on = [module.gke]
prefix = "${var.name_prefix}-generic"
@@ -147,7 +147,7 @@ module "generic_nodepool" {
# 2.2 Create Materialize-dedicated node pool with taints
module "materialize_nodepool" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/nodepool?ref=v13.2.1"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/nodepool?ref=v13.12.1"
depends_on = [module.gke]
prefix = "${var.name_prefix}-mz"
@@ -170,7 +170,7 @@ module "materialize_nodepool" {
# 3. Set up PostgreSQL database instance for Materialize metadata storage
module "database" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/database?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/database?ref=v13.12.1"
depends_on = [module.networking]
databases = [local.database_config.database]
@@ -191,7 +191,7 @@ module "database" {
# 4. Create Google Cloud Storage bucket for Materialize persistent data storage
module "storage" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/storage?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/storage?ref=v13.12.1"
project_id = var.project_id
region = var.region
@@ -205,7 +205,7 @@ module "storage" {
# 5. Install cert-manager for SSL certificate management and create cluster issuer
module "cert_manager" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/cert-manager?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/cert-manager?ref=v13.12.1"
node_selector = local.generic_node_labels
@@ -216,7 +216,7 @@ module "cert_manager" {
}
module "self_signed_cluster_issuer" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/self-signed-cluster-issuer?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/self-signed-cluster-issuer?ref=v13.12.1"
name_prefix = var.name_prefix
@@ -227,7 +227,7 @@ module "self_signed_cluster_issuer" {
# 6. Install Materialize Kubernetes operator for managing Materialize instances
module "operator" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/operator?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//gcp/modules/operator?ref=v13.12.1"
name_prefix = var.name_prefix
region = var.region
@@ -259,7 +259,7 @@ module "operator" {
# 7. Deploy Materialize instance with configured backend connections
module "materialize_instance" {
- source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/materialize-instance?ref=v13.10.0"
+ source = "git::https://github.com/MaterializeInc/materialize-terraform-self-managed.git//kubernetes/modules/materialize-instance?ref=v13.12.1"
instance_name = local.materialize_instance_name
instance_namespace = local.materialize_instance_namespace
diff --git a/test/testdrive/catalog.td b/test/testdrive/catalog.td
index d0baa38912fd3..ea2b6ae857f02 100644
--- a/test/testdrive/catalog.td
+++ b/test/testdrive/catalog.td
@@ -613,7 +613,6 @@ mz_optimizer_notices ""
mz_replacements ""
mz_replica_hydration_history ""
mz_sessions ""
-mz_source_references ""
mz_storage_usage_by_shard ""
mz_subscriptions ""
mz_type_pg_metadata ""
@@ -740,6 +739,7 @@ mz_pending_cluster_replicas mz_catalog_server ""
mz_postgres_source_tables mz_catalog_server ""
mz_postgres_sources mz_catalog_server ""
mz_replica_system_parameters mz_catalog_server ""
+mz_source_references mz_catalog_server ""
mz_sql_server_source_tables mz_catalog_server ""
# Check default sources, tables, and views in mz_introspection.
@@ -838,7 +838,7 @@ test_table ""
# `SHOW TABLES` and `mz_tables` should agree.
> SELECT COUNT(*) FROM mz_tables WHERE id LIKE 's%'
-29
+28
# There is one entry in mz_indexes for each field_number/expression of the index.
> SELECT COUNT(id) FROM mz_indexes WHERE id LIKE 's%'