diff --git a/Cargo.lock b/Cargo.lock index 38ba6908284b4..bdbd43576308d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6736,6 +6736,7 @@ dependencies = [ "anyhow", "async-stream", "bytesize", + "chrono", "columnar", "columnation", "core_affinity", @@ -7818,6 +7819,7 @@ dependencies = [ "serde_json", "sha2", "tempfile", + "time", "timely", "tokio", "tokio-postgres", diff --git a/bin/ci-builder b/bin/ci-builder index 6665fe5053502..2aa0816ff0fc6 100755 --- a/bin/ci-builder +++ b/bin/ci-builder @@ -253,6 +253,14 @@ case "$cmd" in npm_dir=$HOME/.npm-builder mkdir -p "$(pwd)/target-xcompile" ~/.kube "$claude_dir" "$codex_dir" "$npm_dir" + container_workdir=$workdir + if is_truthy "${CI_BUILDER_FIXED_WORKDIR:-${CI:-0}}"; then + if [[ "$workdir" != "$(pwd)" ]]; then + die "the fixed workdir requires invoking ci-builder from the repository root (set CI_BUILDER_FIXED_WORKDIR=0 to opt out)" + fi + container_workdir=/workspace + fi + args=( --cidfile "$cid_file" --rm --interactive @@ -262,8 +270,8 @@ case "$cmd" in --volume "$HOME/.claude.json:/tmp/.claude.json" --volume "$codex_dir:/tmp/.codex" --volume "$npm_dir:/tmp/.npm" - --volume "$workdir:$workdir" - --workdir "$workdir" + --volume "$workdir:$container_workdir" + --workdir "$container_workdir" --env XDG_CACHE_HOME=/mnt/build/cache --env MZ_SOFT_ASSERTIONS --env NO_COLOR diff --git a/bin/migrate-gh-to-linear b/bin/migrate-gh-to-linear new file mode 100755 index 0000000000000..e35032ebb18f0 --- /dev/null +++ b/bin/migrate-gh-to-linear @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. +# +# migrate-gh-to-linear - Migrate GitHub issues to Linear. + +exec "$(dirname "$0")"/pyactivate -m materialize.cli.migrate_gh_to_linear "$@" diff --git a/ci/mkpipeline.py b/ci/mkpipeline.py index 5f105595e2466..909a2d7de7ebd 100644 --- a/ci/mkpipeline.py +++ b/ci/mkpipeline.py @@ -306,9 +306,15 @@ def fetch_hashes() -> None: and fail_build_reason is None ): trim_test_selection_id(pipeline, set()) - # Make the website always deploy + # lint-docs is required for website deploy, the others are free because an agent is always around anyway for step in steps(pipeline): - if step.get("id") == "lint-docs": + if step.get("id") in ( + "lint-docs", + "lint-macos", + "lint-fast", + "lint-clippy", + "lint-doctests", + ): step.pop("skip", None) # Surface label-driven changes as a Buildkite annotation, since otherwise diff --git a/ci/plugins/cloudtest/hooks/command b/ci/plugins/cloudtest/hooks/command index b153e2840668e..d323ab84239ef 100644 --- a/ci/plugins/cloudtest/hooks/command +++ b/ci/plugins/cloudtest/hooks/command @@ -14,6 +14,11 @@ set -euo pipefail . misc/shlib/shlib.bash . test/cloudtest/config.bash +# The kind cluster setup hands repository paths to the host Docker daemon as +# bind mount sources, which requires container paths to equal host paths. See +# CI_BUILDER_FIXED_WORKDIR in bin/ci-builder. +export CI_BUILDER_FIXED_WORKDIR=0 + run_args=( "--junitxml=junit_cloudtest_$BUILDKITE_JOB_ID.xml" ) diff --git a/ci/plugins/mzcompose/hooks/command b/ci/plugins/mzcompose/hooks/command index c6956bfe7dfb5..a4fa58e6eecd7 100644 --- a/ci/plugins/mzcompose/hooks/command +++ b/ci/plugins/mzcompose/hooks/command @@ -13,6 +13,11 @@ set -euo pipefail . misc/shlib/shlib.bash +# mzcompose (and the compositions it runs) hands repository paths to the host +# Docker daemon as bind mount sources, which requires container paths to equal +# host paths. See CI_BUILDER_FIXED_WORKDIR in bin/ci-builder. +export CI_BUILDER_FIXED_WORKDIR=0 + builder=${BUILDKITE_PLUGIN_MZCOMPOSE_CI_BUILDER:-min} if is_truthy "${CI_HEAP_PROFILES:-}"; then diff --git a/ci/test/coverage_report.sh b/ci/test/coverage_report.sh index 321b7da20de01..fcf5d4bcddba9 100755 --- a/ci/test/coverage_report.sh +++ b/ci/test/coverage_report.sh @@ -29,7 +29,12 @@ bin/ci-annotate-errors junit_coverage*.xml || true ci_unimportant_heading "Create coverage report" REPORT=coverage_without_unittests_"$BUILDKITE_BUILD_ID" REPORT_UNITTESTS=coverage_with_unittests_"$BUILDKITE_BUILD_ID" -find coverage -name '*.lcov' -exec sed -i "s#SF:/var/lib/buildkite-agent/builds/buildkite-[^/]*/materialize/[^/]*/#SF:#" {} + +# Binaries embed the checkout path of the agent that compiled them: /workspace +# for builds with CI_BUILDER_FIXED_WORKDIR, the Buildkite checkout path +# otherwise (e.g. the cargo-test job). +find coverage -name '*.lcov' -exec sed -i \ + -e "s#SF:/workspace/#SF:#" \ + -e "s#SF:/var/lib/buildkite-agent/builds/buildkite-[^/]*/materialize/[^/]*/#SF:#" {} + find coverage -name '*.lcov' -not -name 'cargotest.lcov' -exec genhtml -o "$REPORT" {} + find coverage -name '*.lcov' -exec genhtml -o "$REPORT_UNITTESTS" {} + diff --git a/ci/test/pipeline.template.yml b/ci/test/pipeline.template.yml index 9554fae407586..6d915c47c1db5 100644 --- a/ci/test/pipeline.template.yml +++ b/ci/test/pipeline.template.yml @@ -787,6 +787,9 @@ steps: queue: linux-x86_64 env: CI: "true" + # The kind cluster setup hands repository paths to the host Docker + # daemon. See CI_BUILDER_FIXED_WORKDIR in bin/ci-builder. + CI_BUILDER_FIXED_WORKDIR: "0" coverage: skip sanitizer: skip ci_glue_exempt: true diff --git a/console/icon/ClaudeLogoIcon.svg b/console/icon/ClaudeLogoIcon.svg new file mode 100644 index 0000000000000..869d807f0c2af --- /dev/null +++ b/console/icon/ClaudeLogoIcon.svg @@ -0,0 +1,2 @@ + + diff --git a/console/icon/CodexLogoIcon.svg b/console/icon/CodexLogoIcon.svg new file mode 100644 index 0000000000000..30ab4e4415ff4 --- /dev/null +++ b/console/icon/CodexLogoIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/console/icon/CursorLogoIcon.svg b/console/icon/CursorLogoIcon.svg new file mode 100644 index 0000000000000..d27970ff2c1a2 --- /dev/null +++ b/console/icon/CursorLogoIcon.svg @@ -0,0 +1,3 @@ + + + diff --git a/console/icon/GeminiLogoIcon.svg b/console/icon/GeminiLogoIcon.svg new file mode 100644 index 0000000000000..0d6944fbf77ca --- /dev/null +++ b/console/icon/GeminiLogoIcon.svg @@ -0,0 +1,2 @@ + + diff --git a/console/icon/Terminal.svg b/console/icon/TerminalIcon.svg similarity index 100% rename from console/icon/Terminal.svg rename to console/icon/TerminalIcon.svg diff --git a/console/icon/VsCodeLogoIcon.svg b/console/icon/VsCodeLogoIcon.svg new file mode 100644 index 0000000000000..e1dfe854af020 --- /dev/null +++ b/console/icon/VsCodeLogoIcon.svg @@ -0,0 +1,2 @@ + + diff --git a/console/icon/WindsurfLogoIcon.svg b/console/icon/WindsurfLogoIcon.svg new file mode 100644 index 0000000000000..6b53cb5d4dcf6 --- /dev/null +++ b/console/icon/WindsurfLogoIcon.svg @@ -0,0 +1,2 @@ + + diff --git a/console/misc/docker/nginx.conf.template b/console/misc/docker/nginx.conf.template index 70ede35edc6c1..052a358aefe0f 100644 --- a/console/misc/docker/nginx.conf.template +++ b/console/misc/docker/nginx.conf.template @@ -48,6 +48,20 @@ ${MZ_NGINX_LISTENER_CONFIG} proxy_set_header X-Webhook-Event $http_x_webhook_event; } + # RFC 9728 OAuth protected-resource metadata. MCP clients pointed at + # /api/mcp follow the resource_metadata URL from the 401 challenge, which + # environmentd publishes on this host, so it must reach environmentd too. + location /.well-known/oauth-protected-resource { + resolver ${NGINX_LOCAL_RESOLVERS} valid=10s; + set $backend ${MZ_ENDPOINT}; + proxy_pass $backend; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Request-ID $request_id; + } + # Cache policy for static assets location ~* \.(js|css|png|jpg|jpeg|gif|svg|woff|woff2)$ { root /usr/share/nginx/html; diff --git a/console/src/access/AppPasswordsPage.tsx b/console/src/access/AppPasswordsPage.tsx index 3ad93817fba39..f1f4f05135a1f 100644 --- a/console/src/access/AppPasswordsPage.tsx +++ b/console/src/access/AppPasswordsPage.tsx @@ -54,7 +54,7 @@ import { hasTenantApiTokenPermissions } from "~/api/auth"; import { ApiToken } from "~/api/frontegg/types"; import Alert from "~/components/Alert"; import { AppErrorBoundary } from "~/components/AppErrorBoundary"; -import ConnectModal from "~/components/ConnectModal"; +import ConnectDrawer from "~/components/connect/ConnectDrawer"; import { SecretCopyableBox } from "~/components/copyableComponents"; import TaggedMultiSelect from "~/components/Dropdown/TaggedComboBox"; import { LoadingContainer } from "~/components/LoadingContainer"; @@ -451,7 +451,7 @@ const ApiTokensTableProps = ({ borderBottomColor={colors.border.primary} > - + @@ -545,13 +545,7 @@ const SecretBox = ({ ); }; -const ConnectAppPasswordModal = ({ - user, - userStr, -}: { - user: User; - userStr: string; -}) => { +const ConnectAppPasswordButton = ({ userStr }: { userStr: string }) => { const { isOpen, onOpen, onClose } = useDisclosure(); return ( @@ -566,10 +560,9 @@ const ConnectAppPasswordModal = ({ > Connect - diff --git a/console/src/components/ConnectInstructions.tsx b/console/src/components/ConnectInstructions.tsx deleted file mode 100644 index 48141e0713671..0000000000000 --- a/console/src/components/ConnectInstructions.tsx +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { BoxProps, Grid, Spinner, Text } from "@chakra-ui/react"; -import { useAtom } from "jotai"; -import React from "react"; -import { useParams } from "react-router-dom"; - -import { User } from "~/external-library-wrappers/frontegg"; -import { ClusterDetailParams } from "~/platform/clusters/ClusterRoutes"; -import { currentEnvironmentState } from "~/store/environments"; -import ConnectionIcon from "~/svg/ConnectionIcon"; -import { MonitorIcon } from "~/svg/Monitor"; -import TerminalIcon from "~/svg/Terminal"; - -import { CopyableBox, TabbedCodeBlock } from "./copyableComponents"; -import McpConnectInstructions from "./McpConnectInstructions"; - -export interface ConnectInstructionsProps extends BoxProps { - /** The user string to display. Falls back to user.email if not provided. */ - userStr?: string; - /** Frontegg user object. Required for cloud mode. */ - user?: User; - /** Override the environmentd address (host:port). */ - environmentdAddress?: string; - /** Override the query params in the psql connection string. */ - psqlQueryParams?: string; - /** Self-managed deployment with OIDC enabled; gets the browser OAuth flow. */ - oidcEnabled?: boolean; - /** Pre-computed Base64 token for MCP configuration (cloud only). */ - mcpBase64Token?: string; - /** Called when the active tab changes. */ - onTabChange?: (title: string) => void; - /** Callback to generate a new MCP token (creates an app password). */ - onGenerateToken?: () => void; - /** Whether token generation is in progress. */ - isGeneratingToken?: boolean; -} - -const ConnectInstructions = ({ - user, - onTabChange, - oidcEnabled, - mcpBase64Token, - onGenerateToken, - isGeneratingToken, - ...props -}: ConnectInstructionsProps): JSX.Element => { - const [currentEnvironment] = useAtom(currentEnvironmentState); - const { clusterName } = useParams(); - - const envAddress = - props.environmentdAddress ?? - (currentEnvironment?.state === "enabled" - ? currentEnvironment.sqlAddress - : undefined); - - if (!envAddress) { - return ; - } - - const userStr = props.userStr || user?.email || ""; - - const defaultClusterOptionString = clusterName - ? `&options=--cluster%3D${clusterName}` - : ""; - - // NOTE(benesch): We'd like to use `sslmode=verify-full` to prevent MITM - // attacks, but that mode requires specifying `sslrootcert=/path/to/cabundle`, - // and that path varies by platform. So instead we use `require`, which is - // at least better than the default of `prefer`. - const queryParams = - props.psqlQueryParams ?? `sslmode=require${defaultClusterOptionString}`; - - const psqlCopyString = `psql "postgres://${encodeURIComponent( - userStr, - )}@${envAddress}/materialize?${queryParams}"`; - - const [host, port] = envAddress.split(":"); - return ( - - Host - - Port - - Database - - User - - - ), - icon: , - }, - { - title: "Terminal", - contents: psqlCopyString, - icon: , - }, - { - title: "MCP Server", - children: ( - - ), - icon: , - }, - ]} - onTabChange={onTabChange} - minHeight="208px" - {...props} - /> - ); -}; - -export default ConnectInstructions; diff --git a/console/src/components/ConnectModal.tsx b/console/src/components/ConnectModal.tsx deleted file mode 100644 index ca2ee13bd31e4..0000000000000 --- a/console/src/components/ConnectModal.tsx +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { - Box, - Button, - Flex, - HStack, - ModalBody, - ModalCloseButton, - ModalContent, - ModalHeader, - ModalOverlay, - Spinner, - Text, - useTheme, - VStack, -} from "@chakra-ui/react"; -import React, { useState } from "react"; - -import ConnectInstructions from "~/components/ConnectInstructions"; -import { Modal } from "~/components/Modal"; -import { User } from "~/external-library-wrappers/frontegg"; -import docUrls from "~/mz-doc-urls.json"; -import { useCreateApiToken } from "~/queries/frontegg"; -import { useListApiTokens } from "~/queries/frontegg"; -import { MaterializeTheme } from "~/theme"; -import { toBase64 } from "~/utils/format"; - -import { SecretCopyableBox } from "./copyableComponents"; -import SupportLink from "./SupportLink"; -import TextLink from "./TextLink"; - -export const NEW_USER_DEFAULT_PASSWORD_NAME = "App password"; - -/** - * A modal that displays Materialize connection instructions - */ -const ConnectModal = ({ - onClose, - isOpen, - forAppPassword, - user, -}: { - onClose: () => void; - isOpen: boolean; - forAppPassword?: { - user: string; - }; - user: User; -}) => { - const { colors } = useTheme(); - const showCreateAppPassword = !forAppPassword; - const [activeTab, setActiveTab] = useState("External tools"); - const { - mutate: createAppPassword, - isPending: createInProgress, - data: newPassword, - } = useCreateApiToken(); - - const mcpUser = forAppPassword?.user ?? user.email; - const mcpBase64Token = newPassword?.password - ? toBase64(`${mcpUser}:${newPassword.password}`) - : undefined; - - const isMcpTab = activeTab === "MCP Server"; - - return ( - - - - Connect To Materialize - - - - Below are the details to connect to Materialize - {forAppPassword && " with this app password"}. If you need more - information you can{" "} - - view the documentation - {" "} - or Contact Support. - - - createAppPassword({ - type: "personal", - description: "MCP token", - }) - } - isGeneratingToken={createInProgress} - mt="4" - /> - {showCreateAppPassword && !isMcpTab && ( - - - - )} - - - - ); -}; - -interface CreateAppPasswordProps { - user: User; - createAppPassword: ReturnType["mutate"]; - createInProgress: boolean; - newPassword: ReturnType["data"]; -} - -const CreateAppPassword = (props: CreateAppPasswordProps) => { - const { colors } = useTheme(); - - return ( - - Loading... - - } - > - - - ); -}; - -const CreateAppPasswordInner = ({ - user, - createAppPassword, - createInProgress, - newPassword, -}: CreateAppPasswordProps) => { - const { data: appPasswords } = useListApiTokens({ user }); - const { colors } = useTheme(); - - if (createInProgress) { - return ( - - - Generating new app password... - - ); - } - - if (newPassword?.password) { - return ( - <> - - - App password - - - - - Copy this somewhere safe. App passwords cannot be displayed after - initial creation. - - - ); - } - - return ( - <> - - - - Create an app password - - - Create a new app password if you don't have one accessible. - - - - - - ); -}; - -export default ConnectModal; diff --git a/console/src/components/McpConnectInstructions.tsx b/console/src/components/McpConnectInstructions.tsx deleted file mode 100644 index caa9845e6d2df..0000000000000 --- a/console/src/components/McpConnectInstructions.tsx +++ /dev/null @@ -1,278 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { - BoxProps, - Button, - Flex, - Spinner, - Text, - useTheme, - VStack, -} from "@chakra-ui/react"; -import { useAtom } from "jotai"; -import React from "react"; - -import { useAppConfig } from "~/config/useAppConfig"; -import docUrls from "~/mz-doc-urls.json"; -import { - currentEnvironmentState, - useEnvironmentGate, -} from "~/store/environments"; -import { MaterializeTheme } from "~/theme"; -import { obfuscateSecret } from "~/utils/format"; - -import { - CopyableBox, - SecretCopyableBox, - TabbedCodeBlock, -} from "./copyableComponents"; -import TextLink from "./TextLink"; - -interface McpConnectInstructionsProps extends BoxProps { - userStr: string; - /** Self-managed deployment with OIDC enabled; gets the browser OAuth flow. */ - oidcEnabled?: boolean; - /** Pre-computed Base64 token for MCP configuration (cloud only). */ - mcpBase64Token?: string; - /** Callback to generate a new MCP token (creates an app password). */ - onGenerateToken?: () => void; - /** Whether token generation is in progress. */ - isGeneratingToken?: boolean; -} - -const McpConnectInstructions = ({ - userStr, - oidcEnabled = false, - mcpBase64Token, - onGenerateToken, - isGeneratingToken, - ...props -}: McpConnectInstructionsProps) => { - const { colors } = useTheme(); - const [currentEnvironment] = useAtom(currentEnvironmentState); - const appConfig = useAppConfig(); - const isCloud = appConfig.mode === "cloud"; - // Envs >= 26.30.0 advertise OAuth via RFC 9728, so MCP clients log in through - // the browser instead of using a Basic-auth token. Applies to cloud (Frontegg) - // and self-managed with OIDC. The version must match region-controller's gate - // for `--frontegg-oauth-issuer-url` (precedence >= 26.30.0). Older/pre-release - // envs, and self-managed without OIDC, stay on the token flow. - const oauthAvailable = - useEnvironmentGate("26.30.0") === true && (isCloud || oidcEnabled); - - const envAddress = - currentEnvironment?.state === "enabled" - ? currentEnvironment.httpAddress - : undefined; - - if (!envAddress) return null; - - // MCP is served on the HTTP endpoint, so build its URL from the HTTP host. - // `envAddress` is `currentEnvironment.httpAddress`, the same address the - // console uses for its own HTTP API calls (for self-managed, the console's - // own host, which nginx proxies to environmentd). The pgwire host advertised - // in `balancerdDnsNames` does not serve MCP. - const baseUrl = isCloud - ? `https://${envAddress.split(":")[0]}` - : `${appConfig.environmentdScheme}://${envAddress}`; - - const user = userStr || ""; - const base64Command = `printf '${user}:' | base64 -w0`; - - const oauthCommand = (ep: "agent" | "developer") => - `claude mcp add --transport http "materialize-${ep}" \\\n "${baseUrl}/api/mcp/${ep}"`; - const tokenCommand = (ep: "agent" | "developer") => - `claude mcp add --transport http "materialize-${ep}" \\\n "${baseUrl}/api/mcp/${ep}" \\\n --header "Authorization: Basic "`; - - // App-password / token acquisition steps, shared by every endpoint that - // offers the token method (the Agent tab always, the Developer tab only when - // OAuth is unavailable). - const tokenInstructions = ( - - {isCloud && onGenerateToken && ( - - - Generate a new token: - - {isGeneratingToken ? ( - - - Generating token... - - ) : mcpBase64Token ? ( - - - - Copy this somewhere safe. Tokens cannot be displayed after - initial creation. - - - ) : ( - <> - - - For service accounts, create a{" "} - - service app password - {" "} - and Base64-encode it below. - - - )} - - )} - - - {isCloud ? ( - - Or Base64-encode an existing app password: - - ) : ( - - Base64-encode your username and password: - - )} - - - - ); - - return ( - - - Connect your AI agent or coding assistant to Materialize using the - built-in MCP server. - - - - Connect your client - - See the{" "} - - documentation - {" "} - for connecting clients like Claude Desktop, Cursor, Windsurf, etc. - - - Or connect to Claude Code now: - - - - Expose real-time data products to AI agents via - Materialize's built-in MCP endpoint. Used by both - service accounts and signed-in users. - - - {oauthAvailable && ( - - - Sign in with your account - - - - Your browser opens to sign in to Materialize. - - - )} - - - {oauthAvailable && ( - - Or use a service account - - )} - {tokenInstructions} - - - - ), - }, - { - title: "Developer", - children: ( - - - Query Materialize system catalog tables for troubleshooting - and observability via the built-in MCP developer endpoint. - - - {oauthAvailable ? ( - - - - Your browser opens to sign in to Materialize. App - passwords also work. See the{" "} - - documentation - - . - - - ) : ( - - {tokenInstructions} - - - )} - - ), - }, - ]} - /> - - - ); -}; - -export default McpConnectInstructions; diff --git a/console/src/components/McpConnectModal.tsx b/console/src/components/McpConnectModal.tsx deleted file mode 100644 index aba97296dd57e..0000000000000 --- a/console/src/components/McpConnectModal.tsx +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { - ModalBody, - ModalCloseButton, - ModalContent, - ModalHeader, - ModalOverlay, -} from "@chakra-ui/react"; -import React from "react"; - -import McpConnectInstructions from "~/components/McpConnectInstructions"; -import { Modal } from "~/components/Modal"; - -export interface McpConnectModalProps { - onClose: () => void; - isOpen: boolean; - userStr?: string; -} - -const McpConnectModal = ({ - onClose, - isOpen, - userStr, -}: McpConnectModalProps) => { - return ( - - - - Connect to MCP Server - - - "} /> - - - - ); -}; - -export default McpConnectModal; diff --git a/console/src/components/OidcConnectModal.tsx b/console/src/components/OidcConnectModal.tsx deleted file mode 100644 index c727758e2bbf3..0000000000000 --- a/console/src/components/OidcConnectModal.tsx +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { - Code, - ModalBody, - ModalCloseButton, - ModalContent, - ModalHeader, - ModalOverlay, - Text, - useTheme, - VStack, -} from "@chakra-ui/react"; -import React from "react"; - -import ConnectInstructions from "~/components/ConnectInstructions"; -import { - SecretCopyableBox, - TabbedCodeBlock, -} from "~/components/copyableComponents"; -import McpConnectInstructions from "~/components/McpConnectInstructions"; -import { Modal } from "~/components/Modal"; -import TextLink from "~/components/TextLink"; -import { useAppConfig } from "~/config/useAppConfig"; -import { type AuthContextProps } from "~/external-library-wrappers/oidc"; -import { useSelfManagedProfile } from "~/hooks/useSelfManagedProfile"; -import ConnectionIcon from "~/svg/ConnectionIcon"; -import { MaterializeTheme } from "~/theme"; -import { obfuscateSecret } from "~/utils/format"; - -const OIDC_USERNAME_PLACEHOLDER = ""; -const HOST_PLACEHOLDER = ""; - -const OidcConnectModal = ({ - onClose, - isOpen, - auth, -}: { - onClose: () => void; - isOpen: boolean; - auth: AuthContextProps; -}) => { - const { colors } = useTheme(); - const appConfig = useAppConfig(); - const idToken = auth.user?.id_token; - - const { sqlRole } = useSelfManagedProfile(auth); - const userStr = sqlRole ?? OIDC_USERNAME_PLACEHOLDER; - - const obfuscated = idToken ? obfuscateSecret(idToken) : ""; - - const balancerdHost = - appConfig.mode === "self-managed" - ? appConfig.balancerdDnsNames?.[0] - : undefined; - const sqlAddress = balancerdHost - ? `${balancerdHost}:6875` - : `${HOST_PLACEHOLDER}:6875`; - - return ( - - - - Connect To Materialize - - - - {!balancerdHost && ( - <> - Replace {HOST_PLACEHOLDER} with your - Materialize SQL endpoint.{" "} - - )} - See the{" "} - - installation docs - {" "} - for full setup details. When prompted for a password, paste the ID - token below. - - {idToken ? ( - - - - ID Token - - - When prompted for a password, paste this ID token. - - - - ) : ( - - ), - icon: , - }, - ]} - minHeight="208px" - /> - )} - - - - ); -}; - -export default OidcConnectModal; diff --git a/console/src/components/PasswordConnectModal.tsx b/console/src/components/PasswordConnectModal.tsx deleted file mode 100644 index 054a363e7364a..0000000000000 --- a/console/src/components/PasswordConnectModal.tsx +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { - ModalBody, - ModalCloseButton, - ModalContent, - ModalHeader, - ModalOverlay, - Text, - useTheme, -} from "@chakra-ui/react"; -import React from "react"; - -import McpConnectInstructions from "~/components/McpConnectInstructions"; -import { Modal } from "~/components/Modal"; -import ConnectionIcon from "~/svg/ConnectionIcon"; -import { MaterializeTheme } from "~/theme"; - -import { TabbedCodeBlock } from "./copyableComponents"; - -const PASSWORD_USERNAME_PLACEHOLDER = ""; - -const PasswordConnectModal = ({ - onClose, - isOpen, -}: { - onClose: () => void; - isOpen: boolean; -}) => { - const { colors } = useTheme(); - - return ( - - - - Connect To Materialize - - - - Use the details below to connect your AI agent to Materialize. - - - ), - icon: , - }, - ]} - minHeight="208px" - /> - - - - ); -}; - -export default PasswordConnectModal; diff --git a/console/src/components/connect/ConnectDrawer.test.tsx b/console/src/components/connect/ConnectDrawer.test.tsx new file mode 100644 index 0000000000000..6d790efe01680 --- /dev/null +++ b/console/src/components/connect/ConnectDrawer.test.tsx @@ -0,0 +1,233 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { http, HttpResponse } from "msw"; +import React from "react"; + +import server from "~/api/mocks/server"; +import { dummyValidUser } from "~/external-library-wrappers/__mocks__/frontegg"; +import { + defaultRegionId, + disabledEnvironment, + healthyEnvironment, + renderComponent, + setFakeEnvironment, +} from "~/test/utils"; +import { parseDbVersion } from "~/version/api"; + +import ConnectDrawer from "./ConnectDrawer"; + +// Matches `healthyEnvironment` in test/utils.tsx. +const ENVIRONMENT_HOST = "8zpze6ltqnsjok9vvf2i99st5.us-east-1.aws.example.com"; + +const renderDrawer = () => + renderComponent( + , + ); + +describe("ConnectDrawer", () => { + it("shows the MCP server instructions by default", async () => { + await renderDrawer(); + + expect(await screen.findByText("Choose your MCP server")).toBeVisible(); + expect(screen.getByText("Select your client")).toBeVisible(); + // The test environment predates the OAuth gate, so the token flow shows. + expect( + screen.getByRole("button", { name: "Generate app password" }), + ).toBeVisible(); + expect(screen.getByText("Add the MCP server")).toBeVisible(); + expect( + screen.getByText(new RegExp(`https://${ENVIRONMENT_HOST}`)), + ).toBeVisible(); + expect(screen.getByText(/materialize-developer/)).toBeVisible(); + expect(screen.getByText("Install agent skills (optional)")).toBeVisible(); + }); + + it("switches MCP config format per client", async () => { + await renderDrawer(); + const user = userEvent.setup(); + + await user.click(await screen.findByRole("button", { name: "MCP client" })); + await user.click(await screen.findByRole("menuitem", { name: "Cursor" })); + + expect(await screen.findByText("Add to ~/.cursor/mcp.json")).toBeVisible(); + expect(screen.getByText(/mcpServers/)).toBeVisible(); + }); + + it("switches to the agent MCP server", async () => { + await renderDrawer(); + const user = userEvent.setup(); + + await user.click(await screen.findByText("Query your data products")); + + expect(await screen.findByText(/materialize-agent/)).toBeVisible(); + expect( + screen.queryByText("Install agent skills (optional)"), + ).not.toBeInTheDocument(); + }); + + it("reports that a disabled region cannot serve connection details", async () => { + await renderComponent( + , + { + initializeState: ({ set }) => + setFakeEnvironment(set, defaultRegionId, disabledEnvironment), + }, + ); + + expect( + await screen.findByText(/Connection details are unavailable/), + ).toBeVisible(); + expect( + screen.queryByText("Choose your MCP server"), + ).not.toBeInTheDocument(); + }); + + it("shows the OAuth flow with an authenticate step on gated environments", async () => { + await renderComponent( + , + { + initializeState: ({ set }) => + setFakeEnvironment(set, defaultRegionId, { + ...healthyEnvironment, + status: { + health: "healthy", + errors: [], + version: parseDbVersion("v26.30.0 (abcdef123)"), + }, + }), + }, + ); + const user = userEvent.setup(); + + expect(await screen.findByText("Authenticate")).toBeVisible(); + expect(screen.getByText("claude /mcp")).toBeVisible(); + expect( + screen.getByText(/Select the materialize-developer server/), + ).toBeVisible(); + // The OAuth command carries no credential. + expect(screen.queryByText(/Authorization/)).not.toBeInTheDocument(); + + await user.click(screen.getByText("Use a token instead")); + + expect( + await screen.findByRole("button", { name: "Generate app password" }), + ).toBeVisible(); + expect(screen.queryByText("Authenticate")).not.toBeInTheDocument(); + expect(screen.getByText("Use OAuth instead")).toBeVisible(); + }); + + it("generates an MCP token and masks it in the command", async () => { + server.use( + http.post("*/frontegg/identity/resources/users/api-tokens/v1", () => + HttpResponse.json({ + clientId: "11111111-1111-1111-1111-111111111111", + secret: "22222222-2222-2222-2222-222222222222", + createdAt: "2026-01-01T00:00:00Z", + description: "MCP token", + metadata: {}, + }), + ), + ); + await renderDrawer(); + const user = userEvent.setup(); + + await user.click( + await screen.findByRole("button", { name: "Generate app password" }), + ); + + expect( + await screen.findByText(/App password created and added to the command/), + ).toBeVisible(); + expect(screen.getByText(/Authorization: Basic \*+/)).toBeVisible(); + }); + + it("shows connection details on the External tools tab", async () => { + await renderDrawer(); + const user = userEvent.setup(); + + await user.click( + await screen.findByRole("button", { name: /External tools/ }), + ); + + expect(await screen.findByLabelText("Host")).toHaveTextContent( + ENVIRONMENT_HOST, + ); + expect(screen.getByLabelText("Port")).toHaveTextContent("6875"); + expect(screen.getByLabelText("Database")).toHaveTextContent("materialize"); + expect(screen.getByLabelText("User")).toHaveTextContent( + dummyValidUser.email, + ); + expect( + screen.getByText(/In DBeaver, create a new PostgreSQL/), + ).toBeVisible(); + }); + + it("creates an app password and substitutes it into tool snippets", async () => { + server.use( + http.post("*/frontegg/identity/resources/users/api-tokens/v1", () => + HttpResponse.json({ + clientId: "11111111-1111-1111-1111-111111111111", + secret: "22222222-2222-2222-2222-222222222222", + createdAt: "2026-01-01T00:00:00Z", + description: "External tools", + metadata: {}, + }), + ), + ); + await renderDrawer(); + const user = userEvent.setup(); + + await user.click( + await screen.findByRole("button", { name: /External tools/ }), + ); + await user.click( + await screen.findByRole("button", { name: "Create new password" }), + ); + const nameInput = await screen.findByLabelText("App password name"); + expect(nameInput).toHaveValue(""); + expect(screen.getByText(/You are naming this password/)).toBeVisible(); + expect(screen.getByRole("button", { name: "Create" })).toBeDisabled(); + await user.type(nameInput, "External tools"); + await user.click(screen.getByRole("button", { name: "Create" })); + + await waitFor(() => { + expect( + screen.getByText(/Copy it now\. It will not be shown again\./), + ).toBeVisible(); + }); + // The tool snippet shows the password masked. Copying yields the real one. + const expectedPassword = `mzp_${"1".repeat(32)}${"2".repeat(32)}`; + expect( + screen.getByText( + new RegExp(`Password\\s+\\*{${expectedPassword.length}}`), + ), + ).toBeVisible(); + expect( + screen.queryByText(new RegExp(expectedPassword)), + ).not.toBeInTheDocument(); + }); + + it("shows the psql command on the Terminal tab", async () => { + await renderDrawer(); + const user = userEvent.setup(); + + await user.click(await screen.findByRole("button", { name: /Terminal/ })); + + expect( + await screen.findByText( + new RegExp( + `psql "postgres://${encodeURIComponent(dummyValidUser.email)}@${ENVIRONMENT_HOST}:6875/materialize\\?sslmode=require"`, + ), + ), + ).toBeVisible(); + }); +}); diff --git a/console/src/components/connect/ConnectDrawer.tsx b/console/src/components/connect/ConnectDrawer.tsx new file mode 100644 index 0000000000000..a5d308fb46f57 --- /dev/null +++ b/console/src/components/connect/ConnectDrawer.tsx @@ -0,0 +1,255 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { Box, Divider, Flex, Grid, Spinner, VStack } from "@chakra-ui/react"; +import { useAtom } from "jotai"; +import React, { useState } from "react"; +import { useParams } from "react-router-dom"; + +import ErrorBox from "~/components/ErrorBox"; +import SideDrawer from "~/components/SideDrawer"; +import { useAppConfig } from "~/config/useAppConfig"; +import { User } from "~/external-library-wrappers/frontegg"; +import { type AuthContextProps } from "~/external-library-wrappers/oidc"; +import { useSelfManagedProfile } from "~/hooks/useSelfManagedProfile"; +import { ConnectionIcon, MonitorIcon, TerminalIcon } from "~/icons"; +import { ClusterDetailParams } from "~/platform/clusters/ClusterRoutes"; +import { + currentEnvironmentState, + Environment, + useEnvironmentGate, +} from "~/store/environments"; + +import { ConnectMethodCard } from "./connectComponents"; +import { ConnectExternalToolsPanel } from "./ConnectExternalToolsPanel"; +import { ConnectMcpPanel } from "./ConnectMcpPanel"; +import { ConnectContext } from "./connectOptions"; +import { ConnectTerminalPanel } from "./ConnectTerminalPanel"; + +const OIDC_USERNAME_PLACEHOLDER = ""; +const PASSWORD_USERNAME_PLACEHOLDER = ""; +const HOST_PLACEHOLDER = ""; + +/** + * First version that advertises OAuth via RFC 9728, letting MCP clients log in + * through the browser instead of using a Basic-auth token. Must match + * region-controller's gate for `--frontegg-oauth-issuer-url`. + */ +const MCP_OAUTH_MIN_VERSION = "26.30.0"; + +type ConnectMethodId = "mcp" | "external-tools" | "terminal"; + +const CONNECT_METHODS: { + id: ConnectMethodId; + label: string; + sublabel: string; + icon: React.ReactNode; +}[] = [ + { + id: "mcp", + label: "MCP Server", + sublabel: "Connect your agent", + icon: , + }, + { + id: "external-tools", + label: "External tools", + sublabel: "Connection details", + icon: , + }, + { + id: "terminal", + label: "Terminal", + sublabel: "psql", + icon: , + }, +]; + +const ConnectPanels = ({ ctx }: { ctx: ConnectContext }) => { + const [methodId, setMethodId] = useState("mcp"); + + return ( + + + {CONNECT_METHODS.map((method) => ( + setMethodId(method.id)} + /> + ))} + + + {methodId === "mcp" && } + {methodId === "external-tools" && } + {methodId === "terminal" && } + + ); +}; + +const LoadingPanel = () => ( + + + +); + +/** Shown when the region cannot serve connection details, so the drawer does + * not spin forever on an environment that will never come up. */ +const UnavailablePanel = () => ( + +); + +/** Environments still resolving get a spinner, ones known not to be running + * get an error. */ +const environmentPanel = (environment: Environment | undefined) => { + if (environment?.state === "disabled") return ; + return ; +}; + +const CloudConnectContent = ({ + user, + forAppPassword, +}: { + user?: User; + forAppPassword?: { user: string }; +}) => { + const [currentEnvironment] = useAtom(currentEnvironmentState); + const { clusterName } = useParams(); + const oauthAvailable = useEnvironmentGate(MCP_OAUTH_MIN_VERSION) === true; + + if (currentEnvironment?.state !== "enabled") { + return environmentPanel(currentEnvironment); + } + + const [host, port] = currentEnvironment.sqlAddress.split(":"); + // MCP is served on the HTTP endpoint, so build its URL from the HTTP host. + // The pgwire host does not serve MCP. + const mcpBaseUrl = `https://${currentEnvironment.httpAddress.split(":")[0]}`; + + const ctx: ConnectContext = { + isCloud: true, + user: forAppPassword?.user ?? user?.email ?? "", + host, + port, + database: "materialize", + ssl: true, + mcpBaseUrl, + oauthAvailable, + clusterName, + canCreateAppPassword: !forAppPassword, + }; + + return ; +}; + +const SelfManagedConnectContent = ({ + auth, + oidcEnabled = false, +}: { + auth?: AuthContextProps; + oidcEnabled?: boolean; +}) => { + const appConfig = useAppConfig(); + const [currentEnvironment] = useAtom(currentEnvironmentState); + const { clusterName } = useParams(); + const { sqlRole } = useSelfManagedProfile(auth); + // The browser flow additionally requires OIDC on self-managed. Password auth + // has no issuer to redirect to, so those deployments stay on tokens. + const oauthAvailable = + useEnvironmentGate(MCP_OAUTH_MIN_VERSION) === true && oidcEnabled; + + if ( + appConfig.mode !== "self-managed" || + currentEnvironment?.state !== "enabled" + ) { + return environmentPanel(currentEnvironment); + } + + const balancerdHost = appConfig.balancerdDnsNames?.[0]; + const fallbackUser = oidcEnabled + ? OIDC_USERNAME_PLACEHOLDER + : PASSWORD_USERNAME_PLACEHOLDER; + // The console's own HTTP address serves MCP (nginx proxies to environmentd). + // The pgwire host advertised in `balancerdDnsNames` does not serve MCP. + const mcpBaseUrl = `${appConfig.environmentdScheme}://${currentEnvironment.httpAddress}`; + + const ctx: ConnectContext = { + isCloud: false, + user: sqlRole ?? fallbackUser, + host: balancerdHost ?? HOST_PLACEHOLDER, + port: "6875", + database: "materialize", + ssl: appConfig.environmentdScheme === "https", + mcpBaseUrl, + oauthAvailable, + clusterName, + canCreateAppPassword: false, + idToken: oidcEnabled ? auth?.user?.id_token : undefined, + }; + + return ; +}; + +export interface ConnectDrawerProps { + isOpen: boolean; + onClose: () => void; + /** Cloud sessions: the signed-in Frontegg user. */ + user?: User; + /** Self-managed sessions: the OIDC auth context, when available. */ + auth?: AuthContextProps; + /** Self-managed deployment with OIDC enabled; gets the browser OAuth flow. */ + oidcEnabled?: boolean; + /** Pin the instructions to a specific user, e.g. a service account whose + * app password was just created. Hides app password creation. */ + forAppPassword?: { + user: string; + }; +} + +/** Side drawer with instructions for connecting agents, external tools, and + * psql to Materialize. */ +const ConnectDrawer = ({ + isOpen, + onClose, + user, + auth, + oidcEnabled, + forAppPassword, +}: ConnectDrawerProps) => { + const appConfig = useAppConfig(); + + return ( + + + {appConfig.mode === "cloud" ? ( + + ) : ( + + )} + + + ); +}; + +export default ConnectDrawer; diff --git a/console/src/components/connect/ConnectExternalToolsPanel.tsx b/console/src/components/connect/ConnectExternalToolsPanel.tsx new file mode 100644 index 0000000000000..854db8d0d66ba --- /dev/null +++ b/console/src/components/connect/ConnectExternalToolsPanel.tsx @@ -0,0 +1,286 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { + Box, + Button, + HStack, + Input, + Text, + useTheme, + VStack, +} from "@chakra-ui/react"; +import React, { useState } from "react"; + +import { SecretCopyableBox } from "~/components/copyableComponents"; +import TextLink from "~/components/TextLink"; +import { useCreateApiToken } from "~/queries/frontegg"; +import { MaterializeTheme } from "~/theme"; +import { obfuscateSecret } from "~/utils/format"; + +import { + ConnectionDetailRow, + ConnectStep, + LabeledCommandBox, +} from "./connectComponents"; +import { + ConnectContext, + EXTERNAL_TOOLS, + ExternalToolId, +} from "./connectOptions"; + +const DETAIL_LABEL_WIDTH = "88px"; + +/** Inline app password creation: name it, create it, copy it once. */ +const CreateAppPasswordRow = ({ + onCreated, +}: { + onCreated: (password: string) => void; +}) => { + const { colors } = useTheme(); + const [isNaming, setIsNaming] = useState(false); + const [name, setName] = useState(""); + const { + mutate: createAppPassword, + isPending, + data: newPassword, + } = useCreateApiToken(); + + React.useEffect(() => { + if (newPassword?.password) { + onCreated(newPassword.password); + } + }, [newPassword, onCreated]); + + const create = () => { + if (name.trim().length === 0 || isPending) return; + createAppPassword({ type: "personal", description: name.trim() }); + }; + + return ( + + + + Password + + {newPassword?.password ? ( + + ) : isNaming ? ( + + setName(event.target.value)} + maxW="220px" + isDisabled={isPending} + placeholder="Password name" + aria-label="App password name" + autoFocus + onKeyDown={(event) => { + if (event.key === "Enter") create(); + }} + /> + + + + ) : ( + + )} + + {isNaming && !newPassword?.password && ( + + You are naming this password. The password itself is generated for + you. + + )} + {newPassword?.password && ( + + Copy it now. It will not be shown again. Manage in{" "} + App Passwords. + + )} + + ); +}; + +/** Self-managed OIDC deployments use the ID token as the SQL password. */ +const IdTokenRow = ({ idToken }: { idToken: string }) => { + const { colors } = useTheme(); + + return ( + + + + Password + + + + + When prompted for a password, paste this ID token. + + + ); +}; + +export interface ConnectExternalToolsPanelProps { + ctx: ConnectContext; +} + +/** External tools tab: connection details plus per-tool config snippets. */ +export const ConnectExternalToolsPanel = ({ + ctx, +}: ConnectExternalToolsPanelProps) => { + const { colors } = useTheme(); + const [toolId, setToolId] = useState("dbeaver"); + const [createdPassword, setCreatedPassword] = useState(); + + const tool = + EXTERNAL_TOOLS.find((candidate) => candidate.id === toolId) ?? + EXTERNAL_TOOLS[0]; + const snippet = tool.buildSnippet({ + host: ctx.host, + port: ctx.port, + database: ctx.database, + user: ctx.user, + password: createdPassword, + ssl: ctx.ssl, + }); + // Mask the password in the rendered snippet. The copy button copies the + // real value. + const displaySnippet = createdPassword + ? snippet.replaceAll(createdPassword, obfuscateSecret(createdPassword)) + : undefined; + + return ( + + + + + + + + {ctx.canCreateAppPassword && ( + + )} + {ctx.idToken && } + + + + + + {EXTERNAL_TOOLS.map((candidate) => { + const isActive = candidate.id === toolId; + return ( + setToolId(candidate.id)} + > + {candidate.label} + + ); + })} + + + + + + ); +}; diff --git a/console/src/components/connect/ConnectMcpPanel.tsx b/console/src/components/connect/ConnectMcpPanel.tsx new file mode 100644 index 0000000000000..bc45d9b4a6d4a --- /dev/null +++ b/console/src/components/connect/ConnectMcpPanel.tsx @@ -0,0 +1,433 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { + Box, + Button, + Flex, + forwardRef, + Grid, + HStack, + Menu, + MenuButton, + MenuItem, + MenuList, + RadioProps, + Spinner, + Text, + useRadio, + useRadioGroup, + useTheme, + VStack, +} from "@chakra-ui/react"; +import React, { useState } from "react"; + +import { SecretCopyableBox } from "~/components/copyableComponents"; +import TextLink from "~/components/TextLink"; +import { + ChevronDownIcon, + ClaudeLogoIcon, + CodexLogoIcon, + CursorLogoIcon, + GeminiLogoIcon, + OverflowMenuIcon, + VsCodeLogoIcon, + WindsurfLogoIcon, +} from "~/icons"; +import { useCreateApiToken } from "~/queries/frontegg"; +import { MaterializeTheme } from "~/theme"; +import { obfuscateSecret, toBase64 } from "~/utils/format"; + +import { ConnectStep, LabeledCommandBox } from "./connectComponents"; +import { + AGENT_SKILLS_COMMAND, + buildBase64TokenCommand, + buildMcpInstallLink, + buildMcpSnippet, + ConnectContext, + MCP_CLIENTS, + MCP_SERVERS, + MCP_TOKEN_PLACEHOLDER, + McpClientId, + McpServerId, + PASSWORD_PLACEHOLDER, + resolveSignInHint, +} from "./connectOptions"; + +const CLIENT_ICONS: Record = { + "claude-code": , + cursor: , + "claude-desktop": , + vscode: , + windsurf: , + codex: , + "gemini-cli": , + other: , +}; + +const McpServerRadioCard = forwardRef< + RadioProps & { label: string; blurb: string }, + "input" +>(({ label, blurb, ...props }, ref) => { + const { colors, shadows } = useTheme(); + const { getInputProps, getRadioProps } = useRadio(props); + + return ( + + + + + {label} + + + {blurb} + + + + ); +}); + +export interface McpServerSelectProps { + value: McpServerId; + onChange: (id: McpServerId) => void; +} + +const McpServerSelect = ({ value, onChange }: McpServerSelectProps) => { + const { getRadioProps, getRootProps } = useRadioGroup({ + name: "mcp-server", + value, + onChange: (val) => onChange(val as McpServerId), + }); + + return ( + + {MCP_SERVERS.map((server) => ( + + ))} + + ); +}; + +export interface McpClientSelectProps { + value: McpClientId; + onChange: (id: McpClientId) => void; +} + +const McpClientSelect = ({ value, onChange }: McpClientSelectProps) => { + const { colors, shadows } = useTheme(); + const selected = + MCP_CLIENTS.find((client) => client.id === value) ?? MCP_CLIENTS[0]; + + return ( + + + + + {CLIENT_ICONS[selected.id]} + + {selected.name} + + + + + + + {MCP_CLIENTS.map((client) => ( + onChange(client.id)} gap="2"> + {CLIENT_ICONS[client.id]} + {client.name} + + ))} + + + ); +}; + +/** Content of the token acquisition step. Cloud sessions mint an app password + * that is substituted into the command, OIDC sessions get the token derived + * from their ID token, password-auth users Base64-encode their password. */ +const McpTokenStep = ({ + ctx, + hasToken, + isGenerating, + onGenerate, +}: { + ctx: ConnectContext; + hasToken: boolean; + isGenerating: boolean; + onGenerate: () => void; +}) => { + const { colors } = useTheme(); + + if (!ctx.canCreateAppPassword) { + if (ctx.idToken) { + const oidcToken = toBase64(`${ctx.user}:${ctx.idToken}`); + return ( + + + + Derived from your username and current ID token. It expires with + your session. + + + ); + } + return ( + + ); + } + + if (isGenerating) { + return ( + + + Generating app password... + + ); + } + + if (hasToken) { + return ( + + App password created and added to the command below. It will not be + shown again. + + ); + } + + return ( + + + + ); +}; + +export interface ConnectMcpPanelProps { + ctx: ConnectContext; +} + +/** MCP Server tab: pick a server, connect a client, install agent skills. */ +export const ConnectMcpPanel = ({ ctx }: ConnectMcpPanelProps) => { + const { colors } = useTheme(); + const [serverId, setServerId] = useState("developer"); + const [clientId, setClientId] = useState("claude-code"); + const [useTokenFlow, setUseTokenFlow] = useState(false); + + const { + mutate: createAppPassword, + isPending: isGeneratingToken, + data: newPassword, + } = useCreateApiToken(); + // Only cloud-generated tokens are substituted into the command. The + // OIDC-derived token is a full JWT, far too long to inline readably, so it + // stays in the token step's secret box. + const mcpToken = newPassword?.password + ? toBase64(`${ctx.user}:${newPassword.password}`) + : undefined; + + const server = + MCP_SERVERS.find((candidate) => candidate.id === serverId) ?? + MCP_SERVERS[0]; + const client = + MCP_CLIENTS.find((candidate) => candidate.id === clientId) ?? + MCP_CLIENTS[0]; + + const oauthActive = ctx.oauthAvailable && !useTokenFlow; + const isDeveloper = server.id === "developer"; + + const snippet = buildMcpSnippet({ + client, + server, + baseUrl: ctx.mcpBaseUrl, + token: oauthActive ? undefined : (mcpToken ?? MCP_TOKEN_PLACEHOLDER), + }); + // Mask the token in the rendered snippet. The copy button copies the real + // value. + const displaySnippet = mcpToken + ? snippet.replace(mcpToken, obfuscateSecret(mcpToken)) + : undefined; + const installLink = + oauthActive && client.oneClickInstall + ? buildMcpInstallLink(client, server, ctx.mcpBaseUrl) + : undefined; + + const clientStep = ( + + + Select your client + + + + ); + + const commandStep = ( + + {installLink && ( + + + + Installs the server config for you. + + + )} + + {ctx.oauthAvailable && ( + + setUseTokenFlow(oauthActive)} + > + {oauthActive ? "Use a token instead" : "Use OAuth instead"} + + + )} + + ); + + const authenticateStep = ( + + {client.authenticateCommand && ( + + )} + + {resolveSignInHint(client, server.serverName)} + + + ); + + const skillsStep = ( + + + Ready-made instructions that help your agent work with Materialize + accurately. + + + + ); + + const steps: { title: string; content: React.ReactNode }[] = [ + { + title: "Choose your MCP server", + content: , + }, + { title: "Connect your client", content: clientStep }, + ]; + if (!oauthActive) { + steps.push({ + title: ctx.canCreateAppPassword + ? "Generate an app password" + : ctx.idToken + ? "Copy your MCP token" + : "Create a token", + content: ( + + createAppPassword({ type: "personal", description: "MCP token" }) + } + /> + ), + }); + } + steps.push({ + title: + !oauthActive && client.tokenConfigLocation + ? client.tokenConfigLocation + : client.configLocation, + content: commandStep, + }); + if (oauthActive) { + steps.push({ title: "Authenticate", content: authenticateStep }); + } + if (isDeveloper) { + steps.push({ + title: "Install agent skills (optional)", + content: skillsStep, + }); + } + + return ( + + {steps.map((step, index) => ( + + {step.content} + + ))} + + ); +}; diff --git a/console/src/components/connect/ConnectTerminalPanel.tsx b/console/src/components/connect/ConnectTerminalPanel.tsx new file mode 100644 index 0000000000000..2b5ef5b0b7276 --- /dev/null +++ b/console/src/components/connect/ConnectTerminalPanel.tsx @@ -0,0 +1,50 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { Text, useTheme, VStack } from "@chakra-ui/react"; +import React from "react"; + +import { MaterializeTheme } from "~/theme"; + +import { LabeledCommandBox } from "./connectComponents"; +import { buildPsqlCommand, ConnectContext } from "./connectOptions"; + +export interface ConnectTerminalPanelProps { + ctx: ConnectContext; +} + +/** Terminal tab: the psql one-liner. */ +export const ConnectTerminalPanel = ({ ctx }: ConnectTerminalPanelProps) => { + const { colors } = useTheme(); + const command = buildPsqlCommand( + { + host: ctx.host, + port: ctx.port, + database: ctx.database, + user: ctx.user, + ssl: ctx.ssl, + }, + ctx.clusterName, + ); + + return ( + + + {ctx.idToken && ( + + When prompted for a password, paste the ID token from the External + tools tab. + + )} + + ); +}; diff --git a/console/src/components/connect/connectComponents.tsx b/console/src/components/connect/connectComponents.tsx new file mode 100644 index 0000000000000..4ba248cb5440e --- /dev/null +++ b/console/src/components/connect/connectComponents.tsx @@ -0,0 +1,178 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { Box, Flex, HStack, Text, useTheme, VStack } from "@chakra-ui/react"; +import React from "react"; + +import { CopyableBox } from "~/components/copyableComponents"; +import { MaterializeTheme } from "~/theme"; + +export interface ConnectStepProps { + stepNumber: number; + title: string; + isLast?: boolean; + children: React.ReactNode; +} + +/** Numbered step with a connector line to the next step. */ +export const ConnectStep = ({ + stepNumber, + title, + isLast = false, + children, +}: ConnectStepProps) => { + const { colors } = useTheme(); + + return ( + + + + {stepNumber} + + {!isLast && ( + + )} + + + + {title} + + {children} + + + ); +}; + +export interface LabeledCommandBoxProps { + contents: string; + label?: React.ReactNode; + /** Shown instead of `contents`, e.g. with secrets masked. Copying always + * copies `contents`. */ + displayContents?: string; +} + +/** A copyable command or config snippet with an optional instruction above. */ +export const LabeledCommandBox = ({ + contents, + label, + displayContents, +}: LabeledCommandBoxProps) => { + const { colors } = useTheme(); + + return ( + + {label && ( + + {label} + + )} + + {displayContents} + + + ); +}; + +export interface ConnectionDetailRowProps { + label: string; + contents: string; +} + +/** Label and copyable value row for connection details. */ +export const ConnectionDetailRow = ({ + label, + contents, +}: ConnectionDetailRowProps) => { + const { colors } = useTheme(); + + return ( + + + {label} + + + + ); +}; + +export interface ConnectMethodCardProps { + isActive: boolean; + label: string; + sublabel: string; + icon: React.ReactNode; + onClick: () => void; +} + +/** Top-level card for switching between connection methods. */ +export const ConnectMethodCard = ({ + isActive, + label, + sublabel, + icon, + onClick, +}: ConnectMethodCardProps) => { + const { colors } = useTheme(); + + return ( + + + {icon} + + + {label} + + + {sublabel} + + + ); +}; diff --git a/console/src/components/connect/connectOptions.test.ts b/console/src/components/connect/connectOptions.test.ts new file mode 100644 index 0000000000000..c004a381eea89 --- /dev/null +++ b/console/src/components/connect/connectOptions.test.ts @@ -0,0 +1,308 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { + buildBase64TokenCommand, + buildMcpInstallLink, + buildMcpSnippet, + buildPsqlCommand, + EXTERNAL_TOOLS, + MCP_CLIENTS, + MCP_SERVERS, + McpClientId, + McpServerId, +} from "./connectOptions"; + +const BASE_URL = "https://region.example.com"; + +const server = (id: McpServerId) => { + const found = MCP_SERVERS.find((candidate) => candidate.id === id); + if (!found) throw new Error(`unknown server ${id}`); + return found; +}; + +const client = (id: McpClientId) => { + const found = MCP_CLIENTS.find((candidate) => candidate.id === id); + if (!found) throw new Error(`unknown client ${id}`); + return found; +}; + +const tool = (id: string) => { + const found = EXTERNAL_TOOLS.find((candidate) => candidate.id === id); + if (!found) throw new Error(`unknown tool ${id}`); + return found; +}; + +const CONNECTION_DETAILS = { + host: "region.example.com", + port: "6875", + database: "materialize", + user: "user@example.com", + ssl: true, +}; + +const FAKE_PASSWORD = "mzp_abc123"; + +describe("buildMcpSnippet", () => { + it("builds a claude mcp add command for the OAuth flow", () => { + const snippet = buildMcpSnippet({ + client: client("claude-code"), + server: server("developer"), + baseUrl: BASE_URL, + }); + expect(snippet).toEqual( + `claude mcp add --transport http materialize-developer \\\n ${BASE_URL}/api/mcp/developer`, + ); + }); + + it("adds a Basic auth header to the command for the token flow", () => { + const snippet = buildMcpSnippet({ + client: client("claude-code"), + server: server("agent"), + baseUrl: BASE_URL, + token: "dG9rZW4=", + }); + expect(snippet).toContain(`${BASE_URL}/api/mcp/agent`); + expect(snippet).toContain('--header "Authorization: Basic dG9rZW4="'); + }); + + it("builds valid mcpServers JSON for JSON-configured clients", () => { + const snippet = buildMcpSnippet({ + client: client("cursor"), + server: server("developer"), + baseUrl: BASE_URL, + token: "dG9rZW4=", + }); + expect(JSON.parse(snippet)).toEqual({ + mcpServers: { + "materialize-developer": { + url: `${BASE_URL}/api/mcp/developer`, + headers: { Authorization: "Basic dG9rZW4=" }, + }, + }, + }); + }); + + it("omits headers from JSON configs for the OAuth flow", () => { + const snippet = buildMcpSnippet({ + client: client("cursor"), + server: server("developer"), + baseUrl: BASE_URL, + }); + expect(JSON.parse(snippet)).toEqual({ + mcpServers: { + "materialize-developer": { + url: `${BASE_URL}/api/mcp/developer`, + }, + }, + }); + }); + + it("roots VS Code configs at servers with an explicit http type", () => { + const snippet = buildMcpSnippet({ + client: client("vscode"), + server: server("developer"), + baseUrl: BASE_URL, + token: "dG9rZW4=", + }); + expect(JSON.parse(snippet)).toEqual({ + servers: { + "materialize-developer": { + type: "http", + url: `${BASE_URL}/api/mcp/developer`, + headers: { Authorization: "Basic dG9rZW4=" }, + }, + }, + }); + }); + + it("uses httpUrl for Gemini CLI configs", () => { + const snippet = buildMcpSnippet({ + client: client("gemini-cli"), + server: server("developer"), + baseUrl: BASE_URL, + token: "dG9rZW4=", + }); + expect(JSON.parse(snippet)).toEqual({ + mcpServers: { + "materialize-developer": { + httpUrl: `${BASE_URL}/api/mcp/developer`, + headers: { Authorization: "Basic dG9rZW4=" }, + }, + }, + }); + }); + + it("uses serverUrl for Windsurf configs", () => { + const snippet = buildMcpSnippet({ + client: client("windsurf"), + server: server("developer"), + baseUrl: BASE_URL, + }); + expect(JSON.parse(snippet)).toEqual({ + mcpServers: { + "materialize-developer": { + serverUrl: `${BASE_URL}/api/mcp/developer`, + }, + }, + }); + }); + + it("shows just the URL for Claude Desktop's OAuth connector flow", () => { + const snippet = buildMcpSnippet({ + client: client("claude-desktop"), + server: server("developer"), + baseUrl: BASE_URL, + }); + expect(snippet).toEqual(`${BASE_URL}/api/mcp/developer`); + }); + + it("proxies Claude Desktop's token flow through mcp-remote", () => { + const snippet = buildMcpSnippet({ + client: client("claude-desktop"), + server: server("developer"), + baseUrl: BASE_URL, + token: "dG9rZW4=", + }); + expect(JSON.parse(snippet)).toEqual({ + mcpServers: { + "materialize-developer": { + command: "npx", + args: [ + "mcp-remote", + `${BASE_URL}/api/mcp/developer`, + "--header", + "Authorization: Basic dG9rZW4=", + ], + }, + }, + }); + }); + + it("builds TOML with underscored table names for Codex", () => { + const snippet = buildMcpSnippet({ + client: client("codex"), + server: server("agent"), + baseUrl: BASE_URL, + token: "dG9rZW4=", + }); + expect(snippet).toEqual( + `[mcp_servers.materialize_agent]\nurl = "${BASE_URL}/api/mcp/agent"\nhttp_headers = { "Authorization" = "Basic dG9rZW4=" }`, + ); + }); +}); + +describe("buildMcpInstallLink", () => { + it("builds a Cursor deep link with Base64-encoded config", () => { + const link = buildMcpInstallLink( + client("cursor"), + server("developer"), + BASE_URL, + ); + expect(link).toBeDefined(); + const url = new URL(link ?? ""); + expect(url.protocol).toEqual("cursor:"); + expect(url.searchParams.get("name")).toEqual("materialize-developer"); + const config = JSON.parse(atob(url.searchParams.get("config") ?? "")); + expect(config).toEqual({ url: `${BASE_URL}/api/mcp/developer` }); + }); + + it("builds a VS Code deep link with URL-encoded config", () => { + const link = buildMcpInstallLink( + client("vscode"), + server("agent"), + BASE_URL, + ); + expect(link).toBeDefined(); + const [scheme, encoded] = (link ?? "").split("?"); + expect(scheme).toEqual("vscode:mcp/install"); + expect(JSON.parse(decodeURIComponent(encoded))).toEqual({ + name: "materialize-agent", + type: "http", + url: `${BASE_URL}/api/mcp/agent`, + }); + }); + + it("returns undefined for clients without one-click install", () => { + expect( + buildMcpInstallLink(client("claude-code"), server("developer"), BASE_URL), + ).toBeUndefined(); + }); +}); + +describe("EXTERNAL_TOOLS", () => { + it("shows a placeholder password until one is created", () => { + const snippet = tool("dbeaver").buildSnippet(CONNECTION_DETAILS); + expect(snippet).toContain("Password "); + expect(snippet).toContain("SSL require"); + }); + + it("substitutes a created app password", () => { + const snippet = tool("dbt").buildSnippet({ + ...CONNECTION_DETAILS, + password: FAKE_PASSWORD, + }); + expect(snippet).toContain(`pass: ${FAKE_PASSWORD}`); + expect(snippet).toContain("sslmode: require"); + }); + + it("builds a DATABASE_URL with the user encoded", () => { + const snippet = tool("env").buildSnippet({ + ...CONNECTION_DETAILS, + password: FAKE_PASSWORD, + }); + // Interpolated rather than written out, so the secret scanner does not + // read the assertion as a real connection string. + expect(snippet).toContain( + `DATABASE_URL=postgres://user%40example.com:${FAKE_PASSWORD}@region.example.com:6875/materialize?sslmode=require`, + ); + }); + + it("disables SSL for deployments without TLS", () => { + const snippet = tool("dbeaver").buildSnippet({ + ...CONNECTION_DETAILS, + ssl: false, + }); + expect(snippet).toContain("SSL disable"); + }); +}); + +describe("buildPsqlCommand", () => { + it("encodes the user and requires SSL", () => { + expect(buildPsqlCommand(CONNECTION_DETAILS)).toEqual( + 'psql "postgres://user%40example.com@region.example.com:6875/materialize?sslmode=require"', + ); + }); + + it("appends the cluster option when set", () => { + expect(buildPsqlCommand(CONNECTION_DETAILS, "quickstart")).toContain( + "sslmode=require&options=--cluster%3Dquickstart", + ); + }); + + it("omits query params without TLS or cluster", () => { + expect(buildPsqlCommand({ ...CONNECTION_DETAILS, ssl: false })).toEqual( + 'psql "postgres://user%40example.com@region.example.com:6875/materialize"', + ); + }); +}); + +describe("buildBase64TokenCommand", () => { + it("embeds the user with an app password placeholder by default", () => { + expect(buildBase64TokenCommand("user@example.com")).toEqual( + "printf '%s' 'user@example.com:' | base64 -w0", + ); + }); + + it("uses the deployment-specific credential placeholder", () => { + expect(buildBase64TokenCommand("user@example.com", "")).toEqual( + "printf '%s' 'user@example.com:' | base64 -w0", + ); + }); +}); diff --git a/console/src/components/connect/connectOptions.ts b/console/src/components/connect/connectOptions.ts new file mode 100644 index 0000000000000..8f4fbffe89af4 --- /dev/null +++ b/console/src/components/connect/connectOptions.ts @@ -0,0 +1,417 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { toBase64 } from "~/utils/format"; + +export const MCP_TOKEN_PLACEHOLDER = ""; +export const APP_PASSWORD_PLACEHOLDER = ""; +export const PASSWORD_PLACEHOLDER = ""; + +/** Deployment-derived inputs the connect panels render from. */ +export interface ConnectContext { + isCloud: boolean; + /** SQL user shown in connection strings. */ + user: string; + host: string; + port: string; + database: string; + /** Whether the SQL endpoint requires TLS. */ + ssl: boolean; + /** Scheme plus host serving the MCP endpoints, without a trailing slash. */ + mcpBaseUrl: string; + /** Whether MCP clients can sign in through the browser instead of using a + * Basic-auth token. */ + oauthAvailable: boolean; + /** Cluster to preselect in the psql connection string. */ + clusterName?: string; + /** Whether the panel offers creating app passwords (cloud sessions only). */ + canCreateAppPassword: boolean; + /** Self-managed OIDC ID token, used as the SQL password. */ + idToken?: string; +} + +export type McpServerId = "developer" | "agent"; + +export interface McpServer { + id: McpServerId; + label: string; + blurb: string; + /** Name the server is registered under in the client's MCP config. */ + serverName: string; + /** Path of the MCP endpoint on the environment's HTTP address. */ + path: string; +} + +export const MCP_SERVERS: McpServer[] = [ + { + id: "developer", + label: "Developer", + blurb: "Troubleshooting & observability", + serverName: "materialize-developer", + path: "/api/mcp/developer", + }, + { + id: "agent", + label: "Agent", + blurb: "Query your data products", + serverName: "materialize-agent", + path: "/api/mcp/agent", + }, +]; + +export type McpClientId = + | "claude-code" + | "cursor" + | "claude-desktop" + | "vscode" + | "windsurf" + | "codex" + | "gemini-cli" + | "other"; + +export type McpConfigKind = "cli" | "json" | "toml" | "connector"; + +export interface McpClient { + id: McpClientId; + name: string; + kind: McpConfigKind; + /** Where the config goes, used as the final step's title. */ + configLocation: string; + /** Overrides `configLocation` for the token flow. Used by clients whose + * token setup goes somewhere else than the OAuth setup. */ + tokenConfigLocation?: string; + /** Command that starts the client's sign-in, when it has one. Shown in the + * Authenticate step of the OAuth flow. */ + authenticateCommand?: (serverName: string) => string; + /** How the sign-in proceeds, shown in the Authenticate step. Use the + * function form when the text names the server. */ + signInHint: string | ((serverName: string) => string); + /** Whether the client supports one-click install via a deep link. */ + oneClickInstall?: boolean; +} + +// Config formats and sign-in behavior verified against each client's docs. +// When a client changes its config schema, this registry is the single place +// to update. +export const MCP_CLIENTS: McpClient[] = [ + { + id: "claude-code", + name: "Claude Code", + kind: "cli", + configLocation: "Add the MCP server", + authenticateCommand: () => "claude /mcp", + signInHint: (serverName) => + `Select the ${serverName} server, then Authenticate. Your browser opens to sign in.`, + }, + { + id: "cursor", + name: "Cursor", + kind: "json", + configLocation: "Add to ~/.cursor/mcp.json", + signInHint: + "Cursor prompts you to sign in when the server first connects. You can also click the server under Settings > MCP.", + oneClickInstall: true, + }, + { + // Claude Desktop's config file only launches local stdio servers. Remote + // servers are added as custom connectors in the app for OAuth, or through + // the mcp-remote proxy for the token flow. + id: "claude-desktop", + name: "Claude Desktop", + kind: "connector", + configLocation: "Add as a custom connector under Settings > Connectors", + tokenConfigLocation: "Add to claude_desktop_config.json", + signInHint: + "Claude Desktop opens your browser to sign in when you add the connector.", + }, + { + id: "vscode", + name: "VS Code", + kind: "json", + configLocation: "Add to .vscode/mcp.json", + signInHint: + "VS Code prompts you to authenticate the server the first time chat uses it.", + oneClickInstall: true, + }, + { + id: "windsurf", + name: "Windsurf", + kind: "json", + configLocation: "Add to ~/.codeium/windsurf/mcp_config.json", + signInHint: + "Windsurf opens your browser to sign in when the server first connects.", + }, + { + id: "codex", + name: "Codex CLI", + kind: "toml", + configLocation: "Add to ~/.codex/config.toml", + authenticateCommand: (serverName) => `codex mcp login ${serverName}`, + signInHint: "Your browser opens to sign in.", + }, + { + id: "gemini-cli", + name: "Gemini CLI", + kind: "json", + configLocation: "Add to ~/.gemini/settings.json", + signInHint: (serverName) => + `Run /mcp auth ${serverName} inside Gemini CLI. Your browser opens to sign in.`, + }, + { + id: "other", + name: "Other", + kind: "json", + configLocation: "Add to your client's MCP config", + signInHint: "Your client opens a browser to sign in on first connect.", + }, +]; + +export const resolveSignInHint = (client: McpClient, serverName: string) => + typeof client.signInHint === "function" + ? client.signInHint(serverName) + : client.signInHint; + +export interface McpSnippetOptions { + client: McpClient; + server: McpServer; + /** Scheme plus host serving the MCP endpoints, without a trailing slash. */ + baseUrl: string; + /** Base64 `user:app-password` token. Omitted for the browser OAuth flow. */ + token?: string; +} + +const mcpServerUrl = (server: McpServer, baseUrl: string) => + `${baseUrl}${server.path}`; + +const buildCliSnippet = ({ server, baseUrl, token }: McpSnippetOptions) => { + const header = token ? ` \\\n --header "Authorization: Basic ${token}"` : ""; + return `claude mcp add --transport http ${server.serverName} \\\n ${mcpServerUrl(server, baseUrl)}${header}`; +}; + +const buildJsonSnippet = ({ + client, + server, + baseUrl, + token, +}: McpSnippetOptions) => { + const url = mcpServerUrl(server, baseUrl); + const headers = token + ? { headers: { Authorization: `Basic ${token}` } } + : undefined; + // Each client expects a slightly different JSON schema for remote servers. + let config: Record; + switch (client.id) { + // VS Code roots at "servers" and needs an explicit transport type, + // otherwise it treats the entry as a stdio command. + case "vscode": + config = { + servers: { [server.serverName]: { type: "http", url, ...headers } }, + }; + break; + // Windsurf's canonical key for remote servers is "serverUrl". + case "windsurf": + config = { + mcpServers: { [server.serverName]: { serverUrl: url, ...headers } }, + }; + break; + // Gemini CLI uses "httpUrl" for the streamable HTTP transport. + case "gemini-cli": + config = { + mcpServers: { [server.serverName]: { httpUrl: url, ...headers } }, + }; + break; + default: + config = { mcpServers: { [server.serverName]: { url, ...headers } } }; + } + return JSON.stringify(config, null, 2); +}; + +/** Claude Desktop: the OAuth flow pastes the URL into the connectors UI, the + * token flow proxies through mcp-remote since the config file is stdio-only. */ +const buildConnectorSnippet = ({ + server, + baseUrl, + token, +}: McpSnippetOptions) => { + const url = mcpServerUrl(server, baseUrl); + if (!token) return url; + return JSON.stringify( + { + mcpServers: { + [server.serverName]: { + command: "npx", + args: [ + "mcp-remote", + url, + "--header", + `Authorization: Basic ${token}`, + ], + }, + }, + }, + null, + 2, + ); +}; + +const buildTomlSnippet = ({ server, baseUrl, token }: McpSnippetOptions) => { + const header = token + ? `\nhttp_headers = { "Authorization" = "Basic ${token}" }` + : ""; + const tableName = server.serverName.replace(/-/g, "_"); + return `[mcp_servers.${tableName}]\nurl = "${mcpServerUrl(server, baseUrl)}"${header}`; +}; + +/** Builds the client-specific MCP config snippet, using + * `MCP_TOKEN_PLACEHOLDER` when the token flow is active but no token has been + * generated yet. */ +export const buildMcpSnippet = (options: McpSnippetOptions): string => { + switch (options.client.kind) { + case "cli": + return buildCliSnippet(options); + case "toml": + return buildTomlSnippet(options); + case "connector": + return buildConnectorSnippet(options); + case "json": + return buildJsonSnippet(options); + } +}; + +/** One-click install link for clients that register a URL scheme handler. + * Only built for the OAuth flow so tokens never end up in URLs. */ +export const buildMcpInstallLink = ( + client: McpClient, + server: McpServer, + baseUrl: string, +): string | undefined => { + const config = { url: mcpServerUrl(server, baseUrl) }; + switch (client.id) { + case "cursor": + return `cursor://anysphere.cursor-deeplink/mcp/install?name=${encodeURIComponent( + server.serverName, + )}&config=${encodeURIComponent(toBase64(JSON.stringify(config)))}`; + case "vscode": + return `vscode:mcp/install?${encodeURIComponent( + JSON.stringify({ name: server.serverName, type: "http", ...config }), + )}`; + default: + return undefined; + } +}; + +export const AGENT_SKILLS_COMMAND = + "npx skills add MaterializeInc/agent-skills"; + +export interface ConnectionDetails { + host: string; + port: string; + database: string; + user: string; + /** Actual app password once created, otherwise callers show a placeholder. */ + password?: string; + /** Whether the endpoint requires TLS (cloud always does). */ + ssl: boolean; +} + +export type ExternalToolId = "dbeaver" | "dbt" | "env"; + +export interface ExternalTool { + id: ExternalToolId; + label: string; + /** Instruction shown above the snippet. */ + instruction: string; + buildSnippet: (details: ConnectionDetails) => string; +} + +const passwordOrPlaceholder = (details: ConnectionDetails) => + details.password ?? APP_PASSWORD_PLACEHOLDER; + +export const EXTERNAL_TOOLS: ExternalTool[] = [ + { + id: "dbeaver", + label: "DBeaver", + instruction: + "In DBeaver, create a new PostgreSQL connection with these settings:", + buildSnippet: (details) => + [ + `Host ${details.host}`, + `Port ${details.port}`, + `Database ${details.database}`, + `User ${details.user}`, + `Password ${passwordOrPlaceholder(details)}`, + `SSL ${details.ssl ? "require" : "disable"}`, + ].join("\n"), + }, + { + id: "dbt", + label: "dbt", + instruction: "Add this profile to your dbt profiles.yml:", + buildSnippet: (details) => + [ + "materialize:", + " target: prod", + " outputs:", + " prod:", + " type: materialize", + ` host: ${details.host}`, + ` port: ${details.port}`, + ` user: ${details.user}`, + ` pass: ${passwordOrPlaceholder(details)}`, + ` database: ${details.database}`, + ` sslmode: ${details.ssl ? "require" : "disable"}`, + ].join("\n"), + }, + { + id: "env", + label: ".env", + instruction: "Copy these variables into your .env file:", + buildSnippet: (details) => + [ + `MZ_HOST=${details.host}`, + `MZ_PORT=${details.port}`, + `MZ_DATABASE=${details.database}`, + `MZ_USER=${details.user}`, + `MZ_PASSWORD=${passwordOrPlaceholder(details)}`, + `DATABASE_URL=postgres://${encodeURIComponent(details.user)}:${passwordOrPlaceholder(details)}@${details.host}:${details.port}/${details.database}${details.ssl ? "?sslmode=require" : ""}`, + ].join("\n"), + }, +]; + +/** Builds the psql connection command. + * + * NOTE(benesch): We'd like to use `sslmode=verify-full` to prevent MITM + * attacks, but that mode requires specifying `sslrootcert=/path/to/cabundle`, + * and that path varies by platform. So instead we use `require`, which is + * at least better than the default of `prefer`. */ +export const buildPsqlCommand = ( + details: Pick< + ConnectionDetails, + "host" | "port" | "database" | "user" | "ssl" + >, + clusterName?: string, +): string => { + const params: string[] = []; + if (details.ssl) { + params.push("sslmode=require"); + } + if (clusterName) { + params.push(`options=--cluster%3D${clusterName}`); + } + const query = params.length > 0 ? `?${params.join("&")}` : ""; + return `psql "postgres://${encodeURIComponent(details.user)}@${details.host}:${details.port}/${details.database}${query}"`; +}; + +/** Command to Base64-encode the user and their credential for the MCP token + * flow. The credential differs by deployment: an app password on cloud, the + * OIDC ID token or the login password on self-managed. */ +export const buildBase64TokenCommand = ( + user: string, + credentialPlaceholder: string = APP_PASSWORD_PLACEHOLDER, +) => `printf '%s' '${user}:${credentialPlaceholder}' | base64 -w0`; diff --git a/console/src/icons/index.tsx b/console/src/icons/index.tsx index 4e6b084c3e974..5a2a4415a7ee2 100644 --- a/console/src/icons/index.tsx +++ b/console/src/icons/index.tsx @@ -23,15 +23,18 @@ export { default as CheckmarkIcon } from "~/icon/CheckmarkIcon.svg?react"; export { default as ChevronDownIcon } from "~/icon/ChevronDownIcon.svg?react"; export { default as ChevronLeftIcon } from "~/icon/ChevronLeftIcon.svg?react"; export { default as ChevronRightIcon } from "~/icon/ChevronRightIcon.svg?react"; +export { default as ClaudeLogoIcon } from "~/icon/ClaudeLogoIcon.svg?react"; export { default as ClockIcon } from "~/icon/ClockIcon.svg?react"; export { default as CloseIcon } from "~/icon/CloseIcon.svg?react"; export { default as ClustersIcon } from "~/icon/ClustersIcon.svg?react"; +export { default as CodexLogoIcon } from "~/icon/CodexLogoIcon.svg?react"; export { default as ColumnIcon } from "~/icon/ColumnIcon.svg?react"; export { default as CommandIcon } from "~/icon/CommandIcon.svg?react"; export { default as ConfluentLogoIcon } from "~/icon/ConfluentLogoIcon.svg?react"; export { default as ConnectionIcon } from "~/icon/ConnectionIcon.svg?react"; export { default as CopyIcon } from "~/icon/CopyIcon.svg?react"; export { default as CrosshairIcon } from "~/icon/CrosshairIcon.svg?react"; +export { default as CursorLogoIcon } from "~/icon/CursorLogoIcon.svg?react"; export { default as DiamondErrorIcon } from "~/icon/DiamondErrorIcon.svg?react"; export { default as DoubleChevronRightIcon } from "~/icon/DoubleChevronRightIcon.svg?react"; export { default as DownloadIcon } from "~/icon/DownloadIcon.svg?react"; @@ -41,6 +44,7 @@ export { default as EyeOpenIcon } from "~/icon/EyeOpenIcon.svg?react"; export { default as FilterIcon } from "~/icon/FilterIcon.svg?react"; export { default as FrameSelectionIcon } from "~/icon/FrameSelectionIcon.svg?react"; export { default as GcpLogoIcon } from "~/icon/GcpLogoIcon.svg?react"; +export { default as GeminiLogoIcon } from "~/icon/GeminiLogoIcon.svg?react"; export { default as GlobeIcon } from "~/icon/GlobeIcon.svg?react"; export { default as HamburgerIcon } from "~/icon/HamburgerIcon.svg?react"; export { default as IndexIcon } from "~/icon/IndexIcon.svg?react"; @@ -78,10 +82,12 @@ export { default as SortIcon } from "~/icon/SortIcon.svg?react"; export { default as SourcesIcon } from "~/icon/SourcesIcon.svg?react"; export { default as StopIcon } from "~/icon/StopIcon.svg?react"; export { default as TableIcon } from "~/icon/TableIcon.svg?react"; -export { default as Terminal } from "~/icon/Terminal.svg?react"; +export { default as TerminalIcon } from "~/icon/TerminalIcon.svg?react"; export { default as ThumbIcon } from "~/icon/ThumbIcon.svg?react"; export { default as ViewIcon } from "~/icon/ViewIcon.svg?react"; +export { default as VsCodeLogoIcon } from "~/icon/VsCodeLogoIcon.svg?react"; export { default as WarningIcon } from "~/icon/WarningIcon.svg?react"; export { default as WebhookIcon } from "~/icon/WebhookIcon.svg?react"; +export { default as WindsurfLogoIcon } from "~/icon/WindsurfLogoIcon.svg?react"; export { default as ZoomInIcon } from "~/icon/ZoomInIcon.svg?react"; export { default as ZoomOutIcon } from "~/icon/ZoomOutIcon.svg?react"; diff --git a/console/src/layouts/NavBar.tsx b/console/src/layouts/NavBar.tsx index 2fd17f015cdf6..e5666d9cf6215 100644 --- a/console/src/layouts/NavBar.tsx +++ b/console/src/layouts/NavBar.tsx @@ -24,11 +24,9 @@ import * as React from "react"; import { Link as RouterLink } from "react-router-dom"; import useResizeObserver from "use-resize-observer"; -import ConnectModal from "~/components/ConnectModal"; +import ConnectDrawer from "~/components/connect/ConnectDrawer"; import FreeTrialNotice from "~/components/FreeTrialNotice"; import { MaterializeLogo } from "~/components/MaterializeLogo"; -import OidcConnectModal from "~/components/OidcConnectModal"; -import PasswordConnectModal from "~/components/PasswordConnectModal"; import { AppConfigSwitch } from "~/config/AppConfigSwitch"; import EnvironmentSelectField from "~/layouts/EnvironmentSelect"; import ProfileDropdown from "~/layouts/ProfileDropdown"; @@ -153,9 +151,9 @@ export interface NavBarProps { export const NavBar = ({ isCollapsed }: NavBarProps) => { const { - isOpen: isConnectModalOpen, - onClose: onCloseConnectModal, - onOpen: onOpenConnectModal, + isOpen: isConnectDrawerOpen, + onClose: onCloseConnectDrawer, + onOpen: onOpenConnectDrawer, } = useDisclosure(); const { isOpen: isMobileNavOpen, @@ -268,13 +266,13 @@ export const NavBar = ({ isCollapsed }: NavBarProps) => { - ) @@ -285,20 +283,18 @@ export const NavBar = ({ isCollapsed }: NavBarProps) => { + - {runtimeConfig.isOidcAvailable ? ( - - ) : ( - - )} ) } diff --git a/console/src/layouts/NavBar/NavMenu.tsx b/console/src/layouts/NavBar/NavMenu.tsx index 6a2ea3c63615f..15b206245e50a 100644 --- a/console/src/layouts/NavBar/NavMenu.tsx +++ b/console/src/layouts/NavBar/NavMenu.tsx @@ -21,10 +21,8 @@ import React from "react"; import { Location, useLocation } from "react-router-dom"; import { useCanViewUsage } from "~/api/auth"; -import ConnectModal from "~/components/ConnectModal"; +import ConnectDrawer from "~/components/connect/ConnectDrawer"; import FreeTrialNotice from "~/components/FreeTrialNotice"; -import OidcConnectModal from "~/components/OidcConnectModal"; -import PasswordConnectModal from "~/components/PasswordConnectModal"; import { AppConfigSwitch, CloudRuntimeConfig } from "~/config/AppConfigSwitch"; import { useFlags } from "~/hooks/useFlags"; import { useIsSuperUser } from "~/hooks/useIsSuperUser"; @@ -270,9 +268,9 @@ const NavMenuMobile = (props: { items: NavItemType[]; }) => { const { - isOpen: isConnectModalOpen, - onClose: onCloseConnectModal, - onOpen: onOpenConnectModal, + isOpen: isConnectDrawerOpen, + onClose: onCloseConnectDrawer, + onOpen: onOpenConnectDrawer, } = useDisclosure(); return ( @@ -304,12 +302,12 @@ const NavMenuMobile = (props: { - @@ -320,21 +318,19 @@ const NavMenuMobile = (props: { - {runtimeConfig.isOidcAvailable ? ( - - ) : ( - - )} + ) } diff --git a/console/vite.config.ts b/console/vite.config.ts index 40253963badc8..eb6f206f1ceb8 100644 --- a/console/vite.config.ts +++ b/console/vite.config.ts @@ -158,6 +158,11 @@ export default defineConfig({ secure: false, ws: true, }, + // OAuth discovery for MCP clients pointed at /api/mcp. + "/.well-known/oauth-protected-resource": { + target: `https://127.0.0.1:${devServerProxyPort}`, + secure: false, + }, } : { "/api/": { @@ -167,6 +172,10 @@ export default defineConfig({ target: `ws://127.0.0.1:${devServerProxyPort}`, ws: true, }, + // OAuth discovery for MCP clients pointed at /api/mcp. + "/.well-known/oauth-protected-resource": { + target: `http://127.0.0.1:${devServerProxyPort}`, + }, }, }, plugins, diff --git a/doc/developer/design/20260708_prometheus_cluster_metrics.md b/doc/developer/design/20260708_prometheus_cluster_metrics.md index aef57850bf8bb..fee18408ec3f0 100644 --- a/doc/developer/design/20260708_prometheus_cluster_metrics.md +++ b/doc/developer/design/20260708_prometheus_cluster_metrics.md @@ -38,7 +38,7 @@ To do this we define a `BuiltinPrometheusSink` static that pairs a SQL query wit This resolves the requirements because: - **(1) envd-down survival:** the scrape path is clusterd → Prometheus, so envd is not on the scrape path. For this to hold, attribution must also be designed so it does not couple the sink's *emission* frontier to envd. That coupling is the central risk and is worked in §6.8. -- **(2) load/hydration:** the dataflow's frontier advances as work completes. The sink emits `mz_metric_sink_frontier_ms` whenever the frontier moves. When a worker stalls, the frontier freezes and this metric goes stale. Prometheus sees the last-good value plus the stale freshness metric. +- **(2) load/hydration:** the dataflow's frontier advances as work completes. The sink emits `mz_compute_metric_sink_frontier_ms` whenever the frontier moves. When a worker stalls, the frontier freezes and this metric goes stale. Prometheus sees the last-good value plus the stale freshness metric. - **(3) attribution:** attribution is resolved on the replica so labels carry both stable identifiers (collection ID) and human-readable names, so dashboards read cleanly and alerts survive object renames. Resolving names without dragging the sink's freshness down to envd's cadence needs care (§6.8). ## 6. Proposed Design @@ -99,7 +99,7 @@ Three sinks, distilled from `COMPUTE_METRIC_QUERIES` in `src/environmentd/src/ht Sink #2's `mz_dataflow_elapsed_seconds_total` is monotonic within a replica lifetime and resets only on replica restart (reconciliation after an envd restart being the sole other mid-flight mutation). Consumers should wrap it in `rate()`, which handles the reset. -Each sink automatically emits the `MetricSink` operator's per-sink health gauges (see §6.5), which cover complementary failure modes: `mz_metric_sink_frontier_ms{sink=""}` (liveness, is the sink advancing?) and `mz_metric_sink_errors{sink=""}` (data-plane, is the sink's SQL producing errors?). +Each sink automatically emits the `MetricSink` operator's per-sink health gauges (see §6.5), which cover complementary failure modes: `mz_compute_metric_sink_frontier_ms{sink=""}` (liveness, is the sink advancing?) and `mz_compute_metric_sink_errors{sink=""}` (data-plane, is the sink's SQL producing errors?). ### 6.3 How this sink differs from existing compute sinks @@ -132,12 +132,12 @@ Because the sink SQL is written by us as part of the database itself, we can con Two failure modes need separate signals, because the sink can only observe one of them from inside its own dataflow. -- **Data-plane errors (sink is running, SQL is producing errors).** SQL-level errors (division by zero, casts, etc.) flow through the standard compute error side channel. The `MetricSink` operator consumes both the OK and ERR streams from its input (the same shape `MaterializedView` uses when it writes OK/ERR shards to persist), emits gauges and counters from the OK stream, and tracks the number of currently-live error rows in `mz_metric_sink_errors{sink=""}`. This is a gauge of net-live errors, not a monotonic counter: it rises as errors appear and falls as they are retracted, and published metric values freeze while it is nonzero. Alert on `mz_metric_sink_errors > 0`, not on its rate. -- **Liveness (sink stopped advancing).** Once the sink is running, the freshness metric `mz_metric_sink_frontier_ms{sink=""}` is the primary signal that it has stalled. If it stops making progress, the frontier stops advancing, the value stops updating, and a staleness alert fires. This does not cover the case where the dataflow is never rendered at all, which is a larger problem. +- **Data-plane errors (sink is running, SQL is producing errors).** SQL-level errors (division by zero, casts, etc.) flow through the standard compute error side channel. The `MetricSink` operator consumes both the OK and ERR streams from its input (the same shape `MaterializedView` uses when it writes OK/ERR shards to persist), emits gauges and counters from the OK stream, and tracks the number of currently-live error rows in `mz_compute_metric_sink_errors{sink=""}`. This is a gauge of net-live errors, not a monotonic counter: it rises as errors appear and falls as they are retracted, and published metric values freeze while it is nonzero. Alert on `mz_compute_metric_sink_errors > 0`, not on its rate. +- **Liveness (sink stopped advancing).** Once the sink is running, the freshness metric `mz_compute_metric_sink_frontier_ms{sink=""}` is the primary signal that it has stalled. If it stops making progress, the frontier stops advancing, the value stops updating, and a staleness alert fires. This does not cover the case where the dataflow is never rendered at all, which is a larger problem. This carries the sink input's **logical frontier in milliseconds**. For the introspection sources these sinks read, logical time is driven by the replica's real-time clock (epoch milliseconds), so `scrape_time - frontier_ms` measures how far the emitted values lag real time, and it grows without bound under a wedged worker. -Staleness detection is opt-in. A Prometheus gauge is stamped fresh on every scrape regardless of when its value last changed, so a consumer only learns a series is stale by comparing `mz_metric_sink_frontier_ms` against scrape time. Dashboards and alerts that ignore the freshness signal read the last-good value as current. We do not make a stale data series read as absent. +Staleness detection is opt-in. A Prometheus gauge is stamped fresh on every scrape regardless of when its value last changed, so a consumer only learns a series is stale by comparing `mz_compute_metric_sink_frontier_ms` against scrape time. Dashboards and alerts that ignore the freshness signal read the last-good value as current. We do not make a stale data series read as absent. The two signals are complementary and neither subsumes the other. Alerting rules should include both. diff --git a/doc/developer/design/20260817_compute_hydration_timestamps.md b/doc/developer/design/20260817_compute_hydration_timestamps.md new file mode 100644 index 0000000000000..3a531b0c47f70 --- /dev/null +++ b/doc/developer/design/20260817_compute_hydration_timestamps.md @@ -0,0 +1,604 @@ +# Compute Hydration Timestamps + +- Associated: + [SQL-632](https://linear.app/materializeinc/issue/SQL-632/write-design-doc), + [Improved Hydration Visibility](https://linear.app/materializeinc/project/improved-hydration-visibility-f8cb205ddd89/overview), + [PRD](https://app.notion.com/p/materialize/Improved-hydration-visibility-3a513f48d37b806cb3a5e7f5b5a9c1e6) + +Scope. This covers the compute half of the PRD, and within that only the +per-worker introspection relation. Compute's job is to observe the hydration +lifecycle accurately and publish it. Rolling the per-worker rows up into a +per-replica relation, and persisting episodes durably, are follow-up work +described at the end and not built here. + +## The Problem + +The PRD asks for per-object and per-replica hydration history: when hydration +started, when it finished, and whether it finished at all. Compute observes none +of that today. It observes a single duration, and only for objects that finished +hydrating. + +`mz_introspection.mz_compute_hydration_times_per_worker` has columns +`(export_id, worker_id, time_ns)`. `time_ns` is NULL until the object is +hydrated, at which point `handle_hydration` in +`src/compute/src/logging/compute.rs` fills it with an `Instant` based elapsed +measurement taken inside the replica process. That has three consequences. + +**No timestamps at all.** There is no start and no finish instant, only a +duration between two unnamed points. Nothing downstream can roll per-object rows +up into a replica or cluster hydration episode, because that requires absolute +times to take a min and a max over. A replica has no notion of being globally +hydrated, so that rollup is the only way to obtain one. + +**In-flight hydration is opaque.** While an object hydrates, `time_ns` is NULL, +which is indistinguishable from an object waiting on its inputs, an object +queued behind `HYDRATION_CONCURRENCY`, and an object whose dataflow was never +installed. "Is this object hydrating, and since when" is unanswerable. + +**The reported duration measures the wrong interval.** +`CollectionLogging::new` emits the `Export` event, which stamps +`ExportState::created_at`, and `handle_create_dataflow` in +`src/compute/src/compute_state.rs` calls it while the dataflow is still suspended +by its `StartSignal` token. That token is released only when `Schedule` arrives at +`handle_schedule`. So `time_ns` spans time spent waiting for input frontiers to +advance, time spent queued behind the sequential hydration limit in +`src/compute-client/src/controller/sequential_hydration.rs`, and the hydration +work itself, with no way to separate them. + +## Success Criteria + +1. For every maintained compute object on every worker, compute publishes the + wallclock instant at which its dataflow was installed, the instant hydration + began, and the instant it completed, or NULL where the event has not happened. +2. Time spent hydrating is distinguishable from time spent waiting to start. +3. An object currently hydrating is distinguishable from one waiting to start and + from one never installed, from the moment its row appears rather than only + once it completes. +4. The published instants are accurate to the event, not quantized to the + introspection logging interval. +5. Values are stable for as long as the replica process lives, including across + an environmentd restart and reconnection. +6. Every row satisfies `installed_at <= started_at <= hydrated_at` where the + values are non-NULL, and no row presents a later stage without an earlier one. +7. No existing relation changes shape, semantics, or values. + `mz_compute_hydration_times`, `mz_compute_hydration_statuses`, and the + per-worker relation all keep returning exactly what they return today. + +## Out of Scope + +- **The per-replica relation.** Aggregating the per-worker rows into a + `mz_compute_hydration_timestamps` storage-managed collection, and the + cross-worker rollup questions that come with it, are follow-up work. See + "Follow-up work" below. +- **Persistence, retention, and history relations.** Writing episodes into + durable append-only relations, deduplicating them, and enforcing retention is + adapter work. +- **Peak memory and disk.** Being built separately, sourced from the + process-global memory limiter in `src/compute/src/memory_limiter.rs` and the + usage collector in `src/clusterd/src/usage_metrics.rs`. It shares this design's + notion of an episode, since the point of a peak is to attribute it to one, but + nothing here depends on it landing first. +- **Prometheus metrics.** A later cloud-team requirement per the PRD. +- **Per-operator progress.** `mz_compute_operator_hydration_statuses` already + provides per-LIR-node booleans, which are the numerator and denominator the + PRD's progress view needs. +- **Storage hydration.** Sources report `rehydration_latency` through their own + statistics path, which is a duration and not a pair of timestamps. Giving + storage a timestamp equivalent so that `mz_hydration_statuses` could carry + timestamps across both halves is a known pre-existing gap and not one this + design closes. +- **Subscribes, peeks, and other transient dataflows.** The per-worker relation + covers whatever the logging dataflows cover, and consumers filter transient IDs + as they do today. + +## Solution Proposal + +Add three wallclock timestamps to the compute hydration log, stamped by the +replica from the logging event time, and retain `time_ns` unchanged alongside +them. + +| Column | Stamped when | Meaning | +| --- | --- | --- | +| `installed_at` | the `Export` event, from `CreateDataflow` | the dataflow exists on this worker, suspended, so this is also the start of queueing | +| `started_at` | the dataflow is unsuspended | hydration is actually running | +| `hydrated_at` | the output frontier passes the as-of | hydration is complete | + +`hydrated_at - started_at` is hydration time as users mean it, and +`started_at - installed_at` is the queueing interval. Today's `time_ns` conflates +the two. + +Two choices shape everything else, each argued in its own section below. The +timestamps are stamped by the replica rather than by the compute controller, for +the reasons in "Why the replica and not the compute controller". And `time_ns` is +kept rather than replaced, for the reasons in "Preserving the existing relations". + +The per-replica episode, once the rollup in "Follow-up work" exists, is +`min(started_at)` to `max(hydrated_at)` over an object set. An object hydrating +right now has a non-NULL `started_at` and a NULL `hydrated_at`. An object waiting +to start has a NULL `started_at`. + +Queueing is made of two distinct waits, and the replica cannot tell them apart. +A collection waits first for its input frontiers to advance, which the controller +enforces by not sending `Schedule` at all from `maybe_schedule_collection`, and +then for a hydration concurrency slot, which the `SequentialHydration` interceptor +enforces by withholding a `Schedule` it has already received. Both appear to the +replica as `Schedule` arriving late. Separating them is possible, because the +interceptor knows when it enqueued the command, but that observation lives in +environmentd and would reset on restart, which is the property this design exists +to preserve. If the distinction is wanted it belongs in a live-only surface or a +Prometheus metric. + +### The replica can already stamp wallclock times + +Compute log event times are Unix-epoch based. `initialize` in +`src/compute/src/logging/initialize.rs` captures +`start_offset = SystemTime::now() - UNIX_EPOCH` alongside an `Instant` and passes +both to timely's `Logger::new`, which computes each event's time as +`offset + now.elapsed()`. So every event time is a `Duration` since the epoch, +advanced monotonically. The comment there states the intent: to let logging +sources be joined against tables and other real-time sources. + +So the replica already knows the wallclock instant of every hydration event. +`handle_hydration` discards it in favour of an elapsed measurement. + +**Stamp the event time, not the update timestamp.** The demux has the event +`Duration` in hand per event, and separately computes `DemuxHandler::ts`, which +rounds up to the next logging interval and is used as the differential timestamp +of the update. The values must come from the event time. The rounding then affects +only when an update becomes visible and whether two transitions in the same +interval consolidate, never the accuracy of a recorded instant. This is +criterion 4, and it is easy to get wrong by reaching for the value already being +computed. + +Two properties of this clock need documenting on the relation. It is monotone, +because it advances off an `Instant` rather than re-reading the system clock, so +it does not jump if NTP steps the wall clock. And `initialize` runs per timely +worker, so the epoch anchor is sampled per worker, and a replica with `scale > 1` +spans processes on different machines. Within a row all three timestamps share one +anchor and are therefore mutually consistent. Any comparison across workers +absorbs anchor skew. That skew is pre-existing across everything derived from +compute logging and has not been observed to be severe, but it is a real risk and +this design is the first to invite direct comparison of absolute times, so it is +acknowledged rather than designed around. + +### A new hydration start event + +There is no event for hydration start today. Add +`ComputeEvent::HydrationStart { export_id }`, logged from the three places a +dataflow's computation becomes unblocked. + +**From `handle_schedule`**, which drops the suspension token. Guard it in the +demux on `started_at` already being set, mirroring the existing guard on +`hydration_time_ns`. That makes it idempotent under reconciliation, where +`reconcile` retains matching dataflows but still forwards `Schedule`, reaching a +`suspended_collections.remove` that returns `None`. + +**From `handle_create_dataflow`, for dataflows with nothing to gate.** +`StartSignal` is attached only to imported sources and imported indexes, and +`Dataflow::import_ids` is exactly the imported index ids chained with the imported +source ids. So a dataflow with no imports, for example a default index on +`SELECT 1` or a constant materialized view, has nothing suspended and begins +computing at creation. `handle_create_dataflow` can test this before rendering and +stamp `started_at` alongside `installed_at`. + +This matters beyond bookkeeping. Such a dataflow can reach hydration before its +`Schedule` ever arrives, and the interceptor anticipates exactly that: *"it is +possible to observe hydration even for collections for which we never sent a +`Schedule` command, if the replica decided to not suspend the dataflow after +creation"*. It then sends the `Schedule` anyway to keep protocol communication +predictable, so a `started_at` stamped only from `handle_schedule` would land +after `hydrated_at` and violate criterion 6. Stamping at creation also means the +row is correct from the moment it appears: a consumer sampling mid-hydration sees +`(installed_at, started_at, NULL)`, which is the truth, rather than +`(installed_at, NULL, NULL)`, which would report the object as queued when nothing +is queueing it. That is criterion 3. + +**From `initialize_logging`, for log collections.** These emit an `Export` event +like any other collection, so they get an `installed_at`, but the controller marks +them scheduled implicitly and never sends `Schedule`, per +`CollectionState::new_log_collection`. They then hydrate immediately at an as-of of +`Timestamp::MIN`. Without a start event they would sit permanently in the illegal +state `(installed_at, NULL, hydrated_at)`, on every replica, for a dozen or so +objects. Emitting the event where they are created is simpler than filtering them +and keeps the state machine total. Nobody should read hydration timings for log +collections, and the relation's documentation should say so. + +Between them, the second and third cases cover every path by which a dataflow can +begin computing without a `Schedule`, so `handle_hydration` should not need to +repair anything. If it nonetheless observes a hydration event with `started_at` +still NULL, that is an unanticipated path: stamp `started_at` from `installed_at` +so the invariant holds, and soft-log, so the surprise is visible rather than +silently absorbed. + +### Log relation shape + +`ComputeLog::HydrationTime` currently describes `(export_id, worker_id, time_ns)` +in `src/compute-client/src/logging.rs`. Widen it: + +``` +export_id text not null +worker_id uint8 not null +time_ns uint8 nullable +installed_at timestamptz not null +started_at timestamptz nullable +hydrated_at timestamptz nullable +key: (export_id, worker_id) +``` + +`ExportState` in the demux gains the three instants next to `created_at` and +`hydration_time_ns`, which both stay so that `time_ns` keeps its current +derivation. Row updates follow the existing retract-and-reinsert pattern in +`handle_hydration`, so an object moves through `(installed_at, NULL, NULL)` or +`(installed_at, started_at, NULL)` depending on whether it is gated, and then to +the complete row. + +Since the durable log id is keyed on the `LogVariant`, in +`src/catalog/src/durable/transaction.rs`, widening in place rather than adding a +variant means no new id and no second log collection producing overlapping data. + +### Preserving the existing relations + +The builtin log currently named `mz_compute_hydration_times_per_worker`, in +`src/catalog/src/builtin/mz_introspection.rs`, is renamed to +`mz_compute_hydration_timestamps_per_worker` and exposes all six columns. A new +builtin view takes the old name and projects the old three columns: + +```sql +SELECT export_id, worker_id, time_ns +FROM mz_introspection.mz_compute_hydration_timestamps_per_worker +``` + +**This is why `time_ns` is kept rather than replaced.** Because it is retained +rather than derived from the timestamps, that projection is exact: no value +changes, no precision is lost, and no cross-worker arithmetic is introduced. +Deriving `time_ns` as `hydrated_at - installed_at` would have moved it to +microsecond precision, since `timestamptz` caps there, where today it is true +nanoseconds. Deriving it after aggregation would additionally have absorbed +cross-worker install skew and the per-worker anchor skew described above. So +`time_ns` remains the authoritative per-worker duration, measured from a single +`Instant` inside one worker, and the timestamps carry episode identity, which +requires absolute times a duration cannot provide. Two columns with two documented +jobs rather than two sources of truth. + +Everything downstream is therefore untouched. The introspection subscribe in +`src/adapter/src/coord/introspection.rs` keeps reading the old name and keeps its +current SQL. `mz_internal.mz_compute_hydration_times` and +`mz_internal.mz_compute_hydration_statuses` are not modified at all, so they keep +their shapes, semantics, values, retained-metrics properties, indexes and shards. +`arrangement_sizes_snapshot`, which decodes that collection positionally from a +persist snapshot in `src/adapter/src/coord/message_handler.rs`, needs no change. +Neither does the console query that joins it by name. + +This is what makes the design purely additive: it adds columns and a relation, and +changes nothing that anything currently reads. + +### Sequence of events + +Two cases matter, and they differ in exactly the way the timestamps are meant to +expose. + +**An index created on a running, already-hydrated cluster.** Everything happens +in one controller turn and one replica turn. + +| Step | Where | What | Stamp | +| --- | --- | --- | --- | +| 1 | controller | `create_dataflow` downgrades input read holds to the as-of and calls `add_collection` per export | | +| 2 | controller | sends `ComputeCommand::CreateDataflow` | | +| 3 | controller | calls `maybe_schedule_collection` immediately. Inputs are already available, so the frontier check passes and `Schedule` goes out in the same turn | | +| 4 | interceptor | forwards `CreateDataflow` and tracks the collection. Withholds `Schedule`, enqueuing it, then `hydrate_collections` re-emits it because the cluster is below `HYDRATION_CONCURRENCY` | | +| 5 | replica | `handle_create_dataflow` calls `CollectionLogging::new`, which logs `Export` | **`installed_at`**, and `started_at` too if the dataflow has no imports | +| 6 | replica | inserts the suspension token and renders the dataflow, whose operators park on the `StartSignal` | | +| 7 | replica | `handle_schedule` drops the token and the operators start | **`started_at`** | +| 8 | replica | the dataflow reads its inputs from the as-of forward and builds arrangements. Nothing is stamped here, this interval is the hydration | | +| 9 | replica | the output frontier passes the as-of and `set_reported_output_frontier` calls `set_hydrated` | **`hydrated_at`** | +| 10 | replica | the demux writes the retract and insert pair, so the per-worker relation carries all three | | +| 11 | controller | separately, a `Frontiers` response arrives and `update_output_frontier` flips the controller's own hydration view, which is what the 0dt caught-up check and the autoscaling signal read. One round trip later, and it stamps nothing | | + +Because steps 2 through 7 collapse into one turn, `installed_at` and `started_at` +are within milliseconds of each other and the queueing interval is approximately +zero. This is the case today's `time_ns` happens to measure correctly. + +**A replica joining a cluster that already hosts N objects.** This is the +rehydration case the PRD is about. + +1. `Instance::add_replica` replays the reduced command history into the new client + and then installs per-replica collection state. The replay is what delivers the + commands, there is no separate trigger. +2. `ComputeCommandHistory::reduce` emits every `CreateDataflow` before any + `Schedule`, so the replica stamps N `installed_at` values in a burst. This is + the moment the replica learned its whole workload. +3. Every `Schedule` is then withheld by the interceptor and released + `HYDRATION_CONCURRENCY` at a time. +4. Objects therefore acquire `started_at` in waves. An object at the back of the + queue has `installed_at` set and `started_at` NULL, and is correctly reported + as not yet started rather than as hydrating slowly. +5. Each completion releases the next queued `Schedule`, so the queue drains at + the rate dataflows hydrate. + +For an object at the back of that queue, today's `time_ns` is mostly queueing +time. Separating the two intervals is the point of stamping `installed_at`. + +### What the timestamps promise, and what they do not + +This is the contract handed to whatever consumes the relation. + +**Episode identity.** `installed_at` is generated by the replica and is stable +across an environmentd restart, a reconnection, and reconciliation, because the +logging dataflows and the demux state holding it are not recreated. It is +`NOT NULL` by construction and stamped at the inception of the episode, before any +waiting. So `(worker_id, export_id, installed_at)`, and its per-replica equivalent +once the rollup exists, identifies an episode without needing an epoch or any +controller-side bookkeeping. `started_at` is deliberately not the key: it is NULL +for the entire pre-start phase of a gated dataflow, which is precisely the state a +consumer most needs to track and re-recognise. + +**What resets it.** A replica process restart recreates the logging dataflows and +yields entirely fresh timestamps for every object, which is a genuinely new +episode. Reconciliation preserves them only for dataflows it actually retains: +`reconcile` requires that a candidate be compatible, uncompacted, free of +subscribes and copy-tos, and have its dependencies retained, and that last +condition cascades, so one base index failing to reconcile replaces every dataflow +downstream of it. Those objects get fresh timestamps on a replica that has been up +for a long time. That is arguably correct, since the dataflows really were +rebuilt, but it is not obvious from the outside. + +**Very short episodes may not be observable at all.** The demux assigns updates a +timestamp rounded up to the logging interval, and the introspection write path +consolidates: the subscribe handler discards the subscribe timestamp and flattens +a batch into one append, and the storage write task consolidates on its own batch +interval. So an object whose transitions all fall inside one such window presents +only its final state, and an object that dies before its rows reach persist leaves +no record. In the limit this is unavoidable, and stamping event time rather than +the rounded update timestamp is what keeps it to a visibility limit rather than an +accuracy one: an episode that is recorded is recorded accurately. + +**A crash is not a compute event and is not reported here.** A replica that dies +cannot report its own death. What the relation shows is that a dropped object's +row is retracted individually while its replica's other rows persist, whereas a +replica going away removes all of its rows at once via +`drop_introspection_subscribes`. Replica lifecycle is recorded in +`mz_cluster_replica_status_history`, which the PRD already wants joined against, +so a consumer distinguishes the two by joining rather than by anything compute +adds. + +**A restarting replica briefly serves stale rows.** When a replica restarts under +the same ID, the subscribe is reinstalled with `deferred_write` set and +`first_data_at` cleared, and until the new subscribe reports, the collection +serves the previous subscribe's data, which the replica may have invalidated by +restarting. Consumers must gate on introspection freshness, as +`mz_object_arrangement_size_history` already does via +`fresh_introspection_replicas`. + +**`REFRESH` materialized views report a refresh interval, not hydration work.** +The reported output frontier is the meet of write and compute frontier, and a +REFRESH MV's write frontier sits at the as-of until the first refresh lands, so +hydration is not considered complete until then. For `REFRESH EVERY '1 day'`, +`hydrated_at - started_at` can be most of a day, nearly all of it idle. The +per-object stamps are still internally consistent, so this is not a defect in the +relation, but any rollup must exclude these objects or it will never close an +episode. The controller already receives `refresh_schedule` in `add_collection`, +so it can mark them. + +**Read-only mode changes what the output frontier means.** In read-only mode the +write frontier is deliberately excluded from the reported output frontier, because +a read-only dataflow cannot push it forward. So `hydrated_at` during a 0dt +read-only window reflects compute progress only, which is the intended reading but +differs from the steady-state one. + +### Why the replica and not the compute controller + +The compute controller evaluates a very similar hydration predicate already in +`ReplicaCollectionState::hydrated`, has a real clock, and has an existing +append-only introspection path it uses for `WallclockLagHistory`. Stamping there +would be less code. + +It is nonetheless wrong here. Controller state is rebuilt from scratch on every +environmentd restart, so every timestamp it stamped would reset on a restart that +did not disturb the replica at all, and every episode would look new. +Controller-stamped values also cannot be made idempotent: `ReplicaState::epoch` +resets to 1 on restart, so `(replica_id, epoch)` collides across restarts. And the +controller learns of hydration one `Frontiers` round trip late, so its durations +run systematically long against the replica's own measurement. + +Two qualifications, so the argument is not overstated. The controller-side and +replica-side predicates are close but not identical: the controller's is +`as_of.is_empty() || as_of < output_frontier`, while `CollectionState::hydrated` on +the replica omits the empty-as-of clause and additionally requires the frontier to +have been reported. The divergence is unobservable, because an empty as-of never +produces a `CreateDataflow` at all. Separately, an ordinary version upgrade reaps +clusterd processes, so timestamps reset on upgrade wherever they are stamped. The +benefit of replica stamping is for same-version environmentd restarts, +reconnections and generation changes, which is still the common case, rather than +for upgrades. + +### Implementation touch points + +- `src/compute/src/logging/compute.rs`: `ComputeEvent::HydrationStart`, the three + `ExportState` fields, the packer, `handle_export`, `handle_export_dropped`, + `handle_hydration` including the soft-logged repair, and a new + `handle_hydration_start`. Also a `CollectionLogging` method alongside + `set_hydrated`. +- `src/compute/src/compute_state.rs`: log the start event from `handle_schedule`, + from `handle_create_dataflow` when `import_ids` is empty, and from + `initialize_logging`. +- `src/compute-client/src/logging.rs`: the widened `RelationDesc`. + `LogVariant::desc` is the only exhaustive match a shape change touches, since + the variant itself is unchanged. +- `src/catalog/src/builtin/mz_introspection.rs`: rename the builtin log and add + the compat view. +- `src/catalog/src/builtin.rs`: `BUILTINS_STATIC` registration for the new view. +- `src/pgrepr-consts/src/oid.rs`: an OID for the new view. +- Goldens that hardcode this relation's identity, columns, OIDs or indexes: + `test/sqllogictest/oid.slt`, `information_schema_tables.slt`, + `mz_catalog_server_index_accounting.slt`, `cluster.slt`, + `catalog_server_explain.slt`, `test/cluster/mzcompose.py`, and the autogenerated + `test/sqllogictest/autogenerated/mz_introspection.slt`. +- Docs: the `mz_introspection` system catalog reference page. + +Not touched, and deliberately so: the introspection subscribe, +`mz_internal.mz_compute_hydration_times`, +`mz_internal.mz_compute_hydration_statuses`, +`src/adapter/src/coord/message_handler.rs`, and +`src/mz-debug/src/system_catalog_dumper.rs`. + +New testdrive coverage worth adding: + +- `installed_at` set and `started_at` NULL for an object gated by + `HYDRATION_CONCURRENCY`. +- `started_at` NULL for an object waiting on an unavailable input. +- An import-free dataflow carrying `started_at` from creation, equal to its + `installed_at`, and satisfying the ordering invariant. +- Log collections having all three timestamps set. +- All timestamps surviving an environmentd restart unchanged. +- A replica restart yielding entirely fresh values. + +The most important tests are compatibility ones: that +`mz_compute_hydration_times_per_worker`, `mz_compute_hydration_times` and +`mz_compute_hydration_statuses` return identical values before and after. The +existing assertions in `test/testdrive/hydration-status.td` and the blue-green +tests are the contract and should be left alone rather than adjusted to fit. + +## Follow-up work + +### `mz_compute_hydration_timestamps`, the per-replica relation + +The per-worker relation is per replica and per worker, and lives in +`mz_introspection`, so it is only queryable against a targeted replica. Making the +timestamps generally useful needs the same treatment +`mz_compute_hydration_times` has: an introspection `SUBSCRIBE` rolling the +per-worker rows up per object and writing them to a storage-managed collection in +`mz_internal`. That is deferred, and it is where the genuinely open questions +live. + +The rollup has to answer questions the per-worker relation does not raise: + +- **Which aggregate for which column.** `hydrated_at` presumably keeps today's + all-or-nothing rule, `max` gated on every worker having reported, since an + object is readable only once all workers have advanced. `installed_at` and + `started_at` are less obvious. An ungated `min` avoids transient NULLs but is + non-monotone, because a min over a growing set of reporting workers can move + earlier as slow workers report, which would make an episode key appear to + change. Gating all three the same way is the conservative choice and needs + checking against how the existing gate behaves in practice. +- **Internally contradictory rows.** Mixing gated and ungated aggregates can + publish a row whose `hydrated_at` is set while `started_at` is NULL, or whose + timestamps cross. Whatever aggregation is chosen has to preserve criterion 6 at + the rolled-up level too. +- **Cross-worker clock anchors.** Aggregating min and max across workers crosses + per-worker epoch anchors, and for multi-process replicas crosses machines. Any + derived interval absorbs that skew, which is one more reason to keep `time_ns` + as the authoritative duration rather than recomputing it from aggregated + timestamps. +- **Object-set scoping.** A replica that gains an object after hydrating reports a + later `max(hydrated_at)`, so a naive rollup conflates an initial hydration + episode with later incremental ones. The information needed to scope an episode, + which objects belonged to the replica's initial set, lives in the controller, + and the PRD states the intended rule: adding an object starts a new replica + episode only if the replica transitions from fully hydrated to not. REFRESH + materialized views and log collections need excluding from the same object set. +- **Freshness.** Rows from a replica that has restarted are briefly stale and must + be gated on `fresh_introspection_replicas`. +- **Naming and compatibility.** If the aggregate relation supersedes + `mz_compute_hydration_times`, the positional persist decode in + `arrangement_sizes_snapshot` and its use of + `resolve_builtin_storage_collection`, which is typed `&'static BuiltinSource`, + both need repointing. Simplest is to add the new relation alongside and leave + the old one alone. + +### Durable history + +Persisting episodes into append-only relations with retention, per the PRD, is +adapter work. What compute provides for it is a stable episode key +(`installed_at`), event-accurate instants, and values that survive an environmentd +restart. What it does not provide is any record of an episode whose rows never +reached persist, or any signal that a replica crashed. + +## Minimal Viable Prototype + +The prototype is the compute and catalog change itself, exercised through +testdrive against a targeted replica. The validating query selects from +`mz_compute_hydration_timestamps_per_worker` on a cluster with a hydration +concurrency limit and a handful of indexes, showing objects moving from waiting, +to hydrating, to hydrated, with the queueing interval visible separately from the +hydration interval. A second run after an environmentd restart shows identical +values, which is the property the design turns on. A third check confirms +`mz_compute_hydration_times` and `mz_compute_hydration_statuses` are byte for byte +unchanged. + +No SQL syntax changes and no new user-facing commands, so there is nothing to +wireframe. + +## Alternatives + +**Stamp in the compute controller.** Rejected for the reasons in "Why the replica +and not the compute controller". + +**Replace `time_ns` with the timestamps rather than keeping both.** Rejected for +the reasons in "Preserving the existing relations". + +**Backfill `started_at` at the hydration event** instead of stamping it at +creation for ungated dataflows. Rejected because the row would be wrong in the +interim, reporting an object as queued while it hydrates, and because it defers a +fact the replica already knows at creation time. Retained only as a soft-logged +repair for paths this design does not anticipate. + +**Narrow `mz_compute_hydration_statuses.hydration_time` to the interval its name +claims.** Rejected. Monitoring and analytics depend on the current values, and the +failure mode is a query that keeps working while silently returning different +numbers. A column whose name overstates what it measures is the cheaper problem. +For the same reason no `queue_time` column is added to it: every interval is +derivable from the new relation, and growing the old surface would attract new +consumers to it. + +**Add a variant instead of widening `ComputeLog::HydrationTime`.** Rejected. It +would need a new durable log id and would leave two log collections producing +overlapping data, which is the two-sources-of-truth problem the design otherwise +avoids. + +**Emit per-transition events into an append-only relation** rather than +maintaining current state, in the shape of `mz_cluster_replica_status_history`. +This is a real alternative, and `WallclockLagHistory` shows that append-only +compute introspection works. It would make short episodes observable, which the +current-state relation cannot do. Rejected for two reasons. Current state is what +consumers actually need in order to answer "is this hydrating now", so an event +log would be in addition to this relation rather than instead of it. And the +compute-side cost is unbounded demux state, because the demux would have to retain +per-object history rather than a fixed-size `ExportState`, on a path that runs +inside every replica. + +**Raise the resolution of existing metrics sampling.** Fails because sampling +cannot observe a transition, only its aftermath, so a hydration that starts and +finishes between two samples is invisible. + +## Settled during review + +Recorded so the reasoning is not relitigated. Each item is argued in the section +named. + +- **Stamping location.** The replica, not the compute controller. See "Why the + replica and not the compute controller". +- **`time_ns` is kept.** See "Preserving the existing relations". +- **`started_at` for ungated dataflows is stamped at creation,** not backfilled at + hydration. See "A new hydration start event". +- **`hydration_time` is left exactly as it is,** and no `queue_time` column is + added. Future consumers read the timestamps. See "Alternatives". +- **Clock skew.** Accepted and documented rather than designed around. + Pre-existing across everything derived from compute logging. +- **Per-export rather than per-dataflow stamping.** Correct for today's + single-export dataflows, which the interceptor asserts outright. If multi-output + dataflows land, the fix is to stamp per dataflow and fan out to exports in a + view. +- **No fourth stamp** for the hydration-slot boundary. The interceptor knows it + but runs in environmentd. +- **Crash detection is not compute's job.** A replica cannot report its own death, + and replica lifecycle is already in `mz_cluster_replica_status_history`. +- **Sub-window episodes are unavoidable in the limit.** Mitigated by stamping + event time so that recorded episodes are accurate, and documented as a + visibility limit. +- **Log collections** get a start event rather than a filter. +- **The per-replica relation is follow-up work,** not part of this design. + +## Open questions + +None outstanding for this design. The open questions all belong to the per-replica +rollup and are enumerated under "Follow-up work". diff --git a/doc/developer/generated/adapter/_crate.md b/doc/developer/generated/adapter/_crate.md index 87f556e7d7de8..f95100c3ef17c 100644 --- a/doc/developer/generated/adapter/_crate.md +++ b/doc/developer/generated/adapter/_crate.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/lib.rs -revision: 9d0a7c3c6f +revision: 4f0805a4d8 --- # adapter @@ -15,8 +15,11 @@ The crate is organized into several major subsystems: * `client` — the external `Client` / `SessionClient` API consumed by pgwire. * `session` — per-connection `Session` state including transaction management and prepared statements. * `config` — system-parameter synchronisation with LaunchDarkly or a JSON file. +* `peek_client` — `PeekClient`, the session-side handle for frontend peek sequencing, holding compute instance channels, catalog snapshot cache, timestamp oracles, and storage collection access. +* `frontend_peek` — frontend-side fast-path peek sequencing that bypasses the coordinator's main loop. +* `frontend_read_then_write` — frontend-side OCC read-then-write sequencing for DML (INSERT, UPDATE, DELETE). -Key public types re-exported from `lib.rs`: `Client`, `SessionClient`, `Handle`, `ExecuteResponse`, `AdapterError`, `AuthenticationError`, `AdapterNotice`, `CollectionIdBundle`, `ReadHolds`, `TimestampContext`, `serve`, `Config`. +Key public types re-exported from `lib.rs`: `Client`, `SessionClient`, `Handle`, `ExecuteResponse`, `AdapterError`, `AuthenticationError`, `AdapterNotice`, `CollectionIdBundle`, `ReadHolds`, `TimestampContext`, `serve`, `Config`, `PeekClient`. Key dependencies: `mz-catalog`, `mz-controller`, `mz-compute-client`, `mz-storage-client`, `mz-sql`, `mz-expr`, `mz-transform`, `mz-persist-client`, `mz-timestamp-oracle`. Downstream consumers: `mz-environmentd` (calls `serve` to start the coordinator), `mz-pgwire` (uses `Client`/`SessionClient`), `mz-balancerd`. diff --git a/doc/developer/generated/adapter/catalog/_module.md b/doc/developer/generated/adapter/catalog/_module.md index f1ae14482cb1c..ec49d249ee0c9 100644 --- a/doc/developer/generated/adapter/catalog/_module.md +++ b/doc/developer/generated/adapter/catalog/_module.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog.rs -revision: f3b4f3f1be +revision: 39dcae2fba --- # adapter::catalog @@ -14,3 +14,4 @@ Re-exports include `InjectedAuditEvent` from `transact`. Builtin materialized vi `Catalog::allocate_storage_usage_id(commit_ts)` is a thin wrapper around `DurableCatalogState::allocate_id` that bumps the `STORAGE_USAGE_ID_ALLOC_KEY` allocator by one and returns the allocated id, committed at `commit_ts`. It bypasses the high-level catalog transaction machinery; one id covers all rows produced in a single collection cycle. `Catalog::allocate_user_replica_ids`, `allocate_system_replica_ids`, and `allocate_replica_ids` delegate to the durable store to pre-allocate replica IDs out-of-band before a catalog transaction. `allocate_replica_ids` dispatches to user or system allocation based on the owning cluster's ID type. `Catalog::register_temporary_namespace` registers a connection's temporary namespace (the UUID-to-connection mapping used to stamp durable temporary items) before the first temporary-item transaction for that session. `Catalog::drop_temporary_namespace` removes the namespace when the session terminates. The `mz_temp` schema itself is created lazily when the first temporary item is applied. +`CommentObjectId::MetricSink` maps to `ObjectType::MetricSink`; `COMMENT ON METRIC SINK` is rejected at parse, so no metric-sink comment id is built in practice. The arm exists only for exhaustiveness. diff --git a/doc/developer/generated/adapter/catalog/apply.md b/doc/developer/generated/adapter/catalog/apply.md index 6e9df3f5686b1..34f985ab2f815 100644 --- a/doc/developer/generated/adapter/catalog/apply.md +++ b/doc/developer/generated/adapter/catalog/apply.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/apply.rs -revision: f3b4f3f1be +revision: 39dcae2fba --- # adapter::catalog::apply @@ -11,7 +11,7 @@ An `InProgressRetractions` struct caches denormalized state stripped during retr The `ApplyState` state machine batches consecutive updates of the same type when beneficial, then delegates to `apply_updates_inner`, which calls per-type `apply_*_update` methods for roles, role auth, databases, schemas, clusters, network policies, items, system configuration, scoped system configuration, and more. Temporary items are stored as regular durable `Item` updates with an `ephemeral_owner_session` UUID identifying the owning session. Before processing each update, `is_nonlocal_ephemeral_item_update` checks whether the update belongs to a session connected to this process; updates for sessions not connected here are skipped. For additions, locality is determined by whether the UUID is registered in `temporary_namespaces`; for retractions, it is determined by whether the corresponding addition was applied (i.e., the item is present in `entry_by_id`). The `sort_updates` helper `merge_item_updates` orders all persistent items before all temporary items; because non-temporary items can never depend on temporary ones, this keeps dependencies before dependents for additions. During temp item updates in `apply_updates_inner`, `set_create_sql` is called to preserve the exact `create_sql` bytes so that retraction consolidation works correctly. After processing retractions that are not replaced by a corresponding addition (i.e., truly dropped items), `apply_updates` calls `drop_optimizer_notices` to clean up any optimizer notices associated with the dropped items and emits the corresponding `mz_notices` retractions. -`CatalogState` methods defined in this module (`set_optimized_plan`, `set_physical_plan`, `set_dataflow_metainfo`, `drop_optimizer_notices`) mutate the plan and notice fields stored directly on `Index` and `MaterializedView` catalog items. `set_physical_plan` accepts a `DataflowDescription`. +`CatalogState` methods defined in this module (`set_optimized_plan`, `set_physical_plan`, `set_dataflow_metainfo`, `drop_optimizer_notices`) mutate the plan and notice fields stored directly on `Index`, `MaterializedView`, and `MetricSink` catalog items. `set_physical_plan` accepts a `DataflowDescription`. `StateUpdateKind::Role`, `StateUpdateKind::DefaultPrivilege`, `StateUpdateKind::SystemPrivilege`, `StateUpdateKind::Comment`, and `StateUpdateKind::AuditLog` updates do not produce builtin table updates; `mz_roles`, `mz_role_parameters`, `mz_default_privileges`, `mz_system_privileges`, `mz_internal.mz_comments`, and `mz_catalog.mz_audit_events` are materialized views backed by `mz_internal.mz_catalog_raw` rather than builtin tables. `StateUpdateKind::ClusterSystemConfiguration` and `StateUpdateKind::ReplicaSystemConfiguration` updates maintain the in-memory `CatalogState::scoped_system_parameters` working copy via `apply_scoped_system_configuration_update`; they produce no builtin table updates because `mz_internal.mz_cluster_system_parameters` and `mz_internal.mz_replica_system_parameters` are materialized views backed by `mz_internal.mz_catalog_raw`. Retraction is conditional on the stored value matching, so a value change (retraction of the old value followed by addition of the new) is order-independent and correct regardless of the order the two updates arrive. Both update kinds are sorted alongside `Cluster` and `ClusterReplica` updates in the pseudo-topological ordering applied before processing. `concretize_replica_location` accepts an `allow_disabled` boolean that, when `true`, permits replica sizes that are marked disabled in the size map; this is used during catalog apply so that existing replicas with disabled sizes can remain registered in memory (and return sensible SQL results via the `mz_cluster_replicas` MV) without panicking. A `soft_panic_or_log!` is emitted when a managed cluster or replica references an unknown size that is absent from the in-memory map. During catalog apply, `concretize_replica_location` is called with `allowed_availability_zones = None` so that the AZ list stored in the durable replica record is kept as-is rather than being overridden by the cluster's current AZ pool; this is correct because a graceful reconfiguration may have the cluster controller distinguish realized-shape from target-shape replicas by their respective AZ lists. diff --git a/doc/developer/generated/adapter/catalog/builtin_table_updates.md b/doc/developer/generated/adapter/catalog/builtin_table_updates.md index 8715de56b4dbb..56e75aebc767c 100644 --- a/doc/developer/generated/adapter/catalog/builtin_table_updates.md +++ b/doc/developer/generated/adapter/catalog/builtin_table_updates.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/builtin_table_updates.rs -revision: 57107078dc +revision: 39dcae2fba --- # adapter::catalog::builtin_table_updates @@ -10,5 +10,5 @@ For each catalog object type, a corresponding `pack_*` function converts a `Cata `BuiltinTableUpdate` wraps either a `Row`-level update or a `ProtoBatch` for bulk ingestion, both resolved to a `CatalogItemId` before being written. `pack_role_auth_update` writes a row into `mz_role_auth` containing the role's password hash and the timestamp it was last updated. `mz_roles`, `mz_role_parameters`, `mz_clusters`, `mz_cluster_replicas`, `mz_cluster_schedules`, `mz_default_privileges`, `mz_system_privileges`, `mz_internal.mz_comments`, `mz_sinks`, `mz_kafka_sinks`, and `mz_iceberg_sinks` are materialized views backed by `mz_internal.mz_catalog_raw`; rows for these objects are not produced by `pack_*` functions here. `MZ_CLUSTER_REPLICA_SIZE_INTERNAL` (a `BuiltinTable`) receives rows via `pack_cluster_replica_size_update` and retains entries for disabled sizes; the `mz_cluster_replicas` MV LEFT JOINs it to resolve the `disk` column. -`CatalogItem::Secret`, `CatalogItem::Log`, `CatalogItem::Connection`, `CatalogItem::Sink`, and `CatalogItem::Index` do not produce direct table updates from `pack_*` functions. `CatalogItem::Secret` and `CatalogItem::Log` return empty update vecs. `CatalogItem::Connection` returns an empty vec because all connection-detail subtables (`mz_kafka_connections`, `mz_ssh_tunnel_connections`, `mz_aws_connections`, `mz_aws_privatelink_connections`) are materialized views over `mz_catalog_raw` derived from each connection's persisted `create_sql`. `CatalogItem::Sink` returns an empty vec because `mz_sinks`, `mz_kafka_sinks`, and `mz_iceberg_sinks` are materialized views over `mz_catalog_raw` derived from each sink's persisted `create_sql`. `CatalogItem::Source` does not produce rows into `MZ_SOURCES` (a materialized view over `mz_catalog_raw`), `mz_postgres_sources` (a materialized view), or `mz_kafka_sources` (a materialized view); for ingestion sources, `pack_item_update` returns an empty vec. Source-type subtable details such as `mz_postgres_source_tables`, `mz_mysql_source_tables`, `mz_sql_server_source_tables`, and `mz_kafka_source_tables` are also materialized views over `mz_catalog_raw`, so ingestion exports and old-syntax subsources do not emit rows from `pack_*` functions either. Sources only emit rows for webhook sources and source-specific progress/subsource metadata. `CatalogItem::Index` does not produce rows into `MZ_INDEXES` (a materialized view); `pack_index_update` does not write to `MZ_INDEXES`. +`CatalogItem::Secret`, `CatalogItem::Log`, `CatalogItem::Connection`, `CatalogItem::Sink`, `CatalogItem::MetricSink`, and `CatalogItem::Index` do not produce direct table updates from `pack_*` functions. `CatalogItem::Secret`, `CatalogItem::Log`, and `CatalogItem::MetricSink` return empty update vecs. `CatalogItem::Connection` returns an empty vec because all connection-detail subtables (`mz_kafka_connections`, `mz_ssh_tunnel_connections`, `mz_aws_connections`, `mz_aws_privatelink_connections`) are materialized views over `mz_catalog_raw` derived from each connection's persisted `create_sql`. `CatalogItem::Sink` returns an empty vec because `mz_sinks`, `mz_kafka_sinks`, and `mz_iceberg_sinks` are materialized views over `mz_catalog_raw` derived from each sink's persisted `create_sql`. `CatalogItem::Source` does not produce rows into `MZ_SOURCES` (a materialized view over `mz_catalog_raw`), `mz_postgres_sources` (a materialized view), or `mz_kafka_sources` (a materialized view); for ingestion sources, `pack_item_update` returns an empty vec. Source-type subtable details such as `mz_postgres_source_tables`, `mz_mysql_source_tables`, `mz_sql_server_source_tables`, and `mz_kafka_source_tables` are also materialized views over `mz_catalog_raw`, so ingestion exports and old-syntax subsources do not emit rows from `pack_*` functions either. Sources only emit rows for webhook sources and source-specific progress/subsource metadata. `CatalogItem::Index` does not produce rows into `MZ_INDEXES` (a materialized view); `pack_index_update` does not write to `MZ_INDEXES`. The `notice` sub-module handles the special `mz_recent_activity_log` and related notice tables separately. diff --git a/doc/developer/generated/adapter/catalog/consistency.md b/doc/developer/generated/adapter/catalog/consistency.md index 6261ddb289344..b22c807c28d28 100644 --- a/doc/developer/generated/adapter/catalog/consistency.md +++ b/doc/developer/generated/adapter/catalog/consistency.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/consistency.rs -revision: 699d823624 +revision: 39dcae2fba --- # adapter::catalog::consistency diff --git a/doc/developer/generated/adapter/catalog/open.md b/doc/developer/generated/adapter/catalog/open.md index 3f872ab7068c9..365506e7e33a6 100644 --- a/doc/developer/generated/adapter/catalog/open.md +++ b/doc/developer/generated/adapter/catalog/open.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/open.rs -revision: f3b4f3f1be +revision: 39dcae2fba --- # adapter::catalog::open diff --git a/doc/developer/generated/adapter/catalog/open/_module.md b/doc/developer/generated/adapter/catalog/open/_module.md index 12cebd00d04f7..80ecb0dee90ea 100644 --- a/doc/developer/generated/adapter/catalog/open/_module.md +++ b/doc/developer/generated/adapter/catalog/open/_module.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/open.rs -revision: f3b4f3f1be +revision: 39dcae2fba --- # adapter::catalog::open diff --git a/doc/developer/generated/adapter/catalog/open/builtin_schema_migration.md b/doc/developer/generated/adapter/catalog/open/builtin_schema_migration.md index a986d5bb37e13..f081383700079 100644 --- a/doc/developer/generated/adapter/catalog/open/builtin_schema_migration.md +++ b/doc/developer/generated/adapter/catalog/open/builtin_schema_migration.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/open/builtin_schema_migration.rs -revision: 57107078dc +revision: 7cc7716810 --- # adapter::catalog::open::builtin_schema_migration @@ -17,6 +17,8 @@ A `Replacement` step at version `26.33.0-dev.0` covers `mz_catalog.mz_audit_even Three `Replacement` steps at version `26.34.0-dev.0` cover: `mz_catalog.mz_indexes` (because adding the `mz_cluster_reconfigurations_ind` and `mz_cluster_auto_scaling_strategies_ind` builtin indexes changes the VALUES set inlined by `make_mz_indexes`, altering the MV's SQL fingerprint); `mz_internal.mz_postgres_sources` (reflecting its conversion from `BuiltinTable` to `BuiltinMaterializedView`); and `mz_catalog.mz_kafka_sources` (reflecting the same conversion). Eight `Replacement` steps at version `26.37.0-dev.0` cover: `mz_internal.mz_postgres_source_tables`, `mz_internal.mz_mysql_source_tables`, `mz_internal.mz_sql_server_source_tables`, and `mz_internal.mz_kafka_source_tables` (reflecting their conversion from `BuiltinTable` to `BuiltinMaterializedView`); and `mz_catalog.mz_kafka_connections`, `mz_catalog.mz_ssh_tunnel_connections`, `mz_internal.mz_aws_connections`, and `mz_catalog.mz_aws_privatelink_connections` (reflecting their conversion from `BuiltinTable` to `BuiltinMaterializedView`, where each is now derived from its connection's persisted `create_sql` via `mz_catalog_raw`). Three `Replacement` steps at version `26.38.0-dev.0` cover: `mz_catalog.mz_sinks`, `mz_catalog.mz_kafka_sinks`, and `mz_catalog.mz_iceberg_sinks` (reflecting their conversion from `BuiltinTable` to `BuiltinMaterializedView`, now derived from each sink's persisted `create_sql` via `mz_catalog_raw`). The earlier `0.160.0` `mz_sinks` step that named the `Table` description was removed at the same time, because the `Table` description no longer resolves; the `26.38.0-dev.0` step covers all upgrade paths that the old step would have handled. +A `Replacement` step at version `26.38.0-rc.2` covers `mz_internal.mz_cluster_reconfigurations`, whose MV definition changed to include `arrangement_compression` in the `changes` diff column. +A `Replacement` step at version `26.39.0-dev.0` covers `mz_catalog.mz_audit_events`, because the MV gained a `metric-sink` arm in its `object_type` CASE expression, changing its SQL fingerprint. When applying replacement migrations, `mz_storage_usage_by_shard` is excluded from data-destroying replacement plans to preserve billing data. When the source and target versions differ and the source version is a dev build, `Migration::run` forces evolution-mode migration even without an explicit `force_migration` config, avoiding version-based filter failures in dev environments. In forced dev-to-dev migrations, `migrate_evolve_one` skips builtins that do not yet have a shard registered. Brand-new builtins in a given version may not have their shards allocated until the leader completes bootstrap; excluding them avoids "missing shard ID" errors on read-only replicas. diff --git a/doc/developer/generated/adapter/catalog/open/builtin_schema_migration_tests.md b/doc/developer/generated/adapter/catalog/open/builtin_schema_migration_tests.md index bf9f7289552c9..27ebaf119a07a 100644 --- a/doc/developer/generated/adapter/catalog/open/builtin_schema_migration_tests.md +++ b/doc/developer/generated/adapter/catalog/open/builtin_schema_migration_tests.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/open/builtin_schema_migration_tests.rs -revision: 57107078dc +revision: 0c76273b9f --- # adapter::catalog::open::builtin_schema_migration_tests diff --git a/doc/developer/generated/adapter/catalog/state.md b/doc/developer/generated/adapter/catalog/state.md index 7f813c7cd32fa..295b7f07b0704 100644 --- a/doc/developer/generated/adapter/catalog/state.md +++ b/doc/developer/generated/adapter/catalog/state.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/state.rs -revision: f3b4f3f1be +revision: 39dcae2fba --- # adapter::catalog::state diff --git a/doc/developer/generated/adapter/catalog/timeline.md b/doc/developer/generated/adapter/catalog/timeline.md index ad9bb78a86e20..bc5de05ea2518 100644 --- a/doc/developer/generated/adapter/catalog/timeline.md +++ b/doc/developer/generated/adapter/catalog/timeline.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/timeline.rs -revision: 9d0a7c3c6f +revision: 39dcae2fba --- # adapter::catalog::timeline diff --git a/doc/developer/generated/adapter/catalog/transact.md b/doc/developer/generated/adapter/catalog/transact.md index 0b9ae03dbe490..886fbbc0d17ad 100644 --- a/doc/developer/generated/adapter/catalog/transact.md +++ b/doc/developer/generated/adapter/catalog/transact.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/catalog/transact.rs -revision: f3b4f3f1be +revision: 39dcae2fba --- # adapter::catalog::transact diff --git a/doc/developer/generated/adapter/client.md b/doc/developer/generated/adapter/client.md index 2850b9cfcff7a..539a542911f27 100644 --- a/doc/developer/generated/adapter/client.md +++ b/doc/developer/generated/adapter/client.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/client.rs -revision: ff68428395 +revision: 00299a05e3 --- # adapter::client @@ -12,7 +12,7 @@ During `startup`, `Client` constructs a `PeekClient` from the `StartupResponse` `Client::update_scoped_system_parameters` sends `Command::UpdateScopedSystemParameters` to the coordinator to reconcile the scoped feature-flag working copy; `Client::install_scoped_system_parameter_frontend` fires and forgets `Command::InstallScopedSystemParameterFrontend` so the coordinator can share the frontend for synchronous create-time resolution. Both are called by the system-parameter sync loop. `SessionClient` wraps a `Session` and provides per-connection operations such as `execute`, `commit`, `declare`, and `inject_audit_events` for manually appending audit log entries. `SessionClient::catalog_snapshot` fetches the catalog through the session-side snapshot cache (delegating to `PeekClient::catalog_snapshot`), avoiding coordinator round-trips when the catalog's transient revision is unchanged. -`SessionClient::execute` unrolls SQL `EXECUTE ` statements via `unroll_sql_execute` before attempting frontend peek sequencing via `try_frontend_peek`; if frontend sequencing declines, it falls back to `Command::Execute` through the coordinator. +`SessionClient::execute` unrolls SQL `EXECUTE ` statements via `unroll_sql_execute` before attempting frontend peek sequencing via `try_frontend_peek`; if the peek path declines, it attempts frontend read-then-write sequencing via `try_frontend_read_then_write_with_cancel` (for INSERT/UPDATE/DELETE statements); if both frontend paths decline, it falls back to `Command::Execute` through the coordinator. Blind writes (constant INSERTs whose source folds to a literal after optimization) are dispatched through `insert_constant`, which stages the rows as session write ops (`TransactionOps::Writes`) committed at transaction end; read-dependent writes (those whose selection depends on persisted collections) go through the OCC path in `PeekClient::frontend_read_then_write`. `Handle` holds the coordinator's background task handle and is used to await coordinator shutdown. `RecordFirstRowStream` is an adapter stream that records the timestamp of the first row for metrics. `TimeoutType` enumerates session-level timeouts (currently `IdleInTransactionSession`); the `Timeout` struct manages active timeout tasks and delivers expired timeouts through `SessionClient::recv_timeout`. diff --git a/doc/developer/generated/adapter/command.md b/doc/developer/generated/adapter/command.md index 5a9f8fb8620b9..19a9dcdc634a5 100644 --- a/doc/developer/generated/adapter/command.md +++ b/doc/developer/generated/adapter/command.md @@ -1,11 +1,11 @@ --- source: src/adapter/src/command.rs -revision: ff68428395 +revision: 39dcae2fba --- # adapter::command Defines the `Command` enum — the complete set of messages that external clients (pgwire, webhooks, internal tools) send to the coordinator — and the `ExecuteResponse` / `ExecuteResponseKind` enums that the coordinator returns. -`Command` covers session lifecycle (`Startup`, `Terminate`), SQL execution (`Execute`, `Commit`), authentication (`AuthenticatePassword`, SASL variants, `CheckRoleCanLogin`), streaming data ingestion (`StartCopyFromStdin`), audit event injection (`InjectAuditEvents`), and several variants that offload work from the coordinator's main loop: `ExecuteSlowPathPeek`, `ExecuteSubscribe`, `ExecuteCopyTo`, `CopyToPreflight`, `ExecuteSideEffectingFunc`, `LookupConnection`, `RegisterFrontendPeek`, `UnregisterFrontendPeek`, `ExplainTimestamp`, and `FrontendStatementLogging`. `LookupConnection` resolves a raw connection ID to its `ConnectionId` handle and authenticated role; holding the returned handle prevents the raw ID from being reused, which the frontend peek sequencing relies on to ensure RBAC checks and the subsequent `ExecuteSideEffectingFunc` act on the same connection. Two variants manage scoped feature-flag overrides: `UpdateScopedSystemParameters` carries the desired sparse `ScopedParameters` and an optional `ScopedParametersScope` bounding which objects' rows may be pruned; `InstallScopedSystemParameterFrontend` shares a new `Arc` with the coordinator so create-time scoped resolution can evaluate synchronously. -`ExecuteResponse` enumerates every possible result of a SQL statement execution, from `CreatedTable` to `SendingRowsStreaming`; `generated_from` maps each `PlanKind` to its permitted response kinds. +`Command` covers session lifecycle (`Startup`, `Terminate`), SQL execution (`Execute`, `Commit`), authentication (`AuthenticatePassword`, SASL variants, `CheckRoleCanLogin`), streaming data ingestion (`StartCopyFromStdin`), audit event injection (`InjectAuditEvents`), and several variants that offload work from the coordinator's main loop: `ExecuteSlowPathPeek`, `ExecuteSubscribe`, `ExecuteCopyTo`, `CopyToPreflight`, `ExecuteSideEffectingFunc`, `LookupConnection`, `RegisterFrontendPeek`, `UnregisterFrontendPeek`, `ExplainTimestamp`, and `FrontendStatementLogging`. `LookupConnection` resolves a raw connection ID to its `ConnectionId` handle and authenticated role; holding the returned handle prevents the raw ID from being reused, which the frontend peek sequencing relies on to ensure RBAC checks and the subsequent `ExecuteSideEffectingFunc` act on the same connection. Two variants manage scoped feature-flag overrides: `UpdateScopedSystemParameters` carries the desired sparse `ScopedParameters` and an optional `ScopedParametersScope` bounding which objects' rows may be pruned; `InstallScopedSystemParameterFrontend` shares a new `Arc` with the coordinator so create-time scoped resolution can evaluate synchronously. `RegisterConnectionCancelWatch` installs a connection-scoped cancellation watch, returning a receiver that fires when cancellation is requested; each registration replaces any prior channel so the caller cannot observe a cancellation aimed at an earlier statement. `CreateInternalSubscribe` creates a subscribe that writes no `mz_subscriptions` row, used by frontend-sequenced read-then-write (DELETE/UPDATE/INSERT...SELECT) operations via OCC. `AttemptWrite` submits a write attempt: `write_ts: Some(ts)` pins the write to an exact timestamp and reports `WriteResult::TimestampPassed` if the table has moved past it; `write_ts: None` rides the next group commit with a coordinator-chosen timestamp. `DropInternalSubscribe` drops an internal subscribe; it is fire-and-forget. +`ExecuteResponse` enumerates every possible result of a SQL statement execution, from `CreatedTable` to `SendingRowsStreaming`; `generated_from` maps each `PlanKind` to its permitted response kinds. `CreateMetricSink` maps to `[CreatedMetricSink]`, and `ExecuteResponse::CreatedMetricSink` carries the tag string `"CREATE METRIC SINK"`. `CopyFromStdinWriter` carries parallel `mpsc` batch sender channels and a `oneshot` completion receiver that pgwire uses to stream raw byte chunks to background batch-builder tasks and then commit the resulting `ProtoBatch` values during `COPY FROM STDIN`. diff --git a/doc/developer/generated/adapter/config/params.md b/doc/developer/generated/adapter/config/params.md index 8b4a1217a3301..69b667b4c8c17 100644 --- a/doc/developer/generated/adapter/config/params.md +++ b/doc/developer/generated/adapter/config/params.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/config/params.rs -revision: 7258dad07f +revision: 8e2fe64727 --- # adapter::config::params diff --git a/doc/developer/generated/adapter/config/sync.md b/doc/developer/generated/adapter/config/sync.md index 2727e3ce87cf2..137825e8327a1 100644 --- a/doc/developer/generated/adapter/config/sync.md +++ b/doc/developer/generated/adapter/config/sync.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/config/sync.rs -revision: dbd2c3fc06 +revision: 8e2fe64727 --- # adapter::config::sync diff --git a/doc/developer/generated/adapter/coord/_module.md b/doc/developer/generated/adapter/coord/_module.md index 65f9c53d66b35..33b863439bd55 100644 --- a/doc/developer/generated/adapter/coord/_module.md +++ b/doc/developer/generated/adapter/coord/_module.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord.rs -revision: 9de9cddf5e +revision: 39dcae2fba --- # adapter::coord @@ -16,7 +16,7 @@ Bootstrap handles derived builtin storage collections (builtin MVs) separately: The `clusters_caught_up_check` built during `serve` excludes both migrated MVs and new builtin MVs (and their transitive dependents) from the caught-up frontier check: migrated MV dataflows do not advance their write frontier in read-only mode, and new builtin MVs have no writer until the deployment promotes, so both sets would otherwise stall the check. The `Message` enum includes a `ClusterControllerRequest(cluster_controller::ClusterControllerRequest)` variant, carrying one pull/apply call from the cluster controller task to be answered on the main coordinator message loop. `Message::ArrangementSizesWrite(Vec)` carries the records prepared by the off-thread arrangement sizes snapshot task back to the coordinator loop for stamping and appending to `mz_object_arrangement_size_history`. -`ship_dataflow`, `try_ship_dataflow`, and `ship_dataflow_and_notice_builtin_table_updates` accept `DataflowDescription` (previously `DataflowDescription`). +`ship_dataflow`, `try_ship_dataflow`, and `ship_dataflow_and_notice_builtin_table_updates` accept `DataflowDescription`. `ExecuteContextInner` carries a `response_barriers: Vec` field (Debug-ignored). `ExecuteContext::from_parts` is a convenience wrapper around `from_parts_with_response_barriers`, which accepts an explicit barrier list. `into_parts` returns the barrier list as a fifth element alongside `tx`, `internal_cmd_tx`, `session`, and `extra`; callers that repack an `ExecuteContext` must preserve and forward these barriers. `delay_response_until` appends a `BuiltinTableAppendCompletion` barrier; on retirement, the response is held until all barriers resolve. The parts returned by `into_parts` lose the `Drop` backstop that answers the client on shutdown, so they must not be held across an await point (a bare `ClientTransmitter` panics when dropped unsent). When `retire` must wait on response barriers, it keeps `self` intact across the wait so the `Drop` backstop remains active; the barriers are cleared before the recursive call so the recursion terminates. `ExplainTimestampStage` has four variants in order: `Optimize`, `RealTimeRecency`, `LinearizeTimestamp` (carries `ExplainTimestampLinearizeTimestamp`: validity, format, optimized_plan, cluster_id, source_ids, when, real_time_recency_ts), and `Finish` (carries `ExplainTimestampFinish`: validity, format, cluster_id, source_ids, when, real_time_recency_ts, timeline_context, oracle_read_ts). The `LinearizeTimestamp` stage reads the oracle timestamp off the coordinator loop and passes it forward as `oracle_read_ts` in `ExplainTimestampFinish`. `SubscribeStage` has five variants in order: `OptimizeMir`, `LinearizeTimestamp` (carries `SubscribeLinearizeTimestamp`: validity, plan, timeline, optimizer, global_mir_plan, dependency_ids, replica_id, explain_ctx), `TimestampOptimizeLir` (carries `SubscribeTimestampOptimizeLir`, which includes `oracle_read_ts: Option`), `Finish`, and `Explain`. The `LinearizeTimestamp` stage reads the oracle timestamp off the coordinator loop before LIR optimization. diff --git a/doc/developer/generated/adapter/coord/appends.md b/doc/developer/generated/adapter/coord/appends.md index f2b0fa30faf35..424b8a7b9abc1 100644 --- a/doc/developer/generated/adapter/coord/appends.md +++ b/doc/developer/generated/adapter/coord/appends.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/appends.rs -revision: 6f5533eaa0 +revision: 39dcae2fba --- # adapter::coord::appends diff --git a/doc/developer/generated/adapter/coord/catalog_implications.md b/doc/developer/generated/adapter/coord/catalog_implications.md index 98187fd452632..8ba8791f1948e 100644 --- a/doc/developer/generated/adapter/coord/catalog_implications.md +++ b/doc/developer/generated/adapter/coord/catalog_implications.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/catalog_implications.rs -revision: f3b4f3f +revision: 39dcae2fba --- # adapter::coord::catalog_implications diff --git a/doc/developer/generated/adapter/coord/catalog_serving.md b/doc/developer/generated/adapter/coord/catalog_serving.md index 12bf72000678f..a5ff09f796b49 100644 --- a/doc/developer/generated/adapter/coord/catalog_serving.md +++ b/doc/developer/generated/adapter/coord/catalog_serving.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/catalog_serving.rs -revision: ab10152eac +revision: 39dcae2fba --- # adapter::coord::catalog_serving diff --git a/doc/developer/generated/adapter/coord/command_handler.md b/doc/developer/generated/adapter/coord/command_handler.md index 8bcecc97a64de..492bff4e70c4a 100644 --- a/doc/developer/generated/adapter/coord/command_handler.md +++ b/doc/developer/generated/adapter/coord/command_handler.md @@ -1,13 +1,13 @@ --- source: src/adapter/src/coord/command_handler.rs -revision: f3b4f3f +revision: 39dcae2fba --- # adapter::coord::command_handler Implements the coordinator's `handle_command` method, which dispatches each `Command` variant to the appropriate handler. -Covers session startup and teardown (`Startup`, `Terminate`), authentication, SQL execution (`Execute`, `Commit`), COPY FROM STDIN setup (`StartCopyFromStdin`), webhook lookup, system variable get/set, consistency checks, frontend-offload variants (`ExecuteSlowPathPeek`, `ExecuteSubscribe`, `ExecuteCopyTo`, `CopyToPreflight`, `ExecuteSideEffectingFunc`, `LookupConnection`, `RegisterFrontendPeek`, `UnregisterFrontendPeek`, `ExplainTimestamp`, `FrontendStatementLogging`) that reduce coordinator bottleneck, and scoped feature-flag management (`UpdateScopedSystemParameters` dispatches to `Coordinator::reconcile_scoped_system_parameters`; `InstallScopedSystemParameterFrontend` stores the shared frontend into `Coordinator::scoped_frontend`). -`LookupConnection` resolves a connection ID to its `ConnectionIdHandle` and authenticated role, used by the frontend peek path to carry the target connection's handle while performing RBAC checks for `pg_cancel_backend`. +Covers session startup and teardown (`Startup`, `Terminate`), authentication, SQL execution (`Execute`, `Commit`), COPY FROM STDIN setup (`StartCopyFromStdin`), webhook lookup, system variable get/set, consistency checks, frontend-offload variants (`ExecuteSlowPathPeek`, `ExecuteSubscribe`, `ExecuteCopyTo`, `CopyToPreflight`, `ExecuteSideEffectingFunc`, `LookupConnection`, `RegisterFrontendPeek`, `UnregisterFrontendPeek`, `ExplainTimestamp`, `FrontendStatementLogging`) that reduce coordinator bottleneck, scoped feature-flag management (`UpdateScopedSystemParameters` dispatches to `Coordinator::reconcile_scoped_system_parameters`; `InstallScopedSystemParameterFrontend` stores the shared frontend into `Coordinator::scoped_frontend`), and the frontend read-then-write variants (`RegisterConnectionCancelWatch` installs a per-connection cancel watch channel; `CreateInternalSubscribe` delegates to `Coordinator::handle_create_internal_subscribe`; `AttemptWrite` delegates to `Coordinator::handle_attempt_write`; `DropInternalSubscribe` delegates to `Coordinator::drop_internal_subscribe`). +`LookupConnection` resolves a raw connection ID to its `ConnectionId` handle and authenticated `RoleId`, used by the frontend peek path to carry the target connection's handle while performing RBAC checks for `pg_cancel_backend`. `handle_startup` performs privilege checks, triggers JWT group-to-role membership sync via `maybe_sync_jwt_groups`, sets session defaults, and emits the initial builtin-table writes for session tracking. The private `handle_startup_inner` encapsulates all fallible work: when it returns an error, no per-connection coordinator state exists (per-connection state is registered only in the `Ok` arm of `handle_startup`), so `handle_terminate` must not be called for a failed startup. Durable catalog changes made before an error (an auto-provisioned role, synced role memberships) are role-scoped rather than connection-scoped and are intentionally kept. `handle_terminate` must only be called for connections that completed a successful startup, i.e. that are present in `active_conns`; this is enforced by the coordinator's startup/teardown protocol and the cleanup guard in `Client::startup` that omits `Command::Terminate` on failure. During session default resolution, if the resolved `transaction_isolation` default names a feature-flagged isolation level whose flag is disabled (e.g. a role default set while `bounded staleness` was enabled, with the flag later turned off), that default is dropped so the session falls back to the built-in default. At the start of each top-level statement execution (i.e. when `outer_context.is_none()`), the coordinator removes any pending cancellation watch for the connection via `connection_cancel_watches.remove`. This discards stale cancellations that arrived while no statement was running, preventing them from cancelling the next statement. Nested executes (e.g. FETCH executing its cursor's statement) skip this removal because the outer statement is still running. For statements that cannot be run inside a transaction, the `OperationProhibitsTransaction` error redacts literals for sensitive statements (those where `StatementKind::from(&*stmt).is_sensitive()` is true) via `stmt.to_ast_string_redacted()`, preventing secret material from leaking into error messages that are persisted in `mz_statement_execution_history`. Other statements use `stmt.to_string()` for a clearer error. diff --git a/doc/developer/generated/adapter/coord/ddl.md b/doc/developer/generated/adapter/coord/ddl.md index 29d45fc1de801..db67c1e95b919 100644 --- a/doc/developer/generated/adapter/coord/ddl.md +++ b/doc/developer/generated/adapter/coord/ddl.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/ddl.rs -revision: f3b4f3f +revision: 39dcae2fba --- # adapter::coord::ddl diff --git a/doc/developer/generated/adapter/coord/indexes.md b/doc/developer/generated/adapter/coord/indexes.md index bad64868c0f42..38700748b4f53 100644 --- a/doc/developer/generated/adapter/coord/indexes.md +++ b/doc/developer/generated/adapter/coord/indexes.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/indexes.rs -revision: 9d0a7c3c6f +revision: 39dcae2fba --- # adapter::coord::indexes diff --git a/doc/developer/generated/adapter/coord/read_then_write.md b/doc/developer/generated/adapter/coord/read_then_write.md index df86fbfe9f669..95e5dbf51a3b3 100644 --- a/doc/developer/generated/adapter/coord/read_then_write.md +++ b/doc/developer/generated/adapter/coord/read_then_write.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/read_then_write.rs -revision: 50f9dad7b2 +revision: 39dcae2fba --- # adapter::coord::read_then_write @@ -14,3 +14,4 @@ User tables (that are not source exports), user-defined views, user-defined mate On failure, the function returns `AdapterError::Unsupported` (for `mz_now()` usage), `AdapterError::ReadThenWriteDependencyLimitExceeded` (when the dependency bound is exceeded), or `AdapterError::InvalidTableMutationSelection` with the offending object's name and type. `Coordinator::handle_create_internal_subscribe` creates a subscribe that introspection does not see (`internal: true` on `ActiveSubscribe`). It takes ownership of `read_holds` and drops them only after the dataflow is shipped, preventing the `since` from advancing past `as_of` in the interim. The dataflow is shipped via `try_ship_dataflow` before the sink is registered, so a failure (e.g. a dependency dropped since optimization) leaves nothing to unwind. Results are delivered through a `response_tx` oneshot; if the receiver is gone when the handler runs, the internal subscribe is immediately retired via `drop_internal_subscribe`. `Coordinator::drop_internal_subscribe` cancels the dataflow on the compute side by delegating to `drop_compute_sink`. +`Coordinator::handle_attempt_write` enqueues a write attempt from the frontend read-then-write path. It checks that the connection is still active and the coordinator is not in read-only mode, then validates the target's current `GlobalId` generation against the caller's `target_global_id`; a generation mismatch sends `WriteResult::TargetChanged`. When `write_ts` is `Some`, the write is submitted as a `TimestampedWriteRequest` via the group committer; when `None`, it rides the next group commit as a blind write. diff --git a/doc/developer/generated/adapter/coord/sequencer.md b/doc/developer/generated/adapter/coord/sequencer.md index e1e7d958d1570..acc3fcdc502b5 100644 --- a/doc/developer/generated/adapter/coord/sequencer.md +++ b/doc/developer/generated/adapter/coord/sequencer.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/sequencer.rs -revision: 38a95cefe2 +revision: 95baa04a85 --- # adapter::coord::sequencer diff --git a/doc/developer/generated/adapter/coord/sequencer/_module.md b/doc/developer/generated/adapter/coord/sequencer/_module.md index 953fe5f576802..32c867f044e3a 100644 --- a/doc/developer/generated/adapter/coord/sequencer/_module.md +++ b/doc/developer/generated/adapter/coord/sequencer/_module.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/sequencer.rs -revision: 38a95cefe2 +revision: 95baa04a85 --- # adapter::coord::sequencer diff --git a/doc/developer/generated/adapter/coord/sequencer/inner.md b/doc/developer/generated/adapter/coord/sequencer/inner.md index cb8212c4f25e4..26daabda5fd2f 100644 --- a/doc/developer/generated/adapter/coord/sequencer/inner.md +++ b/doc/developer/generated/adapter/coord/sequencer/inner.md @@ -1,19 +1,20 @@ --- source: src/adapter/src/coord/sequencer/inner.rs -revision: 6f5533eaa0 +revision: 00299a05e3 --- # adapter::coord::sequencer::inner Implements the majority of `sequence_*` methods: general DDL (CREATE/DROP/ALTER for sources, sinks, connections, tables, types, roles, schemas, databases, network policies), DML (INSERT, UPDATE, DELETE via `sequence_read_then_write`), transaction management, cursor operations, COPY FROM, SHOW, SET/RESET, FETCH, RAISE, and more. Each method validates the plan against the current catalog and session state, performs the catalog transaction, applies catalog implications, and returns the appropriate `ExecuteResponse`. -The `inner` sub-modules split out the most complex individual statement types (peek, subscribe, cluster, index, materialized view, view, copy_from, secret, explain_timestamp) into separate files. +The `inner` sub-modules split out the most complex individual statement types (peek, subscribe, cluster, index, materialized view, view, copy_from, secret, explain_timestamp, create_metric_sink) into separate files. Also defines the `sequence_staged` generic driver and `Staged` / `StagedContext` traits used across all multi-stage sequencing pipelines. `validate_role_attributes` permits the `LOGIN` attribute even when password auth is disabled, restricting the unavailable-feature gate to `SUPERUSER` and `PASSWORD` attributes. `CREATE CONNECTION ... VALIDATE` and `ALTER CONNECTION ... VALIDATE` tasks are wrapped in `ore_catch_unwind` to convert panics (e.g., from malformed TLS material) into `AdapterError::Internal` rather than crashing the coordinator. Connection secret content is validated through `check_connection_secret_content_guards`, which iterates `details.secret_content_guards()` for a connection and reads each referenced secret via `caching_secrets_reader`. This is called for both `CREATE CONNECTION` and `ALTER CONNECTION` before the catalog entry is installed. The symmetric helper `check_secret_content_guards_of_dependents` validates proposed new secret contents against every connection that references the secret, and is called from `sequence_alter_secret` before the new value is persisted. Privilege grant/revoke operations (`sequence_grant_privileges`) group all grantee changes for the same target object into a single `Op::UpdatePrivilege` with a `privileges: Vec` field, so a bulk grant/revoke touching one object is a single durable write rather than one per grantee. `sequence_revoke_role` applies the same grouping: privilege revokes are collected by target into a `BTreeMap` before emitting ops. Write operations (`sequence_insert`, `sequence_read_then_write`) reject sessions using the bounded staleness isolation level with `AdapterError::BoundedStalenessReadOnly`. `sequence_insert` includes a test-only failpoint `insert_after_pack_before_commit` that parks a blind INSERT after its rows are packed but before the implicit commit stages them for group commit, enabling tests to land a concurrent `ALTER TABLE` in that window. `sequence_set_variable` rejects combinations of `transaction_isolation = 'bounded staleness'` and `real_time_recency = on` with `AdapterError::BoundedStalenessRealTimeRecencyConflict`. +`sequence_read_then_write` returns `AdapterError::Internal` immediately when `self.frontend_read_then_write_enabled` is true; the lock-based coordinator path and the OCC frontend path do not synchronize with each other, so routing a statement to the coordinator's read-then-write path while the frontend is enabled is a bug. `sequence_read_then_write` validates all transitive dependencies of the selection in a single call to `validate_read_then_write_dependencies`, passing the full set of dependency `CatalogItemId`s and a bound read from the `READ_THEN_WRITE_MAX_DEPENDENCIES` dyncfg; exceeding the bound returns `AdapterError::ReadThenWriteDependencyLimitExceeded`. `await_real_time_recent_timestamp` and the private `real_time_recent_timestamp_error` helper convert `StorageError::RtrTimeout` and `StorageError::RtrDropFailure` to the dedicated `AdapterError::RtrTimeout` / `AdapterError::RtrDropFailure` variants (with humanized collection names) before propagating; these helpers are called from the RTR-awaiting tasks in `peek`, `explain_timestamp`, and `command_handler`. `sequence_side_effecting_func` handles `PgCancelBackend` with a `NULL` connection-id argument by returning `NULL` immediately (matching PostgreSQL semantics), before attempting to look up or cancel any connection. diff --git a/doc/developer/generated/adapter/coord/sequencer/inner/_module.md b/doc/developer/generated/adapter/coord/sequencer/inner/_module.md index 97609f81d74dd..03d4998d25f29 100644 --- a/doc/developer/generated/adapter/coord/sequencer/inner/_module.md +++ b/doc/developer/generated/adapter/coord/sequencer/inner/_module.md @@ -1,12 +1,12 @@ --- source: src/adapter/src/coord/sequencer/inner.rs -revision: 6f5533eaa0 +revision: 00299a05e3 --- # adapter::coord::sequencer::inner Houses the per-statement sequencing implementations split into child files for the most complex statement types. -`inner.rs` itself handles the majority of DDL and DML statements; the child modules (`peek`, `subscribe`, `cluster`, `copy_from`, `create_index`, `create_materialized_view`, `create_view`, `secret`, `explain_timestamp`) each own one focused area of the sequencing logic. +`inner.rs` itself handles the majority of DDL and DML statements; the child modules (`peek`, `subscribe`, `cluster`, `copy_from`, `create_index`, `create_materialized_view`, `create_metric_sink`, `create_view`, `secret`, `explain_timestamp`) each own one focused area of the sequencing logic. Together they implement the full `sequence_plan` dispatch surface for every SQL plan kind. The generic `sequence_staged` driver and the `Staged` / `StagedContext` / `StageResult` traits live in `inner.rs`, providing the common loop that advances multi-stage plans either immediately or by spawning background tasks and re-queuing via the coordinator's message channel. `validate_role_attributes` permits the `LOGIN` attribute even when password auth is disabled, restricting the unavailable-feature gate to `SUPERUSER` and `PASSWORD` attributes. @@ -15,5 +15,6 @@ The generic `sequence_staged` driver and the `Staged` / `StagedContext` / `Stage `execute_side_effecting_func` (used by the frontend peek path) performs no RBAC check itself; RBAC is pre-checked by the caller via `rbac::check_plan` before `Command::ExecuteSideEffectingFunc` is sent. Connection secret content is validated through `check_connection_secret_content_guards` for `CREATE CONNECTION` and `ALTER CONNECTION`, and through `check_secret_content_guards_of_dependents` when a secret's value changes, before any catalog entry is installed or persisted. `inner.rs` includes a test-only failpoint `insert_after_pack_before_commit` that parks a blind INSERT after its rows are packed but before the implicit commit stages them for group commit, enabling tests to land a concurrent `ALTER TABLE` in that window. +`sequence_read_then_write` returns `AdapterError::Internal` immediately when `self.frontend_read_then_write_enabled` is true; the lock-based coordinator path and the OCC frontend path do not synchronize with each other, so reaching the coordinator's read-then-write path while the frontend is enabled is a routing bug. Privilege grant/revoke operations group all grantee changes for the same target object into a single `Op::UpdatePrivilege` carrying a `privileges: Vec`, so a bulk grant/revoke affecting one object is a single durable write rather than one per grantee. `sequence_alter_sink` handles `ALTER SINK` operations including `SET FROM` and option changes (e.g. `COMMIT INTERVAL`). It syncs `resolved_ids` to match the new `create_sql` and `from` target before constructing the updated `Sink` and emitting `Op::UpdateItem`. The `resolved_ids` derived from the old `create_sql` still references the old input; without this sync the in-memory catalog disagrees with `create_sql` until the next reload, and the temporary-dependency check in `Op::UpdateItem` (which reads `uses()`) would not see the new input. Option edits from the plan's `set_options` and `reset_options` fields are applied to the `CREATE SINK` statement via `plan::apply_sink_option_edits` after name resolution. diff --git a/doc/developer/generated/adapter/coord/sequencer/inner/cluster.md b/doc/developer/generated/adapter/coord/sequencer/inner/cluster.md index e55bfa8c84b71..7a60d5bf9df37 100644 --- a/doc/developer/generated/adapter/coord/sequencer/inner/cluster.md +++ b/doc/developer/generated/adapter/coord/sequencer/inner/cluster.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/coord/sequencer/inner/cluster.rs -revision: a4e7587de2 +revision: e3fe017dcb --- # adapter::coord::sequencer::inner::cluster @@ -12,8 +12,8 @@ Replica IDs are pre-allocated out-of-band via `Catalog::allocate_replica_ids` be Cluster and replica creation and deletion are applied through the catalog implications pipeline rather than directly in this module. For `CREATE CLUSTER` (both managed and unmanaged variants), `scoped_overrides_create_op` is called with `ClusterEvalContext` and `ReplicaEvalContext` values built from plan data and pre-allocated replica IDs, and the resulting `Op::UpdateScopedSystemParameters` is folded into the same catalog transaction that creates the cluster. This makes the committed diff drive the replica-scoped controller push before `create_replica`, so the new cluster's optimizer and each new replica's render-frozen flags take effect before any dataflow renders, rather than falling back to environment-wide values until the next sync tick. `CREATE CLUSTER` validates the `HYDRATION SIZE` of an `AUTO SCALING STRATEGY` against the allowed replica sizes for the session role, just as `SIZE` itself is validated. This prevents a typo from failing silently at burst-arm time and prevents a size-restricted role from bursting at a size it may not `CREATE` with. -A shape-changing managed-to-managed `ALTER CLUSTER` (one that changes `SIZE`, `AVAILABILITY ZONES`, or `INTROSPECTION`) reshapes into a durable `reconfiguration` record via `reshape_alter_cluster_managed` instead of going through the legacy 3-stage machine. The record carries the full target config shape, a deadline, and an `on_timeout` action (`ROLLBACK` by default). Non-shape fields (`workload_class`, `schedule`, `auto_scaling_strategy`, etc.) are applied to the realized config immediately. The controller converges the replica set onto the target and cuts the realized config over at hydration. `ClusterStage::AwaitReconfiguration` (carrying `AlterClusterAwaitReconfiguration`) is a foreground wait-shim that polls the durable record until it reaches a terminal status, then reports success or failure depending on whether the realized config reached the target; it is used when `ENABLE_BACKGROUND_ALTER_CLUSTER` is off. The legacy foreground `AlterClusterWaitForHydrated` readiness check additionally requires every pending replica to be online (per `mz_cluster_replica_statuses`) before declaring the reconfiguration ready to cut over, so a cluster hosting only single-replica sources (which are never placed on a pending replica until cut-over) does not declare readiness vacuously the moment the pending replicas are created. -When a reconfiguration is in flight, a subsequent shape-changing `ALTER` folds onto the in-flight target via `fold_reconfiguration_target`: dimensions the `ALTER` set explicitly take the new value, while dimensions left `Unchanged` keep the in-flight target's value (not the realized config's value, which would silently revert the in-flight transition). The `deadline` and `on_timeout` are handled separately: when the folding `ALTER` carries an explicit `WAIT` clause, the clause's deadline and action overwrite the in-flight record's (allowing a later `ALTER` to steer the deadline without discarding hydration progress already made on the target); when the folding `ALTER` carries no `WAIT` clause, the in-flight record's `deadline` and `on_timeout` are preserved so that an unrelated shape-changing `ALTER` does not silently reset the user-set deadline. When there is no reconfiguration in flight and no `WAIT` clause, the system-default timeout and implicit `ROLLBACK` action apply. An `ALTER` back to the realized shape cancels the record. Changing the replication factor while a reconfiguration is in flight is refused with `AdapterError::AlterClusterReplicationFactorWhileReconfiguring`. +A shape-changing managed-to-managed `ALTER CLUSTER` (one that changes `SIZE`, `AVAILABILITY ZONES`, `INTROSPECTION`, or `ARRANGEMENT COMPRESSION`) reshapes into a durable `reconfiguration` record via `reshape_alter_cluster_managed` instead of going through the legacy 3-stage machine. The record carries the full target config shape, a deadline, and an `on_timeout` action (`ROLLBACK` by default). Non-shape fields (`workload_class`, `schedule`, `auto_scaling_strategy`, etc.) are applied to the realized config immediately. The controller converges the replica set onto the target and cuts the realized config over at hydration. `ClusterStage::AwaitReconfiguration` (carrying `AlterClusterAwaitReconfiguration`) is a foreground wait-shim that polls the durable record until it reaches a terminal status, then reports success or failure depending on whether the realized config reached the target; it is used when `ENABLE_BACKGROUND_ALTER_CLUSTER` is off. The legacy foreground `AlterClusterWaitForHydrated` readiness check additionally requires every pending replica to be online (per `mz_cluster_replica_statuses`) before declaring the reconfiguration ready to cut over, so a cluster hosting only single-replica sources (which are never placed on a pending replica until cut-over) does not declare readiness vacuously the moment the pending replicas are created. +When a reconfiguration is in flight, a subsequent shape-changing `ALTER` folds onto the in-flight target via `fold_reconfiguration_target`: dimensions the `ALTER` set explicitly take the new value, while dimensions left `Unchanged` keep the in-flight target's value (not the realized config's value, which would silently revert the in-flight transition). `ReconfigurationTarget` carries `size`, `replication_factor`, `availability_zones`, `logging`, and `arrangement_compression` as its shape dimensions; `ReconfigurationDimensionsUnchanged` tracks which of those are unchanged by a given `ALTER`. The `deadline` and `on_timeout` are handled separately: when the folding `ALTER` carries an explicit `WAIT` clause, the clause's deadline and action overwrite the in-flight record's (allowing a later `ALTER` to steer the deadline without discarding hydration progress already made on the target); when the folding `ALTER` carries no `WAIT` clause, the in-flight record's `deadline` and `on_timeout` are preserved so that an unrelated shape-changing `ALTER` does not silently reset the user-set deadline. When there is no reconfiguration in flight and no `WAIT` clause, the system-default timeout and implicit `ROLLBACK` action apply. An `ALTER` back to the realized shape cancels the record. Changing the replication factor while a reconfiguration is in flight is refused with `AdapterError::AlterClusterReplicationFactorWhileReconfiguring`. `ALTER CLUSTER` validates a newly set or changed `AUTO SCALING STRATEGY`'s `HYDRATION SIZE` like `SIZE` itself: it must name a real replica size the session role may use. Only a changed strategy is checked, so an existing policy does not block unrelated `ALTER`s if the size allow-list later shrinks. Additionally, if a reconfiguration is in flight, `HYDRATION SIZE` must differ from the in-flight reconfiguration's target `SIZE`: equality would end the reshape with a no-op burst shape and a stored statement that fails its own re-plan. Unmanaged clusters reject `AUTO SCALING STRATEGY` changes with an error, mirroring how other managed-only options are handled. Converting a cluster to unmanaged while a hydration burst is in flight is refused with `AdapterError::AlterClusterUnmanagedWhileBursting`. The unmanaged variant carries no burst field, so converting would drop the record without a `Finished` audit event and strand the burst replica as an ordinary unmanaged replica nothing ever tears down. `validate_reconfiguration_resource_limits` checks that the transient peak credit rate (realized plus target shapes running simultaneously) and per-cluster replica count fit the environment-wide budget before the record is written. `validate_reconfiguration_credit_peak` computes the peak by excluding the reconfiguring cluster's own replicas from the base consumption rate to avoid double-counting an already-materialized overlap. diff --git a/doc/developer/generated/adapter/coord/sequencer/inner/create_metric_sink.md b/doc/developer/generated/adapter/coord/sequencer/inner/create_metric_sink.md new file mode 100644 index 0000000000000..13bcf0ae516dc --- /dev/null +++ b/doc/developer/generated/adapter/coord/sequencer/inner/create_metric_sink.md @@ -0,0 +1,13 @@ +--- +source: src/adapter/src/coord/sequencer/inner/create_metric_sink.rs +revision: 39dcae2fba +--- + +# adapter::coord::sequencer::inner::create_metric_sink + +Implements `CREATE METRIC SINK` sequencing. +`Coordinator::sequence_create_metric_sink` allocates a user id, constructs a `CatalogItem::MetricSink` from the plan, and commits it via `catalog_transact`. No dataflow is optimized or shipped: the catalog item records the definition for future use and survives restarts because `create_sql` is re-parsed on boot. + +`ensure_metric_sink_prefix_is_free` enforces a prefix-free uniqueness constraint on all metric sinks bound to the target cluster. The published metric name is `prefix + metric_name`, so two prefixes where one is a prefix of the other would publish colliding names; Prometheus silently merges same-named families. The check is performed at sequencing time rather than at plan time, because planning is not serialized against catalog writes: two concurrent `CREATE METRIC SINK` statements can plan against the same state, but the coordinator sequences one statement at a time and nothing commits between this check and `catalog_transact`. A sink already holding the same item name is skipped, since the create is then a no-op (`IF NOT EXISTS`) or an "already exists" error. + +`IF NOT EXISTS` is handled by matching `ErrorKind::Sql(CatalogError::ItemAlreadyExists)` from the catalog transaction and adding an `AdapterNotice::ObjectAlreadyExists` notice rather than returning an error. diff --git a/doc/developer/generated/adapter/error.md b/doc/developer/generated/adapter/error.md index b5ca1e2315a01..3218c4cb676b7 100644 --- a/doc/developer/generated/adapter/error.md +++ b/doc/developer/generated/adapter/error.md @@ -1,12 +1,12 @@ --- source: src/adapter/src/error.rs -revision: f4650d00b +revision: 4f0805a4d8 --- # adapter::error Defines `AdapterError`, the central error type for the coordinator, with variants covering planning errors, catalog errors, storage/compute controller errors, RBAC violations, timestamp errors, authentication errors, and miscellaneous unstructured errors. -Notable variants include `ReadThenWriteDependencyLimitExceeded { max_rw_dependencies: usize }` (maps to `SqlState::PROGRAM_LIMIT_EXCEEDED`) when a read-then-write statement's selection transitively depends on more objects than the configured `max_rw_dependencies` limit; the error message includes the limit and a hint to raise `read_then_write_max_dependencies`. `ConcurrentDependencyDrop { dependency_kind, dependency_id }` for mid-flight dependency drops, `ConcurrentDependencyMutation { dependency_id }` (maps to `SqlState::T_R_SERIALIZATION_FAILURE`) when a dependency's `create_sql` changed between plan time and commit (detected by `PlanValidity`'s optional hash check), or when a staged table write's rows were packed against a `RelationDesc` that a concurrent `ALTER TABLE` has since replaced (detected by `Coordinator::stale_write_target` during group commit), `AuthenticationError` (with an inner `AuthenticationError` enum covering `InvalidCredentials`, `NonLogin`, `RoleNotFound`, `PasswordRequired`), `ReplacementSchemaMismatch(RelationDescDiff)` for schema-incompatible replacements, `ReplaceMaterializedViewSealed`, `CollectionUnreadable`, `AlterClusterWhilePendingReplicas`, `AlterClusterUnmanagedWhileReconfiguring` (maps to `SqlState::OBJECT_IN_USE`) for attempts to convert a cluster to unmanaged while a graceful reconfiguration is in progress, `AlterClusterUnmanagedWhileBursting` (maps to `SqlState::OBJECT_IN_USE`) for attempts to convert a cluster to unmanaged while a hydration burst is in flight, `AlterClusterReplicationFactorWhileReconfiguring` (maps to `SqlState::OBJECT_IN_USE`) for attempts to change a cluster's replication factor while a graceful reconfiguration is in progress, `AlterClusterScheduleWhileReconfiguring` (maps to `SqlState::OBJECT_IN_USE`) for attempts to change a cluster's schedule while a graceful reconfiguration is in progress, `AlterClusterWaitOnScheduledCluster` (maps to `SqlState::FEATURE_NOT_SUPPORTED`) for attempts to use `WAIT` on an `ALTER` of a scheduled (non-`MANUAL`) cluster whose replica set the controller reshapes in the background, `ImpossibleTimestampConstraints`, `OidcGroupSyncFailed(String)` for OIDC group-to-role sync failures in strict mode (maps to `SqlState::INTERNAL_ERROR`), `ClusterStateChanged { cluster_id }` (maps to `SqlState::T_R_SERIALIZATION_FAILURE`) produced by `Op::CheckClusterState` when a conditional cluster-config write fails its precondition, and four bounded-staleness variants: `BoundedStalenessExceeded { bound, gap_ms, slowest_input }` (maps to `SqlState::T_R_SERIALIZATION_FAILURE`), `BoundedStalenessReadOnly` (maps to `SqlState::READ_ONLY_SQL_TRANSACTION`), `BoundedStalenessRealTimeRecencyConflict` (maps to `SqlState::FEATURE_NOT_SUPPORTED`), and `BoundedStalenessTimelineUnsupported` (maps to `SqlState::FEATURE_NOT_SUPPORTED`). +Notable variants include `ReadThenWriteDependencyLimitExceeded { max_rw_dependencies: usize }` (maps to `SqlState::PROGRAM_LIMIT_EXCEEDED`) when a read-then-write statement's selection transitively depends on more objects than the configured `max_rw_dependencies` limit; the error message includes the limit and a hint to raise `read_then_write_max_dependencies`. `ReadThenWriteContention` (maps to `SqlState::T_R_SERIALIZATION_FAILURE`) is returned when a frontend read-then-write operation exhausts its OCC retry budget; since every attempt was refused before any append, the statement is retryable. `ConcurrentDependencyDrop { dependency_kind, dependency_id }` for mid-flight dependency drops, `ConcurrentDependencyMutation { dependency_id }` (maps to `SqlState::T_R_SERIALIZATION_FAILURE`) when a dependency's `create_sql` changed between plan time and commit (detected by `PlanValidity`'s optional hash check), or when a staged table write's rows were packed against a `RelationDesc` that a concurrent `ALTER TABLE` has since replaced (detected by `Coordinator::stale_write_target` during group commit), `AuthenticationError` (with an inner `AuthenticationError` enum covering `InvalidCredentials`, `NonLogin`, `RoleNotFound`, `PasswordRequired`), `ReplacementSchemaMismatch(RelationDescDiff)` for schema-incompatible replacements, `ReplaceMaterializedViewSealed`, `CollectionUnreadable`, `AlterClusterWhilePendingReplicas`, `AlterClusterUnmanagedWhileReconfiguring` (maps to `SqlState::OBJECT_IN_USE`) for attempts to convert a cluster to unmanaged while a graceful reconfiguration is in progress, `AlterClusterUnmanagedWhileBursting` (maps to `SqlState::OBJECT_IN_USE`) for attempts to convert a cluster to unmanaged while a hydration burst is in flight, `AlterClusterReplicationFactorWhileReconfiguring` (maps to `SqlState::OBJECT_IN_USE`) for attempts to change a cluster's replication factor while a graceful reconfiguration is in progress, `AlterClusterScheduleWhileReconfiguring` (maps to `SqlState::OBJECT_IN_USE`) for attempts to change a cluster's schedule while a graceful reconfiguration is in progress, `AlterClusterWaitOnScheduledCluster` (maps to `SqlState::FEATURE_NOT_SUPPORTED`) for attempts to use `WAIT` on an `ALTER` of a scheduled (non-`MANUAL`) cluster whose replica set the controller reshapes in the background, `ImpossibleTimestampConstraints`, `OidcGroupSyncFailed(String)` for OIDC group-to-role sync failures in strict mode (maps to `SqlState::INTERNAL_ERROR`), `ClusterStateChanged { cluster_id }` (maps to `SqlState::T_R_SERIALIZATION_FAILURE`) produced by `Op::CheckClusterState` when a conditional cluster-config write fails its precondition, and four bounded-staleness variants: `BoundedStalenessExceeded { bound, gap_ms, slowest_input }` (maps to `SqlState::T_R_SERIALIZATION_FAILURE`), `BoundedStalenessReadOnly` (maps to `SqlState::READ_ONLY_SQL_TRANSACTION`), `BoundedStalenessRealTimeRecencyConflict` (maps to `SqlState::FEATURE_NOT_SUPPORTED`), and `BoundedStalenessTimelineUnsupported` (maps to `SqlState::FEATURE_NOT_SUPPORTED`). Implements `From` conversions from many downstream error types (`PlanError`, `StorageError`, `OptimizerError`, `VarError`, etc.) and maps each variant to a PostgreSQL `SqlState` error code via `code()`; error details and hints are provided through `detail()` and `hint()`. The `From` conversion maps `OptimizerError::RestrictedFunction` to `AdapterError::Unauthorized(UnauthorizedError::RestrictedSystemObject)` rather than the generic `Optimizer` variant; within `code()`, `OptimizerError::RestrictedFunction` maps to `SqlState::INSUFFICIENT_PRIVILEGE`. Several constructor helpers on `AdapterError` (e.g. `concurrent_dependency_drop_from_instance_missing`, `concurrent_dependency_drop_from_peek_error`) perform explicit error-kind conversions that are intentionally not automatic `From` impls. diff --git a/doc/developer/generated/adapter/frontend_read_then_write.md b/doc/developer/generated/adapter/frontend_read_then_write.md new file mode 100644 index 0000000000000..0a06297a02473 --- /dev/null +++ b/doc/developer/generated/adapter/frontend_read_then_write.md @@ -0,0 +1,48 @@ +--- +source: src/adapter/src/frontend_read_then_write.rs +revision: 00299a05e3 +--- + +# adapter::frontend_read_then_write + +Implements `INSERT [...] SELECT`, `DELETE`, and `UPDATE` using a subscribe with optimistic concurrency control (OCC), sequenced from the session task rather than the coordinator's main loop. +`PeekClient::frontend_read_then_write` is the main entry point. It validates the plan, acquires an OCC semaphore permit and a read hold, optimizes the selection into a subscribe dataflow, runs the OCC loop, and then handles the outcome by staging rows, submitting a blind write, or advancing the session's write timestamp. + +The OCC semaphore serializes concurrent read-then-write operations process-wide, bounding the number that hold read holds simultaneously. The permit is acquired before the read hold so that queued operations do not pin compaction while waiting. + +## Whether the write reads persisted state + +Two predicates determine this. `validate_selection_dependencies` inspects `depends_on()` syntactically before the dataflow runs, because a read-dependent write inside a transaction must be refused while refusal is still possible. The subscribe answers dynamically at runtime: a channel that closes on its own means the selection's output frontier reached the empty antichain and the diffs are frontier-independent. + +The syntactic answer decides transaction membership. A write that reads nothing stages its rows as `TransactionOps::Writes` and commits atomically with the rest of the transaction. A write that reads persisted state is refused inside an explicit transaction; in an extended-protocol pipeline it ends its own implicit transaction rather than spanning the rest of the pipeline, matching PostgreSQL's treatment of statements that cannot run in a transaction block. + +The dynamic answer decides only how the write is submitted: a timestamped write from inside the OCC loop, or a blind submission after it. `end_own_transaction` clears staged ops for statements that committed on their own, preventing `TransactionStatus::may_span_pipeline` from holding the pipeline open. + +Disagreement between the two predicates is caught by `soft_assert_or_log!` in the `Committed` and `NoRowsMatched` arms of the outcome handler. + +## The OCC loop (`run_occ_loop`) + +The loop drains the subscribe, accumulates diffs in `OccState::all_diffs`, and consolidates incrementally on each progress message. When the consolidated state is ready to write (progress past `as_of` and nonempty diffs), it submits a `Command::AttemptWrite` at the observed frontier as the write timestamp. + +On `WriteResult::TimestampPassed` (concurrent write advanced the target's upper), the loop clears `write_submitted`, increments `retry_count`, and waits for the subscribe to advance before retrying. Retries are bounded by `max_occ_retries`; exceeding the limit yields `AdapterError::ReadThenWriteContention`. + +A subscribe channel that closes on its own exits via `OccOutcome::Blind`, returning frontier-independent diffs for the caller to stage or submit. A consolidated empty result at a progress timestamp past `as_of` exits via `OccOutcome::NoRowsMatched`. + +`process_message` handles the subscribe output format (`mz_timestamp`, `mz_progressed`, `mz_diff`, data columns), validates constraints on inserted rows via `RelationDesc::constraints_met`, and enforces `max_result_size`. Format mismatches are surfaced as `AdapterError::Internal` rather than a panic. + +## Cancellation and write safety (`FrontendWriteAttemptState`) + +`FrontendWriteAttemptState` coordinates between the attempt and its cancellation wrapper. `write_submitted` is set just before `Command::AttemptWrite` is sent and cleared only on `TimestampPassed` (the only outcome where the write definitively did not land). Cancellation and statement timeout check `write_submitted` and, if set, await the definitive write result rather than synthesizing an error, because the write may already be durable. + +## Read linearization + +`ensure_read_linearized` blocks until the governing timeline's oracle has advanced to `as_of`, implementing the strict-serializable read guarantee. It uses `governing_timeline` (not `TimelineContext::timeline()`) so that a timestamp-independent selection still linearizes against the `EpochMilliseconds` oracle. A single group-commit nudge is issued at the start of the wait to avoid polling for a full `default_timestamp_interval`. Linearization runs before the subscribe is created and, for `NoRowsMatched` with an observed timestamp, after the OCC loop, with the OCC permit released before the second wait. + +## Key helpers + +- `validate_read_then_write`: enforces the `mz_now()` ban, the dependency cap, timeline compatibility (EpochMilliseconds only), cluster liveness, and replica pin; returns a `ValidationResult` carrying cluster, replica, timeline, and table descriptor. +- `optimize_mir_read_then_write`: applies the mutation to the MIR selection via `apply_mutation_to_mir`, prepares unmaterializable functions one-shot, and runs the subscribe optimizer to produce a `GlobalMirPlan`. +- `apply_mutation_to_mir`: DELETE negates the expression; UPDATE wraps it in a `Let` binding and unions negated old rows with mapped new rows; INSERT passes through unchanged. +- `build_success_response`: builds the `ExecuteResponse` before writing, enforcing result-size caps row-by-row for RETURNING expressions to bound temporary allocation. +- `submit_blind_write`: sends `Command::AttemptWrite` with `write_ts: None`, letting group commit choose the timestamp. +- `classify_write_result`: maps `WriteResult` to `WriteOutcome`, translating `TargetChanged` to `ConcurrentDependencyMutation` for consistent client-visible error codes. diff --git a/doc/developer/generated/adapter/metrics.md b/doc/developer/generated/adapter/metrics.md index 8fe81d12e48f5..06288a70a2777 100644 --- a/doc/developer/generated/adapter/metrics.md +++ b/doc/developer/generated/adapter/metrics.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/metrics.rs -revision: 38a95cefe2 +revision: 4f0805a4d8 --- # adapter::metrics @@ -15,3 +15,4 @@ Several public metrics carry `MetricTag` annotations for categorization: `mz_que Helper functions `session_type_label_value`, `statement_type_label_value`, and `subscribe_output_label_value` produce the label strings used for partitioning these metrics. `Metrics` includes a `subscribe_outputs` `IntCounterVec` (labeled via `subscribe_output_label_value`) counting subscribe output rows; `SessionMetrics` vends per-call counters from it via `Metrics::subscribe_outputs`. `Metrics` includes `active_internal_subscribes: IntGaugeVec` (labeled by `session_type`) tracking the number of active internal subscribes, which serve frontend-sequenced read-then-write operations. Internal subscribes are not reflected in `active_subscribes` or in the `mz_subscriptions` builtin table. +`Metrics` includes `occ_retry_count: Histogram` recording the number of OCC retry attempts made per frontend read-then-write execution before it either succeeds or exhausts its retry budget. diff --git a/doc/developer/generated/adapter/notice.md b/doc/developer/generated/adapter/notice.md index 1d871c2fecab6..507abca1db741 100644 --- a/doc/developer/generated/adapter/notice.md +++ b/doc/developer/generated/adapter/notice.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/notice.rs -revision: a4e7587de2 +revision: 00299a05e3 --- # adapter::notice diff --git a/doc/developer/generated/adapter/optimize/dataflows.md b/doc/developer/generated/adapter/optimize/dataflows.md index 5fea016dcdc52..173241d12b804 100644 --- a/doc/developer/generated/adapter/optimize/dataflows.md +++ b/doc/developer/generated/adapter/optimize/dataflows.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/optimize/dataflows.rs -revision: 34813f0a3f +revision: 39dcae2fba --- # adapter::optimize::dataflows diff --git a/doc/developer/generated/adapter/optimize/subscribe.md b/doc/developer/generated/adapter/optimize/subscribe.md index da15fbc688368..7549d066283fb 100644 --- a/doc/developer/generated/adapter/optimize/subscribe.md +++ b/doc/developer/generated/adapter/optimize/subscribe.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/optimize/subscribe.rs -revision: b9097f8a3d +revision: 4f0805a4d8 --- # adapter::optimize::subscribe diff --git a/doc/developer/generated/adapter/peek_client.md b/doc/developer/generated/adapter/peek_client.md index fc48bcb3a2fb4..764f0cc48b075 100644 --- a/doc/developer/generated/adapter/peek_client.md +++ b/doc/developer/generated/adapter/peek_client.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/peek_client.rs -revision: ff68428395 +revision: 4f0805a4d8 --- # adapter::peek_client diff --git a/doc/developer/generated/adapter/session.md b/doc/developer/generated/adapter/session.md index 6b5a102515ba8..14abd7c1c3e40 100644 --- a/doc/developer/generated/adapter/session.md +++ b/doc/developer/generated/adapter/session.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/session.rs -revision: b5624d20bb +revision: 00299a05e3 --- # adapter::session diff --git a/doc/developer/generated/adapter/statement_logging.md b/doc/developer/generated/adapter/statement_logging.md index 8c57d76cec84d..6fab4745c9d3f 100644 --- a/doc/developer/generated/adapter/statement_logging.md +++ b/doc/developer/generated/adapter/statement_logging.md @@ -1,6 +1,6 @@ --- source: src/adapter/src/statement_logging.rs -revision: 10a94621ed +revision: 39dcae2fba --- # adapter::statement_logging diff --git a/doc/developer/generated/audit-log/_crate.md b/doc/developer/generated/audit-log/_crate.md index d44365baa0234..15526e3ef3096 100644 --- a/doc/developer/generated/audit-log/_crate.md +++ b/doc/developer/generated/audit-log/_crate.md @@ -1,6 +1,6 @@ --- source: src/audit-log/src/lib.rs -revision: 8598d82c1c +revision: 39dcae2fba --- # mz-audit-log @@ -17,7 +17,7 @@ All event types and their detail payloads are defined here and serialized as JSO The crate is a single `lib.rs` with no submodules. It defines two independent versioned hierarchies: -* **Audit events** — `VersionedEvent` (currently `V1`) wraps `EventV1`, which carries an `EventType` (Create/Drop/Alter/Grant/Revoke/Comment), an `ObjectType` (Cluster, ClusterReplica, Connection, ContinualTask, Database, Func, Index, MaterializedView, NetworkPolicy, Role, Secret, Schema, Sink, Source, System, Table, Type, View), and an `EventDetails` variant holding the event-specific payload struct. +* **Audit events** — `VersionedEvent` (currently `V1`) wraps `EventV1`, which carries an `EventType` (Create/Drop/Alter/Grant/Revoke/Comment), an `ObjectType` (Cluster, ClusterReplica, Connection, ContinualTask, Database, Func, Index, MaterializedView, MetricSink, NetworkPolicy, Role, Secret, Schema, Sink, Source, System, Table, Type, View), and an `EventDetails` variant holding the event-specific payload struct. * **Storage usage snapshots** — `VersionedStorageUsage` (currently `V1`) wraps `StorageUsageV1`, recording per-shard byte sizes at a point in time. Each `EventDetails` variant is a versioned struct (e.g. `CreateClusterReplicaV1` through `V4`). diff --git a/doc/developer/generated/avro/schema.md b/doc/developer/generated/avro/schema.md index a011b24271b27..8024591a94995 100644 --- a/doc/developer/generated/avro/schema.md +++ b/doc/developer/generated/avro/schema.md @@ -1,6 +1,6 @@ --- source: src/avro/src/schema.rs -revision: 548ddc2a3f +revision: f0cdedca04 --- Parses JSON Avro schemas into an Arena-based `Schema` struct and performs schema resolution. diff --git a/doc/developer/generated/catalog-debug/_crate.md b/doc/developer/generated/catalog-debug/_crate.md index 39769c921ba0e..41be40a0519fb 100644 --- a/doc/developer/generated/catalog-debug/_crate.md +++ b/doc/developer/generated/catalog-debug/_crate.md @@ -1,6 +1,6 @@ --- source: src/catalog-debug/src/main.rs -revision: 253293ef87 +revision: 39dcae2fba --- # mz-catalog-debug diff --git a/doc/developer/generated/catalog-protos/_crate.md b/doc/developer/generated/catalog-protos/_crate.md index 6f46c4d6961ed..1aad1b8b59757 100644 --- a/doc/developer/generated/catalog-protos/_crate.md +++ b/doc/developer/generated/catalog-protos/_crate.md @@ -1,12 +1,12 @@ --- source: src/catalog-protos/src/lib.rs -revision: f3b4f3f +revision: 39dcae2fba --- # mz-catalog-protos Provides all Rust types durably persisted in the Materialize catalog, along with `RustType` conversion impls bridging those types to protobuf. -The crate exposes the current schema as `objects` (currently v91) plus frozen snapshots `objects_v74` through `objects_v91` used for migrations. -`CATALOG_VERSION` (91) and `MIN_CATALOG_VERSION` (74) constants bound the supported migration range; the build script validates file hashes to prevent accidental mutation of snapshots. +The crate exposes the current schema as `objects` (currently v92) plus frozen snapshots `objects_v74` through `objects_v92` used for migrations. +`CATALOG_VERSION` (92) and `MIN_CATALOG_VERSION` (74) constants bound the supported migration range; the build script validates file hashes to prevent accidental mutation of snapshots. The crate has an optional `proptest` feature; `derive(Arbitrary)` on catalog types is compiled only when the `test` cfg or the `proptest` feature is enabled. Key dependencies are `mz-proto`, `mz-repr`, `mz-sql`, `mz-audit-log`, `mz-compute-types`, and `mz-storage-types`; the primary consumer is `mz-catalog`. diff --git a/doc/developer/generated/catalog-protos/audit_log.md b/doc/developer/generated/catalog-protos/audit_log.md index a40886e0ba1ac..d8c11d5f5b4b1 100644 --- a/doc/developer/generated/catalog-protos/audit_log.md +++ b/doc/developer/generated/catalog-protos/audit_log.md @@ -1,6 +1,6 @@ --- source: src/catalog-protos/src/audit_log.rs -revision: 8598d82c1c +revision: 39dcae2fba --- # mz-catalog-protos::audit_log diff --git a/doc/developer/generated/catalog-protos/objects.md b/doc/developer/generated/catalog-protos/objects.md index df03321463157..d201916c6d072 100644 --- a/doc/developer/generated/catalog-protos/objects.md +++ b/doc/developer/generated/catalog-protos/objects.md @@ -1,11 +1,11 @@ --- source: src/catalog-protos/src/objects.rs -revision: f3b4f3f +revision: 39dcae2fba --- # mz-catalog-protos::objects -Defines the current (v91) set of Rust structs and enums that represent all durably persisted catalog objects, generated from protobuf definitions. +Defines the current (v92) set of Rust structs and enums that represent all durably persisted catalog objects, generated from protobuf definitions. This file is the canonical snapshot of the current catalog schema; `objects_v.rs` files are frozen snapshots used as migration sources. Key types include `ConfigKey`, `ConfigValue`, `SettingKey`, `SettingValue`, `IdAllocKey`, `RoleId`, `DatabaseId`, `SchemaId`, `AutoProvisionSource`, `RoleAttributes`, and many more covering every catalog entity. Scoped system configuration is represented by `ClusterSystemConfigurationKey`/`ClusterSystemConfigurationValue` (keyed by `ClusterId` and parameter name) and `ReplicaSystemConfigurationKey`/`ReplicaSystemConfigurationValue` (keyed by `ReplicaId` and parameter name), along with their corresponding `ClusterSystemConfiguration` and `ReplicaSystemConfiguration` wrapper structs and variants in `StateUpdateKind`. The `ManagedCluster` struct carries durable cluster autoscaling state: `auto_scaling_strategy` (`Option`), `reconfiguration` (`Option`), and `burst` (`Option`). `AutoScalingStrategy` holds an optional `OnHydration` sub-policy. `ReconfigurationState` records an in-flight graceful reconfiguration with a `ReconfigurationTarget`, deadline, and `OnTimeoutAction`. `BurstState` records an active hydration burst. The managed `ReplicaLocation` (`ManagedLocation`) stores `availability_zones` as a list rather than a single optional zone. `ItemValue` carries an `ephemeral_owner_session: Option` field; `Some(uuid)` marks a temporary item owned by and only visible to the session with that UUID, while `None` denotes a normal durable item. The `audit_log_event_v1` submodule includes `CreateRoleV1`, `AlterAddColumnV1`, `AlterSourceTimestampIntervalV1`, `AlterClusterReconfigurationV1` (with `ReconfigurationLifecycleV1` variants `Started`, `Finalized`, `TimedOut`, `Cancelled`, `ResourceExhausted` and `ClusterReplicaLoggingV1`), and `ClusterHydrationBurstV1` (with `HydrationBurstLifecycleV1` and `BurstFinishCauseV1` variants `LingerElapsed`, `NoLongerWarranted`) for audit-logging corresponding catalog events. `CreateOrDropClusterReplicaReasonV1` carries reasons `Manual`, `Schedule`, `System`, `Reconfiguration`, `HydrationBurst`, and `Retired`. diff --git a/doc/developer/generated/catalog-protos/serialization.md b/doc/developer/generated/catalog-protos/serialization.md index 6815a1221b088..f1ec2f6fe84aa 100644 --- a/doc/developer/generated/catalog-protos/serialization.md +++ b/doc/developer/generated/catalog-protos/serialization.md @@ -1,6 +1,6 @@ --- source: src/catalog-protos/src/serialization.rs -revision: 80f8711523 +revision: 39dcae2fba --- # mz-catalog-protos::serialization diff --git a/doc/developer/generated/catalog/builtin/mz_catalog.md b/doc/developer/generated/catalog/builtin/mz_catalog.md index 5f02712bf230f..5b47bbf9b6661 100644 --- a/doc/developer/generated/catalog/builtin/mz_catalog.md +++ b/doc/developer/generated/catalog/builtin/mz_catalog.md @@ -1,6 +1,6 @@ --- source: src/catalog/src/builtin/mz_catalog.rs -revision: 57107078d +revision: 39dcae2fba --- # catalog::builtin::mz_catalog @@ -13,7 +13,7 @@ This module exports 76 public items: types, tables, materialized views, views, s **Tables** — `BuiltinTable` statics covering type system metadata, operator and function definitions, column and type details, and cluster sizing information. -**Materialized views** — Core catalog entities backed by queries over `mz_internal.mz_catalog_raw`: `MZ_DATABASES`, `MZ_SCHEMAS`, `MZ_CONNECTIONS`, `MZ_SECRETS`, `MZ_TABLES`, `MZ_COLUMNS`, `MZ_VIEWS`, `MZ_SOURCES`, `MZ_SINKS`, `MZ_CLUSTERS`, `MZ_CLUSTER_REPLICAS`, `MZ_ROLES`, `MZ_ROLE_PARAMETERS`, `MZ_OBJECTS`, `MZ_ALL_OBJECTS`, `MZ_DEFAULT_PRIVILEGES`, `MZ_SYSTEM_PRIVILEGES`, `MZ_AUDIT_EVENTS`, `MZ_KAFKA_SOURCES`, `MZ_KAFKA_SINKS`, `MZ_ICEBERG_SINKS`, and others. `MZ_KAFKA_CONNECTIONS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_KAFKA_CONNECTIONS_OID`) that reads `Item` rows from `mz_catalog_raw`, pulls broker addresses and the progress topic out of the persisted `create_sql` via `parse_connection_details`, and keeps only kafka connections. The default progress topic is reconstructed in SQL to match `KafkaConnection::progress_topic` in the storage types crate. `MZ_SSH_TUNNEL_CONNECTIONS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_SSH_TUNNEL_CONNECTIONS_OID`) that derives SSH tunnel connection details (public keys) by parsing `create_sql` via `parse_connection_details`. `MZ_AWS_PRIVATELINK_CONNECTIONS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_AWS_PRIVATELINK_CONNECTIONS_OID`) that reconstructs the principal ARN from the environment-context functions `mz_aws_account_id()` and `mz_aws_external_id_prefix()`; the ARN format mirrors `AwsPrincipalContext::to_principal_string`. It keeps a `WHERE principal IS NOT NULL` guard so no row is emitted when the AWS context is absent, matching the packer it replaces. `MZ_KAFKA_SOURCES` is a `BuiltinMaterializedView` (OID constant `MV_MZ_KAFKA_SOURCES_OID`) that derives Kafka source details (topic and computed group ID) by parsing the raw catalog create SQL via `mz_internal.parse_kafka_source_details`; the `group_id_prefix` column holds the full computed `group.id`, not just the user-supplied prefix. `MZ_KAFKA_SINKS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_KAFKA_SINKS_OID`) that derives the Kafka sink topic by parsing `create_sql` via `mz_internal.parse_catalog_create_sql`. `MZ_ICEBERG_SINKS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_ICEBERG_SINKS_OID`) that derives the Iceberg sink namespace and table name from `create_sql` via `mz_internal.parse_catalog_create_sql`. `MZ_SINKS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_SINKS_OID`) that derives all sink metadata from the persisted `create_sql` via `mz_internal.parse_catalog_create_sql`; the `size` column is deprecated and always NULL. `MZ_AUDIT_EVENTS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_AUDIT_EVENTS_OID`) whose `event_type` column covers `create`, `drop`, `alter`, `grant`, `revoke`, and `comment`, and whose `object_type` column covers `cluster`, `cluster-replica`, `connection`, `database`, `func`, `index`, `materialized-view`, `role`, `secret`, `schema`, `sink`, `source`, `table`, `type`, `view`, `system`, `continual-task`, and `network-policy`. `MZ_CLUSTERS` and `MZ_CLUSTER_REPLICAS` are `BuiltinMaterializedView` objects that derive cluster and replica metadata from the durable catalog raw source; `MZ_CLUSTER_REPLICAS` resolves the `disk` column via a LEFT JOIN against `mz_cluster_replica_size_internal` (which retains rows for disabled sizes), so replicas with unknown sizes return `NULL` for `disk` rather than causing an error. `MZ_DEFAULT_PRIVILEGES` decodes `DefaultPrivileges` rows from `mz_catalog_raw`, mapping the proto enum numeric `object_type` to its SQL display name via a CASE expression. `MZ_SYSTEM_PRIVILEGES` decodes `SystemPrivileges` rows and unnests them into `mz_aclitem` values. These carry `Ontology` annotations with entity names, descriptions, and `OntologyLink` relationships for the catalog graph. `MZ_INDEXES` is also a `BuiltinMaterializedView` but is generated dynamically via the `pub(super) make_mz_indexes` function rather than declared as a static, so it is not counted among the module's exported statics. +**Materialized views** — Core catalog entities backed by queries over `mz_internal.mz_catalog_raw`: `MZ_DATABASES`, `MZ_SCHEMAS`, `MZ_CONNECTIONS`, `MZ_SECRETS`, `MZ_TABLES`, `MZ_COLUMNS`, `MZ_VIEWS`, `MZ_SOURCES`, `MZ_SINKS`, `MZ_CLUSTERS`, `MZ_CLUSTER_REPLICAS`, `MZ_ROLES`, `MZ_ROLE_PARAMETERS`, `MZ_OBJECTS`, `MZ_ALL_OBJECTS`, `MZ_DEFAULT_PRIVILEGES`, `MZ_SYSTEM_PRIVILEGES`, `MZ_AUDIT_EVENTS`, `MZ_KAFKA_SOURCES`, `MZ_KAFKA_SINKS`, `MZ_ICEBERG_SINKS`, and others. `MZ_KAFKA_CONNECTIONS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_KAFKA_CONNECTIONS_OID`) that reads `Item` rows from `mz_catalog_raw`, pulls broker addresses and the progress topic out of the persisted `create_sql` via `parse_connection_details`, and keeps only kafka connections. The default progress topic is reconstructed in SQL to match `KafkaConnection::progress_topic` in the storage types crate. `MZ_SSH_TUNNEL_CONNECTIONS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_SSH_TUNNEL_CONNECTIONS_OID`) that derives SSH tunnel connection details (public keys) by parsing `create_sql` via `parse_connection_details`. `MZ_AWS_PRIVATELINK_CONNECTIONS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_AWS_PRIVATELINK_CONNECTIONS_OID`) that reconstructs the principal ARN from the environment-context functions `mz_aws_account_id()` and `mz_aws_external_id_prefix()`; the ARN format mirrors `AwsPrincipalContext::to_principal_string`. It keeps a `WHERE principal IS NOT NULL` guard so no row is emitted when the AWS context is absent, matching the packer it replaces. `MZ_KAFKA_SOURCES` is a `BuiltinMaterializedView` (OID constant `MV_MZ_KAFKA_SOURCES_OID`) that derives Kafka source details (topic and computed group ID) by parsing the raw catalog create SQL via `mz_internal.parse_kafka_source_details`; the `group_id_prefix` column holds the full computed `group.id`, not just the user-supplied prefix. `MZ_KAFKA_SINKS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_KAFKA_SINKS_OID`) that derives the Kafka sink topic by parsing `create_sql` via `mz_internal.parse_catalog_create_sql`. `MZ_ICEBERG_SINKS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_ICEBERG_SINKS_OID`) that derives the Iceberg sink namespace and table name from `create_sql` via `mz_internal.parse_catalog_create_sql`. `MZ_SINKS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_SINKS_OID`) that derives all sink metadata from the persisted `create_sql` via `mz_internal.parse_catalog_create_sql`; the `size` column is deprecated and always NULL. `MZ_AUDIT_EVENTS` is a `BuiltinMaterializedView` (OID constant `MV_MZ_AUDIT_EVENTS_OID`) whose `event_type` column covers `create`, `drop`, `alter`, `grant`, `revoke`, and `comment`, and whose `object_type` column covers `cluster`, `cluster-replica`, `connection`, `database`, `func`, `index`, `materialized-view`, `metric-sink`, `network-policy`, `role`, `secret`, `schema`, `sink`, `source`, `system`, `table`, `type`, `view`, and `continual-task`. `MZ_CLUSTERS` and `MZ_CLUSTER_REPLICAS` are `BuiltinMaterializedView` objects that derive cluster and replica metadata from the durable catalog raw source; `MZ_CLUSTER_REPLICAS` resolves the `disk` column via a LEFT JOIN against `mz_cluster_replica_size_internal` (which retains rows for disabled sizes), so replicas with unknown sizes return `NULL` for `disk` rather than causing an error. `MZ_DEFAULT_PRIVILEGES` decodes `DefaultPrivileges` rows from `mz_catalog_raw`, mapping the proto enum numeric `object_type` to its SQL display name via a CASE expression. `MZ_SYSTEM_PRIVILEGES` decodes `SystemPrivileges` rows and unnests them into `mz_aclitem` values. These carry `Ontology` annotations with entity names, descriptions, and `OntologyLink` relationships for the catalog graph. `MZ_INDEXES` is also a `BuiltinMaterializedView` but is generated dynamically via the `pub(super) make_mz_indexes` function rather than declared as a static, so it is not counted among the module's exported statics. **Views** — Additional SQL views for derived catalog information such as `MZ_TIMEZONE_ABBREVIATIONS` and `MZ_TIMEZONE_NAMES`. diff --git a/doc/developer/generated/catalog/builtin/mz_internal.md b/doc/developer/generated/catalog/builtin/mz_internal.md index 7d6451ffa205f..7d43f6e926975 100644 --- a/doc/developer/generated/catalog/builtin/mz_internal.md +++ b/doc/developer/generated/catalog/builtin/mz_internal.md @@ -1,6 +1,6 @@ --- source: src/catalog/src/builtin/mz_internal.rs -revision: fd1dd6e62b +revision: e3fe017dcb --- # catalog::builtin::mz_internal diff --git a/doc/developer/generated/catalog/durable/initialize.md b/doc/developer/generated/catalog/durable/initialize.md index 9eae6e45d457c..7909d37e99ea7 100644 --- a/doc/developer/generated/catalog/durable/initialize.md +++ b/doc/developer/generated/catalog/durable/initialize.md @@ -1,6 +1,6 @@ --- source: src/catalog/src/durable/initialize.rs -revision: fca741734d +revision: 39dcae2fba --- # catalog::durable::initialize diff --git a/doc/developer/generated/catalog/durable/objects/_module.md b/doc/developer/generated/catalog/durable/objects/_module.md index 7e3d86e792b31..4ead9184be04f 100644 --- a/doc/developer/generated/catalog/durable/objects/_module.md +++ b/doc/developer/generated/catalog/durable/objects/_module.md @@ -1,6 +1,6 @@ --- source: src/catalog/src/durable/objects.rs -revision: f3b4f3f +revision: 39dcae2fba --- # catalog::durable::objects diff --git a/doc/developer/generated/catalog/durable/upgrade/_module.md b/doc/developer/generated/catalog/durable/upgrade/_module.md index f4b022fa97a43..8603fc4525727 100644 --- a/doc/developer/generated/catalog/durable/upgrade/_module.md +++ b/doc/developer/generated/catalog/durable/upgrade/_module.md @@ -1,12 +1,12 @@ --- source: src/catalog/src/durable/upgrade.rs -revision: f3b4f3f +revision: 39dcae2fba --- # catalog::durable::upgrade Orchestrates catalog schema migrations by replaying version-specific upgrade functions over a sequence of protobuf snapshots. -`CATALOG_VERSION` is the current schema version (91); `run_upgrade` chains individual `v{N}_to_v{N+1}` functions (v74 through v91) until the stored data matches the current version. +`CATALOG_VERSION` is the current schema version (92); `run_upgrade` chains individual `v{N}_to_v{N+1}` functions (v74 through v92) until the stored data matches the current version. Each version-specific submodule operates entirely on frozen `objects_v{N}` types, ensuring that future code changes cannot retroactively break old migrations. The `objects!` macro generates per-version support code, handling both old protobuf-based snapshots (v74--v78) and newer serde-based snapshots (v79+). The `json_compatible` submodule provides helpers for reading old JSON-encoded state when proto types have changed. @@ -19,3 +19,4 @@ The v87→v88 migration runs through `run_versioned_upgrade` and is a no-op: new The v88→v89 migration runs through `run_versioned_upgrade` and adds `ReconfigurationState::status`, backfilling any in-flight reconfiguration records as `InProgress`. All other v88→v89 changes (new `ReconfigurationLifecycleV1::ResourceExhausted` variant, `ClusterReplicaLoggingV1`, `BurstFinishCauseV1`, updated `AlterClusterReconfigurationV1` and `ClusterHydrationBurstV1` audit event fields) are additive and confined to the append-only audit log; no existing record changes shape beyond the reconfiguration status backfill. The v89→v90 migration runs through `run_versioned_upgrade` and adds the `arrangement_compression` flag to managed clusters and cluster replicas, backfilling it as `false`. Managed `Cluster` and `ClusterReplica` records are rewritten to include the new field; unmanaged clusters and all other records pass through unchanged. The flag also appears on a managed cluster's in-flight `reconfiguration` target, backfilled the same way. The v90→v91 migration runs through `run_versioned_upgrade` and adds `ephemeral_owner_session: Option` to `ItemValue`, backfilling it as `None` for every existing `Item` record. All other record types are unchanged. The explicit rewrite is required so that future retractions (which encode with the new field) match the stored records; without it, retraction and record would differ and leave negative multiplicity. +The v91→v92 migration runs through `run_versioned_upgrade` and is a no-op: it adds a `MetricSink` variant to four existing enums (`CatalogItemType`, `ObjectType`, `CommentObject`, and the audit log's `ObjectType`). No record stored under v91 can reference a variant that did not exist at that time, so every v91 record is already valid v92. Metric sinks are stored as ordinary `Item` records; `durable::objects::item_type` identifies the type by parsing `create_sql`. diff --git a/doc/developer/generated/catalog/memory/objects.md b/doc/developer/generated/catalog/memory/objects.md index edaa5c86cbd4f..ce4b755c6339b 100644 --- a/doc/developer/generated/catalog/memory/objects.md +++ b/doc/developer/generated/catalog/memory/objects.md @@ -1,15 +1,15 @@ --- source: src/catalog/src/memory/objects.rs -revision: f3b4f3f +revision: 39dcae2fba --- # catalog::memory::objects Defines all in-memory catalog object types, which are richer and more consumer-friendly than the durable `objects` types. -Key types include `CatalogItem` (an enum over Table, Source, Log, View, MaterializedView, Sink, Index, Type, Func, Secret, Connection), `CatalogEntry` (pairs a `CatalogItem` with its metadata), `Database`, `Schema`, `Role`, `Cluster`, `ClusterReplica`, `NetworkPolicy`, `ReconfigurationState`, and `BurstState`. +Key types include `CatalogItem` (an enum over Table, Source, Log, View, MaterializedView, Sink, Index, Type, Func, Secret, Connection, MetricSink), `CatalogEntry` (pairs a `CatalogItem` with its metadata), `Database`, `Schema`, `Role`, `Cluster`, `ClusterReplica`, `NetworkPolicy`, `ReconfigurationState`, and `BurstState`. `View` and `MaterializedView` store their locally optimized MIR expression in `locally_optimized_expr`. `Index` and `MaterializedView` additionally carry `optimized_plan` (global MIR dataflow), `physical_plan` (`DataflowDescription`, i.e. the LIR dataflow), and `dataflow_metainfo` (optimizer notices) fields, all `#[serde(skip)]` since they are populated after catalog replay and are not part of the durable representation. -`CatalogItem` exposes `optimized_plan()`, `physical_plan()`, and `dataflow_metainfo()` accessors for the plan fields, and `plan_fields_mut()` for mutable access to all three plan fields at once on plan-bearing items (`Index`, `MaterializedView`); it returns `None` for other item kinds. `CatalogItem::set_create_sql` overwrites the `create_sql` of a view, index, or table without replanning, used during temporary item updates to keep the byte-identical `create_sql` for retraction consolidation; it panics for item kinds that cannot be temporary. `CatalogItem::update_timestamp_interval` updates the timestamp interval for a source and returns the previous `WithOptionValue` if one was set, or `None`; it returns `Err(())` if called on a non-source item. `CatalogItem::is_hydratable` returns `true` for items that run a dataflow on cluster replicas and therefore have hydration state: indexes, materialized views, sinks, and ingestion sources. Non-ingestion sources (webhooks, ingestion exports) are excluded. `CatalogEntry` exposes `item_mut()` for in-place mutation. +`CatalogItem` exposes `optimized_plan()`, `physical_plan()`, and `dataflow_metainfo()` accessors for the plan fields, and `plan_fields_mut()` for mutable access to all three plan fields at once on plan-bearing items (`Index`, `MaterializedView`, `MetricSink`); it returns `None` for other item kinds. `CatalogItem::set_create_sql` overwrites the `create_sql` of a view, index, or table without replanning, used during temporary item updates to keep the byte-identical `create_sql` for retraction consolidation; it panics for item kinds that cannot be temporary. `CatalogItem::update_timestamp_interval` updates the timestamp interval for a source and returns the previous `WithOptionValue` if one was set, or `None`; it returns `Err(())` if called on a non-source item. `CatalogItem::is_hydratable` returns `true` for items that run a dataflow on cluster replicas and therefore have hydration state: indexes, materialized views, sinks, and ingestion sources. Non-ingestion sources (webhooks, ingestion exports) are excluded. `CatalogEntry` exposes `item_mut()` for in-place mutation. `ClusterVariantManaged` carries `auto_scaling_strategy` (`Option`), `reconfiguration` (`Option`), and `burst` (`Option`) in addition to the base cluster shape fields. `ClusterVariantManaged::has_unwarranted_burst_record` returns `true` when the in-flight `burst` record is no longer consistent with the current config: the `ON HYDRATION` policy was removed or re-sized, or `replication_factor` was set to 0. `ReconfigurationState` is the in-memory mirror of the durable type, recording the target config, deadline, `on_timeout` action, and `status` (`ReconfigurationStatus`: `InProgress`, `Finalized`, `TimedOut`, `Cancelled`, `ResourceExhausted`) for a graceful reconfiguration. `BurstState` is the in-memory mirror of the durable burst-state record, carrying `burst_size`, `linger_duration`, and an optional `steady_hydrated_at` timestamp. `Cluster::try_to_plan` rebuilds the `CreateClusterPlan` that the original `CREATE CLUSTER` statement would have produced. Clusters are stored as a structured config rather than SQL text, so `SHOW CREATE CLUSTER` uses this method as the first half of the config-to-statement roundtrip; `unplan_create_cluster` then converts the plan back into a statement. The rebuilt plan always sets `if_not_exists: false` because `IF NOT EXISTS` is a property of the original statement, not of the cluster itself. `ClusterReplicaProcessStatus` records the process status, a cumulative `restart_count` mirrored from the orchestrator, and the time of the most recent change to either field. @@ -17,5 +17,6 @@ Key types include `CatalogItem` (an enum over Table, Source, Log, View, Material `StateUpdate` and `StateUpdateKind` represent in-memory deltas applied during catalog replay and incremental updates. `StateUpdateKind` includes `ClusterSystemConfiguration` and `ReplicaSystemConfiguration` variants carrying the corresponding durable objects. Temporary items are represented as ordinary `Item` variants in `StateUpdateKind`, identified by `Item::ephemeral_owner_session`; there is no separate `TemporaryItem` variant. `DataSourceDesc` and `DataSource` describe how a source or table obtains its data (ingestion, introspection, webhook, progress, etc.). `Sink::envelope` returns `Some("append")` for the `SinkEnvelope::Append` variant in addition to `Some("debezium")` and `Some("upsert")`. +`MetricSink` represents a metric sink catalog item: a relation exported as Prometheus metrics, bound to a cluster. It carries `create_sql`, `global_id`, `cluster_id`, `resolved_ids`, and optional plan fields (`optimized_plan`, `physical_plan`, `dataflow_metainfo`) populated after catalog replay; the plan fields are `#[serde(skip)]`. `MetricSink` is a compute object on its cluster (`is_compute_object_on_cluster` returns `Some(cluster_id)`) but reports `is_hydratable` as `false` until per-replica dataflows are implemented. `CatalogEntry::is_metric_sink` returns `true` for entries holding a `MetricSink` item. `plan_fields_mut` exposes all three plan fields for mutable access, consistent with `Index` and `MaterializedView`. `Cluster::auto_scaling_strategy` (implementing `CatalogCluster`) returns the `AutoScalingStrategy` from a managed cluster's config, or `None` for unmanaged clusters. The `UpdateFrom` trait drives the incremental update pattern throughout this module. diff --git a/doc/developer/generated/cluster-controller/ctx.md b/doc/developer/generated/cluster-controller/ctx.md index 1e76c6cba5e83..03e923e695cea 100644 --- a/doc/developer/generated/cluster-controller/ctx.md +++ b/doc/developer/generated/cluster-controller/ctx.md @@ -1,6 +1,6 @@ --- source: src/cluster-controller/src/ctx.rs -revision: 767b86caef +revision: e3fe017dcb --- # cluster-controller::ctx diff --git a/doc/developer/generated/clusterd-test-driver/script.md b/doc/developer/generated/clusterd-test-driver/script.md index e75b212f1b783..2adafcd97cc0d 100644 --- a/doc/developer/generated/clusterd-test-driver/script.md +++ b/doc/developer/generated/clusterd-test-driver/script.md @@ -1,6 +1,6 @@ --- source: src/clusterd-test-driver/src/script.rs -revision: 141cb2a0a5 +revision: cd52678037 --- # mz-clusterd-test-driver::script diff --git a/doc/developer/generated/clusterd-test-driver/text.md b/doc/developer/generated/clusterd-test-driver/text.md index e2c1ee28b1728..c05f74c7aa9fb 100644 --- a/doc/developer/generated/clusterd-test-driver/text.md +++ b/doc/developer/generated/clusterd-test-driver/text.md @@ -1,6 +1,6 @@ --- source: src/clusterd-test-driver/src/text.rs -revision: 141cb2a0a5 +revision: cd52678037 --- # mz-clusterd-test-driver::text diff --git a/doc/developer/generated/compute-client/controller/instance.md b/doc/developer/generated/compute-client/controller/instance.md index 99d65d611a261..8c42a9517ae6f 100644 --- a/doc/developer/generated/compute-client/controller/instance.md +++ b/doc/developer/generated/compute-client/controller/instance.md @@ -1,6 +1,6 @@ --- source: src/compute-client/src/controller/instance.rs -revision: 94054eb165 +revision: 59ca7517a9 --- # mz-compute-client::controller::instance @@ -10,4 +10,5 @@ It drives the compute protocol by sending commands to replicas, processing their Key internal types include `CollectionState` (per-collection frontier and read-hold tracking), `ReplicaState` (per-replica client and metrics), `PendingPeek`, and `ActiveSubscribe`. Error types `ReplicaExists`, `ReplicaMissing`, `DataflowCreationError`, and `ReadPolicyError` cover the various failure modes. Storage metadata resolution for `MaterializedViewSinkConnection` is performed during dataflow creation; subscribe and copy-to sinks pass their connection data through unmodified. `Instance` holds a `replica_dyncfg_overrides: BTreeMap` field (empty by default) that stores per-replica dyncfg overrides from the scoped feature flags layer. The `send` method applies overrides via `specialize_command_for_replica`, which for `UpdateConfiguration` merges the replica's `ConfigUpdates` into the update, and for `CreateInstance` captures the current instance-wide dyncfg with the override applied on top as `initial_config`. The base (un-specialized) command is recorded in history so specialization is re-applied at replay time in `add_replica`. `update_replica_dyncfg_overrides` replaces the overrides map. `add_replica_state` returns `Err(ReadHoldIssuerHungUp)` when storage read hold issuers have closed during process shutdown; the replica is still inserted to keep bookkeeping consistent with the controller, and `initiate_shutdown` is called to terminate the `Instance::run` loop by replacing `command_rx` with a sender-less channel. Unlike `shutdown`, `initiate_shutdown` does not assert that no replicas remain. +`collections_hydrated_on_replicas` collects all non-hydrated collections before returning, so that when it logs at INFO it names every collection the caller is waiting on in a single log line rather than one line per collection. `InstanceClient` (in `instance_client`) provides the externally-visible interface, while this module houses the core state machine running as an async task. diff --git a/doc/developer/generated/compute/logging/differential.md b/doc/developer/generated/compute/logging/differential.md index 25ba981edc7c5..1d8af29026223 100644 --- a/doc/developer/generated/compute/logging/differential.md +++ b/doc/developer/generated/compute/logging/differential.md @@ -1,6 +1,6 @@ --- source: src/compute/src/logging/differential.rs -revision: 225aeaa79f +revision: f0cdedca04 --- # mz-compute::logging::differential diff --git a/doc/developer/generated/compute/render/reduce.md b/doc/developer/generated/compute/render/reduce.md index 839fa66d5082a..ba5b3f75a9f6f 100644 --- a/doc/developer/generated/compute/render/reduce.md +++ b/doc/developer/generated/compute/render/reduce.md @@ -1,6 +1,6 @@ --- source: src/compute/src/render/reduce.rs -revision: 1c55de49eb +revision: f0cdedca04 --- # mz-compute::render::reduce diff --git a/doc/developer/generated/compute/render/top_k.md b/doc/developer/generated/compute/render/top_k.md index b5e83a970f2b6..3693f776c0ef4 100644 --- a/doc/developer/generated/compute/render/top_k.md +++ b/doc/developer/generated/compute/render/top_k.md @@ -1,6 +1,6 @@ --- source: src/compute/src/render/top_k.rs -revision: 1c55de49eb +revision: f0cdedca04 --- # mz-compute::render::top_k diff --git a/doc/developer/generated/compute/sink/correction.md b/doc/developer/generated/compute/sink/correction.md index 98e87c42cbf98..3cf755b8879b2 100644 --- a/doc/developer/generated/compute/sink/correction.md +++ b/doc/developer/generated/compute/sink/correction.md @@ -1,6 +1,6 @@ --- source: src/compute/src/sink/correction.rs -revision: f9be03fda1 +revision: f0cdedca04 --- # mz-compute::sink::correction diff --git a/doc/developer/generated/compute/sink/correction_v2.md b/doc/developer/generated/compute/sink/correction_v2.md index 862add3d74d72..056e9c5740afe 100644 --- a/doc/developer/generated/compute/sink/correction_v2.md +++ b/doc/developer/generated/compute/sink/correction_v2.md @@ -1,6 +1,6 @@ --- source: src/compute/src/sink/correction_v2.rs -revision: c96d4ef054 +revision: f0cdedca04 --- # mz-compute::sink::correction_v2 diff --git a/doc/developer/generated/environmentd/_crate.md b/doc/developer/generated/environmentd/_crate.md index cbbd33954fa78..66d665cde59d2 100644 --- a/doc/developer/generated/environmentd/_crate.md +++ b/doc/developer/generated/environmentd/_crate.md @@ -1,6 +1,6 @@ --- source: src/environmentd/src/lib.rs -revision: d92a868ee9 +revision: 84f88ca968 --- # environmentd diff --git a/doc/developer/generated/environmentd/http/sql.md b/doc/developer/generated/environmentd/http/sql.md index 2acf319446102..53d23ad81a164 100644 --- a/doc/developer/generated/environmentd/http/sql.md +++ b/doc/developer/generated/environmentd/http/sql.md @@ -1,6 +1,6 @@ --- source: src/environmentd/src/http/sql.rs -revision: e4df9977da +revision: 39dcae2fba --- # environmentd::http::sql diff --git a/doc/developer/generated/environmentd/test_util.md b/doc/developer/generated/environmentd/test_util.md index 2c24b6d8c9423..ec68ea2cfb652 100644 --- a/doc/developer/generated/environmentd/test_util.md +++ b/doc/developer/generated/environmentd/test_util.md @@ -1,6 +1,6 @@ --- source: src/environmentd/src/test_util.rs -revision: fd1dd6e62b +revision: 4f0805a4d8 --- # environmentd::test_util diff --git a/doc/developer/generated/expr/interpret.md b/doc/developer/generated/expr/interpret.md index d49697c3177ba..b0c5b0490d0cc 100644 --- a/doc/developer/generated/expr/interpret.md +++ b/doc/developer/generated/expr/interpret.md @@ -1,6 +1,6 @@ --- source: src/expr/src/interpret.rs -revision: 32c1e1d39 +revision: 95baa04a85 --- # mz-expr::interpret diff --git a/doc/developer/generated/expr/scalar/func/_module.md b/doc/developer/generated/expr/scalar/func/_module.md index 139ac03de2830..b1a075b5fd3a1 100644 --- a/doc/developer/generated/expr/scalar/func/_module.md +++ b/doc/developer/generated/expr/scalar/func/_module.md @@ -1,6 +1,6 @@ --- source: src/expr/src/scalar/func.rs -revision: f4650d00b +revision: 32c1e1d399 --- # mz-expr::scalar::func diff --git a/doc/developer/generated/expr/scalar/func/impls/jsonb.md b/doc/developer/generated/expr/scalar/func/impls/jsonb.md index ac141378d2f4d..c6f6eec2ea254 100644 --- a/doc/developer/generated/expr/scalar/func/impls/jsonb.md +++ b/doc/developer/generated/expr/scalar/func/impls/jsonb.md @@ -1,9 +1,10 @@ --- source: src/expr/src/scalar/func/impls/jsonb.rs -revision: f4650d00b +revision: 39dcae2fba --- # mz-expr::scalar::func::impls::jsonb Provides scalar function implementations for `jsonb` datums: subscript operators, `jsonb_array_elements`, `jsonb_each`, `jsonb_object_keys`, type-coercion casts, `jsonb_build_object`, `jsonb_build_array`, `jsonb_pretty`, and containment checks. -Also provides `parse_catalog_id` (converts catalog JSON-serialized IDs into string format like `u1`, `s2`, `p`), `parse_catalog_privileges` (converts catalog JSON-serialized privilege arrays into `mz_aclitem[]`), `parse_catalog_acl_mode` (converts a catalog JSON-serialized `AclMode` bitflags object of shape `{"bitflags": }` into a PostgreSQL ACL char-code string, e.g. `{"bitflags": 514}` → `"ar"`), `parse_catalog_create_sql` (parses a catalog `create_sql` string into a JSONB object with type and additional fields depending on the statement: for connections, a `connection_type` field; for materialized views, `cluster_id` and `definition`; for sources (`CreateSource`), `source_type`, `cluster_id`, `connection_id`, `envelope_type`, `key_format`, and `value_format`; for webhook sources, `source_type` and `cluster_id`; for subsources, `source_type` and `of_source_id`), `parse_postgres_source_details` (decodes a Postgres source `create_sql` to extract `slot` and `timeline_id` from the hex-encoded `DETAILS` proto option, returning `{"slot": , "timeline_id": }`; returns JSON `null` for non-Postgres-source statements), and `parse_kafka_source_details` (extracts `topic`, `group_id_prefix`, and `connection_id` from a Kafka source `create_sql`, returning `{"topic": , "group_id_prefix": , "connection_id": }`; returns JSON `null` for non-Kafka-source statements), and `parse_source_export_details` (extracts source-export metadata from a `CREATE TABLE ... FROM SOURCE` or non-progress `CREATE SUBSOURCE ... OF SOURCE ...` statement, returning `{"source_id": , "external_reference": [, ...], "envelope_type": , "key_format": , "value_format": }`; `envelope_type`, `key_format`, and `value_format` are always null for `CREATE SUBSOURCE` statements; returns JSON `null` for progress subsources and any non-source-export statement). Helper functions include `jsonb_datum_to_u64`, `jsonb_datum_to_acl_mode` (shared decoder for `AclMode` objects used by both `parse_catalog_privileges` and `parse_catalog_acl_mode`), and `jsonb_datum_to_role_id`. The `PostgresPublicationDetailsSubset` struct is a local prost message that re-declares the two proto tags needed (`slot` at tag 2, `timeline_id` at tag 3) to avoid a dependency cycle with `mz-storage-types`. +Also provides `parse_catalog_id` (converts catalog JSON-serialized IDs into string format like `u1`, `s2`, `p`), `parse_catalog_privileges` (converts catalog JSON-serialized privilege arrays into `mz_aclitem[]`), `parse_catalog_acl_mode` (converts a catalog JSON-serialized `AclMode` bitflags object of shape `{"bitflags": }` into a PostgreSQL ACL char-code string, e.g. `{"bitflags": 514}` → `"ar"`), `parse_catalog_create_sql` (parses a catalog `create_sql` string into a JSONB object with type and additional fields depending on the statement: for connections, a `connection_type` field; for materialized views, `cluster_id` and `definition`; for sources (`CreateSource`), `source_type`, `cluster_id`, `connection_id`, `envelope_type`, `key_format`, and `value_format`; for webhook sources, `source_type` and `cluster_id`; for subsources, `source_type` and `of_source_id`; for metric sinks (`CreateMetricSink`), `cluster_id` and `from_id`), `parse_postgres_source_details` (decodes a Postgres source `create_sql` to extract `slot` and `timeline_id` from the hex-encoded `DETAILS` proto option, returning `{"slot": , "timeline_id": }`; returns JSON `null` for non-Postgres-source statements), and `parse_kafka_source_details` (extracts `topic`, `group_id_prefix`, and `connection_id` from a Kafka source `create_sql`, returning `{"topic": , "group_id_prefix": , "connection_id": }`; returns JSON `null` for non-Kafka-source statements), and `parse_source_export_details` (extracts source-export metadata from a `CREATE TABLE ... FROM SOURCE` or non-progress `CREATE SUBSOURCE ... OF SOURCE ...` statement, returning `{"source_id": , "external_reference": [, ...], "envelope_type": , "key_format": , "value_format": }`; `envelope_type`, `key_format`, and `value_format` are always null for `CREATE SUBSOURCE` statements; returns JSON `null` for progress subsources and any non-source-export statement). Also provides `parse_connection_details` (extracts connection metadata from a `CREATE CONNECTION` `create_sql` string; for Kafka connections returns `{"brokers": [...], "progress_topic": }`; for SSH connections returns `{"public_key_1": , "public_key_2": }`; for AWS connections returns `{"auth_kind": "assume-role"|"credentials", "endpoint": , "region": , "access_key_id": , "access_key_id_secret_id": , "secret_access_key_secret_id": , "session_token": , "session_token_secret_id": , "assume_role_arn": , "assume_role_session_name": }`; returns JSON `null` for other connection types and non-connection statements). +Helper functions include `jsonb_datum_to_u64`, `jsonb_datum_to_acl_mode` (shared decoder for `AclMode` objects used by both `parse_catalog_privileges` and `parse_catalog_acl_mode`), and `jsonb_datum_to_role_id`. The `PostgresPublicationDetailsSubset` struct is a local prost message that re-declares the two proto tags needed (`slot` at tag 2, `timeline_id` at tag 3) to avoid a dependency cycle with `mz-storage-types`. diff --git a/doc/developer/generated/expr/scalar/func/impls/range.md b/doc/developer/generated/expr/scalar/func/impls/range.md index 85b56fa222443..d0e7d04dc705b 100644 --- a/doc/developer/generated/expr/scalar/func/impls/range.md +++ b/doc/developer/generated/expr/scalar/func/impls/range.md @@ -1,6 +1,6 @@ --- source: src/expr/src/scalar/func/impls/range.rs -revision: 94ee2d5448 +revision: 95baa04a85 --- # mz-expr::scalar::func::impls::range @@ -8,7 +8,7 @@ revision: 94ee2d5448 Provides scalar function implementations for PostgreSQL range types. Key types: - `CastRangeToString` -- converts a range to its text representation via `stringify_datum`, parameterized by `SqlScalarType`, implementing `LazyUnaryFunc` directly. -- `RangeLower` / `RangeUpper` -- extract the lower/upper bound from a range (via `#[sqlfunc]`); `RangeLower` is marked `is_monotone = true`. +- `RangeLower` / `RangeUpper` -- extract the lower/upper bound from a range (via `#[sqlfunc]`); `RangeLower` is marked `is_monotone = true`. The monotone claim survives the function mapping empty and unbounded-lower ranges to NULL because those NULL-producing inputs form a downward-closed prefix of the range ordering (`None` inner sorts below `Some`, and a `None` lower bound sorts below every finite one), so a range whose endpoints both yield values contains no NULL-yielding interior. `RangeUpper` is not marked monotone. - `RangeEmpty` -- returns whether the range is empty. - `RangeLowerInc` / `RangeUpperInc` -- return whether the lower/upper bound is inclusive. - `RangeLowerInf` / `RangeUpperInf` -- return whether the lower/upper bound is unbounded. diff --git a/doc/developer/generated/interchange/avro/encode.md b/doc/developer/generated/interchange/avro/encode.md index 8b6f0f5c0df53..ad4abda45cf88 100644 --- a/doc/developer/generated/interchange/avro/encode.md +++ b/doc/developer/generated/interchange/avro/encode.md @@ -1,6 +1,6 @@ --- source: src/interchange/src/avro/encode.rs -revision: 57cbba8867 +revision: f0cdedca04 --- # interchange::avro::encode diff --git a/doc/developer/generated/kafka-util/client.md b/doc/developer/generated/kafka-util/client.md index 6065fd0e4641e..3d6b7b70de587 100644 --- a/doc/developer/generated/kafka-util/client.md +++ b/doc/developer/generated/kafka-util/client.md @@ -1,6 +1,6 @@ --- source: src/kafka-util/src/client.rs -revision: d5cab9feec +revision: d9bb748bcd --- # mz-kafka-util::client diff --git a/doc/developer/generated/mysql-util/_crate.md b/doc/developer/generated/mysql-util/_crate.md index 9ccf4a90e3492..cd89e518583a4 100644 --- a/doc/developer/generated/mysql-util/_crate.md +++ b/doc/developer/generated/mysql-util/_crate.md @@ -1,6 +1,6 @@ --- source: src/mysql-util/src/lib.rs -revision: a2661591b3 +revision: de8eac0d16 --- # mysql-util diff --git a/doc/developer/generated/mysql-util/partition.md b/doc/developer/generated/mysql-util/partition.md index 5588a68ee035d..79b4a7a3b63b9 100644 --- a/doc/developer/generated/mysql-util/partition.md +++ b/doc/developer/generated/mysql-util/partition.md @@ -1,6 +1,6 @@ --- source: src/mysql-util/src/partition.rs -revision: a2661591b3 +revision: de8eac0d16 --- # mysql-util::partition @@ -9,7 +9,7 @@ Primary key space partitioner for MySQL string columns. Divides a `utf8mb4_bin` ## `partition_table` -The public entry point. Accepts a live MySQL connection (inside a `REPEATABLE READ` transaction), a qualified table reference, the primary key column name, the desired worker count, an estimated total row count, and a `min_split_threshold`. Returns up to `num_workers - 1` exclusive boundary strings that divide the key space into `num_workers` partitions. +The public entry point. Accepts a live MySQL connection (inside a `REPEATABLE READ` transaction), a qualified table reference, the primary key column name, the desired worker count, an estimated total row count, a `min_split_threshold`, and a `max_probed_prefixes` budget. Returns up to `num_workers - 1` exclusive boundary strings that divide the key space into `num_workers` partitions. The function delegates partitioning to the internal `partition` function and traces the resulting boundaries at the `TRACE` level, redacting them outside CI. @@ -21,9 +21,9 @@ The function delegates partitioning to the internal `partition` function and tra ### BFS prefix expansion (`compute_boundaries`) -Starting from a single root prefix covering the entire key space, the algorithm iterates BFS rounds. In each round every prefix whose `estimated_rows` exceeds `target_max_rows_per_prefix` is expanded into child prefixes one character longer via `children_prefixes`; prefixes within the target are carried forward unchanged. The loop stops when no prefix is split in a round. +Starting from a single root prefix covering the entire key space, the algorithm iterates BFS rounds. In each round every prefix whose `estimated_rows` exceeds `target_max_rows_per_prefix` is expanded into child prefixes one character longer via `children_prefixes`, subject to a `max_probed_prefixes` budget that is decremented once per prefix probed. When the budget is exhausted mid-walk, the partial split is discarded and the parent prefix is kept as a leaf. Prefixes within the target are carried forward unchanged. The loop stops when no prefix is split in a round. -After BFS, the algorithm sums the estimated row counts across all leaf prefixes and assigns worker boundaries greedily: it walks the prefixes in key order, accumulating rows, and emits a boundary at each prefix whose cumulative total crosses a multiple of `total / workers`. +After BFS, the algorithm recomputes the total estimated rows across all leaf prefixes (which can diverge from the original top-level estimate) and assigns worker boundaries greedily: it walks the prefixes in key order, accumulating rows, and emits a boundary at each prefix whose cumulative total crosses a multiple of `total / workers`. ### Child prefix enumeration (`children_prefixes`) diff --git a/doc/developer/generated/mysql-util/probe.md b/doc/developer/generated/mysql-util/probe.md index 9d032fb10280d..ea6c6dd540e0c 100644 --- a/doc/developer/generated/mysql-util/probe.md +++ b/doc/developer/generated/mysql-util/probe.md @@ -1,6 +1,6 @@ --- source: src/mysql-util/src/probe.rs -revision: a2661591b3 +revision: de8eac0d16 --- # mysql-util::probe diff --git a/doc/developer/generated/mz-deploy/cli/commands/clusters.md b/doc/developer/generated/mz-deploy/cli/commands/clusters.md index efa2ce1ec057f..dc2a594785c67 100644 --- a/doc/developer/generated/mz-deploy/cli/commands/clusters.md +++ b/doc/developer/generated/mz-deploy/cli/commands/clusters.md @@ -1,6 +1,6 @@ --- source: src/mz-deploy/src/cli/commands/clusters.rs -revision: 3f3bdb0535 +revision: 2e6c03ac43 --- # mz-deploy::cli::commands::clusters @@ -9,6 +9,6 @@ Clusters apply command: converge live cluster state to match cluster definitions `plan` iterates over definitions and calls `plan_cluster` for each. `plan_cluster` checks whether the cluster exists, creates it if absent, or computes drift and emits `ALTER CLUSTER RESET` then `ALTER CLUSTER SET` statements if options have changed. RESET runs before SET to avoid server-side validation errors when simultaneously dropping an autoscaling policy and changing the cluster size. Grant and comment reconciliation follows the create/alter step. -`diff_cluster_options` computes which options to SET and which to RESET by comparing a `ClusterDefinition` against the live `Cluster`. Autoscaling policy drift is only computed when the region supports autoscaling strategies; on older regions the live policy is unknowable and left unchanged. +`diff_cluster_options` computes which options to SET and which to RESET by comparing a `ClusterDefinition` against the live cluster state. The comparison is generic over cluster option names: an option the definition declares is SET when its value differs from the live one; an option the definition omits is RESET unless the live value is already the server default. `default_options` supplies the server defaults, including `EXPERIMENTAL ARRANGEMENT COMPRESSION = false`. The `CanonicalIntervals` visitor normalizes interval-valued literals so that spelling differences (e.g. `'60s'` vs `'00:01:00'`) do not appear as drift. `is_discarded` filters out options the server drops (such as `DISK` and empty-block spellings), preventing false drift. `run` is the CLI entry point: it connects, calls `plan`, and executes the result unless `dry_run` is set. diff --git a/doc/developer/generated/mz-deploy/cli/commands/stage.md b/doc/developer/generated/mz-deploy/cli/commands/stage.md index 73b10584fb0ee..4f8a4bc98ecef 100644 --- a/doc/developer/generated/mz-deploy/cli/commands/stage.md +++ b/doc/developer/generated/mz-deploy/cli/commands/stage.md @@ -1,6 +1,6 @@ --- source: src/mz-deploy/src/cli/commands/stage.rs -revision: b9e4651bb1 +revision: 2e6c03ac43 --- # mz-deploy::cli::commands::stage diff --git a/doc/developer/generated/mz-deploy/client/_module.md b/doc/developer/generated/mz-deploy/client/_module.md index b188dfabc9a43..462176872d4cc 100644 --- a/doc/developer/generated/mz-deploy/client/_module.md +++ b/doc/developer/generated/mz-deploy/client/_module.md @@ -1,6 +1,6 @@ --- source: src/mz-deploy/src/client.rs -revision: 3f3bdb0535 +revision: 2e6c03ac43 --- # mz-deploy::client @@ -15,8 +15,6 @@ Scoped sub-clients: * `deployment_ops` — blue/green lifecycle: staging, hydration monitoring, cutover, abort. * `validation` — pre-deployment checks that the target environment matches expected state. * `type_info` — `SHOW COLUMNS` queries to generate/refresh the `types.lock` data-contract file. -* `auto_scaling` — helper submodule for converting autoscaling strategy values to/from SQL AST and catalog JSON. - Supporting submodules: `models` (shared data structures), `errors` (`ConnectionError`, `DatabaseValidationError`), `dev_overlays` (`DevOverlaysClient`). `quote_identifier` double-quotes a SQL identifier, escaping embedded double quotes. diff --git a/doc/developer/generated/mz-deploy/client/auto_scaling.md b/doc/developer/generated/mz-deploy/client/auto_scaling.md deleted file mode 100644 index c18102632b6a8..0000000000000 --- a/doc/developer/generated/mz-deploy/client/auto_scaling.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -source: src/mz-deploy/src/client/auto_scaling.rs -revision: 3f3bdb0535 ---- - -# mz-deploy::client::auto_scaling - -Conversions between the `AUTO SCALING STRATEGY` cluster option and the structured `AutoScalingStrategy` policy. - -Three representations meet here: - -- The SQL option in a `CREATE CLUSTER` / `ALTER CLUSTER` statement (`ClusterAutoScalingStrategyOptionValue` AST). -- The structured policy (`mz_sql::plan::AutoScalingStrategy`), used for drift comparison. -- The `strategy` jsonb column of `mz_internal.mz_cluster_auto_scaling_strategies`, whose serde shape matches the plan type. - -Key functions: - -- `strategy_from_option_value` — converts an AST option value into a structured policy. An empty block `AUTO SCALING STRATEGY = ()` maps to `None` (autoscaling disabled). -- `strategy_to_cluster_option` — renders a structured policy back into a `CREATE CLUSTER` / `ALTER CLUSTER ... SET` option. -- `strategy_from_catalog_json` — parses the `strategy` jsonb column. A JSON `null` value maps to `None`, matching a cluster whose policy was just removed while a burst still lingers. diff --git a/doc/developer/generated/mz-deploy/client/connection.md b/doc/developer/generated/mz-deploy/client/connection.md index 67c8135cdf1fc..c13fae198f5dd 100644 --- a/doc/developer/generated/mz-deploy/client/connection.md +++ b/doc/developer/generated/mz-deploy/client/connection.md @@ -1,6 +1,6 @@ --- source: src/mz-deploy/src/client/connection.rs -revision: 3f3bdb0535 +revision: 2e6c03ac43 --- # mz-deploy::client::connection diff --git a/doc/developer/generated/mz-deploy/client/introspection.md b/doc/developer/generated/mz-deploy/client/introspection.md index b72fd8a795bdf..4000161bfe78b 100644 --- a/doc/developer/generated/mz-deploy/client/introspection.md +++ b/doc/developer/generated/mz-deploy/client/introspection.md @@ -1,6 +1,6 @@ --- source: src/mz-deploy/src/client/introspection.rs -revision: 3f3bdb0535 +revision: 2e6c03ac43 --- # mz-deploy::client::introspection @@ -10,5 +10,3 @@ Read-only catalog introspection queries. Methods on `IntrospectionClient` query `mz_catalog` and `information_schema` to inspect the live environment without modifying it. Provides batch existence checks for schemas, clusters, and objects, as well as dependency lookups used during deployment planning and sink repointing. `DependentSink` identifies a sink that depends on an object in a schema being dropped; it is used during apply to find sinks that must be repointed before old schemas are dropped with CASCADE. - -Cluster queries include autoscaling strategy support via `auto_scaling_query_parts`, which returns SQL fragments for joining `mz_internal.mz_cluster_auto_scaling_strategies` on regions that support the feature, or a `NULL` column on older regions. `parse_auto_scaling_strategy` deserializes the strategy JSON from a query row. diff --git a/doc/developer/generated/mz-deploy/client/models.md b/doc/developer/generated/mz-deploy/client/models.md index 765d2d0d82c4a..917dda846956e 100644 --- a/doc/developer/generated/mz-deploy/client/models.md +++ b/doc/developer/generated/mz-deploy/client/models.md @@ -1,6 +1,6 @@ --- source: src/mz-deploy/src/client/models.rs -revision: 3f3bdb0535 +revision: 2e6c03ac43 --- # mz-deploy::client::models @@ -11,8 +11,6 @@ Domain models for Materialize catalog objects. `DeploymentMode` is the mode tag stored alongside each deployment record. Currently only `Stage` exists; kept as an enum for schema compatibility with the `mode` column in `_mz_deploy.tables.deployments`. -`Cluster` represents a live compute cluster from the Materialize catalog. The `auto_scaling_strategy` field is `None` for unmanaged clusters, clusters without an autoscaling policy, and regions that predate the autoscaling feature. +`Cluster` represents a live compute cluster from the Materialize catalog. -`ClusterOptions` holds the desired managed-cluster configuration (size, replication factor, autoscaling policy). `ClusterOptions::from_cluster` constructs options from a live `Cluster`, propagating its autoscaling policy. - -`ClusterConfig` is an enum distinguishing `Managed` clusters (with `ClusterOptions` and grants) from `Unmanaged` clusters (with replica definitions and grants). It is used during blue/green staging to recreate production clusters in the staging environment. +`ClusterConfig` is an enum distinguishing `Managed` clusters (with the canonical `CREATE CLUSTER` statement and grants) from `Unmanaged` clusters (with replica definitions and grants). It is used during blue/green staging to recreate production clusters in the staging environment. diff --git a/doc/developer/generated/mz-deploy/client/provisioning.md b/doc/developer/generated/mz-deploy/client/provisioning.md index a1aa1f8de4774..04ad3e9533f5e 100644 --- a/doc/developer/generated/mz-deploy/client/provisioning.md +++ b/doc/developer/generated/mz-deploy/client/provisioning.md @@ -1,6 +1,6 @@ --- source: src/mz-deploy/src/client/provisioning.rs -revision: 3f3bdb0535 +revision: 2e6c03ac43 --- # mz-deploy::client::provisioning @@ -11,4 +11,4 @@ Methods on `ProvisioningClient` issue idempotent `CREATE … IF NOT EXISTS` and Provisioning must follow referential order: databases must be created before schemas, and schemas before clusters. Callers (e.g. `DeploymentExecutor`) are responsible for invoking methods in the correct order. -`create_cluster` builds a `CREATE CLUSTER` statement with the requested size and replication factor, appending an autoscaling strategy clause via `strategy_to_cluster_option` when one is configured. `create_cluster_with_config` dispatches between managed and unmanaged cluster creation based on the `ClusterConfig` variant, then applies any associated replica and grant statements. +`create_cluster_with_config` dispatches between managed and unmanaged cluster creation based on the `ClusterConfig` variant. For managed clusters, it replays the production cluster's canonical `CREATE CLUSTER` statement under the new name. For unmanaged clusters, it issues `CREATE CLUSTER ... REPLICAS ()` followed by individual `CREATE CLUSTER REPLICA` statements. After the cluster is created, any associated grant statements are applied. diff --git a/doc/developer/generated/mz-deploy/project/clusters.md b/doc/developer/generated/mz-deploy/project/clusters.md index a23718c726cee..c73597c7abb48 100644 --- a/doc/developer/generated/mz-deploy/project/clusters.md +++ b/doc/developer/generated/mz-deploy/project/clusters.md @@ -1,6 +1,6 @@ --- source: src/mz-deploy/src/project/clusters.rs -revision: 3f3bdb0535 +revision: 2e6c03ac43 --- # mz-deploy::project::clusters @@ -12,5 +12,3 @@ Cluster definition loading and validation. `load_clusters` scans `/clusters/` for `.sql` files, parses each file, and validates it via `classify_cluster_statements`. Profile overrides are resolved (a file matching the active profile name takes precedence over the default variant). After validation, if a `profile_suffix` is given, `apply_cluster_suffix` rewrites all cluster name references in the definition so the live cluster name carries the suffix. All variants of each cluster file are validated independently before the active variant is resolved, so structural errors in non-active profile variants are caught even when they would not be used. - -`extract_size`, `extract_replication_factor`, and `extract_auto_scaling_strategy` are helpers that pull specific option values out of a `CreateClusterStatement` for use by the apply command's drift detection. diff --git a/doc/developer/generated/mz-deploy/project/compiler/typecheck/executor.md b/doc/developer/generated/mz-deploy/project/compiler/typecheck/executor.md index 0d0fb07c567e5..28e3f53d5f8e9 100644 --- a/doc/developer/generated/mz-deploy/project/compiler/typecheck/executor.md +++ b/doc/developer/generated/mz-deploy/project/compiler/typecheck/executor.md @@ -1,6 +1,6 @@ --- source: src/mz-deploy/src/project/compiler/typecheck/executor.rs -revision: 673fdb9d44 +revision: f0cdedca04 --- # mz-deploy::project::compiler::typecheck::executor diff --git a/doc/developer/generated/ore/assert.md b/doc/developer/generated/ore/assert.md index 351c4f35ff37c..2233c2ac274df 100644 --- a/doc/developer/generated/ore/assert.md +++ b/doc/developer/generated/ore/assert.md @@ -1,6 +1,6 @@ --- source: src/ore/src/assert.rs -revision: 1ec49c0396 +revision: f0cdedca04 --- # mz-ore::assert diff --git a/doc/developer/generated/ore/pool/_module.md b/doc/developer/generated/ore/pool/_module.md index 487aae1a0b89b..404bcee080cf7 100644 --- a/doc/developer/generated/ore/pool/_module.md +++ b/doc/developer/generated/ore/pool/_module.md @@ -1,6 +1,6 @@ --- source: src/ore/src/pool.rs -revision: 26305d8cb0 +revision: f0cdedca04 --- # mz-ore::pool diff --git a/doc/developer/generated/ore/pool/extent.md b/doc/developer/generated/ore/pool/extent.md index 5878ccf3bdaf2..a8e66027d42c5 100644 --- a/doc/developer/generated/ore/pool/extent.md +++ b/doc/developer/generated/ore/pool/extent.md @@ -1,6 +1,6 @@ --- source: src/ore/src/pool/extent.rs -revision: f2082d0163 +revision: f0cdedca04 --- # mz-ore::pool::extent diff --git a/doc/developer/generated/ore/pool/region.md b/doc/developer/generated/ore/pool/region.md index 6b93e127e13f7..6d47c3e3f22cf 100644 --- a/doc/developer/generated/ore/pool/region.md +++ b/doc/developer/generated/ore/pool/region.md @@ -1,6 +1,6 @@ --- source: src/ore/src/pool/region.rs -revision: f2082d0163 +revision: f0cdedca04 --- # mz-ore::pool::region diff --git a/doc/developer/generated/ore/secure.md b/doc/developer/generated/ore/secure.md index 94953d6c1c15d..2d9790f0b7830 100644 --- a/doc/developer/generated/ore/secure.md +++ b/doc/developer/generated/ore/secure.md @@ -1,6 +1,6 @@ --- source: src/ore/src/secure.rs -revision: b69d9bb412 +revision: 07cec92c1c --- # ore::secure @@ -28,7 +28,7 @@ Downstream crates should import these from `mz_ore::secure` rather than adding a A `String` wrapper that: - Is zeroed from memory when dropped (`ZeroizeOnDrop`) - Redacts its contents in `Debug` and `Display` output -- Does **not** implement `Clone` (prevents untracked copies of secrets) +- Implements `Clone`; clones also carry `ZeroizeOnDrop`, so copies are zeroed independently on drop Use `unsecure()` to access the inner `&str` when the plaintext is needed. diff --git a/doc/developer/generated/ore/tracing.md b/doc/developer/generated/ore/tracing.md index 0e97462e7a70e..5af341360a9b7 100644 --- a/doc/developer/generated/ore/tracing.md +++ b/doc/developer/generated/ore/tracing.md @@ -1,6 +1,6 @@ --- source: src/ore/src/tracing.rs -revision: 8ef14a9866 +revision: b19c78fa51 --- # mz-ore::tracing diff --git a/doc/developer/generated/persist-client/cache.md b/doc/developer/generated/persist-client/cache.md index 792189c9aca61..97d05202a4967 100644 --- a/doc/developer/generated/persist-client/cache.md +++ b/doc/developer/generated/persist-client/cache.md @@ -1,6 +1,6 @@ --- source: src/persist-client/src/cache.rs -revision: b8c325f035 +revision: f0cdedca04 --- # persist-client::cache diff --git a/doc/developer/generated/persist-client/fetch.md b/doc/developer/generated/persist-client/fetch.md index 529dfda3b12df..c69a289a5ba5f 100644 --- a/doc/developer/generated/persist-client/fetch.md +++ b/doc/developer/generated/persist-client/fetch.md @@ -1,12 +1,13 @@ --- source: src/persist-client/src/fetch.rs -revision: 70f75e4f0e +revision: 95baa04a85 --- # persist-client::fetch Implements reading of individual batch parts from blob storage, including `LeasedBatchPart` (a leased reference to a part), `FetchedPart` (the decoded columnar data), and `EncodedPart` (the raw bytes as fetched). Applies time-range filtering (`FetchBatchFilter`) and optional stats-based pushdown to skip fetching parts entirely. +`FetchedBlob::stats()` decodes part stats via `try_decode()` and returns `None` on failure (undecodable stats due to version skew or corruption), causing the caller to treat the part as unconstrained and fetch it rather than discarding it. A semaphore-based permit system (`FETCH_SEMAPHORE_PERMIT_ADJUSTMENT`) bounds the total number of bytes being decoded in-flight to avoid memory overload. `LeasedBatchPart` carries the `LeasedReaderId` of the reader that minted it; when a blob fetch fails, `missing_blob_diagnostics` refreshes shard state and reports whether that reader is still present, distinguishing an expired lease from a GC bug. `fetch_batch_part_blob` adds context (the blob key) to any error returned by `Blob::get`, so that retry log entries produced by `retry_external` identify which blob and therefore which shard is stuck when a GET stalls indefinitely. diff --git a/doc/developer/generated/persist-client/internal/_module.md b/doc/developer/generated/persist-client/internal/_module.md index ce7774efe845e..fb707f6b83cee 100644 --- a/doc/developer/generated/persist-client/internal/_module.md +++ b/doc/developer/generated/persist-client/internal/_module.md @@ -1,6 +1,6 @@ --- source: src/persist-client/src/internal/state.rs -revision: 253293ef87 +revision: 95baa04a85 --- # persist-client::internal diff --git a/doc/developer/generated/persist-client/internal/cache.md b/doc/developer/generated/persist-client/internal/cache.md index ead817c04d8be..d09721e5cb615 100644 --- a/doc/developer/generated/persist-client/internal/cache.md +++ b/doc/developer/generated/persist-client/internal/cache.md @@ -1,6 +1,6 @@ --- source: src/persist-client/src/internal/cache.rs -revision: e757b4d11b +revision: f0cdedca04 --- # persist-client::internal::cache diff --git a/doc/developer/generated/persist-client/internal/compact.md b/doc/developer/generated/persist-client/internal/compact.md index bf753d3b6bdcf..e94b7754d39ef 100644 --- a/doc/developer/generated/persist-client/internal/compact.md +++ b/doc/developer/generated/persist-client/internal/compact.md @@ -1,6 +1,6 @@ --- source: src/persist-client/src/internal/compact.rs -revision: 4267863081 +revision: f0cdedca04 --- # persist-client::internal::compact diff --git a/doc/developer/generated/persist-client/internal/encoding.md b/doc/developer/generated/persist-client/internal/encoding.md index 985eafcefa553..472145677ec66 100644 --- a/doc/developer/generated/persist-client/internal/encoding.md +++ b/doc/developer/generated/persist-client/internal/encoding.md @@ -1,11 +1,11 @@ --- source: src/persist-client/src/internal/encoding.rs -revision: c450b59e6d +revision: 95baa04a85 --- # persist-client::internal::encoding Implements proto encoding/decoding for all persist internal types: `State`, `StateDiff`, `Trace`, `HollowBatch`, rollups, reader/writer state, and schemas. Provides `LazyProto` (deferred proto decoding), `LazyPartStats` (deferred stats decoding), and `Schemas` (a pair of key/value schema handles). -`LazyPartStats` stores encoded stats bytes without validating them on ingestion. Its `Debug` impl uses `try_decode()` to surface malformed encodings as a rendered error rather than panicking, since the bytes may arrive straight off blob and pass through `Trace::unflatten`'s validation path before any decode is attempted. +`LazyPartStats` stores encoded stats bytes without validating them on ingestion. Its `Debug` impl uses `try_decode()` to surface malformed encodings as a rendered error rather than panicking, since the bytes may arrive straight off blob and pass through `Trace::unflatten`'s validation path before any decode is attempted. Stats from a newer version that use an unknown `oneof` variant decode as an error rather than as a misread value; every read path therefore fails open on them (the filter keeps the part, `stats()` accessors return `None`, and `EXPLAIN FILTER PUSHDOWN` reports the part as selected). Also owns codec version compatibility checks that prevent reading data written by a newer version of persist. diff --git a/doc/developer/generated/persist-client/internal/machine.md b/doc/developer/generated/persist-client/internal/machine.md index 6964f902e9169..f078154c29cfd 100644 --- a/doc/developer/generated/persist-client/internal/machine.md +++ b/doc/developer/generated/persist-client/internal/machine.md @@ -1,6 +1,6 @@ --- source: src/persist-client/src/internal/machine.rs -revision: 1af97ffb4d +revision: 0c5d22e13a --- # persist-client::internal::machine @@ -9,7 +9,7 @@ Implements `Machine`, the core state-machine driver that sequences all shard ope Each operation is a pure state transition function applied to a `State` snapshot; `Machine` handles the retry loop, maintenance scheduling, and error propagation. Heartbeat timestamps used in compare-and-append, since downgrade, and reader registration are sampled fresh inside each retry closure via `(cfg.now)()` rather than being passed in as parameters; this ensures that a long-running retry sequence commits a recent timestamp rather than one computed before the first attempt. Also provides the `retry_external` and `retry_determinate` helpers that standardize backoff behavior for transient storage errors. -`retry_external` logs at INFO after `INFO_MIN_ATTEMPTS` (3) failures and escalates to WARN after `WARN_MIN_ATTEMPTS` (30) failures; at roughly five minutes of continuous failure using the persist backoff (clamped at 16 s), a WARN log signals that the operation is likely wedged rather than experiencing a transient error. +`retry_external` logs at INFO after `INFO_MIN_ATTEMPTS` (3) failures and escalates to WARN after `WARN_MIN_ATTEMPTS` (30) failures; at roughly five minutes of continuous failure using the persist backoff (clamped at 16 s), a WARN log signals that the operation is likely wedged rather than experiencing a transient error. All error log lines use `display_with_causes()` to show the full error chain, including in the indeterminate `compare_and_append` retry log. Includes configurable compaction claiming via `CLAIM_UNCLAIMED_COMPACTIONS`, `CLAIM_COMPACTION_PERCENT`, and `CLAIM_COMPACTION_MIN_VERSION` dynamic configs. `upgrade_version` is a no-op when the shard is a tombstone: it returns `Ok(())` immediately via `NoOpStateTransition` rather than attempting to commit a new state, which `compute_next_state_locked` would reject on a finalized shard. `wait_for_upper_past` uses `StateWatch::wait_for_upper_past` so that snapshot and listen waiters wake only on shard upper advances, not on seqno bumps from GC, rollups, or since-downgrades. diff --git a/doc/developer/generated/persist-client/internal/state.md b/doc/developer/generated/persist-client/internal/state.md index a8ebfa2d59111..24a0ff7aebdb0 100644 --- a/doc/developer/generated/persist-client/internal/state.md +++ b/doc/developer/generated/persist-client/internal/state.md @@ -1,6 +1,6 @@ --- source: src/persist-client/src/internal/state.rs -revision: 253293ef87 +revision: 95baa04a85 --- # persist-client::internal::state diff --git a/doc/developer/generated/persist-client/internal/state_diff.md b/doc/developer/generated/persist-client/internal/state_diff.md index d669010086bc8..24f0b866c13c6 100644 --- a/doc/developer/generated/persist-client/internal/state_diff.md +++ b/doc/developer/generated/persist-client/internal/state_diff.md @@ -1,6 +1,6 @@ --- source: src/persist-client/src/internal/state_diff.rs -revision: 253293ef87 +revision: f0cdedca04 --- # persist-client::internal::state_diff diff --git a/doc/developer/generated/persist-client/internal/trace.md b/doc/developer/generated/persist-client/internal/trace.md index c281b9982b585..8280ebf3d5018 100644 --- a/doc/developer/generated/persist-client/internal/trace.md +++ b/doc/developer/generated/persist-client/internal/trace.md @@ -1,6 +1,6 @@ --- source: src/persist-client/src/internal/trace.rs -revision: c450b59e6d +revision: f0cdedca04 --- # persist-client::internal::trace diff --git a/doc/developer/generated/persist-client/operators/_module.md b/doc/developer/generated/persist-client/operators/_module.md index da05ecc388efd..c21a9bd0fb81f 100644 --- a/doc/developer/generated/persist-client/operators/_module.md +++ b/doc/developer/generated/persist-client/operators/_module.md @@ -1,6 +1,6 @@ --- source: src/persist-client/src/operators/shard_source.rs -revision: 70f75e4f0e +revision: 95baa04a85 --- # persist-client::operators diff --git a/doc/developer/generated/persist-client/operators/shard_source.md b/doc/developer/generated/persist-client/operators/shard_source.md index cc67c5e2f9ccf..f59f781c5e206 100644 --- a/doc/developer/generated/persist-client/operators/shard_source.md +++ b/doc/developer/generated/persist-client/operators/shard_source.md @@ -1,6 +1,6 @@ --- source: src/persist-client/src/operators/shard_source.rs -revision: 70f75e4f0e +revision: 95baa04a85 --- # persist-client::operators::shard_source diff --git a/doc/developer/generated/persist-client/schema.md b/doc/developer/generated/persist-client/schema.md index 88d57f84171d0..016ec8a5862ac 100644 --- a/doc/developer/generated/persist-client/schema.md +++ b/doc/developer/generated/persist-client/schema.md @@ -1,6 +1,6 @@ --- source: src/persist-client/src/schema.rs -revision: 161628c089 +revision: f0cdedca04 --- # persist-client::schema diff --git a/doc/developer/generated/persist-client/usage.md b/doc/developer/generated/persist-client/usage.md index 6ee86dc35a7b9..93ddac44844e6 100644 --- a/doc/developer/generated/persist-client/usage.md +++ b/doc/developer/generated/persist-client/usage.md @@ -1,6 +1,6 @@ --- source: src/persist-client/src/usage.rs -revision: 1671bbe147 +revision: f0cdedca04 --- # persist-client::usage diff --git a/doc/developer/generated/persist-types/arrow.md b/doc/developer/generated/persist-types/arrow.md index b6037fe3bb53c..efb37360dd270 100644 --- a/doc/developer/generated/persist-types/arrow.md +++ b/doc/developer/generated/persist-types/arrow.md @@ -1,6 +1,6 @@ --- source: src/persist-types/src/arrow.rs -revision: 25802a51f9 +revision: f0cdedca04 --- # persist-types::arrow diff --git a/doc/developer/generated/persist-types/codec_impls.md b/doc/developer/generated/persist-types/codec_impls.md index ae7223928a8d1..9041d1ec34bc4 100644 --- a/doc/developer/generated/persist-types/codec_impls.md +++ b/doc/developer/generated/persist-types/codec_impls.md @@ -1,6 +1,6 @@ --- source: src/persist-types/src/codec_impls.rs -revision: 901d0526a1 +revision: 1c250c3597 --- # persist-types::codec_impls diff --git a/doc/developer/generated/persist/file.md b/doc/developer/generated/persist/file.md index d58d957fffb03..3d94cc08ca882 100644 --- a/doc/developer/generated/persist/file.md +++ b/doc/developer/generated/persist/file.md @@ -1,6 +1,6 @@ --- source: src/persist/src/file.rs -revision: a298f30bbb +revision: f0cdedca04 --- # persist::file diff --git a/doc/developer/generated/persist/s3.md b/doc/developer/generated/persist/s3.md index 60eb617660710..17b0c0b7ac287 100644 --- a/doc/developer/generated/persist/s3.md +++ b/doc/developer/generated/persist/s3.md @@ -1,6 +1,6 @@ --- source: src/persist/src/s3.rs -revision: f8cd2af31b +revision: f0cdedca04 --- # persist::s3 diff --git a/doc/developer/generated/pgcopy/copy.md b/doc/developer/generated/pgcopy/copy.md index 87d3a72079894..95bb29c598396 100644 --- a/doc/developer/generated/pgcopy/copy.md +++ b/doc/developer/generated/pgcopy/copy.md @@ -1,6 +1,6 @@ --- source: src/pgcopy/src/copy.rs -revision: 10a94621ed +revision: 95baa04a85 --- # mz-pgcopy::copy @@ -9,4 +9,4 @@ Implements encoding and decoding of PostgreSQL COPY text, CSV, and binary format `encode_copy_format` and `decode_copy_format` dispatch on `CopyFormatParams` (Text, Csv, Binary, Parquet) to per-format helpers; `CopyTextFormatParser` / `RawIterator` handle escape sequences for the text format; `CopyCsvFormatParams` and `CopyTextFormatParams` carry configurable delimiters, quotes, escape characters, and null sentinels. `encode_copy_format` accepts a `TextEncodeSettings` argument that is forwarded to `Value::encode_text` for text and CSV formats; callers that encode outside a session context (such as `COPY TO ` in the dataflow layer) must pass `TextEncodeSettings::STABLE`. `encode_copy_format_header` writes a CSV header row when requested; it uses `TextEncodeSettings::STABLE` internally. -CSV decoding uses `csv-core` rather than the higher-level `csv` crate so that per-field quote state is available; `DecodedField` records the `start..end` range into the decode output buffer and a `quoted` flag indicating whether the field's first input byte was the configured quote character. The NULL-marker and end-of-copy checks both require the field to be unquoted, preventing a quoted empty string (`""`) from silently decoding to SQL NULL and preventing a quoted `"\."` from being mistaken for the bare `\.` terminator. +CSV decoding uses `csv-core` rather than the higher-level `csv` crate so that per-field quote state is available; `DecodedField` records the `start..end` range into the decode output buffer and a `quoted` flag indicating whether the field's first input byte was the configured quote character. The NULL-marker and end-of-copy checks both require the field to be unquoted, preventing a quoted empty string (`""`) from silently decoding to SQL NULL and preventing a quoted `"\."` from being mistaken for the bare `\.` terminator. An `at_record_start` flag tracks whether the decoder is at the beginning of a record; when it is, any orphan `\r`/`\n` bytes (left by `csv-core` after consuming the `\r` half of a CRLF terminator) are skipped before inspecting the quote byte, preventing the orphan from being mistaken for the field's first character and misclassifying a quoted field as unquoted. diff --git a/doc/developer/generated/pgwire/protocol.md b/doc/developer/generated/pgwire/protocol.md index 3c016d48b2ec3..54bb5575773da 100644 --- a/doc/developer/generated/pgwire/protocol.md +++ b/doc/developer/generated/pgwire/protocol.md @@ -1,6 +1,6 @@ --- source: src/pgwire/src/protocol.rs -revision: f9ffa16cba +revision: 39dcae2fba --- # pgwire::protocol diff --git a/doc/developer/generated/repr/adt/interval.md b/doc/developer/generated/repr/adt/interval.md index 7334aec80376d..6eea2648993e0 100644 --- a/doc/developer/generated/repr/adt/interval.md +++ b/doc/developer/generated/repr/adt/interval.md @@ -1,6 +1,6 @@ --- source: src/repr/src/adt/interval.rs -revision: 26305d8cb0 +revision: 84f88ca968 --- # mz-repr::adt::interval diff --git a/doc/developer/generated/repr/relation.md b/doc/developer/generated/repr/relation.md index 5c77389a28b55..18d42473b1c0d 100644 --- a/doc/developer/generated/repr/relation.md +++ b/doc/developer/generated/repr/relation.md @@ -1,6 +1,6 @@ --- source: src/repr/src/relation.rs -revision: 37d0a1c9ac +revision: 95baa04a85 --- # mz-repr::relation @@ -13,6 +13,7 @@ Defines a dual-type system for relation and column metadata: `SqlColumnType::backport_nullability` reconciles nullability information from a `ReprColumnType` back into an `SqlColumnType`, including nested record fields. `RelationDesc` (an ordered sequence of named, typed columns using `SqlColumnType`) remains the primary schema descriptor. `RelationDescBuilder` provides a fluent construction API. +`RelationDesc::apply_demand` asserts via `debug_assert!` that the desc is "dense" (every `ColumnIndex` equals the column's position and `typ_idx`, i.e. no columns have been dropped). This invariant is required because `apply_demand` filters `metadata` by raw `ColumnIndex` and `typ` by position, which only agree when the desc is dense; a sparse desc would silently attach types, statistics, and filter specs to wrong columns downstream. `RelationDescDiff` and `VersionedRelationDesc` support schema evolution by tracking changes between relation versions. `ColumnName`, `ColumnIndex`, and `NotNullViolation` provide naming, indexing, and validation support. `PropRelationDescDiff` and the `arb_relation_desc_diff`, `arb_relation_desc_projection`, and `arb_row_for_relation` helpers are gated behind `#[cfg(any(test, feature = "proptest"))]` and support property-based testing. diff --git a/doc/developer/generated/repr/row/_module.md b/doc/developer/generated/repr/row/_module.md index f572fc76c79f4..6bc76415d68fa 100644 --- a/doc/developer/generated/repr/row/_module.md +++ b/doc/developer/generated/repr/row/_module.md @@ -1,6 +1,6 @@ --- source: src/repr/src/row.rs -revision: f4650d00b +revision: f0cdedca04 --- # mz-repr::row diff --git a/doc/developer/generated/repr/row/encode.md b/doc/developer/generated/repr/row/encode.md index 12c8c8a6ebc3b..7cb64b8e2c409 100644 --- a/doc/developer/generated/repr/row/encode.md +++ b/doc/developer/generated/repr/row/encode.md @@ -1,6 +1,6 @@ --- source: src/repr/src/row/encode.rs -revision: 37d0a1c9ac +revision: 95baa04a85 --- # mz-repr::row::encode diff --git a/doc/developer/generated/repr/scalar.md b/doc/developer/generated/repr/scalar.md index 70dfcb0010ad9..74bd35c297e53 100644 --- a/doc/developer/generated/repr/scalar.md +++ b/doc/developer/generated/repr/scalar.md @@ -1,6 +1,6 @@ --- source: src/repr/src/scalar.rs -revision: f4650d00b +revision: 95baa04a85 --- # mz-repr::scalar diff --git a/doc/developer/generated/repr/stats.md b/doc/developer/generated/repr/stats.md index 022a5a0824844..6b4490b9e6bb2 100644 --- a/doc/developer/generated/repr/stats.md +++ b/doc/developer/generated/repr/stats.md @@ -1,6 +1,6 @@ --- source: src/repr/src/stats.rs -revision: 32c1e1d39 +revision: 95baa04a85 --- # mz-repr::stats @@ -8,3 +8,4 @@ revision: 32c1e1d39 Provides persist statistics implementations for non-primitive `mz-repr` types: decodes Arrow column statistics (min/max) for `Numeric`, timestamps, intervals, JSONB, and other complex types that need custom codec-aware stat extraction. These stats are used by persist's pushdown filters to skip reading shards or batches that provably cannot match a query's filter predicates. `col_values` extracts a `(lower, upper)` datum pair from a `ColumnStatKinds`. For `Float32` and `Float64` columns, bounds are first passed through `float_bounds`, which returns `None` (treating the column as unconstrained) when the lower bound is a negative NaN but the upper bound is not: persist records float stats in IEEE-754 total order where `-NaN` sorts below `-Infinity`, while `Datum` ordering ranks every NaN above every finite value, so such bounds cannot be expressed as a valid datum interval. +Legacy V0 stats use `AtomicBytesStats` (a `BytesStats::Atomic` variant), which carries bounds encoded as `ProtoDatum` bytes without a type tag. `col_values` decodes these bytes and validates that the resulting `Datum` matches the expected column type before using them. A mismatch (which can occur under version skew or schema change) is logged via `soft_panic_or_log!` and returns `None` rather than using a wrong-typed bound that could exclude valid column values. diff --git a/doc/developer/generated/repr/strconv.md b/doc/developer/generated/repr/strconv.md index 51e648b707e7a..ac1e80a7f692f 100644 --- a/doc/developer/generated/repr/strconv.md +++ b/doc/developer/generated/repr/strconv.md @@ -1,6 +1,6 @@ --- source: src/repr/src/strconv.rs -revision: 32c1e1d39 +revision: f0cdedca04 --- # mz-repr::strconv diff --git a/doc/developer/generated/repr/timestamp.md b/doc/developer/generated/repr/timestamp.md index 73a11d207d6cb..d8efca529ab0f 100644 --- a/doc/developer/generated/repr/timestamp.md +++ b/doc/developer/generated/repr/timestamp.md @@ -1,6 +1,6 @@ --- source: src/repr/src/timestamp.rs -revision: 669b67e3d1 +revision: f0cdedca04 --- # mz-repr::timestamp diff --git a/doc/developer/generated/row-spine/_crate.md b/doc/developer/generated/row-spine/_crate.md index 266de19c3a7a8..25e2f9efaaed0 100644 --- a/doc/developer/generated/row-spine/_crate.md +++ b/doc/developer/generated/row-spine/_crate.md @@ -1,6 +1,6 @@ --- source: src/row-spine/src/lib.rs -revision: 12181a5639 +revision: f0cdedca04 --- # mz-row-spine diff --git a/doc/developer/generated/sql-parser/ast/defs/statement.md b/doc/developer/generated/sql-parser/ast/defs/statement.md index 958ba7ca3fc35..2e19caa52703c 100644 --- a/doc/developer/generated/sql-parser/ast/defs/statement.md +++ b/doc/developer/generated/sql-parser/ast/defs/statement.md @@ -1,13 +1,15 @@ --- source: src/sql-parser/src/ast/defs/statement.rs -revision: 447da2b53e +revision: 39dcae2fba --- # mz-sql-parser::ast::defs::statement -Defines `Statement`, the top-level enum of all SQL statement types supported by Materialize, including DML (SELECT, INSERT, UPDATE, DELETE, COPY), DDL (CREATE/ALTER/DROP for connections, databases, schemas, sources, sinks, tables, views, materialized views, indexes, secrets, clusters, roles, types, functions, network policies; ALTER MATERIALIZED VIEW APPLY REPLACEMENT; DISCARD), and control statements (SUBSCRIBE, EXPLAIN PLAN, EXPLAIN PUSHDOWN, EXPLAIN TIMESTAMP, EXPLAIN SINK SCHEMA, EXPLAIN ANALYZE OBJECT, EXPLAIN ANALYZE CLUSTER, SHOW, SET, RESET, BEGIN, COMMIT, ROLLBACK, PREPARE, EXECUTE, EXECUTE UNIT TEST, DEALLOCATE, DECLARE, FETCH, CLOSE, RAISE, GRANT/REVOKE ROLE, GRANT/REVOKE PRIVILEGES, ALTER DEFAULT PRIVILEGES, REASSIGN OWNED, VALIDATE CONNECTION, COMMENT). +Defines `Statement`, the top-level enum of all SQL statement types supported by Materialize, including DML (SELECT, INSERT, UPDATE, DELETE, COPY), DDL (CREATE/ALTER/DROP for connections, databases, schemas, sources, sinks, metric sinks, tables, views, materialized views, indexes, secrets, clusters, roles, types, functions, network policies; ALTER MATERIALIZED VIEW APPLY REPLACEMENT; DISCARD), and control statements (SUBSCRIBE, EXPLAIN PLAN, EXPLAIN PUSHDOWN, EXPLAIN TIMESTAMP, EXPLAIN SINK SCHEMA, EXPLAIN ANALYZE OBJECT, EXPLAIN ANALYZE CLUSTER, SHOW, SET, RESET, BEGIN, COMMIT, ROLLBACK, PREPARE, EXECUTE, EXECUTE UNIT TEST, DEALLOCATE, DECLARE, FETCH, CLOSE, RAISE, GRANT/REVOKE ROLE, GRANT/REVOKE PRIVILEGES, ALTER DEFAULT PRIVILEGES, REASSIGN OWNED, VALIDATE CONNECTION, COMMENT). Each statement variant has a corresponding struct with its specific fields. +`CreateMetricSinkStatement` represents a `CREATE METRIC SINK [IF NOT EXISTS] [IN CLUSTER ] FROM [WITH (PREFIX = '...')]` statement. Fields: `name: UnresolvedItemName`, `in_cluster: Option`, `if_not_exists: bool`, `from: T::ItemName`, `with_options: Vec>`. `CreateMetricSinkOptionName` has a single variant `Prefix`; its `redact_value()` returns `false`. The `AstDisplay` impl persists the `WITH (PREFIX = ...)` clause in `create_sql` so the prefix survives a restart. + `ExecuteUnitTestStatement` represents an `EXECUTE UNIT TEST FOR [AT TIME ] [MOCK , ...] EXPECTED ` statement. Fields: `name: Ident`, `target: T::ItemName`, `at_time: Option>`, `mocks: Vec>`, `expected: ExpectedResultDef`. `MockViewDef` represents a single `MOCK () AS ()` clause within an `EXECUTE UNIT TEST` statement. Fields: `name: T::ItemName`, `columns: Vec>`, `query: Query`. diff --git a/doc/developer/generated/sql-parser/parser.md b/doc/developer/generated/sql-parser/parser.md index c0a431e2868b0..1b530be228af1 100644 --- a/doc/developer/generated/sql-parser/parser.md +++ b/doc/developer/generated/sql-parser/parser.md @@ -1,6 +1,6 @@ --- source: src/sql-parser/src/parser.rs -revision: 447da2b53e +revision: 39dcae2fba --- # mz-sql-parser::parser @@ -26,3 +26,4 @@ The `Precedence` enum is `pub(crate)` and serves as the single source of truth f `parse_rows_from` uses `parse_windowless_function` (a private method) for each function inside `ROWS FROM (...)`. `parse_windowless_function` parses a function name and argument list without consuming `DISTINCT`, `FILTER`, or `OVER`, ensuring that table functions in `ROWS FROM` never carry those clauses. `parse_cluster_option_name` recognizes `AUTO SCALING STRATEGY` as a cluster option, dispatching to `parse_cluster_option_auto_scaling_strategy`. That method parses a paren-enclosed list of strategy sub-policies; the only supported sub-policy in v1 is `ON HYDRATION (HYDRATION SIZE = '...' [, LINGER DURATION = '...'])`. An empty list `()` disables autoscaling. Each sub-policy may appear at most once; a duplicate `ON HYDRATION` entry is a parse error. The result is a `ClusterOption` with value `WithOptionValue::ClusterAutoScalingStrategyOptionValue`. `GRANT/REVOKE ON ALL` parsing recognizes `NETWORK POLICY` as an object type in addition to the other supported types. +`CREATE METRIC SINK [IF NOT EXISTS] [IN CLUSTER ] FROM [WITH (PREFIX = ...)]` is parsed by `parse_create_metric_sink`, which is dispatched from `parse_create` when the next two tokens are `METRIC SINK`. The only supported `WITH` option is `PREFIX`, parsed by `parse_create_metric_sink_option`. diff --git a/doc/developer/generated/sql/ast/transform.md b/doc/developer/generated/sql/ast/transform.md index 271da64494ad8..5480e98514270 100644 --- a/doc/developer/generated/sql/ast/transform.md +++ b/doc/developer/generated/sql/ast/transform.md @@ -1,6 +1,6 @@ --- source: src/sql/src/ast/transform.rs -revision: 699d823624 +revision: 39dcae2fba --- # mz-sql::ast::transform @@ -8,3 +8,4 @@ revision: 699d823624 Provides AST mutation utilities for catalog rename and ID-replacement operations: `create_stmt_rename` renames an item in its own `CREATE` statement; `create_stmt_rename_refs` rewrites all references to a renamed item in dependent statements; `create_stmt_rename_schema_refs` rewrites schema references across CREATE statements; and `create_stmt_replace_ids` replaces `CatalogItemId`s inline. The module uses the `mz-sql-parser` visitor infrastructure (`VisitMut`) to walk and rewrite raw SQL ASTs without reparsing. `CreateSqlRewriteSchema` implements a manual `visit_data_type_mut` because the generated visitor treats `DataType` as an opaque associated type and produces a no-op. The manual implementation descends into `RawDataType::Array`, `List`, `Map`, and `Other` variants so that schema-qualified type references in casts, column types, and nested element types are rewritten alongside other item references during a schema rename. +`CreateMetricSinkStatement` is handled in the rename and schema-rename visitors so that metric sink `FROM` references are rewritten alongside all other item references. diff --git a/doc/developer/generated/sql/catalog.md b/doc/developer/generated/sql/catalog.md index 7b17d24ba9523..75a4247c5eca3 100644 --- a/doc/developer/generated/sql/catalog.md +++ b/doc/developer/generated/sql/catalog.md @@ -1,13 +1,13 @@ --- source: src/sql/src/catalog.rs -revision: fd1dd6e62b +revision: 39dcae2fba --- # mz-sql::catalog Defines the `SessionCatalog` trait — the abstraction layer between the SQL planner and any concrete catalog implementation. The trait covers resolution (converting partial names to fully-qualified names), lookup (retrieving metadata about databases, schemas, items, roles, clusters, etc.), and session management (variables, notices, privilege queries, and access-restriction flags such as `restrict_to_user_objects`). -Supporting types such as `CatalogItem`, `CatalogItemType` (variants: `Table`, `Source`, `Sink`, `View`, `MaterializedView`, `Index`, `Type`, `Func`, `Secret`, `Connection`), `CatalogDatabase`, `CatalogSchema`, `CatalogCluster`, `CatalogRole`, `CatalogType`, and `CatalogError` are all defined here, making this the central interface contract for the entire SQL layer. +Supporting types such as `CatalogItem`, `CatalogItemType` (variants: `Table`, `Source`, `Sink`, `View`, `MaterializedView`, `Index`, `Type`, `Func`, `Secret`, `Connection`, `MetricSink`), `CatalogDatabase`, `CatalogSchema`, `CatalogCluster`, `CatalogRole`, `CatalogType`, and `CatalogError` are all defined here, making this the central interface contract for the entire SQL layer. `CatalogCluster` exposes `auto_scaling_strategy() -> Option<&AutoScalingStrategy>`, returning the user-configured autoscaling policy when the cluster is managed and has one set. `ObjectType` covers the full set of catalog object kinds visible in SQL; `CatalogConfig` carries session-level configuration including `connection_context`, optional `helm_chart_version`, and `aws_account_id` (the AWS account ID the environment runs under, folded to a literal by `mz_aws_account_id()` and `None` on non-cloud/local environments). Also defines `AutoProvisionSource` (enum tracking whether a role was auto-provisioned by OIDC, Frontegg, or None), `RoleAttributes` and `RoleAttributesRaw` (which include an `auto_provision_source` field), and `PasswordAction`. diff --git a/doc/developer/generated/sql/names.md b/doc/developer/generated/sql/names.md index 585ed27220b02..5a9ccde9b0414 100644 --- a/doc/developer/generated/sql/names.md +++ b/doc/developer/generated/sql/names.md @@ -1,6 +1,6 @@ --- source: src/sql/src/names.rs -revision: 4868442183 +revision: 39dcae2fba --- # mz-sql::names @@ -10,7 +10,7 @@ The `Aug` type parameterizes the SQL AST after name resolution and is the key di Also contains `NameResolver` (the AST fold that performs name resolution against a `SessionCatalog`) and `NameSimplifier` (which collapses fully-qualified names back to the shortest unambiguous form for display). `NameResolver` allocates CTE `LocalId`s from a monotonically increasing `next_cte_id` counter via `allocate_cte_id()`. This guarantees every CTE in a statement receives a unique id, even when CTE names shadow each other across nested scopes. Later phases (HIR lowering's `CteMap`, the planner's `qcx.ctes`) key CTEs by `LocalId` and rely on this uniqueness. `ResolvedItemName` has three variants: `Item` (a fully-resolved catalog item), `Cte` (a CTE local ID), and `Error` (a sentinel for name-resolution failures). -`CommentObjectId` identifies the subject of a `COMMENT ON` statement; variants cover tables, sources, views, materialized views, sinks, indexes, connections, types, secrets, roles, databases, schemas, clusters, cluster replicas, and network policies. New variants require corresponding additions to `mz_catalog_protos::objects::CommentObject` and to both CASE expressions in the `mz_internal.mz_comments` materialized view, which reads `CommentObject` out of `mz_catalog_raw` as serde JSON. +`CommentObjectId` identifies the subject of a `COMMENT ON` statement; variants cover tables, sources, views, materialized views, sinks, metric sinks, indexes, connections, types, secrets, roles, databases, schemas, clusters, cluster replicas, and network policies. New variants require corresponding additions to `mz_catalog_protos::objects::CommentObject` and to both CASE expressions in the `mz_internal.mz_comments` materialized view, which reads `CommentObject` out of `mz_catalog_raw` as serde JSON. The `NameResolver`'s `Fold` impl handles all `WithOptionValue` variants; `KafkaMatchingBrokerRule(x)` is resolved by calling `self.fold_kafka_matching_broker_rule(x)`. `ClusterAutoScalingStrategyOptionValue(value)` is passed through unchanged (it carries no names to resolve). `fold_network_policy_name` extracts the raw identifier string from a `RawNetworkPolicyName` before calling `catalog.resolve_network_policy`, using `ident.as_str()` for unresolved names and `s.as_str()` for already-resolved names rather than calling `to_string()` on the outer enum. This ensures quoted identifiers (e.g. `"hyphenated-name"`) are resolved without spurious quote characters that would cause a lookup failure. In `resolve_item_name_id`, the `None` (no version specified) arm guards version pinning with `id.is_user()`: only user items are pinned to a specific version, mirroring the by-name path in `fold_item_name`. The `Some(_)` (explicit version specified) arm additionally resolves to `RelationVersionSelector::Latest` when the item is not a user item (`!id.is_user()`), allowing catalogs that carry stale builtin version pins to open successfully rather than failing with `PlanError::InvalidVersion`. diff --git a/doc/developer/generated/sql/normalize.md b/doc/developer/generated/sql/normalize.md index 1e04050563033..95b2361442aab 100644 --- a/doc/developer/generated/sql/normalize.md +++ b/doc/developer/generated/sql/normalize.md @@ -1,9 +1,9 @@ --- source: src/sql/src/normalize.rs -revision: 68852854f4 +revision: 39dcae2fba --- # mz-sql::normalize Converts loosely-typed AST nodes (identifiers, unresolved names, option lists) into structured Rust types used by the planner. -Key functions: `ident` / `ident_ref` (case-folding), `unresolved_item_name` / `unresolved_schema_name` (converts multi-part names to `PartialItemName`/`PartialSchemaName`), `create_statement` (canonicalizes `CREATE` statements for catalog storage, returning `PlanError::Internal` for unexpected statement types), and the `generate_extracted_config!` macro used throughout the crate to parse `WITH` option blocks. +Key functions: `ident` / `ident_ref` (case-folding), `unresolved_item_name` / `unresolved_schema_name` (converts multi-part names to `PartialItemName`/`PartialSchemaName`), `create_statement` (canonicalizes `CREATE` statements for catalog storage, returning `PlanError::Internal` for unexpected statement types; handles `CREATE METRIC SINK` via the `CreateMetricSink` arm), and the `generate_extracted_config!` macro used throughout the crate to parse `WITH` option blocks. diff --git a/doc/developer/generated/sql/plan.md b/doc/developer/generated/sql/plan.md index 0037fc402a383..68ffff2936a00 100644 --- a/doc/developer/generated/sql/plan.md +++ b/doc/developer/generated/sql/plan.md @@ -1,6 +1,6 @@ --- source: src/sql/src/plan.rs -revision: 447da2b53e +revision: 39dcae2fba --- # mz-sql::plan @@ -8,6 +8,7 @@ revision: 447da2b53e Defines the `Plan` enum and all plan-specific data types produced by the SQL planner and consumed by the adapter. The file contains ~2000 lines of type definitions covering every statement kind (DDL, DML, ACL, SCL, TCL) plus shared context types (`PlanContext`, `QueryContext`, `Params`, `QueryLifetime`). `SubscribeFrom::Query` carries an `HirRelationExpr` (not a `MirRelationExpr`); decorrelation happens downstream. +`CreateMetricSinkPlan` carries `name: QualifiedItemName`, `metric_sink: MetricSink`, and `if_not_exists: bool`. `MetricSink` holds `create_sql`, `from: GlobalId` (the collection the sink reads), `cluster_id: ClusterId`, and `prefix: String` (the Prometheus metric name prefix). `DropObjectsPlan` covers `MetricSink` via the `ObjectType::MetricSink` variant. `TryFromValue` is re-exported from the `with_options` submodule for callers that need to convert `WithOptionValue` items outside the planner. The module layout is documented inline: `handle_statement` (in `statement`) is the entry point; `SELECT` queries flow through `query`; all plans involve `hir` + `lowering`; supporting utilities live in `error`, `notice`, `literal`, `plan_utils`, `scope`, `with_options`, `explain`, and `typeconv`. `ConnectionDetails` includes a `Gcp(GcpConnection)` variant for GCP connections, and a `GlueSchemaRegistry(GlueSchemaRegistryConnection)` variant for AWS Glue Schema Registry connections. diff --git a/doc/developer/generated/sql/plan/_module.md b/doc/developer/generated/sql/plan/_module.md index b607d58171ac9..e32d2f03d2f8d 100644 --- a/doc/developer/generated/sql/plan/_module.md +++ b/doc/developer/generated/sql/plan/_module.md @@ -1,12 +1,12 @@ --- source: src/sql/src/plan.rs -revision: 447da2b53e +revision: 39dcae2fba --- # mz-sql::plan Defines all `Plan` variants produced by the SQL planner and consumed by the adapter, plus the shared infrastructure for planning. -Key types exported from `plan.rs`: `Plan` (the top-level enum covering every statement kind), `PlanContext`, `QueryContext`, `QueryLifetime`, `Params`, and dozens of plan-specific structs (`CreateSourcePlan`, `SelectPlan`, `SubscribePlan`, etc.). +Key types exported from `plan.rs`: `Plan` (the top-level enum covering every statement kind), `PlanContext`, `QueryContext`, `QueryLifetime`, `Params`, and dozens of plan-specific structs (`CreateSourcePlan`, `SelectPlan`, `SubscribePlan`, `CreateMetricSinkPlan`, etc.). `SubscribeFrom::Query` carries an `HirRelationExpr` (not a `MirRelationExpr`); lowering to MIR happens downstream of planning. `TryFromValue` is re-exported from the `with_options` submodule for callers that need to convert `WithOptionValue` items outside the planner. The module is organized into: `query` (query planning), `hir` + `lowering` (HIR IR and HIR→MIR translation), `statement` (statement dispatch), `func` helpers (`typeconv`, `side_effecting_func`), `transform_ast`/`transform_hir` (rewrites), and supporting utilities (`error`, `notice`, `literal`, `plan_utils`, `scope`, `with_options`, `explain`, `virtual_syntax`). diff --git a/doc/developer/generated/sql/plan/hir.md b/doc/developer/generated/sql/plan/hir.md index 9e8d9e395daf2..453956aa3ab4e 100644 --- a/doc/developer/generated/sql/plan/hir.md +++ b/doc/developer/generated/sql/plan/hir.md @@ -1,6 +1,6 @@ --- source: src/sql/src/plan/hir.rs -revision: 5b0bee10fd +revision: 449b2eead4 --- # mz-sql::plan::hir @@ -12,3 +12,4 @@ Key types include `ColumnRef` (a leveled column reference), `JoinKind` (inner/le `relation_node_count` counts the number of `HirRelationExpr` nodes in an expression tree (including those reachable through scalar subqueries) as a structural size metric for comparing two expressions against each other; scalar nodes themselves are not counted. `HirScalarExpr` implements `VisitChildren` to expose its immediate subquery bodies (`Exists`/`Select` arms) without descending further; this is the asymmetric counterpart of `VisitChildren` on `HirRelationExpr`, which traverses scalars into subqueries at any depth. HIR is converted to MIR via `HirRelationExpr::lower()`, which is defined in `plan::lowering` as an `impl HirRelationExpr` method. +The deprecated `visit_mut_fallible` method is wrapped in `stack::maybe_grow` so that traversals over user-controlled-depth relation trees (long JOIN chains, deep CTE nesting) do not overflow the stack. The deprecated `visit_fallible` method is similarly guarded. `split_subquery_predicates` (in `transform_hir`) uses `visit_mut_fallible` internally, so the guard applies there too. diff --git a/doc/developer/generated/sql/plan/query.md b/doc/developer/generated/sql/plan/query.md index 239a7596be610..32156202a3042 100644 --- a/doc/developer/generated/sql/plan/query.md +++ b/doc/developer/generated/sql/plan/query.md @@ -1,6 +1,6 @@ --- source: src/sql/src/plan/query.rs -revision: 0b5864836a +revision: 9dcba0f701 --- # mz-sql::plan::query @@ -16,3 +16,4 @@ When a `JOIN ... USING (col) AS alias` is planned via `plan_using_constraint`, t `plan_nested_query` treats a nested query as an unordered relation: its `ORDER BY` is dropped rather than materialized into a `TopK` unless combined with a `LIMIT`/`OFFSET` clause. This diverges from PostgreSQL, where order-sensitive aggregates observe a sorted subquery's output as an executor artifact. Callers that need a specific aggregation order must use the in-aggregate `agg(value ORDER BY ...)` form. `INTERSECT` planning calls `HirRelationExpr::relation_node_count` on both inputs and places the smaller one on the left. Because the planner duplicates the left input, a left-deep INTERSECT chain without this swap would produce exponential plan size; with it, plan size is bounded at O(n^log2(3)). `MAX_TYPE_NESTING_DEPTH` (128) and `MAX_TYPE_RESOLUTION_NODES` (100,000) are constants bounding custom type resolution. `scalar_type_from_catalog` delegates to `scalar_type_from_catalog_inner` via `TypeResolutionBudget`, rejecting custom types that are nested deeper than `MAX_TYPE_NESTING_DEPTH` or that require more than `MAX_TYPE_RESOLUTION_NODES` sub-type resolutions, preventing stack overflow and memory exhaustion from pathological types. +When planning a `SELECT` with table functions in the SELECT list alongside a `GROUP BY`, aggregates, or `HAVING`, table functions are joined on top of the reduce (after GROUP BY and aggregation), not before it. The planner speculatively joins the table function before the reduce so that its columns are in scope during SELECT list expansion and GROUP BY planning, then checks whether any group key or aggregate actually references the table function columns. If none do, the join is rolled back, the reduce is applied on the pre-join relation, and Step 8.5 re-plans the table function join on top of the reduce output, rebinding column references to the reduced relation. diff --git a/doc/developer/generated/sql/plan/statement.md b/doc/developer/generated/sql/plan/statement.md index 0a558c621e624..525fde332d5cd 100644 --- a/doc/developer/generated/sql/plan/statement.md +++ b/doc/developer/generated/sql/plan/statement.md @@ -1,6 +1,6 @@ --- source: src/sql/src/plan/statement.rs -revision: 234d77f6b5 +revision: 39dcae2fba --- # mz-sql::plan::statement diff --git a/doc/developer/generated/sql/plan/statement/_module.md b/doc/developer/generated/sql/plan/statement/_module.md index 1c2c3c24502a9..fc271deeb2c54 100644 --- a/doc/developer/generated/sql/plan/statement/_module.md +++ b/doc/developer/generated/sql/plan/statement/_module.md @@ -1,6 +1,6 @@ --- source: src/sql/src/plan/statement.rs -revision: 234d77f6b5 +revision: 39dcae2fba --- # mz-sql::plan::statement diff --git a/doc/developer/generated/sql/plan/statement/acl.md b/doc/developer/generated/sql/plan/statement/acl.md index b3683aa9248bd..487263bd47552 100644 --- a/doc/developer/generated/sql/plan/statement/acl.md +++ b/doc/developer/generated/sql/plan/statement/acl.md @@ -1,6 +1,6 @@ --- source: src/sql/src/plan/statement/acl.rs -revision: 4fd96c4a39 +revision: 39dcae2fba --- # mz-sql::plan::statement::acl @@ -8,3 +8,4 @@ revision: 4fd96c4a39 Plans access-control statements: `GRANT`/`REVOKE` privileges, `GRANT`/`REVOKE` role membership, `ALTER DEFAULT PRIVILEGES`, `ALTER OWNER`, and `REASSIGN OWNED`. Each statement has a `describe_*` function (returns `StatementDesc`) and a `plan_*` function (returns a `Plan` variant such as `GrantPrivilegesPlan` or `AlterOwnerPlan`). `GRANT/REVOKE ALL` now includes network policies: `plan_update_privilege` iterates over `scx.catalog.get_network_policies()` and maps each to `ObjectId::NetworkPolicy`. +`ObjectType::MetricSink` is grouped with `Sink`, `ClusterReplica`, `Role`, and `Func` in the `ALTER DEFAULT PRIVILEGES` validation: metric sinks do not have privileges and produce an error if named in that context. diff --git a/doc/developer/generated/sql/plan/statement/ddl.md b/doc/developer/generated/sql/plan/statement/ddl.md index 1da003762df22..7d23e5ef38b5d 100644 --- a/doc/developer/generated/sql/plan/statement/ddl.md +++ b/doc/developer/generated/sql/plan/statement/ddl.md @@ -1,6 +1,6 @@ --- source: src/sql/src/plan/statement/ddl.rs -revision: 4868442183 +revision: 39dcae2fba --- # mz-sql::plan::statement::ddl @@ -28,3 +28,4 @@ The `iceberg_sink_builder` function accepts an optional `storage_connection: Opt `AvroSchema::Glue { connection, seed, .. }` in a source format is fully planned: the connection is resolved and must be a `Connection::GlueSchemaRegistry` item, and the `seed` must be present (populated by purification); a missing seed produces the error `"Avro Glue seed resolution has not been performed"`. The resulting `Schema` carries `wire_format: WireFormat::Glue { registry: Some(glue_connection_id) }`. `get_encoding_inner` builds the `Schema::wire_format` field directly inside each `AvroSchema` match arm rather than from a legacy `(csr_connection, confluent_wire_format)` pair. For `AvroSchema::InlineSchema`, the `CONFLUENT WIRE FORMAT` option controls whether `WireFormat::Confluent { registry: None }` or `WireFormat::None` is produced. For `AvroSchema::Csr`, the error message for a non-CSR connection item now reads `"is not a Confluent Schema Registry connection"`. `generate_extracted_config!(GlueAvroOption, ...)` is generated at module level, producing `GlueAvroOptionExtracted` with fields `schema_name`, `key_schema_name`, `value_schema_name`, `key_compatibility_level`, and `value_compatibility_level` (all `Option`); this struct is used in both planning and purification. `kafka_sink_builder` handles `Format::Avro(AvroSchema::Glue { ... })` for Kafka sinks: it resolves the connection (must be a `Connection::GlueSchemaRegistry` item), rejects `SCHEMA NAME` (source-only, replaced by per-side options), rejects key-side options without a corresponding KEY, parses the compatibility level via `mz_ccsr::CompatibilityLevel`, generates the Avro schema via `AvroSchemaGenerator`, and returns `KafkaSinkFormatType::Avro` with `wire_format: WireFormat::Glue { registry: Some(glue_connection_id) }`. `plan_create_type` validates nested type references using a shared `TypeResolutionBudget`, rejecting types that exceed `MAX_TYPE_NESTING_DEPTH` (128) or `MAX_TYPE_RESOLUTION_NODES` (100,000) at `CREATE TYPE` time with graceful planning errors. Map key types are validated under a separate per-root budget (since the key type is not part of the resolved type tree at runtime), while the value type and record fields draw from the function's main budget. +`plan_create_metric_sink` plans `CREATE METRIC SINK` statements (gated by `ENABLE_METRIC_SINK`). The planner validates that the `FROM` relation exposes five required columns: `metric_name` (`String`), `metric_type` (`String`), `labels` (`Map`), `value` (`Float64`), and `help` (`String`). The `PREFIX` option is required and must be a non-empty string whose first character is an ASCII letter, underscore, or colon, and whose remaining characters are ASCII alphanumeric, underscore, or colon (a valid start of a Prometheus metric family name). `relation_desc()` returning `None` on the `FROM` item doubles as the rejection filter for item types that cannot be read from (such as indexes). The resulting `CreateMetricSinkPlan` contains a `MetricSink` carrying `create_sql`, `from: GlobalId`, `cluster_id`, and `prefix`. diff --git a/doc/developer/generated/sql/plan/statement/ddl/_module.md b/doc/developer/generated/sql/plan/statement/ddl/_module.md index 5799c3a45ea39..71b842e33f8e8 100644 --- a/doc/developer/generated/sql/plan/statement/ddl/_module.md +++ b/doc/developer/generated/sql/plan/statement/ddl/_module.md @@ -1,6 +1,6 @@ --- source: src/sql/src/plan/statement/ddl.rs -revision: 4868442183 +revision: 39dcae2fba --- # mz-sql::plan::statement::ddl @@ -21,3 +21,4 @@ The `iceberg_sink_builder` function accepts an optional `storage_connection: Opt `plan_alter_sink` handles `AlterSinkAction::SetOptions` and `AlterSinkAction::ResetOptions`, currently restricted to the `CommitInterval` option name. A `SET` identical to the current with-options returns `Plan::AlterNoop`. A `RESET` of an option that is not set is rejected. `iceberg_sink_builder` enforces a minimum `COMMIT INTERVAL` of 1 second; intervals shorter than 1 second produce the error `"COMMIT INTERVAL must be at least 1 second"`. `plan_create_type` validates nested type references using a shared `TypeResolutionBudget`, rejecting types that exceed the nesting depth limit (128) or total resolution node limit (100,000) with graceful planning errors. +`plan_create_metric_sink` plans `CREATE METRIC SINK` (gated by `ENABLE_METRIC_SINK`), validating that the `FROM` relation exposes the five required columns (`metric_name`, `metric_type`, `labels`, `value`, `help`) with the correct types, and that the required `PREFIX` option is a valid start of a Prometheus metric family name. diff --git a/doc/developer/generated/sql/plan/statement/show.md b/doc/developer/generated/sql/plan/statement/show.md index 85f836357e05b..6dab5c4a0c0a4 100644 --- a/doc/developer/generated/sql/plan/statement/show.md +++ b/doc/developer/generated/sql/plan/statement/show.md @@ -1,6 +1,6 @@ --- source: src/sql/src/plan/statement/show.rs -revision: 6c07c975be +revision: 39dcae2fba --- # mz-sql::plan::statement::show @@ -13,3 +13,4 @@ The `ensure_no_from` helper enforces that `FROM` is absent for `SHOW` commands t `plan_show_create_type` uses the fully-qualified type name (schema + item) in the output row rather than just the bare item name, and rejects system types with an error. `humanize_sql_for_show_create` strips the `Version` option from `CREATE SINK` SQL output, since that option does not roundtrip (a SQL string containing `VERSION` cannot be used as-is to recreate the sink). It also strips the internal `DETAILS` option from `CREATE TABLE ... FROM SOURCE` statements, retaining only `TEXT COLUMNS`, `EXCLUDE COLUMNS`, `PARTITION BY`, and `RETAIN HISTORY`; the omitted options are not user-typeable but remain accessible in the raw `create_sql` and `redacted_create_sql` columns of catalog builtins. In addition, column definitions populated during purification (the `Defined` variant of `TableFromSourceColumns`) and constraints are stripped from `CREATE TABLE ... FROM SOURCE` output, since `CREATE TABLE ... FROM SOURCE` rejects column definitions as input; a user-typed `Named` column list is left intact. The full column schema remains available via `SHOW COLUMNS`, `mz_columns`, and the raw `create_sql` column of `mz_catalog.mz_tables`. Role names embedded in `pg_has_role` filter expressions are passed through `escaped_string_literal` to ensure correct SQL quoting. +`CatalogItemType::MetricSink` is grouped with other non-relation item types (indexes, functions, secrets, types, sinks) in `show_columns`: attempting `SHOW COLUMNS` on a metric sink produces the error that it does not have columns. diff --git a/doc/developer/generated/sql/plan/transform_hir.md b/doc/developer/generated/sql/plan/transform_hir.md index 0166e018c1787..5939ffb4c8953 100644 --- a/doc/developer/generated/sql/plan/transform_hir.md +++ b/doc/developer/generated/sql/plan/transform_hir.md @@ -1,11 +1,11 @@ --- source: src/sql/src/plan/transform_hir.rs -revision: b6b85cfcef +revision: 449b2eead4 --- # mz-sql::plan::transform_hir Applies HIR-level rewrites before decorrelation. `split_subquery_predicates` hoists subquery predicates out of conjunctions so that cheaper non-subquery filters are pushed into the outer relation first. -`try_simplify_quantified_comparisons` rewrites `ANY`/`ALL` subquery predicates into semi/anti-join `EXISTS` form; it accepts a `simplify_join_on` flag that also enables this rewrite inside `JOIN ON` clauses. +`try_simplify_quantified_comparisons` rewrites `ANY`/`ALL` subquery predicates into semi/anti-join `EXISTS` form; it accepts a `simplify_join_on` flag that also enables this rewrite inside `JOIN ON` clauses. It short-circuits immediately when the expression tree contains no subquery (the common case), because `walk_relation` recomputes `input.typ()` at every level (O(depth^2) over a deep relation tree) and would stall the coordinator on a query with no quantified comparisons. When a subquery is present, `walk_relation` wraps each recursive call in `stack::maybe_grow` to prevent stack overflow on deep relation trees. `fuse_window_functions` is another pass in this module that merges groups of value window function calls and window aggregations with identical partition/order/frame/options into fused calls, reducing the overhead of the MIR window function pattern. diff --git a/doc/developer/generated/sql/rbac.md b/doc/developer/generated/sql/rbac.md index 2a9463a379785..a85a8c8d538d2 100644 --- a/doc/developer/generated/sql/rbac.md +++ b/doc/developer/generated/sql/rbac.md @@ -1,6 +1,6 @@ --- source: src/sql/src/rbac.rs -revision: 447da2b53e +revision: 39dcae2fba --- # mz-sql::rbac @@ -14,3 +14,4 @@ System users (`mz_system`, `mz_support`) bypass most checks, and individual chec `generate_read_privileges_inner` uses an iterative worklist traversal rather than recursion; view dependency chains are user-controlled and can be arbitrarily deep, so recursion risks a stack overflow. `ALTER ROLE ... SET restrict_to_user_objects` requires superuser: the `PlannedAlterRoleOption::Variable` match arm in `generate_rbac_requirements` enforces this with a case-insensitive name comparison. `Plan::CreateMaterializedView` in `generate_rbac_requirements` requires ownership of both `replace` (the `CREATE OR REPLACE` target, if set) and `materialized_view.replacement_target` (the `FOR ` replacement target, if set). Both are independent and optional; ownership of whichever are present is required, mirroring the ownership check for `ALTER ... APPLY REPLACEMENT` and `CREATE INDEX`. +`Plan::CreateMetricSink` in `generate_rbac_requirements` requires `CREATE` on the target schema, `CREATE` on the cluster, and read privileges on the `FROM` relation (via `generate_read_privileges`). Metric sinks are treated as an egress path for the source relation's contents, so read access on that relation is the guard, not ownership of it. diff --git a/doc/developer/generated/sql/session/vars.md b/doc/developer/generated/sql/session/vars.md index bf789d08cb043..457a2391037bb 100644 --- a/doc/developer/generated/sql/session/vars.md +++ b/doc/developer/generated/sql/session/vars.md @@ -1,6 +1,6 @@ --- source: src/sql/src/session/vars.rs -revision: 38a95cefe2 +revision: 4f0805a4d8 --- # mz-sql::session::vars diff --git a/doc/developer/generated/sql/session/vars/_module.md b/doc/developer/generated/sql/session/vars/_module.md index 2f9632cdc9462..11e2a1e56ca6d 100644 --- a/doc/developer/generated/sql/session/vars/_module.md +++ b/doc/developer/generated/sql/session/vars/_module.md @@ -1,6 +1,6 @@ --- source: src/sql/src/session/vars.rs -revision: 38a95cefe2 +revision: 4f0805a4d8 --- # mz-sql::session::vars diff --git a/doc/developer/generated/sql/session/vars/constraints.md b/doc/developer/generated/sql/session/vars/constraints.md index 0fcea9d6fe8c1..b710729860d49 100644 --- a/doc/developer/generated/sql/session/vars/constraints.md +++ b/doc/developer/generated/sql/session/vars/constraints.md @@ -1,6 +1,6 @@ --- source: src/sql/src/session/vars/constraints.rs -revision: 2b5b078528 +revision: 4f0805a4d8 --- # mz-sql::session::vars::constraints diff --git a/doc/developer/generated/sql/session/vars/definitions.md b/doc/developer/generated/sql/session/vars/definitions.md index 6f9658fc5af82..975149823a736 100644 --- a/doc/developer/generated/sql/session/vars/definitions.md +++ b/doc/developer/generated/sql/session/vars/definitions.md @@ -1,6 +1,6 @@ --- source: src/sql/src/session/vars/definitions.rs -revision: 38a95cefe2 +revision: 39dcae2fba --- # mz-sql::session::vars::definitions @@ -11,5 +11,5 @@ The `feature_flags!` macro accepts an optional `scope: ,` field per flag e The `lazy_value!` and `value!` macros (from `polyfill`) are used extensively to express default values that cannot be computed at compile time. This file is the authoritative source of truth for which variables exist and their defaults. `RESTRICT_TO_USER_OBJECTS` is a read-only `bool` session variable (default `false`) that restricts queries from accessing system catalog objects; it is designed to be set only via `ALTER ROLE ... SET` by superusers. -Feature flags include `enable_repeat_row_non_negative` (guards the `repeat_row_non_negative` table function), `enable_storage_introspection_logs` (guards forwarding storage timely logging events into the compute introspection dataflow), `enable_kafka_broker_matching_rules` (guards `MATCHING` broker rules in `BROKERS` for Kafka PrivateLink connections), `enable_bounded_staleness_isolation` (guards the `bounded staleness ` transaction isolation level; defaults to `true`), `enable_fixed_correlated_cte_lowering` (gates the corrected HIR-to-MIR lowering path that uses CTE-aware branch keys when decorrelating CTEs referenced from nested correlated scopes; defaults to `true`), `enable_statement_arrival_logging` (when enabled, SQL frontends log incoming statements at info level as they arrive, before processing, with literals redacted; intended as an emergency diagnostic for incidents where statements crash the process before reaching the statement log; defaults to `false`), `unsafe_enable_unbounded_custom_type_resolution` (default `false`, `enable_for_item_parsing: true`; when enabled, custom type resolution proceeds without the nesting-depth and total-node limits that ordinarily bound resolution work), `enable_auto_scaling_strategy` (guards the `AUTO SCALING STRATEGY` cluster option; default `true`, `enable_for_item_parsing: true`; `RESET (AUTO SCALING STRATEGY)` is intentionally not gated by this flag so clusters can shed an autoscaling policy after a flag rollback), `unsafe_enable_incomplete_view_column_lists` (default `false`, `enable_for_item_parsing: true`; when enabled, `CREATE VIEW` and `CREATE MATERIALIZED VIEW` accept a column-name list shorter than the query's arity, applying the names as a prefix rename; this flag is force-enabled during bootstrap so views that an earlier version accepted with incomplete column lists continue to re-plan without crashing environmentd), `enable_extended_protocol_implicit_transaction` (default `true`; when enabled, an implicit write transaction started by the extended-query protocol spans the whole pipeline up to the client's Sync message so the pipeline commits or rolls back atomically as in PostgreSQL; disabling is the escape hatch for clients that pipeline statements Materialize cannot run in one transaction, such as a read or DDL after a write, which would otherwise fail rather than silently committing the staged writes), and a set of optimizer feature flags that wire into `OptimizerFeatures` including `enable_less_reduce_in_eqprop`, `enable_dequadratic_eqprop_map`, `enable_eq_classes_withholding_errors`, `enable_fast_path_plan_insights`, `enable_cast_elimination`, `enable_case_literal_transform`, `enable_simplify_quantified_comparisons`, `enable_simplify_from_less_existence`, `enable_coalesce_case_transform`, `enable_will_distinct_propagation`, `disallow_unmaterializable_functions_as_of`, `enable_with_ordinality_legacy_fallback`, `enable_frontend_peek_sequencing`, and `enable_replacement_materialized_views`. +Feature flags include `enable_metric_sink` (guards `CREATE METRIC SINK`; default `false`; `enable_for_item_parsing: true` so that already-created metric sinks remain parseable after a flag rollback, which would otherwise take down the catalog), `enable_repeat_row_non_negative` (guards the `repeat_row_non_negative` table function), `enable_storage_introspection_logs` (guards forwarding storage timely logging events into the compute introspection dataflow), `enable_kafka_broker_matching_rules` (guards `MATCHING` broker rules in `BROKERS` for Kafka PrivateLink connections), `enable_bounded_staleness_isolation` (guards the `bounded staleness ` transaction isolation level; defaults to `true`), `enable_fixed_correlated_cte_lowering` (gates the corrected HIR-to-MIR lowering path that uses CTE-aware branch keys when decorrelating CTEs referenced from nested correlated scopes; defaults to `true`), `enable_statement_arrival_logging` (when enabled, SQL frontends log incoming statements at info level as they arrive, before processing, with literals redacted; intended as an emergency diagnostic for incidents where statements crash the process before reaching the statement log; defaults to `false`), `unsafe_enable_unbounded_custom_type_resolution` (default `false`, `enable_for_item_parsing: true`; when enabled, custom type resolution proceeds without the nesting-depth and total-node limits that ordinarily bound resolution work), `enable_auto_scaling_strategy` (guards the `AUTO SCALING STRATEGY` cluster option; default `true`, `enable_for_item_parsing: true`; `RESET (AUTO SCALING STRATEGY)` is intentionally not gated by this flag so clusters can shed an autoscaling policy after a flag rollback), `unsafe_enable_incomplete_view_column_lists` (default `false`, `enable_for_item_parsing: true`; when enabled, `CREATE VIEW` and `CREATE MATERIALIZED VIEW` accept a column-name list shorter than the query's arity, applying the names as a prefix rename; this flag is force-enabled during bootstrap so views that an earlier version accepted with incomplete column lists continue to re-plan without crashing environmentd), `enable_extended_protocol_implicit_transaction` (default `true`; when enabled, an implicit write transaction started by the extended-query protocol spans the whole pipeline up to the client's Sync message so the pipeline commits or rolls back atomically as in PostgreSQL; disabling is the escape hatch for clients that pipeline statements Materialize cannot run in one transaction, such as a read or DDL after a write, which would otherwise fail rather than silently committing the staged writes), and a set of optimizer feature flags that wire into `OptimizerFeatures` including `enable_less_reduce_in_eqprop`, `enable_dequadratic_eqprop_map`, `enable_eq_classes_withholding_errors`, `enable_fast_path_plan_insights`, `enable_cast_elimination`, `enable_case_literal_transform`, `enable_simplify_quantified_comparisons`, `enable_simplify_from_less_existence`, `enable_coalesce_case_transform`, `enable_will_distinct_propagation`, `disallow_unmaterializable_functions_as_of`, `enable_with_ordinality_legacy_fallback`, `enable_frontend_peek_sequencing`, and `enable_replacement_materialized_views`. `DEFAULT_TIMESTAMP_INTERVAL`, `SSH_CHECK_INTERVAL`, and `STORAGE_STATISTICS_INTERVAL` each carry a `NON_ZERO_DURATION` constraint, so setting any of them to zero raises an `InvalidParameterValue` error. diff --git a/doc/developer/generated/sqllogictest/_crate.md b/doc/developer/generated/sqllogictest/_crate.md index e2f4a85379fda..413d02ed607d8 100644 --- a/doc/developer/generated/sqllogictest/_crate.md +++ b/doc/developer/generated/sqllogictest/_crate.md @@ -1,6 +1,6 @@ --- source: src/sqllogictest/src/lib.rs -revision: 30d929249e +revision: 84f88ca968 --- # sqllogictest diff --git a/doc/developer/generated/sqllogictest/bin/sqllogictest.md b/doc/developer/generated/sqllogictest/bin/sqllogictest.md index c3a61629a872f..fbb61287e8939 100644 --- a/doc/developer/generated/sqllogictest/bin/sqllogictest.md +++ b/doc/developer/generated/sqllogictest/bin/sqllogictest.md @@ -1,6 +1,6 @@ --- source: src/sqllogictest/src/bin/sqllogictest.rs -revision: 38a95cefe2 +revision: 84f88ca968 --- # sqllogictest binary diff --git a/doc/developer/generated/storage-operators/persist_source.md b/doc/developer/generated/storage-operators/persist_source.md index 1de00adf44424..a76d8414fa78f 100644 --- a/doc/developer/generated/storage-operators/persist_source.md +++ b/doc/developer/generated/storage-operators/persist_source.md @@ -1,6 +1,6 @@ --- source: src/storage-operators/src/persist_source.rs -revision: 32c1e1d39 +revision: 95baa04a85 --- # storage-operators::persist_source @@ -11,4 +11,5 @@ It wraps the lower-level `shard_source` from `mz_persist_client`, adding storage Defines `Subtime`, an opaque sub-timestamp token that enables finer-grained frontier progress within a single timestamp, used for granular backpressure flow control. `Subtime` implements `differential_dataflow::lattice::Lattice` (join as max, meet as min), `Maximum` (returning `u64::MAX`), and `columnation::Columnation` (using `CopyRegion`), allowing it to participate in differential dataflow lattice operations and columnar arrangements. The `persist_source_core` function handles the inner scoped dataflow, wiring up `shard_source`, optional `txns_progress`, and the `decode_and_mfp` operator that deserializes `SourceData` and applies remaining MFP logic. +The `decode_and_mfp` operator's inner processing loop audits parts that were selected for `filter_pushdown_audit`: for `SourceData(Ok(row))` rows, if the MFP produces output the part should not have been discarded; for `SourceData(Err(_))` rows, the mere presence of an error row is a pushdown violation since errors must surface regardless of any filter. Both cases log a Sentry-tagged error (redacting the MFP output and the error value to avoid leaking user data) and optionally panic when `STATS_AUDIT_PANIC` is set. A `filter_pushdown_audit` proptest module in the `#[cfg(test)]` block provides end-to-end soundness checks for persist filter pushdown: it builds a part from actual rows, computes the real production column statistics, then asserts that `filter_result` never returns `FilterResult::Discard` for a part whose MFP produces output on some actual row. This exercises the full path from stats derivation through `col_stats` to the interpreter, catching both interpreter bugs and stats-derivation bugs. The module also includes targeted regression tests via `assert_part_kept`, which verifies that a specific predicate keeps a part when real rows match it; one such test covers the case where a float column's `PrimitiveStats` bounds are unordered (e.g., `-NaN` as the lower bound in arrow total order, which sorts above finite values in `OrderedFloat` order), ensuring those bounds are not misread as an empty range and incorrectly discarding the part. diff --git a/doc/developer/generated/storage-operators/stats.md b/doc/developer/generated/storage-operators/stats.md index fd3b86fd3d046..63e565523ae54 100644 --- a/doc/developer/generated/storage-operators/stats.md +++ b/doc/developer/generated/storage-operators/stats.md @@ -1,10 +1,10 @@ --- source: src/storage-operators/src/stats.rs -revision: a55caae279 +revision: 95baa04a85 --- # storage-operators::stats Provides `StatsCursor`, a streaming-consolidating cursor over a persist shard specialized to `RelationDesc`. -It maintains separate sub-cursors for errors and data, using pushdown statistics to skip fetching parts that contain neither errors nor rows matching the provided MFP plan. +It maintains separate sub-cursors for errors and data, using pushdown statistics to skip fetching parts that contain neither errors nor rows matching the provided MFP plan. When part stats are present but fail to decode (e.g., written by a newer version), the cursor treats the part as unconstrained and fetches it (fail open). Errors are yielded before data to match Materialize's standard lookup semantics. diff --git a/doc/developer/generated/storage-types/dyncfgs.md b/doc/developer/generated/storage-types/dyncfgs.md index d8d2b17f95ea5..171d0078b5f09 100644 --- a/doc/developer/generated/storage-types/dyncfgs.md +++ b/doc/developer/generated/storage-types/dyncfgs.md @@ -1,6 +1,6 @@ --- source: src/storage-types/src/dyncfgs.rs -revision: a694e29cb3 +revision: de8eac0d16 --- # storage-types::dyncfgs @@ -14,7 +14,7 @@ Covers: * **AWS**: `AWS_PREFETCH_STS_CONNECT_TIMEOUT` — connect timeout for the AssumeRole credentials prefetcher's STS calls, defaulting to the SDK's 3.1 second default. Read once when a connection is set up; running sinks are unaffected until their dataflow restarts. * **Networking**: `ENFORCE_EXTERNAL_ADDRESSES` — whether to reject private/local addresses when resolving external connections (defaults `false`). * **Kafka**: `KAFKA_CLIENT_ID_ENRICHMENT_RULES` (JSON array of pattern/payload rules for enriching `client.id`), `KAFKA_POLL_MAX_WAIT`, `KAFKA_LOW_WATERMARK_CHECK` (gates whether Kafka sources check the partition low watermark and error when the start offset/resume upper has been compacted away), `KAFKA_DEFAULT_AWS_PRIVATELINK_ENDPOINT_IDENTIFICATION_ALGORITHM` (defaults to `"none"`), `KAFKA_BUFFERED_EVENT_RESIZE_THRESHOLD_ELEMENTS`, `KAFKA_RETRY_BACKOFF`, `KAFKA_RETRY_BACKOFF_MAX`, `KAFKA_RECONNECT_BACKOFF`, `KAFKA_RECONNECT_BACKOFF_MAX` (defaults 30 s to reduce reconnection churn), `KAFKA_SINK_MESSAGE_MAX_BYTES`, `KAFKA_SINK_BATCH_SIZE`, and `KAFKA_SINK_BATCH_NUM_MESSAGES`. -* **MySQL**: `MYSQL_REPLICATION_HEARTBEAT_INTERVAL`, `MYSQL_SOURCE_SNAPSHOT_PARALLELISM` (gates PK-range splitting of snapshot reads across workers, defaults `false`), and `MYSQL_SOURCE_SNAPSHOT_EXACT_COUNT_MAX_ROWS` (tables whose optimizer row estimate exceeds this threshold report the estimate for the snapshot size gauge rather than running `COUNT(*)`; defaults to 1,000,000). +* **MySQL**: `MYSQL_REPLICATION_HEARTBEAT_INTERVAL`, `MYSQL_SOURCE_SNAPSHOT_PARALLELISM` (gates PK-range splitting of snapshot reads across workers, defaults `false`), `MYSQL_SOURCE_SNAPSHOT_EXACT_COUNT_MAX_ROWS` (tables whose optimizer row estimate exceeds this threshold report the estimate for the snapshot size gauge rather than running `COUNT(*)`; defaults to 1,000,000), `MYSQL_SOURCE_SNAPSHOT_PARTITION_MIN_ROWS` (smallest estimated row count the snapshot partitioner attempts to subdivide; defaults to 50,000), and `MYSQL_SOURCE_SNAPSHOT_PARTITION_PROBED_PREFIXES_PER_BILLION_ROWS` (cap on PK-prefix probes per billion estimated rows; limits time spent on high-cardinality partitioning and yields coarser boundaries when exhausted; defaults to 1,000). * **PostgreSQL**: `PG_FETCH_SLOT_RESUME_LSN_INTERVAL`, `PG_SCHEMA_VALIDATION_INTERVAL`, and `PG_SOURCE_VALIDATE_TIMELINE` (whether a timeline switch is treated as a definite error, defaults `true`). * **SQL Server**: `SQL_SERVER_SOURCE_VALIDATE_RESTORE_HISTORY` (whether a restore-history change is treated as a definite error, defaults `true`), plus additional SQL Server configs registered from `crate::sources::sql_server` (`CDC_CLEANUP_CHANGE_TABLE`, `CDC_CLEANUP_CHANGE_TABLE_MAX_DELETES`, `MAX_LSN_WAIT`, `SNAPSHOT_PROGRESS_REPORT_INTERVAL`). * **Upsert**: `STORAGE_UPSERT_PREVENT_SNAPSHOT_BUFFERING` (defaults `true`), `STORAGE_ROCKSDB_USE_MERGE_OPERATOR` (defaults `true`), `STORAGE_UPSERT_MAX_SNAPSHOT_BATCH_BUFFERING`, `ENABLE_UPSERT_PAGED_SPILL` (allows the upsert-v2 stash to spill chunks to the shared buffer pool, defaults `false`), `STORAGE_USE_CONTINUAL_FEEDBACK_UPSERT` (defaults `true`), and `ENABLE_UPSERT_V2` (defaults `false`). diff --git a/doc/developer/generated/storage-types/errors.md b/doc/developer/generated/storage-types/errors.md index 253777d603dcf..427d55e234ff3 100644 --- a/doc/developer/generated/storage-types/errors.md +++ b/doc/developer/generated/storage-types/errors.md @@ -1,6 +1,6 @@ --- source: src/storage-types/src/errors.rs -revision: f4650d00b +revision: f0cdedca04 --- # storage-types::errors diff --git a/doc/developer/generated/storage-types/sources/casts.md b/doc/developer/generated/storage-types/sources/casts.md index 66bfb25edce83..3f740aaf5a63a 100644 --- a/doc/developer/generated/storage-types/sources/casts.md +++ b/doc/developer/generated/storage-types/sources/casts.md @@ -1,6 +1,6 @@ --- source: src/storage-types/src/sources/casts.rs -revision: 32c1e1d39 +revision: 1c250c3597 --- # storage-types::sources::casts diff --git a/doc/developer/generated/storage-types/stats.md b/doc/developer/generated/storage-types/stats.md index 4ea100721704c..aa6c2ab751e39 100644 --- a/doc/developer/generated/storage-types/stats.md +++ b/doc/developer/generated/storage-types/stats.md @@ -1,10 +1,13 @@ --- source: src/storage-types/src/stats.rs -revision: c94586e255 +revision: 95baa04a85 --- # storage-types::stats Defines `RelationPartStats`, which bridges persist's `PartStats` to the `ResultSpec`-based filter pushdown used by `mz-expr`. `may_match_mfp` evaluates a `MapFilterProject` against column-level statistics to decide whether a persist part can be skipped entirely during reads. +Before evaluating column predicates, `may_match_mfp` consults `err_count()`: if the error count is absent (missing or malformed err-column stats) or nonzero, the part is kept unconditionally, matching the storage read path's `filter_result` behavior. Only parts that are confirmed error-free proceed to column-level pushdown. +`err_count` derives the error count from the `err` column's null-count statistics: the null count gives the number of ok rows, and subtracting from the total gives errors. Wrong-shaped err-column stats (`try_as_optional_bytes` fails) or an ok count that exceeds the part length both return `None`, which fails open to keeping the part. +Column stats are accessed through two paths: `col_values` reads scalar bounds directly from the ok struct column, and `col_json` supplements those with JSON-specific range narrowing for `Jsonb` columns. Both paths treat mismatched or corrupt column shapes as unknown range rather than panicking, incrementing the `PartStatsMetrics::mismatched_count` counter and returning `None`. This is the primary mechanism for predicate pushdown into persist for storage collections. diff --git a/doc/developer/generated/storage/sink/kafka.md b/doc/developer/generated/storage/sink/kafka.md index e9f76dfac5ca4..abfc8aa55e041 100644 --- a/doc/developer/generated/storage/sink/kafka.md +++ b/doc/developer/generated/storage/sink/kafka.md @@ -1,6 +1,6 @@ --- source: src/storage/src/sink/kafka.rs -revision: 253293ef87 +revision: d9bb748bcd --- # mz-storage::sink::kafka diff --git a/doc/developer/generated/storage/source/mysql/snapshot.md b/doc/developer/generated/storage/source/mysql/snapshot.md index be4f403669a3f..c8b5c0b0aaaa4 100644 --- a/doc/developer/generated/storage/source/mysql/snapshot.md +++ b/doc/developer/generated/storage/source/mysql/snapshot.md @@ -1,6 +1,6 @@ --- source: src/storage/src/source/mysql/snapshot.rs -revision: 253293ef87 +revision: de8eac0d16 --- # mz-storage::source::mysql::snapshot @@ -14,7 +14,7 @@ Handles resumption correctly by skipping tables whose outputs have already been The leader's setup is encapsulated in `lock_and_prepare_snapshot`, which: 1. Verifies output schemas against planning-time descriptors via `verify_output_schemas`. -2. Reads row counts and, for tables with a supported single-column primary key, computes PK-range split boundaries via `sample_pk_bounds` (runs concurrently over at most `worker_count` connections). The `mysql_source_snapshot_parallelism` dyncfg disables splitting, putting every table in single-worker fallback mode. For tables whose optimizer estimate exceeds `MYSQL_SOURCE_SNAPSHOT_EXACT_COUNT_MAX_ROWS` (default 1,000,000 rows), `collect_table_statistics` reads the optimizer's row estimate from `information_schema.tables` instead of running `COUNT(*)`; for smaller tables an exact count is used. The counts feed both the PK-range sampling stride and the snapshot size gauge. +2. Reads row counts and, for tables with a supported single-column primary key, computes PK-range split boundaries via `sample_pk_bounds` (runs concurrently over at most `worker_count` connections). The `mysql_source_snapshot_parallelism` dyncfg disables splitting, putting every table in single-worker fallback mode. Splitting is only attempted for tables whose single-column primary key is a `CHAR` or `VARCHAR` column of at most 768 characters with `utf8mb4_bin` collation; tables with integer PKs or other collations fall back to single-worker-per-table mode. The `MYSQL_SOURCE_SNAPSHOT_PARTITION_MIN_ROWS` dyncfg sets the smallest estimated row count the partitioner attempts to subdivide, and `MYSQL_SOURCE_SNAPSHOT_PARTITION_PROBED_PREFIXES_PER_BILLION_ROWS` caps the number of prefix probes, scaled by the estimated row count, to limit time spent on high-cardinality tables. For tables whose optimizer estimate exceeds `MYSQL_SOURCE_SNAPSHOT_EXACT_COUNT_MAX_ROWS` (default 1,000,000 rows), `collect_table_statistics` reads the optimizer's row estimate from `information_schema.tables` instead of running `COUNT(*)`; for smaller tables an exact count is used. The counts feed both the PK-range sampling stride and the snapshot size gauge. 3. Acquires `LOCK TABLES … READ` and reads `@@global.gtid_executed` as the snapshot upper via `lock_tables_and_read_gtid_set`. This helper optionally sets `@@session.lock_wait_timeout` before locking. 4. Broadcasts the resulting `SnapshotInfo` (GTID set, per-table PK boundaries, and any schema-errored outputs) to all workers via a timely feedback loop. diff --git a/doc/developer/generated/storage/source/sql_server/_module.md b/doc/developer/generated/storage/source/sql_server/_module.md index 33010929918ab..8027dbd2b9826 100644 --- a/doc/developer/generated/storage/source/sql_server/_module.md +++ b/doc/developer/generated/storage/source/sql_server/_module.md @@ -1,6 +1,6 @@ --- source: src/storage/src/source/sql_server.rs -revision: 5427dc5764 +revision: 946b68f676 --- # mz-storage::source::sql_server diff --git a/doc/developer/generated/storage/source/sql_server/progress.md b/doc/developer/generated/storage/source/sql_server/progress.md index 946596eacc828..9d99efa4dfb78 100644 --- a/doc/developer/generated/storage/source/sql_server/progress.md +++ b/doc/developer/generated/storage/source/sql_server/progress.md @@ -1,10 +1,11 @@ --- source: src/storage/src/source/sql_server/progress.rs -revision: af5f8f85e1 +revision: 946b68f676 --- # mz-storage::source::sql_server::progress Implements the progress-tracking operator for SQL Server CDC sources. On startup, validates the upstream restore history ID against the value stored in `SqlServerSourceExtras`; if a restore is detected and `SQL_SERVER_SOURCE_VALIDATE_RESTORE_HISTORY` is enabled, the operator exits early so the replication operator's definite error can propagate downstream. -Periodically probes the upstream server for its current maximum LSN to compute `offset_known`, listens to the resume-upper stream to update `offset_committed`, and optionally cleans up already-ingested rows from the upstream change tables when `CDC_CLEANUP_CHANGE_TABLE` is enabled. +On startup, fetches the upstream maximum LSN eagerly to seed both `offset_known` and `offset_committed` before entering the main loop: `offset_committed` is seeded from each output's resumption LSN (falling back to the current upstream max LSN when snapshotting), and `offset_known` is set to the current upstream max LSN; this prevents a large bogus ingestion-lag reading during the initial snapshot phase. +Periodically probes the upstream server for its current maximum LSN to update `offset_known`, listens to the resume-upper stream to update `offset_committed` (never regressing below the seeded value), and optionally cleans up already-ingested rows from the upstream change tables when `CDC_CLEANUP_CHANGE_TABLE` is enabled. diff --git a/doc/developer/generated/storage/source/sql_server/replication.md b/doc/developer/generated/storage/source/sql_server/replication.md index 1c30314d4c7d2..9377aefb322d6 100644 --- a/doc/developer/generated/storage/source/sql_server/replication.md +++ b/doc/developer/generated/storage/source/sql_server/replication.md @@ -1,6 +1,6 @@ --- source: src/storage/src/source/sql_server/replication.rs -revision: 253293ef87 +revision: 946b68f676 --- # mz-storage::source::sql_server::replication diff --git a/doc/developer/generated/storage/upsert/_module.md b/doc/developer/generated/storage/upsert/_module.md index 8ab6775936e28..ad78bb1b4d36c 100644 --- a/doc/developer/generated/storage/upsert/_module.md +++ b/doc/developer/generated/storage/upsert/_module.md @@ -1,6 +1,6 @@ --- source: src/storage/src/upsert.rs -revision: 7ffd40e1ab +revision: f0cdedca04 --- # mz-storage::upsert diff --git a/doc/developer/generated/testdrive/action/_module.md b/doc/developer/generated/testdrive/action/_module.md index 03f90d3b7e9bd..9e05000322784 100644 --- a/doc/developer/generated/testdrive/action/_module.md +++ b/doc/developer/generated/testdrive/action/_module.md @@ -1,6 +1,6 @@ --- source: src/testdrive/src/action.rs -revision: fd1dd6e62b +revision: 84f88ca968 --- # testdrive::action diff --git a/doc/developer/generated/testdrive/action/consistency.md b/doc/developer/generated/testdrive/action/consistency.md index 1dab3820c1658..1c6ca93e5b564 100644 --- a/doc/developer/generated/testdrive/action/consistency.md +++ b/doc/developer/generated/testdrive/action/consistency.md @@ -1,6 +1,6 @@ --- source: src/testdrive/src/action/consistency.rs -revision: 5b2cefc829 +revision: 84f88ca968 --- # testdrive::action::consistency diff --git a/doc/developer/generated/timely-util/column_pager/_module.md b/doc/developer/generated/timely-util/column_pager/_module.md index c48f6d466e860..e15477b7b0c7f 100644 --- a/doc/developer/generated/timely-util/column_pager/_module.md +++ b/doc/developer/generated/timely-util/column_pager/_module.md @@ -1,6 +1,6 @@ --- source: src/timely-util/src/column_pager.rs -revision: 5d6bfccaf8 +revision: f0cdedca04 --- # timely-util::column_pager diff --git a/doc/developer/generated/timely-util/columnar/_module.md b/doc/developer/generated/timely-util/columnar/_module.md index 49d82a2e2bae9..6c661394abb4a 100644 --- a/doc/developer/generated/timely-util/columnar/_module.md +++ b/doc/developer/generated/timely-util/columnar/_module.md @@ -1,12 +1,13 @@ --- source: src/timely-util/src/columnar.rs -revision: 26305d8cb0 +revision: f0cdedca04 --- # timely-util::columnar Provides the `Column` container type and related infrastructure for storing columnar data in timely dataflow pipelines. `Column` is a tri-variant enum (`Typed`, `Bytes`, `Align`) that can hold data as a typed columnar container, raw Timely network bytes, or a u64-aligned heap allocation; it implements timely's `ContainerBytes`, `DrainContainer`, `PushInto`, `Accountable`, and `SizableContainer` traits. +`Column::is_empty()` returns whether the column holds no records; `Typed` answers directly from the container while `Bytes` and `Align` reconstruct a borrow. The `SizableContainer::at_capacity` implementation uses `at_serialized_capacity`, a shared helper (also used by `ColumnBuilder`) that returns `true` once the serialized size reaches 10% under the 2 MiB (`SHIP_WORDS`) threshold. The signal is monotone: once fired it stays fired, aligning chunk-size decisions between the builder and merger paths. The `batcher` submodule provides `Chunker` and `ColumnChunker` for sorting and consolidating columnar updates, the `builder` submodule provides `ColumnBuilder` for assembling batched aligned allocations, the `builder_input` submodule provides input container types for column builders, the `chunk` submodule provides `ColumnChunk`, a pool-spillable chunk type over `Column`-shaped sorted updates, the `consolidate` submodule provides consolidation utilities for columnar containers, the `merge_batcher` submodule provides `ColumnMergeBatcher`, a pager-aware merge batcher, and the `unload` submodule provides `UnloadChunk` and `UnloadBatch`, a bulk-read interface for extracting updates matching a sorted probe set from a chunk or batch without holding a reference into pool memory across calls. `Col2ValBatcher` and `Col2KeyBatcher` type aliases tie these pieces together, using `MergeBatcher>` where `ColInternalMerger` is the merge strategy from `mz_timely_util::columnation`. diff --git a/doc/developer/generated/timely-util/columnar/batcher.md b/doc/developer/generated/timely-util/columnar/batcher.md index 6dbb8f39807a4..8ca1ffae7892f 100644 --- a/doc/developer/generated/timely-util/columnar/batcher.md +++ b/doc/developer/generated/timely-util/columnar/batcher.md @@ -1,6 +1,6 @@ --- source: src/timely-util/src/columnar/batcher.rs -revision: 4bd76235f0 +revision: f0cdedca04 --- # timely-util::columnar::batcher diff --git a/doc/developer/generated/timely-util/columnar/chunk.md b/doc/developer/generated/timely-util/columnar/chunk.md index 7bc25b0add27c..60d1dea817f0a 100644 --- a/doc/developer/generated/timely-util/columnar/chunk.md +++ b/doc/developer/generated/timely-util/columnar/chunk.md @@ -1,6 +1,6 @@ --- source: src/timely-util/src/columnar/chunk.rs -revision: ea7b8b595a +revision: f0cdedca04 --- # timely-util::columnar::chunk diff --git a/doc/developer/generated/transform/analysis/_module.md b/doc/developer/generated/transform/analysis/_module.md index fa74519455566..7709ca584012a 100644 --- a/doc/developer/generated/transform/analysis/_module.md +++ b/doc/developer/generated/transform/analysis/_module.md @@ -1,6 +1,6 @@ --- source: src/transform/src/analysis.rs -revision: 5d046b3ab6 +revision: f0cdedca04 --- # mz-transform::analysis diff --git a/doc/developer/generated/transform/equivalence_propagation.md b/doc/developer/generated/transform/equivalence_propagation.md index f2d287106d70b..94e104075a503 100644 --- a/doc/developer/generated/transform/equivalence_propagation.md +++ b/doc/developer/generated/transform/equivalence_propagation.md @@ -1,6 +1,6 @@ --- source: src/transform/src/equivalence_propagation.rs -revision: 5d046b3ab6 +revision: 84f88ca968 --- # mz-transform::equivalence_propagation diff --git a/doc/developer/generated/txn-wal/txn_cache.md b/doc/developer/generated/txn-wal/txn_cache.md index 7840f1cedaa3c..3677bd401f840 100644 --- a/doc/developer/generated/txn-wal/txn_cache.md +++ b/doc/developer/generated/txn-wal/txn_cache.md @@ -1,6 +1,6 @@ --- source: src/txn-wal/src/txn_cache.rs -revision: a55caae279 +revision: f0cdedca04 --- # mz-txn-wal::txn_cache diff --git a/doc/user/content/self-managed-deployments/deployment-guidelines/aws-deployment-guidelines.md b/doc/user/content/self-managed-deployments/deployment-guidelines/aws-deployment-guidelines.md index 550139e2f320e..f4efa9c9f7625 100644 --- a/doc/user/content/self-managed-deployments/deployment-guidelines/aws-deployment-guidelines.md +++ b/doc/user/content/self-managed-deployments/deployment-guidelines/aws-deployment-guidelines.md @@ -47,10 +47,10 @@ For the RDS PostgreSQL metadata database, we recommend: - **Multi-AZ** for production. - **gp3** storage. -| Deployment size | Instance | vCPU / memory | Continuously-active objects (~60% CPU) | -|---|---|---|---| -| Entry / small production | `db.r6g.large` | 2 / 16 GiB | ~4,500 | -| Recommended default | `db.r6g.2xlarge` | 8 / 64 GiB | ~18,000 | +| Deployment size | Instance | vCPU / memory | Storage | Provisioned IOPS | Continuously-active objects (~60% CPU) | +|---|---|---|---|---|---| +| Entry / small production | `db.r6g.large` | 2 / 16 GiB | 200 GiB | 3,000 (baseline) | ~4,500 | +| Recommended default | `db.r6g.2xlarge` | 8 / 64 GiB | 400 GiB | 6,000 | ~18,000 | ## TLS diff --git a/doc/user/content/self-managed-deployments/deployment-guidelines/azure-deployment-guidelines.md b/doc/user/content/self-managed-deployments/deployment-guidelines/azure-deployment-guidelines.md index a83bc08a91584..2ca6fd39c834a 100644 --- a/doc/user/content/self-managed-deployments/deployment-guidelines/azure-deployment-guidelines.md +++ b/doc/user/content/self-managed-deployments/deployment-guidelines/azure-deployment-guidelines.md @@ -69,11 +69,13 @@ database, we recommend: - The **Memory Optimized** tier (E-series), which provides the 1:8 vCore-to-memory ratio recommended for the metadata database. - **Zone-redundant high availability** for production. +- **Premium SSD v2** storage, which includes 3,000 IOPS and 125 MB/s at any + size. -| Deployment size | `sku_name` | vCores / memory | Continuously-active objects (~60% CPU) | -|---|---|---|---| -| Entry / small production | `MO_Standard_E4ds_v5` | 4 / 32 GiB | ~4,500 | -| Recommended default | `MO_Standard_E16ds_v5` | 16 / 128 GiB | ~18,000 | +| Deployment size | `sku_name` | vCores / memory | Storage | Provisioned IOPS | Continuously-active objects (~60% CPU) | +|---|---|---|---|---|---| +| Entry / small production | `MO_Standard_E4ds_v5` | 4 / 32 GiB | 128 GiB | 3,000 (included) | ~4,500 | +| Recommended default | `MO_Standard_E16ds_v5` | 16 / 128 GiB | 512 GiB | 6,000 | ~18,000 | ## TLS diff --git a/doc/user/content/self-managed-deployments/deployment-guidelines/gcp-deployment-guidelines.md b/doc/user/content/self-managed-deployments/deployment-guidelines/gcp-deployment-guidelines.md index 9e196370239f2..ed8c39e9412cc 100644 --- a/doc/user/content/self-managed-deployments/deployment-guidelines/gcp-deployment-guidelines.md +++ b/doc/user/content/self-managed-deployments/deployment-guidelines/gcp-deployment-guidelines.md @@ -100,11 +100,13 @@ recommend: metadata database. Avoid shared-core machine types (`db-f1-micro`, `db-g1-small`) in production. - A **regional (highly available)** configuration for production. +- **SSD** storage. IOPS and throughput cannot be configured independently: they + scale with the provisioned size at 30 IOPS and 0.48 MB/s per GB. -| Deployment size | `tier` | vCPU / memory | Continuously-active objects (~60% CPU) | -|---|---|---|---| -| Entry / small production | `db-perf-optimized-N-4` | 4 / 32 GB | ~4,500 | -| Recommended default | `db-perf-optimized-N-16` | 16 / 128 GB | ~18,000 | +| Deployment size | `tier` | vCPU / memory | Storage | Provisioned IOPS | Continuously-active objects (~60% CPU) | +|---|---|---|---|---|---| +| Entry / small production | `db-perf-optimized-N-4` | 4 / 32 GB | 200 GB | 6,000 (set by size) | ~4,500 | +| Recommended default | `db-perf-optimized-N-16` | 16 / 128 GB | 500 GB | 15,000 (set by size) | ~18,000 | ## TLS diff --git a/doc/user/content/transform-data/patterns/temporal-filters.md b/doc/user/content/transform-data/patterns/temporal-filters.md index 94db8be9d6270..60d11d579343d 100644 --- a/doc/user/content/transform-data/patterns/temporal-filters.md +++ b/doc/user/content/transform-data/patterns/temporal-filters.md @@ -290,6 +290,16 @@ Unfortunately, this record does not pass the filter and is excluded from process In conclusion: if you want to account for late arriving data up to some given time duration, you must adjust your temporal filter to allow for such records to make an appearance in the result set. This is often referred to as a **grace period**. + + + +{{< hide >}} + ## Temporal filter pushdown All of the queries in the previous examples only return results based on recently-added events. @@ -325,3 +335,5 @@ Some common functions, such as casting from a string to a timestamp, can prevent {{< note >}} See the guide on [partitioning and filter pushdown](/transform-data/patterns/partition-by/) for a **private preview** feature that can make the filter pushdown optimization more predictable. {{< /note >}} + +{{< /hide >}} diff --git a/misc/dbt-materialize/pytest.ini b/misc/dbt-materialize/pytest.ini index b84e28a52fffc..edf66d488db47 100644 --- a/misc/dbt-materialize/pytest.ini +++ b/misc/dbt-materialize/pytest.ini @@ -18,4 +18,5 @@ filterwarnings = ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning ignore:unclosed file .*:ResourceWarning testpaths = + tests/unit tests/adapter diff --git a/misc/dbt-materialize/tests/unit/test_configs.py b/misc/dbt-materialize/tests/unit/test_configs.py new file mode 100644 index 0000000000000..c62d87184afd4 --- /dev/null +++ b/misc/dbt-materialize/tests/unit/test_configs.py @@ -0,0 +1,92 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License in the LICENSE file at the +# root of this repository, or online at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from dbt.adapters.materialize.exceptions import ( + PartitionByConfigError, + RefreshIntervalConfigError, +) +from dbt.adapters.materialize.impl import ( + MaterializeAdapter, + MaterializeIndexConfig, + MaterializeRefreshIntervalConfig, +) + + +class TestIndexConfig: + def test_none(self): + assert MaterializeIndexConfig.parse(None) is None + + def test_columns(self): + index = MaterializeIndexConfig.parse({"columns": ["a", "b"]}) + + assert index.columns == ["a", "b"] + assert index.default is False + assert index.name is None + assert index.cluster is None + + def test_all_fields(self): + index = MaterializeIndexConfig.parse( + { + "columns": ["a"], + "default": True, + "name": "my_idx", + "cluster": "my_cluster", + } + ) + + assert index.default is True + assert index.name == "my_idx" + assert index.cluster == "my_cluster" + + +class TestRefreshIntervalConfig: + def test_none(self): + assert MaterializeRefreshIntervalConfig.parse(None) is None + + def test_every(self): + refresh_interval = MaterializeRefreshIntervalConfig.parse( + {"every": "1 day", "aligned_to": "2024-01-01"} + ) + + assert refresh_interval.every == "1 day" + assert refresh_interval.aligned_to == "2024-01-01" + assert refresh_interval.on_commit is False + + def test_invalid_field_type(self): + with pytest.raises(RefreshIntervalConfigError): + MaterializeRefreshIntervalConfig.parse({"every": 1}) + + +class TestPartitionByConfig: + @pytest.fixture + def adapter(self): + # parse_partition_by only validates the shape of the config, so it does + # not need a connected adapter. + return MaterializeAdapter.__new__(MaterializeAdapter) + + def test_none(self, adapter): + assert adapter.parse_partition_by(None) is None + + def test_empty_list(self, adapter): + assert adapter.parse_partition_by([]) is None + + def test_columns(self, adapter): + assert adapter.parse_partition_by(["a", "b"]) == ["a", "b"] + + @pytest.mark.parametrize("raw", ["a", {"columns": ["a"]}, ["a", 1]]) + def test_invalid(self, adapter, raw): + with pytest.raises(PartitionByConfigError): + adapter.parse_partition_by(raw) diff --git a/misc/dbt-materialize/tests/unit/test_connections.py b/misc/dbt-materialize/tests/unit/test_connections.py new file mode 100644 index 0000000000000..9aa489ac56955 --- /dev/null +++ b/misc/dbt-materialize/tests/unit/test_connections.py @@ -0,0 +1,75 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License in the LICENSE file at the +# root of this repository, or online at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dbt.adapters.materialize.connections import ( + DEFAULT_SESSION_PARAMETERS, + MaterializeCredentials, + _build_options_string, +) + + +def parse_options(options_string): + """Turn a libpq options string back into a dictionary, undoing the escaping + of spaces.""" + parsed = {} + for part in options_string.replace("\\ ", "\x00").split(" "): + key, _, value = part.partition("=") + parsed[key.lstrip("-")] = value.replace("\x00", " ") + return parsed + + +def test_defaults_are_applied(): + options = parse_options(_build_options_string(None, None)) + + assert options == DEFAULT_SESSION_PARAMETERS + + +def test_user_options_win_over_defaults(): + options = parse_options( + _build_options_string({"welcome_message": "on", "cluster": "my_cluster"}, None) + ) + + assert options["welcome_message"] == "on" + assert options["cluster"] == "my_cluster" + # Defaults the user did not override are still there. + assert options["auto_route_catalog_queries"] == "on" + + +def test_search_path_is_appended(): + options = parse_options(_build_options_string(None, "my_schema")) + + assert options["search_path"] == "my_schema" + + +def test_values_with_spaces_are_escaped(): + options_string = _build_options_string({"application_name": "my app"}, None) + + assert "--application_name=my\\ app" in options_string + assert parse_options(options_string)["application_name"] == "my app" + + +def test_credentials_type(): + credentials = MaterializeCredentials( + host="localhost", + port=6875, + database="materialize", + schema="public", + user="materialize", + password="", + ) + + assert credentials.type == "materialize" + assert "cluster" in credentials._connection_keys() + assert "options" in credentials._connection_keys() diff --git a/misc/dbt-materialize/tests/unit/test_relation.py b/misc/dbt-materialize/tests/unit/test_relation.py new file mode 100644 index 0000000000000..2fac6e4ed9ff3 --- /dev/null +++ b/misc/dbt-materialize/tests/unit/test_relation.py @@ -0,0 +1,60 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License in the LICENSE file at the +# root of this repository, or online at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from dbt.adapters.materialize.relation import MaterializeRelation + + +def make_relation(relation_type): + return MaterializeRelation.create( + database="materialize", + schema="public", + identifier="my_relation", + type=relation_type, + ) + + +@pytest.mark.parametrize( + "relation_type,expected", + [ + ("materialized_view", True), + # The legacy materialization name is still supported. + ("materializedview", True), + ("view", False), + ("table", False), + ], +) +def test_is_materialized_view(relation_type, expected): + assert make_relation(relation_type).is_materialized_view is expected + + +@pytest.mark.parametrize( + "relation_type,attribute", + [ + ("source", "is_source"), + ("source_table", "is_source_table"), + ("sink", "is_sink"), + ], +) +def test_materialize_specific_types(relation_type, attribute): + relation = make_relation(relation_type) + + assert getattr(relation, attribute) is True + assert relation.is_view is False + + +def test_relation_max_name_length(): + # Materialize does not have PostgreSQL's 63 character limit. + assert make_relation("view").relation_max_name_length() == 255 diff --git a/misc/python/materialize/checks/all_checks/metric_sink.py b/misc/python/materialize/checks/all_checks/metric_sink.py index 4f51d941260f4..f171edda46f72 100644 --- a/misc/python/materialize/checks/all_checks/metric_sink.py +++ b/misc/python/materialize/checks/all_checks/metric_sink.py @@ -15,9 +15,10 @@ class MetricSink(Check): - """A metric sink ships no dataflow, so surviving a restart is the whole of - what it does. Boot has to re-parse its `create_sql` back into a catalog item - and rebuild the dependency edge to the relation it reads.""" + """Boot has to re-parse a metric sink's `create_sql` back into a catalog + item, rebuild the dependency edge to the relation it reads, and re-render + the sink's dataflow. A sink whose dataflow does not come back publishes no + metrics, so the check probes for the dataflow, not just the item.""" def _can_run(self, e: Executor) -> bool: return self.base_version >= MzVersion.parse_mz("v26.39.0-dev") @@ -32,7 +33,7 @@ def initialize(self) -> Testdrive: > CREATE SCHEMA metric_sink_schema - > CREATE METRIC SINK metric_sink_schema.metric_sink_one FROM metric_sink_view WITH (PREFIX = 'one_') + > CREATE METRIC SINK metric_sink_schema.metric_sink_one FROM metric_sink_view WITH (PREFIX = 'mz_metric_sink_one_') """)) def manipulate(self) -> list[Testdrive]: @@ -42,12 +43,12 @@ def manipulate(self) -> list[Testdrive]: """ > INSERT INTO metric_sink_table VALUES ('b', 'counter', '{x=>z}', 2, 'help b') - > CREATE METRIC SINK metric_sink_two IN CLUSTER quickstart FROM metric_sink_view WITH (PREFIX = 'two_') + > CREATE METRIC SINK metric_sink_two IN CLUSTER quickstart FROM metric_sink_view WITH (PREFIX = 'mz_metric_sink_two_') """, """ > INSERT INTO metric_sink_table VALUES ('c', 'gauge', '{}', 3, 'help c') - > CREATE METRIC SINK IF NOT EXISTS metric_sink_three FROM metric_sink_view WITH (PREFIX = 'three_') + > CREATE METRIC SINK IF NOT EXISTS metric_sink_three FROM metric_sink_view WITH (PREFIX = 'mz_metric_sink_three_') # A sink is never the subject of a rename, but it is a bystander of every # rename of the relation it reads or of the schema it lives in, and both @@ -70,13 +71,13 @@ def validate(self) -> Testdrive: # current probe is a proxy: it confirms the catalog item was re-parsed on # boot without needing a system connection. return Testdrive(dedent(""" - ! CREATE METRIC SINK metric_sink_schema_renamed.metric_sink_one FROM metric_sink_view_renamed WITH (PREFIX = 'one_') + ! CREATE METRIC SINK metric_sink_schema_renamed.metric_sink_one FROM metric_sink_view_renamed WITH (PREFIX = 'mz_metric_sink_one_') contains:metric sink "materialize.metric_sink_schema_renamed.metric_sink_one" already exists - ! CREATE METRIC SINK metric_sink_two FROM metric_sink_view_renamed WITH (PREFIX = 'two_') + ! CREATE METRIC SINK metric_sink_two FROM metric_sink_view_renamed WITH (PREFIX = 'mz_metric_sink_two_') contains:metric sink "materialize.public.metric_sink_two" already exists - ! CREATE METRIC SINK metric_sink_three FROM metric_sink_view_renamed WITH (PREFIX = 'three_') + ! CREATE METRIC SINK metric_sink_three FROM metric_sink_view_renamed WITH (PREFIX = 'mz_metric_sink_three_') contains:metric sink "materialize.public.metric_sink_three" already exists # The FROM edge came back too, so the view is still pinned, under the @@ -86,4 +87,29 @@ def validate(self) -> Testdrive: > SELECT count(*) FROM metric_sink_view_renamed 3 + + # Each re-rendered dataflow registers a collector that stamps + # its own sink id on a frontier gauge, so three distinct ids + # means all three dataflows came back. The registry is only + # sampled every `compute_prometheus_introspection_scrape_interval`, + # so give the scrape room to land. + $ set-sql-timeout duration=60s + + # Every replica has its own registry, so this introspection + # relation can only be read with a replica targeted. + $ set-from-sql var=replica-name + SELECT r.name + FROM mz_catalog.mz_cluster_replicas r + JOIN mz_catalog.mz_clusters c ON c.id = r.cluster_id + WHERE c.name = 'quickstart' + ORDER BY r.name + LIMIT 1 + + > SET cluster_replica = ${replica-name} + + > SELECT count(DISTINCT labels -> 'sink') FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_compute_metric_sink_frontier_ms' + 3 + + > RESET cluster_replica """)) diff --git a/misc/python/materialize/cli/ci_coverage_pr_report.py b/misc/python/materialize/cli/ci_coverage_pr_report.py index 2380d954adccf..51f5783f29d62 100755 --- a/misc/python/materialize/cli/ci_coverage_pr_report.py +++ b/misc/python/materialize/cli/ci_coverage_pr_report.py @@ -25,14 +25,16 @@ # - Negative values indicate that the line has only been covered in unit tests. Coverage = dict[str, OrderedDict[int, int | None]] # Normalizes an absolute lcov SF path to a repo-relative path so it can be -# matched against the keys from find_modified_lines(). Only the buildkite -# alternative yields an actual repo file (starting with `src/`). The external +# matched against the keys from find_modified_lines(). Only the /workspace +# (CI_BUILDER_FIXED_WORKDIR builds) and buildkite alternatives yield an actual +# repo file (starting with `src/`). The external # and rustlib alternatives exist so the assert below does not fire on # dependency sources, they never match a modified repo file. SOURCE_RE = re.compile( r""" ( external/(.*$) | /usr/local/lib/rustlib/(.*$) + | /workspace/(src/.*$) | /var/lib/buildkite-agent/builds/buildkite-.*/materialize/[^/]*/(src/.*$) )""", re.VERBOSE, diff --git a/misc/python/materialize/cli/migrate_gh_to_linear.py b/misc/python/materialize/cli/migrate_gh_to_linear.py new file mode 100644 index 0000000000000..1bf4bda3c0120 --- /dev/null +++ b/misc/python/materialize/cli/migrate_gh_to_linear.py @@ -0,0 +1,382 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +import argparse +import json +import os +import re +import subprocess +import sys +from typing import Any + +import requests + +from materialize import spawn + +GITHUB_REPO = "MaterializeInc/database-issues" +LINEAR_API_URL = "https://api.linear.app/graphql" +LINEAR_MAX_BODY = 100_000 # Linear rejects longer descriptions and comment bodies. + +LABEL_MAP = { + "ci-flake": "CI Flake", + "test-blocker": "Test Blocker", + "release-blocker": "Release Blocker", + "T-security": "Security", + "T-correctness": "correctness", + "C-bug": "Bug", + "C-feature": "Feature", +} + +LABEL_TO_TEAM = { + "A-sql": "SQL", + "A-optimization": "SQL", + "A-coordinator": "SQL", + "A-ADAPTER": "SQL", + "A-storage": "SS", + "T-sources": "SS", + "T-sinks": "SS", + "T-kafka": "SS", + "T-PG": "SS", + "T-mysql": "SS", + "T-debezium": "SS", + "T-s3": "SS", + "T-ICEBERG": "SS", + "T-subsource": "SS", + "T-sqlserver": "SS", + "A-webhooks": "SS", + "A-compute": "CPU", + "A-dataflow": "CPU", + "A-CLUSTER": "CPU", + "A-PERSIST": "DB", + "A-console": "CNS", + "A-infra": "QAR", + "A-build": "QAR", + "T-testing": "QAR", + "A-monitoring": "DB", + "T-observability": "DB", + "A-tracing": "DB", + "A-docs": "DEX", + "A-backend": "DB", + "A-controller": "DB", + "A-platform": "DB", +} + + +GH_TO_LINEAR_USER = { + "aljoscha": "9b290678-4120-4ad4-a69f-df4f1e8e9c64", + "Alphadelta14": "2ddf091f-20aa-4db7-9c09-6efa78470d5f", + "antiguru": "0dcfe0c6-0631-4b4a-b409-91cea985a27b", + "bosconi": "2721aa79-f4f4-43bc-b5d1-0fd4717f6e23", + "DAlperin": "f69e5038-fec2-4b57-9512-7fac8e7387de", + "def-": "6d6e4bba-365d-406e-98a1-2538ffd1a882", + "doy-materialize": "21a8819b-aa96-4c70-946c-ff4af431352d", + "ggevay": "3fd3b7f5-038d-422f-9682-bf38185102d4", + "jubrad": "ba122e7c-6b3f-4daf-98e8-91373cd8bf77", + "martykulma": "18e88ba3-4257-4df5-89e7-a011b27eaa38", + "mgree": "7d8fbcc5-713b-4162-91f2-9f9f5b188fb6", + "mtabebe": "fb996665-32a5-4e2a-903b-8b67c010d62c", + "patrickwwbutler": "dbfeb9d9-b913-4f10-820d-cc3de0acba35", + "petrosagg": "713e3034-6a5d-4c42-8676-f8298d3af904", + "SangJunBak": "41416df0-176c-49fb-a872-1d0e50f9cecc", +} + +LANG_ALIASES = { + "python3": "python", + "py": "python", + "rs": "rust", + "js": "javascript", + "ts": "typescript", + "sh": "bash", + "zsh": "bash", +} + + +def normalize_code_blocks(text: str) -> str: + def replace_lang(m: re.Match[str]) -> str: + lang = m.group(1) + return f"```{LANG_ALIASES.get(lang, lang)}" + + return re.sub(r"```(\w+)", replace_lang, text) + + +def gh_fetch_issue(repo: str, number: int) -> dict[str, Any]: + data = json.loads(spawn.capture(["gh", "api", f"repos/{repo}/issues/{number}"])) + return { + "number": data["number"], + "title": data["title"], + "state": data["state"], + "url": data["html_url"], + "body": data.get("body") or "", + "labels": [l["name"] for l in data.get("labels", [])], + "assignees": [a["login"] for a in data.get("assignees", [])], + "author": data["user"]["login"], + } + + +def gh_fetch_comments(repo: str, number: int) -> list[dict[str, Any]]: + raw = spawn.capture( + ["gh", "api", f"repos/{repo}/issues/{number}/comments", "--paginate"] + ) + return [ + { + "author": c["user"]["login"], + "date": c["created_at"][:10], + "body": c.get("body") or "", + } + for c in json.loads(raw) + ] + + +def linear_gql( + api_key: str, query: str, variables: dict[str, Any] | None = None +) -> dict[str, Any]: + resp = requests.post( + LINEAR_API_URL, + headers={"Authorization": api_key, "Content-Type": "application/json"}, + json={"query": query, "variables": variables or {}}, + timeout=30, + ) + resp.raise_for_status() + data = resp.json() + if data.get("errors"): + raise RuntimeError(f"Linear API error: {data['errors']}") + return data["data"] + + +def linear_find_team(api_key: str, team_key: str) -> dict[str, Any]: + data = linear_gql( + api_key, + "query($f:TeamFilter){teams(filter:$f){nodes{id name key}}}", + {"f": {"key": {"eq": team_key}}}, + ) + teams = data["teams"]["nodes"] + if not teams: + raise RuntimeError(f"No Linear team with key '{team_key}'") + return teams[0] + + +def linear_find_label(api_key: str, team_id: str, label_name: str) -> str | None: + q = "query($f:IssueLabelFilter){issueLabels(filter:$f){nodes{id}}}" + for filt in [ + {"name": {"eq": label_name}, "team": {"id": {"eq": team_id}}}, + {"name": {"eq": label_name}, "team": {"null": True}}, + ]: + nodes = linear_gql(api_key, q, {"f": filt})["issueLabels"]["nodes"] + if nodes: + return nodes[0]["id"] + return None + + +def linear_fetch_team_states(api_key: str, team_id: str) -> dict[str, str]: + data = linear_gql( + api_key, + "query($f:WorkflowStateFilter){workflowStates(filter:$f){nodes{id name}}}", + {"f": {"team": {"id": {"eq": team_id}}}}, + ) + return {s["name"]: s["id"] for s in data["workflowStates"]["nodes"]} + + +def linear_create_issue(api_key: str, inp: dict[str, Any]) -> dict[str, Any]: + data = linear_gql( + api_key, + "mutation($i:IssueCreateInput!){issueCreate(input:$i){success issue{id identifier url}}}", + {"i": inp}, + ) + if not data["issueCreate"]["success"]: + raise RuntimeError(f"Failed to create issue: {inp.get('title')}") + return data["issueCreate"]["issue"] + + +def linear_add_attachment(api_key: str, issue_id: str, url: str, title: str) -> None: + linear_gql( + api_key, + "mutation($i:AttachmentCreateInput!){attachmentCreate(input:$i){success}}", + {"i": {"issueId": issue_id, "url": url, "title": title}}, + ) + + +def linear_add_comment(api_key: str, issue_id: str, body: str) -> None: + linear_gql( + api_key, + "mutation($i:CommentCreateInput!){commentCreate(input:$i){success}}", + {"i": {"issueId": issue_id, "body": body}}, + ) + + +def clamp_body(body: str, url: str, kind: str) -> str: + """Trim body to Linear's maximum length, linking to GitHub for the rest. + + Linear rejects descriptions and comment bodies longer than LINEAR_MAX_BODY + characters. The full text stays on the linked GitHub issue, so trimming + here loses nothing. + """ + if len(body) <= LINEAR_MAX_BODY: + return body + note = f"\n\n---\n\n*{kind} truncated, see {url}*" + return body[: LINEAR_MAX_BODY - len(note)] + note + + +def split_long_block(block: str, limit: int) -> list[str]: + """Split a comment block into parts no longer than limit characters. + + Splits only on line boundaries so no line is cut in half. When a split + falls inside a fenced code block, the fence is closed on the part that ends + and reopened on the part that continues, so each part renders on its own. + """ + if len(block) <= limit: + return [block] + fence = "```" + # Reserve room to close and reopen a fence around a split. + budget = limit - 2 * (len(fence) + 1) + parts: list[str] = [] + lines: list[str] = [] + length = 0 + in_fence = False + for line in block.split("\n"): + if lines and length + len(line) + 1 > budget: + ended = "\n".join(lines) + if in_fence: + ended += "\n" + fence + parts.append(ended) + lines = [fence] if in_fence else [] + length = len(fence) + 1 if in_fence else 0 + lines.append(line) + length += len(line) + 1 + if line.lstrip().startswith(fence): + in_fence = not in_fence + parts.append("\n".join(lines)) + return parts + + +def pack_comment_bodies(header: str, blocks: list[str]) -> list[str]: + """Pack per-comment blocks into as few Linear comment bodies as possible. + + Each block is one migrated GitHub comment. Blocks are concatenated under a + repeated header until the next part would exceed LINEAR_MAX_BODY, then a new + body starts. A block longer than the limit is split by split_long_block, so + no comment text is lost. + """ + sep = "\n\n---\n\n" + limit = LINEAR_MAX_BODY - len(header) - len(sep) + bodies: list[str] = [] + current = header + for block in blocks: + for part in split_long_block(block, limit): + if len(current) + len(sep) + len(part) > LINEAR_MAX_BODY: + bodies.append(current) + current = header + current += sep + part + bodies.append(current) + return bodies + + +def migrate_issue(api_keys: dict[str, str], gh: dict[str, Any], dry_run: bool) -> None: + api_key = api_keys.get(gh["author"], api_keys["default"]) + labels = gh["labels"] + team_key = next((LABEL_TO_TEAM[l] for l in labels if l in LABEL_TO_TEAM), "DB") + mapped = [LABEL_MAP[l] for l in labels if l in LABEL_MAP] + unmapped = [l for l in labels if l not in LABEL_MAP and l not in LABEL_TO_TEAM] + + label_info = f"labels=[{', '.join(labels) or 'none'}] -> team={team_key}" + if mapped: + label_info += f", linear=[{', '.join(mapped)}]" + if unmapped: + label_info += f", skipped=[{', '.join(unmapped)}]" + assignee_info = "" + if gh["assignees"]: + assignee_info = f", assignees=[{', '.join(gh['assignees'])}]" + print(f"#{gh['number']}: {gh['title']} ({label_info}{assignee_info})", flush=True) + if gh["state"] == "closed": + print(" skipped: already closed", flush=True) + return + if dry_run: + print(" [dry-run] would create Linear issue and close GH issue", flush=True) + return + + team = linear_find_team(api_key, team_key) + label_ids = [] + for l in labels: + if l in LABEL_MAP: + lid = linear_find_label(api_key, team["id"], LABEL_MAP[l]) + if lid: + label_ids.append(lid) + else: + print(f" warning: Linear label '{LABEL_MAP[l]}' not found", flush=True) + + state_id = linear_fetch_team_states(api_key, team["id"]).get("Backlog") + desc = clamp_body(normalize_code_blocks(gh["body"]), gh["url"], "Description") + inp: dict[str, Any] = { + "teamId": team["id"], + "title": gh["title"], + "description": desc, + } + if label_ids: + inp["labelIds"] = label_ids + if state_id: + inp["stateId"] = state_id + for assignee in gh["assignees"]: + if assignee in GH_TO_LINEAR_USER: + inp["assigneeId"] = GH_TO_LINEAR_USER[assignee] + break + else: + print(f" warning: no Linear user for GH assignee '{assignee}'", flush=True) + + issue = linear_create_issue(api_key, inp) + linear_add_attachment(api_key, issue["id"], gh["url"], "GitHub Issue") + print(f" -> {issue['identifier']} {issue['url']}", flush=True) + + comments = gh_fetch_comments(GITHUB_REPO, gh["number"]) + if comments: + header = f"**Migrated comments from GitHub#{gh['number']}:**" + blocks = [ + f"**{c['author']}** ({c['date']}):\n\n{normalize_code_blocks(c['body'])}" + for c in comments + ] + for body in pack_comment_bodies(header, blocks): + linear_add_comment(api_key, issue["id"], body) + print(f" -> migrated {len(comments)} comment(s)", flush=True) + + subprocess.run( + [ + "gh", + "issue", + "close", + str(gh["number"]), + "--repo", + GITHUB_REPO, + "--reason", + "not planned", + "--comment", + f"Migrated to Linear: {issue['url']}", + ], + check=True, + capture_output=True, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Migrate GitHub issues to Linear") + parser.add_argument("issue_id", nargs="+", type=int) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + api_keys: dict[str, str] = {} + api_keys["default"] = os.environ.get("LINEAR_API_KEY", "") + if os.environ.get("LINEAR_API_KEY_DEF"): + api_keys["def-"] = os.environ["LINEAR_API_KEY_DEF"] + if not api_keys["default"] and not args.dry_run: + print("Error: set LINEAR_API_KEY environment variable", file=sys.stderr) + sys.exit(1) + + for num in args.issue_id: + migrate_issue(api_keys, gh_fetch_issue(GITHUB_REPO, num), args.dry_run) + + +if __name__ == "__main__": + main() diff --git a/src/adapter/src/catalog/apply.rs b/src/adapter/src/catalog/apply.rs index 17a5e16215d1d..4a4ca65a423c4 100644 --- a/src/adapter/src/catalog/apply.rs +++ b/src/adapter/src/catalog/apply.rs @@ -1506,7 +1506,7 @@ impl CatalogState { /// Set the optimized plan for the item identified by `id`. /// /// # Panics - /// If the item is not an `Index` or `MaterializedView`. + /// If the item is not an `Index`, `MaterializedView`, or `MetricSink`. pub(super) fn set_optimized_plan( &mut self, id: GlobalId, @@ -1525,7 +1525,7 @@ impl CatalogState { /// Set the physical plan for the item identified by `id`. /// /// # Panics - /// If the item is not an `Index` or `MaterializedView`. + /// If the item is not an `Index`, `MaterializedView`, or `MetricSink`. pub(super) fn set_physical_plan( &mut self, id: GlobalId, @@ -1544,7 +1544,7 @@ impl CatalogState { /// Set the `DataflowMetainfo` for the item identified by `id`. /// /// # Panics - /// If the item is not an `Index` or `MaterializedView`. + /// If the item is not an `Index`, `MaterializedView`, or `MetricSink`. pub(super) fn set_dataflow_metainfo( &mut self, id: GlobalId, diff --git a/src/adapter/src/coord.rs b/src/adapter/src/coord.rs index cbbdaca9a63b4..7678759fcb636 100644 --- a/src/adapter/src/coord.rs +++ b/src/adapter/src/coord.rs @@ -408,6 +408,11 @@ pub enum Message { span: Span, stage: CreateIndexStage, }, + CreateMetricSinkStageReady { + ctx: ExecuteContext, + span: Span, + stage: CreateMetricSinkStage, + }, CreateViewStageReady { ctx: ExecuteContext, span: Span, @@ -542,6 +547,7 @@ impl Message { Message::PeekStageReady { .. } => "peek_stage_ready", Message::ExplainTimestampStageReady { .. } => "explain_timestamp_stage_ready", Message::CreateIndexStageReady { .. } => "create_index_stage_ready", + Message::CreateMetricSinkStageReady { .. } => "create_metric_sink_stage_ready", Message::CreateViewStageReady { .. } => "create_view_stage_ready", Message::CreateMaterializedViewStageReady { .. } => { "create_materialized_view_stage_ready" @@ -786,6 +792,31 @@ pub struct CreateIndexExplain { explain_ctx: ExplainPlanContext, } +#[derive(Debug)] +pub enum CreateMetricSinkStage { + Optimize(CreateMetricSinkOptimize), + Finish(CreateMetricSinkFinish), +} + +#[derive(Debug)] +pub struct CreateMetricSinkOptimize { + validity: PlanValidity, + plan: plan::CreateMetricSinkPlan, + resolved_ids: ResolvedIds, +} + +#[derive(Debug)] +pub struct CreateMetricSinkFinish { + validity: PlanValidity, + item_id: CatalogItemId, + global_id: GlobalId, + plan: plan::CreateMetricSinkPlan, + resolved_ids: ResolvedIds, + global_mir_plan: optimize::metric_sink::GlobalMirPlan, + global_lir_plan: optimize::metric_sink::GlobalLirPlan, + optimizer_features: OptimizerFeatures, +} + #[derive(Debug)] pub enum CreateViewStage { Optimize(CreateViewOptimize), @@ -2745,6 +2776,32 @@ impl Coordinator { self.allow_writes(mview.cluster_id, mview.global_id_writes()); } } + CatalogItem::MetricSink(metric_sink) => { + let df_desc = self + .catalog() + .try_get_physical_plan(&metric_sink.global_id) + .expect("added in `bootstrap_dataflow_plans`") + .clone(); + + let df_meta = self + .catalog() + .try_get_dataflow_metainfo(&metric_sink.global_id) + .expect("added in `bootstrap_dataflow_plans`"); + + if self.catalog().state().system_config().enable_mz_notices() { + // Collect optimization hint updates. + self.catalog().state().pack_optimizer_notices( + &mut builtin_table_updates, + df_meta.optimizer_notices.iter(), + Diff::ONE, + ); + } + + // No read policy to set: the export is a sink, not a readable collection, so + // `ship_dataflow` has no index export to initialize a policy for. + self.ship_dataflow(df_desc, metric_sink.cluster_id, None) + .await; + } CatalogItem::Sink(sink) => { policies_to_set .entry(CompactionWindow::Default) @@ -2767,8 +2824,7 @@ impl Coordinator { CatalogItem::Log(_) | CatalogItem::Type(_) | CatalogItem::Func(_) - | CatalogItem::Secret(_) - | CatalogItem::MetricSink(_) => {} + | CatalogItem::Secret(_) => {} } } @@ -3758,6 +3814,99 @@ impl Coordinator { compute_instance.insert_collection(mv.global_id_writes()); } + CatalogItem::MetricSink(metric_sink) => { + // Collect optimizer parameters. + let compute_instance = instance_snapshots + .entry(metric_sink.cluster_id) + .or_insert_with(|| { + self.instance_snapshot(metric_sink.cluster_id) + .expect("compute instance exists") + }); + let global_id = metric_sink.global_id; + let optimizer_config = optimizer_config(&self.catalog, metric_sink.cluster_id); + + let (optimized_plan, physical_plan, metainfo) = match cached_global_exprs + .remove(&global_id) + { + Some(global_expressions) + if global_expressions.optimizer_features + == optimizer_config.features => + { + debug!("global expression cache hit for {global_id:?}"); + ( + global_expressions.global_mir, + global_expressions.physical_plan, + global_expressions.dataflow_metainfos, + ) + } + Some(_) | None => { + // A transient id for the view the optimizer builds over `from` to + // shape its rows (see `optimize::metric_sink::shape_metric_sink_source`). + // The id only needs to be unique within this dataflow, so a cached plan + // reusing a transient id from a previous boot is safe: build ids are + // dataflow-local on the worker and never registered in the controller's + // instance-global collections (only export ids are). + let (_, view_id) = self.allocate_transient_id(); + + let (optimized_plan, global_lir_plan) = { + let mut optimizer = optimize::metric_sink::Optimizer::new( + self.owned_catalog(), + compute_instance.clone(), + view_id, + global_id, + optimizer_config.clone(), + self.optimizer_metrics(), + ); + + // MIR ⇒ MIR optimization (global) + let metric_sink_plan = optimize::metric_sink::MetricSink::new( + entry.name().clone(), + metric_sink.from, + metric_sink.prefix.clone(), + ); + let global_mir_plan = optimizer.optimize(metric_sink_plan)?; + let optimized_plan = global_mir_plan.df_desc().clone(); + + // MIR ⇒ LIR lowering and LIR ⇒ LIR optimization (global) + let global_lir_plan = optimizer.optimize(global_mir_plan)?; + + (optimized_plan, global_lir_plan) + }; + + let (physical_plan, metainfo) = global_lir_plan.unapply(); + let metainfo = { + // Pre-allocate a vector of transient GlobalIds for each notice. + let notice_ids = + std::iter::repeat_with(|| self.allocate_transient_id()) + .map(|(_item_id, gid)| gid) + .take(metainfo.optimizer_notices.len()) + .collect::>(); + // Return a metainfo with rendered notices. + self.catalog() + .render_notices(metainfo, notice_ids, Some(global_id)) + }; + uncached_expressions.insert( + global_id, + GlobalExpressions { + global_mir: optimized_plan.clone(), + physical_plan: physical_plan.clone(), + dataflow_metainfos: metainfo.clone(), + optimizer_features: optimizer_config.features.clone(), + }, + ); + (optimized_plan, physical_plan, metainfo) + } + }; + + let catalog = self.catalog_mut(); + catalog.set_optimized_plan(global_id, optimized_plan); + catalog.set_physical_plan(global_id, physical_plan); + catalog.set_dataflow_metainfo(global_id, metainfo); + + // NOTE: No `insert_collection` for the export. A metric sink writes to the + // metrics registry rather than to a readable collection, so no later dataflow + // can import it. + } CatalogItem::Table(_) | CatalogItem::Source(_) | CatalogItem::Log(_) @@ -3766,8 +3915,7 @@ impl Coordinator { | CatalogItem::Type(_) | CatalogItem::Func(_) | CatalogItem::Secret(_) - | CatalogItem::Connection(_) - | CatalogItem::MetricSink(_) => (), + | CatalogItem::Connection(_) => (), } } @@ -3791,6 +3939,7 @@ impl Coordinator { let gid = match entry.item() { CatalogItem::Index(idx) => idx.global_id(), CatalogItem::MaterializedView(mv) => mv.global_id_writes(), + CatalogItem::MetricSink(metric_sink) => metric_sink.global_id, CatalogItem::Table(_) | CatalogItem::Source(_) | CatalogItem::Log(_) @@ -3799,8 +3948,7 @@ impl Coordinator { | CatalogItem::Type(_) | CatalogItem::Func(_) | CatalogItem::Secret(_) - | CatalogItem::Connection(_) - | CatalogItem::MetricSink(_) => continue, + | CatalogItem::Connection(_) => continue, }; if let Some(plan) = self.catalog.try_get_physical_plan(&gid) { catalog_ids.push(gid); diff --git a/src/adapter/src/coord/catalog_implications.rs b/src/adapter/src/coord/catalog_implications.rs index ca4bc8b21fb25..961f8cafaed48 100644 --- a/src/adapter/src/coord/catalog_implications.rs +++ b/src/adapter/src/coord/catalog_implications.rs @@ -35,7 +35,7 @@ use itertools::Itertools; use mz_adapter_types::compaction::CompactionWindow; use mz_catalog::memory::objects::{ CatalogItem, Cluster, ClusterReplica, Connection, DataSourceDesc, Index, MaterializedView, - Secret, Sink, Source, StateDiff, Table, TableDataSource, View, + MetricSink, Secret, Sink, Source, StateDiff, Table, TableDataSource, View, }; use mz_cloud_resources::VpcEndpointConfig; use mz_compute_client::logging::LogVariant; @@ -447,6 +447,23 @@ impl Coordinator { indexes_to_drop.push((index.cluster_id, index.global_id())); dropped_item_names.insert(index.global_id(), full_name); } + CatalogImplication::MetricSink(CatalogImplicationKind::Added(_metric_sink)) => { + // Nothing to do, mirroring `Index`: shipping the dataflow at create time is + // the sequencer's job (`create_metric_sink_finish`), and re-rendering it after + // a restart happens during bootstrap (`bootstrap_dataflow_plans`). + } + CatalogImplication::MetricSink(CatalogImplicationKind::Altered { .. }) => { + // Nothing to do: owner, privilege, and rename changes are catalog-only. + } + CatalogImplication::MetricSink(CatalogImplicationKind::Dropped( + metric_sink, + full_name, + )) => { + // A metric sink is a non-readable leaf compute dataflow, like an MV's write + // side, so it drops through the same path as other compute sinks. + compute_sinks_to_drop.push((metric_sink.cluster_id, metric_sink.global_id)); + dropped_item_names.insert(metric_sink.global_id, full_name); + } CatalogImplication::MaterializedView(CatalogImplicationKind::Added(mv)) => { tracing::debug!(?mv, "not handling AddMaterializedView in here yet"); } @@ -598,6 +615,7 @@ impl Coordinator { | CatalogImplication::Source(CatalogImplicationKind::None) | CatalogImplication::Sink(CatalogImplicationKind::None) | CatalogImplication::Index(CatalogImplicationKind::None) + | CatalogImplication::MetricSink(CatalogImplicationKind::None) | CatalogImplication::MaterializedView(CatalogImplicationKind::None) | CatalogImplication::View(CatalogImplicationKind::None) | CatalogImplication::Secret(CatalogImplicationKind::None) @@ -1683,6 +1701,7 @@ enum CatalogImplication { Source(CatalogImplicationKind<(Source, Option)>), Sink(CatalogImplicationKind), Index(CatalogImplicationKind), + MetricSink(CatalogImplicationKind), MaterializedView(CatalogImplicationKind), View(CatalogImplicationKind), Secret(CatalogImplicationKind), @@ -1836,10 +1855,16 @@ impl CatalogImplication { CatalogItem::Connection(connection) => { self.absorb_connection(connection, None, catalog_update.diff); } + CatalogItem::MetricSink(metric_sink) => { + self.absorb_metric_sink( + metric_sink, + Some(parsed_full_name), + catalog_update.diff, + ); + } CatalogItem::Log(_) => {} CatalogItem::Type(_) => {} CatalogItem::Func(_) => {} - CatalogItem::MetricSink(_) => {} }, ParsedStateUpdateKind::Cluster { durable_cluster: _, @@ -1886,6 +1911,7 @@ impl CatalogImplication { ); impl_absorb_method!(absorb_sink, Sink, Sink); impl_absorb_method!(absorb_index, Index, Index); + impl_absorb_method!(absorb_metric_sink, MetricSink, MetricSink); impl_absorb_method!(absorb_materialized_view, MaterializedView, MaterializedView); impl_absorb_method!(absorb_view, View, View); diff --git a/src/adapter/src/coord/message_handler.rs b/src/adapter/src/coord/message_handler.rs index 71c81667b1055..8f80962510230 100644 --- a/src/adapter/src/coord/message_handler.rs +++ b/src/adapter/src/coord/message_handler.rs @@ -210,6 +210,9 @@ impl Coordinator { Message::CreateIndexStageReady { ctx, span, stage } => { self.sequence_staged(ctx, span, stage).boxed_local().await; } + Message::CreateMetricSinkStageReady { ctx, span, stage } => { + self.sequence_staged(ctx, span, stage).boxed_local().await; + } Message::CreateViewStageReady { ctx, span, stage } => { self.sequence_staged(ctx, span, stage).boxed_local().await; } diff --git a/src/adapter/src/coord/sequencer.rs b/src/adapter/src/coord/sequencer.rs index 4d057fc7e1121..d86d66acbfb2f 100644 --- a/src/adapter/src/coord/sequencer.rs +++ b/src/adapter/src/coord/sequencer.rs @@ -293,10 +293,8 @@ impl Coordinator { self.sequence_create_index(ctx, plan, resolved_ids).await; } Plan::CreateMetricSink(plan) => { - let result = self - .sequence_create_metric_sink(ctx.session(), plan, resolved_ids) + self.sequence_create_metric_sink(ctx, plan, resolved_ids) .await; - ctx.retire(result); } Plan::CreateType(plan) => { let result = self diff --git a/src/adapter/src/coord/sequencer/inner.rs b/src/adapter/src/coord/sequencer/inner.rs index 83749b8a28930..3a8ccfd116224 100644 --- a/src/adapter/src/coord/sequencer/inner.rs +++ b/src/adapter/src/coord/sequencer/inner.rs @@ -18,6 +18,7 @@ use anyhow::anyhow; use futures::future::{BoxFuture, FutureExt}; use futures::{Future, StreamExt, future}; use itertools::Itertools; +use maplit::btreemap; use mz_adapter_types::compaction::CompactionWindow; use mz_adapter_types::connection::ConnectionId; use mz_adapter_types::dyncfgs::{ @@ -28,6 +29,9 @@ use mz_catalog::memory::error::ErrorKind; use mz_catalog::memory::objects::{ CatalogItem, Connection, DataSourceDesc, Sink, Source, Table, TableDataSource, Type, }; +use mz_compute_types::ComputeInstanceId; +use mz_compute_types::dataflows::DataflowDescription; +use mz_compute_types::plan::LirRelationExpr; use mz_expr::{ CollectionPlan, Eval, MapFilterProject, OptimizedMirRelationExpr, ResultSpec, RowSetFinishing, }; @@ -39,12 +43,12 @@ use mz_ore::tracing::OpenTelemetryContext; use mz_ore::{assert_none, instrument}; use mz_repr::adt::jsonb::Jsonb; use mz_repr::adt::mz_acl_item::{MzAclItem, PrivilegeMap}; -use mz_repr::explain::ExprHumanizer; use mz_repr::explain::json::json_string; +use mz_repr::explain::{ExprHumanizer, ExprHumanizerExt, TransientItem}; use mz_repr::role_id::RoleId; use mz_repr::{ - CatalogItemId, Datum, Diff, GlobalId, RelationVersion, RelationVersionSelector, Row, RowArena, - RowIterator, Timestamp, + CatalogItemId, Datum, Diff, GlobalId, RelationDesc, RelationVersion, RelationVersionSelector, + Row, RowArena, RowIterator, Timestamp, }; use mz_secrets::SecretsReader; use mz_sql::ast::{ @@ -101,7 +105,9 @@ use timely::progress::Antichain; use tokio::sync::{oneshot, watch}; use tracing::{Instrument, Span, info, warn}; -use crate::catalog::{self, Catalog, ConnCatalog, DropObjectInfo, UpdatePrivilegeVariant}; +use crate::catalog::{ + self, Catalog, CatalogState, ConnCatalog, DropObjectInfo, UpdatePrivilegeVariant, +}; use crate::command::{ExecuteResponse, Response}; use crate::coord::appends::{ BuiltinTableAppendNotify, DeferredOp, DeferredPlan, PendingWriteTxn, UserWriteResponder, @@ -124,7 +130,7 @@ use crate::session::{ WriteLocks, WriteOp, }; use crate::util::{ClientTransmitter, ResultExt, viewable_variables}; -use crate::{PeekResponseUnary, ReadHolds}; +use crate::{CollectionIdBundle, PeekResponseUnary, ReadHolds}; /// A future that resolves to a real-time recency timestamp. type RtrTimestampFuture = BoxFuture<'static, Result>; @@ -5055,6 +5061,73 @@ impl Coordinator { emit_optimizer_notices(&*self.catalog, ctx.session(), notices); } + /// Renders `raw_df_meta`'s optimizer notices against a humanizer that resolves the + /// about-to-be-created item's own `global_id` to `name`, rather than to the bare transient id + /// a system-session humanizer would produce. `source_desc` supplies the column names the + /// notices humanize against, and is the desc of the relation the new item reads. + /// + /// Callers render before the catalog transaction that creates the item, so the persisted + /// notice text can already refer to the item by its intended name. The raw notices stay with + /// the caller: they go to the user session only once the transaction succeeds, so a failed + /// transaction does not tell the user about an item that was never created. + fn render_create_item_notices( + &self, + name: &QualifiedItemName, + global_id: GlobalId, + source_desc: &RelationDesc, + raw_df_meta: &DataflowMetainfo, + ) -> DataflowMetainfo> { + let notice_ids = std::iter::repeat_with(|| self.allocate_transient_id()) + .map(|(_item_id, notice_id)| notice_id) + .take(raw_df_meta.optimizer_notices.len()) + .collect::>(); + + let system_catalog = self.catalog().for_system_session(); + let full_name = self.catalog().resolve_full_name(name, None); + let transient_items = btreemap! { + global_id => TransientItem::new( + Some(full_name.into_parts()), + Some(source_desc.iter_names().map(|c| c.to_string()).collect()), + ) + }; + let humanizer = ExprHumanizerExt::new(transient_items, &system_catalog); + CatalogState::render_notices_core( + &humanizer, + (self.catalog().config().now)(), + raw_df_meta, + notice_ids, + Some(global_id), + ) + } + + /// Sets `df_desc`'s as-of from a read hold on `id_bundle`, ships the dataflow, and drops the + /// hold once compute has taken its own (compute puts in its own read holds during + /// `create_dataflow`, so it is safe to release this one right after shipping). + /// + /// The read hold across shipping keeps the since of `id_bundle` from advancing underneath the + /// as-of just picked. + async fn ship_new_dataflow( + &mut self, + id_bundle: &CollectionIdBundle, + mut df_desc: DataflowDescription, + instance: ComputeInstanceId, + notice_builtin_updates_fut: Option, + ) { + let read_holds = self.acquire_read_holds(id_bundle); + let since = read_holds.least_valid_read(); + df_desc.set_as_of(since); + + self.ship_dataflow_and_notice_builtin_table_updates( + df_desc, + instance, + notice_builtin_updates_fut, + None, + ) + .await; + + drop(read_holds); + } + /// Persist already-rendered optimizer notices for a newly created /// non-transient dataflow. /// diff --git a/src/adapter/src/coord/sequencer/inner/create_index.rs b/src/adapter/src/coord/sequencer/inner/create_index.rs index 249021981ab11..be46743380721 100644 --- a/src/adapter/src/coord/sequencer/inner/create_index.rs +++ b/src/adapter/src/coord/sequencer/inner/create_index.rs @@ -23,7 +23,6 @@ use mz_sql::plan; use mz_sql::session::metadata::SessionMetadata; use tracing::Span; -use crate::catalog::CatalogState; use crate::command::ExecuteResponse; use crate::coord::sequencer::inner::return_if_err; use crate::coord::{ @@ -478,47 +477,14 @@ impl Coordinator { owner_id, }]; - // Pre-allocate a vector of transient GlobalIds for each notice. - let notice_ids = std::iter::repeat_with(|| self.allocate_transient_id()) - .map(|(_item_id, global_id)| global_id) - .take(global_lir_plan.df_meta().optimizer_notices.len()) - .collect::>(); - - // Render optimizer notices before the catalog transaction, using an - // `ExprHumanizerExt` that knows about the to-be-created index. This - // way the notice text produced here (and persisted in - // `mz_optimizer_notices`) resolves the new index's own `global_id` to - // its intended human-readable name, rather than to the bare transient - // id that `for_system_session()` would produce on its own. - // // We keep `raw_df_meta` live so that on success we can emit its raw // notices to the user session (rendered against the user's - // session-aware humanizer). We deliberately do NOT emit to the user - // here, so that if the catalog transaction below fails the user - // isn't shown confusing notices about an item that wasn't actually - // created. - let (mut df_desc, raw_df_meta) = global_lir_plan.unapply(); - let df_meta = { - let system_catalog = self.catalog().for_system_session(); - let full_name = self.catalog().resolve_full_name(&name, None); - let on_desc = on_entry - .relation_desc() - .expect("can only create indexes on items with a valid description"); - let transient_items = btreemap! { - global_id => TransientItem::new( - Some(full_name.into_parts()), - Some(on_desc.iter_names().map(|c| c.to_string()).collect()), - ) - }; - let humanizer = ExprHumanizerExt::new(transient_items, &system_catalog); - CatalogState::render_notices_core( - &humanizer, - (self.catalog().config().now)(), - &raw_df_meta, - notice_ids, - Some(global_id), - ) - }; + // session-aware humanizer). + let (df_desc, raw_df_meta) = global_lir_plan.unapply(); + let on_desc = on_entry + .relation_desc() + .expect("can only create indexes on items with a valid description"); + let df_meta = self.render_create_item_notices(&name, global_id, &on_desc, &raw_df_meta); // Populate the durable expression cache before the catalog // transaction and await the write. This way any other envd (or a @@ -549,31 +515,19 @@ impl Coordinator { let notice_builtin_updates_fut = coord.persist_dataflow_metainfo(df_meta, global_id); - // We're putting in place read holds, such that ship_dataflow, - // below, which calls update_read_capabilities, can successfully - // do so. Otherwise, the since of dependencies might move along - // concurrently, pulling the rug from under us! - // - // TODO: Maybe in the future, pass those holds on to compute, to - // hold on to them and downgrade when possible? - let read_holds = coord.acquire_read_holds(&id_bundle); - let since = read_holds.least_valid_read(); - df_desc.set_as_of(since); - + // TODO: Maybe in the future, pass the read holds + // `ship_new_dataflow` takes on to compute, to hold on to them + // and downgrade when possible? coord - .ship_dataflow_and_notice_builtin_table_updates( + .ship_new_dataflow( + &id_bundle, df_desc, cluster_id, notice_builtin_updates_fut, - None, ) .await; // No `allow_writes` here because indexes do not modify external state. - // Drop read holds after the dataflow has been shipped, at which - // point compute will have put in its own read holds. - drop(read_holds); - coord.update_compute_read_policy( cluster_id, item_id, diff --git a/src/adapter/src/coord/sequencer/inner/create_metric_sink.rs b/src/adapter/src/coord/sequencer/inner/create_metric_sink.rs index d40db59ac0852..e476fe3f47328 100644 --- a/src/adapter/src/coord/sequencer/inner/create_metric_sink.rs +++ b/src/adapter/src/coord/sequencer/inner/create_metric_sink.rs @@ -9,48 +9,207 @@ //! `CREATE METRIC SINK` sequencing. //! -//! All this does is write the catalog item. Nothing gets optimized or shipped, so a metric sink -//! created here publishes no metrics. The item is still worth having on its own: it holds the -//! definition compute needs, and because `create_sql` is re-parsed on boot the sink survives a -//! restart. -//! -//! TODO: optimize and ship the dataflow here, and re-render it during bootstrap. +//! Staged like `CREATE INDEX`: optimization runs off the coordinator thread, then the finish stage +//! writes the durable catalog item and ships the dataflow inside one catalog transaction. use anyhow::anyhow; use mz_catalog::memory::error::ErrorKind; use mz_catalog::memory::objects::{CatalogItem, MetricSink}; use mz_controller_types::ClusterId; use mz_ore::instrument; +use mz_repr::optimize::OverrideFrom; use mz_sql::catalog::CatalogError; use mz_sql::names::{QualifiedItemName, ResolvedIds}; use mz_sql::plan; use mz_sql::session::metadata::SessionMetadata; +use tracing::Span; -use crate::AdapterNotice; -use crate::catalog; use crate::command::ExecuteResponse; -use crate::coord::Coordinator; +use crate::coord::sequencer::inner::return_if_err; +use crate::coord::{ + Coordinator, CreateMetricSinkFinish, CreateMetricSinkOptimize, CreateMetricSinkStage, Message, + PlanValidity, StageResult, Staged, +}; use crate::error::AdapterError; +use crate::optimize::dataflows::dataflow_import_id_bundle; +use crate::optimize::{self, Optimize}; use crate::session::Session; +use crate::{AdapterNotice, ExecuteContext, catalog}; + +impl Staged for CreateMetricSinkStage { + type Ctx = ExecuteContext; + + fn validity(&mut self) -> &mut PlanValidity { + match self { + Self::Optimize(stage) => &mut stage.validity, + Self::Finish(stage) => &mut stage.validity, + } + } + + async fn stage( + self, + coord: &mut Coordinator, + ctx: &mut ExecuteContext, + ) -> Result>, AdapterError> { + match self { + CreateMetricSinkStage::Optimize(stage) => { + coord.create_metric_sink_optimize(stage).await + } + CreateMetricSinkStage::Finish(stage) => { + coord.create_metric_sink_finish(ctx, stage).await + } + } + } + + fn message(self, ctx: ExecuteContext, span: Span) -> Message { + Message::CreateMetricSinkStageReady { + ctx, + span, + stage: self, + } + } + + fn cancel_enabled(&self) -> bool { + true + } +} impl Coordinator { #[instrument] pub(crate) async fn sequence_create_metric_sink( &mut self, + ctx: ExecuteContext, + plan: plan::CreateMetricSinkPlan, + resolved_ids: ResolvedIds, + ) { + let stage = return_if_err!( + self.create_metric_sink_validate(ctx.session(), plan, resolved_ids), + ctx + ); + self.sequence_staged(ctx, Span::current(), stage).await; + } + + #[instrument] + fn create_metric_sink_validate( + &self, session: &Session, plan: plan::CreateMetricSinkPlan, resolved_ids: ResolvedIds, - ) -> Result { - let plan::CreateMetricSinkPlan { - name, - metric_sink, - if_not_exists, - } = plan; + ) -> Result { + // Track the target cluster and resolved dependencies so concurrent drops are caught + // between stages instead of panicking later when the dataflow is shipped. + let validity = PlanValidity::new( + self.catalog(), + resolved_ids.items().copied().collect(), + Some(plan.metric_sink.cluster_id), + None, + session.role_metadata().clone(), + ); + Ok(CreateMetricSinkStage::Optimize(CreateMetricSinkOptimize { + validity, + plan, + resolved_ids, + })) + } - self.ensure_metric_sink_prefix_is_free(&name, metric_sink.cluster_id, &metric_sink.prefix)?; + #[instrument] + async fn create_metric_sink_optimize( + &mut self, + CreateMetricSinkOptimize { + validity, + plan, + resolved_ids, + }: CreateMetricSinkOptimize, + ) -> Result>, AdapterError> { + let cluster_id = plan.metric_sink.cluster_id; + // Collect optimizer parameters. + let compute_instance = self + .instance_snapshot(cluster_id) + .expect("compute instance does not exist"); let (item_id, global_id) = self.allocate_user_id().await?; - let op = catalog::Op::CreateItem { + // A transient id for the view the optimizer builds over `from` to shape its rows (see + // `optimize::metric_sink::shape_metric_sink_source`); scoped to this dataflow, not durable. + let (_, view_id) = self.allocate_transient_id(); + + let optimizer_config = optimize::OptimizerConfig::from(self.catalog().system_config()) + .override_from(&self.catalog.get_cluster(cluster_id).config.features()) + .override_from(&self.cluster_scoped_optimizer_overrides(cluster_id)); + let optimizer_features = optimizer_config.features.clone(); + + // Build an optimizer for this METRIC SINK. + let mut optimizer = optimize::metric_sink::Optimizer::new( + self.owned_catalog(), + compute_instance, + view_id, + global_id, + optimizer_config, + self.optimizer_metrics(), + ); + let span = Span::current(); + Ok(StageResult::Handle(mz_ore::task::spawn_blocking( + || "optimize create metric sink", + move || { + span.in_scope(|| { + let metric_sink = optimize::metric_sink::MetricSink::new( + plan.name.clone(), + plan.metric_sink.from, + plan.metric_sink.prefix.clone(), + ); + + // MIR ⇒ MIR optimization (global) + let global_mir_plan = optimizer.catch_unwind_optimize(metric_sink)?; + // MIR ⇒ LIR lowering and LIR ⇒ LIR optimization (global) + let global_lir_plan = + optimizer.catch_unwind_optimize(global_mir_plan.clone())?; + + let stage = CreateMetricSinkStage::Finish(CreateMetricSinkFinish { + validity, + item_id, + global_id, + plan, + resolved_ids, + global_mir_plan, + global_lir_plan, + optimizer_features, + }); + Ok(Box::new(stage)) + }) + }, + ))) + } + + #[instrument] + async fn create_metric_sink_finish( + &mut self, + ctx: &mut ExecuteContext, + stage: CreateMetricSinkFinish, + ) -> Result>, AdapterError> { + let CreateMetricSinkFinish { + item_id, + global_id, + plan: + plan::CreateMetricSinkPlan { + name, + metric_sink, + if_not_exists, + }, + resolved_ids, + global_mir_plan, + global_lir_plan, + optimizer_features, + .. + } = stage; + let cluster_id = metric_sink.cluster_id; + let id_bundle = dataflow_import_id_bundle(global_lir_plan.df_desc(), cluster_id); + + // Run the authoritative prefix-free check here in the finish stage, not in optimize: + // optimize runs off the coordinator thread, so another sink could commit between the two + // stages. See `ensure_metric_sink_prefix_is_free`. + self.ensure_metric_sink_prefix_is_free(&name, cluster_id, &metric_sink.prefix)?; + + let owner_id = *ctx.session().current_role_id(); + let ops = vec![catalog::Op::CreateItem { id: item_id, name: name.clone(), item: CatalogItem::MetricSink(MetricSink { @@ -58,25 +217,82 @@ impl Coordinator { global_id, from: metric_sink.from, resolved_ids, - cluster_id: metric_sink.cluster_id, + cluster_id, prefix: metric_sink.prefix, optimized_plan: None, physical_plan: None, dataflow_metainfo: None, }), - owner_id: *session.current_role_id(), - }; + owner_id, + }]; + + // Render optimizer notices before the catalog transaction: this way notice text resolves + // the new sink's own `global_id` to its intended human-readable name rather than a bare + // transient id. + let (df_desc, raw_df_meta) = global_lir_plan.unapply(); + let from_entry = self.catalog().get_entry_by_global_id(&metric_sink.from); + let from_desc = from_entry + .relation_desc() + .expect("can only create a metric sink on items with a valid description"); + let df_meta = self.render_create_item_notices(&name, global_id, &from_desc, &raw_df_meta); + + // Populate the durable expression cache before the catalog transaction and await the + // write. This way any other envd (or a subsequent bootstrap here) will observe the cached + // plans + rendered notices as soon as the item becomes visible. Metric sinks have no local + // MIR (the pipeline starts from a `GlobalId`), so there is no local expression to cache. + self.catalog() + .cache_expressions( + global_id, + None, + global_mir_plan.df_desc().clone(), + df_desc.clone(), + df_meta.clone(), + optimizer_features, + ) + .await; + + let transact_result = self + .catalog_transact_with_side_effects(Some(ctx), ops, move |coord, _ctx| { + Box::pin(async move { + // Save plan structures. + coord + .catalog_mut() + .set_optimized_plan(global_id, global_mir_plan.df_desc().clone()); + coord + .catalog_mut() + .set_physical_plan(global_id, df_desc.clone()); + + let notice_builtin_updates_fut = + coord.persist_dataflow_metainfo(df_meta, global_id); - match self.catalog_transact(Some(session), vec![op]).await { - Ok(()) => Ok(ExecuteResponse::CreatedMetricSink), + coord + .ship_new_dataflow( + &id_bundle, + df_desc, + cluster_id, + notice_builtin_updates_fut, + ) + .await; + // No `allow_writes` here: metric sinks write to the in-process metrics + // registry, not to external/persist state. + }) + }) + .await; + + match transact_result { + Ok(_) => { + self.emit_raw_optimizer_notices_to_user(ctx, &raw_df_meta.optimizer_notices); + Ok(StageResult::Response(ExecuteResponse::CreatedMetricSink)) + } Err(AdapterError::Catalog(mz_catalog::memory::error::Error { kind: ErrorKind::Sql(CatalogError::ItemAlreadyExists(_, _)), })) if if_not_exists => { - session.add_notice(AdapterNotice::ObjectAlreadyExists { - name: name.item, - ty: "metric sink", - }); - Ok(ExecuteResponse::CreatedMetricSink) + ctx.session() + .add_notice(AdapterNotice::ObjectAlreadyExists { + name: name.item, + ty: "metric sink", + }); + Ok(StageResult::Response(ExecuteResponse::CreatedMetricSink)) } Err(err) => Err(err), } diff --git a/src/adapter/src/optimize.rs b/src/adapter/src/optimize.rs index 6138c1c6feeb0..4c845def2ba12 100644 --- a/src/adapter/src/optimize.rs +++ b/src/adapter/src/optimize.rs @@ -56,7 +56,7 @@ pub mod copy_to; pub mod dataflows; pub mod index; pub mod materialized_view; -mod metric_sink; +pub mod metric_sink; pub mod peek; pub mod subscribe; pub mod view; diff --git a/src/adapter/src/optimize/index.rs b/src/adapter/src/optimize/index.rs index dfbd40e42afc5..22b3b634ce3e7 100644 --- a/src/adapter/src/optimize/index.rs +++ b/src/adapter/src/optimize/index.rs @@ -128,10 +128,6 @@ impl GlobalLirPlan { pub fn df_desc(&self) -> &LirDataflowDescription { &self.df_desc } - - pub fn df_meta(&self) -> &DataflowMetainfo { - &self.df_meta - } } impl Optimize for Optimizer { diff --git a/src/adapter/src/optimize/metric_sink.rs b/src/adapter/src/optimize/metric_sink.rs index d0bc8eadd8619..7baa79e90ead9 100644 --- a/src/adapter/src/optimize/metric_sink.rs +++ b/src/adapter/src/optimize/metric_sink.rs @@ -18,31 +18,265 @@ //! family-conflict counting) stays in the operator, because it needs the frontier-gated fold that //! a per-row `Map` in MIR can't express. +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use mz_compute_types::plan::LirRelationExpr; +use mz_compute_types::sinks::{ComputeSinkConnection, ComputeSinkDesc, MetricSinkConnection}; use mz_expr::func::variadic::Coalesce; use mz_expr::{MirRelationExpr, MirScalarExpr, func}; +use mz_repr::explain::trace_plan; use mz_repr::{ ColumnName, Datum, GlobalId, RelationDesc, ReprRelationType, ReprScalarType, Row, SqlScalarType, }; +use mz_sql::names::QualifiedItemName; +use mz_sql::optimizer_metrics::OptimizerMetrics; +use mz_transform::TransformCtx; +use mz_transform::dataflow::DataflowMetainfo; +use mz_transform::normalize_lets::normalize_lets; +use mz_transform::typecheck::{SharedTypecheckingContext, empty_typechecking_context}; +use timely::progress::Antichain; + +use crate::optimize::dataflows::{ + ComputeInstanceSnapshot, DataflowBuilder, ExprPrep, ExprPrepMaintained, +}; +use crate::optimize::{ + LirDataflowDescription, MirDataflowDescription, Optimize, OptimizerCatalog, OptimizerConfig, + OptimizerError, optimize_mir_local, +}; /// Matches Prometheus's metric name grammar: `[a-zA-Z_:][a-zA-Z0-9_:]*`. /// /// Expressed in MIR (see `shape_metric_sink_source`) rather than parsed from a `&str` on the /// operator's hot path. -// NOTE: `shape_metric_sink_source` (and this pattern) have no caller yet, hence the -// `#[allow(dead_code)]`, but adding them alongside the compute operator -// (`mz_compute::sink::metric_sink`) that reads the `metric_kind`/`name_valid` -// column contract, so they live together. -#[allow(dead_code)] const METRIC_NAME_PATTERN: &str = "^[a-zA-Z_:][a-zA-Z0-9_:]*$"; +/// Optimizer for `CREATE METRIC SINK`. +/// +/// Like `CREATE INDEX`, the pipeline starts directly from the `GlobalId` of the collection to +/// export rather than lowering a new relational expression from HIR. Unlike a materialized view +/// sink, there is no persist shard, so there is no storage-metadata stage. +pub struct Optimizer { + /// A representation typechecking context to use throughout the optimizer pipeline. + typecheck_ctx: SharedTypecheckingContext, + /// A snapshot of the catalog state. + catalog: Arc, + /// A snapshot of the cluster that will run the dataflow. + compute_instance: ComputeInstanceSnapshot, + /// A transient GlobalId for the shaped view built over the sink's source relation (see + /// `shape_metric_sink_source`). + view_id: GlobalId, + /// A durable GlobalId to be used with the exported metric sink. + sink_id: GlobalId, + /// Optimizer config. + config: OptimizerConfig, + /// Optimizer metrics. + metrics: OptimizerMetrics, + /// The time spent performing optimization so far. + duration: Duration, +} + +impl Optimizer { + pub fn new( + catalog: Arc, + compute_instance: ComputeInstanceSnapshot, + view_id: GlobalId, + sink_id: GlobalId, + config: OptimizerConfig, + metrics: OptimizerMetrics, + ) -> Self { + Self { + typecheck_ctx: empty_typechecking_context(), + catalog, + compute_instance, + view_id, + sink_id, + config, + metrics, + duration: Default::default(), + } + } +} + +/// A wrapper of metric sink parts needed to start the optimization process. +pub struct MetricSink { + name: QualifiedItemName, + from: GlobalId, + /// Prepended to every row's `metric_name` to form the published name. Validated at plan time + /// as a valid start of a Prometheus metric name. + prefix: String, +} + +impl MetricSink { + /// Construct a new [`MetricSink`]. Arguments are recorded as-is. + pub fn new(name: QualifiedItemName, from: GlobalId, prefix: String) -> Self { + Self { name, from, prefix } + } +} + +/// The (sealed intermediate) result after embedding a [`MetricSink`] into a +/// [`MirDataflowDescription`], inlining referenced views, and jointly optimizing the `MIR` plans. +#[derive(Clone, Debug)] +pub struct GlobalMirPlan { + df_desc: MirDataflowDescription, + df_meta: DataflowMetainfo, +} + +impl GlobalMirPlan { + pub fn df_desc(&self) -> &MirDataflowDescription { + &self.df_desc + } +} + +/// The (final) result after MIR ⇒ LIR lowering and optimizing the resulting +/// `DataflowDescription` with `LIR` plans. +#[derive(Clone, Debug)] +pub struct GlobalLirPlan { + df_desc: LirDataflowDescription, + df_meta: DataflowMetainfo, +} + +impl GlobalLirPlan { + pub fn df_desc(&self) -> &LirDataflowDescription { + &self.df_desc + } +} + +impl Optimize for Optimizer { + type To = GlobalMirPlan; + + fn optimize(&mut self, metric_sink: MetricSink) -> Result { + let time = Instant::now(); + + let from_entry = self.catalog.get_entry(&metric_sink.from); + let full_name = self + .catalog + .resolve_full_name(&metric_sink.name, from_entry.conn_id()); + let from_desc = from_entry + .relation_desc() + .expect("can only create a metric sink on items with a valid description") + .into_owned(); + + let mut df_builder = { + let compute = self.compute_instance.clone(); + DataflowBuilder::new(&*self.catalog, compute).with_config(&self.config) + }; + let mut df_desc = MirDataflowDescription::new(full_name.to_string()); + let mut df_meta = DataflowMetainfo::default(); + + df_builder.import_into_dataflow(&metric_sink.from, &mut df_desc, &self.config.features)?; + df_builder.maybe_reoptimize_imported_views(&mut df_desc, &self.config)?; + + // Push the pure row-wise shaping (coalesce identity elements, classify the metric kind, + // validate the metric name) into MIR, so the operator only does the cross-row logic + // (dedup/collision/family-conflict) that needs the fold. See `shape_metric_sink_source`. + let (shaped_expr, shaped_desc) = + shape_metric_sink_source(metric_sink.from, &from_desc, &metric_sink.prefix); + let mut local_ctx = TransformCtx::local( + &self.config.features, + &self.typecheck_ctx, + &mut df_meta, + Some(&mut self.metrics), + Some(self.view_id), + ); + let shaped_expr = optimize_mir_local(shaped_expr, &mut local_ctx)?; + + df_builder.import_view_into_dataflow( + &self.view_id, + &shaped_expr, + &mut df_desc, + &self.config.features, + )?; + df_builder.maybe_reoptimize_imported_views(&mut df_desc, &self.config)?; + + let sink_description = ComputeSinkDesc { + from: self.view_id, + from_desc: shaped_desc, + connection: ComputeSinkConnection::MetricSink(MetricSinkConnection {}), + with_snapshot: true, + up_to: Antichain::new(), + non_null_assertions: Vec::new(), + refresh_schedule: None, + }; + df_desc.export_sink(self.sink_id, sink_description); + + // Prepare expressions in the assembled dataflow. + let style = ExprPrepMaintained; + df_desc.visit_children( + |r| style.prep_relation_expr(r), + |s| style.prep_scalar_expr(s), + )?; + + // Construct TransformCtx for global optimization. + let mut transform_ctx = TransformCtx::global( + &df_builder, + &mz_transform::EmptyStatisticsOracle, + &self.config.features, + &self.typecheck_ctx, + &mut df_meta, + Some(&mut self.metrics), + ); + // Run global optimization. + mz_transform::optimize_dataflow(&mut df_desc, &mut transform_ctx, false)?; + + self.duration += time.elapsed(); + + Ok(GlobalMirPlan { df_desc, df_meta }) + } +} + +impl Optimize for Optimizer { + type To = GlobalLirPlan; + + fn optimize(&mut self, plan: GlobalMirPlan) -> Result { + let time = Instant::now(); + + let GlobalMirPlan { + mut df_desc, + df_meta, + } = plan; + + // Ensure all expressions are normalized before finalizing. + for build in df_desc.objects_to_build.iter_mut() { + normalize_lets(&mut build.plan.0, &self.config.features)? + } + + // Finalize the dataflow: MIR ⇒ LIR lowering and LIR ⇒ LIR transforms. + let df_desc = LirRelationExpr::finalize_dataflow( + df_desc, + &self.config.features, + Some(self.metrics.lowering()), + )?; + + // Trace the pipeline output under `optimize`. + trace_plan(&df_desc); + + self.duration += time.elapsed(); + self.metrics + .observe_e2e_optimization_time("metric_sink", self.duration); + + Ok(GlobalLirPlan { df_desc, df_meta }) + } +} + +impl GlobalLirPlan { + /// Unwraps the parts of the final result of the optimization pipeline. + pub fn unapply(self) -> (LirDataflowDescription, DataflowMetainfo) { + (self.df_desc, self.df_meta) + } +} + /// Extends the metric sink's imported relation with the row-wise shaping the operator otherwise -/// has to do in Rust: coalesces `labels`/`help` to their identity element, and adds two columns -/// the operator reads instead of parsing strings on its hot path: +/// has to do in Rust: prepends the user `prefix` to `metric_name` to form the published name, +/// coalesces `labels`/`help` to their identity element, and adds two columns the operator reads +/// instead of parsing strings on its hot path: /// /// * `metric_kind` (`Int32`, nullable): `0` for `gauge`, `1` for `counter`, `NULL` for any other /// `metric_type`. -/// * `name_valid` (`Bool`, nullable): whether `metric_name` matches the Prometheus metric-name -/// grammar (see `METRIC_NAME_PATTERN`). The operator treats a `NULL` the same as `false`. +/// * `name_valid` (`Bool`, nullable): whether the published name (`prefix + metric_name`) matches +/// the Prometheus metric-name grammar (see `METRIC_NAME_PATTERN`). The operator treats a `NULL` +/// the same as `false`. Validating the published name, not the bare `metric_name`, is what lets a +/// row name start with a digit: the prefix supplies the valid leading character. /// /// No row is dropped or filtered here: the operator still needs every row, including the ones /// this marks invalid, to count `skipped`/`null_values`. Only the pure per-row shaping moves to @@ -53,15 +287,15 @@ const METRIC_NAME_PATTERN: &str = "^[a-zA-Z_:][a-zA-Z0-9_:]*$"; /// via `Reduce` + `FirstValue`), collapsing the operator to a plain fold over the live set. That /// full move is deferred: the tiebreak fidelity that logic needs is easier to keep correct /// hand-written and unit-tested for now. -#[allow(dead_code)] fn shape_metric_sink_source( from_id: GlobalId, from_desc: &RelationDesc, + prefix: &str, ) -> (MirRelationExpr, RelationDesc) { // Precondition: the source relation exposes the canonical metric-sink columns (`metric_name`, - // `metric_type`, `labels`, `value`, `help`). No in-tree caller enforces this yet (see the NOTE - // on `METRIC_NAME_PATTERN`); the SQL planner will, once the CREATE METRIC SINK planning path - // lands. + // `metric_type`, `labels`, `value`, `help`). `CREATE METRIC SINK` planning enforces this (see + // `validate_metric_sink_desc` in `mz_sql::plan::statement::ddl`), so a missing column here is + // a planner bug, not user error. let get_idx = |name: &str| { from_desc .get_by_name(&ColumnName::from(name)) @@ -111,12 +345,28 @@ fn shape_metric_sink_source( ), ); - // The regexp requires at least one leading character, so an empty name fails it without a - // separate `!= ""` check. `is_null().not()` keeps a null name concretely `false`, not `NULL`. - let name_valid = MirScalarExpr::column(metric_name_idx) + // The published name is `prefix + metric_name`. Planning requires the prefix to start with the + // reserved marker (see `validate_metric_sink_prefix`), so every published family lands in the + // `mz_metric_sink_` lane nothing else in the process registry writes. `TextConcat` (the `||` + // operator) propagates nulls, so a null `metric_name` stays null and is skipped, never + // published as the bare prefix. + let prefixed_name = MirScalarExpr::literal_ok(Datum::String(prefix), ReprScalarType::String) + .call_binary( + MirScalarExpr::column(metric_name_idx), + func::TextConcatBinary, + ); + + // The regexp requires at least one leading character, so an empty published name fails it + // without a separate `!= ""` check. `is_null().not()` keeps a null name concretely `false`, + // not `NULL`. The shaping map appends `prefixed_name` at `arity + 2`, and a map scalar may + // reference earlier-appended columns, so this reads that column instead of rebuilding the + // concat. + let published_name = MirScalarExpr::column(arity + 2); + let name_valid = published_name + .clone() .call_is_null() .not() - .and(MirScalarExpr::column(metric_name_idx).call_binary( + .and(published_name.call_binary( MirScalarExpr::literal_ok(Datum::String(METRIC_NAME_PATTERN), ReprScalarType::String), func::IsRegexpMatchCaseSensitive, )); @@ -125,17 +375,18 @@ fn shape_metric_sink_source( .map(vec![ labels_coalesced, help_coalesced, + prefixed_name, metric_kind, name_valid, ]) .project(vec![ - metric_name_idx, + arity + 2, // prefixed metric_name metric_type_idx, arity, // coalesced labels value_idx, arity + 1, // coalesced help - arity + 2, // metric_kind - arity + 3, // name_valid + arity + 3, // metric_kind + arity + 4, // name_valid ]); let mut labels_shaped_ct = labels_ct.clone(); @@ -157,8 +408,23 @@ fn shape_metric_sink_source( #[cfg(test)] mod tests { + use std::collections::BTreeMap; + + use mz_catalog::memory::objects::{CatalogEntry, CatalogItem, Table, TableDataSource}; + use mz_controller_types::ClusterId; use mz_expr::Eval; - use mz_repr::{RowArena, SqlColumnType}; + use mz_ore::metrics::MetricsRegistry; + use mz_repr::adt::mz_acl_item::PrivilegeMap; + use mz_repr::role_id::RoleId; + use mz_repr::{ + CatalogItemId, RelationVersion, RelationVersionSelector, RowArena, SqlColumnType, + VersionedRelationDesc, + }; + use mz_sql::names::{ + FullItemName, ItemQualifiers, RawDatabaseSpecifier, ResolvedDatabaseSpecifier, ResolvedIds, + SchemaId, SchemaSpecifier, + }; + use mz_sql::session::vars::SystemVars; use super::*; @@ -185,7 +451,8 @@ mod tests { #[mz_ore::test] fn shaped_desc_column_contract() { - let (_expr, desc) = shape_metric_sink_source(GlobalId::Transient(0), &source_desc()); + let (_expr, desc) = + shape_metric_sink_source(GlobalId::Transient(0), &source_desc(), "app_"); let cols: Vec<(String, SqlColumnType)> = desc .iter() @@ -231,9 +498,10 @@ mod tests { #[mz_ore::test] fn shaped_expr_projects_seven_columns() { - let (expr, _desc) = shape_metric_sink_source(GlobalId::Transient(0), &source_desc()); + let (expr, _desc) = + shape_metric_sink_source(GlobalId::Transient(0), &source_desc(), "app_"); - // The shaping is a `Map` of four new columns followed by a `Project` down to the seven + // The shaping is a `Map` of five new columns followed by a `Project` down to the seven // canonical columns. match &expr { MirRelationExpr::Project { outputs, .. } => { @@ -243,11 +511,10 @@ mod tests { } } - /// The four scalars the shaping `Map` appends, in order: - /// `[labels_coalesced, help_coalesced, metric_kind, name_valid]`. Lets the two classification - /// scalars be evaluated directly against an input row. - fn shaped_map_scalars(desc: &RelationDesc) -> Vec { - let (expr, _desc) = shape_metric_sink_source(GlobalId::Transient(0), desc); + /// The five scalars the shaping `Map` appends, in order: + /// `[labels_coalesced, help_coalesced, prefixed_name, metric_kind, name_valid]`. + fn shaped_map_scalars(desc: &RelationDesc, prefix: &str) -> Vec { + let (expr, _desc) = shape_metric_sink_source(GlobalId::Transient(0), desc, prefix); match expr { MirRelationExpr::Project { input, .. } => match *input { MirRelationExpr::Map { scalars, .. } => scalars, @@ -257,10 +524,26 @@ mod tests { } } + /// Evaluate the shaping `Map`'s appended scalars in order, extending the row with each result. + /// A later scalar may reference an earlier appended column (`name_valid` reads the published-name + /// column), so they must be evaluated cumulatively rather than in isolation. + fn eval_shaped_row<'a>( + scalars: &'a [MirScalarExpr], + input: &[Datum<'a>], + arena: &'a RowArena, + ) -> Vec> { + let mut row = input.to_vec(); + for scalar in scalars { + let datum = scalar.eval(&row, arena).expect("scalar eval succeeds"); + row.push(datum); + } + row + } + #[mz_ore::test] fn metric_kind_classifies_type() { - let scalars = shaped_map_scalars(&source_desc()); - let metric_kind = &scalars[2]; + let scalars = shaped_map_scalars(&source_desc(), "app_"); + let metric_kind = &scalars[3]; let arena = RowArena::new(); // Row layout matches `source_desc`: [metric_name, metric_type, labels, value, help, extra]. for (metric_type, expected) in [ @@ -289,18 +572,21 @@ mod tests { #[mz_ore::test] fn name_valid_matches_prometheus_grammar() { - let scalars = shaped_map_scalars(&source_desc()); - let name_valid = &scalars[3]; + // Validation is on the published name `prefix + metric_name`, not the bare `metric_name`. + // The plan-time-validated prefix supplies the valid leading character, so a row name may + // start with a digit (or be empty, publishing the bare prefix). A dash is invalid anywhere, + // and a null name stays null. + let scalars = shaped_map_scalars(&source_desc(), "app_"); let arena = RowArena::new(); for (metric_name, expected) in [ (Datum::String("http_requests_total"), Datum::True), (Datum::String("with:colons_and_1_digit"), Datum::True), - (Datum::String("1_leading_digit"), Datum::False), + (Datum::String("1_leading_digit"), Datum::True), (Datum::String("has-a-dash"), Datum::False), - (Datum::String(""), Datum::False), + (Datum::String(""), Datum::True), (Datum::Null, Datum::False), ] { - let row = [ + let input = [ metric_name, Datum::Null, Datum::Null, @@ -308,13 +594,158 @@ mod tests { Datum::Null, Datum::Null, ]; + // `name_valid` is the last appended scalar and reads the published-name column, so + // evaluate the whole appended row and read its final column. + let row = eval_shaped_row(&scalars, &input, &arena); assert_eq!( - name_valid - .eval(&row, &arena) - .expect("name_valid eval succeeds"), + *row.last().expect("row has appended columns"), expected, "metric_name = {metric_name:?}", ); } } + + /// The smallest catalog the optimizer needs: one table, at `TABLE_GID`, exposing the canonical + /// metric-sink columns. + #[derive(Debug)] + struct SingleTableCatalog { + entry: CatalogEntry, + } + + const TABLE_ITEM_ID: CatalogItemId = CatalogItemId::User(1); + const TABLE_GID: GlobalId = GlobalId::User(1); + const SINK_GID: GlobalId = GlobalId::User(2); + + impl SingleTableCatalog { + fn new() -> Self { + let table = Table { + create_sql: None, + desc: VersionedRelationDesc::new(source_desc()), + collections: BTreeMap::from([(RelationVersion::root(), TABLE_GID)]), + conn_id: None, + resolved_ids: ResolvedIds::empty(), + custom_logical_compaction_window: None, + is_retained_metrics_object: false, + data_source: TableDataSource::TableWrites { + defaults: Vec::new(), + }, + }; + let entry = CatalogEntry { + item: CatalogItem::Table(table), + referenced_by: Vec::new(), + used_by: Vec::new(), + id: TABLE_ITEM_ID, + oid: 20_000, + name: QualifiedItemName { + qualifiers: ItemQualifiers { + database_spec: ResolvedDatabaseSpecifier::Ambient, + schema_spec: SchemaSpecifier::Id(SchemaId::User(1)), + }, + item: "t".to_string(), + }, + owner_id: RoleId::User(1), + privileges: PrivilegeMap::default(), + }; + Self { entry } + } + } + + impl OptimizerCatalog for SingleTableCatalog { + fn get_entry(&self, _id: &GlobalId) -> mz_catalog::memory::objects::CatalogCollectionEntry { + mz_catalog::memory::objects::CatalogCollectionEntry { + entry: self.entry.clone(), + version: RelationVersionSelector::Latest, + } + } + + fn get_entry_by_item_id(&self, _id: &CatalogItemId) -> &CatalogEntry { + &self.entry + } + + fn resolve_full_name( + &self, + name: &QualifiedItemName, + _conn_id: Option<&mz_adapter_types::connection::ConnectionId>, + ) -> FullItemName { + FullItemName { + database: RawDatabaseSpecifier::Ambient, + schema: "public".to_string(), + item: name.item.clone(), + } + } + + fn get_indexes_on( + &self, + _id: GlobalId, + _cluster: ClusterId, + ) -> Box + '_> + { + Box::new(std::iter::empty()) + } + } + + /// The assembled dataflow exports exactly one `MetricSink`, reading the shaped view rather + /// than the source relation directly. + #[mz_ore::test] + fn optimizer_exports_one_metric_sink() { + let catalog = Arc::new(SingleTableCatalog::new()); + let cluster_id = ClusterId::user(1).expect("valid cluster id"); + let compute_instance = ComputeInstanceSnapshot::new_without_collections(cluster_id); + let view_id = GlobalId::Transient(1); + let config = OptimizerConfig::from(&SystemVars::default()); + let metrics = OptimizerMetrics::register_into(&MetricsRegistry::new(), Duration::MAX); + + let mut optimizer = Optimizer::new( + catalog, + compute_instance, + view_id, + SINK_GID, + config, + metrics, + ); + + let name = QualifiedItemName { + qualifiers: ItemQualifiers { + database_spec: ResolvedDatabaseSpecifier::Ambient, + schema_spec: SchemaSpecifier::Id(SchemaId::User(1)), + }, + item: "s".to_string(), + }; + let global_mir_plan = optimizer + .optimize(MetricSink::new(name, TABLE_GID, "app_".to_string())) + .expect("MIR optimization succeeds"); + let global_lir_plan = optimizer + .optimize(global_mir_plan) + .expect("LIR optimization succeeds"); + + let df_desc = global_lir_plan.df_desc(); + assert!(df_desc.index_exports.is_empty()); + let sink_exports: Vec<_> = df_desc.sink_exports.iter().collect(); + assert_eq!(sink_exports.len(), 1); + let (sink_id, sink_desc) = sink_exports[0]; + assert_eq!(*sink_id, SINK_GID); + assert!(matches!( + sink_desc.connection, + ComputeSinkConnection::MetricSink(_) + )); + // The sink reads the shaped view, whose desc carries the operator's column contract. + assert_eq!(sink_desc.from, view_id); + let shaped_names: Vec<&str> = sink_desc + .from_desc + .iter_names() + .map(|n| n.as_str()) + .collect(); + assert_eq!( + shaped_names, + vec![ + "metric_name", + "metric_type", + "labels", + "value", + "help", + "metric_kind", + "name_valid", + ] + ); + } } diff --git a/src/catalog/src/builtin/mz_introspection.rs b/src/catalog/src/builtin/mz_introspection.rs index fbb38b7bd4ec2..d5ad93b478809 100644 --- a/src/catalog/src/builtin/mz_introspection.rs +++ b/src/catalog/src/builtin/mz_introspection.rs @@ -324,7 +324,8 @@ pub static MZ_COMPUTE_HYDRATION_TIMES_PER_WORKER: LazyLock = access: vec![PUBLIC_SELECT], ontology: Some(Ontology { entity_name: "hydration_time_per_worker", - description: "Time in nanoseconds for each compute export to hydrate per worker.", + description: "Hydration duration and lifecycle timestamps for each compute export \ + per worker.", links: &const { [OntologyLink { name: "hydration_time_of", diff --git a/src/catalog/src/memory/objects.rs b/src/catalog/src/memory/objects.rs index 7acccc9492e23..200eb401441fe 100644 --- a/src/catalog/src/memory/objects.rs +++ b/src/catalog/src/memory/objects.rs @@ -2540,16 +2540,15 @@ impl CatalogItem { /// per-replica hydration check would resolve vacuously. pub fn is_hydratable(&self) -> bool { match self { - CatalogItem::Index(_) | CatalogItem::MaterializedView(_) | CatalogItem::Sink(_) => true, + CatalogItem::Index(_) + | CatalogItem::MaterializedView(_) + | CatalogItem::Sink(_) + | CatalogItem::MetricSink(_) => true, CatalogItem::Source(source) => matches!( source.data_source, DataSourceDesc::Ingestion { .. } | DataSourceDesc::OldSyntaxIngestion { .. } ), - // TODO(SQL-571): return true once metric sinks run a per-replica dataflow, and - // move this arm out then. Reporting `false` until then is safe: a sink with no - // real hydration state lets graceful reconfiguration cut over without waiting. - CatalogItem::MetricSink(_) - | CatalogItem::Table(_) + CatalogItem::Table(_) | CatalogItem::Log(_) | CatalogItem::View(_) | CatalogItem::Type(_) diff --git a/src/clusterd/src/lib.rs b/src/clusterd/src/lib.rs index 148cf0471d961..4d202a6fcaabd 100644 --- a/src/clusterd/src/lib.rs +++ b/src/clusterd/src/lib.rs @@ -21,7 +21,7 @@ use hyper_util::rt::TokioIo; use mz_build_info::{BuildInfo, build_info}; use mz_cloud_resources::AwsExternalIdPrefix; use mz_cluster_client::client::TimelyConfig; -use mz_compute::server::ComputeInstanceContext; +use mz_compute::server::{ComputeInstanceContext, ComputeRuntimeRole}; use mz_http_util::DynamicFilterTarget; use mz_orchestrator_tracing::{StaticTracingConfig, TracingCliArgs}; use mz_ore::cli::{self, CliConfig}; @@ -470,6 +470,7 @@ async fn run(args: Args) -> Result<(), anyhow::Error> { // Start compute server. let compute_client_builder = mz_compute::server::serve( compute_timely_config, + ComputeRuntimeRole::Solo, &metrics_registry, persist_clients, txns_ctx, diff --git a/src/compute-client/src/controller.rs b/src/compute-client/src/controller.rs index 33c3d9a1a2672..aa4c1707b899e 100644 --- a/src/compute-client/src/controller.rs +++ b/src/compute-client/src/controller.rs @@ -778,7 +778,10 @@ impl ComputeController { /// Creates the described dataflow and initializes state for its output. /// - /// Only materialized views and subscribes are allowed to have a `target_replica`. + /// Only sink exports are allowed to have a `target_replica`: materialized views and subscribes. + /// Metric sinks are sink exports too, and nothing here forbids targeting one, but by caller + /// convention they always pass `target_replica: None` so each replica renders the sink into its + /// own registry for per-replica introspection. /// /// Panics if called with a dataflow description that has index exports /// when `target_replica` is set. diff --git a/src/compute-client/src/logging.rs b/src/compute-client/src/logging.rs index ae35867136cc6..d3b788af63efe 100644 --- a/src/compute-client/src/logging.rs +++ b/src/compute-client/src/logging.rs @@ -355,6 +355,18 @@ impl LogVariant { .with_column("export_id", SqlScalarType::String.nullable(false)) .with_column("worker_id", SqlScalarType::UInt64.nullable(false)) .with_column("time_ns", SqlScalarType::UInt64.nullable(true)) + .with_column( + "installed_at", + SqlScalarType::TimestampTz { precision: None }.nullable(false), + ) + .with_column( + "started_at", + SqlScalarType::TimestampTz { precision: None }.nullable(true), + ) + .with_column( + "hydrated_at", + SqlScalarType::TimestampTz { precision: None }.nullable(true), + ) .with_key(vec![0, 1]) .finish(), diff --git a/src/compute/Cargo.toml b/src/compute/Cargo.toml index eaed8bd235694..9300ebd57da87 100644 --- a/src/compute/Cargo.toml +++ b/src/compute/Cargo.toml @@ -13,6 +13,7 @@ workspace = true anyhow.workspace = true async-stream.workspace = true bytesize.workspace = true +chrono.workspace = true columnar.workspace = true columnation.workspace = true dec = { workspace = true, features = ["serde"] } diff --git a/src/compute/src/compute_state.rs b/src/compute/src/compute_state.rs index e20ef9266ec9f..3a2f7b42e2697 100644 --- a/src/compute/src/compute_state.rs +++ b/src/compute/src/compute_state.rs @@ -668,6 +668,16 @@ impl<'a> ActiveComputeState<'a> { .chain(dataflow.copy_to_ids()) .collect(); + // `StartSignal` is attached only to imported sources and imported indexes, and + // `import_ids` is exactly those two sets, so a dataflow with no imports has nothing + // suspended and begins computing as soon as it is rendered. Such a dataflow can reach + // hydration before its `Schedule` arrives, and the controller sends one anyway to keep + // protocol communication predictable, so a `started_at` stamped only from + // `handle_schedule` would land after `hydrated_at`. Stamping it here also keeps the row + // truthful from the moment it appears, rather than reporting the object as queued while + // nothing is queueing it. + let starts_immediately = dataflow.import_ids().next().is_none(); + // Initialize compute and logging state for each object. for object_id in dataflow.export_ids() { let is_subscribe_or_copy = subscribe_copy_ids.contains(&object_id); @@ -686,6 +696,9 @@ impl<'a> ActiveComputeState<'a> { *dataflow_index, dataflow.import_ids(), ); + if starts_immediately { + logging.set_hydration_start(); + } collection.logging = Some(logging); } @@ -727,6 +740,12 @@ impl<'a> ActiveComputeState<'a> { // scheduled. let suspension_token = self.compute_state.suspended_collections.remove(&id); drop(suspension_token); + + if let Some(collection) = self.compute_state.collections.get(&id) { + if let Some(logging) = &collection.logging { + logging.set_hydration_start(); + } + } } fn handle_allow_compaction(&mut self, id: GlobalId, frontier: Antichain) { @@ -871,6 +890,11 @@ impl<'a> ActiveComputeState<'a> { let logging = CollectionLogging::new(id, logger.clone(), *dataflow_index, std::iter::empty()); + // Log collections are never suspended and the controller marks them scheduled + // implicitly, so no `Schedule` command ever arrives for them. Record their hydration + // start here, or they would sit permanently in the illegal state of being hydrated + // without having started. + logging.set_hydration_start(); collection.logging = Some(logging); let existing = self.compute_state.collections.insert(id, collection); diff --git a/src/compute/src/logging/compute.rs b/src/compute/src/logging/compute.rs index 54acf1019eea2..065d51fceb240 100644 --- a/src/compute/src/logging/compute.rs +++ b/src/compute/src/logging/compute.rs @@ -15,12 +15,14 @@ use std::fmt::{Display, Write}; use std::rc::Rc; use std::time::{Duration, Instant}; +use chrono::DateTime; use columnar::{Columnar, Index, Ref}; use differential_dataflow::VecCollection; use differential_dataflow::collection::AsCollection; use differential_dataflow::trace::{BatchReader, Cursor, Navigable}; use mz_compute_types::plan::LirId; use mz_ore::cast::CastFrom; +use mz_repr::adt::timestamp::CheckedTimestamp; use mz_repr::{Datum, Diff, GlobalId, Row, RowRef, Timestamp}; use mz_timely_util::columnar::batcher; use mz_timely_util::columnar::builder::ColumnBuilder; @@ -156,6 +158,13 @@ pub struct ErrorCount { pub diff: Diff, } +/// An export started hydrating. +#[derive(Debug, Clone, PartialOrd, PartialEq, Columnar)] +pub struct HydrationStart { + /// Identifier of the export. + pub export_id: GlobalId, +} + /// An export is hydrated. #[derive(Debug, Clone, PartialOrd, PartialEq, Columnar)] pub struct Hydration { @@ -225,6 +234,8 @@ pub enum ComputeEvent { DataflowShutdown(DataflowShutdown), /// The number of errors in a dataflow export has changed. ErrorCount(ErrorCount), + /// A dataflow export started hydrating, i.e. its dataflow was unsuspended. + HydrationStart(HydrationStart), /// A dataflow export was hydrated. Hydration(Hydration), /// A dataflow operator's hydration status changed. @@ -468,6 +479,21 @@ where Datum::String(scratch) } +/// Pack an offset from the Unix epoch into a `Datum::TimestampTz`. +/// +/// The offset is rounded to microseconds, the maximum resolution of `timestamptz`. Rounding is +/// monotone, so it cannot reorder two offsets that were ordered before. +fn epoch_offset_datum(offset: Duration) -> Datum<'static> { + let secs = i64::try_from(offset.as_secs()).expect("must fit"); + let datetime = DateTime::from_timestamp(secs, offset.subsec_nanos()) + .expect("epoch offset is in range for `DateTime`"); + let timestamp = CheckedTimestamp::try_from(datetime) + .expect("epoch offset is a valid timestamp") + .round_to_precision(None) + .expect("epoch offset is far from the maximum timestamp"); + Datum::TimestampTz(timestamp) +} + /// State maintained by the demux operator. struct DemuxState { /// The timely activations handle. @@ -615,16 +641,25 @@ impl DemuxState { ]) } - /// Pack a hydration time update key-value for the given export ID and hydration time. + /// Pack a hydration time update key-value for the given export ID, hydration time, and + /// hydration lifecycle timestamps. fn pack_hydration_time_update( &mut self, export_id: GlobalId, time_ns: Option, + timestamps: &HydrationTimestamps, ) -> (&RowRef, &RowRef) { self.hydration_time_packer.pack_slice(&[ make_string_datum(export_id, &mut self.scratch_string_a), Datum::UInt64(u64::cast_from(self.worker_id)), Datum::from(time_ns), + epoch_offset_datum(timestamps.installed_at), + timestamps + .started_at + .map_or(Datum::Null, epoch_offset_datum), + timestamps + .hydrated_at + .map_or(Datum::Null, epoch_offset_datum), ]) } @@ -721,6 +756,28 @@ impl DemuxState { } } +/// Wallclock instants of an export's hydration lifecycle, as durations since the Unix epoch. +/// +/// These are sampled from compute logging event times, which advance off an `Instant` anchored to +/// the epoch once per worker when logging is initialized. They are therefore monotone within a +/// worker and unaffected by system clock steps, but they carry that worker's anchor, so comparing +/// them across workers absorbs anchor skew. +/// +/// The invariant `installed_at <= started_at <= hydrated_at` holds over the non-NULL values, and no +/// later stage is ever set without its predecessors. +#[derive(Debug, Clone, Copy)] +struct HydrationTimestamps { + /// When this export's dataflow was installed, still suspended. Also the start of queueing. + installed_at: Duration, + /// When hydration work began, that is, when the dataflow was unsuspended. + /// + /// Equals `installed_at` for a dataflow that was never suspended, since such a dataflow starts + /// running the moment it is built. + started_at: Option, + /// When this export's output frontier passed the dataflow as-of. + hydrated_at: Option, +} + /// State tracked for each dataflow export. struct ExportState { /// The ID of the dataflow maintaining this export. @@ -734,17 +791,28 @@ struct ExportState { created_at: Instant, /// Whether the exported collection is hydrated. hydration_time_ns: Option, + /// Wallclock instants of this export's hydration lifecycle. + /// + /// Distinct from `created_at`/`hydration_time_ns`, which remain the authoritative per-worker + /// duration at nanosecond precision. These instants exist to identify and bound hydration + /// episodes, which a duration cannot do. + hydration_timestamps: HydrationTimestamps, /// Hydration status of operators feeding this export. operator_hydration: BTreeMap, } impl ExportState { - fn new(dataflow_index: usize) -> Self { + fn new(dataflow_index: usize, installed_at: Duration) -> Self { Self { dataflow_index, error_count: Diff::ZERO, created_at: Instant::now(), hydration_time_ns: None, + hydration_timestamps: HydrationTimestamps { + installed_at, + started_at: None, + hydrated_at: None, + }, operator_hydration: BTreeMap::new(), } } @@ -818,6 +886,7 @@ impl DemuxHandler<'_, '_, '_> { } DataflowShutdown(shutdown) => self.handle_dataflow_shutdown(shutdown), ErrorCount(error_count) => self.handle_error_count(error_count), + HydrationStart(hydration) => self.handle_hydration_start(hydration), Hydration(hydration) => self.handle_hydration(hydration), OperatorHydration(hydration) => self.handle_operator_hydration(hydration), LirMapping(mapping) => self.handle_lir_mapping(mapping), @@ -837,16 +906,27 @@ impl DemuxHandler<'_, '_, '_> { let datum = self.state.pack_export_update(export_id, dataflow_index); self.output.export.give((datum, ts, Diff::ONE)); + // Stamp the event time, not `ts`, which is rounded up to the logging interval. The rounding + // then only delays when an update becomes visible, rather than skewing recorded instants. + let installed_at = self.time; + let existing = self .state .exports - .insert(export_id, ExportState::new(dataflow_index)); + .insert(export_id, ExportState::new(dataflow_index, installed_at)); if existing.is_some() { error!(%export_id, "export already registered"); } // Insert hydration time logging for this export. - let datum = self.state.pack_hydration_time_update(export_id, None); + let timestamps = HydrationTimestamps { + installed_at, + started_at: None, + hydrated_at: None, + }; + let datum = self + .state + .pack_hydration_time_update(export_id, None, ×tamps); self.output.hydration_time.give((datum, ts, Diff::ONE)); } @@ -875,9 +955,11 @@ impl DemuxHandler<'_, '_, '_> { } // Remove hydration time logging for this export. - let datum = self - .state - .pack_hydration_time_update(export_id, export.hydration_time_ns); + let datum = self.state.pack_hydration_time_update( + export_id, + export.hydration_time_ns, + &export.hydration_timestamps, + ); self.output .hydration_time .give((datum, ts, Diff::MINUS_ONE)); @@ -961,8 +1043,47 @@ impl DemuxHandler<'_, '_, '_> { } } + fn handle_hydration_start( + &mut self, + HydrationStartReference { export_id }: Ref<'_, HydrationStart>, + ) { + let ts = self.ts(); + // Stamp the event time rather than `ts`, as in `handle_export`. + let started_at = self.time; + let export_id = Columnar::into_owned(export_id); + + let Some(export) = self.state.exports.get_mut(&export_id) else { + error!(%export_id, "hydration start event for unknown export"); + return; + }; + if export.hydration_timestamps.started_at.is_some() { + // A `Schedule` command is re-sent for dataflows retained across reconciliation, and is + // sent even for dataflows that were never suspended and have hydrated already. Ignore + // the repeats, mirroring the guard in `handle_hydration`. + return; + } + + let old_timestamps = export.hydration_timestamps; + export.hydration_timestamps.started_at = Some(started_at); + let new_timestamps = export.hydration_timestamps; + let time_ns = export.hydration_time_ns; + + let retraction = self + .state + .pack_hydration_time_update(export_id, time_ns, &old_timestamps); + self.output + .hydration_time + .give((retraction, ts, Diff::MINUS_ONE)); + let insertion = self + .state + .pack_hydration_time_update(export_id, time_ns, &new_timestamps); + self.output.hydration_time.give((insertion, ts, Diff::ONE)); + } + fn handle_hydration(&mut self, HydrationReference { export_id }: Ref<'_, Hydration>) { let ts = self.ts(); + // Stamp the event time rather than `ts`, as in `handle_export`. + let hydrated_at = self.time; let export_id = Columnar::into_owned(export_id); let Some(export) = self.state.exports.get_mut(&export_id) else { @@ -979,13 +1100,31 @@ impl DemuxHandler<'_, '_, '_> { let nanos = u64::try_from(duration.as_nanos()).expect("must fit"); export.hydration_time_ns = Some(nanos); - let retraction = self.state.pack_hydration_time_update(export_id, None); + let old_timestamps = export.hydration_timestamps; + export.hydration_timestamps.hydrated_at = Some(hydrated_at); + // A dataflow can reach hydration before its `Schedule` arrives, and not only when it has + // no imports to suspend: an index over an already-hydrated arrangement reports hydration + // while still suspended, which happens for a handful of `mz_catalog_server` indexes on + // every bootstrap. So this is a normal path, not a repair for an exotic one. + // + // Stamp `started_at` from `installed_at`, which keeps `installed_at <= started_at <= + // hydrated_at` total and reports the queueing interval as zero. Stamping `hydrated_at` + // instead would invert it, charging the whole life to queueing and reporting zero + // hydration time for a dataflow that only ever hydrated. + if export.hydration_timestamps.started_at.is_none() { + export.hydration_timestamps.started_at = Some(export.hydration_timestamps.installed_at); + } + let new_timestamps = export.hydration_timestamps; + + let retraction = self + .state + .pack_hydration_time_update(export_id, None, &old_timestamps); self.output .hydration_time .give((retraction, ts, Diff::MINUS_ONE)); - let insertion = self - .state - .pack_hydration_time_update(export_id, Some(nanos)); + let insertion = + self.state + .pack_hydration_time_update(export_id, Some(nanos), &new_timestamps); self.output.hydration_time.give((insertion, ts, Diff::ONE)); } @@ -1394,6 +1533,17 @@ impl CollectionLogging { } } + /// Record that the collection's dataflow was unsuspended, so hydration work has begun. + /// + /// Repeated calls are ignored by the demux, so callers need not track whether the dataflow was + /// already unsuspended. + pub fn set_hydration_start(&self) { + self.logger + .log(&ComputeEvent::HydrationStart(HydrationStart { + export_id: self.export_id, + })); + } + /// Set the collection as hydrated. pub fn set_hydrated(&self) { self.logger.log(&ComputeEvent::Hydration(Hydration { diff --git a/src/compute/src/metrics.rs b/src/compute/src/metrics.rs index 9ca9f085e2ff8..16861f1b378e3 100644 --- a/src/compute/src/metrics.rs +++ b/src/compute/src/metrics.rs @@ -12,7 +12,9 @@ use std::sync::{Arc, Mutex}; use mz_compute_client::metrics::{CommandMetrics, HistoryMetrics}; use mz_ore::cast::CastFrom; use mz_ore::metric; -use mz_ore::metrics::{MetricTag, MetricVisibility, MetricsRegistry, UIntGauge, raw}; +use mz_ore::metrics::{ + MakeCollectorOpts, MetricTag, MetricVisibility, MetricsRegistry, UIntGauge, raw, +}; use mz_repr::{GlobalId, SharedRow}; use prometheus::core::{AtomicF64, GenericCounter}; use prometheus::proto::LabelPair; @@ -85,8 +87,32 @@ pub struct ComputeMetrics { subscribe_snapshots_skipped_total: IntCounter, } +/// Applies the per-role const label to `opts`, unless `role` is `Solo`. +/// +/// The two named roles (maintenance, interactive) each get a distinct `role` label so a second +/// compute runtime in the same process registers a distinct series rather than colliding with the +/// first. `Solo` omits the label so a single-runtime deployment registers exactly as it did before +/// a second runtime existed. +fn with_role( + mut opts: MakeCollectorOpts, + role: crate::server::ComputeRuntimeRole, +) -> MakeCollectorOpts { + if let Some(label) = role.label() { + opts.opts = opts.opts.const_label("role", label); + } + opts +} + impl ComputeMetrics { - pub fn register_with(registry: &MetricsRegistry) -> Self { + /// Registers the compute metrics for `role` into `registry`. + /// + /// The two named roles carry a `role` const label so that a second compute runtime in the same + /// process registers a distinct series rather than colliding with the first. `Solo` carries no + /// such label. + pub fn register_with( + registry: &MetricsRegistry, + role: crate::server::ComputeRuntimeRole, + ) -> Self { let workload_class = Arc::new(Mutex::new(None)); let mut index_peek_row_buckets = prometheus::exponential_buckets(1.0, 2.0, 25).expect("valid parameters"); @@ -94,164 +120,170 @@ impl ComputeMetrics { // Apply a `workload_class` label to all metrics in the registry when we // have a known workload class. - registry.register_postprocessor({ - let workload_class = Arc::clone(&workload_class); - move |metrics| { - let workload_class: Option = - workload_class.lock().expect("lock poisoned").clone(); - let Some(workload_class) = workload_class else { - return; - }; - for metric in metrics { - for metric in metric.mut_metric() { - let mut label = LabelPair::default(); - label.set_name("workload_class".into()); - label.set_value(workload_class.clone()); - - let mut labels = metric.take_label(); - labels.push(label); - metric.set_label(labels); + // + // The postprocessor rewrites every metric in the whole registry, so only the maintenance + // runtime registers it. A second registration from the interactive runtime would push the + // label twice onto each metric and produce a duplicate-label scrape error. + if role.owns_process_globals() { + registry.register_postprocessor({ + let workload_class = Arc::clone(&workload_class); + move |metrics| { + let workload_class: Option = + workload_class.lock().expect("lock poisoned").clone(); + let Some(workload_class) = workload_class else { + return; + }; + for metric in metrics { + for metric in metric.mut_metric() { + let mut label = LabelPair::default(); + label.set_name("workload_class".into()); + label.set_value(workload_class.clone()); + + let mut labels = metric.take_label(); + labels.push(label); + metric.set_label(labels); + } } } - } - }); + }); + } Self { workload_class, - history_command_count: registry.register(metric!( + history_command_count: registry.register(with_role(metric!( name: "mz_compute_replica_history_command_count", help: "The number of commands in the replica's command history.", var_labels: ["worker_id", "command_type"], - )), - history_dataflow_count: registry.register(metric!( + ), role)), + history_dataflow_count: registry.register(with_role(metric!( name: "mz_compute_replica_history_dataflow_count", help: "The number of dataflows in the replica's command history.", var_labels: ["worker_id"], visibility: MetricVisibility::Public, tags: [MetricTag::Compute], - )), - reconciliation_reused_dataflows_count_total: registry.register(metric!( + ), role)), + reconciliation_reused_dataflows_count_total: registry.register(with_role(metric!( name: "mz_compute_reconciliation_reused_dataflows_count_total", help: "The total number of dataflows that were reused during compute reconciliation.", var_labels: ["worker_id"], - )), - reconciliation_replaced_dataflows_count_total: registry.register(metric!( + ), role)), + reconciliation_replaced_dataflows_count_total: registry.register(with_role(metric!( name: "mz_compute_reconciliation_replaced_dataflows_count_total", help: "The total number of dataflows that were replaced during compute reconciliation.", var_labels: ["worker_id", "reason"], - )), - arrangement_maintenance_seconds_total: registry.register(metric!( + ), role)), + arrangement_maintenance_seconds_total: registry.register(with_role(metric!( name: "mz_arrangement_maintenance_seconds_total", help: "The total time spent maintaining arrangements.", var_labels: ["worker_id"], visibility: MetricVisibility::Public, tags: [MetricTag::Compute], - )), - arrangement_maintenance_active_info: registry.register(metric!( + ), role)), + arrangement_maintenance_active_info: registry.register(with_role(metric!( name: "mz_arrangement_maintenance_active_info", help: "Whether maintenance is currently occuring.", var_labels: ["worker_id"], - )), - timely_step_duration_seconds: registry.register(metric!( + ), role)), + timely_step_duration_seconds: registry.register(with_role(metric!( name: "mz_timely_step_duration_seconds", help: "The time spent in each compute step_or_park call", const_labels: {"cluster" => "compute"}, var_labels: ["worker_id"], buckets: mz_ore::stats::histogram_seconds_buckets(0.000_128, 32.0), - )), - shared_row_heap_capacity_bytes: registry.register(metric!( + ), role)), + shared_row_heap_capacity_bytes: registry.register(with_role(metric!( name: "mz_dataflow_shared_row_heap_capacity_bytes", help: "The heap capacity of the shared row.", var_labels: ["worker_id"], - )), - persist_peek_seconds: registry.register(metric!( + ), role)), + persist_peek_seconds: registry.register(with_role(metric!( name: "mz_persist_peek_seconds", help: "Time spent in (experimental) Persist fast-path peeks.", var_labels: ["worker_id"], buckets: mz_ore::stats::histogram_seconds_buckets(0.000_128, 8.0), - )), - stashed_peek_seconds: registry.register(metric!( + ), role)), + stashed_peek_seconds: registry.register(with_role(metric!( name: "mz_stashed_peek_seconds", help: "Time spent reading a peek result and stashing it in the peek result stash (aka. persist blob).", var_labels: ["worker_id"], buckets: mz_ore::stats::histogram_seconds_buckets(0.000_128, 8.0), - )), - handle_command_duration_seconds: registry.register(metric!( + ), role)), + handle_command_duration_seconds: registry.register(with_role(metric!( name: "mz_cluster_handle_command_duration_seconds", help: "Time spent in handling commands.", const_labels: {"cluster" => "compute"}, var_labels: ["worker_id", "command_type"], buckets: mz_ore::stats::histogram_seconds_buckets(0.000_128, 8.0), - )), - index_peek_total_seconds: registry.register(metric!( + ), role)), + index_peek_total_seconds: registry.register(with_role(metric!( name: "mz_index_peek_total_seconds", help: "Total time processing index peeks, from process_peek entry to response. Excluding peeks that use the peek response stash.", buckets: mz_ore::stats::histogram_seconds_buckets(0.000_128, 8.0), - )), - index_peek_seek_fulfillment_seconds: registry.register(metric!( + ), role)), + index_peek_seek_fulfillment_seconds: registry.register(with_role(metric!( name: "mz_index_peek_seek_fulfillment_seconds", help: "Time in seek_fulfillment method including frontier checks and data collection.", buckets: mz_ore::stats::histogram_seconds_buckets(0.000_128, 8.0), - )), - index_peek_error_scan_seconds: registry.register(metric!( + ), role)), + index_peek_error_scan_seconds: registry.register(with_role(metric!( name: "mz_index_peek_error_scan_seconds", help: "Time scanning the error trace for errors.", buckets: mz_ore::stats::histogram_seconds_buckets(0.000_128, 8.0), - )), - index_peek_cursor_setup_seconds: registry.register(metric!( + ), role)), + index_peek_cursor_setup_seconds: registry.register(with_role(metric!( name: "mz_index_peek_cursor_setup_seconds", help: "Time setting up cursor and literal constraints.", buckets: mz_ore::stats::histogram_seconds_buckets(0.000_128, 8.0), - )), - index_peek_row_iteration_seconds: registry.register(metric!( + ), role)), + index_peek_row_iteration_seconds: registry.register(with_role(metric!( name: "mz_index_peek_row_iteration_seconds", help: "Time iterating rows and evaluating MFP.", buckets: mz_ore::stats::histogram_seconds_buckets(0.000_128, 8.0), - )), - index_peek_row_iteration_rows: registry.register(metric!( + ), role)), + index_peek_row_iteration_rows: registry.register(with_role(metric!( name: "mz_index_peek_row_iteration_rows", help: "Number of arrangement rows evaluated by the index peek result iterator.", buckets: index_peek_row_buckets.clone(), - )), - index_peek_result_sort_seconds: registry.register(metric!( + ), role)), + index_peek_result_sort_seconds: registry.register(with_role(metric!( name: "mz_index_peek_result_sort_seconds", help: "Time sorting intermediate results during peek collection.", buckets: mz_ore::stats::histogram_seconds_buckets(0.000_128, 8.0), - )), - index_peek_result_sort_rows: registry.register(metric!( + ), role)), + index_peek_result_sort_rows: registry.register(with_role(metric!( name: "mz_index_peek_result_sort_rows", help: "Number of intermediate result rows sorted during peek collection, summed across sort operations.", buckets: index_peek_row_buckets, - )), - index_peek_frontier_check_seconds: registry.register(metric!( + ), role)), + index_peek_frontier_check_seconds: registry.register(with_role(metric!( name: "mz_index_peek_frontier_check_seconds", help: "Time checking trace frontiers.", buckets: mz_ore::stats::histogram_seconds_buckets(0.000_128, 8.0), - )), - index_peek_row_collection_seconds: registry.register(metric!( + ), role)), + index_peek_row_collection_seconds: registry.register(with_role(metric!( name: "mz_index_peek_row_collection_seconds", help: "Time constructing RowCollection from peek results.", buckets: mz_ore::stats::histogram_seconds_buckets(0.000_128, 8.0), - )), - replica_expiration_timestamp_seconds: registry.register(metric!( + ), role)), + replica_expiration_timestamp_seconds: registry.register(with_role(metric!( name: "mz_dataflow_replica_expiration_timestamp_seconds", help: "The replica expiration timestamp in seconds since epoch.", var_labels: ["worker_id"], - )), - replica_expiration_remaining_seconds: registry.register(metric!( + ), role)), + replica_expiration_remaining_seconds: registry.register(with_role(metric!( name: "mz_dataflow_replica_expiration_remaining_seconds", help: "The remaining seconds until replica expiration. Can go negative, can lag behind.", var_labels: ["worker_id"], - )), - collection_count: registry.register(metric!( + ), role)), + collection_count: registry.register(with_role(metric!( name: "mz_compute_collection_count", help: "The number and hydration status of maintained compute collections.", var_labels: ["worker_id", "type", "hydrated"], - )), - subscribe_snapshots_skipped_total: registry.register(metric!( + ), role)), + subscribe_snapshots_skipped_total: registry.register(with_role(metric!( name: "mz_subscribe_snapshots_skipped_total", help: "The number of collection snapshots that were skipped by the subscribe snapshot optimization.", - )), + ), role)), } } @@ -537,3 +569,64 @@ impl Drop for CollectionMetrics { .dec_collection_count(self.collection_type, self.collection_hydrated); } } + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use mz_ore::metrics::MetricsRegistry; + + use super::ComputeMetrics; + use crate::server::ComputeRuntimeRole; + + /// The `Solo` (single-runtime) role registers exactly as compute did before a second runtime + /// existed: no metric carries a `role` label, so single-runtime dashboards and exact-match + /// alerts are byte-unchanged. + #[mz_ore::test] + fn solo_runtime_omits_role_label() { + let registry = MetricsRegistry::new(); + let metrics = ComputeMetrics::register_with(®istry, ComputeRuntimeRole::Solo); + // Instantiate the per-worker children so the `*Vec` families emit rows to inspect. + let _worker = metrics.for_worker(0); + + for family in registry.gather() { + for metric in family.get_metric() { + for label in metric.get_label() { + assert_ne!( + label.name(), + "role", + "solo metric {} unexpectedly carries a role label", + family.name(), + ); + } + } + } + } + + /// The two named roles each carry their own `role` label, so two runtimes in one process + /// register distinct series rather than colliding. Registering both on one registry also + /// exercises the non-collision that lets them coexist. + #[mz_ore::test] + fn named_roles_carry_distinct_role_label() { + let registry = MetricsRegistry::new(); + let maintenance = ComputeMetrics::register_with(®istry, ComputeRuntimeRole::Maintenance); + let interactive = ComputeMetrics::register_with(®istry, ComputeRuntimeRole::Interactive); + let _maintenance_worker = maintenance.for_worker(0); + let _interactive_worker = interactive.for_worker(0); + + let mut roles = BTreeSet::new(); + for family in registry.gather() { + for metric in family.get_metric() { + let role = metric + .get_label() + .iter() + .find(|label| label.name() == "role") + .unwrap_or_else(|| panic!("metric {} missing a role label", family.name())); + roles.insert(role.value().to_string()); + } + } + + assert!(roles.contains("maintenance"), "roles seen: {roles:?}"); + assert!(roles.contains("interactive"), "roles seen: {roles:?}"); + } +} diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs index 3588044852835..3fdbd78e0060c 100644 --- a/src/compute/src/server.rs +++ b/src/compute/src/server.rs @@ -56,6 +56,70 @@ pub struct ComputeInstanceContext { pub connection_context: ConnectionContext, } +/// Which of a process's compute runtimes a given runtime is. +/// +/// A clusterd process runs a single `Solo` runtime by default. When an interactive runtime is +/// configured, the process instead runs a `Maintenance` and an `Interactive` runtime side by side. +/// The named roles share per-process resources (persist cache, metrics registry, log spans). The +/// role distinguishes them so that only the globals-owning runtime runs the non-idempotent +/// process-global initializers, and so metric series and log spans do not collide. +/// +/// `Solo` exists so the single-runtime default stays behaviorally identical to a deployment without +/// a second runtime: no `role` metric label, and it owns the process globals just as the sole +/// runtime always has. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ComputeRuntimeRole { + /// The sole runtime of a single-runtime process. Owns index maintenance and the process-global + /// initializers. + Solo, + /// The maintenance runtime of a two-runtime process. Owns index maintenance and the + /// process-global initializers. + Maintenance, + /// The interactive runtime of a two-runtime process. Shares the process globals owned by + /// maintenance and serves reads. + /// + /// Test-only until the interactive runtime exists to construct it. It is present because the + /// `role` label's entire purpose is that two named roles register into one process registry + /// without colliding, and nothing else can express that: `Solo` registers the same metric names + /// with no `role` label, so prometheus rejects it alongside a named role for differing label + /// dimensions rather than treating it as a second series. Verifying non-collision therefore + /// needs a second *named* role. + /// + /// TODO: drop the `cfg` when the interactive runtime lands and constructs this. + #[cfg(test)] + Interactive, +} + +impl ComputeRuntimeRole { + /// The `role` metric/log label for this role, or `None` for `Solo`. + /// + /// `Solo` omits the label so a single-runtime deployment registers exactly as it did before a + /// second runtime existed, keeping exact-match dashboards and alerts unchanged. + pub fn label(self) -> Option<&'static str> { + match self { + ComputeRuntimeRole::Solo => None, + ComputeRuntimeRole::Maintenance => Some("maintenance"), + #[cfg(test)] + ComputeRuntimeRole::Interactive => Some("interactive"), + } + } + + /// Whether this role runs the non-idempotent, process-global initializers. + /// + /// `Solo` and `Maintenance` run them. An interactive runtime shares the same process and + /// inherits the globals maintenance installs, so re-running them would either double-apply a + /// non-idempotent effect or race maintenance. + /// + /// NOTE: every role a release build can construct owns the globals, so this is constantly true + /// outside tests. The distinction becomes load-bearing when the interactive runtime lands. + pub fn owns_process_globals(self) -> bool { + matches!( + self, + ComputeRuntimeRole::Solo | ComputeRuntimeRole::Maintenance + ) + } +} + /// Type alias for the storage timely log reader. pub(crate) type StorageTimelyLogReader = Arc>>; @@ -84,6 +148,7 @@ struct Config { /// Initiates a timely dataflow computation, processing compute commands. pub async fn serve( timely_config: TimelyConfig, + role: ComputeRuntimeRole, metrics_registry: &MetricsRegistry, persist_clients: Arc, txns_ctx: TxnsContext, @@ -111,7 +176,7 @@ pub async fn serve( persist_clients, txns_ctx, tracing_handle, - metrics: ComputeMetrics::register_with(metrics_registry), + metrics: ComputeMetrics::register_with(metrics_registry, role), context, metrics_registry: metrics_registry.clone(), workers_per_process, diff --git a/src/compute/src/sink/correction.rs b/src/compute/src/sink/correction.rs index 78c0fc696e142..e046f46da27ce 100644 --- a/src/compute/src/sink/correction.rs +++ b/src/compute/src/sink/correction.rs @@ -17,13 +17,12 @@ use std::ops::{AddAssign, Bound, RangeBounds, SubAssign}; use differential_dataflow::consolidation::{consolidate, consolidate_updates}; use differential_dataflow::logging::{BatchEvent, DropEvent}; -use itertools::Itertools; +use itertools::{Either, Itertools}; use mz_compute_types::dyncfgs::{ CONSOLIDATING_VEC_GROWTH_DAMPENER, CORRECTION_V2_CHAIN_PROPORTIONALITY, CORRECTION_V2_CHUNK_SIZE, ENABLE_CORRECTION_V2, }; use mz_dyncfg::ConfigSet; -use mz_ore::iter::IteratorExt; use mz_persist_client::metrics::{SinkMetrics, SinkWorkerMetrics, UpdateDelta}; use mz_repr::{Diff, Timestamp}; use timely::PartialOrder; @@ -101,6 +100,33 @@ impl Correction { } } + /// Consolidate the updates before the given `upper`. + /// + /// This is the expensive half of [`Correction::updates_before`], split out so callers that + /// must not run unbounded CPU work inline can perform it elsewhere. + pub fn consolidate_before(&mut self, upper: &Antichain) { + match self { + Self::V1(c) => c.consolidate_before(upper), + Self::V2(c) => c.consolidate_before(upper), + } + } + + /// Return the updates before the given `upper`, as consolidated by a preceding + /// [`Correction::consolidate_before`] call. + /// + /// The caller must have invoked `consolidate_before` with the same `upper` and must not have + /// mutated the buffer since. Otherwise the returned updates are neither consolidated nor + /// necessarily complete. + pub fn consolidated_updates_before<'a>( + &'a self, + upper: &Antichain, + ) -> impl Iterator + Send + use<'a, D> { + match self { + Self::V1(c) => Either::Left(c.consolidated_updates_before(upper)), + Self::V2(c) => Either::Right(c.consolidated_updates_before(upper)), + } + } + /// Advance the since frontier. /// /// # Panics @@ -251,42 +277,42 @@ impl CorrectionV1 { self.update_metrics(new_size); } - /// Consolidate and return updates within the given bounds. - /// - /// # Panics - /// - /// Panics if `lower` is not less than or equal to `upper`. - pub fn updates_within<'a>( - &'a mut self, - lower: &Antichain, - upper: &Antichain, - ) -> impl Iterator + ExactSizeIterator + use<'a, D> { - assert!(PartialOrder::less_equal(lower, upper)); - - let start = match lower.as_option() { - Some(ts) => Bound::Included(*ts), - None => Bound::Excluded(Timestamp::MAX), - }; + /// The range of stored times before the given `upper`. + fn range_before(upper: &Antichain) -> (Bound, Bound) { + let start = Bound::Included(Timestamp::MIN); let end = match upper.as_option() { Some(ts) => Bound::Excluded(*ts), None => Bound::Unbounded, }; + (start, end) + } - let update_count = self.consolidate((start, end)); + /// Consolidate the updates before the given `upper`. + pub fn consolidate_before(&mut self, upper: &Antichain) { + let _ = self.consolidate(Self::range_before(upper)); + } - let range = self.updates.range((start, end)); - range + /// Return the updates before the given `upper`, as consolidated by a preceding + /// [`CorrectionV1::consolidate_before`] call. + /// + /// The caller must have invoked `consolidate_before` with the same `upper` and must not have + /// mutated the buffer since. Otherwise the returned updates are not consolidated. + pub fn consolidated_updates_before<'a>( + &'a self, + upper: &Antichain, + ) -> impl Iterator + Send + use<'a, D> { + self.updates + .range(Self::range_before(upper)) .flat_map(|(t, data)| data.iter().map(|(d, r)| (d.clone(), *t, *r))) - .exact_size(update_count) } /// Consolidate and return updates before the given `upper`. pub fn updates_before<'a>( &'a mut self, upper: &Antichain, - ) -> impl Iterator + ExactSizeIterator + Send + use<'a, D> { - let lower = Antichain::from_elem(Timestamp::MIN); - self.updates_within(&lower, upper) + ) -> impl Iterator + Send + use<'a, D> { + self.consolidate_before(upper); + self.consolidated_updates_before(upper) } /// Consolidate the updates at the times in the given range. diff --git a/src/compute/src/sink/correction_v2.rs b/src/compute/src/sink/correction_v2.rs index 7a159bb906eab..b8913c38d5112 100644 --- a/src/compute/src/sink/correction_v2.rs +++ b/src/compute/src/sink/correction_v2.rs @@ -358,16 +358,27 @@ impl CorrectionV2 { &'a mut self, upper: &Antichain, ) -> impl Iterator + Send + 'a { + self.consolidate_before(upper); + self.consolidated_updates_before(upper) + } + + /// Return the updates before the given `upper`, as consolidated by a preceding + /// [`CorrectionV2::consolidate_before`] call. + /// + /// The caller must have invoked `consolidate_before` with the same `upper` and must not have + /// mutated the buffer since. Otherwise the returned updates are neither consolidated nor + /// necessarily complete. + pub fn consolidated_updates_before<'a>( + &'a self, + upper: &Antichain, + ) -> impl Iterator + Send + use<'a, D> { // All contained times are advanced to at least the `since`, so a read at an `upper` that - // is not beyond the `since` is always empty. Short-circuit to avoid the eager peel, merge, - // and `boundary` advancement that `consolidate_before` would otherwise perform. Normal - // reads and `consolidate_at_since` always pass an `upper` beyond the `since`. + // is not beyond the `since` is always empty. This mirrors the short-circuit in + // `consolidate_before`, which leaves `emitted` untouched in that case. if !PartialOrder::less_than(&self.since, upper) { return None.into_iter().flatten(); } - self.consolidate_before(upper); - // After `consolidate_before`, `emitted` holds exactly the updates before `upper`: every // path that populates it splits at `upper` (pushing the remainder to `pending_low`), and // the guard above guarantees `upper > since`, so advancing stale times to the `since` @@ -385,9 +396,17 @@ impl CorrectionV2 { /// Consolidate all updates before the given `upper` into the `emitted` chain. /// /// Once this method returns, `emitted` contains all updates at times before `upper`, - /// consolidated. It can also contain updates at times at or beyond `upper` if `upper` is not - /// beyond the `since`. - fn consolidate_before(&mut self, upper: &Antichain) { + /// consolidated. + /// + /// Does nothing if `upper` is not beyond the `since`: all contained times are advanced to at + /// least the `since`, so such a read is empty anyway, and skipping avoids an eager peel, + /// merge, and `boundary` advancement. Normal reads and `consolidate_at_since` always pass an + /// `upper` beyond the `since`. + pub fn consolidate_before(&mut self, upper: &Antichain) { + if !PartialOrder::less_than(&self.since, upper) { + return; + } + if let Some(mut ready) = self.stage.flush() { self.route(&mut ready); } diff --git a/src/compute/src/sink/materialized_view_v2.rs b/src/compute/src/sink/materialized_view_v2.rs index 6a6cf1d9afbb9..841a46a862c62 100644 --- a/src/compute/src/sink/materialized_view_v2.rs +++ b/src/compute/src/sink/materialized_view_v2.rs @@ -551,6 +551,9 @@ mod write { } } + /// The correction buffers owned by the Tokio write task. + type Corrections = OkErr, Correction>; + /// A response from the Tokio write task back to the Timely operator. struct WriteResponse { /// The written batch, or `None` if the corrections buffer had no updates. @@ -628,7 +631,7 @@ mod write { // Construct corrections on the Timely thread (reads ConfigSet), then move to the // Tokio task. The ChannelLogging sends events back to the Timely thread. let worker_metrics = sink_metrics.for_worker(worker_id); - let mut corrections: OkErr, Correction> = OkErr::new( + let mut corrections: Corrections = OkErr::new( Correction::new( sink_metrics.clone(), worker_metrics.clone(), @@ -685,7 +688,8 @@ mod write { let mut writer = persist_api.open_writer().await; while let Some(cmd) = cmd_rx.recv().await { - apply_command(&mut corrections, &mut writer, cmd, &resp_tx).await; + corrections = + apply_command(sink_id, corrections, &mut writer, cmd, &resp_tx).await; // Activate the operator to drain logging events and process batch responses. // ArcActivator suppresses redundant activations, so this is cheap. activator.activate(); @@ -831,62 +835,57 @@ mod write { batches_output_stream } - /// Apply a single command to the task state. + /// Apply a single command to the task state, returning the correction buffers. /// /// `desired` updates enter `corrections` as positive contributions and `persist` updates as /// negative contributions, so the buffer contains `desired - persist`, i.e. the updates that /// need to be written to bring the shard in line with `desired`. + /// + /// Correction maintenance is CPU-bound and unbounded in duration: an insert can merge chains + /// spanning the whole buffer and a consolidation sorts it, neither with an await point in + /// between. Running that inline occupies a Tokio worker thread for the entire time, which + /// stops it from polling every other task scheduled on it. It therefore runs on a blocking + /// thread, which the OS can preempt. The buffers move into the blocking closure and back out + /// again because this task owns them exclusively. async fn apply_command( - corrections: &mut OkErr, Correction>, + sink_id: GlobalId, + mut corrections: Corrections, writer: &mut WriteHandle, cmd: WriteCommand, resp_tx: &mpsc::UnboundedSender, - ) { + ) -> Corrections { match cmd { - WriteCommand::Batch(mut batch) => { - // Apply the same logical sequence of operations that the per-chunk commands - // used to: positive desired inserts, negated persist inserts, then optional - // frontier advancement and forced consolidation. - if !batch.desired_ok.is_empty() { - corrections.ok.insert(&mut batch.desired_ok); - } - if !batch.desired_err.is_empty() { - corrections.err.insert(&mut batch.desired_err); - } - if !batch.persist_ok.is_empty() { - corrections.ok.insert_negated(&mut batch.persist_ok); - } - if !batch.persist_err.is_empty() { - corrections.err.insert_negated(&mut batch.persist_err); - } - if let Some(frontier) = batch.persist_frontier { - // We will only emit times at or after the `persist` frontier, so now is a - // good time to advance the times of stashed updates. - corrections.ok.advance_since(frontier.clone()); - corrections.err.advance_since(frontier); - } - if batch.force_consolidation { - corrections.ok.consolidate_at_since(); - corrections.err.consolidate_at_since(); - } - } + WriteCommand::Batch(batch) => apply_batch(sink_id, corrections, batch).await, WriteCommand::WriteBatch(desc) => { - // Chain ok and err correction iterators directly, avoiding an - // intermediate Vec allocation. + let upper = desc.upper.clone(); + corrections = mz_ore::task::spawn_blocking( + || operator_name(sink_id, "write::consolidate"), + move || { + corrections.ok.consolidate_before(&upper); + corrections.err.consolidate_before(&upper); + corrections + }, + ) + .await; + + // Reading the consolidated updates back only clones them into the batch builder, + // which awaits at every part flush, so it stays on the async task. Chain ok and + // err correction iterators directly, avoiding an intermediate Vec allocation. let oks = corrections .ok - .updates_before(&desc.upper) + .consolidated_updates_before(&desc.upper) .map(|(d, t, r)| ((SourceData(Ok(d)), ()), t, r.into_inner())); let errs = corrections .err - .updates_before(&desc.upper) + .consolidated_updates_before(&desc.upper) .map(|(d, t, r)| ((SourceData(Err(d.deserialize())), ()), t, r.into_inner())); let mut updates = oks.chain(errs).peekable(); if updates.peek().is_none() { // No corrections to write. let _ = resp_tx.send(WriteResponse { batch: None }); - return; + drop(updates); + return corrections; } let batch = writer @@ -901,10 +900,65 @@ mod write { writer.batch_from_transmittable_batch(err.0.batch.expect("just sent")); batch.delete().await; } + + corrections } } } + /// Apply a coalesced batch of updates to the correction buffers, on a blocking thread. + /// + /// See [`apply_command`] for why this work must not run on a Tokio worker thread. + async fn apply_batch( + sink_id: GlobalId, + mut corrections: Corrections, + mut batch: BatchUpdates, + ) -> Corrections { + mz_ore::task::spawn_blocking( + || operator_name(sink_id, "write::apply_batch"), + move || { + // Stands in for the page-in storm this code produces when the process is under + // memory pressure: the buffer's chunks come back one blocking read at a time, from + // inside the sorts and merges below, where no await point can be placed. The + // failpoint's `sleep` action blocks its thread the same way, so it has to sit on + // the same side of the `spawn_blocking` boundary as the work it stands for. + // + // NOTE: activate it with the `FAILPOINTS` env var on the clusterd process, e.g. + // `FAILPOINTS=mv_sink_correction=sleep(30000)`. The `failpoints` session variable + // only reaches environmentd, never a cluster. + fail::fail_point!("mv_sink_correction"); + + // Apply the same logical sequence of operations that the per-chunk commands + // used to: positive desired inserts, negated persist inserts, then optional + // frontier advancement and forced consolidation. + if !batch.desired_ok.is_empty() { + corrections.ok.insert(&mut batch.desired_ok); + } + if !batch.desired_err.is_empty() { + corrections.err.insert(&mut batch.desired_err); + } + if !batch.persist_ok.is_empty() { + corrections.ok.insert_negated(&mut batch.persist_ok); + } + if !batch.persist_err.is_empty() { + corrections.err.insert_negated(&mut batch.persist_err); + } + if let Some(frontier) = batch.persist_frontier { + // We will only emit times at or after the `persist` frontier, so now is a good + // time to advance the times of stashed updates. + corrections.ok.advance_since(frontier.clone()); + corrections.err.advance_since(frontier); + } + if batch.force_consolidation { + corrections.ok.consolidate_at_since(); + corrections.err.consolidate_at_since(); + } + corrections + }, + ) + .await + } + /// State maintained by the `write` operator on the Timely thread. struct State { sink_id: GlobalId, @@ -1115,6 +1169,104 @@ mod write { Some((desc, cap)) } } + + #[cfg(test)] + mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::{Duration, Instant}; + + use mz_ore::metrics::MetricsRegistry; + use mz_persist_client::cfg::PersistConfig; + use mz_persist_client::metrics::Metrics; + + use super::*; + + /// One stall per worker thread would leave scheduling to chance, so oversubscribe: with + /// more stalls than workers every worker is guaranteed to pick one up. + const WORKER_THREADS: usize = 2; + const STALLS: usize = WORKER_THREADS * 2; + /// How long a stalled `apply_batch` blocks its thread. + const STALL: Duration = Duration::from_millis(1_000); + /// The largest tick gap the canary may observe. Generous enough to absorb scheduling noise + /// on a loaded CI host, while far below the `STALL` an inline correction pass produces. + const MAX_GAP: Duration = Duration::from_millis(500); + const TICK: Duration = Duration::from_millis(10); + + fn corrections() -> Corrections { + let registry = MetricsRegistry::new(); + let metrics = Metrics::new(&PersistConfig::new_for_tests(), ®istry); + let sink_metrics = metrics.sink.clone(); + let worker_metrics = sink_metrics.for_worker(0); + let config = mz_dyncfgs::all_dyncfgs(); + OkErr::new( + Correction::new(sink_metrics.clone(), worker_metrics.clone(), None, &config), + Correction::new(sink_metrics, worker_metrics, None, &config), + ) + } + + /// Tick every [`TICK`], recording the largest gap between consecutive ticks in millis. + /// + /// A gap only opens up if no worker thread was available to poll this task. + async fn canary(max_gap_millis: Arc) { + let mut last = Instant::now(); + loop { + tokio::time::sleep(TICK).await; + let now = Instant::now(); + let gap = u64::try_from(now.duration_since(last).as_millis()).unwrap_or(u64::MAX); + max_gap_millis.fetch_max(gap, Ordering::Relaxed); + last = now; + } + } + + /// Correction maintenance must not occupy a Tokio worker thread. + /// + /// Run inline, a pass over a large buffer pins a worker for its whole duration. Under + /// memory pressure it is blocked in the kernel paging chunks back in rather than + /// computing, which is why yielding cannot fix it. + /// + /// Stalls every worker thread inside `apply_batch` and asserts an unrelated task keeps + /// getting polled throughout. + #[mz_ore::test(tokio::test(flavor = "multi_thread", worker_threads = 2))] + #[cfg_attr(miri, ignore)] // depends on real thread scheduling + async fn correction_work_does_not_stall_the_runtime() { + let max_gap_millis = Arc::new(AtomicU64::new(0)); + let _canary = mz_ore::task::spawn(|| "canary", canary(Arc::clone(&max_gap_millis))) + .abort_on_drop(); + + // Let the canary settle, then discard the gaps observed while it did. + tokio::time::sleep(TICK * 5).await; + max_gap_millis.store(0, Ordering::Relaxed); + + let action = format!("sleep({})", STALL.as_millis()); + fail::cfg("mv_sink_correction", &action).expect("valid failpoint action"); + + let stalls: Vec<_> = (0..STALLS) + .map(|i| { + // A forced consolidation is the operation that hurt: it sweeps the whole + // buffer, so its cost is unbounded by the size of the incoming batch. + let batch = BatchUpdates { + force_consolidation: true, + ..BatchUpdates::new() + }; + let sink_id = GlobalId::User(u64::cast_from(i)); + mz_ore::task::spawn(|| "stall", apply_batch(sink_id, corrections(), batch)) + }) + .collect(); + for stall in stalls { + let _ = stall.await; + } + + fail::remove("mv_sink_correction"); + + let max_gap = Duration::from_millis(max_gap_millis.load(Ordering::Relaxed)); + assert!( + max_gap < MAX_GAP, + "runtime went unpolled for {max_gap:?}, so correction work is occupying a \ + worker thread", + ); + } + } } /// Implementation of the `append` operator. diff --git a/src/compute/src/sink/metric_sink.rs b/src/compute/src/sink/metric_sink.rs index 89f6bebda90ff..5c0a73ecfce97 100644 --- a/src/compute/src/sink/metric_sink.rs +++ b/src/compute/src/sink/metric_sink.rs @@ -24,6 +24,7 @@ //! family-conflict counting stay here because they need the cross-row state of the fold. use std::any::Any; +use std::cell::RefCell; use std::collections::BTreeMap; use std::rc::Rc; use std::sync::{Arc, Mutex}; @@ -110,11 +111,31 @@ impl<'scope> SinkRender<'scope> for MetricSinkConnection { }), ); + // The frontier this worker reports to the controller. Every worker reports, not just the + // active one: the controller meets the per-worker frontiers, so a worker that never + // advanced would pin the input's since forever. + let sink_frontier = Rc::new(RefCell::new(Antichain::from_elem(Timestamp::MIN))); + let shared_frontier = Rc::clone(&sink_frontier); + op.build(move |_capabilities| { // Recycled across activations: unpacking a row into `Datum`s otherwise allocates a // fresh `Vec` per row on this hot path. let mut datum_vec = DatumVec::new(); move |frontiers| { + // Combined ok+err input frontier. A timestamp is closed once neither input can + // still produce data at it, so folding a closed time observes all of its diffs. + let mut frontier = Antichain::new(); + for f in frontiers { + frontier.extend(f.frontier().iter().copied()); + } + // NOTE: the reported frontier is published here, before the active worker folds + // through it below (`st.integrate` at the end of this closure). This is only safe + // because the active worker must complete that fold in the same activation that + // observed the frontier. An early return added on the active-worker path between + // here and `st.integrate` would advertise progress the sink has not folded, + // downgrading the input's since ahead of the data actually consumed. + shared_frontier.borrow_mut().clone_from(&frontier); + if worker_id != active_worker_id { // Drain so the operator isn't rescheduled forever. There is no state to // fold into on this worker. @@ -152,13 +173,6 @@ impl<'scope> SinkRender<'scope> for MetricSinkConnection { } }); - // Combined ok+err input frontier. A timestamp is closed once neither input can - // still produce data at it, so folding a closed time observes all of its diffs. - let mut frontier = Antichain::new(); - for f in frontiers { - frontier.extend(f.frontier().iter().copied()); - } - st.integrate(&frontier); st.frontier_ms = frontier .as_option() @@ -168,6 +182,12 @@ impl<'scope> SinkRender<'scope> for MetricSinkConnection { } }); + // Report frontier updates to the `ComputeState`. A metric sink writes to the metrics + // registry rather than to a collection, so its "write" frontier is the input frontier it + // has folded through. + let collection = compute_state.expect_collection_mut(sink_id); + collection.sink_write_frontier = Some(sink_frontier); + Some(Rc::new(drop_handle)) } } @@ -605,7 +625,7 @@ fn build_families(published: &BTreeMap) -> Vec) -> BoxFuture<'static, azure_core::Result> + + Send + + Sync, +>; + +/// A shared slot holding the current access token for one scope set. +type TokenSlot = Arc>; + +/// A [TokenCredential] for AKS-style workload identity that re-reads the +/// projected service account token file on every AAD access token refresh. +/// +/// `azure_identity`'s `WorkloadIdentityCredential` reads +/// `AZURE_FEDERATED_TOKEN_FILE` once at construction and holds the contents +/// for the life of the process. Kubernetes rotates the projected token, so +/// once the last cached AAD access token expires, every refresh presents an +/// expired client assertion and fails, permanently locking a long-running +/// process out of blob storage. Deferring the file read to refresh time picks +/// up rotations. +struct RefreshingWorkloadIdentityCredential { + federated_token_file: PathBuf, + exchange: ExchangeFn, + /// One token slot and refresh task per requested scope set. The task + /// keeps the slot fresh, so [TokenCredential::get_token] only blocks on + /// the first use of a scope set. + cache: RwLock, (TokenSlot, AbortOnDropHandle<()>)>>, + refresh_buffer: Duration, + retry_interval: Duration, +} + +impl Debug for RefreshingWorkloadIdentityCredential { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RefreshingWorkloadIdentityCredential") + .field("federated_token_file", &self.federated_token_file) + .finish_non_exhaustive() + } +} + +impl RefreshingWorkloadIdentityCredential { + /// Returns a credential if the workload identity environment variables + /// are present, or `None` to indicate that a different credential type + /// must be used. + fn from_env() -> Option> { + // A token provided directly via AZURE_FEDERATED_TOKEN is static, so + // there is nothing to re-read. `azure_identity`'s credential chain + // prefers it over the token file, defer to it to preserve that + // precedence. + if std::env::var(AZURE_FEDERATED_TOKEN).is_ok() { + return None; + } + let (Ok(tenant_id), Ok(client_id), Ok(token_file)) = ( + std::env::var(AZURE_TENANT_ID), + std::env::var(AZURE_CLIENT_ID), + std::env::var(AZURE_FEDERATED_TOKEN_FILE), + ) else { + return None; + }; + Some(Self::new(tenant_id, client_id, PathBuf::from(token_file))) + } + + fn new( + tenant_id: String, + client_id: String, + federated_token_file: PathBuf, + ) -> azure_core::Result { + let options = TokenCredentialOptions::default(); + let http_client = options.http_client(); + let authority_host = options.authority_host()?; + let exchange: ExchangeFn = Arc::new(move |assertion, scopes| { + let http_client = Arc::clone(&http_client); + let authority_host = authority_host.clone(); + let tenant_id = tenant_id.clone(); + let client_id = client_id.clone(); + async move { + let scopes: Vec<&str> = scopes.iter().map(String::as_str).collect(); + let res = federated_credentials_flow::perform( + http_client, + &client_id, + &assertion, + &scopes, + &tenant_id, + &authority_host, + ) + .await + .map_err(|err| { + azure_core::error::Error::full( + ErrorKind::Credential, + err, + "request token error", + ) + })?; + Ok(AccessToken::new( + res.access_token().clone(), + OffsetDateTime::now_utc() + Duration::from_secs(res.expires_in), + )) + } + .boxed() + }); + Ok(Self::with_exchange( + federated_token_file, + exchange, + TOKEN_REFRESH_BUFFER, + TOKEN_REFRESH_RETRY_INTERVAL, + )) + } + + fn with_exchange( + federated_token_file: PathBuf, + exchange: ExchangeFn, + refresh_buffer: Duration, + retry_interval: Duration, + ) -> Self { + Self { + federated_token_file, + exchange, + cache: RwLock::new(BTreeMap::new()), + refresh_buffer, + retry_interval, + } + } +} + +/// Reads the projected service account token file and exchanges its contents +/// for an AAD access token. +async fn fetch_token( + federated_token_file: &Path, + exchange: &ExchangeFn, + scopes: Vec, +) -> azure_core::Result { + let assertion = tokio::fs::read_to_string(federated_token_file) + .await + .map_err(|err| { + azure_core::error::Error::full( + ErrorKind::Credential, + err, + format!( + "failed to read federated token from file {}", + federated_token_file.display() + ), + ) + })?; + // Kubernetes writes the projected token without surrounding whitespace, + // but a hand-provisioned file may have a trailing newline, which would + // corrupt the client assertion. + (exchange)(assertion.trim().to_string(), scopes).await +} + +/// Keeps `slot` holding an unexpired token by fetching a replacement within +/// `refresh_buffer` of the current token's expiry. A failed fetch leaves the +/// current token in place and is retried after `retry_interval`. +async fn refresh_task( + federated_token_file: PathBuf, + exchange: ExchangeFn, + slot: TokenSlot, + scopes: Vec, + refresh_buffer: Duration, + retry_interval: Duration, +) { + loop { + let refresh_at = slot.read().expect("lock poisoned").expires_on - refresh_buffer; + let wait = refresh_at - OffsetDateTime::now_utc(); + let wait = if wait.is_positive() { + wait.unsigned_abs() + } else { + Duration::ZERO + }; + tokio::time::sleep(wait.max(retry_interval)).await; + match fetch_token(&federated_token_file, &exchange, scopes.clone()).await { + Ok(token) => *slot.write().expect("lock poisoned") = token, + Err(err) => { + warn!("failed to refresh Azure workload identity token, will retry: {err}") + } + } + } +} + +#[async_trait] +impl TokenCredential for RefreshingWorkloadIdentityCredential { + async fn get_token(&self, scopes: &[&str]) -> azure_core::Result { + let scopes_key: Vec = scopes.iter().map(ToString::to_string).collect(); + + { + let cache = self.cache.read().await; + if let Some((slot, _refresh)) = cache.get(&scopes_key) { + return Ok(slot.read().expect("lock poisoned").clone()); + } + } + + let mut cache = self.cache.write().await; + if let Some((slot, _refresh)) = cache.get(&scopes_key) { + return Ok(slot.read().expect("lock poisoned").clone()); + } + + // First use of this scope set: fetch the initial token, then hand + // the slot to a task that keeps it fresh. A failed initial fetch is + // not cached, the next call retries it. + let token = fetch_token( + &self.federated_token_file, + &self.exchange, + scopes_key.clone(), + ) + .await?; + let slot = Arc::new(std::sync::RwLock::new(token.clone())); + let refresh = mz_ore::task::spawn( + || "azure-workload-identity-token-refresh", + refresh_task( + self.federated_token_file.clone(), + Arc::clone(&self.exchange), + Arc::clone(&slot), + scopes_key.clone(), + self.refresh_buffer, + self.retry_interval, + ), + ) + .abort_on_drop(); + cache.insert(scopes_key, (slot, refresh)); + Ok(token) + } + + async fn clear_cache(&self) -> azure_core::Result<()> { + // Dropping the entries aborts their refresh tasks with them. + self.cache.write().await.clear(); + Ok(()) + } +} + +/// Returns the token credential to use when the blob URL carries no SAS +/// token. +/// +/// Prefers [RefreshingWorkloadIdentityCredential] when its environment +/// variables are present, because the workload identity credential in +/// `azure_identity`'s default chain never re-reads the rotated token file. +/// Otherwise falls back to the default chain, whose remaining credential +/// types (e.g. managed identity via IMDS) refresh correctly. +fn token_credential() -> Arc { + match RefreshingWorkloadIdentityCredential::from_env() { + Some(credential) => { + info!("azure: using refreshing workload identity credentials"); + Arc::new(credential.expect("Azure workload identity credentials")) + } + None => create_default_credential().expect("Azure default credentials"), + } +} + /// Configuration for opening an [AzureBlob]. #[derive(Clone, Debug)] pub struct AzureBlobConfig { @@ -92,13 +366,9 @@ impl AzureBlobConfig { warn!("Failed to parse SAS token: {err}"); // TODO: should we fallback here? Or can we fully rely on query params // to determine whether a SAS token was provided? - StorageCredentials::token_credential( - create_default_credential().expect("Azure default credentials"), - ) + StorageCredentials::token_credential(token_credential()) } - None => StorageCredentials::token_credential( - create_default_credential().expect("Azure default credentials"), - ), + None => StorageCredentials::token_credential(token_credential()), }; ClientBuilder::new(account, credentials) @@ -108,9 +378,10 @@ impl AzureBlobConfig { .blob_service_client() .container_client(container); - // TODO: some auth modes like user-delegated SAS tokens are time-limited - // and need to be refreshed. This can be done through `service_client.update_credentials` - // but there'll be a fair bit of plumbing needed to make each mode work + // NOTE: a SAS token provided via the URL query string is static and + // never refreshed, so callers must provision one that outlives the + // process. Token credentials (workload identity and managed identity) + // refresh themselves. Ok(AzureBlobConfig { metrics, @@ -376,12 +647,145 @@ impl Blob for AzureBlob { #[cfg(test)] mod tests { + use azure_core::auth::Secret; + use std::sync::Mutex; use tracing::info; use crate::location::tests::blob_impl_test; use super::*; + /// A [MockExchange] wrapped for sharing with the credential's exchange + /// closure. + struct MockExchange { + /// Client assertions passed to each exchange call. + assertions: Vec, + /// Whether the next exchange calls fail. + fail: bool, + } + + fn mock_exchange(state: &Arc>) -> ExchangeFn { + let state = Arc::clone(state); + Arc::new(move |assertion, _scopes| { + let state = Arc::clone(&state); + async move { + let mut state = state.lock().unwrap(); + state.assertions.push(assertion); + if state.fail { + return Err(azure_core::error::Error::message( + ErrorKind::Credential, + "mock exchange failure", + )); + } + Ok(AccessToken::new( + Secret::new(format!("aad-{}", state.assertions.len())), + OffsetDateTime::now_utc() + Duration::from_secs(3600), + )) + } + .boxed() + }) + } + + /// Tests that the token file is re-read (and trimmed) on every fetch, + /// that fetched tokens are served from the slot without further + /// exchanges, and that a failed initial fetch is not cached. + #[mz_ore::test(tokio::test)] + async fn refreshing_workload_identity_credential() { + let token_file = tempfile::NamedTempFile::new().expect("create temp token file"); + std::fs::write(token_file.path(), "token-a\n").expect("write token file"); + + let state = Arc::new(Mutex::new(MockExchange { + assertions: Vec::new(), + fail: false, + })); + let credential = RefreshingWorkloadIdentityCredential::with_exchange( + token_file.path().to_path_buf(), + mock_exchange(&state), + TOKEN_REFRESH_BUFFER, + TOKEN_REFRESH_RETRY_INTERVAL, + ); + let scopes = &["https://storage.azure.com/"]; + + let token = credential.get_token(scopes).await.expect("token"); + assert_eq!(token.token.secret(), "aad-1"); + let token = credential.get_token(scopes).await.expect("token"); + assert_eq!(token.token.secret(), "aad-1"); + assert_eq!(state.lock().unwrap().assertions, vec!["token-a"]); + + // A failed initial fetch surfaces the error without caching it, and + // the rotated token file is re-read on the next fetch. + std::fs::write(token_file.path(), "token-b").expect("write token file"); + credential.clear_cache().await.expect("clear cache"); + state.lock().unwrap().fail = true; + assert!(credential.get_token(scopes).await.is_err()); + state.lock().unwrap().fail = false; + let token = credential.get_token(scopes).await.expect("token"); + assert_eq!(token.token.secret(), "aad-3"); + assert_eq!( + state.lock().unwrap().assertions, + vec!["token-a", "token-b", "token-b"] + ); + } + + /// Tests that the background task refreshes the slot with fresh token + /// file contents and keeps the last good token through failed refreshes. + #[mz_ore::test(tokio::test)] + async fn workload_identity_credential_background_refresh() { + let token_file = tempfile::NamedTempFile::new().expect("create temp token file"); + std::fs::write(token_file.path(), "token-a").expect("write token file"); + + let state = Arc::new(Mutex::new(MockExchange { + assertions: Vec::new(), + fail: false, + })); + // A refresh buffer longer than the issued validity makes every token + // immediately due, so refreshes run continuously at the (shortened) + // retry interval. + let credential = RefreshingWorkloadIdentityCredential::with_exchange( + token_file.path().to_path_buf(), + mock_exchange(&state), + Duration::from_secs(7200), + Duration::from_millis(10), + ); + let scopes = &["https://storage.azure.com/"]; + + let token = credential.get_token(scopes).await.expect("token"); + assert_eq!(token.token.secret(), "aad-1"); + + // The background task picks up the rotated token file without any + // caller blocking on the refresh. + std::fs::write(token_file.path(), "token-b").expect("write token file"); + tokio::time::timeout(Duration::from_secs(30), async { + loop { + let token = credential.get_token(scopes).await.expect("token"); + if token.token.secret() != "aad-1" { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("token refreshed within timeout"); + assert_eq!( + state.lock().unwrap().assertions.last().map(String::as_str), + Some("token-b") + ); + + // Failed refreshes keep the last good token in the slot and retry. + state.lock().unwrap().fail = true; + let held = credential.get_token(scopes).await.expect("token"); + let calls_when_failing = state.lock().unwrap().assertions.len(); + tokio::time::timeout(Duration::from_secs(30), async { + while state.lock().unwrap().assertions.len() <= calls_when_failing + 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("retries within timeout"); + let token = credential.get_token(scopes).await.expect("token"); + assert_eq!(token.token.secret(), held.token.secret()); + } + #[cfg_attr(miri, ignore)] // error: unsupported operation: can't call foreign function `TLS_method` on OS `linux` #[mz_ore::test(tokio::test(flavor = "multi_thread"))] async fn azure_blob() -> Result<(), ExternalError> { diff --git a/src/sql/src/plan/statement/ddl.rs b/src/sql/src/plan/statement/ddl.rs index 790f589c0ff59..cdaafdfac1a26 100644 --- a/src/sql/src/plan/statement/ddl.rs +++ b/src/sql/src/plan/statement/ddl.rs @@ -4250,7 +4250,16 @@ const METRIC_SINK_SOURCE_COLUMNS: &[(&str, fn(&SqlScalarType) -> bool)] = &[ generate_extracted_config!(CreateMetricSinkOption, (Prefix, String)); -/// Rejects a prefix that could not start a Prometheus metric name. +/// The reserved namespace every metric sink prefix must start with. It confines the family names a +/// sink publishes (`prefix + name`) to the `mz_metric_sink_` lane, which nothing else in a +/// replica's Prometheus registry writes: platform metrics use other `mz_*` names, third-party +/// collectors namespace away from `mz_*`, and the sink's own health gauges are +/// `mz_compute_metric_sink_*`. Requiring the marker rather than blocklisting platform names cannot +/// rot as new platform collectors are added. +const METRIC_SINK_PREFIX_MARKER: &str = "mz_metric_sink_"; + +/// Rejects a prefix that could not start a Prometheus metric name, or that escapes the reserved +/// `mz_metric_sink_` lane (see [`METRIC_SINK_PREFIX_MARKER`]). /// /// The sink prepends this to every name it publishes, so `prefix + name` must stay a legal /// family name (`[a-zA-Z_:][a-zA-Z0-9_:]*`, the same grammar the runtime checks each row's @@ -4272,6 +4281,13 @@ fn validate_metric_sink_prefix(prefix: &str) -> Result<(), PlanError> { prefix )); } + if !prefix.starts_with(METRIC_SINK_PREFIX_MARKER) { + return Err(sql_err!( + "metric sink prefix {:?} must start with {:?}", + prefix, + METRIC_SINK_PREFIX_MARKER + )); + } Ok(()) } diff --git a/src/storage-types/fuzz/fuzz_targets/pushdown_soundness.rs b/src/storage-types/fuzz/fuzz_targets/pushdown_soundness.rs index b38ebf3faaea4..166471a645bb6 100644 --- a/src/storage-types/fuzz/fuzz_targets/pushdown_soundness.rs +++ b/src/storage-types/fuzz/fuzz_targets/pushdown_soundness.rs @@ -36,9 +36,7 @@ use mz_persist_types::codec_impls::UnitSchema; use mz_persist_types::part::PartBuilder; use mz_persist_types::stats::{PartStats, PartStatsMetrics}; use mz_repr::adt::numeric::Numeric; -use mz_repr::{ - Datum, Diff, RelationDesc, ReprScalarType, Row, RowArena, SqlScalarType, Timestamp, -}; +use mz_repr::{Datum, Diff, RelationDesc, ReprScalarType, Row, RowArena, SqlScalarType, Timestamp}; use mz_storage_types::errors::DataflowError; use mz_storage_types::sources::SourceData; use mz_storage_types::stats::RelationPartStats; @@ -81,13 +79,28 @@ enum Cmp { #[derive(Arbitrary, Debug)] enum Pred { /// `col lit`, the literal drawn like a row value. - CmpColLit { col: u8, cmp: Cmp, lit: FuzzDatum }, - IsNull { col: u8, negate: bool }, + CmpColLit { + col: u8, + cmp: Cmp, + lit: FuzzDatum, + }, + IsNull { + col: u8, + negate: bool, + }, /// `(json ->(>) 'key') IS NULL` over the Nested specs from real map stats. - JsonbKey { key: u8, stringify: bool }, + JsonbKey { + key: u8, + stringify: bool, + }, /// `(c_f64 + a) * b c`, aimed at the infinity guard and NaN /// arithmetic, with fuzzer-chosen bit patterns. - FloatArith { cmp: Cmp, a: u64, b: u64, c: u64 }, + FloatArith { + cmp: Cmp, + a: u64, + b: u64, + c: u64, + }, And(Box, Box), Or(Box, Box), Not(Box), @@ -103,7 +116,10 @@ struct Input { fn schema() -> RelationDesc { RelationDesc::builder() - .with_column("c_num", SqlScalarType::Numeric { max_scale: None }.nullable(true)) + .with_column( + "c_num", + SqlScalarType::Numeric { max_scale: None }.nullable(true), + ) .with_column("c_f64", SqlScalarType::Float64.nullable(true)) .with_column("c_str", SqlScalarType::String.nullable(true)) .with_column("c_json", SqlScalarType::Jsonb.nullable(true)) diff --git a/src/timely-util/src/column_pager/policy.rs b/src/timely-util/src/column_pager/policy.rs index 502b05b99d3bb..aacf78faf81b0 100644 --- a/src/timely-util/src/column_pager/policy.rs +++ b/src/timely-util/src/column_pager/policy.rs @@ -133,7 +133,7 @@ impl TieredPolicy { let shrink = prev - new_total; let _ = self .budget - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |cur| { + .try_update(Ordering::Relaxed, Ordering::Relaxed, |cur| { Some(cur.saturating_sub(shrink)) }); } diff --git a/test/cluster/mzcompose.py b/test/cluster/mzcompose.py index 2dd95f27e7b3d..c879958dd3a3a 100644 --- a/test/cluster/mzcompose.py +++ b/test/cluster/mzcompose.py @@ -4288,6 +4288,94 @@ def workflow_test_subscribe_hydration_status( """)) +def workflow_test_hydration_timestamps(c: Composition) -> None: + """ + Test that compute hydration timestamps are stamped by the replica. + + They must therefore survive an environmentd restart, which does not disturb + the replica, and reset on a replica restart, which rebuilds the dataflows + they describe. + """ + + c.down(destroy_volumes=True) + c.up("materialized", "clusterd1") + + c.sql( + "ALTER SYSTEM SET unsafe_enable_unorchestrated_cluster_replicas = true;", + port=6877, + user="mz_system", + ) + + c.sql(""" + CREATE CLUSTER cluster1 REPLICAS (replica1 ( + STORAGECTL ADDRESSES ['clusterd1:2100'], + STORAGE ADDRESSES ['clusterd1:2103'], + COMPUTECTL ADDRESSES ['clusterd1:2101'], + COMPUTE ADDRESSES ['clusterd1:2102'], + WORKERS 2 + )); + SET cluster = cluster1; + CREATE TABLE t (a int); + CREATE INDEX idx ON t (a); + """) + + def collect_timestamps() -> list[tuple]: + """Return one `(worker_id, installed_at, started_at, hydrated_at)` row per worker. + + Retries until every worker reports a complete set of timestamps, since + dataflows take an unknown time to hydrate and the introspection + relations are updated asynchronously. + """ + deadline = time.time() + 120 + while True: + with c.sql_cursor() as cursor: + cursor.execute("SET cluster = cluster1") + cursor.execute(""" + SELECT h.worker_id, h.installed_at, h.started_at, h.hydrated_at + FROM mz_introspection.mz_compute_hydration_times_per_worker h + JOIN mz_indexes i ON (i.id = h.export_id) + WHERE i.name = 'idx' + ORDER BY h.worker_id + """) + rows = [tuple(row) for row in cursor.fetchall()] + if len(rows) == 2 and all(all(v is not None for v in row) for row in rows): + return rows + assert time.time() < deadline, f"idx did not hydrate, last saw: {rows}" + time.sleep(1) + + before = collect_timestamps() + for worker_id, installed_at, started_at, hydrated_at in before: + assert ( + installed_at <= started_at <= hydrated_at + ), f"worker {worker_id} timestamps out of order: {installed_at}, {started_at}, {hydrated_at}" + + # An environmentd restart triggers a reconciliation on clusterd, which + # retains the dataflow, so the timestamps describe the same episode and must + # not change. + c.kill("materialized") + c.up("materialized") + + after_environmentd_restart = collect_timestamps() + assert before == after_environmentd_restart, ( + "hydration timestamps changed across an environmentd restart: " + f"{before} vs {after_environmentd_restart}" + ) + + # A replica restart rebuilds the dataflow, which is a new hydration episode, + # so every timestamp must be fresh. + c.kill("clusterd1") + c.up("clusterd1") + + after_replica_restart = collect_timestamps() + for (_, before_installed, _, _), (worker_id, after_installed, _, _) in zip( + before, after_replica_restart + ): + assert after_installed > before_installed, ( + f"worker {worker_id} kept its installed_at across a replica restart: " + f"{before_installed} vs {after_installed}" + ) + + def workflow_cluster_drop_concurrent( c: Composition, parser: WorkflowArgumentParser ) -> None: diff --git a/test/sql-server-cdc/73-ingestion-lag-during-connection-delay.td b/test/sql-server-cdc/73-ingestion-lag-during-connection-delay.td index e5db751cc7714..aa96e9efc2e06 100644 --- a/test/sql-server-cdc/73-ingestion-lag-during-connection-delay.td +++ b/test/sql-server-cdc/73-ingestion-lag-during-connection-delay.td @@ -44,7 +44,18 @@ ALTER SYSTEM SET storage_statistics_interval = 2000; FROM SQL SERVER CONNECTION conn FOR TABLES (dbo.t73) -# Stall the first connection past several statistics intervals. +# Snapshot before stalling anything. Only a source that has durably committed +# has a resumption LSN above the minimum, and only then does the seeded +# `offset_committed` differ from the seeded `offset_known`. +> CREATE CLUSTER REPLICA storage.r1 SIZE = 'scale=1,workers=1' + +> SELECT * FROM t73 +one + +> DROP CLUSTER REPLICA storage.r1 + +# Stall the reconnect. Establishing a connection takes several round trips, so +# the replica below stays stalled for well over a minute. $ http-request method=POST url=http://toxiproxy:8474/proxies/sql-server/toxics content-type=application/json { "name": "startup-delay", @@ -54,13 +65,34 @@ $ http-request method=POST url=http://toxiproxy:8474/proxies/sql-server/toxics c > CREATE CLUSTER REPLICA storage.r1 SIZE = 'scale=1,workers=1' -# The correct state remains zero, so there is no nonzero value to poll for. -$ sleep-is-probably-flaky-i-have-justified-my-need-with-a-comment duration="5s" +$ set-from-sql var=replica_id +SELECT cr.id + FROM mz_clusters c JOIN mz_cluster_replicas cr ON c.id = cr.cluster_id + WHERE c.name = 'storage' AND cr.name = 'r1' + +# The persist sink publishes this row without touching the upstream, so it +# appears during the stall. +> SELECT COUNT(*) + FROM mz_sources s + JOIN mz_internal.mz_source_statistics u ON s.id = u.id + WHERE s.name = 's' AND u.replica_id = '${replica_id}' +1 -$ set-sql-timeout duration=10s +# `force` is required: without it the timeout is raised to the default, and +# retrying past the stall would read the settled post-connect state instead. +$ set-sql-timeout duration=5s force=true -> SELECT u.offset_known - u.offset_committed +> SELECT u.offset_known >= u.offset_committed FROM mz_sources s JOIN mz_internal.mz_source_statistics u ON s.id = u.id - WHERE s.name = 's' -0 + WHERE s.name = 's' AND u.replica_id = '${replica_id}' +true + +$ set-sql-timeout duration=default force=true + +> DROP SOURCE s CASCADE +> DROP CLUSTER storage CASCADE +> DROP CONNECTION conn +> DROP SECRET pass + +$ http-request method=DELETE url=http://toxiproxy:8474/proxies/sql-server diff --git a/test/sqllogictest/metric_sink.slt b/test/sqllogictest/metric_sink.slt index 83dccab596947..a03c57af4b0bf 100644 --- a/test/sqllogictest/metric_sink.slt +++ b/test/sqllogictest/metric_sink.slt @@ -11,8 +11,9 @@ mode cockroach reset-server -# `CREATE METRIC SINK` only writes a catalog item. No dataflow is installed, so -# nothing here actually publishes metrics. +# These tests exercise `CREATE METRIC SINK` planning and catalog behavior. The +# sink's dataflow and the metrics it publishes are covered in +# test/testdrive/metric-sink.td. simple conn=mz_system,user=mz_system ALTER SYSTEM SET enable_metric_sink = true @@ -28,7 +29,7 @@ CREATE VIEW shaped AS SELECT * FROM raw # All five columns present with the right types, so this plans. statement ok -CREATE METRIC SINK ms FROM shaped WITH (PREFIX = 'ms_') +CREATE METRIC SINK ms FROM shaped WITH (PREFIX = 'mz_metric_sink_ms_') statement ok DROP METRIC SINK ms @@ -39,7 +40,7 @@ statement ok CREATE VIEW reordered AS SELECT help, value, labels, metric_type, metric_name FROM raw statement ok -CREATE METRIC SINK ms_reordered FROM reordered WITH (PREFIX = 'reordered_') +CREATE METRIC SINK ms_reordered FROM reordered WITH (PREFIX = 'mz_metric_sink_reordered_') statement ok DROP METRIC SINK ms_reordered @@ -50,7 +51,7 @@ statement ok CREATE VIEW extra AS SELECT *, 1 AS unrelated FROM raw statement ok -CREATE METRIC SINK ms_extra FROM extra WITH (PREFIX = 'extra_') +CREATE METRIC SINK ms_extra FROM extra WITH (PREFIX = 'mz_metric_sink_extra_') statement ok DROP METRIC SINK ms_extra @@ -61,7 +62,7 @@ statement ok CREATE VIEW missing_help AS SELECT metric_name, metric_type, labels, value FROM raw statement error metric sink source must expose column "help" -CREATE METRIC SINK bad FROM missing_help WITH (PREFIX = 'bad_') +CREATE METRIC SINK bad FROM missing_help WITH (PREFIX = 'mz_metric_sink_bad_') # Neither is one of the wrong type. @@ -70,7 +71,7 @@ CREATE VIEW wrong_value_type AS SELECT metric_name, metric_type, labels, value::text AS value, help FROM raw statement error metric sink source column "value" is not of the required type -CREATE METRIC SINK bad FROM wrong_value_type WITH (PREFIX = 'bad_') +CREATE METRIC SINK bad FROM wrong_value_type WITH (PREFIX = 'mz_metric_sink_bad_') # `labels` has to be a text=>text map, not just any map. @@ -79,7 +80,7 @@ CREATE VIEW wrong_labels_type AS SELECT metric_name, metric_type, '{a=>1}'::map[text=>int] AS labels, value, help FROM raw statement error metric sink source column "labels" is not of the required type -CREATE METRIC SINK bad FROM wrong_labels_type WITH (PREFIX = 'bad_') +CREATE METRIC SINK bad FROM wrong_labels_type WITH (PREFIX = 'mz_metric_sink_bad_') # You can't read from something that has no rows to read. @@ -87,7 +88,7 @@ statement ok CREATE SECRET s AS 'x' statement error cannot create metric sink from s because it is a secret -CREATE METRIC SINK bad FROM s WITH (PREFIX = 'bad_') +CREATE METRIC SINK bad FROM s WITH (PREFIX = 'mz_metric_sink_bad_') # An index is in the same boat, and the error should say so instead of being # vague about it. @@ -96,7 +97,7 @@ statement ok CREATE INDEX raw_idx ON raw (metric_name) statement error cannot create metric sink from raw_idx because it is a index -CREATE METRIC SINK bad FROM raw_idx WITH (PREFIX = 'bad_') +CREATE METRIC SINK bad FROM raw_idx WITH (PREFIX = 'mz_metric_sink_bad_') # The prefix is required. The names a sink publishes come from its query's rows # and so are unknown at plan time, which makes the prefix the only handle we get @@ -117,20 +118,27 @@ CREATE METRIC SINK ms_bad_prefix FROM shaped WITH (PREFIX = 'has-a-dash') statement error metric sink prefix must not be empty CREATE METRIC SINK ms_bad_prefix FROM shaped WITH (PREFIX = '') +# The prefix must live in the reserved `mz_metric_sink_` namespace, the one lane +# nothing else in a replica's registry writes. A well-formed prefix outside it is +# rejected, so a sink can never publish onto a platform metric or its own gauges. + +statement error metric sink prefix "app_" must start with "mz_metric_sink_" +CREATE METRIC SINK ms_unmarked FROM shaped WITH (PREFIX = 'app_') + # Prefixes on one cluster have to be prefix-free, not merely distinct: `app_` # plus a row named `svc_x` publishes exactly what `app_svc_` plus a row named # `x` does, and the Prometheus registry merges same-named families silently. statement ok -CREATE METRIC SINK ms_app FROM shaped WITH (PREFIX = 'app_') +CREATE METRIC SINK ms_app FROM shaped WITH (PREFIX = 'mz_metric_sink_app_') -statement error metric sink prefix "app_svc_" conflicts with prefix "app_" of metric sink ms_app on cluster quickstart -CREATE METRIC SINK ms_app_svc FROM shaped WITH (PREFIX = 'app_svc_') +statement error metric sink prefix "mz_metric_sink_app_svc_" conflicts with prefix "mz_metric_sink_app_" of metric sink ms_app on cluster quickstart +CREATE METRIC SINK ms_app_svc FROM shaped WITH (PREFIX = 'mz_metric_sink_app_svc_') # Either direction, so an existing prefix cannot be swallowed by a shorter one. -statement error metric sink prefix "ap" conflicts with prefix "app_" of metric sink ms_app on cluster quickstart -CREATE METRIC SINK ms_ap FROM shaped WITH (PREFIX = 'ap') +statement error metric sink prefix "mz_metric_sink_ap" conflicts with prefix "mz_metric_sink_app_" of metric sink ms_app on cluster quickstart +CREATE METRIC SINK ms_ap FROM shaped WITH (PREFIX = 'mz_metric_sink_ap') # The registry a sink publishes into is process-local and every replica of a # cluster runs the same sinks, so uniqueness is only ever a per-cluster @@ -140,7 +148,7 @@ statement ok CREATE CLUSTER other SIZE 'scale=1,workers=1', REPLICATION FACTOR 0 statement ok -CREATE METRIC SINK ms_other_app IN CLUSTER other FROM shaped WITH (PREFIX = 'app_') +CREATE METRIC SINK ms_other_app IN CLUSTER other FROM shaped WITH (PREFIX = 'mz_metric_sink_app_') statement ok DROP METRIC SINK ms_other_app @@ -155,13 +163,13 @@ DROP METRIC SINK ms_app # existing sink already has is not a conflict with itself. statement ok -CREATE METRIC SINK ms FROM shaped WITH (PREFIX = 'ms_') +CREATE METRIC SINK ms FROM shaped WITH (PREFIX = 'mz_metric_sink_ms_') statement ok -CREATE METRIC SINK IF NOT EXISTS ms FROM shaped WITH (PREFIX = 'ms_') +CREATE METRIC SINK IF NOT EXISTS ms FROM shaped WITH (PREFIX = 'mz_metric_sink_ms_') statement error metric sink "materialize.public.ms" already exists -CREATE METRIC SINK ms FROM shaped WITH (PREFIX = 'ms_') +CREATE METRIC SINK ms FROM shaped WITH (PREFIX = 'mz_metric_sink_ms_') # A metric sink can't be indexed. @@ -280,7 +288,7 @@ DROP VIEW shaped CASCADE # against `raw`. statement ok -CREATE METRIC SINK ms FROM raw WITH (PREFIX = 'ms_') +CREATE METRIC SINK ms FROM raw WITH (PREFIX = 'mz_metric_sink_ms_') statement ok DROP METRIC SINK ms @@ -297,7 +305,7 @@ statement ok CREATE SCHEMA ms_rename_schema statement ok -CREATE METRIC SINK ms_rename_schema.rsink FROM ms_rename_src WITH (PREFIX = 'ren_') +CREATE METRIC SINK ms_rename_schema.rsink FROM ms_rename_src WITH (PREFIX = 'mz_metric_sink_ren_') statement ok ALTER VIEW ms_rename_src RENAME TO ms_rename_src2 @@ -364,7 +372,7 @@ REVOKE SELECT ON TABLE rbac_src FROM rbac_owner COMPLETE 0 simple conn=rbac_owner,user=rbac_owner -CREATE METRIC SINK rbac_ms FROM rbac_src WITH (PREFIX = 'rbac_') +CREATE METRIC SINK rbac_ms FROM rbac_src WITH (PREFIX = 'mz_metric_sink_rbac_') ---- db error: ERROR: permission denied for TABLE "materialize.public.rbac_src" DETAIL: The 'rbac_owner' role needs SELECT privileges on TABLE "materialize.public.rbac_src" @@ -377,7 +385,7 @@ GRANT SELECT ON TABLE rbac_src TO rbac_reader COMPLETE 0 simple conn=rbac_reader,user=rbac_reader -CREATE METRIC SINK rbac_ms FROM rbac_src WITH (PREFIX = 'rbac_') +CREATE METRIC SINK rbac_ms FROM rbac_src WITH (PREFIX = 'mz_metric_sink_rbac_') ---- COMPLETE 0 diff --git a/test/sqllogictest/mz_catalog_server_index_accounting.slt b/test/sqllogictest/mz_catalog_server_index_accounting.slt index 3c108d65f8048..95138d7704663 100644 --- a/test/sqllogictest/mz_catalog_server_index_accounting.slt +++ b/test/sqllogictest/mz_catalog_server_index_accounting.slt @@ -345,6 +345,9 @@ mz_compute_hydration_times object_id mz_compute_hydration_times replica_id mz_compute_hydration_times time_ns mz_compute_hydration_times_per_worker export_id +mz_compute_hydration_times_per_worker hydrated_at +mz_compute_hydration_times_per_worker installed_at +mz_compute_hydration_times_per_worker started_at mz_compute_hydration_times_per_worker time_ns mz_compute_hydration_times_per_worker worker_id mz_compute_import_frontiers_per_worker export_id diff --git a/test/testdrive/hydration-timestamps.td b/test/testdrive/hydration-timestamps.td new file mode 100644 index 0000000000000..b4fb9c131926d --- /dev/null +++ b/test/testdrive/hydration-timestamps.td @@ -0,0 +1,215 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# Test the hydration lifecycle timestamps reported by +# `mz_introspection.mz_compute_hydration_times_per_worker`, alongside the +# `time_ns` duration the relation has always carried. +# +# These tests rely on testdrive's retry feature, as dataflows take an unknown +# (but hopefully small) time to be installed and to hydrate. The one section +# that must not retry, so that a transient invariant violation is not retried +# away, comes last, since `set-max-tries` has no way to restore the default. + +$ set-sql-timeout duration=60s + +> CREATE CLUSTER test SIZE 'scale=1,workers=2' +> SET cluster = test + +# Log collections are never suspended and never receive a `Schedule` command, +# so they must still report a complete set of timestamps. + +> SELECT DISTINCT + installed_at IS NOT NULL, + started_at IS NOT NULL, + hydrated_at IS NOT NULL + FROM mz_introspection.mz_compute_hydration_times_per_worker + WHERE export_id LIKE 's%' +true true true + +# A hydrated object reports all three timestamps, ordered. + +> CREATE TABLE t (a int) +> CREATE INDEX idx IN CLUSTER test ON t (a) + +> SELECT DISTINCT + h.installed_at IS NOT NULL, + h.started_at >= h.installed_at, + h.hydrated_at >= h.started_at, + h.time_ns IS NOT NULL + FROM mz_introspection.mz_compute_hydration_times_per_worker h + JOIN mz_indexes i ON (i.id = h.export_id) + WHERE i.name = 'idx' +true true true true + +# The timestamps are wallclock instants, not offsets from some arbitrary +# origin, so they must land in the recent past. + +> SELECT DISTINCT h.installed_at BETWEEN now() - '1 hour'::interval AND now() + FROM mz_introspection.mz_compute_hydration_times_per_worker h + JOIN mz_indexes i ON (i.id = h.export_id) + WHERE i.name = 'idx' +true + +# `time_ns` and the timestamps measure the same interval by different means: +# `time_ns` from an `Instant` taken when the export state is created, the +# timestamps from logging event times rounded to microseconds. They must agree, +# which is what makes the retained `time_ns` column and the new columns tell one +# consistent story. The tolerance is generous because the point is to catch a +# wrong interval or a units error, not to pin down the rounding. + +> SELECT DISTINCT + abs(extract(epoch FROM (h.hydrated_at - h.installed_at)) - h.time_ns / 1e9) < 1 + FROM mz_introspection.mz_compute_hydration_times_per_worker h + JOIN mz_indexes i ON (i.id = h.export_id) + WHERE i.name = 'idx' +true + +# An object whose inputs are unavailable is never scheduled, so it reports an +# `installed_at` but no `started_at`. A cluster with no replicas never advances +# the frontiers of the collections it maintains, which is what makes the input +# unavailable here. + +> CREATE CLUSTER source_cluster SIZE 'scale=1,workers=1', REPLICATION FACTOR 0 +> CREATE MATERIALIZED VIEW mv_unavailable IN CLUSTER source_cluster AS SELECT * FROM t + +> CREATE INDEX idx_waiting IN CLUSTER test ON mv_unavailable (a) + +> SELECT DISTINCT + h.installed_at IS NOT NULL, + h.started_at IS NULL, + h.hydrated_at IS NULL, + h.time_ns IS NULL + FROM mz_introspection.mz_compute_hydration_times_per_worker h + JOIN mz_indexes i ON (i.id = h.export_id) + WHERE i.name = 'idx_waiting' +true true true true + +# Once the input becomes available the object is scheduled and hydrates, and +# the queueing interval it spent waiting is visible as +# `started_at - installed_at`. + +> ALTER CLUSTER source_cluster SET (REPLICATION FACTOR 1) + +> SELECT DISTINCT + h.started_at > h.installed_at, + h.hydrated_at >= h.started_at + FROM mz_introspection.mz_compute_hydration_times_per_worker h + JOIN mz_indexes i ON (i.id = h.export_id) + WHERE i.name = 'idx_waiting' +true true + +> DROP INDEX idx_waiting +> DROP MATERIALIZED VIEW mv_unavailable +> DROP CLUSTER source_cluster + +# An object queued behind `compute_hydration_concurrency` reports an +# `installed_at` but no `started_at`, exactly as one waiting on its inputs +# does. `mv_stuck` oscillates between one row and no rows, so it never +# converges and holds the single hydration slot indefinitely, in constant +# memory. + +$ postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr} +ALTER SYSTEM SET compute_hydration_concurrency = 1; + +> CREATE CLUSTER queued SIZE 'scale=1,workers=1' +> SET cluster = queued + +> CREATE MATERIALIZED VIEW mv_stuck IN CLUSTER queued AS + WITH MUTUALLY RECURSIVE + x (a int) AS ( + SELECT 1 + EXCEPT ALL + SELECT a FROM x + ) + SELECT * FROM x + +# An in-flight object is distinguishable from one that never started: it has a +# `started_at` but no `hydrated_at`. +> SELECT DISTINCT h.started_at IS NOT NULL, h.hydrated_at IS NULL + FROM mz_introspection.mz_compute_hydration_times_per_worker h + JOIN mz_materialized_views mv ON (mv.id = h.export_id) + WHERE mv.name = 'mv_stuck' +true true + +> CREATE INDEX idx_queued IN CLUSTER queued ON t (a) + +> SELECT DISTINCT + h.installed_at IS NOT NULL, + h.started_at IS NULL, + h.hydrated_at IS NULL + FROM mz_introspection.mz_compute_hydration_times_per_worker h + JOIN mz_indexes i ON (i.id = h.export_id) + WHERE i.name = 'idx_queued' +true true true + +> DROP INDEX idx_queued +> DROP MATERIALIZED VIEW mv_stuck +> SET cluster = test +> DROP CLUSTER queued + +$ postgres-execute connection=postgres://mz_system:materialize@${testdrive.materialize-internal-sql-addr} +ALTER SYSTEM RESET compute_hydration_concurrency; + +# An import-free dataflow has nothing to suspend, so it carries `started_at` +# from creation rather than acquiring it when its `Schedule` lands. That keeps the +# row truthful while it hydrates: it reads as hydrating rather than as queued +# behind something that is not queueing it. +# +# An index is used rather than a materialized view because a constant +# materialized view runs to the empty frontier and its dataflow shuts down, +# taking the row with it. The two stamps land in the same turn but are separate +# log events, so they are microseconds apart rather than equal. + +> CREATE VIEW v_const AS SELECT 1 AS a +> CREATE INDEX idx_const IN CLUSTER test ON v_const (a) + +> SELECT DISTINCT + h.installed_at IS NOT NULL, + h.started_at >= h.installed_at, + h.started_at - h.installed_at < '1 second'::interval, + h.hydrated_at >= h.started_at + FROM mz_introspection.mz_compute_hydration_times_per_worker h + JOIN mz_indexes i ON (i.id = h.export_id) + WHERE i.name = 'idx_const' +true true true true + +> DROP INDEX idx_const +> DROP VIEW v_const + +# The ordering invariant `installed_at <= started_at <= hydrated_at` must hold +# for every row, and no row may report a later stage without its predecessors. +# Assert it repeatedly while import-free dataflows are created and torn down, +# since those are the paths that stamp outside `handle_schedule`. Retries are +# disabled from here on, so that a violation cannot be retried away. + +$ set-max-tries max-tries=1 + +> CREATE MATERIALIZED VIEW mv_const1 IN CLUSTER test AS SELECT 1 +> SELECT count(*) FROM mz_introspection.mz_compute_hydration_times_per_worker + WHERE installed_at IS NULL + OR (started_at IS NULL AND hydrated_at IS NOT NULL) + OR started_at < installed_at + OR hydrated_at < started_at +0 + +> CREATE MATERIALIZED VIEW mv_const2 IN CLUSTER test AS SELECT 2 +> SELECT count(*) FROM mz_introspection.mz_compute_hydration_times_per_worker + WHERE installed_at IS NULL + OR (started_at IS NULL AND hydrated_at IS NOT NULL) + OR started_at < installed_at + OR hydrated_at < started_at +0 + +> CREATE MATERIALIZED VIEW mv_const3 IN CLUSTER test AS SELECT 3 +> SELECT count(*) FROM mz_introspection.mz_compute_hydration_times_per_worker + WHERE installed_at IS NULL + OR (started_at IS NULL AND hydrated_at IS NOT NULL) + OR started_at < installed_at + OR hydrated_at < started_at +0 diff --git a/test/testdrive/metric-sink.td b/test/testdrive/metric-sink.td new file mode 100644 index 0000000000000..9aede34f74e60 --- /dev/null +++ b/test/testdrive/metric-sink.td @@ -0,0 +1,185 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. + +# End-to-end test for CREATE METRIC SINK: verifies that the series computed by +# a metric sink's dataflow actually reach the cluster replica's Prometheus +# registry, as observed through `mz_introspection.mz_cluster_prometheus_metrics` +# (the same registry the sink's collector registers into, scraped on +# `compute_prometheus_introspection_scrape_interval`), and that DROP METRIC +# SINK removes both the user series and the sink's companion gauges. +# +# This test does not depend on the deferred `mz_metric_sinks` catalog view; it +# reads the sink's id from the `sink` label the operator stamps on its own +# companion gauges. + +# A scrape has to land between a write and the assertion that observes it, so +# the default timeout is too tight here. +$ set-sql-timeout duration=60s + +> CREATE TABLE t (k text NOT NULL, val double) + +> INSERT INTO t VALUES ('a', 1), ('b', 2) + +> CREATE VIEW v AS + SELECT 'mz_metric_sink_test_value'::text AS metric_name, + 'gauge'::text AS metric_type, + map_build(LIST[ROW('k', k)])::map[text=>text] AS labels, + val AS value, + 'a metric emitted by the metric sink e2e test'::text AS help + FROM t + +# The FROM relation must expose the canonical metric-sink columns. A relation +# missing `value` (or with a wrongly-typed column) is rejected at plan time. +> CREATE VIEW bad_missing_value AS + SELECT 'm'::text AS metric_name, + 'gauge'::text AS metric_type, + map_build(LIST[ROW('k', k)])::map[text=>text] AS labels, + 'help'::text AS help + FROM t + +! CREATE METRIC SINK bad IN CLUSTER quickstart FROM bad_missing_value WITH (PREFIX = 'mz_metric_sink_bad_') +contains:metric sink source must expose column "value" + +> CREATE VIEW bad_wrong_type AS + SELECT 'm'::text AS metric_name, + 'gauge'::text AS metric_type, + map_build(LIST[ROW('k', k)])::map[text=>text] AS labels, + val::int AS value, + 'help'::text AS help + FROM t + +! CREATE METRIC SINK bad IN CLUSTER quickstart FROM bad_wrong_type WITH (PREFIX = 'mz_metric_sink_bad_') +contains:metric sink source column "value" is not of the required type + +> DROP VIEW bad_missing_value + +> DROP VIEW bad_wrong_type + +> CREATE METRIC SINK s IN CLUSTER quickstart FROM v WITH (PREFIX = 'mz_metric_sink_e2e_') + +# One family, two series: one per distinct label set. +> SELECT count(*) FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_metric_sink_e2e_mz_metric_sink_test_value' +2 + +> SELECT value = 1 FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_metric_sink_e2e_mz_metric_sink_test_value' AND labels -> 'k' = 'a' +true + +> SELECT value = 2 FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_metric_sink_e2e_mz_metric_sink_test_value' AND labels -> 'k' = 'b' +true + +> SELECT count(*) FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_metric_sink_e2e_mz_metric_sink_test_value' + AND metric_type = 'gauge' + AND help = 'a metric emitted by the metric sink e2e test' +2 + +# Read the sink's id off the companion gauge it stamps with a `sink` label. +# `LIMIT 1` guards the single-row contract of `set-from-sql`: today `s` is the +# only metric sink, but a second one (user or builtin) sharing this scrape +# window would otherwise make the capture fail. +$ set-from-sql var=sink-id +SELECT labels -> 'sink' FROM mz_introspection.mz_cluster_prometheus_metrics +WHERE metric_name = 'mz_compute_metric_sink_frontier_ms' +LIMIT 1 + +# The companion gauges exist, are labeled with this sink's id, and read healthy +# (no errors, no skipped/conflicting/colliding series). +> SELECT value > 0 FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_compute_metric_sink_frontier_ms' AND labels -> 'sink' = '${sink-id}' +true + +# Regression test for the compute write-frontier report (`sink_write_frontier` +# in `mz_compute::sink::metric_sink`). A metric sink writes to the process-local +# registry, not to a storage collection, but it still owns a compute write +# frontier the controller must observe. Without the report, `report_frontiers` +# logs `collection without write frontier` and skips the sink, so the controller +# never sees progress and pins the since of the relation the sink reads forever. +# A metric sink is not a storage sink, so it appears in `mz_frontiers`; its write +# frontier must advance past the value captured just after creation. Absent the +# report the frontier stays frozen at its creation `as_of` and this times out. +$ set-from-sql var=wf-before +SELECT write_frontier::text FROM mz_internal.mz_frontiers WHERE object_id = '${sink-id}' + +> SELECT write_frontier > ${wf-before}::mz_timestamp + FROM mz_internal.mz_frontiers WHERE object_id = '${sink-id}' +true + +> SELECT value = 0 FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_compute_metric_sink_errors' AND labels -> 'sink' = '${sink-id}' +true + +> SELECT value = 0 FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_compute_metric_sink_skipped' AND labels -> 'sink' = '${sink-id}' +true + +# A null `value` does not publish a zero: the `(metric_name, labels)` series +# for 'c' is absent from the registry (a gap) while its only live value is +# null, and the sink counts it in `mz_compute_metric_sink_null_values`. +> INSERT INTO t VALUES ('c', NULL) + +> SELECT count(*) FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_metric_sink_e2e_mz_metric_sink_test_value' +2 + +> SELECT count(*) FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_metric_sink_e2e_mz_metric_sink_test_value' AND labels -> 'k' = 'c' +0 + +> SELECT value = 1 FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_compute_metric_sink_null_values' AND labels -> 'sink' = '${sink-id}' +true + +# The gap closes once a non-null value arrives for the same series. +> UPDATE t SET val = 3 WHERE k = 'c' + +> SELECT count(*) FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_metric_sink_e2e_mz_metric_sink_test_value' +3 + +> SELECT value = 3 FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_metric_sink_e2e_mz_metric_sink_test_value' AND labels -> 'k' = 'c' +true + +> SELECT value = 0 FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_compute_metric_sink_null_values' AND labels -> 'sink' = '${sink-id}' +true + +# The sink pins the relation it reads, so the view can only go with the sink. +! DROP VIEW v +contains:still depended upon by metric sink "s" + +> DROP METRIC SINK s + +# The user series and all of this sink's companion gauges are retracted from +# the registry once the next scrape observes the collector is gone. +> SELECT count(*) FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_metric_sink_e2e_mz_metric_sink_test_value' +0 + +> SELECT count(*) FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE labels -> 'sink' = '${sink-id}' +0 + +# CASCADE takes the sink down with the relation it reads. +> CREATE METRIC SINK s_cascade IN CLUSTER quickstart FROM v WITH (PREFIX = 'mz_metric_sink_e2e_') + +> SELECT count(*) FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_metric_sink_e2e_mz_metric_sink_test_value' +3 + +> DROP VIEW v CASCADE + +> SELECT count(*) FROM mz_introspection.mz_cluster_prometheus_metrics + WHERE metric_name = 'mz_metric_sink_e2e_mz_metric_sink_test_value' +0 + +> DROP TABLE t