From 547323897bf69d692591e8548e7a42ea967b8ace Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Wed, 22 Jul 2026 13:24:44 +0200 Subject: [PATCH 01/15] ci: Run spec-sheet on Release (LTO) builds (#37802) Noticed in https://materializeinc.slack.com/archives/C097A6PRLQ2/p1784629759878079 --- ci/mkpipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/mkpipeline.py b/ci/mkpipeline.py index d481d1966fcca..d40eac5f20e11 100644 --- a/ci/mkpipeline.py +++ b/ci/mkpipeline.py @@ -208,6 +208,7 @@ def main() -> int: lto = ( pipeline.get("env", {}).get("CI_LTO", 0) == 1 or bool(os.environ["BUILDKITE_TAG"]) + or args.pipeline == "spec-sheet" or ( not ui.env_is_truthy("BUILDKITE_PULL_REQUEST") and args.pipeline in ("nightly", "release-qualification") From e4fe59bdca822c3364be5d963c68b92bc8f31514 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Wed, 22 Jul 2026 13:49:21 +0200 Subject: [PATCH 02/15] Introduce bin/ci-failures and tell mz-debug-ci skill to use it (#37632) Builds on top of https://github.com/MaterializeInc/ci-dashboard/pull/36 & https://github.com/MaterializeInc/i2/pull/3545 --- .agents/skills/mz-debug-ci/SKILL.md | 12 ++ bin/ci-failures | 14 ++ misc/python/materialize/cli/ci_failures.py | 146 +++++++++++++++++++++ 3 files changed, 172 insertions(+) create mode 100755 bin/ci-failures create mode 100644 misc/python/materialize/cli/ci_failures.py diff --git a/.agents/skills/mz-debug-ci/SKILL.md b/.agents/skills/mz-debug-ci/SKILL.md index d4f70fec3cf6f..0b9f361219561 100644 --- a/.agents/skills/mz-debug-ci/SKILL.md +++ b/.agents/skills/mz-debug-ci/SKILL.md @@ -24,6 +24,18 @@ If either tool is missing or unauthenticated, **stop immediately** and tell the Both `gh` and `bk` make network requests that are blocked by the default sandbox. All Bash commands in this workflow must use `dangerouslyDisableSandbox: true`. +## Search existing failures by pattern + +Search for already recorded failures in CI with a short, stable error substring: + +```bash +bin/ci-failures 'foobar' +``` + +Use `--search` for the global search. Narrow results with `--branch main`, +`--version`, `--issue`, `--test`, `--build`, `--start-date`, or `--end-date`. +The command explains how to create a token when needed. + ## Step 1: Extract PR number Parse `$ARGUMENTS` to get the PR number. Handle both formats: diff --git a/bin/ci-failures b/bin/ci-failures new file mode 100755 index 0000000000000..41557a608ea13 --- /dev/null +++ b/bin/ci-failures @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. +# +# ci-failures - Search historical CI failures. + +exec "$(dirname "$0")"/pyactivate -m materialize.cli.ci_failures "$@" diff --git a/misc/python/materialize/cli/ci_failures.py b/misc/python/materialize/cli/ci_failures.py new file mode 100644 index 0000000000000..4d2e97e3628a3 --- /dev/null +++ b/misc/python/materialize/cli/ci_failures.py @@ -0,0 +1,146 @@ +# Copyright Materialize, Inc. and contributors. All rights reserved. +# +# Use of this software is governed by the Business Source License +# included in the LICENSE file at the root of this repository. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0. +# +# ci_failures.py - Search historical CI failures. + +import argparse +import json +import os +import sys +from datetime import date +from typing import Any + +import requests + +API_URL = "https://ci.dev.materialize.com/api/failures" +TOKEN_URL = "https://ci.dev.materialize.com/tokens" + + +def fetch_failures( + *, + filters: list[dict[str, str]], + search: str | None, + start: int, + size: int, + start_date: str | None, + end_date: str | None, + token: str, +) -> Any: + params: dict[str, str | int] = { + "start": start, + "size": size, + "sorting": json.dumps( + [{"id": "build_date", "desc": True}], separators=(",", ":") + ), + } + if filters: + params["filters"] = json.dumps(filters, separators=(",", ":")) + if search: + params["globalFilter"] = search + if start_date: + params["dateMin"] = start_date + if end_date: + params["dateMax"] = end_date + + response = requests.get( + API_URL, + headers={"Authorization": f"Bearer {token}"}, + params=params, + timeout=30, + ) + response.raise_for_status() + return response.json() + + +def parse_date(value: str) -> str: + try: + date.fromisoformat(value) + except ValueError as error: + raise argparse.ArgumentTypeError("expected YYYY-MM-DD") from error + return value + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="ci-failures", + description="Search historical Materialize CI failures. All filters are ANDed.", + ) + parser.add_argument( + "pattern", nargs="?", help="case-insensitive failure content pattern" + ) + parser.add_argument( + "--search", + help="search build, test, issue, content, and branch", + ) + parser.add_argument("--branch", help="only return failures from this branch") + parser.add_argument("--version", help="only return failures from this version") + parser.add_argument("--issue", help="only return failures with this issue") + parser.add_argument("--test", help="only return failures from this test suite") + parser.add_argument("--build", help="only return failures from this build") + parser.add_argument( + "--start-date", type=parse_date, help="earliest build date (YYYY-MM-DD)" + ) + parser.add_argument( + "--end-date", type=parse_date, help="latest build date (YYYY-MM-DD)" + ) + parser.add_argument("--start", type=int, default=0, help="result offset") + parser.add_argument( + "--size", type=int, default=31, help="number of results, from 1 to 100" + ) + args = parser.parse_args(argv) + + if args.start < 0: + parser.error("--start must be nonnegative") + if not 1 <= args.size <= 100: + parser.error("--size must be between 1 and 100") + if args.start_date and args.end_date and args.start_date > args.end_date: + parser.error("--end-date must not be earlier than --start-date") + + token = os.getenv("CI_DASHBOARD_TOKEN") + if not token: + parser.error( + "CI_DASHBOARD_TOKEN is not set. " + f"Create a token manually at {TOKEN_URL}, then export it as " + "CI_DASHBOARD_TOKEN." + ) + + filters = [] + if args.pattern: + filters.append({"id": "content", "value": args.pattern}) + for argument, column in ( + (args.branch, "branch"), + (args.version, "mz_version"), + (args.issue, "issue"), + (args.test, "test_suite"), + (args.build, "build_identifier"), + ): + if argument: + filters.append({"id": column, "value": argument}) + + try: + failures = fetch_failures( + filters=filters, + search=args.search, + start=args.start, + size=args.size, + start_date=args.start_date, + end_date=args.end_date, + token=token, + ) + except requests.RequestException as error: + print(f"ci-failures: request failed: {error}", file=sys.stderr) + return 1 + + json.dump(failures, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 18d4571699f43b26eaba75edde03d33886cb707c Mon Sep 17 00:00:00 2001 From: Moritz Hoffmann Date: Wed, 22 Jul 2026 15:20:14 +0200 Subject: [PATCH 03/15] compute: scope column_paged_batcher_swap_pageout to Replica (#37799) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation `COLUMN_PAGED_BATCHER_SWAP_PAGEOUT` lacked the `.scoped(ParameterScope::Replica)` annotation its three siblings carry (`enable_column_paged_batcher`, `enable_column_paged_batcher_spill`, `column_paged_batcher_lz4`), so it defaulted to `ParameterScope::Environment`. The sync loop never built a replica context for it, so per-replica or per-size-family LaunchDarkly rules could not bind and every replica resolved the environment-wide value. This blocked isolating the eager `MADV_PAGEOUT` layer on a single replica: in the pager hydration variant ladder a `paged_pageout` replica meant to add only the pageout layer silently equalled the `paged_lz4` rung. Fixes CPU-177. ### Description Add `.scoped(ParameterScope::Replica)` to match the sibling flags. ### Verification Compiles clean (`cargo check`, `cargo clippy`). The LaunchDarkly segment and flag rule for the pageout rung are already in place and will bind once a binary carrying the scope is deployed to staging. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.8 --- src/compute-types/src/dyncfgs.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/compute-types/src/dyncfgs.rs b/src/compute-types/src/dyncfgs.rs index e1e8a96349d18..c8b800ff7cb77 100644 --- a/src/compute-types/src/dyncfgs.rs +++ b/src/compute-types/src/dyncfgs.rs @@ -116,7 +116,8 @@ pub const COLUMN_PAGED_BATCHER_SWAP_PAGEOUT: Config = Config::new( "Eagerly evict the column-paged batcher's lz4-compressed swap-backend spill chunks from RSS \ via `MADV_PAGEOUT` (they otherwise receive no madvise and are reclaimed only lazily). Only \ meaningful when `column_paged_batcher_lz4 = true` and the swap backend is active.", -); +) +.scoped(ParameterScope::Replica); /// Whether rendering should use `mz_join_core` rather than DD's `JoinCore::join_core`. pub const ENABLE_MZ_JOIN_CORE: Config = Config::new( From 4bc482a2d5b44cf1e398ed0db383565e9e0853f3 Mon Sep 17 00:00:00 2001 From: Kay Kim Date: Wed, 22 Jul 2026 09:26:12 -0400 Subject: [PATCH 04/15] edu-60: add redirects for self-managed/v25 docs (#37798) --- doc/user/config.toml | 3 - doc/user/content/concepts/_index.md | 1 + doc/user/content/concepts/clusters.md | 1 + doc/user/content/concepts/indexes.md | 1 + doc/user/content/concepts/sinks.md | 1 + doc/user/content/concepts/sources.md | 1 + doc/user/content/concepts/views.md | 1 + doc/user/content/console/_index.md | 3 + doc/user/content/ingest-data/_index.md | 3 + .../content/ingest-data/cdc-cockroachdb.md | 2 + doc/user/content/ingest-data/kafka/_index.md | 2 + .../content/ingest-data/kafka/amazon-msk.md | 1 + .../ingest-data/kafka/kafka-self-hosted.md | 1 + .../ingest-data/monitoring-data-ingestion.md | 2 + doc/user/content/ingest-data/mysql/_index.md | 2 + .../network-security/ssh-tunnel.md | 1 + .../content/ingest-data/postgres/_index.md | 2 + .../content/ingest-data/troubleshooting.md | 1 + doc/user/content/integrations/_index.md | 16 +++-- doc/user/content/integrations/cli/_index.md | 2 + .../integrations/cli/reference/_index.md | 2 + .../integrations/cli/reference/profile.md | 2 + .../integrations/cli/reference/user.md | 2 + doc/user/content/license.md | 2 + doc/user/content/manage/_index.md | 19 ++++++ doc/user/content/manage/dbt/_index.md | 2 + doc/user/content/releases/_index.md | 4 ++ .../content/security/self-managed/_index.md | 8 +++ .../self-managed-deployments/_index.md | 24 ++++++++ doc/user/content/self-managed/_index.md | 10 --- doc/user/content/self-managed/v25.2/_index.md | 61 ------------------- .../self-managed/v25.2/installation/_index.md | 10 --- .../v25.2/installation/configuration.md | 10 --- .../v25.2/installation/install-on-aws.md | 9 --- .../v25.2/installation/install-on-gcp.md | 9 --- .../installation/install-on-local-kind.md | 10 --- .../installation/install-on-local-minikube.md | 9 --- .../installation/operational-guidelines.md | 10 --- .../v25.2/installation/troubleshooting.md | 10 --- .../v25.2/installation/upgrading.md | 7 --- doc/user/content/serve-results/_index.md | 4 ++ doc/user/content/serve-results/sink/_index.md | 8 +++ doc/user/content/sql/_index.md | 24 ++++++++ doc/user/content/sql/create-source/_index.md | 8 +++ doc/user/content/sql/types/_index.md | 3 + doc/user/content/support.md | 2 + doc/user/content/transform-data/_index.md | 1 + .../dataflow-troubleshooting.md | 2 + doc/user/content/transform-data/faq.md | 2 + .../idiomatic-materialize-sql/_index.md | 9 +++ .../patterns/durable-subscriptions.md | 1 + .../transform-data/patterns/partition-by.md | 1 + .../transform-data/patterns/percentiles.md | 1 + .../transform-data/patterns/rules-engine.md | 1 + .../patterns/temporal-filters.md | 1 + 55 files changed, 172 insertions(+), 163 deletions(-) delete mode 100644 doc/user/content/self-managed/_index.md delete mode 100644 doc/user/content/self-managed/v25.2/_index.md delete mode 100644 doc/user/content/self-managed/v25.2/installation/_index.md delete mode 100644 doc/user/content/self-managed/v25.2/installation/configuration.md delete mode 100644 doc/user/content/self-managed/v25.2/installation/install-on-aws.md delete mode 100644 doc/user/content/self-managed/v25.2/installation/install-on-gcp.md delete mode 100644 doc/user/content/self-managed/v25.2/installation/install-on-local-kind.md delete mode 100644 doc/user/content/self-managed/v25.2/installation/install-on-local-minikube.md delete mode 100644 doc/user/content/self-managed/v25.2/installation/operational-guidelines.md delete mode 100644 doc/user/content/self-managed/v25.2/installation/troubleshooting.md delete mode 100644 doc/user/content/self-managed/v25.2/installation/upgrading.md diff --git a/doc/user/config.toml b/doc/user/config.toml index f3bf3bdb75727..debb0084a363e 100644 --- a/doc/user/config.toml +++ b/doc/user/config.toml @@ -139,6 +139,3 @@ include = "docs/**" # # include = "**.html" # would only include files with ".html" suffix # exclude = "**.{jpg, png}" # would exclude files with ".jpg" or ".png" suffix - -# Avoid deleting self-managed docs which are deployed from the self-managed-docs branches. -exclude = "docs/{self-managed/v25.2}/**" diff --git a/doc/user/content/concepts/_index.md b/doc/user/content/concepts/_index.md index 9eec6f6396947..48c02df590017 100644 --- a/doc/user/content/concepts/_index.md +++ b/doc/user/content/concepts/_index.md @@ -13,6 +13,7 @@ aliases: - /get-started/key-concepts/ - /overview - /self-managed/v25.1/concepts/ + - /self-managed/v25.2/concepts/ --- The pages in this section introduces some of the key concepts in Materialize: diff --git a/doc/user/content/concepts/clusters.md b/doc/user/content/concepts/clusters.md index 5d9632964a35c..141f0d082accd 100644 --- a/doc/user/content/concepts/clusters.md +++ b/doc/user/content/concepts/clusters.md @@ -8,6 +8,7 @@ menu: identifier: 'concepts-clusters' aliases: - /get-started/key-concepts/#clusters + - /self-managed/v25.2/concepts/clusters/ --- ## Overview diff --git a/doc/user/content/concepts/indexes.md b/doc/user/content/concepts/indexes.md index 7a198d69fec95..371f20271f07d 100644 --- a/doc/user/content/concepts/indexes.md +++ b/doc/user/content/concepts/indexes.md @@ -9,6 +9,7 @@ menu: aliases: - /get-started/key-concepts/#indexes - /self-managed/v25.1/concepts/indexes/ + - /self-managed/v25.2/concepts/indexes/ --- ## Overview diff --git a/doc/user/content/concepts/sinks.md b/doc/user/content/concepts/sinks.md index 2a589331aea66..8e3355e39e716 100644 --- a/doc/user/content/concepts/sinks.md +++ b/doc/user/content/concepts/sinks.md @@ -8,6 +8,7 @@ menu: identifier: 'concepts-sinks' aliases: - /get-started/key-concepts/#sinks + - /self-managed/v25.2/concepts/sinks/ --- ## Overview diff --git a/doc/user/content/concepts/sources.md b/doc/user/content/concepts/sources.md index 36281d1d13b39..d58b0dee693a6 100644 --- a/doc/user/content/concepts/sources.md +++ b/doc/user/content/concepts/sources.md @@ -8,6 +8,7 @@ menu: identifier: 'concepts-sources' aliases: - /get-started/key-concepts/#sources + - /self-managed/v25.2/concepts/sources/ --- ## Overview diff --git a/doc/user/content/concepts/views.md b/doc/user/content/concepts/views.md index b147dee7b2c71..fd5e6c9575bb2 100644 --- a/doc/user/content/concepts/views.md +++ b/doc/user/content/concepts/views.md @@ -8,6 +8,7 @@ menu: identifier: 'concepts-views' aliases: - /get-started/key-concepts/#views + - /self-managed/v25.2/concepts/views/ --- ## Overview diff --git a/doc/user/content/console/_index.md b/doc/user/content/console/_index.md index 5c2b8b170663c..ec203c1eaf4af 100644 --- a/doc/user/content/console/_index.md +++ b/doc/user/content/console/_index.md @@ -7,6 +7,9 @@ menus: main: weight: 5 identifier: 'console' +aliases: + - /self-managed/v25.2/console/ + - /self-managed/v25.2/console/integrations/ --- The Materialize Console is a graphical user diff --git a/doc/user/content/ingest-data/_index.md b/doc/user/content/ingest-data/_index.md index 11b686308567c..c257690b37d81 100644 --- a/doc/user/content/ingest-data/_index.md +++ b/doc/user/content/ingest-data/_index.md @@ -9,6 +9,9 @@ menu: weight: 11 aliases: - /self-managed/v25.1/ingest-data/ + - /self-managed/v25.2/ingest-data/kafka/amazon-msk/ + - /self-managed/v25.2/ingest-data/kafka/kafka-self-hosted/ + - /self-managed/v25.2/ingest-data/troubleshooting/ --- You can ingest data into Materialize from various external systems: diff --git a/doc/user/content/ingest-data/cdc-cockroachdb.md b/doc/user/content/ingest-data/cdc-cockroachdb.md index b0626399ada59..d23eb52b79f3d 100644 --- a/doc/user/content/ingest-data/cdc-cockroachdb.md +++ b/doc/user/content/ingest-data/cdc-cockroachdb.md @@ -7,6 +7,8 @@ menu: name: "Using Kafka and Changefeeds" identifier: "crdb-kafka-changefeeds" weight: 5 +aliases: + - /self-managed/v25.2/ingest-data/cdc-cockroachdb/ --- {{< tip >}} diff --git a/doc/user/content/ingest-data/kafka/_index.md b/doc/user/content/ingest-data/kafka/_index.md index cd5bedd3b4b9f..1ae9d0a0c04b3 100644 --- a/doc/user/content/ingest-data/kafka/_index.md +++ b/doc/user/content/ingest-data/kafka/_index.md @@ -7,6 +7,8 @@ menu: parent: 'ingest-data' identifier: 'kafka' weight: 20 +aliases: + - /self-managed/v25.2/ingest-data/kafka/ --- Materialize provides native connector for Kafka message broker. To ingest data diff --git a/doc/user/content/ingest-data/kafka/amazon-msk.md b/doc/user/content/ingest-data/kafka/amazon-msk.md index 5eacf6567ac25..d9abe67ba70a5 100644 --- a/doc/user/content/ingest-data/kafka/amazon-msk.md +++ b/doc/user/content/ingest-data/kafka/amazon-msk.md @@ -6,6 +6,7 @@ aliases: - /integrations/amazon-msk/ - /connect-sources/amazon-msk/ - /ingest-data/amazon-msk/ + - /self-managed/v25.2/ingest-data/kafka/amazon-msk/ menu: main: parent: "kafka" diff --git a/doc/user/content/ingest-data/kafka/kafka-self-hosted.md b/doc/user/content/ingest-data/kafka/kafka-self-hosted.md index 1daff9b8afc42..1428987363cc6 100644 --- a/doc/user/content/ingest-data/kafka/kafka-self-hosted.md +++ b/doc/user/content/ingest-data/kafka/kafka-self-hosted.md @@ -7,6 +7,7 @@ aliases: - /connect-sources/amazon-kafka/ - /ingest-data/kafka-self-hosted/ - /ingest-data/kafka/upstash-kafka/ + - /self-managed/v25.2/ingest-data/kafka/kafka-self-hosted/ menu: main: parent: "kafka" diff --git a/doc/user/content/ingest-data/monitoring-data-ingestion.md b/doc/user/content/ingest-data/monitoring-data-ingestion.md index 1d9d660691522..0149e8136482b 100644 --- a/doc/user/content/ingest-data/monitoring-data-ingestion.md +++ b/doc/user/content/ingest-data/monitoring-data-ingestion.md @@ -6,6 +6,8 @@ menu: identifier: ingest-monitoring parent: ingest-data weight: 39 +aliases: + - /self-managed/v25.2/ingest-data/monitoring-data-ingestion/ --- ## Monitoring the snapshotting progress diff --git a/doc/user/content/ingest-data/mysql/_index.md b/doc/user/content/ingest-data/mysql/_index.md index 81a57a11ef8c6..2ead882971a75 100644 --- a/doc/user/content/ingest-data/mysql/_index.md +++ b/doc/user/content/ingest-data/mysql/_index.md @@ -7,6 +7,8 @@ menu: parent: "ingest-data" identifier: "mysql" weight: 5 +aliases: + - /self-managed/v25.2/ingest-data/mysql/ --- ## Change Data Capture (CDC) diff --git a/doc/user/content/ingest-data/network-security/ssh-tunnel.md b/doc/user/content/ingest-data/network-security/ssh-tunnel.md index cf2489fe91152..467750d02fe58 100644 --- a/doc/user/content/ingest-data/network-security/ssh-tunnel.md +++ b/doc/user/content/ingest-data/network-security/ssh-tunnel.md @@ -9,6 +9,7 @@ aliases: - /integrations/postgres-bastion/ - /ops/network-security/ssh-tunnel/ - /connect-sources/ssh-tunnel/ + - /self-managed/v25.2/ingest-data/network-security/ssh-tunnel/ --- {{< tabs >}} diff --git a/doc/user/content/ingest-data/postgres/_index.md b/doc/user/content/ingest-data/postgres/_index.md index d0860e13fd36f..cefdc20d646cf 100644 --- a/doc/user/content/ingest-data/postgres/_index.md +++ b/doc/user/content/ingest-data/postgres/_index.md @@ -9,6 +9,8 @@ menu: identifier: 'postgresql' aliases: - /self-managed/v25.1/ingest-data/postgres/ + - /self-managed/v25.2/ingest-data/postgres/ + - /self-managed/v25.2/ingest-data/postgres/amazon-aurora/ --- ## Change Data Capture (CDC) diff --git a/doc/user/content/ingest-data/troubleshooting.md b/doc/user/content/ingest-data/troubleshooting.md index 9e9d050273e33..8da3c5beee13d 100644 --- a/doc/user/content/ingest-data/troubleshooting.md +++ b/doc/user/content/ingest-data/troubleshooting.md @@ -10,6 +10,7 @@ menu: aliases: - /ops/diagnosing-using-sql/ - /ops/troubleshooting/ + - /self-managed/v25.2/ingest-data/troubleshooting/ --- As you wire up data ingestion in Materialize, you might run into some snags or diff --git a/doc/user/content/integrations/_index.md b/doc/user/content/integrations/_index.md index 4d40ed41689f3..6d6cbaafad745 100644 --- a/doc/user/content/integrations/_index.md +++ b/doc/user/content/integrations/_index.md @@ -4,11 +4,17 @@ description: "Get details about third-party tools and integrations supported by make_table_row_headers_searchable: true disable_list: true aliases: - - /third-party/supported-tools/ - - /third-party/ - - /third-party/postgres-cloud/ - - /guides/postgres-cloud/ - - /guides/ + - /third-party/supported-tools/ + - /third-party/ + - /third-party/postgres-cloud/ + - /guides/postgres-cloud/ + - /guides/ + - /self-managed/v25.2/integrations/mz-debug/ + - /self-managed/v25.2/integrations/mz-debug/self-managed/ + - /self-managed/v25.2/integrations/mz-debug/emulator/ + - /self-managed/v25.2/integrations/http-api/ + - /self-managed/v25.2/integrations/websocket-api/ + - /self-managed/v25.2/integrations/llm/ menu: main: identifier: "integrations" diff --git a/doc/user/content/integrations/cli/_index.md b/doc/user/content/integrations/cli/_index.md index e166a9b7e8708..045c6d25390dd 100644 --- a/doc/user/content/integrations/cli/_index.md +++ b/doc/user/content/integrations/cli/_index.md @@ -8,6 +8,8 @@ menu: identifier: cli weight: 99 disable_list: true +aliases: + - /self-managed/v25.2/integrations/cli/ --- The Materialize command-line interface (CLI), lets you interact with diff --git a/doc/user/content/integrations/cli/reference/_index.md b/doc/user/content/integrations/cli/reference/_index.md index 044e81e98ff73..bc5f14cf40506 100644 --- a/doc/user/content/integrations/cli/reference/_index.md +++ b/doc/user/content/integrations/cli/reference/_index.md @@ -7,4 +7,6 @@ menu: name: "`mz` Reference" identifier: cli-reference weight: 6 +aliases: + - /self-managed/v25.2/integrations/cli/reference/ --- diff --git a/doc/user/content/integrations/cli/reference/profile.md b/doc/user/content/integrations/cli/reference/profile.md index db727bbe50c28..6cc361ad9d95f 100644 --- a/doc/user/content/integrations/cli/reference/profile.md +++ b/doc/user/content/integrations/cli/reference/profile.md @@ -5,6 +5,8 @@ menu: main: parent: cli-reference weight: 1 +aliases: + - /self-managed/v25.2/integrations/cli/reference/profile/ --- The `mz profile` command manages [authentication profiles] for `mz`. diff --git a/doc/user/content/integrations/cli/reference/user.md b/doc/user/content/integrations/cli/reference/user.md index b0c17a19fb78d..07889e22a409e 100644 --- a/doc/user/content/integrations/cli/reference/user.md +++ b/doc/user/content/integrations/cli/reference/user.md @@ -5,6 +5,8 @@ menu: main: parent: cli-reference weight: 1 +aliases: + - /self-managed/v25.2/integrations/cli/reference/user/ --- The `mz user` command manages users in your organization. diff --git a/doc/user/content/license.md b/doc/user/content/license.md index 578d0d8cdd4cc..f2e9d47111964 100644 --- a/doc/user/content/license.md +++ b/doc/user/content/license.md @@ -5,6 +5,8 @@ menu: main: parent: 'about' weight: 15 +aliases: + - /self-managed/v25.2/license/ --- ## Materialize Cloud diff --git a/doc/user/content/manage/_index.md b/doc/user/content/manage/_index.md index 13068dbea24e6..cd85c660bbe23 100644 --- a/doc/user/content/manage/_index.md +++ b/doc/user/content/manage/_index.md @@ -6,6 +6,25 @@ menu: main: identifier: "manage" weight: 50 +aliases: + - /self-managed/v25.2/manage/ + - /self-managed/v25.2/manage/authentication/ + - /self-managed/v25.2/manage/monitor/ + - /self-managed/v25.2/manage/monitor/prometheus/ + - /self-managed/v25.2/manage/monitor/datadog/ + - /self-managed/v25.2/manage/monitor/alerting/ + - /self-managed/v25.2/manage/dbt/ + - /self-managed/v25.2/manage/dbt/get-started/ + - /self-managed/v25.2/manage/dbt/development-workflows/ + - /self-managed/v25.2/manage/dbt/blue-green-deployments/ + - /self-managed/v25.2/manage/dbt/slim-deployments/ + - /self-managed/v25.2/manage/operational-guidelines/ + - /self-managed/v25.2/manage/terraform/ + - /self-managed/v25.2/manage/terraform/get-started/ + - /self-managed/v25.2/manage/terraform/manage-resources/ + - /self-managed/v25.2/manage/terraform/manage-cloud-modules/ + - /self-managed/v25.2/manage/terraform/appendix-secret-stores/ + - /self-managed/v25.2/administration/usage/ --- This section contains various resources for managing Materialize. diff --git a/doc/user/content/manage/dbt/_index.md b/doc/user/content/manage/dbt/_index.md index df1e161e35dab..0606f3478eda7 100644 --- a/doc/user/content/manage/dbt/_index.md +++ b/doc/user/content/manage/dbt/_index.md @@ -6,6 +6,8 @@ aliases: - /guides/dbt/ - /third-party/dbt/ - /integrations/dbt/ + - /self-managed/v25.2/manage/dbt/get-started/ + - /self-managed/v25.2/manage/dbt/ menu: main: parent: manage diff --git a/doc/user/content/releases/_index.md b/doc/user/content/releases/_index.md index afab6a680ee34..5c2213df79dad 100644 --- a/doc/user/content/releases/_index.md +++ b/doc/user/content/releases/_index.md @@ -8,6 +8,10 @@ menu: weight: 80 aliases: - /self-managed/v25.1/release-notes/ + - /self-managed/v25.2/release-notes/ + - /self-managed/v25.2/release-notes/compatibility/ + - /self-managed/v25.2/release-notes/versioning_lifecycle/ + - /self-managed/v25.2/release-notes/compatibility/ --- {{< note >}} diff --git a/doc/user/content/security/self-managed/_index.md b/doc/user/content/security/self-managed/_index.md index 35d8ed7b59ff9..0bd2f3c0fd6f6 100644 --- a/doc/user/content/security/self-managed/_index.md +++ b/doc/user/content/security/self-managed/_index.md @@ -8,6 +8,14 @@ menu: parent: "security" identifier: "security-sm" weight: 20 +aliases: + - /self-managed/v25.2/manage/access-control/ + - /self-managed/v25.2/manage/access-control/manage-roles/ + - /self-managed/v25.2/manage/access-control/appendix-privileges/ + - /self-managed/v25.2/manage/access-control/appendix-command-privileges/ + - /self-managed/v25.2/manage/access-control/appendix-built-in-roles/ + - /self-managed/v25.2/manage/access-control/rbac-terraform-tutorial/ + - /self-managed/v25.2/manage/appendix-alternative-cluster-architectures/ --- This section covers security for Self-Managed Materialize. diff --git a/doc/user/content/self-managed-deployments/_index.md b/doc/user/content/self-managed-deployments/_index.md index 03d46e946cc50..2fe6951743984 100644 --- a/doc/user/content/self-managed-deployments/_index.md +++ b/doc/user/content/self-managed-deployments/_index.md @@ -5,6 +5,30 @@ disable_list: true aliases: - /self-hosted/concepts/ - /self-managed-deployments/appendix/legacy/appendix-legacy-terraform-releases/ + - /self-managed/ + - /self-managed/v25.2/ + - /self-managed/v25.2/installation/ + - /self-managed/v25.2/installation/configuration/ + - /self-managed/v25.2/installation/install-on-aws/ + - /self-managed/v25.2/installation/install-on-aws/upgrade-on-aws/ + - /self-managed/v25.2/installation/install-on-aws/appendix-aws-configuration/ + - /self-managed/v25.2/installation/install-on-awsappendix-deployment-guidelines/ + - /self-managed/v25.2/installation/install-on-gcp/ + - /self-managed/v25.2/installation/install-on-gcp/appendix-gcp-configuration/ + - /self-managed/v25.2/installation/install-on-azure/ + - /self-managed/v25.2/installation/install-on-azureappendix-deployment-guidelines/ + - /self-managed/v25.2/installation/install-on-azureappendix-azure-configuration/ + - /self-managed/v25.2/installation/install-on-azure/upgrade-on-azure/ + - /self-managed/v25.2/installation/install-on-local-kind/ + - /self-managed/v25.2/installation/install-on-local-minikube/ + - /self-managed/v25.2/installation/install-on-local-minikubeupgrade-on-local-minikube/ + - /self-managed/v25.2/installation/operational-guidelines/ + - /self-managed/v25.2/installation/troubleshooting/ + - /self-managed/v25.2/installation/appendix-terraforms/ + - /self-managed/v25.2/installation/upgrading/ + - /self-managed/v25.2/installation/troubleshooting/ + - /self-managed/v25.2/installation/upgrade-to-swap/ + menu: main: identifier: "sm-deployments" diff --git a/doc/user/content/self-managed/_index.md b/doc/user/content/self-managed/_index.md deleted file mode 100644 index c2ede8fb60ce6..0000000000000 --- a/doc/user/content/self-managed/_index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: "Self-managed Materialize" -description: "" -aliases: - - /self-hosted/ -robots: "noindex, nofollow" -draft: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/_index.md b/doc/user/content/self-managed/v25.2/_index.md deleted file mode 100644 index 5030a6cdee6e8..0000000000000 --- a/doc/user/content/self-managed/v25.2/_index.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: "Self-managed Materialize" -description: "" -aliases: - - /self-managed/ - - /lts/sql/create-source/file/ - - /lts/sql/create-source/kafka/ - - /lts/sql/types/float/ - - /lts/sql/create-source/s3/ - - /lts/sql/types/integer/ - - /lts/sql/show-index/ - - /lts/overview/timelines/ - - /lts/sql/create-source/kinesis/ - - /lts/integrations/sql-clients/ - - /lts/sql/create-source/pubnub/ - - /lts/sql/types/timestamp/ - - /lts/ - - /lts/sql/create-source/materialize-cdc/ - - /lts/install/ - - /lts/ops/monitoring/ - - /lts/overview/architecture/ - - /lts/integrations/aws-aurora/ - - /lts/overview/arrangements/ - - /lts/sql/create-source/ - - /lts/sql/functions/date-bin-hopping/ - - /lts/sql/functions/list_agg/ - - /lts/sql/patterns/top-k/ - - /lts/sql/types/array/ - - /lts/sql/types/text/ - - /lts/get-started/ - - /lts/integrations/debezium/ - - /lts/license/ - - /lts/overview/ - - /lts/sql/ - - /lts/sql/alter-index/ - - /lts/sql/execute/ - - /lts/sql/functions/csv_extract/ - - /lts/sql/functions/justify-days/ - - /lts/sql/prepare/ - - /lts/sql/show-columns/ - - /lts/sql/show-create-sink/ - - /lts/reference/system-catalog/ - - /lts/sql/types/interval/ - - /lts/support/ - - /lts/integrations/ - - /lts/integrations/aws-msk/ - - /lts/integrations/aws-rds/ - - /lts/integrations/dbt/ - - /lts/integrations/golang/ - - /lts/integrations/metabase/ - - /lts/ops/optimization/ - - /lts/ops/troubleshooting/ - - /lts/overview/key-concepts/ - - /lts/quickstarts/live-analytics/ - - /lts/quickstarts/log-parsing/ - - /lts/sql/commit/ - - /lts/sql/copy-from/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/_index.md b/doc/user/content/self-managed/v25.2/installation/_index.md deleted file mode 100644 index eccf9287b802f..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/_index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: "Installation" -description: "Installation guides for Self-Managed Materialize." -disable_list: true -aliases: - - /self-managed/installation/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/configuration.md b/doc/user/content/self-managed/v25.2/installation/configuration.md deleted file mode 100644 index ff086bf308ad1..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/configuration.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: "Materialize Operator Configuration" -description: "" -aliases: - - /self-hosted/configuration/ - - /self-managed/configuration/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/install-on-aws.md b/doc/user/content/self-managed/v25.2/installation/install-on-aws.md deleted file mode 100644 index 8c94084270466..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/install-on-aws.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Install on AWS" -description: "" -aliases: - - /self-managed/installation/install-on-aws/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/install-on-gcp.md b/doc/user/content/self-managed/v25.2/installation/install-on-gcp.md deleted file mode 100644 index 7badbb5930455..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/install-on-gcp.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Install on GCP" -description: "" -aliases: - - /self-managed/installation/install-on-gcp/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/install-on-local-kind.md b/doc/user/content/self-managed/v25.2/installation/install-on-local-kind.md deleted file mode 100644 index 25f8b8b7b5186..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/install-on-local-kind.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: "Install locally on kind" -description: "" -aliases: - - /self-hosted/install-on-local-kind/ - - /self-managed/installation/install-on-local-kind/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/install-on-local-minikube.md b/doc/user/content/self-managed/v25.2/installation/install-on-local-minikube.md deleted file mode 100644 index 1f275c2158dbf..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/install-on-local-minikube.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Install locally on minikube" -description: "" -aliases: - - /self-managed/installation/install-on-local-minikube/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/operational-guidelines.md b/doc/user/content/self-managed/v25.2/installation/operational-guidelines.md deleted file mode 100644 index 160da0f2cac39..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/operational-guidelines.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: "Operational guidelines" -description: "" -robots: "noindex, nofollow" -aliases: - - /self-managed/operational-guidelines/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/troubleshooting.md b/doc/user/content/self-managed/v25.2/installation/troubleshooting.md deleted file mode 100644 index 1b8d2346afd9e..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/troubleshooting.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: "Troubleshooting" -description: "" -aliases: - - /self-hosted/troubleshooting/ - - /self-managed/troubleshooting/ -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/self-managed/v25.2/installation/upgrading.md b/doc/user/content/self-managed/v25.2/installation/upgrading.md deleted file mode 100644 index b2d8cfeb6082e..0000000000000 --- a/doc/user/content/self-managed/v25.2/installation/upgrading.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: "Upgrading" -description: "Upgrading Helm chart and Materialize." -suppress_breadcrumb: true ---- - - diff --git a/doc/user/content/serve-results/_index.md b/doc/user/content/serve-results/_index.md index 2afd678e99e54..00a2184cf5904 100644 --- a/doc/user/content/serve-results/_index.md +++ b/doc/user/content/serve-results/_index.md @@ -6,6 +6,10 @@ menus: main: weight: 15 identifier: 'serve-results' +aliases: + - /self-managed/v25.2/serve-results/ + - /self-managed/v25.2/serve-results/query-results/ + - /self-managed/v25.2/serve-results/bi-tools/ --- In Materialize, indexed views and materialized views maintain up-to-date query diff --git a/doc/user/content/serve-results/sink/_index.md b/doc/user/content/serve-results/sink/_index.md index 10a053706d3c2..70e8b4f7d8cd6 100644 --- a/doc/user/content/serve-results/sink/_index.md +++ b/doc/user/content/serve-results/sink/_index.md @@ -8,6 +8,14 @@ menu: name: "Sink results" identifier: 'sink' weight: 40 +aliases: + - /self-managed/v25.2/serve-results/sink/ + - /self-managed/v25.2/serve-results/sink/s3/ + - /self-managed/v25.2/serve-results/sink/s3_compatible/ + - /self-managed/v25.2/serve-results/sink/census/ + - /self-managed/v25.2/serve-results/sink/kafka/ + - /self-managed/v25.2/serve-results/sink/snowflake/ + - /self-managed/v25.2/serve-results/sink/sink-troubleshooting/ --- A [sink](/concepts/sinks/) describes the external system you want Materialize to diff --git a/doc/user/content/sql/_index.md b/doc/user/content/sql/_index.md index a2e6618fdae5a..9f498378309bf 100644 --- a/doc/user/content/sql/_index.md +++ b/doc/user/content/sql/_index.md @@ -9,6 +9,30 @@ menu: weight: 100 aliases: - /sql/alter-owner/ + - /self-managed/v25.2/sql/ + - /self-managed/v25.2/sql/alter-system-set/ + - /self-managed/v25.2/sql/alter-system-reset/ + - /self-managed/v25.2/sql/alter-swap/ + - /self-managed/v25.2/sql/alter-role/ + - /self-managed/v25.2/sql/alter-owner/ + - /self-managed/v25.2/sql/alter-rename/ + - /self-managed/v25.2/sql/create-materialized-view/ + - /self-managed/v25.2/sql/create-cluster/ + - /self-managed/v25.2/sql/create-connection/ + - /self-managed/v25.2/sql/create-role/ + - /self-managed/v25.2/sql/create-sink/ + - /self-managed/v25.2/sql/create-sink/kafka/ + - /self-managed/v25.2/sql/set/ + - /self-managed/v25.2/sql/show/ + - /self-managed/v25.2/sql/show-roles/ + - /self-managed/v25.2/sql/appendix-cluster-sizes/ + - /self-managed/v25.2/sql/system-clusters/ + - /self-managed/v25.2/sql/system-catalog/ + - /self-managed/v25.2/sql/system-catalog/mz_catalog/ + - /self-managed/v25.2/sql/system-catalog/pg_catalog/ + - /self-managed/v25.2/sql/system-catalog/mz_internal/ + - /self-managed/v25.2/sql/system-catalog/mz_introspection/ + - /self-managed/v25.2/sql/system-catalog/information_schema/ --- ## Create/Alter/Drop Objects diff --git a/doc/user/content/sql/create-source/_index.md b/doc/user/content/sql/create-source/_index.md index 168869569c7bc..5f46384a21757 100644 --- a/doc/user/content/sql/create-source/_index.md +++ b/doc/user/content/sql/create-source/_index.md @@ -6,6 +6,14 @@ menu: main: parent: commands identifier: 'create-source' +aliases: + - /self-managed/v25.2/sql/create-source/ + - /self-managed/v25.2/sql/create-source/kafka/ + - /self-managed/v25.2/sql/create-source/mysql/ + - /self-managed/v25.2/sql/create-source/postgres/ + - /self-managed/v25.2/sql/create-source/sql-server/ + - /self-managed/v25.2/sql/create-source/load-generator/ + - /self-managed/v25.2/sql/create-source/webhook/ --- {{% include-headless "/headless/source-definition" %}} diff --git a/doc/user/content/sql/types/_index.md b/doc/user/content/sql/types/_index.md index 71456bd50fd66..6a996305a8bc0 100644 --- a/doc/user/content/sql/types/_index.md +++ b/doc/user/content/sql/types/_index.md @@ -8,6 +8,9 @@ menu: parent: sql weight: 110 disable_list: true +aliases: + - /self-managed/v25.2/sql/types/list/ + - /self-managed/v25.2/sql/types/ --- Materialize's type system consists of two classes of types: diff --git a/doc/user/content/support.md b/doc/user/content/support.md index aabb77f693c2f..fc3a5a5ac243d 100644 --- a/doc/user/content/support.md +++ b/doc/user/content/support.md @@ -5,6 +5,8 @@ menu: main: parent: 'about' weight: 10 +aliases: + - /self-managed/v25.2/support/ --- Whether you're building, scaling, or debugging, we're here to help. diff --git a/doc/user/content/transform-data/_index.md b/doc/user/content/transform-data/_index.md index 0938eaa48b89c..b8c475fb5ec03 100644 --- a/doc/user/content/transform-data/_index.md +++ b/doc/user/content/transform-data/_index.md @@ -9,6 +9,7 @@ menu: identifier: transform-overview aliases: - /self-managed/v25.1/transform-data/troubleshooting/ + - /self-managed/v25.2/transform-data/ --- With Materialize, you use SQL to transform your fast-changing data into **live diff --git a/doc/user/content/transform-data/dataflow-troubleshooting.md b/doc/user/content/transform-data/dataflow-troubleshooting.md index 140cccb1aa23e..5419ff6acd580 100644 --- a/doc/user/content/transform-data/dataflow-troubleshooting.md +++ b/doc/user/content/transform-data/dataflow-troubleshooting.md @@ -7,6 +7,8 @@ menu: identifier: dataflow-troubleshooting parent: transform-data weight: 88 +aliases: + - /self-managed/v25.2/transform-data/troubleshooting/ --- If you're unable to troubleshoot your issue using the [`Ingest data`](/ingest-data/troubleshooting/) diff --git a/doc/user/content/transform-data/faq.md b/doc/user/content/transform-data/faq.md index 95dac85be933b..4b921c6b0ddb8 100644 --- a/doc/user/content/transform-data/faq.md +++ b/doc/user/content/transform-data/faq.md @@ -7,6 +7,8 @@ menu: identifier: faq-indexes parent: transform-data weight: 100 +aliases: + - /self-managed/v25.2/transform-data/faq/ --- ## Do indexes in Materialize support `ORDER BY`? diff --git a/doc/user/content/transform-data/idiomatic-materialize-sql/_index.md b/doc/user/content/transform-data/idiomatic-materialize-sql/_index.md index 12a9615418c8a..c861cf4b2e0de 100644 --- a/doc/user/content/transform-data/idiomatic-materialize-sql/_index.md +++ b/doc/user/content/transform-data/idiomatic-materialize-sql/_index.md @@ -10,6 +10,15 @@ menu: aliases: - /transform-data/patterns/window-functions/ + - /self-managed/v25.2/transform-data/idiomatic-materialize-sql/ + - /self-managed/v25.2/transform-data/idiomatic-materialize-sql/first-value/ + - /self-managed/v25.2/transform-data/idiomatic-materialize-sql/last-value/ + - /self-managed/v25.2/transform-data/idiomatic-materialize-sql/any/ + - /self-managed/v25.2/transform-data/idiomatic-materialize-sql/lag/ + - /self-managed/v25.2/transform-data/idiomatic-materialize-sql/lead/ + - /self-managed/v25.2/transform-data/idiomatic-materialize-sql/mz_now/ + - /self-managed/v25.2/transform-data/idiomatic-materialize-sql/top-k/ + - /self-managed/v25.2/transform-data/idiomatic-materialize-sql/appendix/ --- Materialize follows the SQL standard (SQL-92) implementation and strives for diff --git a/doc/user/content/transform-data/patterns/durable-subscriptions.md b/doc/user/content/transform-data/patterns/durable-subscriptions.md index d41440aaaa2a1..6549ae70028c2 100644 --- a/doc/user/content/transform-data/patterns/durable-subscriptions.md +++ b/doc/user/content/transform-data/patterns/durable-subscriptions.md @@ -7,6 +7,7 @@ menu: aliases: - /transform-data/patterns/time-travel-queries/ - /transform-data/patterns/time-travel-queries/#history-retention-period + - /self-managed/v25.2/transform-data/patterns/durable-subscriptions/ --- [//]: # "TODO: Move to Serve results section" diff --git a/doc/user/content/transform-data/patterns/partition-by.md b/doc/user/content/transform-data/patterns/partition-by.md index e5fe1098b94ed..8ba93b4fd7fba 100644 --- a/doc/user/content/transform-data/patterns/partition-by.md +++ b/doc/user/content/transform-data/patterns/partition-by.md @@ -4,6 +4,7 @@ description: "Declare how collections are stored." aliases: - /guides/partition-by/ - /sql/patterns/partition-by/ + - /self-managed/v25.2/transform-data/patterns/partition-by/ menu: main: parent: 'sql-patterns' diff --git a/doc/user/content/transform-data/patterns/percentiles.md b/doc/user/content/transform-data/patterns/percentiles.md index 64f58f176c6fa..bb734b61d0a10 100644 --- a/doc/user/content/transform-data/patterns/percentiles.md +++ b/doc/user/content/transform-data/patterns/percentiles.md @@ -3,6 +3,7 @@ title: "Percentile calculation" description: "How to use histograms to efficiently calculate percentiles in Materialize." aliases: - /sql/patterns/percentiles/ + - /self-managed/v25.2/transform-data/patterns/percentiles/ menu: main: parent: 'sql-patterns' diff --git a/doc/user/content/transform-data/patterns/rules-engine.md b/doc/user/content/transform-data/patterns/rules-engine.md index 1d8e48cf1d69e..85a1894927152 100644 --- a/doc/user/content/transform-data/patterns/rules-engine.md +++ b/doc/user/content/transform-data/patterns/rules-engine.md @@ -4,6 +4,7 @@ description: "Encode rules as data and apply them using lateral joins." aliases: - /guides/rules-engine/ - /sql/patterns/rules-engine/ + - /self-managed/v25.2/transform-data/patterns/rules-engine/ menu: main: parent: 'sql-patterns' diff --git a/doc/user/content/transform-data/patterns/temporal-filters.md b/doc/user/content/transform-data/patterns/temporal-filters.md index 76b3da7358a87..a6ba58ad5c78d 100644 --- a/doc/user/content/transform-data/patterns/temporal-filters.md +++ b/doc/user/content/transform-data/patterns/temporal-filters.md @@ -4,6 +4,7 @@ description: "Perform time-windowed computation over temporal data." aliases: - /guides/temporal-filters/ - /sql/patterns/temporal-filters/ + - /self-managed/v25.2/transform-data/patterns/temporal-filters/ menu: main: parent: 'sql-patterns' From 9cd16275c8c0a32ca92fcaf736229c23ea5a28e3 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Wed, 22 Jul 2026 15:37:16 +0200 Subject: [PATCH 05/15] tests: Make parallel-workload more stressful (#37269) --- .../materialize/parallel_workload/database.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/misc/python/materialize/parallel_workload/database.py b/misc/python/materialize/parallel_workload/database.py index be35090a72168..32f7aa3d57bb1 100644 --- a/misc/python/materialize/parallel_workload/database.py +++ b/misc/python/materialize/parallel_workload/database.py @@ -58,24 +58,24 @@ from materialize.parallel_workload.expression import ExprKind, expression from materialize.parallel_workload.settings import Complexity, Scenario -MAX_COLUMNS = 5 +MAX_COLUMNS = 50 MAX_INCLUDE_HEADERS = 5 -MAX_ROWS = 50 +MAX_ROWS = 500 MAX_CLUSTERS = 4 MAX_CLUSTER_REPLICAS = 2 -MAX_DBS = 5 -MAX_SCHEMAS = 5 -MAX_TABLES = 5 -MAX_VIEWS = 15 -MAX_INDEXES = 15 -MAX_ROLES = 15 -MAX_WEBHOOK_SOURCES = 5 -MAX_KAFKA_SOURCES = 5 -MAX_MYSQL_SOURCES = 5 -MAX_SQL_SERVER_SOURCES = 5 -MAX_POSTGRES_SOURCES = 5 -MAX_KAFKA_SINKS = 5 -MAX_ICEBERG_SINKS = 5 +MAX_DBS = 50 +MAX_SCHEMAS = 50 +MAX_TABLES = 50 +MAX_VIEWS = 150 +MAX_INDEXES = 150 +MAX_ROLES = 150 +MAX_WEBHOOK_SOURCES = 50 +MAX_KAFKA_SOURCES = 50 +MAX_MYSQL_SOURCES = 50 +MAX_SQL_SERVER_SOURCES = 50 +MAX_POSTGRES_SOURCES = 50 +MAX_KAFKA_SINKS = 50 +MAX_ICEBERG_SINKS = 50 MAX_INITIAL_DBS = 1 MAX_INITIAL_SCHEMAS = 1 From cde048058b66031e187906bda8acef1379db53c3 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Wed, 22 Jul 2026 15:43:54 +0200 Subject: [PATCH 06/15] zippy: Fix various issues (#37522) * Remove debug prints accidentally introduced in 7986bb0a96. * Keep CI_MZ_SYSTEM_PARAMETER_DEFAULT across MzRestart, Mz0dtDeploy and BackupAndRestore instead of silently reverting the flags under test on the first restart. The parsing now lives in one place. * Make --seed reproducible: pick free object names in sorted order (set iteration depends on PYTHONHASHSEED) and replace the unseeded np.random.zipf pad sizes with random.paretovariate. * Restore the 50% chance for upstream tables without a primary key. has_pk was always True, which made all ShiftForward/ShiftBackward actions permanent no-ops, so UPDATEs never flowed through any CDC source. Debezium sources now only pick tables with a primary key since ENVELOPE DEBEZIUM requires a key. * Let KillClusterd alternatively require TableExists so it can fire in CopyToFromS3, which creates no views. * Trim unexecuted actions' capabilities using the instances recorded at generation time. provides() may return fresh instances and removal compares by identity, so *IsRunning capabilities were never removed. * Join KafkaInsertParallel background threads at the end of the test and fail on their errors instead of dropping them silently. The threads also target the current mz_service now. * Pass mz_service in CreateDebeziumSource, it otherwise routes through balancerd's potentially stale resolver. * Re-point balancerd at the new leader after Mz0dtDeploy, its static resolver otherwise keeps routing to the stopped old generation. * parse_timedelta: support seconds, reject unparseable input. * Only bootstrap Polaris in scenarios that can create Iceberg sinks. * Remove mid-test BackupAndRestore from UserTables, sink state is not recorded across a backup&restore cycle (database-issues#9589). * Remove unused actions (the *Stop actions for external services and MzStartParameterized) and the write-only State.iceberg_* fields. * Add DropTable/DropView actions, guarded so that no depended-upon object is dropped, and wire them into UserTables and KafkaSources. User objects were never dropped before. Test run: https://buildkite.com/materialize/nightly/builds/17089 Co-authored-by: Claude Fable 5 --- .../zippy/backup_and_restore_actions.py | 1 + .../materialize/zippy/balancerd_actions.py | 30 ++++--- .../materialize/zippy/debezium_actions.py | 89 +++++++++---------- misc/python/materialize/zippy/framework.py | 53 +++++++++-- .../materialize/zippy/iceberg_actions.py | 2 - .../python/materialize/zippy/kafka_actions.py | 34 ++++--- .../python/materialize/zippy/mysql_actions.py | 17 +--- misc/python/materialize/zippy/mz_actions.py | 70 ++++----------- .../materialize/zippy/postgres_actions.py | 17 +--- misc/python/materialize/zippy/scenarios.py | 30 +++++-- .../materialize/zippy/sql_server_actions.py | 17 +--- .../python/materialize/zippy/table_actions.py | 34 +++++++ misc/python/materialize/zippy/view_actions.py | 40 +++++++++ test/zippy/mzcompose.py | 26 +++--- 14 files changed, 250 insertions(+), 210 deletions(-) diff --git a/misc/python/materialize/zippy/backup_and_restore_actions.py b/misc/python/materialize/zippy/backup_and_restore_actions.py index 130aac4576952..3762ac3813edc 100644 --- a/misc/python/materialize/zippy/backup_and_restore_actions.py +++ b/misc/python/materialize/zippy/backup_and_restore_actions.py @@ -39,6 +39,7 @@ def run(self, c: Composition, state: State) -> None: system_parameter_defaults=state.system_parameter_defaults, sanity_restart=False, restart="on-failure", + additional_system_parameter_defaults=state.additional_system_parameter_defaults, metadata_store="cockroach", default_replication_factor=2, ) diff --git a/misc/python/materialize/zippy/balancerd_actions.py b/misc/python/materialize/zippy/balancerd_actions.py index 9f931bf7d7d55..e74b55aec8761 100644 --- a/misc/python/materialize/zippy/balancerd_actions.py +++ b/misc/python/materialize/zippy/balancerd_actions.py @@ -15,6 +15,20 @@ from materialize.zippy.mz_capabilities import MzIsRunning +def balancerd_service(state: State) -> Balancerd: + """Balancerd configured to route to the current Mz service.""" + return Balancerd( + https_resolver_template=f"{state.mz_service}:6876", + static_resolver_addr=f"{state.mz_service}:6875", + ) + + +def restart_balancerd(c: Composition, state: State) -> None: + with c.override(balancerd_service(state)): + c.kill("balancerd") + c.up("balancerd") + + class BalancerdStart(Action): """Starts balancerd""" @@ -27,12 +41,7 @@ def incompatible_with(cls) -> set[type[Capability]]: return {BalancerdIsRunning} def run(self, c: Composition, state: State) -> None: - with c.override( - Balancerd( - https_resolver_template=f"{state.mz_service}:6876", - static_resolver_addr=f"{state.mz_service}:6875", - ) - ): + with c.override(balancerd_service(state)): c.up("balancerd") def provides(self) -> list[Capability]: @@ -65,11 +74,4 @@ def requires(cls) -> set[type[Capability]]: return {BalancerdIsRunning, MzIsRunning} def run(self, c: Composition, state: State) -> None: - with c.override( - Balancerd( - https_resolver_template=f"{state.mz_service}:6876", - static_resolver_addr=f"{state.mz_service}:6875", - ) - ): - c.kill("balancerd") - c.up("balancerd") + restart_balancerd(c, state) diff --git a/misc/python/materialize/zippy/debezium_actions.py b/misc/python/materialize/zippy/debezium_actions.py index 6711744bfcc07..785ae32675a9e 100644 --- a/misc/python/materialize/zippy/debezium_actions.py +++ b/misc/python/materialize/zippy/debezium_actions.py @@ -35,20 +35,6 @@ def run(self, c: Composition, state: State) -> None: c.up("debezium") -class DebeziumStop(Action): - """Stop the Debezium instance.""" - - @classmethod - def requires(cls) -> set[type[Capability]]: - return {DebeziumRunning} - - def withholds(self) -> set[type[Capability]]: - return {DebeziumRunning} - - def run(self, c: Composition, state: State) -> None: - c.kill("debezium") - - class CreateDebeziumSource(Action): """Creates a Debezium source in Materialized.""" @@ -63,39 +49,47 @@ def requires(cls) -> set[type[Capability]]: } def __init__(self, capabilities: Capabilities) -> None: - # To avoid conflicts, we make sure the postgres table and the debezium source have matching names - postgres_table = random.choice(capabilities.get(PostgresTableExists)) - cluster_name = random.choice(source_capable_clusters(capabilities)) - debezium_source_name = f"debezium_source_{postgres_table.name}" - this_debezium_source = DebeziumSourceExists(name=debezium_source_name) - - existing_debezium_sources = [ - s - for s in capabilities.get(DebeziumSourceExists) - if s.name == this_debezium_source.name - ] - - if len(existing_debezium_sources) == 0: - self.new_debezium_source = True - - self.debezium_source = this_debezium_source - self.postgres_table = postgres_table - self.debezium_source.postgres_table = self.postgres_table - self.cluster_name = cluster_name - elif len(existing_debezium_sources) == 1: - self.new_debezium_source = False - - self.debezium_source = existing_debezium_sources[0] - assert self.debezium_source.postgres_table is not None - self.postgres_table = self.debezium_source.postgres_table - else: - raise RuntimeError("More than one Debezium source exists") + # ENVELOPE DEBEZIUM requires a key, so only Postgres tables with a + # primary key can back a Debezium source. + pk_tables = [t for t in capabilities.get(PostgresTableExists) if t.has_pk] + + self.new_debezium_source = False + self.debezium_source: DebeziumSourceExists | None = None + + if pk_tables: + # To avoid conflicts, we make sure the postgres table and the debezium source have matching names + postgres_table = random.choice(pk_tables) + cluster_name = random.choice(source_capable_clusters(capabilities)) + debezium_source_name = f"debezium_source_{postgres_table.name}" + this_debezium_source = DebeziumSourceExists(name=debezium_source_name) + + existing_debezium_sources = [ + s + for s in capabilities.get(DebeziumSourceExists) + if s.name == this_debezium_source.name + ] + + if len(existing_debezium_sources) == 0: + self.new_debezium_source = True + + self.debezium_source = this_debezium_source + self.postgres_table = postgres_table + self.debezium_source.postgres_table = self.postgres_table + self.cluster_name = cluster_name + elif len(existing_debezium_sources) == 1: + self.debezium_source = existing_debezium_sources[0] + assert self.debezium_source.postgres_table is not None + self.postgres_table = self.debezium_source.postgres_table + else: + raise RuntimeError("More than one Debezium source exists") super().__init__(capabilities) def run(self, c: Composition, state: State) -> None: if self.new_debezium_source: - c.testdrive(dedent(f""" + assert self.debezium_source is not None + c.testdrive( + dedent(f""" $ http-request method=POST url=http://debezium:8083/connectors content-type=application/json {{ "name": "{self.debezium_source.name}", @@ -113,7 +107,7 @@ def run(self, c: Composition, state: State) -> None: "publication.name": "dbz_publication_{self.debezium_source.name}", "publication.autocreate.mode": "filtered", "slot.name" : "slot_{self.postgres_table.name}", - "database.history.kafka.bootstrap.servers": "kafka:9092", + "database.history.kafka.bootstrap.servers": "redpanda:9092", "database.history.kafka.topic": "schema-changes.history", "truncate.handling.mode": "include", "decimal.handling.mode": "precise", @@ -134,7 +128,12 @@ def run(self, c: Composition, state: State) -> None: > CREATE TABLE {self.debezium_source.get_name_for_query()} FROM SOURCE {self.debezium_source.name} (REFERENCE "postgres.public.{self.postgres_table.name}") FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_conn ENVELOPE DEBEZIUM - """)) + """), + mz_service=state.mz_service, + ) def provides(self) -> list[Capability]: - return [self.debezium_source] if self.new_debezium_source else [] + if self.new_debezium_source: + assert self.debezium_source is not None + return [self.debezium_source] + return [] diff --git a/misc/python/materialize/zippy/framework.py b/misc/python/materialize/zippy/framework.py index 8ab253b4f5a9e..14344a837b115 100644 --- a/misc/python/materialize/zippy/framework.py +++ b/misc/python/materialize/zippy/framework.py @@ -7,7 +7,9 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. +import os import random +import threading from collections.abc import Sequence from datetime import datetime, timedelta from typing import TYPE_CHECKING, TypeVar @@ -19,6 +21,20 @@ from materialize.zippy.scenarios import Scenario +def ci_additional_system_parameter_defaults() -> dict[str, str]: + """Parse the CI_MZ_SYSTEM_PARAMETER_DEFAULT environment variable.""" + defaults = {} + system_parameter_default = os.getenv("CI_MZ_SYSTEM_PARAMETER_DEFAULT", "") + if system_parameter_default: + for val in system_parameter_default.split(";"): + x = val.split("=", maxsplit=1) + assert ( + len(x) == 2 + ), f"CI_MZ_SYSTEM_PARAMETER_DEFAULT '{val}' should be the format =" + defaults[x[0]] = x[1] + return defaults + + class State: mz_service: str deploy_generation: int @@ -28,8 +44,15 @@ def __init__(self): self.mz_service = "materialized" self.deploy_generation = 0 self.system_parameter_defaults = get_default_system_parameters() - self.iceberg_username: str | None = None - self.iceberg_key: str | None = None + # Every action that (re)starts Mz must pass these along, otherwise a + # feature flag under test silently reverts to its builtin default. + self.additional_system_parameter_defaults = ( + ci_additional_system_parameter_defaults() + ) + # Threads spawned by actions (e.g. parallel Kafka ingestion). They are + # joined at the end of the test and any errors they collected fail it. + self.background_threads: list[threading.Thread] = [] + self.background_errors: list[Exception] = [] class Capability: @@ -101,8 +124,10 @@ def get_free_capability_name( ] existing_object_names = self.get_capability_names(capability) remaining_object_names = set(all_object_names) - set(existing_object_names) + # sorted() so that the same --seed picks the same names regardless of + # set iteration order (which depends on PYTHONHASHSEED). return ( - random.choice(list(remaining_object_names)) + random.choice(sorted(remaining_object_names)) if len(remaining_object_names) > 0 else None ) @@ -191,6 +216,10 @@ def __init__( self._actions: list[Action] = [] self._final_actions: list[Action] = [] self._capabilities = Capabilities() + # The exact Capability instances each action added at generation time. + # Trimming in run() removes by identity, and provides() may build new + # instances on each call, so the originals must be remembered here. + self._provided_capabilities: dict[Action, list[Capability]] = {} self._actions_with_weight: dict[ActionOrFactory, float] = ( self._scenario.actions_with_weight() ) @@ -215,11 +244,10 @@ def generate_actions(self, action_def: ActionOrFactory) -> list[Action]: ) for action in actions: - print("test:", action) - self._capabilities._extend(action.provides()) - print(" - ", self._capabilities, action.provides()) + provided = action.provides() + self._provided_capabilities[action] = provided + self._capabilities._extend(provided) self._capabilities._remove(action.withholds()) - print(" - ", self._capabilities, action.withholds()) return actions @@ -237,10 +265,19 @@ def run(self, c: Composition) -> None: executed_count = i + 1 break + # Surface failures from background threads instead of dropping them. + for thread in self._state.background_threads: + thread.join() + if self._state.background_errors: + raise self._state.background_errors[0] + # Remove capability effects of actions that were never executed, # so that finalization only validates objects that actually exist. + # Capabilities withheld by unexecuted actions are not restored. That + # is fine because only the *IsRunning capabilities are ever withheld + # and finalization unconditionally starts those services again. for action in self._actions[executed_count:]: - for cap in action.provides(): + for cap in self._provided_capabilities[action]: self._capabilities.remove_capability_instance(cap) # Generate finalization actions now, after the main loop, so that diff --git a/misc/python/materialize/zippy/iceberg_actions.py b/misc/python/materialize/zippy/iceberg_actions.py index 5fd70f96f780d..e8f89aeed34a4 100644 --- a/misc/python/materialize/zippy/iceberg_actions.py +++ b/misc/python/materialize/zippy/iceberg_actions.py @@ -45,8 +45,6 @@ def run(self, c: Composition, state: State) -> None: Service("polaris", idle=True), ) username, key = setup_polaris_for_iceberg(c) - state.iceberg_username = username - state.iceberg_key = key c.testdrive( dedent(f""" diff --git a/misc/python/materialize/zippy/kafka_actions.py b/misc/python/materialize/zippy/kafka_actions.py index e9cafd4bba471..743c22af6a99c 100644 --- a/misc/python/materialize/zippy/kafka_actions.py +++ b/misc/python/materialize/zippy/kafka_actions.py @@ -12,8 +12,6 @@ import threading from textwrap import dedent -import numpy as np - from materialize.mzcompose.composition import Composition from materialize.zippy.framework import ( Action, @@ -55,20 +53,6 @@ def run(self, c: Composition, state: State) -> None: c.up("redpanda") -class KafkaStop(Action): - """Stop the Kafka instance.""" - - @classmethod - def requires(cls) -> set[type[Capability]]: - return {KafkaRunning} - - def withholds(self) -> set[type[Capability]]: - return {KafkaRunning} - - def run(self, c: Composition, state: State) -> None: - c.kill("redpanda") - - class CreateTopicParameterized(ActionFactory): """Creates a Kafka topic and decides on the envelope that will be used.""" @@ -139,8 +123,10 @@ def requires(cls) -> set[type[Capability]]: def __init__(self, capabilities: Capabilities) -> None: self.topic = random.choice(capabilities.get(TopicExists)) self.delta = random.randint(1, 10000) - # This gives 67% pads of up to 10 bytes, 25% of up to 100 bytes and outliers up to 256 bytes - self.pad = min(np.random.zipf(1.6, 1)[0], 256) * random.choice( + # Heavy-tailed pad sizes: mostly up to 10 bytes, some up to 100 bytes + # and outliers up to 256 bytes. int(paretovariate(0.6)) approximates a + # zipf(1.6) sample while staying reproducible under random.seed(). + self.pad = min(int(random.paretovariate(0.6)), 256) * random.choice( string.ascii_letters ) super().__init__(capabilities) @@ -167,7 +153,17 @@ def run(self, c: Composition, state: State) -> None: """) if self.parallel(): - threading.Thread(target=c.testdrive, args=[testdrive_str]).start() + mz_service = state.mz_service + + def ingest() -> None: + try: + c.testdrive(testdrive_str, mz_service=mz_service) + except Exception as e: + state.background_errors.append(e) + + thread = threading.Thread(target=ingest) + thread.start() + state.background_threads.append(thread) else: c.testdrive(testdrive_str, mz_service=state.mz_service) diff --git a/misc/python/materialize/zippy/mysql_actions.py b/misc/python/materialize/zippy/mysql_actions.py index 9d343b4c776a6..706618c78d590 100644 --- a/misc/python/materialize/zippy/mysql_actions.py +++ b/misc/python/materialize/zippy/mysql_actions.py @@ -40,20 +40,6 @@ def run(self, c: Composition, state: State) -> None: ) -class MySqlStop(Action): - """Stop the MySQL instance.""" - - @classmethod - def requires(cls) -> set[type[Capability]]: - return {MySqlRunning} - - def withholds(self) -> set[type[Capability]]: - return {MySqlRunning} - - def run(self, c: Composition, state: State) -> None: - c.kill("mysql") - - class MySqlRestart(Action): """Restart the MySql instance.""" @@ -80,8 +66,7 @@ def __init__(self, capabilities: Capabilities) -> None: if len(existing_mysql_tables) == 0: self.new_mysql_table = True - # A PK is now required for Debezium - this_mysql_table.has_pk = True + this_mysql_table.has_pk = random.choice([True, False]) self.mysql_table = this_mysql_table elif len(existing_mysql_tables) == 1: diff --git a/misc/python/materialize/zippy/mz_actions.py b/misc/python/materialize/zippy/mz_actions.py index 03389996dab7d..4a76858f1a830 100644 --- a/misc/python/materialize/zippy/mz_actions.py +++ b/misc/python/materialize/zippy/mz_actions.py @@ -8,54 +8,27 @@ # by the Apache License, Version 2.0. -import os - from materialize.mzcompose.composition import Composition from materialize.mzcompose.services.materialized import ( LEADER_STATUS_HEALTHCHECK, DeploymentStatus, Materialized, ) +from materialize.zippy.balancerd_actions import restart_balancerd from materialize.zippy.balancerd_capabilities import BalancerdIsRunning from materialize.zippy.blob_store_capabilities import BlobStoreIsRunning from materialize.zippy.crdb_capabilities import CockroachIsRunning from materialize.zippy.framework import ( Action, - ActionFactory, - Capabilities, Capability, Mz0dtDeployBaseAction, State, ) from materialize.zippy.mz_capabilities import MzIsRunning +from materialize.zippy.table_capabilities import TableExists from materialize.zippy.view_capabilities import ViewExists -class MzStartParameterized(ActionFactory): - """Starts a Mz instance with custom paramters.""" - - @classmethod - def requires(cls) -> set[type[Capability]]: - return {CockroachIsRunning, BlobStoreIsRunning} - - @classmethod - def incompatible_with(cls) -> set[type[Capability]]: - return {MzIsRunning} - - def __init__( - self, additional_system_parameter_defaults: dict[str, str] = {} - ) -> None: - self.additional_system_parameter_defaults = additional_system_parameter_defaults - - def new(self, capabilities: Capabilities) -> list[Action]: - return [ - MzStart( - capabilities=capabilities, - additional_system_parameter_defaults=self.additional_system_parameter_defaults, - ) - ] - - class MzStart(Action): """Starts a Mz instance (all components are running in the same container).""" @@ -67,31 +40,9 @@ def requires(cls) -> set[type[Capability]]: def incompatible_with(cls) -> set[type[Capability]]: return {MzIsRunning} - def __init__( - self, - capabilities: Capabilities, - additional_system_parameter_defaults: dict[str, str] = {}, - ) -> None: - if additional_system_parameter_defaults: - self.additional_system_parameter_defaults = ( - additional_system_parameter_defaults - ) - else: - self.additional_system_parameter_defaults = {} - system_parameter_default = os.getenv("CI_MZ_SYSTEM_PARAMETER_DEFAULT", "") - if system_parameter_default: - for val in system_parameter_default.split(";"): - x = val.split("=", maxsplit=1) - assert ( - len(x) == 2 - ), f"CI_MZ_SYSTEM_PARAMETER_DEFAULT '{val}' should be the format =" - self.additional_system_parameter_defaults[x[0]] = x[1] - - super().__init__(capabilities) - def run(self, c: Composition, state: State) -> None: print( - f"Starting Mz with additional_system_parameter_defaults = {self.additional_system_parameter_defaults}" + f"Starting Mz with additional_system_parameter_defaults = {state.additional_system_parameter_defaults}" ) with c.override( @@ -104,7 +55,7 @@ def run(self, c: Composition, state: State) -> None: system_parameter_defaults=state.system_parameter_defaults, sanity_restart=False, restart="on-failure", - additional_system_parameter_defaults=self.additional_system_parameter_defaults, + additional_system_parameter_defaults=state.additional_system_parameter_defaults, metadata_store="cockroach", default_replication_factor=2, ) @@ -183,6 +134,7 @@ def run(self, c: Composition, state: State) -> None: system_parameter_defaults=state.system_parameter_defaults, sanity_restart=False, restart="on-failure", + additional_system_parameter_defaults=state.additional_system_parameter_defaults, metadata_store="cockroach", default_replication_factor=2, ) @@ -218,6 +170,7 @@ def run(self, c: Composition, state: State) -> None: sanity_restart=False, restart="on-failure", healthcheck=LEADER_STATUS_HEALTHCHECK, + additional_system_parameter_defaults=state.additional_system_parameter_defaults, metadata_store="cockroach", default_replication_factor=2, ), @@ -237,13 +190,20 @@ def run(self, c: Composition, state: State) -> None: wait=True, ) + # Balancerd's resolver is fixed at startup and still points at the + # previous generation. Re-point it at the new leader. + if c.is_running("balancerd"): + restart_balancerd(c, state) + class KillClusterd(Action): """Kills the clusterd processes in the environmentd container. The process orchestrator will restart them.""" @classmethod - def requires(cls) -> set[type[Capability]]: - return {MzIsRunning, ViewExists} + def requires(cls) -> list[set[type[Capability]]]: + # Only kill once a dataflow-bearing object exists, so the kill has + # something to disrupt. + return [{MzIsRunning, ViewExists}, {MzIsRunning, TableExists}] def run(self, c: Composition, state: State) -> None: # Depending on the workload, clusterd may not be running, hence the || true diff --git a/misc/python/materialize/zippy/postgres_actions.py b/misc/python/materialize/zippy/postgres_actions.py index 512335cb5969c..c30454cee6fc2 100644 --- a/misc/python/materialize/zippy/postgres_actions.py +++ b/misc/python/materialize/zippy/postgres_actions.py @@ -27,20 +27,6 @@ def run(self, c: Composition, state: State) -> None: c.up("postgres") -class PostgresStop(Action): - """Stop the Postgres instance.""" - - @classmethod - def requires(cls) -> set[type[Capability]]: - return {PostgresRunning} - - def withholds(self) -> set[type[Capability]]: - return {PostgresRunning} - - def run(self, c: Composition, state: State) -> None: - c.kill("postgres") - - class PostgresRestart(Action): """Restart the Postgres instance.""" @@ -69,8 +55,7 @@ def __init__(self, capabilities: Capabilities) -> None: if len(existing_postgres_tables) == 0: self.new_postgres_table = True - # A PK is now required for Debezium - this_postgres_table.has_pk = True + this_postgres_table.has_pk = random.choice([True, False]) self.postgres_table = this_postgres_table elif len(existing_postgres_tables) == 1: diff --git a/misc/python/materialize/zippy/scenarios.py b/misc/python/materialize/zippy/scenarios.py index 2169b06d0d82e..2351b43120035 100644 --- a/misc/python/materialize/zippy/scenarios.py +++ b/misc/python/materialize/zippy/scenarios.py @@ -75,21 +75,36 @@ StoragedStart, ) from materialize.zippy.copy_s3_actions import CopyFromS3, CopyToS3 -from materialize.zippy.table_actions import DML, CreateTableParameterized, ValidateTable -from materialize.zippy.view_actions import CreateViewParameterized, ValidateView +from materialize.zippy.table_actions import ( + DML, + CreateTableParameterized, + DropTable, + ValidateTable, +) +from materialize.zippy.view_actions import ( + CreateViewParameterized, + DropView, + ValidateView, +) class Scenario: def bootstrap(self) -> list[ActionOrFactory]: - return [ + bootstrap: list[ActionOrFactory] = [ KafkaStart, CockroachStart, BlobStoreStart, MzStart, StoragedStart, BalancerdStart, - IcebergStart, ] + # Polaris is only needed if the scenario can create Iceberg sinks. + if any( + isinstance(action, CreateIcebergSinkParameterized) + for action in self.actions_with_weight() + ): + bootstrap.append(IcebergStart) + return bootstrap def actions_with_weight(self) -> dict[ActionOrFactory, float]: raise RuntimeError @@ -120,6 +135,7 @@ def actions_with_weight(self) -> dict[ActionOrFactory, float]: CreateTopicParameterized(): 5, CreateSourceParameterized(): 5, CreateViewParameterized(max_inputs=2): 5, + DropView: 5, CreateSinkParameterized(): 5, CreateIcebergSinkParameterized(): 5, ValidateView: 10, @@ -163,12 +179,16 @@ def actions_with_weight(self) -> dict[ActionOrFactory, float]: Mz0dtDeploy: 10, BalancerdStop: 1, BalancerdRestart: 1, - BackupAndRestore: 1, + # No mid-test BackupAndRestore since the state of the sinks is not + # recorded across a backup&restore cycle, see + # https://github.com/MaterializeInc/database-issues/issues/9589 KillClusterd: 10, # Disabled because a separate clusterd is not supported by Mz0dtDeploy yet # StoragedRestart: 5, CreateTableParameterized(): 10, CreateViewParameterized(): 10, + DropTable: 5, + DropView: 5, CreateSinkParameterized(): 10, CreateIcebergSinkParameterized(): 5, ValidateTable: 20, diff --git a/misc/python/materialize/zippy/sql_server_actions.py b/misc/python/materialize/zippy/sql_server_actions.py index 046a607e38b61..1b0952c854780 100644 --- a/misc/python/materialize/zippy/sql_server_actions.py +++ b/misc/python/materialize/zippy/sql_server_actions.py @@ -45,20 +45,6 @@ def run(self, c: Composition, state: State) -> None: ) -class SqlServerStop(Action): - """Stop the SQL Server instance.""" - - @classmethod - def requires(cls) -> set[type[Capability]]: - return {SqlServerRunning} - - def withholds(self) -> set[type[Capability]]: - return {SqlServerRunning} - - def run(self, c: Composition, state: State) -> None: - c.kill("sql-server") - - class SqlServerRestart(Action): """Restart the SqlServer instance.""" @@ -87,8 +73,7 @@ def __init__(self, capabilities: Capabilities) -> None: if len(existing_sql_server_tables) == 0: self.new_sql_server_table = True - # A PK is now required for Debezium - this_sql_server_table.has_pk = True + this_sql_server_table.has_pk = random.choice([True, False]) self.sql_server_table = this_sql_server_table elif len(existing_sql_server_tables) == 1: diff --git a/misc/python/materialize/zippy/table_actions.py b/misc/python/materialize/zippy/table_actions.py index 5316a3a1901e5..e88efe9d95f28 100644 --- a/misc/python/materialize/zippy/table_actions.py +++ b/misc/python/materialize/zippy/table_actions.py @@ -21,6 +21,7 @@ ) from materialize.zippy.mz_capabilities import MzIsRunning from materialize.zippy.table_capabilities import TableExists +from materialize.zippy.view_capabilities import ViewExists MAX_ROWS_PER_ACTION = 10000 @@ -131,6 +132,39 @@ def run(self, c: Composition, state: State) -> None: ) +class DropTable(Action): + """Drops a table that no view reads from.""" + + @classmethod + def requires(cls) -> set[type[Capability]]: + return {BalancerdIsRunning, MzIsRunning, TableExists} + + def __init__(self, capabilities: Capabilities) -> None: + referenced: set[str] = set() + for view in capabilities.get(ViewExists): + referenced.update(input.name for input in view.inputs) + + candidates = [ + t for t in capabilities.get(TableExists) if t.name not in referenced + ] + self.table: TableExists | None = ( + random.choice(candidates) if candidates else None + ) + if self.table is not None: + capabilities.remove_capability_instance(self.table) + super().__init__(capabilities) + + def __str__(self) -> str: + return f"{Action.__str__(self)} {self.table.name if self.table else ''}" + + def run(self, c: Composition, state: State) -> None: + if self.table is not None: + c.testdrive( + f"> DROP TABLE {self.table.name};", + mz_service=state.mz_service, + ) + + class DML(Action): """Performs an INSERT, DELETE or UPDATE against a table.""" diff --git a/misc/python/materialize/zippy/view_actions.py b/misc/python/materialize/zippy/view_actions.py index 475546c482457..71608044d50b2 100644 --- a/misc/python/materialize/zippy/view_actions.py +++ b/misc/python/materialize/zippy/view_actions.py @@ -20,9 +20,11 @@ Capability, State, ) +from materialize.zippy.iceberg_capabilities import IcebergSinkExists from materialize.zippy.mysql_cdc_capabilities import MySqlCdcTableExists from materialize.zippy.mz_capabilities import MzIsRunning from materialize.zippy.pg_cdc_capabilities import PostgresCdcTableExists +from materialize.zippy.sink_capabilities import SinkExists from materialize.zippy.source_capabilities import SourceExists from materialize.zippy.storaged_capabilities import StoragedRunning from materialize.zippy.table_capabilities import TableExists @@ -137,6 +139,44 @@ def provides(self) -> list[Capability]: return [self.view] +class DropView(Action): + """Drops a view that no sink and no other view depends on.""" + + @classmethod + def requires(cls) -> set[type[Capability]]: + return {BalancerdIsRunning, MzIsRunning, ViewExists} + + def __init__(self, capabilities: Capabilities) -> None: + referenced: set[str] = set() + for sink in capabilities.get(SinkExists): + referenced.add(sink.source_view.name) + referenced.add(sink.dest_view.name) + for iceberg_sink in capabilities.get(IcebergSinkExists): + referenced.add(iceberg_sink.source_view.name) + for view in capabilities.get(ViewExists): + referenced.update( + input.name for input in view.inputs if isinstance(input, ViewExists) + ) + + candidates = [ + v for v in capabilities.get(ViewExists) if v.name not in referenced + ] + self.view: ViewExists | None = random.choice(candidates) if candidates else None + if self.view is not None: + capabilities.remove_capability_instance(self.view) + super().__init__(capabilities) + + def __str__(self) -> str: + return f"{Action.__str__(self)} {self.view.name if self.view else ''}" + + def run(self, c: Composition, state: State) -> None: + if self.view is not None: + c.testdrive( + f"> DROP MATERIALIZED VIEW {self.view.name};", + mz_service=state.mz_service, + ) + + class ValidateView(Action): """Validates a view.""" diff --git a/test/zippy/mzcompose.py b/test/zippy/mzcompose.py index 5dd73d0e7889a..7d07106d10350 100644 --- a/test/zippy/mzcompose.py +++ b/test/zippy/mzcompose.py @@ -13,7 +13,6 @@ expected state it can verify results for correctness. """ -import os import random import re import time @@ -47,7 +46,7 @@ setup_default_ssh_test_connection, ) from materialize.mzcompose.services.testdrive import Testdrive -from materialize.zippy.framework import Test +from materialize.zippy.framework import Test, ci_additional_system_parameter_defaults from materialize.zippy.mz_actions import Mz0dtDeploy from materialize.zippy.scenarios import * # noqa: F401 F403 @@ -120,12 +119,19 @@ def __str__(self) -> str: def parse_timedelta(arg: str) -> timedelta: p = re.compile( - (r"((?P-?\d+)d)?" r"((?P-?\d+)h)?" r"((?P-?\d+)m)?"), + ( + r"((?P-?\d+)d)?" + r"((?P-?\d+)h)?" + r"((?P-?\d+)m)?" + r"((?P-?\d+)s)?" + ), re.IGNORECASE, ) - m = p.match(arg) - assert m is not None + m = p.fullmatch(arg) + assert ( + m is not None + ), f"invalid timedelta '{arg}', expected a format like '1d2h3m4s'" parts = {k: int(v) for k, v in m.groupdict().items() if v} td = timedelta(**parts) @@ -212,15 +218,7 @@ def workflow_default(c: Composition, parser: WorkflowArgumentParser) -> None: print(f"Using seed {args.seed}") random.seed(args.seed) - additional_system_parameter_defaults = {} - system_parameter_default = os.getenv("CI_MZ_SYSTEM_PARAMETER_DEFAULT", "") - if system_parameter_default: - for val in system_parameter_default.split(";"): - x = val.split("=", maxsplit=1) - assert ( - len(x) == 2 - ), f"CI_MZ_SYSTEM_PARAMETER_DEFAULT '{val}' should be the format =" - additional_system_parameter_defaults[x[0]] = x[1] + additional_system_parameter_defaults = ci_additional_system_parameter_defaults() with c.override( Cockroach( From 3b68da713d04625e8bf6683867be305d67e7d183 Mon Sep 17 00:00:00 2001 From: Roberto Jimeno Date: Wed, 22 Jul 2026 09:49:11 -0400 Subject: [PATCH 07/15] console: [SAS-128] Add per-account/per-cluster cost breakdown to the billing page (#37286) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > [!IMPORTANT] > **Mature enough to merge into `main`, and intentionally open to change in Phase 5.** > > This is Phase 1 of the Orb account-hierarchy console work. It is purely **additive**: it introduces the new "Spend by account & cluster" section and leaves the existing spend total, daily chart, and "Spend between…" breakdown **untouched**, so it cannot regress today's page and is trivially revertable (the console redeploys in ~5 min and this section is not load-bearing). > > Please review for **correctness and safety-to-merge, not feature-completeness.** A few details are deliberately deferred to **Phase 5** and tracked as follow-ups below — friendly account names (SAS-141/142) and the full per-account Compute/Storage drilldown (blocked on the breakdown endpoint carrying usage quantities and replica sizes). Landing this now unblocks the staged rollout (Phases 3–4). ### Motivation Phase 1 of Orb account-hierarchy support in the console (Linear: SAS-128). Today the billing page shows only an org-wide spend total. With sub-accounts, a parent org needs to see how spend breaks down across its own and its child accounts, per cluster, bucketed by day — driven by `GET /api/costs/breakdown/daily`. ### Description - Adds a **"Spend by account & cluster"** section to Usage → Overview, sourced entirely from `GET /api/costs/breakdown/daily` (`useDailyCostsBreakdown` in `queries.ts`). The old aggregate `GET /api/costs/breakdown` endpoint and its frontend usage (`getCostsBreakdown`/`useCostsBreakdown`/the old `AccountClusterBreakdown` component) have been removed — this is bucketed-by-UTC-day from the start, not a follow-on aggregate. - **`AccountSpendBreakdown`**: a stacked-by-account daily bar chart (visx `BarStack`, one accent color per account, per-day tooltip) above a total-spend figure. A parent org sees every child account stacked in the same chart; a child or standalone org sees a single-account stack. - **`AccountSpendLedger`** (own full-width grid row, SAS-154): every account as an expandable row — share of total, a trend sparkline, and total cost — expanding to region-qualified per-cluster costs (`aws/us-east-1 / quickstart.r1`, `aws/us-east-1 / Storage`). Biggest spender first. A final Total row sums every account. All accounts are visible at once; there's no chip-switcher or select-to-drill-down. - **`PlanDetails`**: a separate box alongside the chart shows the account's plan info, credit balance, and a "Last 30 days" per-account breakdown, independent of the chart's selected time range. - Each account currently renders its Orb **external-customer UUID**; friendly names are a deliberate follow-up (backend SAS-141, frontend SAS-142). - **Shared `LedgerTable`/`LedgerGroup`/`LedgerCell`/`LedgerCaret`** component (`LedgerTable.tsx`) replaces duplicated cell/caret/Safari-subgrid-collapse styling previously spread across `SpendBreakdown.tsx` and this PR's `AccountSpendBreakdown.tsx` — a reviewer suggestion, landed as a net -185 lines across the two table files despite the new shared component. Also picks up keyboard (`Enter`/`Space`) + `aria-expanded` support on both tables' expand/collapse rows, which neither had before. - **Type generation:** the breakdown path/schemas are spliced into the committed `global-api` types rather than a full `yarn gen:api` re-sync, because the console's committed spec has drifted from cloud (regions / self-managed-subscription, tracked separately as SAS-161). On a clean rebase once the cloud spec lands, `yarn gen:api` reproduces these. ### Verification - **Component tests:** billing test suite, 43 tests across 7 files (`AccountSpendBreakdown`/`UsagePage`/`PlanDetails`/`dailyBreakdown`/`SpendBreakdown`/`DailyUsageChart`/`UsageRoutes`/`BillingPage`) — covers parent/child/standalone rendering, region-qualified cluster rows, storage/egress rows, error and empty states, and the account share-of-total column. Typecheck + lint clean. - **Manual, end-to-end:** ran this console locally against a personal stack (`rjimeno.dev`) wired to the cloud SAS-136 backend, logged in to a root/leaf Orb hierarchy (`+mock-root@` / `+mock-leaf@`). Confirmed parent + child account groups and region-qualified cluster/storage rows from live Orb data. Linear: SAS-128 --------- Co-authored-by: Claude Opus 4.8 (1M context) --- console/src/api/cloudGlobalApi.ts | 38 +- .../src/api/mocks/cloudGlobalApiHandlers.ts | 22 +- .../billing/AccountSpendBreakdown.tsx | 708 ++++++++++++++++++ console/src/platform/billing/BillingPage.tsx | 29 - .../platform/billing/DailyUsageChart.test.tsx | 287 ------- .../src/platform/billing/DailyUsageChart.tsx | 425 ----------- console/src/platform/billing/LedgerTable.tsx | 250 +++++++ .../src/platform/billing/PlanDetails.test.tsx | 11 +- console/src/platform/billing/PlanDetails.tsx | 154 +++- console/src/platform/billing/PricingPage.tsx | 44 ++ .../platform/billing/SpendBreakdown.test.tsx | 204 ----- .../src/platform/billing/SpendBreakdown.tsx | 547 -------------- .../src/platform/billing/TimeRangeSelect.tsx | 2 +- .../src/platform/billing/UsagePage.test.tsx | 586 ++++++++++++--- console/src/platform/billing/UsagePage.tsx | 227 ++---- .../src/platform/billing/UsageRoutes.test.tsx | 171 +++++ console/src/platform/billing/UsageRoutes.tsx | 55 +- console/src/platform/billing/constants.ts | 27 +- .../platform/billing/dailyBreakdown.test.ts | 309 ++++++++ .../src/platform/billing/dailyBreakdown.ts | 269 +++++++ console/src/platform/billing/queries.ts | 55 +- console/src/platform/billing/types.ts | 23 - console/src/platform/billing/utils.ts | 228 +----- console/src/util.test.ts | 32 + console/src/util.ts | 16 + console/src/utils/format.ts | 4 +- console/src/utils/graph.test.ts | 39 + console/src/utils/graph.ts | 8 +- 28 files changed, 2651 insertions(+), 2119 deletions(-) create mode 100644 console/src/platform/billing/AccountSpendBreakdown.tsx delete mode 100644 console/src/platform/billing/DailyUsageChart.test.tsx delete mode 100644 console/src/platform/billing/DailyUsageChart.tsx create mode 100644 console/src/platform/billing/LedgerTable.tsx create mode 100644 console/src/platform/billing/PricingPage.tsx delete mode 100644 console/src/platform/billing/SpendBreakdown.test.tsx delete mode 100644 console/src/platform/billing/SpendBreakdown.tsx create mode 100644 console/src/platform/billing/UsageRoutes.test.tsx create mode 100644 console/src/platform/billing/dailyBreakdown.test.ts create mode 100644 console/src/platform/billing/dailyBreakdown.ts create mode 100644 console/src/util.test.ts create mode 100644 console/src/utils/graph.test.ts diff --git a/console/src/api/cloudGlobalApi.ts b/console/src/api/cloudGlobalApi.ts index 50353d637f0a7..0c2b4b51ef3b6 100644 --- a/console/src/api/cloudGlobalApi.ts +++ b/console/src/api/cloudGlobalApi.ts @@ -7,10 +7,11 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { formatRFC3339 } from "date-fns"; import createClient from "openapi-fetch"; import { NOT_SUPPORTED_MESSAGE } from "~/config/AppConfig"; +import { addUtcDays } from "~/util"; +import { formatDateInUtc } from "~/utils/dateFormat"; import { apiClient } from "./apiClient"; import { @@ -27,6 +28,13 @@ export type Region = components["schemas"]["Region"]; export type Regions = components["schemas"]["Paginated_Region"]; export type CreditBlock = components["schemas"]["CreditBlock"]; export type DailyCosts = components["schemas"]["DailyCostResponse"]; +export type CostBreakdownAccount = + components["schemas"]["CostBreakdownAccount"]; +export type CostBreakdownCluster = + components["schemas"]["CostBreakdownCluster"]; +export type CostBreakdownDay = components["schemas"]["CostBreakdownDay"]; +export type DailyCostBreakdown = + components["schemas"]["DailyCostBreakdownResponse"]; export type AllCosts = components["schemas"]["AllCosts"]; export type DailyCostKey = keyof components["schemas"]["AllCosts"]; export type Prices = @@ -147,23 +155,31 @@ export async function getCredits(requestOptions: OpenApiRequestOptions = {}) { return handleOpenApiResponseWithBody(data, response); } -export async function getDailyCosts( +export async function getCostsBreakdownDaily( startDate: Date, endDate: Date, requestOptions: OpenApiRequestOptions = {}, ) { const { headers, ...options } = requestOptions; - const { data, response } = await getClient().GET("/api/costs/daily", { - params: { - query: { - startDate: formatRFC3339(startDate), - endDate: formatRFC3339(endDate), + const { data, response } = await getClient().GET( + "/api/costs/breakdown/daily", + { + params: { + query: { + // Both query params are inclusive calendar days (bare `YYYY-MM-DD`, + // no time component — see `DateRange` in cloud's interface.rs), + // unlike every other date param on this client. `startDate`/`endDate` + // here are the internal [start, end) convention used throughout the + // console, so the exclusive `endDate` becomes the day before it. + startDate: formatDateInUtc(startDate, "yyyy-MM-dd"), + endDate: formatDateInUtc(addUtcDays(endDate, -1), "yyyy-MM-dd"), + }, }, + signal: requestOptions?.signal, + headers, + ...options, }, - signal: requestOptions?.signal, - headers, - ...options, - }); + ); return handleOpenApiResponseWithBody(data, response); } diff --git a/console/src/api/mocks/cloudGlobalApiHandlers.ts b/console/src/api/mocks/cloudGlobalApiHandlers.ts index 19a26df3e5635..1daeaaa071e9c 100644 --- a/console/src/api/mocks/cloudGlobalApiHandlers.ts +++ b/console/src/api/mocks/cloudGlobalApiHandlers.ts @@ -7,18 +7,18 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { addDays, differenceInDays, formatISO, parseISO } from "date-fns"; +import { addDays, differenceInDays, formatISO } from "date-fns"; import { http, HttpResponse } from "msw"; import { CreditBlock, + DailyCostBreakdown, DailyCosts, Invoice, Organization, Region, Regions, } from "~/api/cloudGlobalApi"; -import { assert } from "~/util"; export const buildCloudRegionsReponse = ( options: { @@ -109,21 +109,11 @@ export function generateDailyCostResponsePayload( return response; } -export const buildDailyCostResponse = ( - options: { payload?: DailyCosts; status?: number } = {}, +export const buildDailyCostBreakdownResponse = ( + options: { payload?: DailyCostBreakdown; status?: number } = {}, ) => - http.get("*/api/costs/daily", (info) => { - let payload = options.payload; - if (!payload) { - const url = new URL(info.request.url); - const startParam = url.searchParams.get("startDate"); - assert(startParam !== null); - const startDate = parseISO(startParam); - const endParam = url.searchParams.get("endDate"); - assert(endParam !== null); - const endDate = parseISO(endParam); - payload = generateDailyCostResponsePayload(startDate, endDate); - } + http.get("*/api/costs/breakdown/daily", () => { + const payload: DailyCostBreakdown = options.payload ?? { days: [] }; return HttpResponse.json(payload, { status: options.status ?? 200 }); }); diff --git a/console/src/platform/billing/AccountSpendBreakdown.tsx b/console/src/platform/billing/AccountSpendBreakdown.tsx new file mode 100644 index 0000000000000..648c558ae1ee8 --- /dev/null +++ b/console/src/platform/billing/AccountSpendBreakdown.tsx @@ -0,0 +1,708 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { Box, chakra, HStack, Text, Tooltip, useTheme } from "@chakra-ui/react"; +import { AxisBottom, AxisLeft, AxisScale } from "@visx/axis"; +import { curveMonotoneX } from "@visx/curve"; +import { localPoint } from "@visx/event"; +import { Group } from "@visx/group"; +import ParentSize, { + ParentSizeProvidedProps, +} from "@visx/responsive/lib/components/ParentSize"; +import { scaleBand, scaleLinear, scaleOrdinal } from "@visx/scale"; +import { BarStack, LinePath } from "@visx/shape"; +import { useTooltip, useTooltipInPortal } from "@visx/tooltip"; +import { parseISO } from "date-fns"; +import { motion } from "framer-motion"; +import React, { useMemo } from "react"; + +import { + CostBreakdownAccount, + CostBreakdownCluster, + CostBreakdownDay, +} from "~/api/cloudGlobalApi"; +import ErrorBox from "~/components/ErrorBox"; +import { GraphEventOverlay, GraphTooltip } from "~/components/graphComponents"; +import { LoadingContainer } from "~/components/LoadingContainer"; +import { + EmptyListHeader, + EmptyListHeaderContents, + EmptyListWrapper, +} from "~/layouts/listPageComponents"; +import { MaterializeTheme } from "~/theme"; +import { DATE_FORMAT_SHORT, formatDateInUtc } from "~/utils/dateFormat"; +import { formatCurrency, formatPercentage } from "~/utils/format"; +import { + buildXYGraphLayoutProps, + calculateTooltipPosition, +} from "~/utils/graph"; + +import { ACCOUNT_SPEND_FETCH_ERROR_MESSAGE } from "./constants"; +import { + accountTotal, + clusterTotal, + pivotBreakdown, + shortAccountId, + StackedDailyRow, +} from "./dailyBreakdown"; +import { + LedgerCaret, + LedgerCell, + LedgerGroup, + LedgerTable, +} from "./LedgerTable"; +import RegionSelect from "./RegionSelect"; +import TimeRangeSelect from "./TimeRangeSelect"; + +const margin = { top: 10, right: 0, bottom: 36, left: 50 }; +const chartHeightPx = 245; +const SPARKLINE_STROKE_WIDTH = 2; + +type AccountSpendBreakdownProps = { + days: CostBreakdownDay[] | null; + isLoading: boolean; + isError: boolean; + error: Error | null; + regionFilter: "all" | string; + setRegionFilter: (val: "all" | string) => void; + timeRange: number; + setTimeRange: (val: number) => void; +}; + +/** + * A cluster's region-qualified label, mirroring the "Spend between …" table: + * "aws/us-east-1 / quickstart.r1", "… / Storage", "… / Egress". Compute rows + * carry the replica grouping key; storage/egress rows have an empty key and a + * `category` instead, so the two stay distinct. "Other" is a defensive + * fallback for the unexpected case where neither is set. + */ +function clusterLabel(cluster: CostBreakdownCluster): string { + const label = cluster.cluster_grouping_key || cluster.category || "Other"; + return `${cluster.region} / ${label}`; +} + +// TODO(SAS-145): `/api/costs/breakdown/daily` returns dollar amounts only, no +// usage quantities (Orb's per-price `quantity` is discarded server-side). Until +// SAS-145 threads it through, the Usage column renders this placeholder rather +// than a number. +const USAGE_PLACEHOLDER = "—"; + +// Cycled through, in order, to color account layers/legend/table swatches. Kept +// visually distinct; wraps if an org has more accounts than colors. +function useAccountColors(accountIds: string[]): Map { + const { colors } = useTheme(); + return useMemo(() => { + const palette = [ + colors.accent.purple, + colors.accent.green, + colors.accent.orange, + colors.accent.blue, + colors.accent.brightPurple, + colors.accent.red, + colors.accent.darkGreen, + colors.accent.darkYellow, + ]; + return new Map( + accountIds.map((id, ix) => [id, palette[ix % palette.length]]), + ); + }, [accountIds, colors]); +} + +function shortDate(dateString: string): string { + return formatDateInUtc(parseISO(dateString), "M/d"); +} + +function longDate(dateString: string): string { + return formatDateInUtc(parseISO(dateString), "M/d/yyyy"); +} + +type ChartTooltipData = { row: StackedDailyRow }; + +type AccountSpendChartGraphProps = ParentSizeProvidedProps & { + rows: StackedDailyRow[]; + accountIds: string[]; + colorFor: Map; +}; + +const AccountSpendChartGraph = ({ + rows, + accountIds, + colorFor, + width, + height, +}: AccountSpendChartGraphProps) => { + const { colors, textStyles } = useTheme(); + const { + svgProps, + axisLeftProps, + axisBottomProps, + graphEventOverlayProps, + xScaleRange, + yScaleRange, + } = useMemo( + () => buildXYGraphLayoutProps({ width, height, margin }), + [width, height], + ); + const { + tooltipOpen, + tooltipLeft, + tooltipTop, + tooltipData, + hideTooltip, + showTooltip, + } = useTooltip(); + const { containerRef, TooltipInPortal } = useTooltipInPortal({ + detectBounds: true, + scroll: true, + }); + + const maxDailyTotal = useMemo( + () => + rows.reduce((currentMax, row) => { + const total = accountIds.reduce( + (sum, id) => sum + (row[id] as number), + 0, + ); + return Math.max(currentMax, total); + }, 0), + [rows, accountIds], + ); + const barPadding = rows.length > 90 ? 0 : rows.length > 7 ? 0.25 : 0.15; + const xScale = useMemo( + () => + scaleBand({ + domain: rows.map((row) => shortDate(row.startDate)), + range: xScaleRange as [number, number], + padding: barPadding, + }), + [rows, xScaleRange, barPadding], + ); + const yScale = useMemo( + () => + scaleLinear({ + domain: [0, Math.ceil(maxDailyTotal)], + range: yScaleRange, + nice: true, + }), + [maxDailyTotal, yScaleRange], + ); + const colorScale = useMemo( + () => + scaleOrdinal({ + domain: accountIds, + range: accountIds.map((id) => colorFor.get(id) ?? colors.accent.purple), + }), + [accountIds, colorFor, colors], + ); + + // scaleBand has no invert() (which findNearestDatum requires), so map the + // pointer's x to a column by stepping from the band range's start. + const handlePointerMove = (event: React.PointerEvent) => { + const svgPoint = localPoint(event); + if (!svgPoint || rows.length === 0) return; + const [rangeStart] = xScale.range(); + const index = Math.min( + rows.length - 1, + Math.max(0, Math.floor((svgPoint.x - rangeStart) / xScale.step())), + ); + showTooltip({ + ...calculateTooltipPosition({ event }), + tooltipData: { row: rows[index] }, + }); + }; + + return ( + + + + + data={rows} + keys={accountIds} + x={(row) => shortDate(row.startDate)} + xScale={xScale} + yScale={yScale} + color={colorScale} + value={(row, key) => row[key] as number} + > + {(barStacks) => + barStacks.map((barStack) => + barStack.bars.map((bar) => { + const isHovered = tooltipData?.row === rows[bar.index]; + const isDemoted = !isHovered && tooltipOpen; + return ( + + ); + }), + ) + } + + + + > + {...axisLeftProps} + hideTicks + scale={yScale} + stroke={colors.border.secondary} + tickFormat={(value, ix, values) => { + if (ix > 0 && ix < values.length - 1) { + return ""; + } + return formatCurrency(value as number, { + trailingZeroDisplay: "stripIfInteger", + useGrouping: false, + notation: "compact", + }); + }} + tickLabelProps={() => ({ + fill: colors.foreground.primary, + fontSize: textStyles["text-small"].fontSize, + textAnchor: "end", + })} + /> + > + {...axisBottomProps} + hideTicks + scale={xScale} + stroke={colors.border.secondary} + labelOffset={8} + tickLabelProps={() => ({ + fill: colors.foreground.primary, + fontSize: textStyles["text-small"].fontSize, + dy: "4px", + textAnchor: "middle", + })} + /> + + hideTooltip()} + /> + + {tooltipOpen && + tooltipData && + tooltipTop != null && + tooltipLeft != null && ( + + + + {longDate(tooltipData.row.startDate)} + + {accountIds + .filter((id) => (tooltipData.row[id] as number) > 0) + .map((id) => ( + + + + {shortAccountId(id)} + + + {formatCurrency(tooltipData.row[id] as number)} + + + ))} + + + )} + + ); +}; + +const AccountSpendChart = (props: { + rows: StackedDailyRow[]; + accountIds: string[]; + colorFor: Map; +}) => ( + + + {(parent) => ( + + )} + + +); + +const TrendSparkline = ({ + points, + color, +}: { + points: number[]; + color: string; +}) => { + const dataPoints = points.map((value, offset) => ({ offset, value })); + const max = Math.max(0, ...points); + return ( + + + {(parent) => { + const xScale = scaleLinear({ + domain: [0, Math.max(1, dataPoints.length - 1)], + range: [0, parent.width], + }); + const yScale = scaleLinear({ + domain: [0, max], + range: [ + parent.height - SPARKLINE_STROKE_WIDTH / 2, + SPARKLINE_STROKE_WIDTH / 2, + ], + }); + return ( + + xScale(d.offset)} + y={(d) => yScale(d.value)} + /> + + ); + }} + + + ); +}; + +/** + * One account rendered as an expandable ledger group: the account is the + * always-visible parent row (caret + color swatch + label, its share of the + * period total, a daily-total trend, and its total cost) and its clusters are + * indented, region-qualified child rows revealed by the disclosure. Groups + * default open so the whole ledger is visible without interaction. + */ +const AccountLedgerGroup = ({ + account, + total, + share, + trend, + color, +}: { + account: CostBreakdownAccount; + total: number; + share: number; + trend: number[]; + color: string; +}) => { + return ( + ( + <> + + + + + + {shortAccountId(account.external_customer_id)} + + + + + + {formatPercentage(share, 1)} + + + + + + {formatCurrency(total)} + + + )} + > + {account.clusters.map((cluster, ix) => { + const isLastElement = ix === account.clusters.length - 1; + return ( + + + {clusterLabel(cluster)} + + + {USAGE_PLACEHOLDER} + + + + + {formatCurrency(clusterTotal(cluster.amounts))} + + + ); + })} + + ); +}; + +/** + * Direction-A unified ledger (SAS-128): one table listing every account + * (biggest spender first) as an expandable row — its share of the period total, + * a daily-total trend, and its cost — expanding to its region-qualified + * per-cluster costs. A final Total row sums every account. + */ +const UnifiedLedger = ({ + accounts, + series, + colorFor, +}: { + accounts: CostBreakdownAccount[]; + series: Map; + colorFor: Map; +}) => { + const { colors } = useTheme(); + const totals = accounts.map((account) => accountTotal(account)); + const grandTotal = totals.reduce((sum, total) => sum + total, 0); + + return ( + + {accounts.map((account, ix) => { + const total = totals[ix]; + const share = grandTotal > 0 ? total / grandTotal : 0; + return ( + + ); + })} + + Total + + + + + {formatCurrency(grandTotal)} + + + + ); +}; + +/** + * Per-account spend (Direction A / SAS-128). A stacked-by-account daily chart + * over a single unified ledger: every account is an expandable row (share, + * trend, cost) that opens to its region-qualified per-cluster costs. Sourced + * from `/api/costs/breakdown/daily` via `useDailyCostsBreakdown`. + * + * The per-cluster Usage column is a placeholder (SAS-145) until the endpoint + * carries usage quantities; the mock's Compute/Storage resource drilldown + * (replica size) needs per-cluster replica sizes the endpoint does not yet + * carry either, so that remains a separate follow-up. + */ +// Shared pivot of the dense day series into everything the chart and the +// ledger render: account order (biggest spender first), colors, stacked chart +// rows, per-account sparkline series, aggregated accounts, period total, and +// the date range. The chart and the ledger live in separate grid cells +// (SAS-154), so both derive from this one hook and can never disagree on +// ordering or colors. +function useAccountSpendPivot( + days: CostBreakdownDay[] | null, + regionFilter: "all" | string, +) { + const { accountIds, rows, orderedAccounts, series, totalSpend } = useMemo( + () => pivotBreakdown(days, regionFilter), + [days, regionFilter], + ); + const colorFor = useAccountColors(accountIds); + + const rangeStart = days?.length + ? formatDateInUtc(parseISO(days[0].startDate), DATE_FORMAT_SHORT) + : null; + const rangeEnd = days?.length + ? formatDateInUtc( + parseISO(days[days.length - 1].startDate), + DATE_FORMAT_SHORT, + ) + : null; + + return { + accountIds, + rows, + colorFor, + series, + orderedAccounts, + totalSpend, + rangeStart, + rangeEnd, + }; +} + +const EmptyState = () => ( + + + + + +); + +const AccountSpendBreakdown = ({ + days, + isLoading, + isError, + regionFilter, + setRegionFilter, + timeRange, + setTimeRange, +}: AccountSpendBreakdownProps) => { + const { accountIds, rows, colorFor, totalSpend } = useAccountSpendPivot( + days, + regionFilter, + ); + + return ( + + +
+ +
+
+ +
+ + {isLoading ? ( + // Reserve the chart's height so the section doesn't jump when the + // data lands. + + ) : isError ? ( + + ) : !days || accountIds.length === 0 ? ( + + ) : ( + + + {formatCurrency(totalSpend)} + + + + )} + + ); +}; + +export type AccountSpendLedgerProps = { + days: CostBreakdownDay[] | null; + isLoading: boolean; + isError: boolean; + regionFilter: "all" | string; +}; + +/** + * The "Spend between …" heading and the unified per-account ledger, split from + * AccountSpendBreakdown so the page can give the table its own full-width grid + * row beneath the chart and Plan details columns (SAS-154). Loading, error, + * and empty states are surfaced by AccountSpendBreakdown, so this renders + * nothing in those states. + */ +export const AccountSpendLedger = ({ + days, + isLoading, + isError, + regionFilter, +}: AccountSpendLedgerProps) => { + const { colors } = useTheme(); + const { + accountIds, + colorFor, + series, + orderedAccounts, + rangeStart, + rangeEnd, + } = useAccountSpendPivot(days, regionFilter); + + if (isLoading || isError || !days || accountIds.length === 0) { + return null; + } + + return ( + + + Spend between{" "} + {rangeStart && rangeEnd && ( + <> + + {rangeStart} + {" "} + and{" "} + + {rangeEnd} + + + )} + + + + ); +}; + +export default AccountSpendBreakdown; diff --git a/console/src/platform/billing/BillingPage.tsx b/console/src/platform/billing/BillingPage.tsx index b2badfc5309c7..41a65dfe46ef5 100644 --- a/console/src/platform/billing/BillingPage.tsx +++ b/console/src/platform/billing/BillingPage.tsx @@ -526,31 +526,6 @@ const BillingDetails = ({ ); }; -const OnDemandPricing = () => { - const { colors } = useTheme(); - return ( - <> - - On-Demand Pricing - - - The following represents our on-demand pricing structure for compute, - storage, and network resources. To calculate your estimated compute - costs, multiply your projected compute credit usage by the listed price - per credit. - - - - - - ); -}; - const CTABanner = ({ organization, onUpgradeAndPay, @@ -687,7 +662,6 @@ const BillingPage = ({ )} - - - diff --git a/console/src/platform/billing/DailyUsageChart.test.tsx b/console/src/platform/billing/DailyUsageChart.test.tsx deleted file mode 100644 index 3e950d8bdae4d..0000000000000 --- a/console/src/platform/billing/DailyUsageChart.test.tsx +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { render, screen } from "@testing-library/react"; -import { TooltipInPortalProps } from "@visx/tooltip/lib/hooks/useTooltipInPortal"; -import React, { ReactElement } from "react"; - -import { DailyCosts } from "~/api/cloudGlobalApi"; -import { createProviderWrapper } from "~/test/utils"; - -import { - DailyUsageChartTooltip, - LegendEntry, - TooltipData, -} from "./DailyUsageChart"; -import { aggregateByDay } from "./utils"; - -const Wrapper = await createProviderWrapper(); - -const renderComponent = (element: ReactElement) => { - return render({element}); -}; - -const legendData: LegendEntry[] = [ - { key: "compute", label: "Compute", color: "purple" }, - { key: "storage", label: "Storage", color: "gray" }, -]; - -const mockData: TooltipData = { - bar: { - day: { - startDate: "2024-01-13T00:00:00Z", - endDate: "2024-01-14T00:00:00Z", - subtotal: "1.23", - total: "1.23", - costs: { - compute: { - subtotal: "1.00", - total: "1.00", - prices: [ - { - subtotal: "0.50", - regionId: "aws/us-east-1", - unitAmount: "1.00", - replicaSize: "xsmall", - }, - { - subtotal: "0.50", - regionId: "aws/eu-west-1", - unitAmount: "1.00", - replicaSize: "small", - }, - ], - }, - storage: { - subtotal: "1.23", - total: "1.23", - prices: [ - { subtotal: "1.00", regionId: "aws/us-east-1", unitAmount: "1.00" }, - { subtotal: "0.23", regionId: "aws/eu-west-1", unitAmount: "1.00" }, - ], - }, - }, - }, - index: 0, - height: 0, - width: 0, - x: 0, - y: 0, - color: "gray", - }, - hoveredIndex: 0, -}; - -const MockTooltip = ({ - children, - ...props -}: React.PropsWithChildren) => { - return
{children}
; -}; - -describe("UsageChart", () => { - it("tooltip displays two content columns when not filtering by region", async () => { - renderComponent( - , - ); - const startDate = await screen.findByTestId("chart-tooltip-start-date"); - expect(startDate).toHaveTextContent("1/13/2024"); - const table = await screen.findByTestId("chart-tooltip-table"); - const headerCells = Array.from(table.querySelectorAll("th")); - expect(headerCells).toHaveLength(3); - expect(headerCells[1]).toHaveTextContent("aws/us-east-1"); - expect(headerCells[2]).toHaveTextContent("aws/eu-west-1"); - const tableCells = Array.from(table.querySelectorAll("td")); - expect(tableCells).toHaveLength(6); - expect( - parseFloat(tableCells[1].textContent?.replace("$", "") ?? "0") + - parseFloat(tableCells[2].textContent?.replace("$", "") ?? "0"), - ).toEqual(parseFloat(mockData.bar.day.costs.compute.subtotal)); - expect( - parseFloat(tableCells[4].textContent?.replace("$", "") ?? "0") + - parseFloat(tableCells[5].textContent?.replace("$", "") ?? "0"), - ).toEqual(parseFloat(mockData.bar.day.costs.storage.subtotal)); - }); - - it("tooltip displays a single content column when filtering by region", async () => { - renderComponent( - , - ); - const table = await screen.findByTestId("chart-tooltip-table"); - const headerCells = Array.from(table.querySelectorAll("th")); - expect(headerCells.length).toEqual(2); - expect(headerCells[1]).toBeEmptyDOMElement(); - }); - - it("daily costs are aggregated by day", () => { - const costData: DailyCosts["daily"] = [ - { - startDate: "2024-01-12T00:00:00Z", - endDate: "2024-01-13T12:00:00Z", - subtotal: "1.00", - total: "1.00", - costs: { - compute: { - subtotal: "0.50", - total: "0.50", - prices: [ - { - regionId: "aws/us-east-1", - replicaSize: "large", - unitAmount: "1.20", - subtotal: "0.50", - }, - ], - }, - storage: { - subtotal: "0.50", - total: "0.50", - prices: [ - { - regionId: "aws/us-east-1", - unitAmount: "0.000081", - subtotal: "0.50", - }, - ], - }, - }, - }, - { - startDate: "2024-01-13T00:00:00Z", - endDate: "2024-01-13T12:00:00Z", - subtotal: "1.00", - total: "1.00", - costs: { - compute: { - subtotal: "0.50", - total: "0.50", - prices: [ - { - regionId: "aws/us-east-1", - replicaSize: "large", - unitAmount: "1.20", - subtotal: "0.50", - }, - ], - }, - storage: { - subtotal: "0.50", - total: "0.50", - prices: [ - { - regionId: "aws/us-east-1", - unitAmount: "0.000081", - subtotal: "0.50", - }, - ], - }, - }, - }, - { - startDate: "2024-01-13T12:00:00Z", - endDate: "2024-01-14T00:00:00Z", - subtotal: "2.00", - total: "2.00", - costs: { - compute: { - subtotal: "1.50", - total: "1.50", - prices: [ - { - regionId: "aws/us-east-1", - replicaSize: "large", - unitAmount: "1.50", - subtotal: "1.50", - }, - ], - }, - storage: { - subtotal: "0.50", - total: "0.50", - prices: [ - { - regionId: "aws/us-east-1", - unitAmount: "0.000081", - subtotal: "0.50", - }, - ], - }, - }, - }, - ]; - const aggregated = aggregateByDay(costData); - expect(aggregated).toHaveLength(2); - - const day1 = aggregated[0]; - expect(day1.startDate).toEqual("2024-01-12T00:00:00Z"); - expect(day1.endDate).toEqual("2024-01-13T00:00:00Z"); - expect(day1.total).toEqual("1.00"); - expect(day1.subtotal).toEqual("1.00"); - expect(day1.costs.compute.subtotal).toEqual("0.50"); - expect(day1.costs.compute.total).toEqual("0.50"); - expect(day1.costs.compute.prices).toEqual([ - { - regionId: "aws/us-east-1", - replicaSize: "large", - unitAmount: "1.20", - subtotal: "0.50", - }, - ]); - expect(day1.costs.storage.subtotal).toEqual("0.50"); - expect(day1.costs.storage.total).toEqual("0.50"); - expect(day1.costs.storage.prices).toEqual([ - { - regionId: "aws/us-east-1", - unitAmount: "0.000081", - subtotal: "0.50", - }, - ]); - - const day2 = aggregated[1]; - expect(day2.startDate).toEqual("2024-01-13T00:00:00Z"); - expect(day2.endDate).toEqual("2024-01-14T00:00:00Z"); - expect(parseFloat(day2.total)).toEqual(3); - expect(parseFloat(day2.subtotal)).toEqual(3); - expect(parseFloat(day2.costs.compute.subtotal)).toEqual(2); - expect(parseFloat(day2.costs.compute.total)).toEqual(2); - expect(day2.costs.compute.prices).toEqual([ - { - regionId: "aws/us-east-1", - replicaSize: "large", - unitAmount: "-1", - subtotal: "2", - }, - ]); - expect(parseFloat(day2.costs.storage.subtotal)).toEqual(1); - expect(parseFloat(day2.costs.storage.total)).toEqual(1); - expect(day2.costs.storage.prices).toEqual([ - { - regionId: "aws/us-east-1", - unitAmount: "0.000081", - subtotal: "1", - }, - ]); - }); -}); diff --git a/console/src/platform/billing/DailyUsageChart.tsx b/console/src/platform/billing/DailyUsageChart.tsx deleted file mode 100644 index 4245d4896227b..0000000000000 --- a/console/src/platform/billing/DailyUsageChart.tsx +++ /dev/null @@ -1,425 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { - Box, - HStack, - StackProps, - Table, - TableContainer, - Tbody, - Td, - Text, - TextProps, - Th, - Thead, - Tr, - useTheme, -} from "@chakra-ui/react"; -import { AxisBottom, AxisLeft, AxisScale } from "@visx/axis"; -import { Group } from "@visx/group"; -import ParentSize, { - ParentSizeProvidedProps, -} from "@visx/responsive/lib/components/ParentSize"; -import { scaleBand, scaleLinear, scaleOrdinal } from "@visx/scale"; -import { BarStack } from "@visx/shape"; -import { useTooltip, useTooltipInPortal } from "@visx/tooltip"; -import { TooltipInPortalProps } from "@visx/tooltip/lib/hooks/useTooltipInPortal"; -import { parseISO } from "date-fns/parseISO"; -import { motion } from "framer-motion"; -import { useAtom } from "jotai"; -import React, { useMemo } from "react"; - -import { DailyCostKey, DailyCosts } from "~/api/cloudGlobalApi"; -import { GraphTooltip } from "~/components/graphComponents"; -import { cloudRegionsSelector, getRegionId } from "~/store/cloudRegions"; -import { MaterializeTheme } from "~/theme"; -import { formatDateInUtc } from "~/utils/dateFormat"; -import { formatCurrency } from "~/utils/format"; -import { - buildXYGraphLayoutProps, - calculateTooltipPosition, -} from "~/utils/graph"; - -import { aggregateByDay } from "./utils"; - -const margin = { top: 10, right: 0, bottom: 36, left: 50 }; -export const legendHeightPx = 18; -export const chartHeightPx = 245; -const TOOLTIP_DISMISS_TIMEOUT_MS = 300; - -export type DailyUsageChartProps = { - data: DailyCosts["daily"]; - region: "all" | string; -}; - -function extractFormattedDateShort(dateString: string): string { - const date = parseISO(dateString); - return formatDateInUtc(date, "M/d"); -} - -function extractFormattedDateLong(dateString: string): string { - const date = parseISO(dateString); - return formatDateInUtc(date, "M/d/yyyy"); -} - -function getKeyCosts( - day: DailyCosts["daily"][0], - costKey: DailyCostKey, - region: "all" | string, -): number { - return day.costs[costKey].prices.reduce((acc, price) => { - if (region === "all" || price.regionId === region) { - acc += parseFloat(price.subtotal); - } - return acc; - }, 0); -} - -export type TooltipData = { - bar: { - day: DailyCosts["daily"][0]; - index: number; - height: number; - width: number; - x: number; - y: number; - color: string; - }; - hoveredIndex: number; -}; - -function calculateMaxDailySpend( - data: DailyCosts["daily"], - region: "all" | string, -): number { - return data.reduce((currentMax, day) => { - if (region === "all") { - return Math.max(currentMax, parseFloat(day.subtotal)); - } - let totalPrice = 0; - for (const costKey of ["compute", "storage"] as const) { - for (const price of day.costs[costKey].prices) { - if (price.regionId === region) { - totalPrice += parseFloat(price.subtotal); - } - } - } - return Math.max(currentMax, totalPrice); - }, 0); -} - -function calculateBarPadding(dayCount: number): number { - if (dayCount > 90) { - return 0; - } else if (dayCount > 7) { - return 0.25; - } - return 0.15; -} - -export type LegendEntry = { - key: DailyCostKey; - label: string; - color: string; -}; - -type DailyUsageChartTooltipProps = { - top: number; - left: number; - legendData: LegendEntry[]; - tooltipData: TooltipData; - availableRegions: string[]; - component: React.FC; - allRegions: boolean; -}; - -const DailyUsageChartTooltip = ({ - legendData, - tooltipData, - availableRegions, - allRegions, - ...props -}: DailyUsageChartTooltipProps) => { - const today = tooltipData.bar.day; - return ( - - - - - - - - {availableRegions.map((region) => ( - - ))} - - - - {legendData.map(({ key, label, color }) => ( - - - {availableRegions.map((region) => ( - - ))} - - ))} - -
- {extractFormattedDateLong(today.startDate)} - - {availableRegions.length > 1 && region} -
- {label} - - {formatCurrency(getKeyCosts(today, key, region))} -
- - - - ); -}; - -const DailyUsageChartGraph = ({ - data, - height, - width, - region, -}: DailyUsageChartProps & ParentSizeProvidedProps) => { - const { svgProps, axisLeftProps, axisBottomProps, xScaleRange, yScaleRange } = - React.useMemo( - () => - buildXYGraphLayoutProps({ - width: width, - height: height, - margin: margin, - }), - [width, height], - ); - const { colors, textStyles } = useTheme(); - const { - tooltipOpen, - tooltipLeft, - tooltipTop, - tooltipData, - hideTooltip, - showTooltip, - } = useTooltip(); - const { containerRef: tooltipContainerRef, TooltipInPortal } = - useTooltipInPortal({ - detectBounds: true, - scroll: true, - }); - const aggregated = useMemo(() => aggregateByDay(data), [data]); - const maxAmount = useMemo( - () => calculateMaxDailySpend(aggregated, region), - [aggregated, region], - ); - const [cloudRegions] = useAtom(cloudRegionsSelector); - const availableRegions = - region === "all" - ? Array.from(cloudRegions.values()).map((details) => getRegionId(details)) - : [region]; - const barPadding = calculateBarPadding(aggregated.length); - const xScale = useMemo( - () => - scaleBand({ - domain: aggregated.map((day) => - extractFormattedDateShort(day.startDate), - ), - range: xScaleRange as [number, number], - padding: barPadding, - }), - [aggregated, xScaleRange, barPadding], - ); - const yScale = useMemo( - () => - scaleLinear({ - // We apply a ceiling to the number so we can drop the fractional amounts - // from the legend - domain: [0, Math.ceil(maxAmount)], - range: yScaleRange, - // We want a "nice" looking tick scale that ends on a good number. - nice: true, - }), - [maxAmount, yScaleRange], - ); - const legendData: LegendEntry[] = [ - { key: "compute", label: "Compute", color: colors.accent.purple }, - { key: "storage", label: "Storage", color: colors.border.secondary }, - ]; - const colorScale = scaleOrdinal({ - domain: legendData.map((cat) => cat.label), - range: legendData.map((cat) => cat.color), - }); - const tooltipTimeoutRef = React.useRef(); - return ( - - - {legendData.map(({ key, label, color }) => ( - - {label} - - ))} - - - - - data={aggregated} - keys={["compute", "storage"]} - x={(day) => extractFormattedDateShort(day.startDate)} - xScale={xScale} - yScale={yScale} - color={colorScale} - value={(datum, key) => getKeyCosts(datum, key, region)} - > - {(barStacks) => - barStacks.map((barStack) => - barStack.bars.map((bar, stackIx) => { - const isHighlighted = tooltipData?.hoveredIndex === stackIx; - const isDemoted = !isHighlighted && tooltipOpen; - return ( - { - tooltipTimeoutRef.current = window.setTimeout( - () => { - hideTooltip(); - }, - // Hold off on dismissing the tooltip to give a little - // accuracy leeway to users when they're mousing over - // the bar. - TOOLTIP_DISMISS_TIMEOUT_MS, - ); - }} - onMouseMove={(event) => { - if (tooltipTimeoutRef.current) { - clearTimeout(tooltipTimeoutRef.current); - tooltipTimeoutRef.current = undefined; - } - showTooltip({ - ...calculateTooltipPosition({ event }), - tooltipData: { - bar: { - day: aggregated[stackIx], - index: barStack.index, - x: bar.x, - y: bar.y, - height: bar.height, - width: bar.width, - color: bar.color, - }, - hoveredIndex: stackIx, - }, - }); - }} - /> - ); - }), - ) - } - - - - > - {...axisLeftProps} - hideTicks - scale={yScale} - stroke={colors.border.secondary} - tickFormat={(value, ix, values) => { - // Only label the first and last ticks. - if (ix > 0 && ix < values.length - 1) { - return ""; - } - return formatCurrency(value, { - trailingZeroDisplay: "stripIfInteger", - useGrouping: false, - notation: "compact", - }); - }} - tickLabelProps={() => ({ - fill: colors.foreground.primary, - fontSize: textStyles["text-small"].fontSize, - textAnchor: "end", - })} - /> - > - {...axisBottomProps} - hideTicks - scale={xScale} - stroke={colors.border.secondary} - labelOffset={8} - tickLabelProps={() => ({ - fill: colors.foreground.primary, - fontSize: textStyles["text-small"].fontSize, - dy: "4px", - textAnchor: "middle", - })} - /> - - - {tooltipTop && tooltipLeft && tooltipOpen && tooltipData && ( - - )} - - ); -}; - -const LegendItem = ({ - children, - color, - textProps, - ...props -}: React.PropsWithChildren< - { - color: string; - textProps?: TextProps; - } & StackProps ->) => { - return ( - - - {children} - - ); -}; - -const DailyUsageChart = (props: DailyUsageChartProps) => { - return ( - - {(parentSizeState) => { - return ( - - ); - }} - - ); -}; - -export { DailyUsageChartTooltip }; -export default DailyUsageChart; diff --git a/console/src/platform/billing/LedgerTable.tsx b/console/src/platform/billing/LedgerTable.tsx new file mode 100644 index 0000000000000..0b8386d31083d --- /dev/null +++ b/console/src/platform/billing/LedgerTable.tsx @@ -0,0 +1,250 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { + Box, + BoxProps, + Collapse, + Grid, + GridProps, + useDisclosure, + useTheme, +} from "@chakra-ui/react"; +import React, { PropsWithChildren, ReactNode } from "react"; + +import ChevronRightIcon from "~/svg/ChevronRightIcon"; +import { MaterializeTheme } from "~/theme"; +import { isSafari } from "~/util"; + +// Cell padding + caret width + caret/label gap, so an indented child label +// lines up just past its parent row's caret. +const INDENTED_PADDING_LEFT = 4 + 4 + 2; + +const baseCellStyles = { + px: 4, + my: "auto", + display: "flex", + alignItems: "center", + height: 8, + borderBottom: "1px solid", +} as const; + +export interface LedgerCellProps extends PropsWithChildren, BoxProps { + variant?: "row" | "columnHeader" | "groupHeader" | "total"; + /** Right-align numeric content (costs, shares). */ + numeric?: boolean; + /** Indent a child row's first cell past the parent's caret. */ + indented?: boolean; + /** Last child row of a group: taller, with bottom padding. */ + isLastRow?: boolean; +} + +export const LedgerCell = ({ + variant = "row", + numeric, + indented, + isLastRow, + children, + ...rest +}: LedgerCellProps) => { + const { colors } = useTheme(); + const variantStyles = { + row: { + borderColor: "transparent", + ...(isLastRow && { height: 10, paddingBottom: "8px" }), + }, + columnHeader: { + height: 10, + textStyle: "text-ui-med", + color: colors.foreground.secondary, + borderColor: colors.border.secondary, + }, + groupHeader: { + height: 16, + textStyle: "heading-xs", + borderBottom: 0, + borderTop: "1px solid", + borderColor: colors.border.secondary, + }, + total: { + height: 12, + textStyle: "text-ui-med", + borderBottom: 0, + borderTop: "1px solid", + borderColor: colors.border.secondary, + }, + }[variant]; + return ( + + {children} + + ); +}; + +export interface LedgerColumn { + label: ReactNode; + numeric?: boolean; + /** Indent to align with an indented child row's label below it. */ + indented?: boolean; +} + +export interface LedgerTableProps extends PropsWithChildren, GridProps { + /** grid-template-columns; every row must render one cell per column. */ + templateColumns: string; + columns: LedgerColumn[]; +} + +/** + * CSS-grid table for grouped spend ledgers. A grid rather than Chakra Table + * or UniversalTable because group collapse must keep child cells + * column-aligned mid-animation, which needs subgrid rows — can't + * animate a run of s. See SafariSafeCollapse. + */ +export const LedgerTable = ({ + templateColumns, + columns, + children, + ...gridProps +}: LedgerTableProps) => { + const { colors } = useTheme(); + return ( + + + {columns.map((column, ix) => ( + + {column.label} + + ))} + + {children} + + ); +}; + +/** Disclosure caret for a group header's first cell. */ +export const LedgerCaret = ({ isOpen }: { isOpen: boolean }) => ( + +); + +export interface LedgerGroupProps extends PropsWithChildren { + /** + * Cells of the always-visible parent row. Put + * `` at the start of the first cell. + */ + renderHeader: (isOpen: boolean) => ReactNode; + /** Child row count; bounds the collapse animation height. */ + rowCount: number; + defaultIsOpen?: boolean; + /** Keep the group collapsed while its data is still loading. */ + isLoading?: boolean; + "data-testid"?: string; + /** data-testid for the collapsible child-row region. */ + contentTestId?: string; +} + +/** An expandable group: clickable parent row plus collapsible child rows. */ +export const LedgerGroup = ({ + renderHeader, + rowCount, + defaultIsOpen = true, + isLoading, + "data-testid": testId, + contentTestId, + children, +}: LedgerGroupProps) => { + const { isOpen, onToggle } = useDisclosure({ defaultIsOpen }); + return ( + <> + { + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + onToggle(); + } + }} + cursor="pointer" + data-testid={testId} + > + {renderHeader(isOpen)} + + + {children} + + + ); +}; + +/** + * Safari fails to layout subgrid elements inside a Collapse; fall back to + * show/hide there. + * https://stackoverflow.com/q/77927259/214197 + */ +export const SafariSafeCollapse = ({ + children, + isCollapsed, + rowCount, + ...otherProps +}: PropsWithChildren<{ isCollapsed: boolean; rowCount: number }>) => { + const collapseProps = { + display: "grid", + gridTemplateColumns: "subgrid", + gridColumn: "1 / -1", + maxHeight: rowCount * 32 + 8, + }; + if (isSafari()) { + return ( + + {children} + + ); + } + return ( + + {children} + + ); +}; diff --git a/console/src/platform/billing/PlanDetails.test.tsx b/console/src/platform/billing/PlanDetails.test.tsx index aea11c1a9dea6..516290a1f7232 100644 --- a/console/src/platform/billing/PlanDetails.test.tsx +++ b/console/src/platform/billing/PlanDetails.test.tsx @@ -23,8 +23,9 @@ import { setFakeEnvironment, } from "~/test/utils"; +import { ROLLING_AVG_TIME_RANGE_LOOKBACK_DAYS } from "./constants"; import { UpgradedPlanDetails } from "./PlanDetails"; -import { getTimeRange } from "./queries"; +import { getDayAlignedRange } from "./queries"; import { summarizePlanCosts } from "./utils"; const Wrapper = await createProviderWrapper({ @@ -42,7 +43,9 @@ describe("UpgradedPlanDetails", () => { }); it("renders successfully for all regions", async () => { - const [startDate, endDate] = getTimeRange(7); + const [startDate, endDate] = getDayAlignedRange( + ROLLING_AVG_TIME_RANGE_LOOKBACK_DAYS, + ); const payload = generateDailyCostResponsePayload(startDate, endDate); renderComponent( , @@ -53,7 +56,9 @@ describe("UpgradedPlanDetails", () => { }); it("hides region breakdown when filtering regions", async () => { - const [startDate, endDate] = getTimeRange(7); + const [startDate, endDate] = getDayAlignedRange( + ROLLING_AVG_TIME_RANGE_LOOKBACK_DAYS, + ); const payload = generateDailyCostResponsePayload(startDate, endDate); renderComponent( { +const PlanDetailsContainer = ({ + children, + testId = "plan-details", +}: PropsWithChildren<{ testId?: string }>) => { const { colors } = useTheme(); return ( sum + accountTotal(account), 0); +} + +/** + * Plan-details box for the Direction-A account-spend section. Mirrors + * `UpgradedPlanDetails`, but sources spend from the per-account daily breakdown + * (`/api/costs/breakdown/daily`) rather than `/api/costs/daily`, so it carries + * no dependency on the legacy daily-costs endpoint. Plan type and balance come + * from the organization and credits, so the box still renders while the + * breakdown is loading or empty. `days` is the selected window (drives total + * spend and daily average); `last30Days` is a fixed 30-day window. Both are + * restricted to `regionFilter` so the totals here match the ledger. + */ +export const AccountSpendPlanDetails = ({ + days, + last30Days, + regionFilter, +}: { + days: CostBreakdownDay[] | null; + last30Days: CostBreakdownDay[] | null; + regionFilter: "all" | string; +}) => { + const { organization } = useCurrentOrganization(); + const { colors } = useTheme(); + const { data: creditBalance } = useCreditBalance(); + + const filteredDays = useMemo( + () => (days ? filterDaysByRegion(days, regionFilter) : null), + [days, regionFilter], + ); + const filteredLast30Days = useMemo( + () => (last30Days ? filterDaysByRegion(last30Days, regionFilter) : null), + [last30Days, regionFilter], + ); + const total = useMemo(() => breakdownTotal(filteredDays), [filteredDays]); + // The window is dense (one bucket per UTC day), so `days.length` is the day + // count the average divides by. + const dailyAverage = + total !== null && filteredDays ? total / filteredDays.length : null; + const last30 = useMemo( + () => breakdownByAccount(filteredLast30Days), + [filteredLast30Days], + ); + + return ( + + + Plan details + + + + + + + + + + + {total !== null && ( + + + + )} + {last30 && ( + + + {last30.accounts.length > 1 && + last30.accounts.map((account) => ( + + ))} + + )} + {dailyAverage !== null && ( + + + + )} + + + + + + + + + Looking to cancel your subscription? + + + Contact Support ⟶ + + + + ); +}; + export const EvaluationPlanDetails = ({ upgradeButtonProps, canManagePayments = true, diff --git a/console/src/platform/billing/PricingPage.tsx b/console/src/platform/billing/PricingPage.tsx new file mode 100644 index 0000000000000..1c82e2b810dea --- /dev/null +++ b/console/src/platform/billing/PricingPage.tsx @@ -0,0 +1,44 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { Text, useTheme } from "@chakra-ui/react"; +import React from "react"; + +import IFrameWrapper from "~/components/IFrameWrapper"; +import { MainContentContainer } from "~/layouts/BaseLayout"; +import { MaterializeTheme } from "~/theme"; + +const PricingPage = () => { + const { colors } = useTheme(); + return ( + + + On-Demand Pricing + + + The following represents our on-demand pricing structure for compute, + storage, and network resources. To calculate your estimated compute + costs, multiply your projected compute credit usage by the listed price + per credit. + + {/* IFrameWrapper passes width/height straight through as native + iframe attributes, not CSS, so this must be a plain pixel number + (browsers parse legacy iframe dimension attributes leniently, + silently truncating anything else, e.g. "80vh" to 80 pixels). */} + + + ); +}; + +export default PricingPage; diff --git a/console/src/platform/billing/SpendBreakdown.test.tsx b/console/src/platform/billing/SpendBreakdown.test.tsx deleted file mode 100644 index a08760adadcd2..0000000000000 --- a/console/src/platform/billing/SpendBreakdown.test.tsx +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { render, screen } from "@testing-library/react"; -import { parseISO } from "date-fns"; -import React, { ReactElement } from "react"; - -import { DailyCosts } from "~/api/cloudGlobalApi"; -import { - buildCloudRegionsReponse, - buildInvoicesResponse, - generateDailyCostResponsePayload, -} from "~/api/mocks/cloudGlobalApiHandlers"; -import server from "~/api/mocks/server"; -import { - createProviderWrapper, - healthyEnvironment, - setFakeEnvironment, -} from "~/test/utils"; -import { assert } from "~/util"; - -import SpendBreakdown from "./SpendBreakdown"; -import { summarizeResourceCosts } from "./utils"; - -const Wrapper = await createProviderWrapper({ - initializeState: ({ set }) => - setFakeEnvironment(set, "aws/us-east-1", healthyEnvironment), -}); - -const renderComponent = (element: ReactElement) => { - return render({element}); -}; - -describe("SpendBreakdown", () => { - beforeEach(() => { - server.use(buildCloudRegionsReponse(), buildInvoicesResponse()); - }); - - it("renders successfully for all regions", async () => { - const startDate = parseISO("2024-01-01T00:00:00.000+00:00"); - const endDate = parseISO("2024-01-07T00:00:00.000+00:00"); - const payload = generateDailyCostResponsePayload(startDate, endDate); - renderComponent( - , - ); - const planDetails = await screen.findByTestId("spend-breakdown-range"); - expect(planDetails).toHaveTextContent( - "Spend between 01-01-24 and 01-06-24", - ); - const computeTable = screen.getByTestId("spend-breakdown-compute-group"); - expect(computeTable).toHaveTextContent("aws/us-east-1"); - expect(computeTable).toHaveTextContent("aws/eu-west-1"); - const storageTable = screen.getByTestId("spend-breakdown-storage-group"); - expect(storageTable).toHaveTextContent("aws/us-east-1"); - expect(storageTable).toHaveTextContent("aws/eu-west-1"); - }); - - it("hides the details when data is being loaded", async () => { - renderComponent( - , - ); - const planDetails = await screen.findByTestId("spend-breakdown-range"); - expect(planDetails).toHaveTextContent("Spend between"); - const computeTable = screen.getByTestId("spend-breakdown-compute-group"); - expect(computeTable).not.toHaveTextContent("aws/us-east-1"); - expect(computeTable).not.toHaveTextContent("aws/eu-west-1"); - const storageTable = screen.getByTestId("spend-breakdown-storage-group"); - expect(storageTable).not.toHaveTextContent("aws/us-east-1"); - expect(storageTable).not.toHaveTextContent("aws/eu-west-1"); - }); - - it("only displays filtered regions", async () => { - const startDate = parseISO("2024-01-01T00:00:00.000+00:00"); - const endDate = parseISO("2024-01-07T00:00:00.000+00:00"); - const payload = generateDailyCostResponsePayload(startDate, endDate); - renderComponent( - , - ); - const computeTable = screen.getByTestId("spend-breakdown-compute-group"); - expect(computeTable).not.toHaveTextContent("aws/us-east-1"); - expect(computeTable).not.toHaveTextContent("aws/eu-west-1"); - expect(computeTable).toHaveTextContent("credits"); - const storageTable = screen.getByTestId("spend-breakdown-storage-group"); - expect(storageTable).not.toHaveTextContent("aws/us-east-1"); - expect(storageTable).not.toHaveTextContent("aws/eu-west-1"); - }); - - it("correctly aggregates midday plan changes", () => { - const costData: DailyCosts["daily"] = [ - { - startDate: "2024-01-12T00:00:00Z", - endDate: "2024-01-13T12:00:00Z", - subtotal: "1.00", - total: "1.00", - costs: { - compute: { - subtotal: "0.50", - total: "0.50", - prices: [ - { - regionId: "aws/us-east-1", - replicaSize: "large", - unitAmount: "1.20", - subtotal: "0.50", - }, - ], - }, - storage: { - subtotal: "0.50", - total: "0.50", - prices: [ - { - regionId: "aws/us-east-1", - unitAmount: "0.000081", - subtotal: "0.50", - }, - ], - }, - }, - }, - { - startDate: "2024-01-13T00:00:00Z", - endDate: "2024-01-13T12:00:00Z", - subtotal: "1.00", - total: "1.00", - costs: { - compute: { - subtotal: "0.50", - total: "0.50", - prices: [ - { - regionId: "aws/us-east-1", - replicaSize: "large", - unitAmount: "1.20", - subtotal: "0.50", - }, - ], - }, - storage: { - subtotal: "0.50", - total: "0.50", - prices: [ - { - regionId: "aws/us-east-1", - unitAmount: "0.000081", - subtotal: "0.50", - }, - ], - }, - }, - }, - { - startDate: "2024-01-13T12:00:00Z", - endDate: "2024-01-14T00:00:00Z", - subtotal: "2.00", - total: "2.00", - costs: { - compute: { - subtotal: "1.50", - total: "1.50", - prices: [ - { - regionId: "aws/us-east-1", - replicaSize: "large", - unitAmount: "1.50", - subtotal: "1.50", - }, - ], - }, - storage: { - subtotal: "0.50", - total: "0.50", - prices: [ - { - regionId: "aws/us-east-1", - unitAmount: "0.000081", - subtotal: "0.50", - }, - ], - }, - }, - }, - ]; - const summarized = summarizeResourceCosts(costData, 2); - expect(summarized).toHaveLength(1); - const region = summarized.get("aws/us-east-1"); - expect(region).toBeDefined(); - assert(region); - expect(region.compute.total.usagePoints).toHaveLength(2); - expect(region.compute.total.totalCost).toEqual(2.5); - expect(region.storage.total.usagePoints).toHaveLength(2); - expect(region.storage.total.totalCost).toEqual(1.5); - }); -}); diff --git a/console/src/platform/billing/SpendBreakdown.tsx b/console/src/platform/billing/SpendBreakdown.tsx deleted file mode 100644 index 2824c86698543..0000000000000 --- a/console/src/platform/billing/SpendBreakdown.tsx +++ /dev/null @@ -1,547 +0,0 @@ -// Copyright Materialize, Inc. and contributors. All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -import { - Box, - BoxProps, - chakra, - Collapse, - Fade, - Grid, - Text, - Tooltip, - useDisclosure, - useTheme, -} from "@chakra-ui/react"; -import { curveMonotoneX } from "@visx/curve"; -import ParentSize from "@visx/responsive/lib/components/ParentSize"; -import { scaleLinear } from "@visx/scale"; -import { LinePath } from "@visx/shape"; -import { parseISO } from "date-fns"; -import React, { PropsWithChildren, useMemo } from "react"; - -import { DailyCostKey, DailyCosts } from "~/api/cloudGlobalApi"; -import ChevronRightIcon from "~/svg/ChevronRightIcon"; -import WarningIcon from "~/svg/WarningIcon"; -import { MaterializeTheme } from "~/theme"; -import { formatBytes, isSafari } from "~/util"; -import { DATE_FORMAT_SHORT, formatDateInUtc } from "~/utils/dateFormat"; -import { formatCurrency } from "~/utils/format"; - -import { costUnits } from "./constants"; -import { ResourceBreakdown, ResourceMeasurement } from "./types"; -import { summarizeResourceCosts } from "./utils"; - -// The first day of plans that introduced storage usage bucketed by region. -const BUCKETED_STORAGE_START_DATE = new Date("2024-02-01T00:00:00Z"); - -type SpendBreakdownProps = { - region: "all" | string; - dailyCosts: DailyCosts["daily"] | null; - totalDays: number; -}; - -type ResourceGroupProps = { - costCaption?: boolean; - costKey: DailyCostKey; - headerProps?: BoxProps; - isLoading: boolean; - isRegionFiltering: boolean; - label: string; - selectedRegions: Array<[string, ResourceBreakdown]>; - warningMessage?: string; -}; - -function safeRound(value: number, digits: number): number { - const rounded = Math.round(Number(`${value}e+${digits}`)); - return Number(`${rounded}e-${digits}`); -} - -const STROKE_WIDTH = 2; - -const SpendSparkline = ({ - color, - points, - min, - max, - height, - width, -}: { - points: number[]; - min: number; - max: number; - color: string; - height: number; - width: number; -}) => { - const dataPoints = points.map((value, ix) => ({ offset: ix, value })); - const xScale = useMemo( - () => - scaleLinear({ - // Cartesian system values (ascending left to right) - domain: [0, dataPoints.length], - // SVG coordinate values (ascending left to right) - range: [0, width], - }), - [width, dataPoints], - ); - const yScale = useMemo( - () => - scaleLinear({ - // Cartesian system values (ascending left to right) - domain: [min, max], - // SVG coordinate values (ascending top to bottom) - // Add extra padding to avoid clipping broad strokes. - range: [height - STROKE_WIDTH / 2, STROKE_WIDTH / 2], - }), - [height, min, max], - ); - return ( - - xScale(d.offset)} - y={(d) => yScale(d.value)} - /> - - ); -}; - -const StorageMetric = ({ usageGb }: { usageGb: number }) => { - const usageBytes = usageGb * Math.pow(1024, 3); - const [amount, unit] = formatBytes(usageBytes); - const rounded = safeRound(amount, 2); - return ( - - - {rounded} {unit} - - - ); -}; - -const CreditsMetric = ({ credits }: { credits: number }) => { - const rounded = safeRound(credits, 2); - return ( - - {rounded} credits - - ); -}; - -const resourceTypePaddingLeft = 4 + 4 + 2; // table cell + width of caret + caret/label gap - -const baseCellStyles = { - px: 4, - my: "auto", - display: "flex", - alignItems: "center", - height: 8, - borderBottom: "1px solid", -}; - -function getMinWithDefault(left: number, right: number): number { - /// We initialize min values with -1, so it always loses. - if (left === -1) { - return right; - } - if (right === -1) { - return left; - } - return Math.min(left, right); -} - -function findMinMax(points: number[]) { - let min = 1; - let max = 0; - for (const point of points) { - min = getMinWithDefault(min, point); - max = Math.max(max, point); - } - return { min, max }; -} - -type ResourceRowProps = { - costCaption?: boolean; - regionId: string; - resourceType: string; - resourceSummary: ResourceMeasurement; - isLastElement: boolean; - isRegionFiltering: boolean; -}; - -const ResourceRow = ({ - regionId, - resourceType, - resourceSummary, - isLastElement, - isRegionFiltering, - costCaption, -}: ResourceRowProps) => { - const { colors } = useTheme(); - const cellStyles = { - ...baseCellStyles, - borderColor: "transparent", - height: isLastElement ? 10 : baseCellStyles.height, - paddingBottom: isLastElement ? "8px" : "unset", - }; - const { min: usageMin, max: usageMax } = useMemo( - () => findMinMax(resourceSummary.usagePoints), - [resourceSummary], - ); - return ( - <> - - {!isRegionFiltering && `${regionId} / `} - {resourceType} - - - {resourceSummary.usageUnits === costUnits.storage ? ( - - ) : ( - - )} - - - - - {(parent) => ( - - )} - - - - - {formatCurrency(resourceSummary.totalCost)} - {costCaption && ( - - (${resourceSummary.rate} {resourceSummary.usageUnits} - /hr) - - )} - - - ); -}; - -function summarizeRegions(regions: ResourceBreakdown[], costKey: DailyCostKey) { - let totalPoints: number[] = []; - for (const region of regions) { - const { usagePoints: regionPoints } = region[costKey].total; - if (totalPoints.length === 0) { - totalPoints = Array(regionPoints.length).fill(0); - } - totalPoints = regionPoints.map((val, ix) => val + totalPoints[ix]); - } - let min = -1; - let max = 0; - for (const point of totalPoints) { - min = getMinWithDefault(min, point); - max = Math.max(max, point); - } - return { min, max, totalPoints }; -} - -const ResourceGroup = ({ - costCaption, - costKey, - headerProps, - isLoading, - isRegionFiltering, - label, - selectedRegions, - warningMessage, -}: ResourceGroupProps) => { - const { colors } = useTheme(); - const { isOpen, onToggle } = useDisclosure(); - const totalUsageMetric = selectedRegions.reduce( - (total, [_regionId, regionSummary]) => - total + regionSummary[costKey].total.usageValue, - 0, - ); - const selectedSummary = summarizeRegions( - selectedRegions.map(([_regionId, breakdown]) => breakdown), - costKey, - ); - const resourceMeasurements = selectedRegions.flatMap( - ([regionId, regionSummary]) => { - return Array.from(regionSummary[costKey].resources.entries()) - .sort(([_l, left], [_r, right]) => left.sort - right.sort) - .map( - ([resourceType, resourceSummary]) => - [regionId, resourceType, resourceSummary] as const, - ); - }, - ); - const groupHeaderStyles = { - ...baseCellStyles, - height: 16, - textStyle: "heading-xs", - borderColor: !isOpen ? colors.border.secondary : "transparent", - borderBottom: 0, - }; - return ( - <> - - - - {label} - - - - - - - - {selectedRegions.length > 0 && costKey === "storage" ? ( - - ) : ( - - )} - - - - - {(parent) => ( - - )} - - - - - {formatCurrency( - selectedRegions.reduce( - (total, [_regionId, regionSummary]) => - total + regionSummary[costKey].total.totalCost, - 0, - ), - )} - - - - {resourceMeasurements.map( - ([regionId, resourceType, resourceSummary], ix) => ( - - ), - )} - - - ); -}; - -/** - * Safari fails to layout the subgrid elements correctly if they're in a - * Collapse component. For Safari, fall back to showing and hiding a simple - * div. Other browsers get the nice animation. - * - * StackOverflow post where someone encounters a similar issue: - * https://stackoverflow.com/q/77927259/214197 - */ -const SafariSafeCollapse = ({ - children, - isCollapsed, - rowCount, - ...otherProps -}: PropsWithChildren<{ - isCollapsed: boolean; - rowCount: number; -}>) => { - const collapseProps = { - display: "grid", - gridTemplateColumns: "subgrid", - gridColumn: "1 / -1", - maxHeight: rowCount * 32 + 8, - }; - if (isSafari()) { - return ( - - {children} - - ); - } else { - return ( - - {children} - - ); - } -}; - -const SpendBreakdown = ({ - region, - dailyCosts, - totalDays, -}: SpendBreakdownProps) => { - const { colors } = useTheme(); - const isLoading = dailyCosts === null; - const startDate = !isLoading - ? formatDateInUtc(parseISO(dailyCosts[0].startDate), DATE_FORMAT_SHORT) - : "-"; - const endDate = !isLoading - ? formatDateInUtc(parseISO(dailyCosts.at(-1)!.startDate), DATE_FORMAT_SHORT) - : "-"; - const hasUnbucketedStorage = - !isLoading && - parseISO(dailyCosts[0].startDate) < BUCKETED_STORAGE_START_DATE; - const resourceCostsByRegion = useMemo( - () => summarizeResourceCosts(dailyCosts ?? [], totalDays), - [dailyCosts, totalDays], - ); - const isRegionFiltering = region !== "all"; - const regions = !isRegionFiltering - ? Array.from(resourceCostsByRegion.keys()) - : [region]; - const selectedRegions = Array.from(resourceCostsByRegion.entries()).filter( - ([regionId, _regionSummary]) => regions.includes(regionId), - ); - - const tableHeaderStyles = { - ...baseCellStyles, - py: 3, - textStyle: "text-ui-med", - color: colors.foreground.secondary, - borderColor: colors.border.secondary, - height: 10, - }; - - return ( - - - Spend between{" "} - {!isLoading && ( - <> - - {startDate} - {" "} - and{" "} - - {endDate} - - - )} - - - - Resource type - - - Usage - - - Usage trend - - - Total cost - - - - - - ); -}; - -export default SpendBreakdown; diff --git a/console/src/platform/billing/TimeRangeSelect.tsx b/console/src/platform/billing/TimeRangeSelect.tsx index 89eccabd095fb..09d275dd4162a 100644 --- a/console/src/platform/billing/TimeRangeSelect.tsx +++ b/console/src/platform/billing/TimeRangeSelect.tsx @@ -23,7 +23,7 @@ const TimeRangeSelect = ({ timeRange, setTimeRange }: TimeRangeSelectProps) => { 14: "Last 14 days", 30: "Last 30 days", 90: "Last 90 days", - 180: "Last 180 days", + 100: "Last 100 days", }; return ( @@ -42,6 +40,17 @@ const renderComponent = (element: ReactElement) => { return render({element}); }; +// Wrap accounts in a single UTC-day bucket — the minimal `/api/costs/breakdown/ +// daily` payload. Per-day cost is additive, so a one-day window reproduces the +// period totals the breakdown asserts on. +const oneDay = (accounts: CostBreakdownAccount[]) => [ + { + startDate: "2024-01-15T00:00:00Z", + endDate: "2024-01-16T00:00:00Z", + accounts, + }, +]; + const buildOrganization = (overrides: Partial = {}) => { return { id: "00000000-0000-0000-0000-000000000000", @@ -63,6 +72,7 @@ describe("UsagePage", () => { buildCloudRegionsReponse(), buildInvoicesResponse(), buildCreditsResponse(), + buildDailyCostBreakdownResponse(), buildCloudOrganizationsResponse({ payload: buildOrganization(), }), @@ -74,104 +84,486 @@ describe("UsagePage", () => { vi.clearAllMocks(); }); - it("renders successfully for all regions", async () => { - const [startDate, endDate] = getTimeRange(7); - const payload = generateDailyCostResponsePayload(startDate, endDate); - expect(payload.daily.length).toEqual(30); - server.use(buildDailyCostResponse({ payload })); + it("renders a unified account & cluster ledger for a parent org", async () => { + server.use( + buildDailyCostBreakdownResponse({ + payload: { + days: oneDay([ + { + external_customer_id: "parent-org", + name: "", + clusters: [ + { + environment_id: "environment-parent-0", + cluster_grouping_key: "quickstart.r1", + category: "", + region: "aws/us-east-1", + amounts: { "price-compute": "10.00" }, + usage: 0, + }, + { + environment_id: "environment-parent-0", + cluster_grouping_key: "compute.r1", + category: "", + region: "aws/us-east-1", + amounts: { "price-compute": "4.00" }, + usage: 0, + }, + ], + }, + { + external_customer_id: "child-org", + name: "", + clusters: [ + { + environment_id: "environment-child-0", + cluster_grouping_key: "prod.r1", + category: "", + region: "aws/us-east-1", + amounts: { "price-compute": "5.00" }, + usage: 0, + }, + ], + }, + ]), + }, + }), + ); renderComponent(); - const regionSelect = await screen.findByTestId( - "region-select", - {}, - { timeout: 5_000 }, + const breakdown = within( + await screen.findByTestId( + "account-spend-breakdown", + {}, + { timeout: 5_000 }, + ), + ); + // The ledger lives in its own full-width grid row below the chart and + // plan-details columns (SAS-154). + const ledger = within(await screen.findByTestId("account-spend-ledger")); + // One expandable row per account, biggest spender first, each showing that + // account's period total (parent 14 = 10 + 4, child 5). + const accountRows = await ledger.findAllByTestId("account-row"); + expect(accountRows).toHaveLength(2); + expect(within(accountRows[0]).getByText(formatCurrency(14))).toBeVisible(); + expect(within(accountRows[1]).getByText(formatCurrency(5))).toBeVisible(); + // Each account row also shows its share of the period total (SAS-144): + // parent 14/19 ≈ 73.7%, child 5/19 ≈ 26.3%. + expect(within(accountRows[0]).getByText("73.7%")).toBeVisible(); + expect(within(accountRows[1]).getByText("26.3%")).toBeVisible(); + // Accounts render expanded, so every cluster row is visible inline, + // region-qualified ("aws/us-east-1 / "). + for (const cluster of ["quickstart.r1", "compute.r1", "prod.r1"]) { + expect( + await ledger.findByText(`aws/us-east-1 / ${cluster}`), + ).toBeVisible(); + } + // The segmented-by-account chart and the grand-total row (19 = 14 + 5) are + // present. + expect(await breakdown.findByTestId("account-spend-chart")).toBeVisible(); + const totalRow = within(await ledger.findByTestId("account-total-row")); + expect(totalRow.getByText(formatCurrency(19))).toBeVisible(); + // A Usage column sits between "Account / cluster" and "Share of total" + // (SAS-145): the endpoint doesn't carry usage quantities yet, so each of + // the 3 cluster rows shows a placeholder rather than a number. + expect(await ledger.findByText("Usage")).toBeVisible(); + expect(ledger.getAllByText("—")).toHaveLength(3); + // The section leads with the period total (19 = 14 + 5), mirroring the + // legacy chart panel, and a "Spend between …" range above the table, + // mirroring the legacy "Spend between …" breakdown. oneDay()'s single + // bucket is 2024-01-15, so the range collapses to that one date. + expect( + within(await breakdown.findByTestId("account-spend-total")).getByText( + formatCurrency(19), + ), + ).toBeVisible(); + const range = within(await ledger.findByTestId("account-spend-range")); + expect(range.getByText("Spend between", { exact: false })).toBeVisible(); + expect(range.getAllByText("01-15-24")).toHaveLength(2); + }); + + it("sends bare inclusive UTC calendar dates to the breakdown endpoint (SAS-151)", async () => { + // Regression guard for the timestamp-serialization bug class: the query + // params must be plain YYYY-MM-DD (no time component, no local-zone + // offset for a shifted wall-clock date to hide in), computed from UTC + // calendar fields, both ends inclusive. + const captured: URLSearchParams[] = []; + server.use( + http.get("*/api/costs/breakdown/daily", ({ request }) => { + captured.push(new URL(request.url).searchParams); + return HttpResponse.json({ days: [] }); + }), ); - expect(regionSelect).toBeVisible(); + // Computed before render so a UTC-midnight rollover mid-test can't skew + // the expectation. + const now = new Date(); + const utcDay = (offset: number) => + new Date( + Date.UTC( + now.getUTCFullYear(), + now.getUTCMonth(), + now.getUTCDate() + offset, + ), + ) + .toISOString() + .slice(0, 10); - const timeRangeSelect = await screen.findByTestId("time-range-select"); - expect(timeRangeSelect).toBeVisible(); + renderComponent(); + // The page fires two breakdown queries: the selected range (default + // "Last 7 days") and the fixed 30-day plan-details window. + await waitFor(() => expect(captured.length).toBeGreaterThanOrEqual(2)); - const spend = await screen.findByTestId("all-spend-amount"); - const timeRangeSubtotal = payload.daily - .slice(-7) - .reduce((subtotal, day) => subtotal + parseFloat(day.subtotal), 0); - expect(spend.textContent).toEqual(formatCurrency(timeRangeSubtotal)); + for (const params of captured) { + expect(params.get("startDate")).toMatch(/^\d{4}-\d{2}-\d{2}$/); + expect(params.get("endDate")).toMatch(/^\d{4}-\d{2}-\d{2}$/); + } + // "Last N days" is the N inclusive UTC days ending today, so exactly N + // buckets can come back — the 31-buckets-for-30-days symptom is pinned + // out here. + const sevenDay = captured.find((p) => p.get("startDate") === utcDay(-6)); + expect(sevenDay?.get("endDate")).toBe(utcDay(0)); + const thirtyDay = captured.find((p) => p.get("startDate") === utcDay(-29)); + expect(thirtyDay?.get("endDate")).toBe(utcDay(0)); + }); - await waitFor(async () => - expect(await screen.findByTestId("chart")).toBeVisible(), + it("shows an error state when the breakdown request fails", async () => { + server.use(buildDailyCostBreakdownResponse({ status: 500 })); + renderComponent(); + expect( + await screen.findByText("An error occurred loading your usage"), + ).toBeVisible(); + }); + + it("shows an empty state when the window has no usage", async () => { + // beforeEach's default handler already returns { days: [] }. + renderComponent(); + expect(await screen.findByTestId("account-breakdown-empty")).toBeVisible(); + }); + + it("renders 0.0% shares for an all-zero period instead of NaN (SAS-144)", async () => { + server.use( + buildDailyCostBreakdownResponse({ + payload: { + days: oneDay([ + { + external_customer_id: "parent-org", + name: "", + clusters: [ + { + environment_id: "environment-parent-0", + cluster_grouping_key: "quickstart.r1", + category: "", + region: "aws/us-east-1", + amounts: { "price-compute": "0.00" }, + usage: 0, + }, + ], + }, + { + external_customer_id: "child-org", + name: "", + clusters: [ + { + environment_id: "environment-child-0", + cluster_grouping_key: "prod.r1", + category: "", + region: "aws/us-east-1", + amounts: { "price-compute": "0.00" }, + usage: 0, + }, + ], + }, + ]), + }, + }), ); + renderComponent(); + + const ledger = within( + await screen.findByTestId("account-spend-ledger", {}, { timeout: 5_000 }), + ); + const accountRows = await ledger.findAllByTestId("account-row"); + expect(accountRows).toHaveLength(2); + for (const row of accountRows) { + expect(within(row).getByText("0.0%")).toBeVisible(); + } + expect(ledger.queryByText(/NaN/)).not.toBeInTheDocument(); }); - it("changing the region filters the totals", async () => { - const [startDate, endDate] = getTimeRange(7); - const payload = generateDailyCostResponsePayload(startDate, endDate); - server.use(buildDailyCostResponse({ payload })); + it("shows a plan-details box beside the breakdown, itemizing last 30 days by account", async () => { + server.use( + buildDailyCostBreakdownResponse({ + payload: { + days: oneDay([ + { + external_customer_id: "parent-org", + name: "", + clusters: [ + { + environment_id: "environment-parent-0", + cluster_grouping_key: "quickstart.r1", + category: "", + region: "aws/us-east-1", + amounts: { "price-compute": "10.00" }, + usage: 0, + }, + { + environment_id: "environment-parent-0", + cluster_grouping_key: "compute.r1", + category: "", + region: "aws/us-east-1", + amounts: { "price-compute": "4.00" }, + usage: 0, + }, + ], + }, + { + external_customer_id: "child-org", + name: "", + clusters: [ + { + environment_id: "environment-child-0", + cluster_grouping_key: "prod.r1", + category: "", + region: "aws/us-east-1", + amounts: { "price-compute": "5.00" }, + usage: 0, + }, + ], + }, + ]), + }, + }), + ); renderComponent(); - const regionSelect = screen.getByTestId("region-select"); - expect(regionSelect).toBeVisible(); - await waitFor(async () => - expect(await screen.findByTestId("chart")).toBeVisible(), + const planDetails = within( + await screen.findByTestId("account-plan-details", {}, { timeout: 5_000 }), + ); + // The box renders before the breakdown resolves, so await the spend rows. + // Every figure is derived from /api/costs/breakdown/daily, not + // /api/costs/daily. + expect(await planDetails.findByText("Total spend")).toBeVisible(); + expect(await planDetails.findByText("Last 30 days")).toBeVisible(); + expect(await planDetails.findByText("Daily average")).toBeVisible(); + // "Last 30 days" is itemized by account (parent 14 = 10 + 4, child 5), + // biggest spender first, under the window total. + expect(await planDetails.findByText(formatCurrency(14))).toBeVisible(); + expect(await planDetails.findByText(formatCurrency(5))).toBeVisible(); + }); + + it("renders storage and egress as distinct region-qualified rows", async () => { + server.use( + buildDailyCostBreakdownResponse({ + payload: { + days: oneDay([ + { + external_customer_id: "standalone-org", + name: "", + clusters: [ + { + environment_id: "environment-standalone-0", + cluster_grouping_key: "default.r1", + category: "", + region: "aws/us-east-1", + amounts: { "price-compute": "3.00" }, + usage: 0, + }, + { + // Storage and egress both have an empty cluster_grouping_key; + // their `category` keeps them on separate rows, rendered as + // " / Storage" and " / Egress". + environment_id: "environment-standalone-0", + cluster_grouping_key: "", + category: "Storage", + region: "aws/us-east-1", + amounts: { "price-storage": "0.50" }, + usage: 0, + }, + { + environment_id: "environment-standalone-0", + cluster_grouping_key: "", + category: "Egress", + region: "aws/us-east-1", + amounts: { "price-egress": "0.25" }, + usage: 0, + }, + ], + }, + ]), + }, + }), ); - const spendElement = await screen.findByTestId("all-spend-amount"); - const allSpend = parseFloat( - spendElement.textContent!.replace("$", "").replace(",", ""), + renderComponent(); + + const ledger = within( + await screen.findByTestId("account-spend-ledger", {}, { timeout: 5_000 }), ); - await selectReactSelectOption(regionSelect, "aws/us-east-1"); - const regionElement = await screen.findByTestId( - "aws/us-east-1-spend-amount", + // The account renders expanded, so its clusters are visible inline. Scope + // lookups to the ledger table. Storage and egress (both empty + // cluster_grouping_key) stay on separate rows via `category`. + expect(await ledger.findByText("aws/us-east-1 / default.r1")).toBeVisible(); + expect(await ledger.findByText("aws/us-east-1 / Storage")).toBeVisible(); + expect(await ledger.findByText("aws/us-east-1 / Egress")).toBeVisible(); + }); + + it("falls back to 'Other' when a row has neither cluster key nor category", async () => { + // Defensive: a non-compute row with an empty cluster_grouping_key and no + // category can only occur against a backend that predates the `category` + // field. It should render " / Other" rather than mislabel as a + // cluster or crash. + server.use( + buildDailyCostBreakdownResponse({ + payload: { + days: oneDay([ + { + external_customer_id: "standalone-org", + name: "", + clusters: [ + { + environment_id: "environment-standalone-0", + cluster_grouping_key: "", + category: "", + region: "aws/us-east-1", + amounts: { "price-storage": "0.50" }, + usage: 0, + }, + ], + }, + ]), + }, + }), ); - const regionSpend = parseFloat( - regionElement.textContent!.replace("$", "").replace(",", ""), + renderComponent(); + + const ledger = within( + await screen.findByTestId("account-spend-ledger", {}, { timeout: 5_000 }), ); - expect(regionSpend, JSON.stringify(payload)).toBeLessThan(allSpend); + // The account renders expanded, so its one cluster row is visible inline. + expect(await ledger.findByText("aws/us-east-1 / Other")).toBeVisible(); }); - it("changing the time range filters the totals", async () => { - let [startDate, endDate] = getTimeRange(7); - let payload = generateDailyCostResponsePayload(startDate, endDate); - server.use(buildDailyCostResponse({ payload })); + it("filters the ledger by region", async () => { + server.use( + buildDailyCostBreakdownResponse({ + payload: { + days: oneDay([ + { + external_customer_id: "east-org", + name: "", + clusters: [ + { + environment_id: "environment-east-0", + cluster_grouping_key: "quickstart.r1", + category: "", + region: "aws/us-east-1", + amounts: { "price-compute": "10.00" }, + usage: 0, + }, + ], + }, + { + external_customer_id: "west-org", + name: "", + clusters: [ + { + environment_id: "environment-west-0", + cluster_grouping_key: "quickstart.r1", + category: "", + region: "aws/eu-west-1", + amounts: { "price-compute": "5.00" }, + usage: 0, + }, + ], + }, + ]), + }, + }), + ); renderComponent(); - const timeRangeSelect = - screen.getByTestId("time-range-select"); - expect(timeRangeSelect).toBeVisible(); - await waitFor(async () => - expect(await screen.findByTestId("chart")).toBeVisible(), + const ledger = within( + await screen.findByTestId("account-spend-ledger", {}, { timeout: 5_000 }), ); + // Both accounts show before filtering. + expect(await ledger.findAllByTestId("account-row")).toHaveLength(2); + const totalRowBefore = within( + await ledger.findByTestId("account-total-row"), + ); + expect(totalRowBefore.getByText(formatCurrency(15))).toBeVisible(); - const spendElement = await screen.findByTestId("all-spend-amount"); - const originalSpend = parseFloat( - spendElement.textContent!.replace("$", "").replace(",", ""), + const planDetails = within( + await screen.findByTestId("account-plan-details", {}, { timeout: 5_000 }), ); + const totalSpendLabel = await planDetails.findByText("Total spend"); + // The plan-details summary must track the same region filter as the + // ledger, so it starts at the unfiltered $15 total too. + expect( + within(totalSpendLabel.parentElement as HTMLElement).getByText( + formatCurrency(15), + ), + ).toBeVisible(); - [startDate, endDate] = getTimeRange(14); - payload = generateDailyCostResponsePayload(startDate, endDate); - server.use(buildDailyCostResponse({ payload })); - await selectReactSelectOption(timeRangeSelect, "Last 14 days"); - await waitFor(async () => - expect(await screen.findByTestId("chart")).toBeVisible(), + const regionSelect = screen.getByTestId( + "account-region-select", ); - const newSpend = parseFloat( - spendElement.textContent!.replace("$", "").replace(",", ""), + await selectReactSelectOption(regionSelect, "aws/us-east-1"); + + // Only the us-east-1 account remains, and the grand total drops to just + // its cost — the eu-west-1 account is filtered out entirely. + await waitFor(async () => { + expect(await ledger.findAllByTestId("account-row")).toHaveLength(1); + }); + const accountRows = await ledger.findAllByTestId("account-row"); + expect(within(accountRows[0]).getByText(formatCurrency(10))).toBeVisible(); + const totalRowAfter = within( + await ledger.findByTestId("account-total-row"), ); - expect(originalSpend).toBeLessThan(newSpend); + expect(totalRowAfter.getByText(formatCurrency(10))).toBeVisible(); + + // The plan-details summary follows the same filter: total spend drops to + // the us-east-1-only figure, matching the ledger. + await waitFor(() => { + expect( + within(totalSpendLabel.parentElement as HTMLElement).getByText( + formatCurrency(10), + ), + ).toBeVisible(); + }); }); - it("displays a generic error if there's an issue querying the server", async () => { - server.use(buildDailyCostResponse({ status: 500 })); + it("displays the invoices table for direct-billed organizations", async () => { + server.use( + buildInvoicesResponse({ + invoices: [ + { + issueDate: "2026-06-02T00:00:00Z", + currency: "usd", + total: "100.00", + amountDue: "100.00", + createdAt: "2026-06-02T00:00:00Z", + status: "paid", + invoiceNumber: "INV-001", + }, + ], + }), + ); renderComponent(); await waitFor(async () => - expect(await screen.findByTestId("chart-error")).toBeVisible(), + expect(await screen.findByTestId("invoice-table")).toBeVisible(), ); }); - it("displays the invoices table for direct-billed organizations", async () => { - server.use(buildDailyCostResponse()); + it("hides the invoice history section entirely for an account with no invoices (e.g. an Orb hierarchy leaf account, which never has its own)", async () => { + server.use(buildInvoicesResponse({ invoices: [] })); renderComponent(); await waitFor(async () => - expect(await screen.findByTestId("invoice-table")).toBeVisible(), + expect( + await screen.findByTestId("account-spend-breakdown"), + ).toBeVisible(), ); + expect(screen.queryByText("Invoice history")).not.toBeInTheDocument(); + expect(screen.queryByTestId("invoice-table")).not.toBeInTheDocument(); }); it("hides the invoices table for AWS Marketplace-billed organizations", async () => { @@ -184,7 +576,6 @@ describe("UsagePage", () => { }, }), }), - buildDailyCostResponse(), ); renderComponent(); @@ -193,59 +584,8 @@ describe("UsagePage", () => { ); }); - it("correctly slices the right number of records for the time range when the range is equal to the number of days", () => { - const dailyCosts: DailyCosts["daily"] = [ - { - startDate: "2024-01-13T00:00:00Z", - endDate: "2024-01-14T00:00:00Z", - costs: { - compute: { prices: [], subtotal: "0.00", total: "0.00" }, - storage: { prices: [], subtotal: "0.00", total: "0.00" }, - }, - total: "0.00", - subtotal: "0.00", - }, - { - startDate: "2024-01-14T00:00:00Z", - endDate: "2024-01-14T12:00:00Z", - costs: { - compute: { prices: [], subtotal: "0.00", total: "0.00" }, - storage: { prices: [], subtotal: "0.00", total: "0.00" }, - }, - total: "0.00", - subtotal: "0.00", - }, - { - startDate: "2024-01-14T12:00:00Z", - endDate: "2024-01-15T00:00:00Z", - costs: { - compute: { prices: [], subtotal: "0.00", total: "0.00" }, - storage: { prices: [], subtotal: "0.00", total: "0.00" }, - }, - total: "0.00", - subtotal: "0.00", - }, - { - startDate: "2024-01-15T00:00:00Z", - endDate: "2024-01-16T00:00:00Z", - costs: { - compute: { prices: [], subtotal: "0.00", total: "0.00" }, - storage: { prices: [], subtotal: "0.00", total: "0.00" }, - }, - total: "0.00", - subtotal: "0.00", - }, - ]; - // Get 3 days of the time range - const sliced = getTimeRangeSlice(dailyCosts, 3); - assert(sliced !== null); - expect(sliced).toHaveLength(4); - expect(sliced[0].startDate).toEqual(dailyCosts[0].startDate); - expect(sliced[3].startDate).toEqual(dailyCosts[3].startDate); - }); - it("generates appropriate time components for a time range", () => { - const [start, end] = getTimeRange(7); + const [start, end] = getDayAlignedRange(7); expect(start.getUTCHours()).toEqual(0); expect(start.getUTCHours()).toEqual(0); expect(start.getUTCMinutes()).toEqual(0); diff --git a/console/src/platform/billing/UsagePage.tsx b/console/src/platform/billing/UsagePage.tsx index 3fee621d538b7..7f213a86d93b6 100644 --- a/console/src/platform/billing/UsagePage.tsx +++ b/console/src/platform/billing/UsagePage.tsx @@ -19,34 +19,41 @@ import { Text, useTheme, } from "@chakra-ui/react"; -import React, { useMemo, useRef, useState } from "react"; +import React, { useRef, useState } from "react"; import { useCurrentOrganization } from "~/api/auth"; -import { DailyCosts, PlanType } from "~/api/cloudGlobalApi"; +import { PlanType } from "~/api/cloudGlobalApi"; import ErrorBox from "~/components/ErrorBox"; import { MainContentContainer } from "~/layouts/BaseLayout"; import { MaterializeTheme } from "~/theme"; import { nowUTC } from "~/util"; -import { formatCurrency } from "~/utils/format"; -import DailyUsageChart, { - chartHeightPx, - legendHeightPx, -} from "./DailyUsageChart"; +import AccountSpendBreakdown, { + AccountSpendLedger, +} from "./AccountSpendBreakdown"; import InvoiceTable from "./InvoiceTable"; -import { UpgradedPlanDetails } from "./PlanDetails"; -import { useDailyCosts, useRecentInvoices } from "./queries"; -import RegionSelect from "./RegionSelect"; -import SpendBreakdown from "./SpendBreakdown"; -import TimeRangeSelect from "./TimeRangeSelect"; -import { getTimeRangeSlice } from "./utils"; +import { AccountSpendPlanDetails } from "./PlanDetails"; +import { useDailyCostsBreakdown, useRecentInvoices } from "./queries"; const DEFAULT_TIME_RANGE_LOOKBACK_DAYS = 7; +// Fixed window behind the account plan-details box's "Last 30 days" row, +// independent of the time-range selector. +const LAST_30_DAYS_LOOKBACK = 30; -const InvoiceTableWrapper = ({ planType }: { planType: PlanType }) => { +// A leaf account's own invoice list is permanently empty (Orb invoices belong +// only to the billing owner, never split per child), so an empty result here +// means there's nothing to show, ever, not just "still loading." Hide the +// heading along with the table rather than leave a header with no rows. +const InvoiceHistorySection = ({ planType }: { planType: PlanType }) => { const { data: invoices, isLoading, isError, error } = useRecentInvoices(); + if (!isLoading && !isError && (invoices?.length ?? 0) === 0) { + return null; + } return ( <> + + Invoice history + {isLoading ? ( ) : isError ? ( @@ -94,188 +101,80 @@ const AwsMarketplaceInvoiceBanner = () => { ); }; -type ChartPanelProps = { - timeRange: number; - setTimeRange: (val: number) => void; - dailyCosts: DailyCosts["daily"] | null; - isLoading: boolean; - isError: boolean; - regionFilter: "all" | string; - setRegionFilter: (val: ChartPanelProps["regionFilter"]) => void; - error: Error | null; -}; - -function getTotalSpend( - dailyCosts: DailyCosts["daily"] | null, - region: "all" | string, -): number { - if (dailyCosts === null) { - return 0; - } - let totalPrice = 0; - for (const day of dailyCosts) { - for (const costKey of ["compute", "storage"] as const) { - for (const price of day.costs[costKey].prices) { - if (region === "all" || price.regionId === region) { - totalPrice += parseFloat(price.subtotal); - } - } - } - } - return totalPrice; -} - -const SelectorPanel = ({ - timeRange, - setTimeRange, - regionFilter, - setRegionFilter, -}: { - timeRange: number; - setTimeRange: (val: number) => void; - regionFilter: "all" | string; - setRegionFilter: (val: ChartPanelProps["regionFilter"]) => void; -}) => { - return ( - -
- -
-
- -
- - ); -}; - -const ChartPanel = ({ - dailyCosts, - regionFilter, - isLoading, - isError, - error, -}: ChartPanelProps) => { - const totalSpend = useMemo( - () => getTotalSpend(dailyCosts, regionFilter), - [dailyCosts, regionFilter], - ); - - return ( - - - - {formatCurrency(totalSpend)} - - - - {isLoading && ( - - )} - {isError && ( - - )} - {!isLoading && !isError && dailyCosts && ( - - )} - - - ); -}; - const UsagePage = () => { const { organization } = useCurrentOrganization(); const [regionFilter, setRegionFilter] = useState<"all" | string>("all"); - const [lastQueryTime, setLastQueryTime] = useState(nowUTC()); + const [lastQueryTime] = useState(nowUTC()); const [timeRangeFilter, setTimeRangeFilter] = useState( DEFAULT_TIME_RANGE_LOOKBACK_DAYS, ); const { - data: dailyCosts, - isLoading: isDailyCostsLoading, - isError: isDailyCostsError, - error: dailyCostsError, - } = useDailyCosts(timeRangeFilter, lastQueryTime); + data: costBreakdownDays, + isLoading: isCostBreakdownLoading, + isError: isCostBreakdownError, + error: costBreakdownError, + } = useDailyCostsBreakdown(timeRangeFilter, lastQueryTime); + // A separate fixed 30-day breakdown feeds the plan-details "Last 30 days" row. + // When the selected range is also 30 days this shares the query cache. + const { data: last30BreakdownDays } = useDailyCostsBreakdown( + LAST_30_DAYS_LOOKBACK, + lastQueryTime, + ); const chartTooltipRef = useRef(null); - const timeRangeCosts = useMemo( - () => getTimeRangeSlice(dailyCosts?.daily ?? null, timeRangeFilter), - [dailyCosts, timeRangeFilter], - ); return ( <> - - - - - + { - setTimeRangeFilter(range); - setLastQueryTime(nowUTC()); - }} - dailyCosts={timeRangeCosts} - isLoading={isDailyCostsLoading} - isError={isDailyCostsError} - error={dailyCostsError} + setTimeRange={setTimeRangeFilter} /> - - + - - + - - Invoice history - {organization?.subscription?.marketplace === "aws" ? ( - + <> + + Invoice history + + + ) : ( - )} diff --git a/console/src/platform/billing/UsageRoutes.test.tsx b/console/src/platform/billing/UsageRoutes.test.tsx new file mode 100644 index 0000000000000..255b1f73896fb --- /dev/null +++ b/console/src/platform/billing/UsageRoutes.test.tsx @@ -0,0 +1,171 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { render, screen, waitFor } from "@testing-library/react"; +import React from "react"; +import { Route, Routes } from "react-router-dom"; + +import { Organization } from "~/api/cloudGlobalApi"; +import { + buildCloudOrganizationsResponse, + buildInvoicesResponse, +} from "~/api/mocks/cloudGlobalApiHandlers"; +import server from "~/api/mocks/server"; +import { CloudRuntimeConfig } from "~/config/AppConfigSwitch"; +import { dummyValidUser } from "~/external-library-wrappers/__mocks__/frontegg"; +import type { + ITeamUserPermission, + User, +} from "~/external-library-wrappers/frontegg"; +import { + createProviderWrapper, + healthyEnvironment, + setFakeEnvironment, +} from "~/test/utils"; + +import UsageRoutes from "./UsageRoutes"; + +// `useCanViewBilling` requires this flag; its real value lives in +// LaunchDarkly, so mock it to isolate the invoices-based gate under test. +vi.mock("~/hooks/useFlags", () => ({ + useFlags: () => ({ "billing-ui-3756": true }), +})); + +// This test only exercises the tab strip / route gating in UsageRoutes +// itself, not the pages' own data fetching, so stub them out rather than +// mocking every endpoint UsagePage and BillingPage otherwise need. +vi.mock("./UsagePage", () => ({ + default: () =>
Usage Page
, +})); +vi.mock("./BillingPage", () => ({ + default: () =>
Billing Page
, +})); + +const adminUser: User = { + ...dummyValidUser, + permissions: [ + { key: "materialize.environment.write" } as ITeamUserPermission, + { key: "materialize.environment.read" } as ITeamUserPermission, + { key: "materialize.invoice.read" } as ITeamUserPermission, + ], +}; + +// `auth`/`authActions` aren't exercised by this test (UsageRoutes only reads +// `isImpersonating` and `user`); the shapes below just satisfy the type. +const runtimeConfig = { + isImpersonating: false as const, + user: adminUser, + auth: { tenantsState: { tenants: [] } }, + authActions: { switchTenant: vi.fn() }, +} as unknown as CloudRuntimeConfig; + +const stripePromise = Promise.resolve(null); + +const buildOrganization = ( + overrides: Partial = {}, +): Organization => ({ + id: "org_id", + name: "Test Organization", + blocked: false, + onboarded: true, + trialExpiresAt: null, + subscription: { type: "on-demand", marketplace: "direct" }, + ...overrides, +}); + +const oneInvoice = [ + { + issueDate: "2026-06-02T00:00:00Z", + currency: "usd", + total: "100.00", + amountDue: "100.00", + createdAt: "2026-06-02T00:00:00Z", + status: "paid", + invoiceNumber: "INV-001", + }, +]; + +const renderAt = async (path: string) => { + const Wrapper = await createProviderWrapper({ + initializeState: ({ set }) => + setFakeEnvironment(set, "aws/us-east-1", healthyEnvironment), + router: { type: "MEMORY_ROUTER", initialRouterEntries: [path] }, + }); + // Mirrors how AuthenticatedRoutes.tsx mounts UsageRoutes (nested under a + // "usage" prefix) so the leading-slash paths inside UsageRoutes ("/overview", + // "/billing") match against the right remainder of the URL. + return render( + + + + } + /> + + , + ); +}; + +describe("UsageRoutes", () => { + afterEach(() => { + server.resetHandlers(); + }); + + it("shows Billing for a paying org with invoices", async () => { + server.use( + buildCloudOrganizationsResponse({ payload: buildOrganization() }), + buildInvoicesResponse({ invoices: oneInvoice }), + ); + await renderAt("/usage/overview"); + + await waitFor(() => { + expect(screen.getByRole("link", { name: "Billing" })).toBeVisible(); + }); + }); + + it("shows Billing for an evaluation org even with no invoices yet", async () => { + server.use( + buildCloudOrganizationsResponse({ + payload: buildOrganization({ + subscription: { type: "evaluation", marketplace: "direct" }, + }), + }), + buildInvoicesResponse({ invoices: [] }), + ); + // Evaluation orgs have no Overview route (see isUpgradedPlan gating). + await renderAt("/usage/billing"); + + await waitFor(() => { + expect(screen.getByRole("link", { name: "Billing" })).toBeVisible(); + }); + }); + + it("hides Billing for a non-evaluation org with no invoices (e.g. a leaf account)", async () => { + server.use( + buildCloudOrganizationsResponse({ payload: buildOrganization() }), + buildInvoicesResponse({ invoices: [] }), + ); + await renderAt("/usage/overview"); + + // Billing reads as hidden from the very first render (invoices loading + // counts as "no invoices"), and stays hidden once the empty result loads. + expect(screen.queryByRole("link", { name: "Billing" })).toBeNull(); + await waitFor(() => { + expect( + screen.queryByRole("link", { name: "Billing" }), + ).not.toBeInTheDocument(); + }); + expect(screen.getByRole("link", { name: "Overview" })).toBeVisible(); + }); +}); diff --git a/console/src/platform/billing/UsageRoutes.tsx b/console/src/platform/billing/UsageRoutes.tsx index 32b0514575deb..7024df82f2894 100644 --- a/console/src/platform/billing/UsageRoutes.tsx +++ b/console/src/platform/billing/UsageRoutes.tsx @@ -7,7 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { HStack, Text, useTheme, VStack } from "@chakra-ui/react"; +import { Box, HStack, Text, useTheme, VStack } from "@chakra-ui/react"; import { Stripe } from "@stripe/stripe-js"; import React from "react"; import { Navigate, Outlet, Route } from "react-router-dom"; @@ -20,9 +20,17 @@ import { SentryRoutes } from "~/sentry"; import { MaterializeTheme } from "~/theme"; import BillingPage from "./BillingPage"; +import PricingPage from "./PricingPage"; +import { useRecentInvoices } from "./queries"; import UsagePage from "./UsagePage"; import { getIsUpgradedPlan } from "./utils"; +// Static reference material, not account-specific data, so it lives to the +// right of the account tabs rather than behind the (root-only) Billing tab. +const PRICING_TAB: Tab[] = [ + { label: "Pricing", href: "/usage/pricing", end: true }, +]; + const UsageLayout = ({ isBillingVisible }: { isBillingVisible: boolean }) => { const { colors } = useTheme(); const { organization } = useCurrentOrganization(); @@ -56,7 +64,30 @@ const UsageLayout = ({ isBillingVisible }: { isBillingVisible: boolean }) => { Contact support - + {/* Each PageTabStrip draws its underline only across its own width, + which would leave gaps in the strips' spacing and the row's + right padding. The same 1px inset line on the row itself keeps + it continuous. Bottom-align the strips (the active tab's 1px + borderBottom makes its strip 1px taller; centering would float + the other strip's underline off the row's) so all the underline + segments land on the same pixel row. */} + + + + + {/* A separate single-tab strip so Pricing shares the tab strip's + hover pill and active underline instead of hand-rolled hover + styling. */} + + + + @@ -74,7 +105,22 @@ const UsageRoutes = ({ const { organization } = useCurrentOrganization(); const isUpgradedPlan = getIsUpgradedPlan(organization?.subscription?.type); - const isBillingVisible = useCanViewBilling({ runtimeConfig }); + const { data: invoices } = useRecentInvoices(); + // Proxy for "is this a leaf account": a leaf's invoices always read as + // empty (SAS-148), same signal InvoiceHistorySection uses. Evaluation orgs + // are also invoice-less pre-upgrade, so carve those out to keep Billing + // reachable for a first-time "Upgrade & Pay". Known gap and the direct fix + // (an is_billing_child field, already written but undeployed): SAS-150. + // NOTE: this reads as "hide" while the query is loading, briefly hiding + // Billing/the route for a real billing owner on first load. Don't "fix" + // that by failing open on loading/error: the route below is gated on this + // same flag, and failing open can flip true->false for a leaf account + // already on /usage/billing, unmounting the route with no fallback and no + // way back. This direction (hidden -> visible, self-correcting) is safer. + const isBillingVisible = + useCanViewBilling({ runtimeConfig }) && + (organization?.subscription?.type === "evaluation" || + (invoices?.length ?? 0) > 0); const shouldRedirectToBilling = isBillingVisible && !isUpgradedPlan; @@ -93,6 +139,9 @@ const UsageRoutes = ({ } /> )} + {/* Static reference material, not account-specific, so it's reachable + regardless of plan type or billing-child status. */} + } /> = {}, +) { + return { + environment_id: "environment-acct-0", + cluster_grouping_key: "quickstart.r1", + category: "", + region: "aws/us-east-1", + amounts: { "price-compute": "10.00" }, + usage: 0, + ...overrides, + }; +} + +describe("aggregateDays", () => { + it("sums a cluster's amounts across days into the period total", () => { + const days: CostBreakdownDay[] = [ + { + startDate: "2026-06-01T00:00:00Z", + endDate: "2026-06-02T00:00:00Z", + accounts: [ + { + external_customer_id: "acct", + name: "", + clusters: [cluster({ amounts: { "price-compute": "10.00" } })], + }, + ], + }, + { + startDate: "2026-06-02T00:00:00Z", + endDate: "2026-06-03T00:00:00Z", + accounts: [ + { + external_customer_id: "acct", + name: "", + clusters: [cluster({ amounts: { "price-compute": "4.50" } })], + }, + ], + }, + ]; + + const { accounts } = aggregateDays(days); + expect(accounts).toHaveLength(1); + expect(accounts[0].clusters).toHaveLength(1); + expect(accounts[0].clusters[0].amounts["price-compute"]).toBe("14.5"); + expect(accountTotal(accounts[0])).toBeCloseTo(14.5); + }); + + it("keeps storage and egress (same empty cluster key) as distinct rows by category", () => { + const clusters = [ + cluster({ + cluster_grouping_key: "", + category: "Storage", + amounts: { "price-storage": "0.50" }, + }), + cluster({ + cluster_grouping_key: "", + category: "Egress", + amounts: { "price-egress": "0.25" }, + }), + ]; + const days: CostBreakdownDay[] = [ + { + startDate: "2026-06-01T00:00:00Z", + endDate: "2026-06-02T00:00:00Z", + accounts: [{ external_customer_id: "acct", name: "", clusters }], + }, + ]; + + const { accounts } = aggregateDays(days); + const categories = accounts[0].clusters.map((c) => c.category).sort(); + expect(categories).toEqual(["Egress", "Storage"]); + }); +}); + +describe("accountDailyTotals", () => { + it("returns a zero-filled per-day series for every account", () => { + const days: CostBreakdownDay[] = [ + { + startDate: "2026-06-01T00:00:00Z", + endDate: "2026-06-02T00:00:00Z", + accounts: [ + { + external_customer_id: "a", + name: "", + clusters: [cluster({ amounts: { c: "3.00" } })], + }, + ], + }, + { + // "b" first appears on day 2; "a" has no usage this day. + startDate: "2026-06-02T00:00:00Z", + endDate: "2026-06-03T00:00:00Z", + accounts: [ + { + external_customer_id: "b", + name: "", + clusters: [cluster({ amounts: { c: "7.00" } })], + }, + ], + }, + ]; + + const series = accountDailyTotals(days); + expect(series.get("a")).toEqual([3, 0]); + expect(series.get("b")).toEqual([0, 7]); + }); + + it("sums rather than overwrites if a day repeats the same account id", () => { + // The backend keys each day's accounts by id before returning them, so + // this shouldn't happen in practice, but the series must not silently + // undercount (and disagree with aggregateDays's total) if it ever does. + const days: CostBreakdownDay[] = [ + { + startDate: "2026-06-01T00:00:00Z", + endDate: "2026-06-02T00:00:00Z", + accounts: [ + { + external_customer_id: "a", + name: "", + clusters: [cluster({ amounts: { c: "3.00" } })], + }, + { + external_customer_id: "a", + name: "", + clusters: [cluster({ amounts: { c: "4.00" } })], + }, + ], + }, + ]; + + const series = accountDailyTotals(days); + expect(series.get("a")).toEqual([7]); + }); +}); + +describe("accountIdsByTotal / stackedDailyRows", () => { + const days: CostBreakdownDay[] = [ + { + startDate: "2026-06-01T00:00:00Z", + endDate: "2026-06-02T00:00:00Z", + accounts: [ + { + external_customer_id: "small", + name: "", + clusters: [cluster({ amounts: { c: "1.00" } })], + }, + { + external_customer_id: "big", + name: "", + clusters: [cluster({ amounts: { c: "5.00" } })], + }, + ], + }, + { + startDate: "2026-06-02T00:00:00Z", + endDate: "2026-06-03T00:00:00Z", + accounts: [ + { + external_customer_id: "big", + name: "", + clusters: [cluster({ amounts: { c: "5.00" } })], + }, + ], + }, + ]; + + it("orders account ids by descending period total", () => { + expect(accountIdsByTotal(accountDailyTotals(days))).toEqual([ + "big", + "small", + ]); + }); + + it("builds one zero-filled row per day, keyed by every account", () => { + const series = accountDailyTotals(days); + const ids = accountIdsByTotal(series); + expect(stackedDailyRows(days, ids, series)).toEqual([ + { startDate: "2026-06-01T00:00:00Z", big: 5, small: 1 }, + { startDate: "2026-06-02T00:00:00Z", big: 5, small: 0 }, + ]); + }); +}); + +describe("breakdownByAccount", () => { + it("breaks a tie in totals by account id, matching accountIdsByTotal's order", () => { + const days: CostBreakdownDay[] = [ + { + startDate: "2026-06-01T00:00:00Z", + endDate: "2026-06-02T00:00:00Z", + accounts: [ + { + external_customer_id: "b-account", + name: "", + clusters: [cluster({ amounts: { "price-compute": "10.00" } })], + }, + { + external_customer_id: "a-account", + name: "", + clusters: [cluster({ amounts: { "price-compute": "10.00" } })], + }, + ], + }, + ]; + + const result = breakdownByAccount(days); + + expect(result?.accounts.map((account) => account.id)).toEqual([ + "a-account", + "b-account", + ]); + }); +}); + +describe("pivotBreakdown", () => { + it("returns empty defaults for null days", () => { + expect(pivotBreakdown(null, "all")).toEqual({ + accountIds: [], + rows: [], + orderedAccounts: [], + series: new Map(), + totalSpend: 0, + }); + }); + + it("orders accounts biggest-spender first and sums to the period total", () => { + const days: CostBreakdownDay[] = [ + { + startDate: "2026-06-01T00:00:00Z", + endDate: "2026-06-02T00:00:00Z", + accounts: [ + { + external_customer_id: "small", + name: "", + clusters: [cluster({ amounts: { c: "1.00" } })], + }, + { + external_customer_id: "big", + name: "", + clusters: [cluster({ amounts: { c: "5.00" } })], + }, + ], + }, + ]; + + const result = pivotBreakdown(days, "all"); + + expect(result.accountIds).toEqual(["big", "small"]); + expect( + result.orderedAccounts.map((account) => account.external_customer_id), + ).toEqual(["big", "small"]); + expect(result.rows).toEqual([ + { startDate: "2026-06-01T00:00:00Z", big: 5, small: 1 }, + ]); + expect(result.series.get("big")).toEqual([5]); + expect(result.totalSpend).toBe(6); + }); + + it("restricts to a single region, matching filterDaysByRegion", () => { + const days: CostBreakdownDay[] = [ + { + startDate: "2026-06-01T00:00:00Z", + endDate: "2026-06-02T00:00:00Z", + accounts: [ + { + external_customer_id: "acct", + name: "", + clusters: [ + cluster({ region: "aws/us-east-1", amounts: { c: "3.00" } }), + cluster({ region: "aws/eu-west-1", amounts: { c: "4.00" } }), + ], + }, + ], + }, + ]; + + const result = pivotBreakdown(days, "aws/us-east-1"); + + expect(result.totalSpend).toBe(3); + }); +}); diff --git a/console/src/platform/billing/dailyBreakdown.ts b/console/src/platform/billing/dailyBreakdown.ts new file mode 100644 index 0000000000000..b5b4b14017806 --- /dev/null +++ b/console/src/platform/billing/dailyBreakdown.ts @@ -0,0 +1,269 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { + CostBreakdownAccount, + CostBreakdownCluster, + CostBreakdownDay, +} from "~/api/cloudGlobalApi"; + +/** + * Names aren't available yet (SAS-141/142), so accounts are labelled by their + * `external_customer_id` UUID. The full id is very wide for a chip or a table + * cell, so callers show this short prefix and keep the whole id in a tooltip. + */ +export function shortAccountId(accountId: string): string { + return `${accountId.slice(0, 8)}…`; +} + +/** Sum a cluster's per-price amounts (dollar strings) into a single total. */ +export function clusterTotal(amounts: { [priceId: string]: string }): number { + return Object.values(amounts).reduce( + (sum, amount) => sum + parseFloat(amount), + 0, + ); +} + +/** Total cost of an account across all of its clusters. */ +export function accountTotal(account: CostBreakdownAccount): number { + return account.clusters.reduce( + (sum, cluster) => sum + clusterTotal(cluster.amounts), + 0, + ); +} + +/** + * Uniquely identifies a cluster row within an account across days. Quoted + * Materialize identifiers can contain arbitrary characters, so this encodes + * the fields as a JSON array rather than joining with a delimiter that could + * itself appear inside one of them. + */ +function clusterKey(cluster: CostBreakdownCluster): string { + return JSON.stringify([ + cluster.environment_id, + cluster.cluster_grouping_key, + cluster.category, + cluster.region, + ]); +} + +/** + * Restrict each day's accounts/clusters to a single region, dropping any + * account left with no clusters. `"all"` returns `days` unchanged. + */ +export function filterDaysByRegion( + days: CostBreakdownDay[], + region: "all" | string, +): CostBreakdownDay[] { + if (region === "all") { + return days; + } + return days.map((day) => ({ + ...day, + accounts: day.accounts + .map((account) => ({ + ...account, + clusters: account.clusters.filter( + (cluster) => cluster.region === region, + ), + })) + .filter((account) => account.clusters.length > 0), + })); +} + +/** + * Collapse the per-day buckets into the period aggregate: one entry per + * account, whose clusters' per-price amounts are summed across every day in the + * window. Per-day cost is additive (SAS-134), so this reproduces the period + * total. Used for the per-account comparison table and the single-account + * cluster drilldown. + */ +export function aggregateDays(days: CostBreakdownDay[]): { + accounts: CostBreakdownAccount[]; +} { + const accounts = new Map< + string, + { name: string; clusters: Map } + >(); + for (const day of days) { + for (const account of day.accounts) { + let entry = accounts.get(account.external_customer_id); + if (!entry) { + // Every day bucket carries the same per-account name (SAS-141), so + // the first occurrence's name is as good as any other's. + entry = { name: account.name, clusters: new Map() }; + accounts.set(account.external_customer_id, entry); + } + for (const cluster of account.clusters) { + const key = clusterKey(cluster); + const existing = entry.clusters.get(key); + if (!existing) { + entry.clusters.set(key, { + ...cluster, + amounts: { ...cluster.amounts }, + }); + continue; + } + for (const [priceId, amount] of Object.entries(cluster.amounts)) { + const prev = parseFloat(existing.amounts[priceId] ?? "0"); + existing.amounts[priceId] = (prev + parseFloat(amount)).toString(); + } + } + } + } + return { + accounts: Array.from(accounts.entries()).map( + ([external_customer_id, { name, clusters }]) => ({ + external_customer_id, + name, + clusters: Array.from(clusters.values()), + }), + ), + }; +} + +/** + * Per-account total cost for each day in the window, in `days` order — the + * series behind the stacked-by-account chart and per-account trend sparklines. + * Every account that appears on any day gets a full-length series, zero-filled + * for days it had no usage. Accumulates rather than overwrites, matching + * `aggregateDays`, in case a day's `accounts` ever repeats the same id twice. + */ +export function accountDailyTotals( + days: CostBreakdownDay[], +): Map { + const series = new Map(); + days.forEach((day, dayIndex) => { + for (const account of day.accounts) { + let daily = series.get(account.external_customer_id); + if (!daily) { + daily = new Array(days.length).fill(0); + series.set(account.external_customer_id, daily); + } + daily[dayIndex] += accountTotal(account); + } + }); + return series; +} + +/** + * Account ids ordered by descending period total (ties broken by id for a + * stable order). Drives the stacked-chart layer order, the legend, and the + * comparison table's row order so the biggest spender is on top. Takes an + * already-computed `accountDailyTotals` series rather than `days` so callers + * that need both don't compute it twice. + */ +export function accountIdsByTotal(series: Map): string[] { + const totals = new Map(); + for (const [id, dailyTotals] of series) { + totals.set( + id, + dailyTotals.reduce((sum, amount) => sum + amount, 0), + ); + } + return Array.from(totals.keys()).sort((a, b) => { + const diff = (totals.get(b) ?? 0) - (totals.get(a) ?? 0); + return diff !== 0 ? diff : a.localeCompare(b); + }); +} + +/** + * Per-account spend over a breakdown window, biggest spender first (ties + * broken by id, matching `accountIdsByTotal`), plus the window total. Used to + * itemize the "Last 30 days" row by account. `null` if there's no data. + */ +export function breakdownByAccount( + days: CostBreakdownDay[] | null, +): { total: number; accounts: { id: string; total: number }[] } | null { + if (!days || days.length === 0) { + return null; + } + const accounts = aggregateDays(days) + .accounts.map((account) => ({ + id: account.external_customer_id, + total: accountTotal(account), + })) + .sort((a, b) => b.total - a.total || a.id.localeCompare(b.id)); + if (accounts.length === 0) { + return null; + } + const total = accounts.reduce((sum, account) => sum + account.total, 0); + return { total, accounts }; +} + +/** One stacked-chart row per day: the day plus each account's total for it. */ +export type StackedDailyRow = { startDate: string } & { + [accountId: string]: number | string; +}; + +/** + * Reshape the per-account daily series into one row per day keyed by account + * id — the row shape visx `BarStack` consumes (`keys` = `accountIds`). Every + * account is present on every row (zero-filled), so stacks line up across + * days. Takes an already-computed `accountDailyTotals` series rather than + * `days` so callers that need both don't compute it twice. + */ +export function stackedDailyRows( + days: CostBreakdownDay[], + accountIds: string[], + series: Map, +): StackedDailyRow[] { + return days.map((day, dayIndex) => { + const row: StackedDailyRow = { startDate: day.startDate }; + for (const id of accountIds) { + row[id] = series.get(id)?.[dayIndex] ?? 0; + } + return row; + }); +} + +/** + * Shared pivot of a breakdown window into everything the chart and the + * ledger render: account order (biggest spender first), stacked chart rows, + * per-account daily series, the aggregated accounts in that same order, and + * the period total. Computes `accountDailyTotals` exactly once and derives + * everything else from it, rather than each piece recomputing its own copy. + */ +export function pivotBreakdown( + days: CostBreakdownDay[] | null, + regionFilter: "all" | string, +): { + accountIds: string[]; + rows: StackedDailyRow[]; + orderedAccounts: CostBreakdownAccount[]; + series: Map; + totalSpend: number; +} { + if (!days) { + return { + accountIds: [], + rows: [], + orderedAccounts: [], + series: new Map(), + totalSpend: 0, + }; + } + const filteredDays = filterDaysByRegion(days, regionFilter); + const series = accountDailyTotals(filteredDays); + const accountIds = accountIdsByTotal(series); + const rows = stackedDailyRows(filteredDays, accountIds, series); + const aggregate = aggregateDays(filteredDays); + const orderedAccounts = accountIds + .map((id) => + aggregate.accounts.find((account) => account.external_customer_id === id), + ) + .filter( + (account): account is CostBreakdownAccount => account !== undefined, + ); + const totalSpend = orderedAccounts.reduce( + (sum, account) => sum + accountTotal(account), + 0, + ); + return { accountIds, rows, orderedAccounts, series, totalSpend }; +} diff --git a/console/src/platform/billing/queries.ts b/console/src/platform/billing/queries.ts index 2cba92856a8ee..d23cf513bfb4e 100644 --- a/console/src/platform/billing/queries.ts +++ b/console/src/platform/billing/queries.ts @@ -8,7 +8,6 @@ // by the Apache License, Version 2.0. import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { addDays, startOfDay, subDays } from "date-fns"; import { useMemo } from "react"; import { queryKeys as authQueryKeys } from "~/api/auth"; @@ -19,15 +18,13 @@ import { import { createStripeSetupIntent, detachPaymentMethod, + getCostsBreakdownDaily, getCredits, - getDailyCosts, Organization, recentInvoices, setDefaultPaymentMethod, } from "~/api/cloudGlobalApi"; -import { nowUTC } from "~/util"; - -import { ROLLING_AVG_TIME_RANGE_LOOKBACK_DAYS } from "./constants"; +import { addUtcDays, nowUTC } from "~/util"; export const creditBalanceQueryKeys = { all: () => buildGlobalQueryKey("credits"), @@ -37,11 +34,11 @@ export const invoiceQueryKeys = { all: () => buildGlobalQueryKey("invoices"), }; -export const dailyCostQueryKeys = { - all: () => buildGlobalQueryKey("dailyCosts"), +export const costBreakdownDailyQueryKeys = { + all: () => buildGlobalQueryKey("costBreakdownDaily"), list: (timeSpan: number, queryTime: Date) => [ - ...dailyCostQueryKeys.all(), + ...costBreakdownDailyQueryKeys.all(), buildQueryKeyPart("list", { timeSpan, queryTime, @@ -76,31 +73,33 @@ export function useRecentInvoices() { }); } -export function getTimeRange(span: number): [Date, Date] { - // Some fields on the usage page, such as the rolling average in the plan - // details component, requires a certain number of days' worth of data. That - // span of time may be greater than what is being visually filtered, so set - // the minimum-queried range to what the rolling average needs. - span = Math.max(span, ROLLING_AVG_TIME_RANGE_LOOKBACK_DAYS); - const endDate = startOfDay(addDays(nowUTC(), 1)); - endDate.setUTCHours(0, 0, 0, 0); - const startDate = new Date( - subDays(endDate, span) - // Since we're bucketing these by day, start from the beginning of the - // UTC day. - .setUTCHours(0, 0, 0, 0), - ); - return [startDate, endDate]; +// `date-fns`'s `startOfDay`/`addDays` read and set calendar fields in the +// browser's local time zone, not UTC. Near UTC midnight, "local tomorrow" can +// actually be UTC "today" (or "the day after tomorrow"), silently shifting the +// whole window by a day. `addUtcDays` stays entirely within UTC calendar +// arithmetic instead. +export function getDayAlignedRange(span: number): [Date, Date] { + const now = nowUTC(); + return [addUtcDays(now, 1 - span), addUtcDays(now, 1)]; } -export function useDailyCosts(timeSpan: number, queryTime: Date) { +/** + * Per-account, per-cluster breakdown bucketed by UTC day (the Direction-B + * source). Returns the dense `days` array from `/api/costs/breakdown/daily`; + * callers pivot it via `dailyBreakdown.ts` into the roll-up / per-account views. + */ +export function useDailyCostsBreakdown(timeSpan: number, queryTime: Date) { return useQuery({ - queryKey: dailyCostQueryKeys.list(timeSpan, queryTime), + queryKey: costBreakdownDailyQueryKeys.list(timeSpan, queryTime), queryFn: async ({ queryKey, signal }) => { const [_, { timeSpan: queryTimeSpan }] = queryKey; - const [startDate, endDate] = getTimeRange(queryTimeSpan); - const response = await getDailyCosts(startDate, endDate, { signal }); - return response.data; + // Day buckets require a UTC-midnight-aligned window, which + // getDayAlignedRange already produces. + const [startDate, endDate] = getDayAlignedRange(queryTimeSpan); + const response = await getCostsBreakdownDaily(startDate, endDate, { + signal, + }); + return response.data.days; }, }); } diff --git a/console/src/platform/billing/types.ts b/console/src/platform/billing/types.ts index 77a74b5844bd8..bb4090d57e7ca 100644 --- a/console/src/platform/billing/types.ts +++ b/console/src/platform/billing/types.ts @@ -7,30 +7,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -import { DailyCostKey } from "~/api/cloudGlobalApi"; - export type RegionGroupedSummary = { total: number; regions: Map; }; - -type ResourceItem = { - usageValue: number; - usageUnits: string; - usagePoints: number[]; - totalCost: number; -}; - -export type ResourceMeasurement = ResourceItem & { - sort: number; - rate: string; -}; - -export type ResourceBreakdown = { - [k in DailyCostKey]: { - total: ResourceItem; - resources: Map; - }; -}; - -export type RegionResourceBreakdown = Map; diff --git a/console/src/platform/billing/utils.ts b/console/src/platform/billing/utils.ts index 6a458c12b09a3..4951ec6bc0435 100644 --- a/console/src/platform/billing/utils.ts +++ b/console/src/platform/billing/utils.ts @@ -8,99 +8,14 @@ // by the Apache License, Version 2.0. import { UTCDate } from "@date-fns/utc"; -import { addDays, addMonths, parseISO, startOfMonth, subDays } from "date-fns"; +import { addMonths, parseISO, startOfMonth, subDays } from "date-fns"; -import { - DailyCostKey, - DailyCosts, - Organization, - Prices, -} from "~/api/cloudGlobalApi"; +import { DailyCosts, Organization } from "~/api/cloudGlobalApi"; import { components } from "~/api/schemas/global-api"; import { CloudRegion } from "~/store/cloudRegions"; -import { assert } from "~/util"; -import { formatDateInUtc, formatUtcIso } from "~/utils/dateFormat"; -import { - costUnits, - replicaSorts, - ROLLING_AVG_TIME_RANGE_LOOKBACK_DAYS, -} from "./constants"; -import { RegionGroupedSummary, RegionResourceBreakdown } from "./types"; - -export function aggregateByDay(data: DailyCosts["daily"]) { - const days = new Map(); - for (const slice of data) { - const startDate = parseISO(slice.startDate); - const formattedDate = formatDateInUtc(startDate); - let day = days.get(formattedDate); - if (!day) { - startDate.setUTCHours(0, 0, 0, 0); - day = { - costs: structuredClone(slice.costs), - startDate: formatUtcIso(startDate), - endDate: formatUtcIso(addDays(startDate, 1)), - subtotal: slice.subtotal, - total: slice.total, - }; - } else { - day = rollupSlice(day, slice); - } - days.set(formattedDate, day); - } - return Array.from(days.values()); -} - -function rollupSlice( - day: DailyCosts["daily"][0], - slice: DailyCosts["daily"][0], -): DailyCosts["daily"][0] { - day.subtotal = sumStringFloats(day.subtotal, slice.subtotal); - day.total = sumStringFloats(day.total, slice.total); - for (const costKey of ["storage", "compute"] as const) { - day.costs[costKey].subtotal = sumStringFloats( - day.costs[costKey].subtotal, - slice.costs[costKey].subtotal, - ); - day.costs[costKey].total = sumStringFloats( - day.costs[costKey].total, - slice.costs[costKey].total, - ); - for (const slicePrice of slice.costs[costKey].prices) { - const dayPrice = day.costs[costKey].prices.find((p) => { - return ( - p.regionId === slicePrice.regionId && - // compare replicaIds for compute prices, otherwise just regionId - ("replicaId" in slicePrice && "replicaId" in p - ? slicePrice.replicaId === p.replicaId - : true) - ); - }); - if (dayPrice) { - dayPrice.subtotal = sumStringFloats( - dayPrice.subtotal, - slicePrice.subtotal, - ); - // If the rate is changed, poison the unit amount so we don't attempt - // to do any calculations based on it. if we end up needing this we can - // update the data structure. - dayPrice.unitAmount = - dayPrice.unitAmount === slicePrice.unitAmount - ? dayPrice.unitAmount - : "-1"; - } else { - (day.costs[costKey].prices as Prices[]).push( - structuredClone(slicePrice), - ); - } - } - } - return day; -} - -function sumStringFloats(left: string, right: string): string { - return (parseFloat(left) + parseFloat(right)).toString(); -} +import { ROLLING_AVG_TIME_RANGE_LOOKBACK_DAYS } from "./constants"; +import { RegionGroupedSummary } from "./types"; export function summarizePlanCosts( dailyCosts: DailyCosts["daily"] | null, @@ -167,141 +82,6 @@ export function summarizePlanCosts( return { spanSummary, last30Summary }; } -export function getTimeRangeSlice( - dailyCosts: DailyCosts["daily"] | null, - timeRangeFilter: number, -): DailyCosts["daily"] | null { - if (dailyCosts === null || dailyCosts.length <= timeRangeFilter) { - return dailyCosts; - } - const lastStartDate = parseISO(dailyCosts[dailyCosts.length - 1].startDate); - // Determine the starting date for the time range (subtract one because - // we're starting from the beginning of the last day). - const firstStartDate = subDays(lastStartDate, timeRangeFilter - 1); - let startOffset = -1 * timeRangeFilter; - // Seed the list with the known-minimum number of records from the tail. - const slices = dailyCosts.slice(startOffset); - startOffset--; - // ... then walk backwards from where we left off, prepending all records that - // fall on or after the start of the time range. Since we're counting back - // from the end of the Array, this is an inclusive guard (otherwise we'd omit - // the last element when calling `.at(-n)`). - while (Math.abs(startOffset) <= dailyCosts.length) { - const slice = dailyCosts.at(startOffset); - if (!slice) break; - const sliceStart = parseISO(slice.startDate); - if (firstStartDate.getTime() <= sliceStart.getTime()) { - slices.unshift(slice); - } else { - break; - } - startOffset--; - } - return slices; -} - -export function summarizeResourceCosts( - dailyCosts: DailyCosts["daily"], - totalDays: number, -) { - const breakdown: RegionResourceBreakdown = new Map(); - let previousDate: string | null = null; - let offset = 0; - for (const day of dailyCosts) { - // If a plan is changed mid-day, we may have multiple slices per day. Only - // advance the the index when a change occurs. - const startDate = formatDateInUtc(parseISO(day.startDate)); - if (previousDate === null) { - previousDate = startDate; - } - // If this slice starts on a new day, advance the day offset. - if (previousDate !== startDate) { - previousDate = startDate; - offset++; - } - summarizeDayCost(breakdown, day.costs, offset, "compute", totalDays); - summarizeDayCost(breakdown, day.costs, offset, "storage", totalDays); - } - return breakdown; -} - -function summarizeDayCost( - breakdown: RegionResourceBreakdown, - costs: DailyCosts["daily"][0]["costs"], - dayIx: number, - costKey: DailyCostKey, - totalDays: number, -) { - for (const priceBlock of costs[costKey].prices) { - if (!breakdown.has(priceBlock.regionId)) { - breakdown.set(priceBlock.regionId, getEmptyBreakdown(totalDays)); - } - const regionSummary = breakdown.get(priceBlock.regionId); - assert(regionSummary); - const costSummary = regionSummary[costKey]; - const priceCost = parseFloat(priceBlock.subtotal); - const creditsCost = priceCost / parseFloat(priceBlock.unitAmount); - // Add this cost value (e.g., storage or compute) to the region-wide total for the cost category - costSummary.total.totalCost += priceCost; - costSummary.total.usageValue += creditsCost; - costSummary.total.usagePoints[dayIx] += priceCost; - const resourceType = - "replicaSize" in priceBlock ? priceBlock.replicaSize : "Storage"; - if (!costSummary.resources.has(resourceType)) { - costSummary.resources.set(resourceType, { - usagePoints: Array(costSummary.total.usagePoints.length).fill(0), - totalCost: 0, - sort: getReplicaSort(resourceType), - usageUnits: costUnits[costKey], - usageValue: 0, - rate: "0.00", - }); - } - // Add this cost value (e.g., storage or compute) to the region-wide total - // for the resource (e.g., `xsmall` replica) - const resourceSummary = costSummary.resources.get(resourceType); - assert(resourceSummary); - resourceSummary.usageValue += creditsCost; - resourceSummary.totalCost += priceCost; - resourceSummary.usagePoints[dayIx] += priceCost; - resourceSummary.rate = priceBlock.unitAmount; - } -} - -function getReplicaSort(replicaName: string): number { - if (replicaName.includes("cc")) { - const size = replicaName.split("cc", 1)[0]; - return parseInt(size); - } else if (replicaName.includes("C")) { - const size = replicaName.split("C", 1)[0]; - return parseInt(size) * 100; - } - return replicaSorts.get(replicaName) ?? 0; -} - -function getEmptyBreakdown(days: number) { - return { - compute: { - total: { - usageValue: 0, - usageUnits: costUnits.compute, - usagePoints: Array(days).fill(0), - totalCost: 0, - }, - resources: new Map(), - }, - storage: { - total: { - usageValue: 0, - usageUnits: costUnits.storage, - usagePoints: Array(days).fill(0), - totalCost: 0, - }, - resources: new Map(), - }, - }; -} - export function getIsUpgradedPlan( planType: components["schemas"]["PlanType"] | undefined, ): planType is Exclude { diff --git a/console/src/util.test.ts b/console/src/util.test.ts new file mode 100644 index 0000000000000..bd40d0ae9ddae --- /dev/null +++ b/console/src/util.test.ts @@ -0,0 +1,32 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { describe, expect, it } from "vitest"; + +import { addUtcDays } from "./util"; + +describe("addUtcDays", () => { + it("adds days across a UTC calendar boundary regardless of local time zone", () => { + // 2026-01-01T23:30:00Z: late in the UTC day, so a naive local-time + // implementation could roll to the wrong day depending on the runner's + // time zone. + const date = new Date("2026-01-01T23:30:00Z"); + expect(addUtcDays(date, 1).toISOString()).toBe("2026-01-02T00:00:00.000Z"); + }); + + it("subtracts days with a negative argument", () => { + const date = new Date("2026-03-01T00:00:00Z"); + expect(addUtcDays(date, -1).toISOString()).toBe("2026-02-28T00:00:00.000Z"); + }); + + it("rolls across a month/year boundary", () => { + const date = new Date("2025-12-31T12:00:00Z"); + expect(addUtcDays(date, 1).toISOString()).toBe("2026-01-01T00:00:00.000Z"); + }); +}); diff --git a/console/src/util.ts b/console/src/util.ts index 0d3d22ee6d096..fc14d774fe537 100644 --- a/console/src/util.ts +++ b/console/src/util.ts @@ -211,6 +211,22 @@ export function nowUTC(): Date { ); } +/** + * UTC midnight of the day `days` after `date`'s UTC calendar day (negative + * `days` for before). Pure calendar arithmetic (`Date.UTC` tolerates a day + * value outside 1-31, rolling into the adjacent month), not millisecond + * addition, so this stays correct regardless of `date`'s time component. + */ +export function addUtcDays(date: Date, days: number): Date { + return new Date( + Date.UTC( + date.getUTCFullYear(), + date.getUTCMonth(), + date.getUTCDate() + days, + ), + ); +} + export function exhaustiveGuard(val: never): never { throw new Error(`Unhandled switch case: ${val}`); } diff --git a/console/src/utils/format.ts b/console/src/utils/format.ts index a65076cc62944..2149bef5228a6 100644 --- a/console/src/utils/format.ts +++ b/console/src/utils/format.ts @@ -215,8 +215,8 @@ export function formatDurationForAxis(durationMs: number): string { } } -export const formatPercentage = (percentage: number) => - `${(percentage * 100).toFixed(2)}%`; +export const formatPercentage = (percentage: number, decimals = 2) => + `${(percentage * 100).toFixed(decimals)}%`; /** Units exposed in human-readable duration controls (filters, inputs). */ export type DurationUnit = "seconds" | "minutes" | "hours"; diff --git a/console/src/utils/graph.test.ts b/console/src/utils/graph.test.ts new file mode 100644 index 0000000000000..98bfd3b4d2c00 --- /dev/null +++ b/console/src/utils/graph.test.ts @@ -0,0 +1,39 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { describe, expect, it } from "vitest"; + +import { buildXYGraphLayoutProps } from "./graph"; + +describe("buildXYGraphLayoutProps", () => { + it("clamps inner width/height to 0 rather than going negative", () => { + // ParentSize renders once with width/height 0 before its ResizeObserver + // reports the real size. A negative value here would flow into an SVG + // rect's width/height attribute (e.g. GraphEventOverlay), which throws. + const { graphEventOverlayProps } = buildXYGraphLayoutProps({ + width: 0, + height: 0, + margin: { top: 10, right: 0, bottom: 36, left: 50 }, + }); + + expect(graphEventOverlayProps.width).toBe(0); + expect(graphEventOverlayProps.height).toBe(0); + }); + + it("computes the normal positive layout when width/height are non-zero", () => { + const { graphEventOverlayProps } = buildXYGraphLayoutProps({ + width: 300, + height: 200, + margin: { top: 10, right: 0, bottom: 36, left: 50 }, + }); + + expect(graphEventOverlayProps.width).toBe(250); + expect(graphEventOverlayProps.height).toBe(154); + }); +}); diff --git a/console/src/utils/graph.ts b/console/src/utils/graph.ts index 53e63c882522a..5412d1a98eda6 100644 --- a/console/src/utils/graph.ts +++ b/console/src/utils/graph.ts @@ -157,8 +157,12 @@ export function buildXYGraphLayoutProps({ height: number; margin: { top: number; right: number; bottom: number; left: number }; }) { - const innerGraphWidth = width - margin.left - margin.right; - const innerGraphHeight = height - margin.top - margin.bottom; + // Clamped to 0: ParentSize renders once with width/height 0 before its + // ResizeObserver reports the real size, and a negative value here would + // flow into an SVG rect's width/height attribute (e.g. GraphEventOverlay), + // which throws. + const innerGraphWidth = Math.max(0, width - margin.left - margin.right); + const innerGraphHeight = Math.max(0, height - margin.top - margin.bottom); // Uses the SVG coordinate system (ascending top to bottom) const innerGraphTop = margin.top; From 61cec161249428c3807eddea6a93e8cf9faf023e Mon Sep 17 00:00:00 2001 From: MkByte <18468315+martykulma@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:53:21 -0400 Subject: [PATCH 08/15] SS-356: docs: document AWS Glue Schema Registry support for Kafka sinks (#37694) Update the user documentation with AWS Glue schema registry for Kafka Sink. - Document schema-name and compatibility-level options and default values - List Confluent to Glue compatibility level mapping - Document set-if-unset compatibility behavior shared with Confluent Schema Registry - Update IAM permissions in `CREATE CONNECTION` to include sink This will be available in the v26.34 release --------- Co-authored-by: Claude Opus 4.8 (1M context) --- doc/user/content/serve-results/sink/kafka.md | 4 +- doc/user/content/sql/create-connection.md | 48 +++++- doc/user/content/sql/create-sink/kafka.md | 166 ++++++++++++++++++- doc/user/data/examples/create_connection.yml | 6 +- doc/user/data/examples/create_sink_kafka.yml | 69 ++++++-- doc/user/data/examples/explain_schema.yml | 2 +- 6 files changed, 257 insertions(+), 38 deletions(-) diff --git a/doc/user/content/serve-results/sink/kafka.md b/doc/user/content/serve-results/sink/kafka.md index d4e11fc9ae667..beaf46889a2da 100644 --- a/doc/user/content/serve-results/sink/kafka.md +++ b/doc/user/content/serve-results/sink/kafka.md @@ -109,8 +109,8 @@ By default, Materialize assigns a partition to each message using the following strategy: 1. Encode the message's key in the specified format. - 2. If the format uses a Confluent Schema Registry, strip out the - schema ID from the encoded bytes. + 2. If the format uses a schema registry (Confluent or AWS Glue), strip out + the header carrying the schema ID from the encoded bytes. 3. Hash the remaining encoded bytes using [SeaHash]. 4. Divide the hash value by the topic's partition count and assign the remainder as the message's partition. diff --git a/doc/user/content/sql/create-connection.md b/doc/user/content/sql/create-connection.md index f27c5287dfa8e..f9d1910c9b4e1 100644 --- a/doc/user/content/sql/create-connection.md +++ b/doc/user/content/sql/create-connection.md @@ -625,8 +625,8 @@ CREATE CONNECTION csr_ssh TO CONFLUENT SCHEMA REGISTRY ( An AWS Glue Schema Registry connection establishes a link to an [AWS Glue Schema Registry]. You can use AWS Glue Schema Registry connections in the `FORMAT` -clause of [`CREATE SOURCE`] statements to decode Avro-encoded messages whose -schemas are managed in AWS Glue. +clause of [`CREATE SOURCE`] statements to decode Avro-encoded messages, and of +[`CREATE SINK`] statements to encode them, with schemas managed in AWS Glue. The connection authenticates to AWS through a separate [AWS connection](#aws), which supplies the credentials and region. See [AWS](#aws) for how to grant @@ -651,15 +651,21 @@ CREATE CONNECTION glue_conn TO AWS GLUE SCHEMA REGISTRY ( #### Permissions {#glue-permissions} -The IAM role assumed by the [AWS connection](#aws) must be allowed to read -schemas from the registry. Materialize uses the following AWS Glue actions: +The IAM role assumed by the [AWS connection](#aws) must be allowed to access the +registry. Sources only read. Sinks also register schemas. Materialize uses the +following AWS Glue actions: | Action | When it is used | |--------|-----------------| | `glue:GetRegistry` | At connection creation, to validate the connection. Only required when `VALIDATE` is `true` (the default). | -| `glue:GetSchemaVersion` | When a source is created, to pin the schema, and at runtime, to fetch the writer schema for each new schema version encountered. Always required. | +| `glue:GetSchemaVersion` | Source: when the source is created, to pin the schema, and at runtime, to fetch the writer schema for each new schema version encountered. Sink: to poll a newly registered schema version until it becomes available. Always required. | +| `glue:GetSchemaByDefinition` | Sink, to find an existing version matching the schema being registered. | +| `glue:RegisterSchemaVersion` | Sink, to add a version to an existing schema. | +| `glue:CreateSchema` | Sink, to create a schema on its first publish. | +| `glue:GetSchema` | Sink, to read an existing schema's compatibility level and warn when it differs from the requested one. Optional. | -A least-privilege policy scoped to a single registry looks like: +A least-privilege policy for a source, scoped to a single registry, grants only +the read actions: ```json { @@ -680,9 +686,35 @@ A least-privilege policy scoped to a single registry looks like: } ``` +A sink additionally needs the schema-write actions on the same resources: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "glue:GetRegistry", + "glue:GetSchemaVersion", + "glue:GetSchemaByDefinition", + "glue:RegisterSchemaVersion", + "glue:CreateSchema", + "glue:GetSchema" + ], + "Resource": [ + "arn:aws:glue:::registry/", + "arn:aws:glue:::schema//*" + ] + } + ] +} +``` + If you create the connection with `WITH (VALIDATE = false)`, you can omit -`glue:GetRegistry` and grant only `glue:GetSchemaVersion`. For details on -creating and authorizing the AWS connection, see [AWS](#aws). +`glue:GetRegistry`. The registry itself must already exist. Materialize sinks +create and version schemas within it but never create the registry. For details +on creating and authorizing the AWS connection, see [AWS](#aws). ### MySQL diff --git a/doc/user/content/sql/create-sink/kafka.md b/doc/user/content/sql/create-sink/kafka.md index 14d497820dd01..f17fd4cb46dad 100644 --- a/doc/user/content/sql/create-sink/kafka.md +++ b/doc/user/content/sql/create-sink/kafka.md @@ -119,7 +119,7 @@ For example, consider a table with two columns named `col-a` and `col@a`. Materialize will use the names `col_a` and `col_a1`, respectively, in the generated Avro schema. -When using a Confluent Schema Registry: +#### Using Confluent Schema Registry * Materialize will automatically publish Avro schemas for the key, if present, and the value to the registry. @@ -134,6 +134,56 @@ When using a Confluent Schema Registry: * You can [add `doc` fields](#avro-schema-documentation) to the Avro schemas. + * You can set the compatibility level for a subject with the `KEY COMPATIBILITY + LEVEL` and `VALUE COMPATIBILITY LEVEL` [options](#schema-compatibility-levels). + The level is applied only when the subject has no compatibility level yet. If + the subject already has one, it is left unchanged. + +#### Using [AWS Glue Schema Registry](/sql/create-connection/#aws-glue-schema-registry) + +{{< private-preview />}} + + * Materialize registers Avro schemas for the key, if present, and the value in + the registry named by the connection. The registry must already exist. + + * By default, each schema is named after the topic (`-value`, and + `-key` when a key is present). You can override these with the + `KEY SCHEMA NAME` and `VALUE SCHEMA NAME` options. + + * You can set the compatibility level applied to a newly created schema with + the `KEY COMPATIBILITY LEVEL` and `VALUE COMPATIBILITY LEVEL` + [options](#syntax), defaulting to `BACKWARD` when omitted. The level is set + only when Materialize creates the schema. If the schema already exists, the + sink adds a new version to it and leaves its compatibility level unchanged. + + * The `AVRO ... FULLNAME`, `NULL DEFAULTS`, and `doc` options are not supported. + +##### Compatibility levels + +To specify compatibility levels, use the Confluent compatibility level names shown +below. The table also shows the equivalent AWS Glue compatibility levels for +reference. AWS Glue's `DISABLED` compatibility level has no equivalent Confluent +compatibility level and is not supported. + +Compatibility level | AWS Glue equivalent +--------------------|-------------------- +`BACKWARD` | `BACKWARD` +`BACKWARD_TRANSITIVE` | `BACKWARD_ALL` +`FORWARD` | `FORWARD` +`FORWARD_TRANSITIVE` | `FORWARD_ALL` +`FULL` | `FULL` +`FULL_TRANSITIVE` | `FULL_ALL` +`NONE` | `NONE` + + +#### Publishing Schemas + +With either registry, Materialize publishes the schemas when the sink starts +running, not when you run `CREATE SINK`. The schema names and definitions are not +validated against the registry at `CREATE SINK` time, so registry errors (for +example a name that collides with an incompatible existing schema) surface once +the sink starts publishing rather than at creation. + SQL types are converted to Avro types according to the following conversion table: @@ -392,8 +442,8 @@ By default, Materialize assigns a partition to each message using the following strategy: 1. Encode the message's key in the specified format. - 2. If the format uses a Confluent Schema Registry, strip out the - schema ID from the encoded bytes. + 2. If the format uses a schema registry (Confluent or AWS Glue), strip out + the header carrying the schema ID from the encoded bytes. 3. Hash the remaining encoded bytes using [SeaHash]. 4. Divide the hash value by the topic's partition count and assign the remainder as the message's partition. @@ -602,7 +652,11 @@ CREATE CONNECTION kafka_connection TO KAFKA ( {{< /tab >}} {{< /tabs >}} -#### Confluent Schema Registry +#### Schema Registry + + +{{< tabs tabID="1" >}} +{{< tab "Confluent" >}} {{< tabs tabID="1" >}} {{< tab "SSL">}} @@ -638,12 +692,31 @@ CREATE CONNECTION csr_basic_http {{< /tab >}} {{< /tabs >}} +{{< /tab >}} +{{< tab "AWS Glue" >}} + +{{< private-preview />}} + +```mzsql +CREATE CONNECTION aws_conn TO AWS ( + ASSUME ROLE ARN = 'arn:aws:iam::123456789000:role/MaterializeGlue' +); + +CREATE CONNECTION glue_conn TO AWS GLUE SCHEMA REGISTRY ( + AWS CONNECTION = aws_conn, + REGISTRY = 'my-registry' +); +``` + +{{< /tab >}} +{{< /tabs >}} + ### Creating a sink #### Upsert envelope {{< tabs >}} -{{< tab "Avro">}} +{{< tab "Avro Confluent">}} ```mzsql CREATE SINK avro_sink @@ -653,6 +726,29 @@ CREATE SINK avro_sink FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_connection ENVELOPE UPSERT; ``` +{{< /tab >}} +{{< tab "Avro AWS Glue">}} + +{{< private-preview />}} + +The registry named by the connection must already exist. The IAM role assumed +by the AWS connection must have the schema-write permissions listed under [AWS +Glue Schema Registry](/sql/create-connection/#aws-glue-schema-registry). + +```mzsql +CREATE SINK glue_sink + IN CLUSTER my_io_cluster + FROM + INTO KAFKA CONNECTION kafka_connection ( + TOPIC 'test_avro_topic' + ) + KEY (key_col) + FORMAT AVRO USING AWS GLUE SCHEMA REGISTRY CONNECTION glue_conn + ENVELOPE UPSERT; +``` + +See [Avro](#avro) for the schema-name defaults and how compatibility levels map +to their AWS Glue equivalents. {{< /tab >}} {{< tab "JSON">}} @@ -672,7 +768,7 @@ CREATE SINK json_sink #### Debezium envelope {{< tabs >}} -{{< tab "Avro">}} +{{< tab "Avro Confluent">}} ```mzsql CREATE SINK avro_sink @@ -681,6 +777,31 @@ CREATE SINK avro_sink FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_connection ENVELOPE DEBEZIUM; ``` + +{{< /tab >}} +{{< tab "Avro AWS Glue">}} + +{{< private-preview />}} +The registry named by the connection must already exist. The IAM role assumed +by the AWS connection must have the schema-write permissions listed under [AWS +Glue Schema Registry](/sql/create-connection/#aws-glue-schema-registry). + +```mzsql +CREATE SINK glue_sink + IN CLUSTER my_io_cluster + FROM + INTO KAFKA CONNECTION kafka_connection ( + TOPIC 'test_avro_topic' + ) + KEY (key_col) + FORMAT AVRO USING AWS GLUE SCHEMA REGISTRY CONNECTION glue_conn + ENVELOPE DEBEZIUM; +``` + +See [Avro](#avro) for the schema-name defaults and how compatibility levels map +to their AWS Glue equivalents. + + {{< /tab >}} {{< /tabs >}} @@ -703,12 +824,16 @@ CREATE SINK custom_topic_sink #### Schema compatibility levels + +{{< tabs >}} +{{< tab "Avro Confluent">}} + ```mzsql CREATE SINK compatibility_level_sink IN CLUSTER my_io_cluster FROM INTO KAFKA CONNECTION kafka_connection ( - TOPIC 'test_avro_topic', + TOPIC 'test_avro_topic' ) FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_connection ( KEY COMPATIBILITY LEVEL 'BACKWARD', @@ -717,8 +842,35 @@ CREATE SINK compatibility_level_sink ENVELOPE UPSERT; ``` +{{< /tab >}} +{{< tab "Avro AWS Glue">}} + +{{< private-preview />}} +```mzsql +CREATE SINK compatibility_level_sink + IN CLUSTER my_io_cluster + FROM + INTO KAFKA CONNECTION kafka_connection ( + TOPIC 'test_avro_topic' + ) + KEY (key_col) + FORMAT AVRO USING AWS GLUE SCHEMA REGISTRY CONNECTION glue_conn ( + KEY COMPATIBILITY LEVEL 'BACKWARD', + VALUE COMPATIBILITY LEVEL 'FULL' + ) + ENVELOPE UPSERT; +``` + +See [Avro](#avro) for the schema-name defaults and how compatibility levels map +to their AWS Glue equivalents. + + +{{< /tab >}} +{{< /tabs >}} + #### Documentation comments + Consider the following sink, `docs_sink`, built on top of a relation `t` with several [SQL comments](/sql/comment-on) attached. diff --git a/doc/user/data/examples/create_connection.yml b/doc/user/data/examples/create_connection.yml index 90b794e5a41af..57486bb95b9cd 100644 --- a/doc/user/data/examples/create_connection.yml +++ b/doc/user/data/examples/create_connection.yml @@ -381,8 +381,10 @@ description: | *Value:* `text`. Required. - The name of the AWS Glue Schema Registry to read schemas from (for - example, `default-registry`). Must not be empty. + The name of the AWS Glue Schema Registry to use (for example, + `default-registry`). Sources read schemas from the registry, and sinks + register schemas in it. The registry must already exist. Must not be + empty. - name: "`WITH ()`" description: | The following `` are supported: diff --git a/doc/user/data/examples/create_sink_kafka.yml b/doc/user/data/examples/create_sink_kafka.yml index d01483037c5e6..121c089421d8d 100644 --- a/doc/user/data/examples/create_sink_kafka.yml +++ b/doc/user/data/examples/create_sink_kafka.yml @@ -15,16 +15,25 @@ ) [KEY ( [, ...] ) [NOT ENFORCED]] [HEADERS ] - FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION [ - ( - [AVRO KEY FULLNAME ''] - [, AVRO VALUE FULLNAME ''] - [, NULL DEFAULTS ] - [, DOC ON [, ...]] - [, KEY COMPATIBILITY LEVEL ''] - [, VALUE COMPATIBILITY LEVEL ''] - ) - ] + FORMAT AVRO + USING CONFLUENT SCHEMA REGISTRY CONNECTION [ + ( + [AVRO KEY FULLNAME ''] + [, AVRO VALUE FULLNAME ''] + [, NULL DEFAULTS ] + [, DOC ON [, ...]] + [, KEY COMPATIBILITY LEVEL ''] + [, VALUE COMPATIBILITY LEVEL ''] + ) + ] + | USING AWS GLUE SCHEMA REGISTRY CONNECTION [ + ( + [KEY SCHEMA NAME ''] + [, VALUE SCHEMA NAME ''] + [, KEY COMPATIBILITY LEVEL ''] + [, VALUE COMPATIBILITY LEVEL ''] + ) + ] [ENVELOPE DEBEZIUM | UPSERT] [WITH (SNAPSHOT = )] syntax_elements: @@ -76,18 +85,34 @@ - name: "**FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION** ``" description: | Encode messages using Avro format with schemas published to the Confluent Schema Registry. + - name: "**FORMAT AVRO USING AWS GLUE SCHEMA REGISTRY CONNECTION** ``" + description: | + ***Private preview.** This feature is under active development.* + + Encode messages using Avro format with schemas registered in the AWS + Glue Schema Registry named by the [AWS Glue Schema Registry + connection](/sql/create-connection/#aws-glue-schema-registry). The + registry must already exist. Only the `KEY SCHEMA NAME`, `VALUE SCHEMA + NAME`, `KEY COMPATIBILITY LEVEL`, and `VALUE COMPATIBILITY LEVEL` + options are supported. See [Avro](#avro) for details. - name: "**AVRO KEY FULLNAME** `''`" description: | - Optional. Default: `row`. Sets the Avro fullname on the generated key schema, if a `KEY` is specified. When used, a value must be specified for `AVRO VALUE FULLNAME`. + Optional, Confluent Schema Registry only. Default: `row`. Sets the Avro fullname on the generated key schema, if a `KEY` is specified. When used, a value must be specified for `AVRO VALUE FULLNAME`. - name: "**AVRO VALUE FULLNAME** `''`" description: | - Optional. Default: `envelope`. Sets the Avro fullname on the generated value schema. When `KEY` is specified, `AVRO KEY FULLNAME` must additionally be specified. + Optional, Confluent Schema Registry only. Default: `envelope`. Sets the Avro fullname on the generated value schema. When `KEY` is specified, `AVRO KEY FULLNAME` must additionally be specified. - name: "**NULL DEFAULTS** ``" description: | - Optional. Default: `false`. Whether to automatically default nullable fields to `null` in the generated schemas. + Optional, Confluent Schema Registry only. Default: `false`. Whether to automatically default nullable fields to `null` in the generated schemas. + - name: "**KEY SCHEMA NAME** `''`" + description: | + Optional, AWS Glue Schema Registry only. The name under which the key schema is registered, if a `KEY` is specified. Default: `-key`. + - name: "**VALUE SCHEMA NAME** `''`" + description: | + Optional, AWS Glue Schema Registry only. The name under which the value schema is registered. Default: `-value`. - name: "**DOC ON** `` [, ...]" description: | - Optional. Add a documentation comment to the generated Avro schemas. + Optional, Confluent Schema Registry only. Add a documentation comment to the generated Avro schemas. | Option | Description | |--------|-------------| @@ -102,10 +127,10 @@ on how documentation comments are added to the generated Avro schemas. - name: "**KEY COMPATIBILITY LEVEL** `''`" description: | - Optional. If specified, set the [Compatibility Level](https://docs.confluent.io/platform/7.6/schema-registry/fundamentals/schema-evolution.html#schema-evolution-and-compatibility) for the generated key schema to one of: `BACKWARD`, `BACKWARD_TRANSITIVE`, `FORWARD`, `FORWARD_TRANSITIVE`, `FULL`, `FULL_TRANSITIVE`, `NONE`. + Optional. If specified, set the [Compatibility Level](https://docs.confluent.io/platform/7.6/schema-registry/fundamentals/schema-evolution.html#schema-evolution-and-compatibility) for the generated key schema to one of: `BACKWARD`, `BACKWARD_TRANSITIVE`, `FORWARD`, `FORWARD_TRANSITIVE`, `FULL`, `FULL_TRANSITIVE`, `NONE`. With AWS Glue, the level is applied only when the sink creates the schema and maps to the [Glue equivalent](#avro). - name: "**VALUE COMPATIBILITY LEVEL** `''`" description: | - Optional. If specified, set the [Compatibility Level](https://docs.confluent.io/platform/7.6/schema-registry/fundamentals/schema-evolution.html#schema-evolution-and-compatibility) for the generated value schema to one of: `BACKWARD`, `BACKWARD_TRANSITIVE`, `FORWARD`, `FORWARD_TRANSITIVE`, `FULL`, `FULL_TRANSITIVE`, `NONE`. + Optional. If specified, set the [Compatibility Level](https://docs.confluent.io/platform/7.6/schema-registry/fundamentals/schema-evolution.html#schema-evolution-and-compatibility) for the generated value schema to one of: `BACKWARD`, `BACKWARD_TRANSITIVE`, `FORWARD`, `FORWARD_TRANSITIVE`, `FULL`, `FULL_TRANSITIVE`, `NONE`. With AWS Glue, the level is applied only when the sink creates the schema and maps to the [Glue equivalent](#avro). - name: "**ENVELOPE** ``" description: | Optional. Specifies how changes to the sink's upstream relation are mapped to Kafka messages. Valid envelope types: @@ -316,6 +341,14 @@ -- [, VALUE COMPATIBILITY LEVEL ''] -- ) -- ] + -- | AVRO USING AWS GLUE SCHEMA REGISTRY CONNECTION [ + -- ( + -- [KEY SCHEMA NAME ''] + -- [, VALUE SCHEMA NAME ''] + -- [, KEY COMPATIBILITY LEVEL ''] + -- [, VALUE COMPATIBILITY LEVEL ''] + -- ) + -- ] -- | JSON | TEXT | BYTES [ENVELOPE DEBEZIUM | UPSERT] [WITH (SNAPSHOT = )] @@ -367,10 +400,10 @@ Optional. A column containing headers to add to each Kafka message emitted by the sink. The column must be of type `map[text => text]` or `map[text => bytea]`. See [Headers](#headers) for details. - name: "**KEY FORMAT** ``" description: | - Set the key encoding explicitly. Supported formats: `AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION `, `JSON`, `TEXT`, `BYTES`. + Set the key encoding explicitly. Supported formats: `AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION `, `AVRO USING AWS GLUE SCHEMA REGISTRY CONNECTION `, `JSON`, `TEXT`, `BYTES`. - name: "**VALUE FORMAT** ``" description: | - Set the value encoding explicitly. Supported formats: `AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION `, `JSON`, `TEXT`, `BYTES`. + Set the value encoding explicitly. Supported formats: `AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION `, `AVRO USING AWS GLUE SCHEMA REGISTRY CONNECTION `, `JSON`, `TEXT`, `BYTES`. - name: "**ENVELOPE** ``" description: | Optional. Specifies how changes to the sink's upstream relation are mapped to Kafka messages. Valid envelope types: diff --git a/doc/user/data/examples/explain_schema.yml b/doc/user/data/examples/explain_schema.yml index 426a83fc540af..d1ba1a4cc6bff 100644 --- a/doc/user/data/examples/explain_schema.yml +++ b/doc/user/data/examples/explain_schema.yml @@ -70,7 +70,7 @@ Optional. A column containing headers to add to each Kafka message emitted by the sink. The column must be of type `map[text => text]` or `map[text => bytea]`. See [Headers](/sql/create-sink/kafka/#headers) for details. - name: "**FORMAT** ``" description: | - Optional. Specifies the format to use for both keys and values: `AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION `, `JSON`, `TEXT`, or `BYTES`. See [Formats](/sql/create-sink/kafka/#formats) for details. + Optional. Specifies the format to use for both keys and values: `AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION `, `AVRO USING AWS GLUE SCHEMA REGISTRY CONNECTION `, `JSON`, `TEXT`, or `BYTES`. See [Formats](/sql/create-sink/kafka/#formats) for details. - name: "**KEY FORMAT** `` **VALUE FORMAT** ``" description: | Optional. Specifies the key format and value formats separately. See [Formats](/sql/create-sink/kafka/#formats) for details. From 5b70402d78742337ddfd0b6090f93101f50e6435 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Wed, 22 Jul 2026 16:02:28 +0200 Subject: [PATCH 09/15] ci: Nightly fixes (2026-07-20) (#37746) Based on https://buildkite.com/materialize/nightly/builds/17443 --- .../materialize/parallel_workload/parallel_workload.py | 7 +++++-- test/launchdarkly-flag-consistency/mzcompose.py | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/misc/python/materialize/parallel_workload/parallel_workload.py b/misc/python/materialize/parallel_workload/parallel_workload.py index 358359b1fe90e..3bdecddac148f 100644 --- a/misc/python/materialize/parallel_workload/parallel_workload.py +++ b/misc/python/materialize/parallel_workload/parallel_workload.py @@ -515,8 +515,10 @@ def print_stats( # attempt when it is rare and the run is DDL-heavy, e.g. SwapSchema), and # the replacement lifecycle refusing a second or a sealed replacement # (ReplaceMaterializedView, whose replacements pile up unfinalized in the - # workload). Tracked separately so such actions don't trip the - # broken-action assertion below. + # workload). "unknown cluster 'dont_exist'" comes from FlipFlags setting the + # default cluster to a nonexistent one to hunt for panics, so any action + # needing a cluster fails with it while that flag is live. Tracked + # separately so such actions don't trip the broken-action assertion below. noise = { "must be owner of", "permission denied for", @@ -524,6 +526,7 @@ def print_stats( "another session modified the catalog while this DDL transaction was open", "because it already has a replacement", "is sealed and thus cannot be replaced", + "unknown cluster 'dont_exist'", } if scenario == Scenario.Rename: # Concurrent renames invalidate the qualified names an action captured diff --git a/test/launchdarkly-flag-consistency/mzcompose.py b/test/launchdarkly-flag-consistency/mzcompose.py index 3dcd0997cf86d..ee11e4d22f5a1 100644 --- a/test/launchdarkly-flag-consistency/mzcompose.py +++ b/test/launchdarkly-flag-consistency/mzcompose.py @@ -538,6 +538,7 @@ KNOWN_CROSS_ENV_DIVERGENCES: set[str] = set(""" allow_real_time_recency allowed_cluster_replica_sizes + column_paged_batcher_budget_fraction compute_dataflow_max_inflight_bytes compute_peek_response_stash_threshold_bytes compute_subscribe_snapshot_optimization From ebe86ac88851638d2b7a6d2c9bee3fa1e4d3a3cb Mon Sep 17 00:00:00 2001 From: MkByte <18468315+martykulma@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:25:08 -0400 Subject: [PATCH 10/15] SS-354: doc: update user docs to provide Azure SQL specific guidance (#37550) Updates Materialize documentation with instructions on configuring Azure SQL as a source. The functionality has been merged, and is expected to GA 7/22. There are some minor differences in role setup, and Azure requires connection encryption. Because of those differences, some of the sections are inlined (SSH tunnel still requires that SSL Mode is set for the Azure connection, and connection policy cannot redirect). --- .../headless/sql-server-deployment-guides.md | 9 + .../content/ingest-data/sql-server/_index.md | 1 + .../ingest-data/sql-server/azure-db.md | 349 ++++++++++++++++++ .../sql/create-source/sql-server-v2.md | 6 +- .../content/sql/create-source/sql-server.md | 6 +- 5 files changed, 369 insertions(+), 2 deletions(-) create mode 100644 doc/user/content/headless/sql-server-deployment-guides.md create mode 100644 doc/user/content/ingest-data/sql-server/azure-db.md diff --git a/doc/user/content/headless/sql-server-deployment-guides.md b/doc/user/content/headless/sql-server-deployment-guides.md new file mode 100644 index 0000000000000..7856f68e96a5c --- /dev/null +++ b/doc/user/content/headless/sql-server-deployment-guides.md @@ -0,0 +1,9 @@ +--- +headless: true +--- +{{% note %}} +The exact configuration steps depend on your SQL Server deployment. For +step-by-step instructions, see the integration guides for +[Azure SQL Database](/ingest-data/sql-server/azure-db/) and +[self-hosted or managed SQL Server](/ingest-data/sql-server/self-hosted/). +{{% /note %}} diff --git a/doc/user/content/ingest-data/sql-server/_index.md b/doc/user/content/ingest-data/sql-server/_index.md index e226bee1525c9..1b36022f37524 100644 --- a/doc/user/content/ingest-data/sql-server/_index.md +++ b/doc/user/content/ingest-data/sql-server/_index.md @@ -37,6 +37,7 @@ Data Capture (CDC) support. ## Integration Guides +- [Azure SQL Database](/ingest-data/sql-server/azure-db/) - [Self-hosted SQL Server](/ingest-data/sql-server/self-hosted/) ## Considerations diff --git a/doc/user/content/ingest-data/sql-server/azure-db.md b/doc/user/content/ingest-data/sql-server/azure-db.md new file mode 100644 index 0000000000000..5f744bab87286 --- /dev/null +++ b/doc/user/content/ingest-data/sql-server/azure-db.md @@ -0,0 +1,349 @@ +--- +title: "Ingest data from Azure SQL Database" +description: "How to stream data from Azure SQL Database to Materialize" +menu: + main: + parent: "sql-server" + name: "Azure SQL Database" + identifier: "sql-server-azure-db" + weight: 70 +--- + +This page shows you how to stream data from [Azure SQL Database](https://azure.microsoft.com/en-us/products/azure-sql/database) +to Materialize using the [SQL Server source](/sql/create-source/sql-server/). + +{{< note >}} +This guide covers **Azure SQL Database**, the single-database service. For +**Azure SQL Managed Instance**, which runs a SQL Server Agent and exposes +`msdb`, follow the [self-hosted SQL Server guide](/ingest-data/sql-server/self-hosted/) +instead. +{{}} + +{{< tip >}} +{{< guided-tour-blurb-for-ingest-data >}} +{{< /tip >}} + +## Before you begin + +- Make sure Change Data Capture (CDC) is available on your Azure SQL Database. + CDC has compute requirements and is not supported on lower service tiers. See + [Azure SQL documentation](https://learn.microsoft.com/en-us/azure/azure-sql/database/change-data-capture-overview?view=azuresql) + for details on service tiers and CDC configuration. + +- Ensure you have access to your database via the [`sqlcmd` client](https://learn.microsoft.com/en-us/sql/tools/sqlcmd/sqlcmd-utility), + or your preferred SQL client, as a member of `db_owner`. + +## A. Configure Azure SQL Database + +{{< note >}} + +To configure Azure SQL Database for data ingestion into Materialize, you must +connect to the database you want to replicate as a member of `db_owner`, which +can enable CDC and create/manage the user, role, and privileges. + +{{}} + +### 1. Create a Materialize user in Azure SQL Database. + +Azure SQL Database is a single-database service. Because it does not provide +reusable server logins or access to the master database for granting +server-scoped permissions, create a [contained database user](https://learn.microsoft.com/en-us/sql/relational-databases/security/contained-database-users-making-your-database-portable) directly in the +database you want to replicate. + +Connect to the database you want to replicate as a member of `db_owner`, then +create the user (replace `` with your own password): + +```sql +CREATE USER materialize WITH PASSWORD = ''; +``` + +Create a gating role for the capture instances and add the user to it: + +```sql +CREATE ROLE materialize_role; +ALTER ROLE materialize_role ADD MEMBER materialize; +``` + +Grant the privileges Materialize needs: + +```sql +-- SELECT on the replicated tables and the CDC change tables. +ALTER ROLE db_datareader ADD MEMBER materialize; + +-- Read access to the transaction-state views used to track replication +-- progress, in place of the server-scoped VIEW SERVER STATE used for +-- self-hosted SQL Server. +GRANT VIEW DATABASE STATE TO materialize; +``` + +{{< note >}} +Unlike self-hosted SQL Server, no explicit grants are issued on the +`sys.fn_cdc_*` functions or the `INFORMATION_SCHEMA` views. They are executable +and readable by default. +{{}} + +### 2. Enable Change-Data-Capture for the database. + +Azure SQL Database drives CDC from an internal scheduler, so **no SQL Server +Agent is required**. Enabling CDC requires the database to be on a service tier +that supports it and that you are a member of `db_owner`. + +Connect to the database you want to replicate and run: + +```sql +EXEC sys.sp_cdc_enable_db; +``` + +For guidance on enabling Change Data Capture on Azure SQL Database, see the +[Azure documentation](https://learn.microsoft.com/en-us/sql/relational-databases/track-changes/about-change-data-capture-sql-server?view=azuresqldb-current). + +### 3. Enable `SNAPSHOT` transaction isolation. + +Enable `SNAPSHOT` transaction isolation for the database. Because Azure SQL +Database connections cannot switch databases, use the `CURRENT` keyword to target +the connected database: + +```sql +ALTER DATABASE CURRENT SET ALLOW_SNAPSHOT_ISOLATION ON; +``` + +### 4. Enable Change-Data-Capture for the tables. + +Enable Change Data Capture for each table you wish to replicate, gated by the +role you created above (replace `` and `` with your +schema and table names): + +```sql +EXEC sys.sp_cdc_enable_table + @source_schema = '', + @source_name = '', + @role_name = 'materialize_role', + @supports_net_changes = 0; +``` + +## B. (Optional) Configure network security + +{{< note >}} +If you are prototyping and your Azure SQL Database is publicly accessible, **you +can skip this step**. For production scenarios, we recommend configuring one of +the network security options below. +{{< /note >}} + +There are various ways to configure your database's network to allow Materialize +to connect: + +- **Allow Materialize IPs:** If your database is publicly accessible, you can + configure your database's firewall to allow connections from a set of + static Materialize IP addresses. + +- **Use an SSH tunnel:** If your database is running in a private network, you + can use an SSH tunnel to connect Materialize to the database. + +Select the option that works best for you. + +{{< tabs >}} + +{{< tab "Allow Materialize IPs">}} + +1. In the [SQL Shell](/console/), or your preferred SQL + client connected to Materialize, find the static egress IP addresses for the + Materialize region you are running in: + + ```mzsql + SELECT * FROM mz_egress_ips; + ``` + +1. Update your [Azure SQL Database firewall rules](https://learn.microsoft.com/en-us/azure/azure-sql/database/firewall-configure?view=azuresql) + to allow traffic from each IP address from the previous step. + +{{< /tab >}} + +{{< tab "Use an SSH tunnel">}} + +This assumes your Azure SQL Database is already reachable over a private IP in a +virtual network via an [Azure Private Endpoint](https://learn.microsoft.com/en-us/azure/azure-sql/database/private-endpoint-overview?view=azuresql), +with the `privatelink.database.windows.net` DNS zone integrated so +`.database.windows.net` resolves to the private IP. + +To create the SSH tunnel, you launch an instance to serve as an SSH bastion host +in that network and configure the bastion host to allow traffic from Materialize. +The bastion forwards traffic to the database's private endpoint. + +1. [Launch a Linux VM with a static public IP address](https://learn.microsoft.com/en-us/azure/virtual-machines/linux/quick-create-portal) +to serve as your SSH bastion host. + + - Make sure the VM is publicly accessible and in the same virtual network as + the private endpoint (or a peered network). + - Add a key pair and note the username. You'll use this username when + connecting Materialize to your bastion host. + - Make sure the VM has a static public IP address. You'll use this IP + address when connecting Materialize to your bastion host. + +1. Configure the SSH bastion host to allow traffic from Materialize. + + 1. In the [SQL Shell](/console/), or your preferred + SQL client connected to Materialize, get the static egress IP addresses for + the Materialize region you are running in: + + ```mzsql + SELECT * FROM mz_egress_ips; + ``` + + 1. Update your SSH bastion host's [firewall rules](https://learn.microsoft.com/en-us/azure/virtual-network/tutorial-filter-network-traffic?toc=%2Fazure%2Fvirtual-machines%2Ftoc.json) + to allow SSH traffic from each IP address from the previous step. + +1. Set the server's [connection policy](https://learn.microsoft.com/en-us/azure/azure-sql/database/connectivity-architecture?view=azuresql#connection-policy) + to **Proxy**: + + ```sh + az sql server conn-policy update \ + --resource-group \ + --server \ + --connection-type Proxy + ``` + + With the `Redirect` policy, the gateway tells the client to reconnect + directly to the backend node on a high port, which bypasses the single port + the SSH tunnel forwards. `Proxy` keeps all traffic on the gateway at port + 1433, which is the port the tunnel forwards. + + If the connection policy is left as `Redirect`, creating the source or + validating the connection fails with an error like: + + ```text + Server requested a connection to an alternative address: + `.worker.database.windows.net:` + ``` + +{{< /tab >}} + +{{< /tabs >}} + +## C. Ingest data in Materialize + +### 1. (Optional) Create a cluster + +{{< note >}} +If you are prototyping and already have a cluster to host your SQL Server +source (e.g. `quickstart`), **you can skip this step**. For production +scenarios, we recommend separating your workloads into multiple clusters for +[resource isolation](/sql/create-cluster/#resource-isolation). +{{< /note >}} + +{{% sql-server-direct/create-a-cluster %}} + +### 2. Create a connection + +Once you have configured your network, create a connection in Materialize per +your networking configuration. Azure SQL Database **requires an encrypted +connection**, so the SQL Server connection must specify `SSL MODE 'required'`. + +{{< tabs >}} + +{{< tab "Allow Materialize IPs">}} +{{% sql-server-direct/ingesting-data/allow-materialize-ips %}} +{{< /tab >}} + +{{< tab "Use an SSH tunnel">}} + +1. In the [SQL Shell](/console/), or your preferred SQL + client connected to Materialize, use the [`CREATE CONNECTION`](/sql/create-connection/#ssh-tunnel) + command to create an SSH tunnel connection: + + ```mzsql + CREATE CONNECTION ssh_connection TO SSH TUNNEL ( + HOST '', + PORT , + USER '' + ); + ``` + + - Replace `` and `` with the public IP + address and port of the SSH bastion host you created + [earlier](#b-optional-configure-network-security). + + - Replace `` with the username for the key pair you created + for your SSH bastion host. + +1. Get Materialize's public keys for the SSH tunnel connection: + + ```mzsql + SELECT * FROM mz_ssh_tunnel_connections; + ``` + +1. Log in to your SSH bastion host and add Materialize's public keys to the + `authorized_keys` file, for example: + + ```sh + # Command for Linux + echo "ssh-ed25519 AAAA...76RH materialize" >> ~/.ssh/authorized_keys + echo "ssh-ed25519 AAAA...hLYV materialize" >> ~/.ssh/authorized_keys + ``` + +1. Back in the SQL client connected to Materialize, validate the SSH tunnel + connection you created using the [`VALIDATE CONNECTION`](/sql/validate-connection) + command: + + ```mzsql + VALIDATE CONNECTION ssh_connection; + ``` + + If no validation error is returned, move to the next step. + +1. Use the [`CREATE SECRET`](/sql/create-secret/) command to securely store the + password for the `materialize` user + [you created](#1-create-a-materialize-user-in-azure-sql-database): + + ```mzsql + CREATE SECRET sqlserver_pass AS ''; + ``` + +1. Use the [`CREATE CONNECTION`](/sql/create-connection/) command to create the + SQL Server connection, routed through the SSH tunnel. Azure SQL Database + requires an encrypted connection, so include `SSL MODE 'required'`: + + ```mzsql + CREATE CONNECTION sqlserver_connection TO SQL SERVER ( + HOST '', + PORT 1433, + USER 'materialize', + PASSWORD SECRET sqlserver_pass, + DATABASE '', + SSL MODE 'required', + SSH TUNNEL ssh_connection + ); + ``` + + - Replace `` with your Azure SQL Database endpoint, and `` + with the database you'd like to connect to. + +{{< /tab >}} + +{{< /tabs >}} + +### 3. Start ingesting data + +{{< note >}} +For a new SQL Server source, if none of the replicating tables +are receiving write queries, snapshotting may take up to an additional 5 minutes +to complete. For details, see [snapshot latency for inactive databases](#snapshot-latency-for-inactive-databases) +{{}} + +{{% include-example file="examples/ingest_data/sql_server/create_source_cloud" example="create-source" %}} + +{{% include-example file="examples/ingest_data/sql_server/create_source_cloud" example="create-source-options" %}} + +{{% include-example file="examples/ingest_data/sql_server/create_source_cloud" +example="schema-changes" %}} + +### 4. Right-size the cluster + +{{% sql-server-direct/right-size-the-cluster %}} + +## D. Explore your data + +{{% sql-server-direct/next-steps %}} + +## Considerations + +{{% include-headless "/headless/sql-server-considerations" %}} diff --git a/doc/user/content/sql/create-source/sql-server-v2.md b/doc/user/content/sql/create-source/sql-server-v2.md index 92bbb0dab3967..28addf9972b85 100644 --- a/doc/user/content/sql/create-source/sql-server-v2.md +++ b/doc/user/content/sql/create-source/sql-server-v2.md @@ -32,6 +32,8 @@ for the database that you would like to replicate. Server](#prerequisite-creating-a-connection-to-sql-server) in Materialize. - The connection setup depends on your network security configuration. +{{% include-headless "/headless/sql-server-deployment-guides" %}} + ## Syntax {{% include-syntax file="examples/create_source_sql_server" example="syntax" %}} @@ -154,7 +156,9 @@ an SSH bastion server to accept connections from Materialize, check #### Creating the source in Materialize -You **must** enable Change Data Capture, see [Enable Change Data Capture SQL Server Instructions](/ingest-data/sql-server/self-hosted/#a-configure-sql-server). +You **must** enable Change Data Capture. See the setup instructions for +[Azure SQL Database](/ingest-data/sql-server/azure-db/#a-configure-azure-sql-database) +or [self-hosted SQL Server](/ingest-data/sql-server/self-hosted/#a-configure-sql-server). Once CDC is enabled for all of the tables you wish to create subsources for, you can create a `SOURCE` in Materialize to begin replicating data! diff --git a/doc/user/content/sql/create-source/sql-server.md b/doc/user/content/sql/create-source/sql-server.md index 4c194f0340f2c..314be876a5358 100644 --- a/doc/user/content/sql/create-source/sql-server.md +++ b/doc/user/content/sql/create-source/sql-server.md @@ -23,6 +23,8 @@ for the database that you would like to replicate. Then [create a connection](#c in Materialize that specifies access and authentication parameters. {{% /create-source/intro %}} +{{% include-headless "/headless/sql-server-deployment-guides" %}} + ## Syntax {{% include-syntax file="examples/create_source_sql_server_legacy" example="syntax" %}} @@ -168,7 +170,9 @@ an SSH bastion server to accept connections from Materialize, check ### Creating a source {#create-source-example} -You **must** enable Change Data Capture, see [Enable Change Data Capture SQL Server Instructions](/ingest-data/sql-server/self-hosted/#a-configure-sql-server). +You **must** enable Change Data Capture. See the setup instructions for +[Azure SQL Database](/ingest-data/sql-server/azure-db/#a-configure-azure-sql-database) +or [self-hosted SQL Server](/ingest-data/sql-server/self-hosted/#a-configure-sql-server). Once CDC is enabled for all of the relevant tables, you can create a `SOURCE` in Materialize to begin replicating data! From 1ba5c6966e32ad26394685ad0cea411192758a80 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Wed, 22 Jul 2026 16:30:05 +0200 Subject: [PATCH 11/15] ci: Disable launchdarkly-flag-consistency test while I'm out (#37804) It's been quite noisy. Alternative would be for someone else to look at it every day. @antiguru Would you want to do that? --- ci/mkpipeline.py | 20 +++++++++++++++++++- ci/nightly/pipeline.template.yml | 1 + 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/ci/mkpipeline.py b/ci/mkpipeline.py index d40eac5f20e11..359dd669d1a4e 100644 --- a/ci/mkpipeline.py +++ b/ci/mkpipeline.py @@ -56,6 +56,24 @@ # err on the side of including too much rather than too little. CI_GLUE_GLOBS = ["bin", "ci", "misc/python/materialize/cli/ci_annotate_errors.py"] + +def ci_glue_globs(current_pipeline: str) -> list[str]: + """The glue globs relevant to `current_pipeline`, i.e. CI_GLUE_GLOBS with + the `pipeline.template.yml` of every *other* pipeline excluded. + + A pipeline template only affects its own pipeline, so a change to, say, the + Nightly template must not count as glue for the test pipeline and force it + to run every step untrimmed. The current pipeline's own template stays in + scope: a template edit can change a step's command in ways the input-based + trimming cannot detect.""" + ci_dir = Path(__file__).parent + return CI_GLUE_GLOBS + [ + f":(exclude)ci/{template.parent.name}/pipeline.template.yml" + for template in sorted(ci_dir.glob("*/pipeline.template.yml")) + if template.parent.name != current_pipeline + ] + + DEFAULT_AGENT = "hetzner-aarch64-4cpu-8gb" @@ -318,7 +336,7 @@ def fetch_hashes() -> None: print("Coverage/Sanitizer build, not trimming pipeline") elif os.environ["BUILDKITE_BRANCH"] == "main" or os.environ["BUILDKITE_TAG"]: print("On main branch or tag, so not trimming pipeline") - elif have_paths_changed(CI_GLUE_GLOBS): + elif have_paths_changed(ci_glue_globs(args.pipeline)): # We still execute pipeline trimming on a copy of the pipeline to # protect against bugs in the pipeline trimming itself. print("[DRY RUN] Trimming unchanged steps from pipeline") diff --git a/ci/nightly/pipeline.template.yml b/ci/nightly/pipeline.template.yml index 655c0ae37f885..52964f2a29bf5 100644 --- a/ci/nightly/pipeline.template.yml +++ b/ci/nightly/pipeline.template.yml @@ -1602,6 +1602,7 @@ steps: - ./ci/plugins/mzcompose: composition: launchdarkly-flag-consistency branches: "main" + skip: "TODO: Reenable when Dennis is back" - id: launchdarkly-reconnect label: "LaunchDarkly reconnect" From f543f7ba5ca56c77a46a597ce6df8100e603ebe0 Mon Sep 17 00:00:00 2001 From: Kynan Rilee Date: Wed, 22 Jul 2026 10:35:34 -0400 Subject: [PATCH 12/15] [SS-348] implement Batches' `for_each_diff_pair` as an iterator (to support async operations) (#37759) We want to write rows to S3 as soon as we read them from the batch instead of queuing them in memory for the next operator (`write_data_files`). That means the body of our `for_each_diff_pair` must support async operations. This PR refactors `for_each_diff_pair` into an iterator so we can do that (in the stacked PR). --- src/interchange/src/envelopes.rs | 282 +++++++++++++++++++++++++------ 1 file changed, 232 insertions(+), 50 deletions(-) diff --git a/src/interchange/src/envelopes.rs b/src/interchange/src/envelopes.rs index 23b8c1d53b6b3..7cd3470800954 100644 --- a/src/interchange/src/envelopes.rs +++ b/src/interchange/src/envelopes.rs @@ -19,80 +19,125 @@ use mz_ore::cast::CastFrom; use mz_repr::{ CatalogItemId, ColumnName, Datum, Diff, Row, RowPacker, SqlColumnType, SqlScalarType, }; +use timely::progress::Antichain; use crate::avro::DiffPair; /// Walks `batch` and invokes `on_diff_pair` for each `DiffPair` at each /// `(key, timestamp)`. /// +/// Thin wrapper around `iter_diff_pairs`. +pub fn for_each_diff_pair(batch: &B, mut on_diff_pair: F) +where + B: BatchReader