[pull] main from MaterializeInc:main

.agents/skills/mz-debug-ci/SKILL.md

@@ -24,6 +24,18 @@ If either tool is missing or unauthenticated, **stop immediately** and tell the
 
 Both `gh` and `bk` make network requests that are blocked by the default sandbox. All Bash commands in this workflow must use `dangerouslyDisableSandbox: true`.
 
+## Search existing failures by pattern
+
+Search for already recorded failures in CI with a short, stable error substring:
+
+```bash
+bin/ci-failures 'foobar'
+```
+
+Use `--search` for the global search. Narrow results with `--branch main`,
+`--version`, `--issue`, `--test`, `--build`, `--start-date`, or `--end-date`.
+The command explains how to create a token when needed.
+
 ## Step 1: Extract PR number
 
 Parse `$ARGUMENTS` to get the PR number. Handle both formats:

bin/ci-failures

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+#
+# ci-failures - Search historical CI failures.
+
+exec "$(dirname "$0")"/pyactivate -m materialize.cli.ci_failures "$@"

ci/mkpipeline.py

@@ -56,6 +56,24 @@
 # err on the side of including too much rather than too little.
 CI_GLUE_GLOBS = ["bin", "ci", "misc/python/materialize/cli/ci_annotate_errors.py"]
 
+
+def ci_glue_globs(current_pipeline: str) -> list[str]:
+    """The glue globs relevant to `current_pipeline`, i.e. CI_GLUE_GLOBS with
+    the `pipeline.template.yml` of every *other* pipeline excluded.
+
+    A pipeline template only affects its own pipeline, so a change to, say, the
+    Nightly template must not count as glue for the test pipeline and force it
+    to run every step untrimmed. The current pipeline's own template stays in
+    scope: a template edit can change a step's command in ways the input-based
+    trimming cannot detect."""
+    ci_dir = Path(__file__).parent
+    return CI_GLUE_GLOBS + [
+        f":(exclude)ci/{template.parent.name}/pipeline.template.yml"
+        for template in sorted(ci_dir.glob("*/pipeline.template.yml"))
+        if template.parent.name != current_pipeline
+    ]
+
+
 DEFAULT_AGENT = "hetzner-aarch64-4cpu-8gb"
 
 
@@ -208,6 +226,7 @@ def main() -> int:
     lto = (
         pipeline.get("env", {}).get("CI_LTO", 0) == 1
         or bool(os.environ["BUILDKITE_TAG"])
+        or args.pipeline == "spec-sheet"
         or (
             not ui.env_is_truthy("BUILDKITE_PULL_REQUEST")
             and args.pipeline in ("nightly", "release-qualification")
@@ -317,7 +336,7 @@ def fetch_hashes() -> None:
             print("Coverage/Sanitizer build, not trimming pipeline")
         elif os.environ["BUILDKITE_BRANCH"] == "main" or os.environ["BUILDKITE_TAG"]:
             print("On main branch or tag, so not trimming pipeline")
-        elif have_paths_changed(CI_GLUE_GLOBS):
+        elif have_paths_changed(ci_glue_globs(args.pipeline)):
             # We still execute pipeline trimming on a copy of the pipeline to
             # protect against bugs in the pipeline trimming itself.
             print("[DRY RUN] Trimming unchanged steps from pipeline")

ci/nightly/pipeline.template.yml

@@ -1602,6 +1602,7 @@ steps:
           - ./ci/plugins/mzcompose:
               composition: launchdarkly-flag-consistency
         branches: "main"
+        skip: "TODO: Reenable when Dennis is back"
 
       - id: launchdarkly-reconnect
         label: "LaunchDarkly reconnect"

console/src/api/cloudGlobalApi.ts

@@ -7,10 +7,11 @@
 // the Business Source License, use of this software will be governed
 // by the Apache License, Version 2.0.
 
-import { formatRFC3339 } from "date-fns";
 import createClient from "openapi-fetch";
 
 import { NOT_SUPPORTED_MESSAGE } from "~/config/AppConfig";
+import { addUtcDays } from "~/util";
+import { formatDateInUtc } from "~/utils/dateFormat";
 
 import { apiClient } from "./apiClient";
 import {
@@ -27,6 +28,13 @@ export type Region = components["schemas"]["Region"];
 export type Regions = components["schemas"]["Paginated_Region"];
 export type CreditBlock = components["schemas"]["CreditBlock"];
 export type DailyCosts = components["schemas"]["DailyCostResponse"];
+export type CostBreakdownAccount =
+  components["schemas"]["CostBreakdownAccount"];
+export type CostBreakdownCluster =
+  components["schemas"]["CostBreakdownCluster"];
+export type CostBreakdownDay = components["schemas"]["CostBreakdownDay"];
+export type DailyCostBreakdown =
+  components["schemas"]["DailyCostBreakdownResponse"];
 export type AllCosts = components["schemas"]["AllCosts"];
 export type DailyCostKey = keyof components["schemas"]["AllCosts"];
 export type Prices =
@@ -147,23 +155,31 @@ export async function getCredits(requestOptions: OpenApiRequestOptions = {}) {
   return handleOpenApiResponseWithBody(data, response);
 }
 
-export async function getDailyCosts(
+export async function getCostsBreakdownDaily(
   startDate: Date,
   endDate: Date,
   requestOptions: OpenApiRequestOptions = {},
 ) {
   const { headers, ...options } = requestOptions;
-  const { data, response } = await getClient().GET("/api/costs/daily", {
-    params: {
-      query: {
-        startDate: formatRFC3339(startDate),
-        endDate: formatRFC3339(endDate),
+  const { data, response } = await getClient().GET(
+    "/api/costs/breakdown/daily",
+    {
+      params: {
+        query: {
+          // Both query params are inclusive calendar days (bare `YYYY-MM-DD`,
+          // no time component — see `DateRange` in cloud's interface.rs),
+          // unlike every other date param on this client. `startDate`/`endDate`
+          // here are the internal [start, end) convention used throughout the
+          // console, so the exclusive `endDate` becomes the day before it.
+          startDate: formatDateInUtc(startDate, "yyyy-MM-dd"),
+          endDate: formatDateInUtc(addUtcDays(endDate, -1), "yyyy-MM-dd"),
+        },
       },
+      signal: requestOptions?.signal,
+      headers,
+      ...options,
     },
-    signal: requestOptions?.signal,
-    headers,
-    ...options,
-  });
+  );
   return handleOpenApiResponseWithBody(data, response);
 }
 

console/src/api/mocks/cloudGlobalApiHandlers.ts

@@ -7,18 +7,18 @@
 // the Business Source License, use of this software will be governed
 // by the Apache License, Version 2.0.
 
-import { addDays, differenceInDays, formatISO, parseISO } from "date-fns";
+import { addDays, differenceInDays, formatISO } from "date-fns";
 import { http, HttpResponse } from "msw";
 
 import {
   CreditBlock,
+  DailyCostBreakdown,
   DailyCosts,
   Invoice,
   Organization,
   Region,
   Regions,
 } from "~/api/cloudGlobalApi";
-import { assert } from "~/util";
 
 export const buildCloudRegionsReponse = (
   options: {
@@ -109,21 +109,11 @@ export function generateDailyCostResponsePayload(
   return response;
 }
 
-export const buildDailyCostResponse = (
-  options: { payload?: DailyCosts; status?: number } = {},
+export const buildDailyCostBreakdownResponse = (
+  options: { payload?: DailyCostBreakdown; status?: number } = {},
 ) =>
-  http.get("*/api/costs/daily", (info) => {
-    let payload = options.payload;
-    if (!payload) {
-      const url = new URL(info.request.url);
-      const startParam = url.searchParams.get("startDate");
-      assert(startParam !== null);
-      const startDate = parseISO(startParam);
-      const endParam = url.searchParams.get("endDate");
-      assert(endParam !== null);
-      const endDate = parseISO(endParam);
-      payload = generateDailyCostResponsePayload(startDate, endDate);
-    }
+  http.get("*/api/costs/breakdown/daily", () => {
+    const payload: DailyCostBreakdown = options.payload ?? { days: [] };
     return HttpResponse.json(payload, { status: options.status ?? 200 });
   });