From d1e452c12254d4087f9e28b98526cd732e8b0490 Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Fri, 30 Oct 2026 16:02:52 +0000 Subject: [PATCH 1/2] fix(macrobenchmarks): retry GKE control plane in other zones on stockout Cluster creation can fail with GCE_STOCKOUT when GKE cannot provision the control plane of a zonal cluster in _ZONE, even though the workload node pool itself is not the constrained resource. create_cluster.sh now tries the control plane in _ZONE first and then in the region's other zones, deleting the failed cluster before the next try. The _MACHINE_TYPE node pool stays pinned to _ZONE via --node-locations, so node placement and zonal-bucket co-location are unchanged. The chosen zone is saved as CLUSTER_LOCATION and used by the later steps. --- .../scripts/cleanup_cluster.sh | 2 +- .../macrobenchmarks/scripts/cleanup_helm.sh | 2 +- .../macrobenchmarks/scripts/create_cluster.sh | 39 ++++++++++++++----- .../macrobenchmarks/scripts/run_workload.sh | 2 +- .../scripts/seed_checkpoint.sh | 2 +- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/cloudbuild/macrobenchmarks/scripts/cleanup_cluster.sh b/cloudbuild/macrobenchmarks/scripts/cleanup_cluster.sh index 6728ddca8..de5004b3b 100755 --- a/cloudbuild/macrobenchmarks/scripts/cleanup_cluster.sh +++ b/cloudbuild/macrobenchmarks/scripts/cleanup_cluster.sh @@ -14,7 +14,7 @@ echo "--- Deleting GKE cluster: ${CLUSTER_NAME} ---" # is the last cleanup step and blocks nothing, so it's worth being patient to # release the cluster's resources rather than leaking them. for i in {1..20}; do - if gcloud container clusters delete "$CLUSTER_NAME" --zone="${_ZONE}" --project="${PROJECT_ID}" --quiet; then + if gcloud container clusters delete "$CLUSTER_NAME" --zone="${CLUSTER_LOCATION:-${_ZONE}}" --project="${PROJECT_ID}" --quiet; then echo "Cluster ${CLUSTER_NAME} deleted successfully." break fi diff --git a/cloudbuild/macrobenchmarks/scripts/cleanup_helm.sh b/cloudbuild/macrobenchmarks/scripts/cleanup_helm.sh index 5de46a491..dcee52395 100755 --- a/cloudbuild/macrobenchmarks/scripts/cleanup_helm.sh +++ b/cloudbuild/macrobenchmarks/scripts/cleanup_helm.sh @@ -7,7 +7,7 @@ fi source "$(dirname "$0")/lib.sh" source "${BUILD_VARS_FILE}" curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash || true -gcloud container clusters get-credentials "$CLUSTER_NAME" --zone="${_ZONE}" --project="${PROJECT_ID}" || true +gcloud container clusters get-credentials "$CLUSTER_NAME" --zone="${CLUSTER_LOCATION:-${_ZONE}}" --project="${PROJECT_ID}" || true helm uninstall "$RUN_ID" || true # Best-effort: the seed release is normally uninstalled by the seed-checkpoint # step; clean up here in case that step failed mid-way. Whole-cluster deletion diff --git a/cloudbuild/macrobenchmarks/scripts/create_cluster.sh b/cloudbuild/macrobenchmarks/scripts/create_cluster.sh index c5c9fc51b..e3dc6140d 100755 --- a/cloudbuild/macrobenchmarks/scripts/create_cluster.sh +++ b/cloudbuild/macrobenchmarks/scripts/create_cluster.sh @@ -28,18 +28,37 @@ gcloud compute networks subnets create "${SUBNET_NAME}" \ # the GKE node label (and the chart's nodeSelector) keys off of. --enable-gvnic # puts the nodes on gVNIC, which C4 requires and which TIER_1 egress (the direct # high-bandwidth path) is gated on. -gcloud container clusters create "$CLUSTER_NAME" \ - --project="${PROJECT_ID}" --zone="${_ZONE}" \ - --machine-type="e2-standard-4" --num-nodes="1" \ - --service-account="${_GKE_SERVICE_ACCOUNT}" \ - --scopes="https://www.googleapis.com/auth/cloud-platform" \ - --private-ipv6-google-access-type=outbound-only \ - --network="${NETWORK_NAME}" --subnetwork="${SUBNET_NAME}" \ - --enable-autoupgrade --quiet +# +# The GKE control plane of a zonal cluster runs in the cluster's zone and can hit +# GCE_STOCKOUT there independently of the node pools. If that happens, retry the +# control plane in the region's other zones; the ${_MACHINE_TYPE} pool stays in +# ${_ZONE}. +ZONES="${_ZONE} $(gcloud compute zones list --project="${PROJECT_ID}" --filter="region:${REGION}" --format="value(name)" | grep -vx "${_ZONE}" || true)" +CLUSTER_LOCATION="" +for zone in $ZONES; do + if gcloud container clusters create "$CLUSTER_NAME" \ + --project="${PROJECT_ID}" --zone="${zone}" \ + --machine-type="e2-standard-4" --num-nodes="1" \ + --service-account="${_GKE_SERVICE_ACCOUNT}" \ + --scopes="https://www.googleapis.com/auth/cloud-platform" \ + --private-ipv6-google-access-type=outbound-only \ + --network="${NETWORK_NAME}" --subnetwork="${SUBNET_NAME}" \ + --enable-autoupgrade --quiet; then + CLUSTER_LOCATION="$zone" + break + fi + gcloud container clusters delete "$CLUSTER_NAME" --zone="${zone}" --project="${PROJECT_ID}" --quiet || true +done +if [ -z "$CLUSTER_LOCATION" ]; then + record_failure create-cluster + exit 1 +fi +echo "export CLUSTER_LOCATION=${CLUSTER_LOCATION}" >> "${BUILD_VARS_FILE}" NODE_POOL_ARGS=( --cluster="$CLUSTER_NAME" --project="${PROJECT_ID}" - --zone="${_ZONE}" + --zone="${CLUSTER_LOCATION}" + --node-locations="${_ZONE}" --machine-type="${_MACHINE_TYPE}" --num-nodes="${_NODES}" --disk-size="200" @@ -54,7 +73,7 @@ if [ "${_ENABLE_TIER1_NETWORKING:-true}" = "true" ]; then NODE_POOL_ARGS+=(--network-performance-configs="total-egress-bandwidth-tier=TIER_1") fi gcloud container node-pools create "${_MACHINE_TYPE}" "${NODE_POOL_ARGS[@]}" -gcloud container clusters get-credentials "$CLUSTER_NAME" --zone="${_ZONE}" --project="${PROJECT_ID}" +gcloud container clusters get-credentials "$CLUSTER_NAME" --zone="${CLUSTER_LOCATION:-${_ZONE}}" --project="${PROJECT_ID}" kubectl apply --server-side -f "https://github.com/kubernetes-sigs/jobset/releases/download/${_JOBSET_VERSION}/manifests.yaml" kubectl rollout status deployment/jobset-controller-manager -n jobset-system --timeout=300s diff --git a/cloudbuild/macrobenchmarks/scripts/run_workload.sh b/cloudbuild/macrobenchmarks/scripts/run_workload.sh index f054e9248..17d492660 100755 --- a/cloudbuild/macrobenchmarks/scripts/run_workload.sh +++ b/cloudbuild/macrobenchmarks/scripts/run_workload.sh @@ -8,7 +8,7 @@ trap 'record_failure run-workload' ERR skip_if_failed source "${BUILD_VARS_FILE}" curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -gcloud container clusters get-credentials "$CLUSTER_NAME" --zone="${_ZONE}" --project="${PROJECT_ID}" +gcloud container clusters get-credentials "$CLUSTER_NAME" --zone="${CLUSTER_LOCATION:-${_ZONE}}" --project="${PROJECT_ID}" CHART="gcsfs/tests/perf/macrobenchmarks/workloads/${_WORKLOAD}/helm_chart" # Restore precedence: an external checkpoint wins; otherwise, when seeding is on, # restore the per-run seed produced by the seed-checkpoint step. Empty => fresh diff --git a/cloudbuild/macrobenchmarks/scripts/seed_checkpoint.sh b/cloudbuild/macrobenchmarks/scripts/seed_checkpoint.sh index 067989043..6306750e4 100755 --- a/cloudbuild/macrobenchmarks/scripts/seed_checkpoint.sh +++ b/cloudbuild/macrobenchmarks/scripts/seed_checkpoint.sh @@ -20,7 +20,7 @@ fi SEED_RUN_ID="${RUN_ID}-seed" SEED_CKPT_DIR="gs://$CHECKPOINT_BUCKET/seed" curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -gcloud container clusters get-credentials "$CLUSTER_NAME" --zone="${_ZONE}" --project="${PROJECT_ID}" +gcloud container clusters get-credentials "$CLUSTER_NAME" --zone="${CLUSTER_LOCATION:-${_ZONE}}" --project="${PROJECT_ID}" CHART="gcsfs/tests/perf/macrobenchmarks/workloads/${_WORKLOAD}/helm_chart" # One optimizer step (gradient accumulation is fixed at 1 in launcher.sh) with From 9903073d7f8e3de20baf7082bab0e0c1e0fc7506 Mon Sep 17 00:00:00 2001 From: Mahalaxmi Date: Mon, 2 Nov 2026 06:05:25 +0000 Subject: [PATCH 2/2] fix(macrobenchmarks): find cluster by name in cleanup, add fallback tests - cleanup-cluster looks the cluster up by name and deletes it in every location found, skipping when none exists (avoids ~10 min of retries on a 404 and covers a leftover cluster from a failed fallback attempt). - Log each zone attempt, the fallback, and the chosen zone. - Move the node-pool comment next to NODE_POOL_ARGS. - Document that _ZONE is the preferred control-plane zone. - Add stubbed-gcloud tests for create-cluster fallback and cleanup. --- cloudbuild/macrobenchmarks/README.md | 4 +- .../metrics/tests/test_create_cluster.py | 177 ++++++++++++++++++ .../scripts/cleanup_cluster.sh | 31 ++- .../macrobenchmarks/scripts/create_cluster.sh | 23 ++- 4 files changed, 214 insertions(+), 21 deletions(-) create mode 100644 cloudbuild/macrobenchmarks/metrics/tests/test_create_cluster.py diff --git a/cloudbuild/macrobenchmarks/README.md b/cloudbuild/macrobenchmarks/README.md index d15f7ef2f..4e579efd4 100644 --- a/cloudbuild/macrobenchmarks/README.md +++ b/cloudbuild/macrobenchmarks/README.md @@ -116,7 +116,7 @@ Before creating the triggers, set up the following in your GCP project. | `init-variables` | Validates all substitutions and the operator-supplied buckets, derives the region and run identifiers. **Fails fast before anything is provisioned or billed.** | | `cleanup-leaked-resources` | Best-effort reap of orphaned infra from prior failed builds with the same `_INFRA_PREFIX`. | | `create-buckets` | Creates the per-run checkpoint bucket and per-run dataset bucket (copying `_DATASET_PATH` in), and ensures the shared results bucket exists. | -| `create-cluster` | Creates the dedicated VPC/subnet, the GKE cluster + a `_MACHINE_TYPE` node pool (with TIER_1 networking when enabled), and installs the JobSet controller. | +| `create-cluster` | Creates the dedicated VPC/subnet, the GKE cluster + a `_MACHINE_TYPE` node pool in `_ZONE` (with TIER_1 networking when enabled), and installs the JobSet controller. If the control plane can't be created in `_ZONE`, it retries in the region's other zones. | | `seed-checkpoint` | When `_SEED_CHECKPOINT=true` and no external checkpoint is given, generates a per-run checkpoint so the measured run exercises the **restore** path. | | `run-workload` | `helm install`s the chart and polls the JobSet to completion, recording start/end timestamps for the scrape. | | `scrape-metrics` | Parses Cloud Logging into raw CSVs, aggregates them into one summary row (retrying for ingestion lag), folds in system metrics, and uploads the summary to the results bucket. | @@ -130,7 +130,7 @@ Before creating the triggers, set up the following in your GCP project. | Substitution | Description | | :----------- | :---------- | | `_INFRA_PREFIX` | Prefix for all created resources (cluster, network, buckets, BigQuery dataset). Also selects the shared results bucket `gs://<_INFRA_PREFIX>-macrobench-results`. | -| `_ZONE` | GCP zone for the cluster and zonal buckets (e.g. `us-central1-a`). The region is derived from it. | +| `_ZONE` | GCP zone for the workload node pool and zonal buckets (e.g. `us-central1-a`), and the preferred zone for the cluster's control plane (which may fall back to another zone in the region). The region is derived from it. | | `_GKE_SERVICE_ACCOUNT` | Email of the node service account (see prerequisites). | | `_DATASET_PATH` | `gs://` directory of `*.parquet` shards (with a `text` column) to train on. Must match the run's region and bucket type. | | `_REQUIREMENTS` | Whitespace-separated immutable pip specs/URLs installed last. Every workload requires the gcsfs build under test; the Ray workload needs nothing else. | diff --git a/cloudbuild/macrobenchmarks/metrics/tests/test_create_cluster.py b/cloudbuild/macrobenchmarks/metrics/tests/test_create_cluster.py new file mode 100644 index 000000000..fc71b7cd9 --- /dev/null +++ b/cloudbuild/macrobenchmarks/metrics/tests/test_create_cluster.py @@ -0,0 +1,177 @@ +"""Tests for the control-plane zone fallback in create_cluster.sh and the +matching lookup in cleanup_cluster.sh. + +The scripts run for real under bash, with stub ``gcloud``/``kubectl``/``sleep`` +executables on PATH that log every call and fail on demand. +""" + +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +SCRIPTS = Path(__file__).resolve().parents[2] / "scripts" + +pytestmark = pytest.mark.skipif(shutil.which("bash") is None, reason="needs bash") + +GCLOUD_STUB = r"""#!/usr/bin/env bash +echo "gcloud $*" >> "$STUB_LOG" +case "$*" in + "compute zones list"*) + [ -n "$STUB_ZONES_LIST_FAIL" ] && exit 1 + printf '%s\n' $STUB_ZONES ;; + "container clusters create"*) + for z in $STUB_FAIL_ZONES; do + [[ "$*" == *"--zone=$z "* ]] && exit 1 + done ;; + "container clusters list"*) + [ -n "$STUB_LIST_FAIL" ] && exit 1 + [ -n "$STUB_CLUSTER_LOCATIONS" ] && printf '%s\n' $STUB_CLUSTER_LOCATIONS ;; +esac +exit 0 +""" + + +@pytest.fixture +def env(tmp_path): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + for name, body in { + "gcloud": GCLOUD_STUB, + "kubectl": '#!/usr/bin/env bash\necho "kubectl $*" >> "$STUB_LOG"\n', + "sleep": "#!/usr/bin/env bash\n", + }.items(): + stub = bin_dir / name + stub.write_text(body) + stub.chmod(0o755) + + build_vars = tmp_path / "build_vars.env" + build_vars.write_text( + "export CLUSTER_NAME=test-cluster\n" + "export NETWORK_NAME=test-net\n" + "export SUBNET_NAME=test-subnet\n" + "export REGION=region1\n" + ) + return { + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "STUB_LOG": str(tmp_path / "calls.log"), + "FAILED_FILE": str(tmp_path / "FAILED"), + "BUILD_VARS_FILE": str(build_vars), + "PROJECT_ID": "test-project", + "_ZONE": "region1-a", + "_MACHINE_TYPE": "test-machine", + "_NODES": "1", + "_GKE_SERVICE_ACCOUNT": "sa@example.com", + "_JOBSET_VERSION": "v0.0.0", + "STUB_ZONES": "region1-a region1-b region1-c", + } + + +def _run(script, env, **overrides): + env = {**env, **overrides} + proc = subprocess.run( + ["bash", str(SCRIPTS / script)], env=env, capture_output=True, text=True + ) + log = Path(env["STUB_LOG"]) + calls = log.read_text().splitlines() if log.exists() else [] + return proc, calls + + +def _calls(calls, prefix): + return [c for c in calls if c.startswith(prefix)] + + +def _failures(env): + failed = Path(env["FAILED_FILE"]) + return failed.read_text().split() if failed.exists() else [] + + +def _cluster_location(env): + for line in Path(env["BUILD_VARS_FILE"]).read_text().splitlines(): + if line.startswith("export CLUSTER_LOCATION="): + return line.split("=", 1)[1] + return None + + +def test_create_succeeds_in_preferred_zone(env): + proc, calls = _run("create_cluster.sh", env) + + assert proc.returncode == 0, proc.stderr + creates = _calls(calls, "gcloud container clusters create") + assert len(creates) == 1 and "--zone=region1-a " in creates[0] + assert not _calls(calls, "gcloud container clusters delete") + assert _cluster_location(env) == "region1-a" + assert _failures(env) == [] + + +def test_create_falls_back_and_keeps_node_pool_in_preferred_zone(env): + proc, calls = _run("create_cluster.sh", env, STUB_FAIL_ZONES="region1-a") + + assert proc.returncode == 0, proc.stderr + creates = _calls(calls, "gcloud container clusters create") + assert ["--zone=region1-a " in c for c in creates] == [True, False] + assert "--zone=region1-b " in creates[1] + deletes = _calls(calls, "gcloud container clusters delete") + assert len(deletes) == 1 and "--zone=region1-a" in deletes[0] + (pool,) = _calls(calls, "gcloud container node-pools create") + assert "--zone=region1-b" in pool and "--node-locations=region1-a" in pool + (creds,) = _calls(calls, "gcloud container clusters get-credentials") + assert "--zone=region1-b" in creds + assert _cluster_location(env) == "region1-b" + assert _failures(env) == [] + + +def test_create_records_failure_once_when_all_zones_fail(env): + proc, calls = _run( + "create_cluster.sh", env, STUB_FAIL_ZONES="region1-a region1-b region1-c" + ) + + assert proc.returncode == 1 + assert len(_calls(calls, "gcloud container clusters create")) == 3 + assert not _calls(calls, "gcloud container node-pools create") + assert _failures(env) == ["create-cluster"] + assert _cluster_location(env) is None + assert "could not create test-cluster" in proc.stderr + + +def test_create_tries_only_preferred_zone_when_zone_list_fails(env): + proc, calls = _run( + "create_cluster.sh", env, STUB_ZONES_LIST_FAIL="1", STUB_FAIL_ZONES="region1-a" + ) + + assert proc.returncode == 1 + creates = _calls(calls, "gcloud container clusters create") + assert len(creates) == 1 and "--zone=region1-a " in creates[0] + assert _failures(env) == ["create-cluster"] + + +def test_cleanup_deletes_cluster_in_every_location_found(env): + proc, calls = _run( + "cleanup_cluster.sh", env, STUB_CLUSTER_LOCATIONS="region1-a region1-b" + ) + + assert proc.returncode == 0, proc.stderr + deletes = _calls(calls, "gcloud container clusters delete") + assert len(deletes) == 2 + assert "--zone=region1-a" in deletes[0] and "--zone=region1-b" in deletes[1] + + +def test_cleanup_skips_delete_when_no_cluster_found(env): + proc, calls = _run("cleanup_cluster.sh", env) + + assert proc.returncode == 0, proc.stderr + assert not _calls(calls, "gcloud container clusters delete") + assert "No cluster test-cluster found" in proc.stdout + + +def test_cleanup_falls_back_to_recorded_location_when_lookup_fails(env): + proc, calls = _run( + "cleanup_cluster.sh", env, STUB_LIST_FAIL="1", CLUSTER_LOCATION="region1-c" + ) + + assert proc.returncode == 0, proc.stderr + (delete,) = _calls(calls, "gcloud container clusters delete") + assert "--zone=region1-c" in delete diff --git a/cloudbuild/macrobenchmarks/scripts/cleanup_cluster.sh b/cloudbuild/macrobenchmarks/scripts/cleanup_cluster.sh index de5004b3b..6b9a96417 100755 --- a/cloudbuild/macrobenchmarks/scripts/cleanup_cluster.sh +++ b/cloudbuild/macrobenchmarks/scripts/cleanup_cluster.sh @@ -7,19 +7,30 @@ fi source "$(dirname "$0")/lib.sh" source "${BUILD_VARS_FILE}" echo "--- Deleting GKE cluster: ${CLUSTER_NAME} ---" +# create-cluster may have tried several zones, and a failed attempt's delete is +# best-effort, so look the cluster up by name and delete it wherever it exists. +# If the lookup itself fails, fall back to the recorded location. An empty +# lookup means there is nothing to delete. +if LOCATIONS=$(gcloud container clusters list --project="${PROJECT_ID}" \ + --filter="name=${CLUSTER_NAME}" --format="value(location)"); then + [ -z "$LOCATIONS" ] && echo "No cluster ${CLUSTER_NAME} found; skipping." +else + LOCATIONS="${CLUSTER_LOCATION:-${_ZONE}}" +fi # Best-effort: just retry the delete until it succeeds. A delete can transiently # fail (e.g. FAILED_PRECONDITION while a cluster operation is still settling); # retrying after a pause handles that without inspecting cluster/operation state. -# A not-found cluster is already the desired end state. Retry generously -- this -# is the last cleanup step and blocks nothing, so it's worth being patient to -# release the cluster's resources rather than leaking them. -for i in {1..20}; do - if gcloud container clusters delete "$CLUSTER_NAME" --zone="${CLUSTER_LOCATION:-${_ZONE}}" --project="${PROJECT_ID}" --quiet; then - echo "Cluster ${CLUSTER_NAME} deleted successfully." - break - fi - echo "Attempt $i to delete cluster ${CLUSTER_NAME} failed. Retrying in 30 seconds..." - sleep 30 +# Retry generously -- this is the last cleanup step and blocks nothing, so it's +# worth being patient to release the cluster's resources rather than leaking them. +for loc in $LOCATIONS; do + for i in {1..20}; do + if gcloud container clusters delete "$CLUSTER_NAME" --zone="${loc}" --project="${PROJECT_ID}" --quiet; then + echo "Cluster ${CLUSTER_NAME} in ${loc} deleted successfully." + break + fi + echo "Attempt $i to delete cluster ${CLUSTER_NAME} in ${loc} failed. Retrying in 30 seconds..." + sleep 30 + done done echo "--- Deleting dedicated subnetwork: ${SUBNET_NAME} ---" diff --git a/cloudbuild/macrobenchmarks/scripts/create_cluster.sh b/cloudbuild/macrobenchmarks/scripts/create_cluster.sh index e3dc6140d..b08a49738 100755 --- a/cloudbuild/macrobenchmarks/scripts/create_cluster.sh +++ b/cloudbuild/macrobenchmarks/scripts/create_cluster.sh @@ -21,14 +21,6 @@ gcloud compute networks subnets create "${SUBNET_NAME}" \ --range="10.0.0.0/20" \ --enable-private-ip-google-access --quiet -# `gcloud container clusters create` has no --node-pool flag and always names -# its initial pool "default-pool", which would not match the workload's -# nodeSelector (cloud.google.com/gke-nodepool=c4-standard-192). Create a small -# system pool for the cluster, then a dedicated ${_MACHINE_TYPE} pool whose name -# the GKE node label (and the chart's nodeSelector) keys off of. --enable-gvnic -# puts the nodes on gVNIC, which C4 requires and which TIER_1 egress (the direct -# high-bandwidth path) is gated on. -# # The GKE control plane of a zonal cluster runs in the cluster's zone and can hit # GCE_STOCKOUT there independently of the node pools. If that happens, retry the # control plane in the region's other zones; the ${_MACHINE_TYPE} pool stays in @@ -36,6 +28,7 @@ gcloud compute networks subnets create "${SUBNET_NAME}" \ ZONES="${_ZONE} $(gcloud compute zones list --project="${PROJECT_ID}" --filter="region:${REGION}" --format="value(name)" | grep -vx "${_ZONE}" || true)" CLUSTER_LOCATION="" for zone in $ZONES; do + echo "--- Creating GKE cluster ${CLUSTER_NAME} in ${zone} ---" if gcloud container clusters create "$CLUSTER_NAME" \ --project="${PROJECT_ID}" --zone="${zone}" \ --machine-type="e2-standard-4" --num-nodes="1" \ @@ -47,13 +40,25 @@ for zone in $ZONES; do CLUSTER_LOCATION="$zone" break fi + echo "Cluster creation in ${zone} failed; deleting the failed cluster and trying the next zone." gcloud container clusters delete "$CLUSTER_NAME" --zone="${zone}" --project="${PROJECT_ID}" --quiet || true done if [ -z "$CLUSTER_LOCATION" ]; then + echo "ERROR: could not create ${CLUSTER_NAME} in any of: ${ZONES}" >&2 record_failure create-cluster exit 1 fi +echo "Cluster ${CLUSTER_NAME} created in ${CLUSTER_LOCATION}." echo "export CLUSTER_LOCATION=${CLUSTER_LOCATION}" >> "${BUILD_VARS_FILE}" + +# `gcloud container clusters create` has no --node-pool flag and always names +# its initial pool "default-pool", which would not match the workload's +# nodeSelector (cloud.google.com/gke-nodepool=c4-standard-192). Create a small +# system pool for the cluster, then a dedicated ${_MACHINE_TYPE} pool whose name +# the GKE node label (and the chart's nodeSelector) keys off of. --enable-gvnic +# puts the nodes on gVNIC, which C4 requires and which TIER_1 egress (the direct +# high-bandwidth path) is gated on. --zone selects the cluster (wherever its +# control plane landed); --node-locations keeps the workers in ${_ZONE}. NODE_POOL_ARGS=( --cluster="$CLUSTER_NAME" --project="${PROJECT_ID}" @@ -73,7 +78,7 @@ if [ "${_ENABLE_TIER1_NETWORKING:-true}" = "true" ]; then NODE_POOL_ARGS+=(--network-performance-configs="total-egress-bandwidth-tier=TIER_1") fi gcloud container node-pools create "${_MACHINE_TYPE}" "${NODE_POOL_ARGS[@]}" -gcloud container clusters get-credentials "$CLUSTER_NAME" --zone="${CLUSTER_LOCATION:-${_ZONE}}" --project="${PROJECT_ID}" +gcloud container clusters get-credentials "$CLUSTER_NAME" --zone="${CLUSTER_LOCATION}" --project="${PROJECT_ID}" kubectl apply --server-side -f "https://github.com/kubernetes-sigs/jobset/releases/download/${_JOBSET_VERSION}/manifests.yaml" kubectl rollout status deployment/jobset-controller-manager -n jobset-system --timeout=300s