From 7add25bf7755832e11c4998e8aa29721fb6a3a0d Mon Sep 17 00:00:00 2001 From: Joshua Long <44989821+jmclong@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:11:29 +0000 Subject: [PATCH 1/2] feat(helm): bundle optional NVMe prerequisites Add optional prerequisite dependencies, rollout verification, registry settings, and DACS build and pipeline integration. Preserve standalone chart publication. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e7adc4d-b2ae-4e92-9e76-2d857187198d --- .pipelines/dacs-integration.yml | 1 + .../scripts/publish/publish-helm-chart.sh | 35 ++-- .../scripts/tests/test-helm-publication.sh | 8 + .pipelines/templates/Dacs.AksTest.Stages.yml | 28 +-- deploy/README.md | 28 ++- deploy/deploy.mk | 36 ++-- deploy/scripts/aks/create.sh | 7 +- helm/cache-server/Chart.yaml | 7 + helm/cache-server/README.md | 21 +++ helm/cache-server/values.yaml | 3 + helm/dacs-prereq/README.md | 90 ++++++++-- helm/dacs-prereq/scripts/rollout-check.sh | 82 +++++++++ helm/dacs-prereq/templates/_helpers.tpl | 29 +-- helm/dacs-prereq/templates/daemonset.yaml | 8 +- helm/dacs-prereq/templates/rbac.yaml | 1 + helm/dacs-prereq/templates/rollout-check.yaml | 112 ++++++++++++ helm/dacs-prereq/values.schema.json | 60 +++++++ helm/dacs-prereq/values.yaml | 20 +++ .../cache-controller/charts/dacs/Chart.yaml | 7 + .../cache-controller/charts/dacs/README.md | 31 ++++ .../cache-controller/charts/dacs/values.yaml | 3 + .../internal/controller/helm_cleanup_test.go | 6 + .../controller/helm_prerequisites_test.go | 170 ++++++++++++++++++ test/README.md | 14 +- test/pkg/common/common.go | 30 +--- user-guide.md | 54 +++--- 26 files changed, 740 insertions(+), 151 deletions(-) create mode 100644 helm/dacs-prereq/scripts/rollout-check.sh create mode 100644 helm/dacs-prereq/templates/rollout-check.yaml create mode 100644 helm/dacs-prereq/values.schema.json create mode 100644 src/subsystems/cache-controller/internal/controller/helm_prerequisites_test.go diff --git a/.pipelines/dacs-integration.yml b/.pipelines/dacs-integration.yml index 6b58e62e..d76c7c41 100644 --- a/.pipelines/dacs-integration.yml +++ b/.pipelines/dacs-integration.yml @@ -254,6 +254,7 @@ extends: - template: /.pipelines/templates/Dacs.AksTest.Stages.yml@self parameters: stageName: AksHelmUpgrade + bundledPrerequisites: false stageDisplayName: AKS Helm Upgrade testPackage: ./test/aks testFilter: aks && helm-upgrade diff --git a/.pipelines/scripts/publish/publish-helm-chart.sh b/.pipelines/scripts/publish/publish-helm-chart.sh index 93ed8a9e..0bab5c50 100755 --- a/.pipelines/scripts/publish/publish-helm-chart.sh +++ b/.pipelines/scripts/publish/publish-helm-chart.sh @@ -58,44 +58,43 @@ package_chart() ( local CHART_NAME="$2" local CHART_PACKAGE="$OUTPUT_DIR/$CHART_NAME-$CHART_VERSION.tgz" - local CHART_DESCRIPTION local CHART_STAGE - CHART_DESCRIPTION="$(grep '^description:' "$CHART_DIR/Chart.yaml" | cut -d: -f2- | xargs)" CHART_STAGE="$(mktemp -d "$OUTPUT_DIR/.${CHART_NAME}.XXXXXX")" trap 'rm -rf -- "$CHART_STAGE"' EXIT cp -a "$CHART_DIR/." "$CHART_STAGE/" - cat > "$CHART_STAGE/Chart.yaml" << EOF -apiVersion: v2 -name: $CHART_NAME -description: $CHART_DESCRIPTION -type: application -version: $CHART_VERSION -appVersion: "$CONTAINER_VERSION" -EOF + if [[ "$CHART_NAME" != dacs-prereq ]]; then + mkdir -p "$CHART_STAGE/charts" + rm -f "$CHART_STAGE/charts"/dacs-prereq-*.tgz + cp -a "$REPOSITORY_ROOT/helm/dacs-prereq" "$CHART_STAGE/charts/dacs-prereq" + sed -i "s|^appVersion:.*|appVersion: \"$CONTAINER_VERSION\"|" \ + "$CHART_STAGE/charts/dacs-prereq/Chart.yaml" + fi # Update values.yaml with the build registry and image versions. - if [[ -f "$CHART_STAGE/values.yaml" ]]; then + local VALUES_FILE + for VALUES_FILE in "$CHART_STAGE/values.yaml" "$CHART_STAGE/charts/dacs-prereq/values.yaml"; do + [[ -f "$VALUES_FILE" ]] || continue sed -i -E \ "s#^([[:space:]]*registry:)[[:space:]]*dacstest\.azurecr\.io\$#\\1 $REGISTRY#" \ - "$CHART_STAGE/values.yaml" + "$VALUES_FILE" sed -i -E \ "s#^([[:space:]]*repository:)[[:space:]]*(acstor/dacs-server|cache-server)\$#\\1 $REGISTRY/dacs/cache-server#" \ - "$CHART_STAGE/values.yaml" + "$VALUES_FILE" sed -i -E \ "s#^([[:space:]]*repository:)[[:space:]]*(acstor/dacs-scheduler|cacheserver-scheduler)\$#\\1 $REGISTRY/dacs/cacheserver-scheduler#" \ - "$CHART_STAGE/values.yaml" + "$VALUES_FILE" sed -i -E \ "s#^([[:space:]]*repository:)[[:space:]]*(acstor/dacs-controller|cache-controller)\$#\\1 $REGISTRY/dacs/cache-controller#" \ - "$CHART_STAGE/values.yaml" + "$VALUES_FILE" sed -i -E \ "s#^([[:space:]]*repository:)[[:space:]]*cache-setup\$#\\1 $REGISTRY/dacs/cache-setup#" \ - "$CHART_STAGE/values.yaml" + "$VALUES_FILE" sed -i -E \ "s#^([[:space:]]*tag:)[[:space:]]*\"\".*\$#\\1 \"$CONTAINER_VERSION\"#" \ - "$CHART_STAGE/values.yaml" - fi + "$VALUES_FILE" + done # Package the chart (empty tag in values.yaml defaults to appVersion) helm package "$CHART_STAGE" -d "$OUTPUT_DIR" \ diff --git a/.pipelines/scripts/tests/test-helm-publication.sh b/.pipelines/scripts/tests/test-helm-publication.sh index b53d698e..c1842cbd 100755 --- a/.pipelines/scripts/tests/test-helm-publication.sh +++ b/.pipelines/scripts/tests/test-helm-publication.sh @@ -32,6 +32,7 @@ assert_safe_labels() { local chart_package="${TEMP_DIR}/${chart_name}-${TEST_CHART_VERSION}.tgz" local rendered + helm dependency build "${chart_dir}" --skip-refresh >/dev/null helm package "${chart_dir}" \ --destination "${TEMP_DIR}" \ --version "${TEST_CHART_VERSION}" \ @@ -69,6 +70,13 @@ grep -Fq "example.azurecr.io/dacs/cache-server:${container_version}" <<<"${rende rendered_prereq="$(helm template test-release "${OUTPUT_DIR}/dacs-prereq-${chart_version}.tgz")" grep -Fq "example.azurecr.io/dacs/cache-setup:${container_version}" <<<"${rendered_prereq}" +for chart in cache-server dacs; do + rendered_parent="$(helm template test-release "${OUTPUT_DIR}/${chart}-${chart_version}.tgz" \ + --set cacheServerPrereq.enabled=true)" + grep -Fq "example.azurecr.io/dacs/cache-setup:${container_version}" <<<"${rendered_parent}" + grep -Fq "helm.sh/hook: post-install,post-upgrade" <<<"${rendered_parent}" +done + grep -Fq "CACHE_CONTROLLER_CHART_VERSION=${chart_version}" "${METADATA_FILE}" grep -Fq "APP_VERSION=$("${REPOSITORY_ROOT}/.pipelines/scripts/get-version.sh" source)" "${METADATA_FILE}" diff --git a/.pipelines/templates/Dacs.AksTest.Stages.yml b/.pipelines/templates/Dacs.AksTest.Stages.yml index c13e5e2d..4c940633 100644 --- a/.pipelines/templates/Dacs.AksTest.Stages.yml +++ b/.pipelines/templates/Dacs.AksTest.Stages.yml @@ -56,6 +56,9 @@ parameters: - name: cacheStaticServers type: number default: 0 + - name: bundledPrerequisites + type: boolean + default: true - name: resourcePrefix type: string default: aks-resilience @@ -182,20 +185,6 @@ stages: RESOURCE_GROUP: $(resourceGroup) RESOURCE_PREFIX: $(resourcePrefix) - - task: AzureCLI@2 - displayName: Deploy Cache Server Prerequisites - inputs: - azureSubscription: ${{ parameters.azureServiceConnection }} - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - set -euo pipefail - echo "Deploying cache-server prerequisites..." - make helm-prereq-install \ - DOCKER_REGISTRY="$DOCKER_REGISTRY" BUILD_NUMBER="$(cacheServerBuildNumber)" - env: - DOCKER_REGISTRY: ${{ parameters.dockerRegistry }} - - task: AzureCLI@2 displayName: Deploy Cache Server inputs: @@ -205,14 +194,24 @@ stages: inlineScript: | set -euo pipefail echo "Deploying with chart=${{ parameters.helmChart }}, build=$(cacheServerBuildNumber)..." + prereq_enabled=true + if [[ "${BUNDLED_PREREQUISITES,,}" != true ]]; then + make helm-prereq-install \ + DOCKER_REGISTRY="$DOCKER_REGISTRY" BUILD_NUMBER="$(cacheServerBuildNumber)" + prereq_enabled=false + fi if [[ "${{ parameters.helmChart }}" == dacs ]]; then make helm-dacs-install \ + CACHE_SERVER_PREREQ_ENABLED="$prereq_enabled" \ + CACHE_SETUP_IMG="$DOCKER_REGISTRY/dacs/cache-setup" \ DOCKER_REGISTRY="$DOCKER_REGISTRY" BUILD_NUMBER="$(cacheServerBuildNumber)" \ DACS_CACHE_CHART_REPO=dacs/helmchart/dacs \ DACS_CACHE_CHART_VERSION="$(cacheServerChartVersion)" \ HELM_ARGS="--set global.registry= --set manager.image.repository=$DOCKER_REGISTRY/dacs/cache-controller --set manager.cacheServerImage.repository=$DOCKER_REGISTRY/dacs/cache-server $EXTRA_HELM_ARGS" else make helm-install \ + CACHE_SERVER_PREREQ_ENABLED="$prereq_enabled" \ + CACHE_SETUP_IMG="$DOCKER_REGISTRY/dacs/cache-setup" \ DOCKER_REGISTRY="$DOCKER_REGISTRY" BASE_REPO="$DOCKER_REGISTRY/dacs" \ BUILD_NUMBER="$(cacheServerBuildNumber)" \ HELM_CHART_REPO=dacs/helmchart/cache-server \ @@ -222,6 +221,7 @@ stages: env: DOCKER_REGISTRY: ${{ parameters.dockerRegistry }} EXTRA_HELM_ARGS: ${{ parameters.helmArgs }} + BUNDLED_PREREQUISITES: ${{ parameters.bundledPrerequisites }} - ${{ if and(eq(parameters.helmChart, 'dacs'), ne(parameters.cacheFixture, '')) }}: - task: AzureCLI@2 diff --git a/deploy/README.md b/deploy/README.md index 38066a55..15c07cbb 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -117,26 +117,38 @@ make unprovision | `DOCKER_REGISTRY` | Docker registry for images | `docker.io` | | `BUILD_NUMBER` | Image tag to use | `local` | | `HELM_ARGS` | Additional arguments to pass to helm | - | +| `HELM_TIMEOUT` | Helm install/upgrade timeout; keep longer than the prerequisite rollout-check timeout | `15m` | +| `CACHE_SERVER_PREREQ_ENABLED` | Enable bundled NVMe prerequisites | `false`; `make deploy` enables it on AKS | +| `CACHE_SETUP_IMG` | NVMe setup image repository | `$(DOCKER_REGISTRY)/dacs/cache-setup` | ### Usage ```bash -# Install dacs-prereq (NVMe RAID setup) -make helm-prereq-install - -# Install cache-server +# Install cache-server without prerequisites make helm-install +# Install either parent chart with NVMe prerequisites +make helm-install CACHE_SERVER_PREREQ_ENABLED=true +make helm-dacs-install CACHE_SERVER_PREREQ_ENABLED=true + # Install with custom image DOCKER_REGISTRY=myregistry.azurecr.io BUILD_NUMBER=v1.2.3 make helm-install -# Uninstall cache-server +# Uninstall cache-server and its bundled prerequisites (host data is retained) make helm-uninstall - -# Uninstall dacs-prereq -make helm-prereq-uninstall ``` +Both charts default prerequisites to off. AKS `make deploy` and the AKS test +pipeline opt in explicitly; Minikube deployments do not. See the +[prerequisite configuration](../helm/dacs-prereq/README.md) for rollout +limits and validation-hook behavior. + +`make helm-dacs-install` and `make deploy CONTROLLER=new` install the +controller and optional prerequisites without creating a Cache. Wait for +prerequisite setup to succeed, then apply a standalone Cache manifest, as shown in the +[user guide](../user-guide.md#step-2-deploy-a-cache-instance). +Minikube needs no NVMe setup and uses the standalone Minikube Cache fixture. + ## Dependencies The deployment targets automatically download required tools: diff --git a/deploy/deploy.mk b/deploy/deploy.mk index 34d3d942..d440ebb3 100644 --- a/deploy/deploy.mk +++ b/deploy/deploy.mk @@ -162,6 +162,8 @@ minikube-clean: ## Delete the Minikube cluster. HELM_NAMESPACE ?= cache-server HELM_ARGS ?= +HELM_TIMEOUT ?= 15m +CACHE_SERVER_PREREQ_ENABLED ?= false # Local image settings for minikube LOCAL_CACHE_SERVER_IMG ?= cache-server @@ -184,13 +186,17 @@ DACS_CACHE_CHART_VERSION ?= $(DACS_CACHE_CHART_BASE_VERSION)-$(BUILD_NUMBER) DACS_CACHE_NAMESPACE ?= cache-server define helm_install +$(HELM_BIN) dependency build ./helm/cache-server $(HELM_BIN) upgrade --install cache-server ./helm/cache-server \ --namespace $(HELM_NAMESPACE) --create-namespace \ --set cacheServer.image.repository=$(1) \ --set cacheServer.image.tag=$(2) \ --set cacheServer.scheduler.image.repository=$(3) \ --set cacheServer.scheduler.image.tag=$(4) \ - --wait --timeout 10m --atomic $(HELM_ARGS) + --set cacheServerPrereq.enabled=$(CACHE_SERVER_PREREQ_ENABLED) \ + --set cacheServerPrereq.nvmeRaidSetup.image.repository=$(CACHE_SETUP_IMG) \ + --set-string cacheServerPrereq.nvmeRaidSetup.image.tag=$(IMAGE_TAG) \ + --wait --timeout $(HELM_TIMEOUT) --atomic $(HELM_ARGS) endef define helm_install_remote @@ -201,7 +207,10 @@ $(HELM_BIN) upgrade --install cache-server oci://$(DOCKER_REGISTRY)/$(HELM_CHART --set cacheServer.image.tag=$(IMAGE_TAG) \ --set cacheServer.scheduler.image.repository=$(SCHEDULER_IMG) \ --set cacheServer.scheduler.image.tag=$(IMAGE_TAG) \ - --wait --timeout 10m --atomic $(HELM_ARGS) + --set cacheServerPrereq.enabled=$(CACHE_SERVER_PREREQ_ENABLED) \ + --set cacheServerPrereq.nvmeRaidSetup.image.repository=$(CACHE_SETUP_IMG) \ + --set-string cacheServerPrereq.nvmeRaidSetup.image.tag=$(IMAGE_TAG) \ + --wait --timeout $(HELM_TIMEOUT) --atomic $(HELM_ARGS) endef # Login to ACR for Helm if the registry is an Azure Container Registry @@ -219,15 +228,15 @@ fi endef .PHONY: helm-prereq-install -helm-prereq-install: helm ## Deploy dacs-prereq chart for NVMe RAID setup. +helm-prereq-install: helm ## Deploy standalone NVMe prerequisites for older parent charts. $(HELM_BIN) upgrade --install dacs-prereq ./helm/dacs-prereq \ --namespace $(HELM_NAMESPACE) --create-namespace \ --set nvmeRaidSetup.image.repository=$(CACHE_SETUP_IMG) \ - --set nvmeRaidSetup.image.tag=$(IMAGE_TAG) \ - --wait --timeout 10m --atomic $(HELM_ARGS) + --set-string nvmeRaidSetup.image.tag=$(IMAGE_TAG) \ + --wait --timeout $(HELM_TIMEOUT) --atomic $(HELM_ARGS) .PHONY: helm-prereq-uninstall -helm-prereq-uninstall: helm ## Uninstall dacs-prereq chart. +helm-prereq-uninstall: helm ## Uninstall standalone NVMe prerequisites. $(HELM_BIN) uninstall dacs-prereq --namespace $(HELM_NAMESPACE) --ignore-not-found --wait .PHONY: teardown-raid @@ -260,6 +269,7 @@ helm-uninstall: helm ## Uninstall cache-server chart. ##@ DACS Cache Controller Helm (New Controller) define helm_install_dacs_cache +$(HELM_BIN) dependency build ./$(DACS_CACHE_CHART_PATH) $(HELM_BIN) upgrade --install dacs ./$(DACS_CACHE_CHART_PATH) \ --namespace $(DACS_CACHE_NAMESPACE) --create-namespace \ --set-string global.registry= \ @@ -267,7 +277,10 @@ $(HELM_BIN) upgrade --install dacs ./$(DACS_CACHE_CHART_PATH) \ --set manager.image.tag=$(2) \ --set manager.cacheServerImage.repository=$(LOCAL_CACHE_SERVER_IMG) \ --set manager.cacheServerImage.tag=$(LOCAL_CACHE_SERVER_TAG) \ - --wait --timeout 5m --atomic $(HELM_ARGS) + --set cacheServerPrereq.enabled=$(CACHE_SERVER_PREREQ_ENABLED) \ + --set cacheServerPrereq.nvmeRaidSetup.image.repository=$(CACHE_SETUP_IMG) \ + --set-string cacheServerPrereq.nvmeRaidSetup.image.tag=$(IMAGE_TAG) \ + --wait --timeout $(HELM_TIMEOUT) --atomic $(HELM_ARGS) endef define helm_install_dacs_cache_remote @@ -277,7 +290,10 @@ $(HELM_BIN) upgrade --install dacs oci://$(DOCKER_REGISTRY)/$(DACS_CACHE_CHART_R --set global.registry=$(DOCKER_REGISTRY) \ --set manager.image.tag=$(IMAGE_TAG) \ --set manager.cacheServerImage.tag=$(IMAGE_TAG) \ - --wait --timeout 5m --atomic $(HELM_ARGS) + --set cacheServerPrereq.enabled=$(CACHE_SERVER_PREREQ_ENABLED) \ + --set cacheServerPrereq.nvmeRaidSetup.image.repository=$(CACHE_SETUP_IMG) \ + --set-string cacheServerPrereq.nvmeRaidSetup.image.tag=$(IMAGE_TAG) \ + --wait --timeout $(HELM_TIMEOUT) --atomic $(HELM_ARGS) endef .PHONY: helm-dacs-install @@ -303,14 +319,14 @@ ifeq ($(CONTROLLER),new) ifeq ($(CLUSTER_TYPE),minikube) @$(MAKE) helm-dacs-install else - @$(MAKE) helm-prereq-install helm-dacs-install + @$(MAKE) helm-dacs-install CACHE_SERVER_PREREQ_ENABLED=true endif else @echo "Deploying with old controller (cache-server helm chart)..." ifeq ($(CLUSTER_TYPE),minikube) @$(MAKE) helm-install else - @$(MAKE) helm-prereq-install helm-install + @$(MAKE) helm-install CACHE_SERVER_PREREQ_ENABLED=true endif endif @echo "Deployment complete (CONTROLLER=$(CONTROLLER))." diff --git a/deploy/scripts/aks/create.sh b/deploy/scripts/aks/create.sh index 69b64edb..224d132f 100755 --- a/deploy/scripts/aks/create.sh +++ b/deploy/scripts/aks/create.sh @@ -213,8 +213,5 @@ echo "" echo "AKS cluster created successfully!" echo "" echo "Next steps:" -echo " 1. Install dacs-prereq chart to set up NVMe RAID:" -echo " make helm-prereq-install DOCKER_REGISTRY=\"${DOCKER_REGISTRY:-dacstest.azurecr.io}\" BUILD_NUMBER=\"${BUILD_NUMBER:-}\"" -echo "" -echo " 2. Install cache-server chart:" -echo " helm install cache-server ./helm/cache-server -n cache-server" +echo " Install cache-server with NVMe prerequisites enabled:" +echo " make deploy DOCKER_REGISTRY=\"${DOCKER_REGISTRY:-dacstest.azurecr.io}\" BUILD_NUMBER=\"${BUILD_NUMBER:-}\"" diff --git a/helm/cache-server/Chart.yaml b/helm/cache-server/Chart.yaml index 29db305b..7f8a2c68 100644 --- a/helm/cache-server/Chart.yaml +++ b/helm/cache-server/Chart.yaml @@ -35,3 +35,10 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. appVersion: 1.16.0 + +dependencies: + - name: dacs-prereq + version: 0.1.0 + repository: file://../dacs-prereq + alias: cacheServerPrereq + condition: cacheServerPrereq.enabled diff --git a/helm/cache-server/README.md b/helm/cache-server/README.md index 4b8f9d2b..326b566b 100644 --- a/helm/cache-server/README.md +++ b/helm/cache-server/README.md @@ -11,9 +11,30 @@ This chart deploys the CacheServer distributed caching system for Kubernetes. ## Installation ```bash +helm dependency build ./helm/cache-server helm install cache-server ./helm/cache-server -n cache-server --create-namespace ``` +### Optional NVMe Prerequisites + +Enable the bundled prerequisite DaemonSet to prepare local NVMe storage: + +```bash +helm upgrade --install cache-server ./helm/cache-server \ + -n cache-server --create-namespace \ + --set cacheServerPrereq.enabled=true \ + --set cacheServerPrereq.nvmeRaidSetup.image.repository="/dacs/cache-setup" \ + --set-string cacheServerPrereq.nvmeRaidSetup.image.tag="" \ + --atomic --timeout 15m +``` + +Prerequisites default to off. Post-install and post-upgrade hooks verify +DaemonSet rollout completion. Rolling updates default to `10%` maximum +unavailable pods, configurable through +`cacheServerPrereq.updateStrategy.rollingUpdate.maxUnavailable`. +See the [shared prerequisite documentation](../dacs-prereq/README.md) +for configuration details. + ## Configuration See `values.yaml` for all configuration options. diff --git a/helm/cache-server/values.yaml b/helm/cache-server/values.yaml index 95047df3..548d91f9 100644 --- a/helm/cache-server/values.yaml +++ b/helm/cache-server/values.yaml @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +cacheServerPrereq: + enabled: false + # -- Node selector for system pod assignment. nodeSelector: dacs.azure.com/processing-unit: system diff --git a/helm/dacs-prereq/README.md b/helm/dacs-prereq/README.md index d3d19035..4c914e4b 100644 --- a/helm/dacs-prereq/README.md +++ b/helm/dacs-prereq/README.md @@ -1,7 +1,7 @@ # dacs-prereq Helm chart that deploys a DaemonSet to prepare NVMe data disks on each node -before the DACS cache server starts. +for the DACS cache server. ## Overview @@ -30,19 +30,29 @@ provided by the cache-setup image; no packages are installed on the host. ## Installation +Both parent charts bundle this chart as an optional dependency, disabled by +default. Enable it when the release should prepare NVMe storage: + ```bash -helm upgrade --install dacs-prereq ./helm/dacs-prereq \ +helm dependency build ./helm/cache-server +helm upgrade --install cache-server ./helm/cache-server \ --namespace cache-server --create-namespace \ - --set nvmeRaidSetup.image.repository="/dacs/cache-setup" \ - --set-string nvmeRaidSetup.image.tag="" \ - --wait --timeout 10m + --set cacheServerPrereq.enabled=true \ + --set cacheServerPrereq.nvmeRaidSetup.image.repository="/dacs/cache-setup" \ + --set-string cacheServerPrereq.nvmeRaidSetup.image.tag="" \ + --atomic --timeout 15m ``` -Or using the Makefile: +For the controller chart, use +`make helm-dacs-install CACHE_SERVER_PREREQ_ENABLED=true`. Wait for storage +preparation to succeed before creating Cache resources in a separate operation. +The rollout hook runs at the end of the release and does not block workloads +created in the same operation. Minikube leaves prerequisites disabled. -```bash -make helm-prereq-install DOCKER_REGISTRY="" BUILD_NUMBER="" -``` +The chart is also published independently and can be installed directly with +`make helm-prereq-install`. Published parent charts contain the build registry's +`dacs/cache-setup` image and stamp the bundled chart's `appVersion` with the +matching container version. Build the image with `make images`, or build it independently: @@ -57,6 +67,10 @@ image version as the rest of the deployment. | Parameter | Description | Default | |-----------|-------------|---------| +| `global.registry` | Registry host with optional project prefix, without a URL scheme | `""` | +| `global.registryMode` | `replace` redirects prerequisite image hosts; `legacy` leaves repositories unchanged | `legacy` | +| `global.imagePullSecrets` | Existing registry credential Secret references in the release namespace | `[]` | +| `name` | Base name for the DaemonSet and RBAC resources | `dacs-prereq` | | `nodeSelector` | Node selector for targeting NVMe nodes | `dacs.azure.com/processing-unit: cpu` | | `tolerations` | Tolerations for the DaemonSet | `[]` | | `mountPoint` | Mount point for the cache directory | `/var/lib/ssd/cacheserver` | @@ -65,6 +79,9 @@ image version as the rest of the deployment. | `updateStrategy.rollingUpdate.maxUnavailable` | Max unavailable during updates | `"10%"` | | `nvmeRaidSetup.image.repository` | Init container image | `cache-setup` | | `nvmeRaidSetup.image.tag` | Init container image tag; empty uses chart appVersion | `""` | +| `rolloutCheck.timeoutSeconds` | Current DaemonSet rollout timeout | `600` | +| `rolloutCheck.image.repository` | Rollout-check kubectl image | `mcr.microsoft.com/oss/v2/kubernetes/kubectl` | +| `rolloutCheck.image.tag` | Rollout-check kubectl image tag | `v1.35.9` | | `nvmeRaidSetup.resources.limits.cpu` | CPU limit for init container | `1` | | `nvmeRaidSetup.resources.limits.memory` | Memory limit for init container | `2Gi` | | `pauseContainer.image.repository` | Pause container image | `mcr.microsoft.com/azurelinux/base/core` | @@ -73,6 +90,31 @@ image version as the rest of the deployment. The init container uses a 512 MiB memory request and a 2 GiB limit. +### Private Registries + +Set these values at the parent chart's top level. Helm passes `global` values +to the bundled prerequisite chart: + +```yaml +cacheServerPrereq: + enabled: true +global: + registry: registry.example.com/dacs + registryMode: replace + imagePullSecrets: + - name: registry-auth +``` + +With `replace`, setup, pause, and rollout-check images use this registry while +preserving repository subpaths and tags. Mirror all three images there. +With `legacy`, their repositories stay unchanged. For mixed registries, leave +`global.registry` empty and override each prerequisite image directly. + +Create referenced Secrets in the release namespace before installation. +An empty list uses node-provided image authentication. These settings cover +the prerequisites and rollout Job, not controller or generated CacheServer +image authentication. + ## How It Works The DaemonSet uses `hostPID`, a privileged init container, and a host `/dev` @@ -86,6 +128,28 @@ Both single-disk and RAID0 formatting explicitly disable `orphan_file` and with Ubuntu 22.04's e2fsck 1.46.5. The host `fstab` entry uses fsck pass `2`, so the host can check the cache filesystem at boot. +### Rollout Verification + +A post-install/post-upgrade Job runs [rollout-check.sh](scripts/rollout-check.sh) +to verify the current DaemonSet rollout on at least one targeted node. +Its dedicated ServiceAccount, Role, and RoleBinding are namespace-scoped. +DaemonSet access is restricted to the prerequisite's name; pod and event lists +are used for failure diagnostics. + +Rollout commands share a 600-second budget by default, with up to three +attempts and five seconds between failures. The final node-count lookup has +a separate 40-second budget. Process-level timeouts preserve automatic +in-cluster API authentication. Diagnostics have a ten-second limit per command. +The Job deadline is the rollout timeout plus 180 seconds. Use a Helm timeout +longer than that deadline; 15 minutes covers the defaults. Persistent failures +fail the hook and an atomic Helm release. + +### Migrating a Standalone Release + +Uninstall a standalone `dacs-prereq` release before enabling the dependency in +a parent release in the same namespace. Helm cannot adopt another release's +resources. Host arrays, filesystems, and data remain in place. + ### Reboot Behavior **Managed OS (persistent root filesystem):** @@ -122,14 +186,16 @@ array, enabling a unified fast-path mount. ## Uninstallation +Uninstall the parent release to remove its bundled prerequisite resources: + ```bash -helm uninstall dacs-prereq --namespace cache-server +helm uninstall cache-server --namespace cache-server ``` -Or using the Makefile: +For a directly installed prerequisite release: ```bash -make helm-prereq-uninstall +helm uninstall dacs-prereq --namespace cache-server ``` **Note:** Uninstalling the chart does not remove the RAID array, filesystem, or diff --git a/helm/dacs-prereq/scripts/rollout-check.sh b/helm/dacs-prereq/scripts/rollout-check.sh new file mode 100644 index 00000000..8d3b57a4 --- /dev/null +++ b/helm/dacs-prereq/scripts/rollout-check.sh @@ -0,0 +1,82 @@ +#!/bin/sh +# Copyright (c) Microsoft Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +diagnostics() { + status=$? + if [ "$status" -ne 0 ]; then + timeout 10s kubectl -n "$NAMESPACE" get daemonset "$DAEMONSET" -o yaml || echo "DaemonSet diagnostics unavailable" >&2 + timeout 10s kubectl -n "$NAMESPACE" get pods -l "$POD_SELECTOR" -o wide || echo "Pod diagnostics unavailable" >&2 + timeout 10s kubectl -n "$NAMESPACE" get events --sort-by=.metadata.creationTimestamp || echo "Event diagnostics unavailable" >&2 + fi + exit "$status" +} + +# Each operation receives its remaining budget; retries never reset the deadline. +retry() ( + deadline=$(($(date +%s) + $1)) + shift + attempt=1 + while :; do + remaining=$((deadline - $(date +%s))) + if [ "$remaining" -le 0 ]; then + echo "$* retry deadline exceeded" >&2 + exit 1 + fi + if output=$("$@" "$remaining"); then + printf '%s\n' "$output" + exit 0 + else + status=$? + fi + # Discard failed stdout from the result, but retain it for diagnostics. + if [ -n "$output" ]; then + printf '%s\n' "$output" >&2 + fi + echo "$* attempt $attempt/3 failed (exit $status)" >&2 + remaining=$((deadline - $(date +%s))) + if [ "$attempt" -ge 3 ] || [ "$remaining" -le 5 ]; then + echo "$* exhausted its retry attempts or time budget" >&2 + exit "$status" + fi + sleep 5 + attempt=$((attempt + 1)) + done +) + +rollout_status() { + # Process timeouts preserve kubectl's automatic in-cluster authentication. + timeout "${1}s" kubectl -n "$NAMESPACE" \ + rollout status "daemonset/$DAEMONSET" --timeout="${1}s" +} + +desired_count() { + timeout "${1}s" kubectl -n "$NAMESPACE" \ + get daemonset "$DAEMONSET" -o jsonpath='{.status.desiredNumberScheduled}' +} + +trap diagnostics EXIT +retry "$ROLLOUT_TIMEOUT_SECONDS" rollout_status +# Check only after rollout status observes the current generation. +desired=$(retry 40 desired_count) +case "$desired" in + ''|*[!0-9]*) echo "Invalid desired node count: $desired" >&2; exit 1 ;; +esac +if [ "$desired" -eq 0 ]; then + echo "No nodes match the prerequisite DaemonSet" >&2 + exit 1 +fi +echo "Prerequisite rollout complete on $desired nodes" diff --git a/helm/dacs-prereq/templates/_helpers.tpl b/helm/dacs-prereq/templates/_helpers.tpl index 364f04eb..264a3b8f 100644 --- a/helm/dacs-prereq/templates/_helpers.tpl +++ b/helm/dacs-prereq/templates/_helpers.tpl @@ -16,23 +16,14 @@ Expand the name of the chart. */}} {{- define "dacs-prereq.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- .Values.name }} {{- end }} {{/* -Create a default fully qualified app name. +Create the rollout checker name. */}} -{{- define "dacs-prereq.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} +{{- define "dacs-prereq.rolloutCheckName" -}} +{{- printf "%s-rollout" .Values.name }} {{- end }} {{/* @@ -63,3 +54,15 @@ Create chart name and version as used by the chart label. {{- define "dacs-prereq.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} + +{{/* +Replace source hosts only when explicitly opted in; retain repository subpaths. +*/}} +{{- define "dacs-prereq.image" -}} +{{- $repository := .image.repository -}} +{{- if and (eq (.global.registryMode | default "legacy") "replace") .global.registry -}} +{{- $repository = regexReplaceAll "^([^/]*[.:][^/]*|localhost)/" $repository "" -}} +{{- $repository = printf "%s/%s" (.global.registry | trimSuffix "/") $repository -}} +{{- end -}} +{{- printf "%s:%s" $repository (.image.tag | default .appVersion) -}} +{{- end }} diff --git a/helm/dacs-prereq/templates/daemonset.yaml b/helm/dacs-prereq/templates/daemonset.yaml index ae8f1c43..0f5a7992 100644 --- a/helm/dacs-prereq/templates/daemonset.yaml +++ b/helm/dacs-prereq/templates/daemonset.yaml @@ -32,6 +32,10 @@ spec: kubectl.kubernetes.io/default-container: nvme-raid-setup spec: hostPID: true + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} {{- if .Values.priorityClassName }} priorityClassName: {{ .Values.priorityClassName }} {{- end }} @@ -46,7 +50,7 @@ spec: serviceAccountName: {{ .Values.name }} initContainers: - name: nvme-raid-setup - image: "{{ .Values.nvmeRaidSetup.image.repository }}:{{ .Values.nvmeRaidSetup.image.tag | default .Chart.AppVersion }}" + image: {{ include "dacs-prereq.image" (dict "global" .Values.global "image" .Values.nvmeRaidSetup.image "appVersion" .Chart.AppVersion) | quote }} imagePullPolicy: {{ .Values.nvmeRaidSetup.image.pullPolicy }} securityContext: privileged: true @@ -115,7 +119,7 @@ spec: {{- toYaml .Values.nvmeRaidSetup.resources | nindent 12 }} containers: - name: pause - image: "{{ .Values.pauseContainer.image.repository }}:{{ .Values.pauseContainer.image.tag }}" + image: {{ include "dacs-prereq.image" (dict "global" .Values.global "image" .Values.pauseContainer.image "appVersion" .Chart.AppVersion) | quote }} imagePullPolicy: {{ .Values.pauseContainer.image.pullPolicy }} command: - sleep diff --git a/helm/dacs-prereq/templates/rbac.yaml b/helm/dacs-prereq/templates/rbac.yaml index e5c770f3..8b5cf5b2 100644 --- a/helm/dacs-prereq/templates/rbac.yaml +++ b/helm/dacs-prereq/templates/rbac.yaml @@ -39,6 +39,7 @@ metadata: subjects: - kind: ServiceAccount name: {{ .Values.name }} + namespace: {{ .Release.Namespace }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role diff --git a/helm/dacs-prereq/templates/rollout-check.yaml b/helm/dacs-prereq/templates/rollout-check.yaml new file mode 100644 index 00000000..a158d1cc --- /dev/null +++ b/helm/dacs-prereq/templates/rollout-check.yaml @@ -0,0 +1,112 @@ +# Copyright (c) Microsoft Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "dacs-prereq.rolloutCheckName" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "dacs-prereq.labels" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "dacs-prereq.rolloutCheckName" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "dacs-prereq.labels" . | nindent 4 }} +rules: + - apiGroups: ["apps"] + resources: ["daemonsets"] + resourceNames: [{{ .Values.name | quote }}] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods", "events"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "dacs-prereq.rolloutCheckName" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "dacs-prereq.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "dacs-prereq.rolloutCheckName" . }} + namespace: {{ .Release.Namespace | quote }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "dacs-prereq.rolloutCheckName" . }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "dacs-prereq.rolloutCheckName" . }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "dacs-prereq.labels" . | nindent 4 }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: 0 + activeDeadlineSeconds: {{ add .Values.rolloutCheck.timeoutSeconds 180 }} + template: + metadata: + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: rollout-check + spec: + restartPolicy: Never + {{- with .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "dacs-prereq.rolloutCheckName" . }} + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: rollout-check + image: {{ include "dacs-prereq.image" (dict "global" .Values.global "image" .Values.rolloutCheck.image "appVersion" .Chart.AppVersion) | quote }} + imagePullPolicy: {{ .Values.rolloutCheck.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + {{- toYaml .Values.rolloutCheck.resources | nindent 12 }} + env: + - name: ROLLOUT_TIMEOUT_SECONDS + value: {{ .Values.rolloutCheck.timeoutSeconds | quote }} + - name: DAEMONSET + value: {{ .Values.name | quote }} + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_SELECTOR + value: {{ printf "app.kubernetes.io/name=%s,app.kubernetes.io/instance=%s" (include "dacs-prereq.name" .) .Release.Name | quote }} + command: + - /bin/sh + - -c + - | + {{- .Files.Get "scripts/rollout-check.sh" | nindent 14 }} diff --git a/helm/dacs-prereq/values.schema.json b/helm/dacs-prereq/values.schema.json new file mode 100644 index 00000000..12dc59ee --- /dev/null +++ b/helm/dacs-prereq/values.schema.json @@ -0,0 +1,60 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["name", "updateStrategy", "rolloutCheck"], + "properties": { + "global": { + "type": "object", + "properties": { + "registry": { + "type": "string", + "pattern": "^$|^[a-zA-Z0-9][a-zA-Z0-9.:-]*(/[a-z0-9._-]+)*/?$" + }, + "registryMode": { + "type": "string", + "enum": ["legacy", "replace"] + }, + "imagePullSecrets": { + "type": "array", + "items": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 253, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" + } + } + } + } + } + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 55, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" + }, + "updateStrategy": { + "type": "object", + "required": ["type"], + "properties": { + "type": { "type": "string", "enum": ["RollingUpdate"] } + } + }, + "rolloutCheck": { + "type": "object", + "required": ["timeoutSeconds"], + "properties": { + "timeoutSeconds": { + "type": "integer", + "minimum": 1, + "maximum": 2147483467 + } + } + } + } +} diff --git a/helm/dacs-prereq/values.yaml b/helm/dacs-prereq/values.yaml index 69898c7a..da4a2c5f 100644 --- a/helm/dacs-prereq/values.yaml +++ b/helm/dacs-prereq/values.yaml @@ -15,6 +15,11 @@ # Default values for dacs-prereq # Base name used for all resources (DaemonSet, ServiceAccount, Role, RoleBinding) +global: + registry: "" + registryMode: legacy + imagePullSecrets: [] + name: dacs-prereq # Node selector for targeting NVMe-capable nodes @@ -38,6 +43,21 @@ updateStrategy: rollingUpdate: maxUnavailable: "10%" +# Post-install/upgrade verification of the latest DaemonSet rollout. +rolloutCheck: + timeoutSeconds: 600 + image: + repository: mcr.microsoft.com/oss/v2/kubernetes/kubectl + tag: v1.35.9 + pullPolicy: IfNotPresent + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + # nvme-raid-setup init container settings nvmeRaidSetup: image: diff --git a/src/subsystems/cache-controller/charts/dacs/Chart.yaml b/src/subsystems/cache-controller/charts/dacs/Chart.yaml index e240d410..2464f421 100644 --- a/src/subsystems/cache-controller/charts/dacs/Chart.yaml +++ b/src/subsystems/cache-controller/charts/dacs/Chart.yaml @@ -21,6 +21,13 @@ version: 0.1.0 # Application version - should match the controller image tag appVersion: "local" +dependencies: + - name: dacs-prereq + version: 0.1.0 + repository: file://../../../../../helm/dacs-prereq + alias: cacheServerPrereq + condition: cacheServerPrereq.enabled + keywords: - cache - storage diff --git a/src/subsystems/cache-controller/charts/dacs/README.md b/src/subsystems/cache-controller/charts/dacs/README.md index 60f54f14..b2dff814 100644 --- a/src/subsystems/cache-controller/charts/dacs/README.md +++ b/src/subsystems/cache-controller/charts/dacs/README.md @@ -10,6 +10,12 @@ A Kubernetes controller that manages distributed cache infrastructure through cu ## Quick Start +When installing from this repository, build the bundled dependency first: + +```bash +helm dependency build ./src/subsystems/cache-controller/charts/dacs +``` + ### Install Controller Only ```bash @@ -44,6 +50,31 @@ performance tests. ## Configuration +### Optional NVMe Prerequisites + +Set `cacheServerPrereq.enabled=true` to include NVMe setup in this release. +It defaults to `false`. For example, from the repository root: + +```bash +helm upgrade --install dacs ./src/subsystems/cache-controller/charts/dacs \ + --namespace dacs-system --create-namespace \ + --set cacheServerPrereq.enabled=true \ + --set cacheServerPrereq.nvmeRaidSetup.image.repository="/dacs/cache-setup" \ + --set-string cacheServerPrereq.nvmeRaidSetup.image.tag="" \ + --atomic --timeout 15m +``` + +Post-install and post-upgrade hooks verify the DaemonSet rollout. The maximum +unavailable pods defaults to `10%`, configurable through +`cacheServerPrereq.updateStrategy.rollingUpdate.maxUnavailable`. +See the [shared prerequisite documentation](../../../../../helm/dacs-prereq/README.md) +for configuration details. + +Wait for the controller/prerequisite release to succeed, then apply a +standalone Cache manifest from [`examples`](examples/README.md). The hook +runs after resources are created, so storage preparation must finish before +creating Cache workloads. + ### Controller Configuration | Parameter | Description | Default | diff --git a/src/subsystems/cache-controller/charts/dacs/values.yaml b/src/subsystems/cache-controller/charts/dacs/values.yaml index c738a971..36e45df1 100644 --- a/src/subsystems/cache-controller/charts/dacs/values.yaml +++ b/src/subsystems/cache-controller/charts/dacs/values.yaml @@ -18,6 +18,9 @@ global: # Name prefix for all resources (e.g., dacs-manager, dacs-webhook) namePrefix: dacs +cacheServerPrereq: + enabled: false + # Configure the controller manager deployment manager: replicas: 1 diff --git a/src/subsystems/cache-controller/internal/controller/helm_cleanup_test.go b/src/subsystems/cache-controller/internal/controller/helm_cleanup_test.go index 594d671f..325aae26 100644 --- a/src/subsystems/cache-controller/internal/controller/helm_cleanup_test.go +++ b/src/subsystems/cache-controller/internal/controller/helm_cleanup_test.go @@ -34,6 +34,9 @@ func TestHelmStandaloneCache(t *testing.T) { if err != nil { t.Skip("helm is required for chart rendering tests") } + if output, err := exec.Command(helm, "dependency", "build", "../../charts/dacs", "--skip-refresh").CombinedOutput(); err != nil { + t.Fatalf("building chart dependencies: %v\n%s", err, output) + } for _, tt := range []struct { name string args []string @@ -68,6 +71,9 @@ func TestHelmCacheCleanup(t *testing.T) { if err != nil { t.Skip("helm is required for chart rendering tests") } + if output, err := exec.Command(helm, "dependency", "build", "../../charts/dacs", "--skip-refresh").CombinedOutput(); err != nil { + t.Fatalf("building chart dependencies: %v\n%s", err, output) + } chart := "../../charts/dacs" script, err := os.ReadFile(filepath.Join(chart, "scripts/pre-delete-cache-cleanup.sh")) if err != nil { diff --git a/src/subsystems/cache-controller/internal/controller/helm_prerequisites_test.go b/src/subsystems/cache-controller/internal/controller/helm_prerequisites_test.go new file mode 100644 index 00000000..1c123162 --- /dev/null +++ b/src/subsystems/cache-controller/internal/controller/helm_prerequisites_test.go @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "bytes" + "io" + "os/exec" + "reflect" + "strings" + "testing" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" +) + +func TestHelmPrerequisites(t *testing.T) { + helm, err := exec.LookPath("helm") + if err != nil { + t.Skip("helm is required for chart rendering tests") + } + if output, err := exec.Command(helm, "dependency", "build", "../../charts/dacs", "--skip-refresh").CombinedOutput(); err != nil { + t.Fatalf("building chart dependencies: %v\n%s", err, output) + } + controllerDefault := []corev1.Toleration{{Key: "CriticalAddonsOnly", Operator: corev1.TolerationOpExists}} + customController := []corev1.Toleration{{Key: "controller-only", Operator: corev1.TolerationOpExists}} + for _, tt := range []struct { + name string + args []string + manager []corev1.Toleration + prereq bool + global string + localized bool + pullSecrets []corev1.LocalObjectReference + invalidProperty string + }{ + {name: "defaults", manager: controllerDefault}, + {name: "independent controller", args: []string{"--set-json", `manager.tolerations=[{"key":"controller-only","operator":"Exists"}]`}, manager: customController}, + {name: "bundled prerequisites", manager: controllerDefault, prereq: true}, + {name: "legacy prerequisite registry", manager: controllerDefault, prereq: true, global: `{"registry":"mirror.example.com:5000/project/","imagePullSecrets":[{"name":"registry-auth"}]}`, pullSecrets: []corev1.LocalObjectReference{{Name: "registry-auth"}}}, + {name: "private prerequisites", manager: controllerDefault, prereq: true, global: `{"registry":"mirror.example.com:5000/project/","registryMode":"replace","imagePullSecrets":[{"name":"registry-auth"}]}`, localized: true, pullSecrets: []corev1.LocalObjectReference{{Name: "registry-auth"}}}, + {name: "private prerequisites without pull secrets", manager: controllerDefault, prereq: true, global: `{"registry":"mirror.example.com:5000/project/","registryMode":"replace","imagePullSecrets":[]}`, localized: true}, + {name: "invalid prerequisite registry", prereq: true, global: `{"registry":"https://mirror.example.com"}`, invalidProperty: "global.registry"}, + {name: "invalid prerequisite registry mode", prereq: true, global: `{"registryMode":"invalid"}`, invalidProperty: "global.registryMode"}, + {name: "invalid prerequisite pull secret", prereq: true, global: `{"imagePullSecrets":[{"name":""}]}`, invalidProperty: "global.imagePullSecrets"}, + } { + t.Run(tt.name, func(t *testing.T) { + args := []string{"template", "prerequisites", "../../charts/dacs"} + if tt.prereq { + args = append(args, "--set", "cacheServerPrereq.enabled=true", + "--set", "cacheServerPrereq.nvmeRaidSetup.image.repository=example.azurecr.io/cache-setup", + "--set-string", "cacheServerPrereq.nvmeRaidSetup.image.tag=20261016.1") + } + if tt.global != "" { + args = append(args, "--set-json", "global="+tt.global) + } + args = append(args, tt.args...) + output, err := exec.Command(helm, args...).CombinedOutput() + if tt.invalidProperty != "" { + if err == nil { + t.Fatal("invalid values passed chart schema validation") + } + property := tt.invalidProperty + if !bytes.Contains(output, []byte(property)) && !bytes.Contains(output, []byte("/"+strings.ReplaceAll(property, ".", "/"))) { + t.Fatalf("chart failed for an unrelated reason: %s", output) + } + return + } + if err != nil { + t.Fatalf("helm template: %v\n%s", err, output) + } + decoder := yamlutil.NewYAMLOrJSONDecoder(bytes.NewReader(output), 4096) + foundManager := false + foundPrereq, foundRollout := false, false + setupImage := "example.azurecr.io/cache-setup:20261016.1" + pauseImage := "mcr.microsoft.com/azurelinux/base/core:3.0" + rolloutImage := "mcr.microsoft.com/oss/v2/kubernetes/kubectl:v1.35.9" + if tt.localized { + setupImage = "mirror.example.com:5000/project/cache-setup:20261016.1" + pauseImage = "mirror.example.com:5000/project/azurelinux/base/core:3.0" + rolloutImage = "mirror.example.com:5000/project/oss/v2/kubernetes/kubectl:v1.35.9" + } + for { + obj := &unstructured.Unstructured{} + if err := decoder.Decode(obj); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + switch obj.GetKind() { + case "DaemonSet": + foundPrereq = true + var daemonset appsv1.DaemonSet + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &daemonset); err != nil { + t.Fatal(err) + } + init := daemonset.Spec.Template.Spec.InitContainers + if len(init) != 1 || init[0].Image != setupImage { + t.Fatalf("prerequisite init containers: %#v", init) + } + if !daemonset.Spec.Template.Spec.HostPID { + t.Fatal("NVMe setup requires host PID access") + } + pod := daemonset.Spec.Template.Spec + if len(pod.Containers) != 1 || pod.Containers[0].Image != pauseImage { + t.Fatalf("prerequisite pause containers: %#v", pod.Containers) + } + if !reflect.DeepEqual(pod.ImagePullSecrets, tt.pullSecrets) { + t.Fatalf("prerequisite pull secrets: got %#v, want %#v", pod.ImagePullSecrets, tt.pullSecrets) + } + case "Job": + if obj.GetAnnotations()["helm.sh/hook"] != "post-install,post-upgrade" { + continue + } + foundRollout = true + var job batchv1.Job + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &job); err != nil { + t.Fatal(err) + } + if job.Spec.ActiveDeadlineSeconds == nil || *job.Spec.ActiveDeadlineSeconds != 780 { + t.Fatalf("rollout-check deadline: %v", job.Spec.ActiveDeadlineSeconds) + } + pod := job.Spec.Template.Spec + if len(pod.Containers) != 1 || pod.Containers[0].Image != rolloutImage { + t.Fatalf("rollout-check containers: %#v", pod.Containers) + } + if strings.Contains(strings.Join(pod.Containers[0].Command, "\n"), "--request-timeout") { + t.Fatal("rollout check must use process timeouts to preserve in-cluster authentication") + } + if !reflect.DeepEqual(pod.ImagePullSecrets, tt.pullSecrets) { + t.Fatalf("rollout-check pull secrets: got %#v, want %#v", pod.ImagePullSecrets, tt.pullSecrets) + } + case "Cache": + t.Fatal("controller chart must not create Cache resources") + case "Deployment": + foundManager = true + deployment := &appsv1.Deployment{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, deployment); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(deployment.Spec.Template.Spec.Tolerations, tt.manager) { + t.Fatalf("controller tolerations changed: got %#v, want %#v", deployment.Spec.Template.Spec.Tolerations, tt.manager) + } + } + } + if !foundManager { + t.Fatal("rendered chart is missing controller Deployment") + } + if foundPrereq != tt.prereq || foundRollout != tt.prereq { + t.Fatalf("bundled prerequisites: daemonset=%t, rollout=%t, want %t", foundPrereq, foundRollout, tt.prereq) + } + }) + } +} diff --git a/test/README.md b/test/README.md index 5e839852..289209df 100644 --- a/test/README.md +++ b/test/README.md @@ -17,20 +17,18 @@ The test framework uses controller-runtime client for cluster interactions. The test framework will: 1. Create the `cache-server` namespace -2. Deploy dacs-prereq (RAID setup) if `INSTALL_PREREQ=true` -3. Deploy cache-server (scheduler + StatefulSet) if `INSTALL=true` -4. Verify all pods are ready -5. Run test scenarios -6. Collect support bundle on completion +2. Deploy cache-server if `INSTALL=true`, including NVMe prerequisites when `CACHE_SERVER_PREREQ_ENABLED=true` +3. Verify all pods are ready +4. Run test scenarios +5. Collect support bundle on completion #### Environment Variables | Variable | Default | Description | |----------|---------|-------------| | `INSTALL` | `false` | Install cache-server helm chart before tests | -| `INSTALL_PREREQ` | `false` | Install dacs-prereq helm chart before tests | +| `CACHE_SERVER_PREREQ_ENABLED` | `false` | Enable the bundled prerequisites when `INSTALL=true` | | `UNINSTALL` | `false` | Uninstall cache-server helm chart after tests | -| `UNINSTALL_PREREQ` | `false` | Uninstall dacs-prereq helm chart after tests | | `SKIP_SUPPORT_BUNDLE` | `false` | Skip collecting support bundle after tests | | `DOCKER_REGISTRY` | - | Docker registry to pull images from | | `BUILD_NUMBER` | - | Image tag to use for cache-server images | @@ -140,7 +138,7 @@ az aks update -g $AKS_RESOURCE_GROUP -n ${AKS_RESOURCE_GROUP}-cluster --attach-a # Build and publish images DOCKER_REGISTRY=myacr.azurecr.io BUILD_NUMBER=v1.0.0 make images publish-images -# Install helm charts (prereq for NVMe RAID setup, then cache-server) +# Install cache-server with bundled NVMe prerequisites DOCKER_REGISTRY=myacr.azurecr.io BUILD_NUMBER=v1.0.0 make deploy # Run e2e tests (auto-detects AKS, runs all tests) diff --git a/test/pkg/common/common.go b/test/pkg/common/common.go index 3b40c2b9..f63edb1d 100644 --- a/test/pkg/common/common.go +++ b/test/pkg/common/common.go @@ -64,18 +64,14 @@ var ( // Optional Environment Variables: // - INSTALL=true: Install DACS components before tests. Default is to // skip installation (assumes components are already deployed). - // - INSTALL_PREREQ=true: Install dacs-prereq before tests. Default - // is to skip installation. + // - CACHE_SERVER_PREREQ_ENABLED=true: Include NVMe prerequisites when + // installing the parent chart. Default is false. // - UNINSTALL=true: Uninstall DACS components after tests. Default is // to skip uninstallation. - // - UNINSTALL_PREREQ=true: Uninstall dacs-prereq after tests. - // Default is to skip uninstallation. // - SKIP_SUPPORT_BUNDLE=true: Skips collecting support bundle after the // test. doInstall = os.Getenv("INSTALL") == trueString - doInstallPrereq = os.Getenv("INSTALL_PREREQ") == trueString doUninstall = os.Getenv("UNINSTALL") == trueString - doUninstallPrereq = os.Getenv("UNINSTALL_PREREQ") == trueString skipSupportBundle = os.Getenv("SKIP_SUPPORT_BUNDLE") == trueString startTime = time.Now() @@ -156,21 +152,12 @@ func Setup(ctx context.Context, k8sClient client.Client, namespace string) { } }).WithContext(ctx).Should(Succeed(), "Namespace creation failed or namespace not found") - if doInstallPrereq { - By("deploying cache server prerequisites") - cmd = exec.CommandContext(ctx, "make", "helm-prereq-install") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to deploy cache server prerequisites") - } else { - log.Info("Skipping dacs-prereq installation (set INSTALL_PREREQ=true to install)") - } - By("deploying cache server") Eventually(func(g Gomega, ctx context.Context) { cmd = exec.CommandContext(ctx, "make", "helm-install") _, err = utils.Run(cmd) g.Expect(err).NotTo(HaveOccurred(), "Failed to deploy cache server") - }).WithTimeout(5*time.Minute).WithContext(ctx).Should(Succeed(), "Failed to deploy cache server") + }).WithTimeout(20*time.Minute).WithContext(ctx).Should(Succeed(), "Failed to deploy cache server") ensureCacheReady(ctx, k8sClient, namespace, LegacyStatefulSetName, false) } @@ -238,17 +225,6 @@ func Teardown(ctx context.Context, k8sClient client.Client, namespace string, su g.Expect(err).NotTo(HaveOccurred(), "Failed to uninstall DACS") }).WithTimeout(5*time.Minute).WithContext(ctx).Should(Succeed(), "Failed to uninstall DACS") - if doUninstallPrereq { - By("uninstalling cache server prerequisites") - Eventually(func(g Gomega, ctx context.Context) { - cmd := exec.CommandContext(ctx, "make", "helm-prereq-uninstall") - _, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred(), "Failed to uninstall cache server prerequisites") - }).WithTimeout(5*time.Minute).WithContext(ctx).Should(Succeed(), "Failed to uninstall cache server prerequisites") - } else { - log.Info("Skipping dacs-prereq uninstallation (set UNINSTALL_PREREQ=true to uninstall)") - } - if namespace != "kube-system" && namespace != "dacs" { By("removing namespace") Eventually(func(g Gomega, ctx context.Context) { diff --git a/user-guide.md b/user-guide.md index fba12451..5876219f 100644 --- a/user-guide.md +++ b/user-guide.md @@ -67,25 +67,25 @@ To install Helm, please follow the official [Helm installation guide](https://he ## Install DACS -Use the same release version for the prerequisite and DACS charts. The DACS -chart's `appVersion` identifies the matching container-safe image tag. If you +The DACS chart bundles optional NVMe prerequisites. Its `appVersion` +identifies the matching container-safe image tag. If you override image settings, keep `dacstest.azurecr.io/dacs/cache-controller`, `dacstest.azurecr.io/dacs/cache-server`, and `dacstest.azurecr.io/dacs/cache-test-client` on that matching version. -### Step 1: Install NVMe RAID Prerequisites (Required) +### Step 1: Install the Controller with NVMe Prerequisites -The cache server requires local NVMe storage to be configured as a RAID array. -Install the prerequisite chart to automatically set up RAID on each node: +Enable the bundled dependency to prepare local NVMe storage: ```sh -helm install dacs-prereq \ - oci://dacstest.azurecr.io/dacs/helmchart/dacs-prereq \ +helm install dacs \ + oci://dacstest.azurecr.io/dacs/helmchart/dacs \ --version \ --namespace cache-server \ --create-namespace \ - --wait --timeout 10m + --set cacheServerPrereq.enabled=true \ + --atomic --timeout 15m ``` When installed, a DaemonSet will automatically: @@ -99,25 +99,14 @@ When installed, a DaemonSet will automatically: > The RAID setup is idempotent. If a RAID array already exists, the script > will detect it and skip creation. > -> The init container requests 512 MiB of memory and has a 2 GiB limit. The -> higher limit accommodates Azure Linux package metadata refreshes while the -> storage utilities are installed. - -### Step 2: Install DACS Cache Controller - -The `dacs` chart deploys a Kubernetes controller that manages cache -infrastructure through a Cache custom resource (CR). +> The init container requests 512 MiB of memory and has a 2 GiB limit. +> Storage tools are preinstalled in the cache-setup image. -```sh -helm install dacs \ - oci://dacstest.azurecr.io/dacs/helmchart/dacs \ - --version \ - --namespace cache-server \ - --create-namespace \ - --wait --timeout 10m -``` +Wait for the release and its prerequisite rollout check to succeed before +creating a Cache. The hook does not block workloads created in the same +operation or stop existing Cache workloads during storage maintenance. -### Step 3: Apply an Example Cache +### Step 2: Deploy a Cache Instance After the controller and prerequisites are ready, apply a standalone Cache: @@ -194,11 +183,14 @@ cache-sample-2 1/1 Running 0 2m The most common configuration options: +These commands update an existing release after storage setup succeeds. + ```sh helm upgrade dacs \ oci://dacstest.azurecr.io/dacs/helmchart/dacs \ --version \ --namespace cache-server \ + --reuse-values \ --set manager.logLevel=debug ``` @@ -307,7 +299,7 @@ kubectl get nodes --show-labels | grep dacs.azure.com/processing-unit Check the prereq logs: ```sh -kubectl logs -n cache-server -l app.kubernetes.io/name=dacs-prereq -c nvme-raid-setup +kubectl logs -n cache-server -l app.kubernetes.io/component=nvme-setup,app.kubernetes.io/instance=dacs -c nvme-raid-setup ``` Common causes: @@ -342,17 +334,11 @@ To uninstall DACS, follow these steps: helm uninstall dacs -n cache-server ``` -3. Uninstall the prerequisites (optional): - - ```sh - helm uninstall dacs-prereq -n cache-server - ``` - > [!NOTE] - > Uninstalling the prereq chart does **not** remove the RAID array or data. + > Uninstalling the parent chart removes its prerequisite resources but does **not** remove the RAID array or data. > The array persists on the node until manually removed. -4. Delete the namespace (optional): +3. Delete the namespace (optional): ```sh kubectl delete namespace cache-server From 6b42db5a32ee22b7ddda5e05de14f41864005044 Mon Sep 17 00:00:00 2001 From: Joshua Long <44989821+jmclong@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:15:37 +0000 Subject: [PATCH 2/2] fix(helm): align rollout Job scheduling with controller Tolerate CriticalAddonsOnly and prefer AKS system nodes without requiring a system pool. Match the controller's Linux amd64/arm64 eligibility, expose independent Job scheduling values, and cover defaults and overrides in Helm regression tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: f5ab036a-4dfa-4f96-809f-5f4a4d7f6b54 --- helm/dacs-prereq/README.md | 11 ++++ helm/dacs-prereq/templates/rollout-check.yaml | 8 +++ helm/dacs-prereq/values.yaml | 25 +++++++++ .../controller/helm_prerequisites_test.go | 56 ++++++++++++++++--- 4 files changed, 92 insertions(+), 8 deletions(-) diff --git a/helm/dacs-prereq/README.md b/helm/dacs-prereq/README.md index 4c914e4b..2febd589 100644 --- a/helm/dacs-prereq/README.md +++ b/helm/dacs-prereq/README.md @@ -80,6 +80,8 @@ image version as the rest of the deployment. | `nvmeRaidSetup.image.repository` | Init container image | `cache-setup` | | `nvmeRaidSetup.image.tag` | Init container image tag; empty uses chart appVersion | `""` | | `rolloutCheck.timeoutSeconds` | Current DaemonSet rollout timeout | `600` | +| `rolloutCheck.tolerations` | Tolerations for the rollout-check Job, independent of the DaemonSet | `CriticalAddonsOnly` with operator `Exists` | +| `rolloutCheck.affinity` | Affinity for the rollout-check Job | Linux amd64/arm64 nodes, with a soft preference for AKS system nodes | | `rolloutCheck.image.repository` | Rollout-check kubectl image | `mcr.microsoft.com/oss/v2/kubernetes/kubectl` | | `rolloutCheck.image.tag` | Rollout-check kubectl image tag | `v1.35.9` | | `nvmeRaidSetup.resources.limits.cpu` | CPU limit for init container | `1` | @@ -136,6 +138,15 @@ Its dedicated ServiceAccount, Role, and RoleBinding are namespace-scoped. DaemonSet access is restricted to the prerequisite's name; pod and event lists are used for failure diagnostics. +The Job's scheduling defaults match the controller Deployment: it tolerates +`CriticalAddonsOnly`, requires Linux amd64/arm64 nodes, and prefers nodes +labeled `kubernetes.azure.com/mode=system`. The system-pool preference is soft, +so it can also run on other eligible nodes. Other taints still need matching +tolerations. +Configure `rolloutCheck.tolerations` and `rolloutCheck.affinity` independently +of the DaemonSet's scheduling settings. For a bundled chart, use the +`cacheServerPrereq.rolloutCheck` values. + Rollout commands share a 600-second budget by default, with up to three attempts and five seconds between failures. The final node-count lookup has a separate 40-second budget. Process-level timeouts preserve automatic diff --git a/helm/dacs-prereq/templates/rollout-check.yaml b/helm/dacs-prereq/templates/rollout-check.yaml index a158d1cc..0a8ad7a0 100644 --- a/helm/dacs-prereq/templates/rollout-check.yaml +++ b/helm/dacs-prereq/templates/rollout-check.yaml @@ -77,6 +77,14 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ include "dacs-prereq.rolloutCheckName" . }} + {{- with .Values.rolloutCheck.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.rolloutCheck.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} securityContext: runAsNonRoot: true runAsUser: 65532 diff --git a/helm/dacs-prereq/values.yaml b/helm/dacs-prereq/values.yaml index da4a2c5f..942fbdde 100644 --- a/helm/dacs-prereq/values.yaml +++ b/helm/dacs-prereq/values.yaml @@ -46,6 +46,31 @@ updateStrategy: # Post-install/upgrade verification of the latest DaemonSet rollout. rolloutCheck: timeoutSeconds: 600 + tolerations: + - key: CriticalAddonsOnly + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - arm64 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: kubernetes.azure.com/mode + operator: In + values: + - system image: repository: mcr.microsoft.com/oss/v2/kubernetes/kubectl tag: v1.35.9 diff --git a/src/subsystems/cache-controller/internal/controller/helm_prerequisites_test.go b/src/subsystems/cache-controller/internal/controller/helm_prerequisites_test.go index 1c123162..a9904b57 100644 --- a/src/subsystems/cache-controller/internal/controller/helm_prerequisites_test.go +++ b/src/subsystems/cache-controller/internal/controller/helm_prerequisites_test.go @@ -40,19 +40,44 @@ func TestHelmPrerequisites(t *testing.T) { } controllerDefault := []corev1.Toleration{{Key: "CriticalAddonsOnly", Operator: corev1.TolerationOpExists}} customController := []corev1.Toleration{{Key: "controller-only", Operator: corev1.TolerationOpExists}} + rolloutAffinity := &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + {Key: "kubernetes.io/os", Operator: corev1.NodeSelectorOpIn, Values: []string{"linux"}}, + {Key: "kubernetes.io/arch", Operator: corev1.NodeSelectorOpIn, Values: []string{"amd64", "arm64"}}, + }, + }}, + }, + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{{ + Weight: 50, + Preference: corev1.NodeSelectorTerm{ + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "kubernetes.azure.com/mode", Operator: corev1.NodeSelectorOpIn, Values: []string{"system"}, + }}, + }, + }}, + }, + } for _, tt := range []struct { - name string - args []string - manager []corev1.Toleration - prereq bool - global string - localized bool - pullSecrets []corev1.LocalObjectReference - invalidProperty string + name string + args []string + manager []corev1.Toleration + prereq bool + global string + localized bool + pullSecrets []corev1.LocalObjectReference + invalidProperty string + overrideRolloutScheduling bool }{ {name: "defaults", manager: controllerDefault}, {name: "independent controller", args: []string{"--set-json", `manager.tolerations=[{"key":"controller-only","operator":"Exists"}]`}, manager: customController}, {name: "bundled prerequisites", manager: controllerDefault, prereq: true}, + {name: "rollout scheduling overridden", manager: controllerDefault, prereq: true, overrideRolloutScheduling: true, args: []string{ + "--set-json", "cacheServerPrereq.rolloutCheck.tolerations=[]", + "--set-json", "cacheServerPrereq.rolloutCheck.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution=[]", + }}, {name: "legacy prerequisite registry", manager: controllerDefault, prereq: true, global: `{"registry":"mirror.example.com:5000/project/","imagePullSecrets":[{"name":"registry-auth"}]}`, pullSecrets: []corev1.LocalObjectReference{{Name: "registry-auth"}}}, {name: "private prerequisites", manager: controllerDefault, prereq: true, global: `{"registry":"mirror.example.com:5000/project/","registryMode":"replace","imagePullSecrets":[{"name":"registry-auth"}]}`, localized: true, pullSecrets: []corev1.LocalObjectReference{{Name: "registry-auth"}}}, {name: "private prerequisites without pull secrets", manager: controllerDefault, prereq: true, global: `{"registry":"mirror.example.com:5000/project/","registryMode":"replace","imagePullSecrets":[]}`, localized: true}, @@ -137,6 +162,21 @@ func TestHelmPrerequisites(t *testing.T) { t.Fatalf("rollout-check deadline: %v", job.Spec.ActiveDeadlineSeconds) } pod := job.Spec.Template.Spec + wantTolerations, wantAffinity := controllerDefault, rolloutAffinity + if tt.overrideRolloutScheduling { + wantTolerations = nil + wantAffinity = rolloutAffinity.DeepCopy() + wantAffinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution = []corev1.PreferredSchedulingTerm{} + } + if !reflect.DeepEqual(pod.Tolerations, wantTolerations) { + t.Fatalf("rollout-check tolerations: got %#v, want %#v", pod.Tolerations, wantTolerations) + } + if !reflect.DeepEqual(pod.Affinity, wantAffinity) { + t.Fatalf("rollout-check affinity: got %#v, want %#v", pod.Affinity, wantAffinity) + } + if len(pod.NodeSelector) != 0 { + t.Fatalf("rollout check must not require the DaemonSet's node selector: %#v", pod.NodeSelector) + } if len(pod.Containers) != 1 || pod.Containers[0].Image != rolloutImage { t.Fatalf("rollout-check containers: %#v", pod.Containers) }