feat(helm)!: separate Cache resources from controller releases

.github/workflows/ci.yml

@@ -319,6 +319,11 @@ jobs:
           make deploy CONTROLLER=new \
             HELM_ARGS="--set manager.cacheServerImage.repository=cache-server --set manager.cacheServerImage.tag=local"
 
+      - name: Apply Cache fixture
+        run: |
+          bash .pipelines/scripts/apply-cache-fixture.sh \
+            src/subsystems/cache-controller/charts/dacs/examples/cache/minikube.yaml
+
       - name: Run controller and perf E2E tests
         run: >-
           make test-e2e
@@ -359,6 +364,9 @@ jobs:
         continue-on-error: true
         run: |
           status=0
+          kubectl delete --namespace cache-server \
+            -f src/subsystems/cache-controller/charts/dacs/examples/cache/minikube.yaml \
+            --ignore-not-found --wait=true --timeout=5m || status=$?
           make undeploy || status=$?
           ./deploy/scripts/minikube/clean.sh || status=$?
           exit "$status"

.pipelines/dacs-integration.yml

@@ -54,6 +54,7 @@ parameters:
   - name: helmUpgradeBaselineVersion
     displayName: Known-good immutable DACS chart version
     type: string
+    # The baseline CRD must support spec.eviction.bucketInterval in the fixtures.
     default: 0.1.0-alpha.0.113+5f43ac1cc927f9
   - name: location
     displayName: AKS region
@@ -184,7 +185,9 @@ extends:
           aksTemplate: ${{ parameters.perfAksTemplate }}
           helmChart: ${{ parameters.helmChart }}
           ${{ if eq(parameters.helmChart, 'dacs') }}:
-            helmArgs: --values src/subsystems/cache-controller/charts/dacs/perf-values.yaml --set cache.scaling.static.numServers=${{ parameters.cacheServerCount }}
+            helmArgs: --values src/subsystems/cache-controller/charts/dacs/perf-values.yaml
+            cacheFixture: src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml
+            cacheStaticServers: ${{ parameters.cacheServerCount }}
           ${{ else }}:
             helmArgs: --values helm/cache-server/perf-values.yaml --set cacheServer.numServers=${{ parameters.cacheServerCount }}
           extraTestArgs: >-
@@ -208,7 +211,7 @@ extends:
           testFilter: aks && aks-upgrade
           testOutputFile: test-aks-upgrade.xml
           resourcePrefix: aks-upgrade
-          helmArgs: --values src/subsystems/cache-controller/charts/dacs/aks-int-${{ parameters.scalingMode }}-values.yaml
+          cacheFixture: src/subsystems/cache-controller/charts/dacs/examples/cache/${{ parameters.scalingMode }}.yaml
           condition: not(canceled())
           azureServiceConnection: ${{ parameters.azureServiceConnection }}
           dockerRegistry: ${{ parameters.dockerRegistry }}
@@ -224,7 +227,7 @@ extends:
           testFilter: aks && aks-scale
           testOutputFile: test-aks-scale.xml
           resourcePrefix: aks-scale
-          helmArgs: --values src/subsystems/cache-controller/charts/dacs/aks-int-${{ parameters.scalingMode }}-values.yaml
+          cacheFixture: src/subsystems/cache-controller/charts/dacs/examples/cache/${{ parameters.scalingMode }}.yaml
           condition: not(canceled())
           azureServiceConnection: ${{ parameters.azureServiceConnection }}
           dockerRegistry: ${{ parameters.dockerRegistry }}
@@ -240,7 +243,7 @@ extends:
           testFilter: aks && aks-node-crash
           testOutputFile: test-aks-node-crash.xml
           resourcePrefix: aks-node-crash
-          helmArgs: --values src/subsystems/cache-controller/charts/dacs/aks-int-${{ parameters.scalingMode }}-values.yaml
+          cacheFixture: src/subsystems/cache-controller/charts/dacs/examples/cache/${{ parameters.scalingMode }}.yaml
           condition: not(canceled())
           azureServiceConnection: ${{ parameters.azureServiceConnection }}
           dockerRegistry: ${{ parameters.dockerRegistry }}
@@ -256,7 +259,8 @@ extends:
           testFilter: aks && helm-upgrade
           testOutputFile: test-aks-helm-upgrade.xml
           resourcePrefix: aks-helm-upgrade
-          helmArgs: --values src/subsystems/cache-controller/charts/dacs/aks-int-${{ parameters.scalingMode }}-values.yaml --version ${{ parameters.helmUpgradeBaselineVersion }} --set manager.image.tag= --set manager.cacheServerImage.tag=
+          cacheFixture: src/subsystems/cache-controller/charts/dacs/examples/cache/${{ parameters.scalingMode }}.yaml
+          helmArgs: --version ${{ parameters.helmUpgradeBaselineVersion }} --set cache.enabled=false --set manager.image.tag= --set manager.cacheServerImage.tag=
           extraTestEnv: |
             export HELM_UPGRADE_TARGET_VERSION="$(cacheServerChartVersion)"
             export HELM_UPGRADE_TARGET_REGISTRY="${{ parameters.dockerRegistry }}/dacs"

.pipelines/scripts/apply-cache-fixture.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Apply a Cache fixture after its admission webhook becomes ready.
+set -euo pipefail
+
+if (( $# < 1 || $# > 2 )); then
+  echo "Usage: $0 FIXTURE [STATIC_SERVER_COUNT]" >&2
+  exit 1
+fi
+
+fixture=$1
+readonly static_servers=${2:-0}
+readonly max_webhook_attempts=24
+
+if [[ ! -f "$fixture" ]]; then
+  echo "Cache fixture does not exist: $fixture" >&2
+  exit 1
+fi
+if [[ ! "$static_servers" =~ ^(0|[1-9][0-9]*)$ ]]; then
+  echo "Cache static server count must be a positive integer or zero to use the fixture default" >&2
+  exit 1
+fi
+
+echo "Applying Cache fixture: $fixture"
+if [[ "$static_servers" != 0 ]]; then
+  current_static_servers=$(kubectl patch --local --filename "$fixture" \
+    --type=merge --patch '{}' --output='jsonpath={.spec.scaling.static.numServers}')
+  if [[ -z "$current_static_servers" ]]; then
+    echo "Cannot set cacheStaticServers on a dynamic Cache fixture; use zero to retain dynamic scaling" >&2
+    exit 1
+  fi
+  rendered_fixture=$(mktemp)
+  trap 'rm -f -- "$rendered_fixture"' EXIT
+  kubectl patch --local --filename "$fixture" --type=json \
+    --patch "[{\"op\":\"replace\",\"path\":\"/spec/scaling/static/numServers\",\"value\":$static_servers}]" \
+    --output=yaml > "$rendered_fixture"
+  fixture=$rendered_fixture
+fi
+
+for attempt in $(seq 1 "$max_webhook_attempts"); do
+  if kubectl apply --dry-run=server --namespace cache-server --filename "$fixture" >/dev/null; then
+    break
+  fi
+  if [[ "$attempt" -eq "$max_webhook_attempts" ]]; then
+    echo "Cache webhook did not become ready after $max_webhook_attempts attempts" >&2
+    exit 1
+  fi
+  echo "Cache webhook is not ready; retrying admission dry-run in 5 seconds"
+  sleep 5
+done
+kubectl apply --namespace cache-server --filename "$fixture"
+kubectl wait --namespace cache-server --filename "$fixture" --for=condition=Ready --timeout=10m

.pipelines/templates/Dacs.AksTest.Stages.yml

@@ -50,6 +50,12 @@ parameters:
   - name: helmArgs
     type: string
     default: ''
+  - name: cacheFixture
+    type: string
+    default: src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml
+  - name: cacheStaticServers
+    type: number
+    default: 0
   - name: resourcePrefix
     type: string
     default: aks-resilience
@@ -205,8 +211,6 @@ stages:
                         DACS_CACHE_CHART_REPO=dacs/helmchart/dacs \
                         DACS_CACHE_CHART_VERSION="$(cacheServerChartVersion)" \
                         HELM_ARGS="--set global.registry= --set manager.image.repository=$DOCKER_REGISTRY/dacs/cache-controller --set manager.cacheServerImage.repository=$DOCKER_REGISTRY/dacs/cache-server $EXTRA_HELM_ARGS"
-                    echo "Waiting for all Cache CRs to be ready..."
-                    kubectl wait --for=condition=Ready caches.storage.azure.com --all --all-namespaces --timeout=10m
                 else
                     make helm-install \
                         DOCKER_REGISTRY="$DOCKER_REGISTRY" BASE_REPO="$DOCKER_REGISTRY/dacs" \
@@ -219,6 +223,20 @@ stages:
               DOCKER_REGISTRY: ${{ parameters.dockerRegistry }}
               EXTRA_HELM_ARGS: ${{ parameters.helmArgs }}
 
+          - ${{ if and(eq(parameters.helmChart, 'dacs'), ne(parameters.cacheFixture, '')) }}:
+              - task: AzureCLI@2
+                displayName: Apply Cache Test Fixture
+                inputs:
+                  azureSubscription: ${{ parameters.azureServiceConnection }}
+                  scriptType: bash
+                  scriptLocation: inlineScript
+                  inlineScript: |
+                    set -euo pipefail
+                    bash .pipelines/scripts/apply-cache-fixture.sh "$CACHE_FIXTURE" "$CACHE_STATIC_SERVERS"
+                env:
+                  CACHE_FIXTURE: ${{ parameters.cacheFixture }}
+                  CACHE_STATIC_SERVERS: ${{ parameters.cacheStaticServers }}
+
           - task: AzureCLI@2
             displayName: Run Test (${{ parameters.testFilter }})
             inputs:

deploy/README.md

@@ -21,6 +21,24 @@ make undeploy
 make unprovision
 ```
 
+For the new controller, install the charts and then apply the desired Cache
+manifest directly:
+
+```bash
+make deploy CONTROLLER=new
+kubectl apply -n cache-server \
+  -f src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml
+kubectl wait -n cache-server \
+  -f src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml \
+  --for=condition=Ready --timeout=10m
+```
+
+Use the dynamic example instead when needed. Cache manifests are not managed
+by Make targets. The controller chart blocks uninstall while Cache resources
+remain. Delete them first, or explicitly enable `cleanup.forceDeleteCaches`
+on the release before uninstalling; see the
+[chart uninstall instructions](../src/subsystems/cache-controller/charts/dacs/README.md#uninstall).
+
 ## Minikube (Local Development)
 
 ```bash

deploy/deploy.mk

@@ -267,7 +267,6 @@ $(HELM_BIN) upgrade --install dacs ./$(DACS_CACHE_CHART_PATH) \
 	--set manager.image.tag=$(2) \
 	--set manager.cacheServerImage.repository=$(LOCAL_CACHE_SERVER_IMG) \
 	--set manager.cacheServerImage.tag=$(LOCAL_CACHE_SERVER_TAG) \
-	--set cache.enabled=true \
 	--wait --timeout 5m --atomic $(HELM_ARGS)
 endef
 
@@ -278,7 +277,6 @@ $(HELM_BIN) upgrade --install dacs oci://$(DOCKER_REGISTRY)/$(DACS_CACHE_CHART_R
 	--set global.registry=$(DOCKER_REGISTRY) \
 	--set manager.image.tag=$(IMAGE_TAG) \
 	--set manager.cacheServerImage.tag=$(IMAGE_TAG) \
-	--set cache.enabled=true \
 	--wait --timeout 5m --atomic $(HELM_ARGS)
 endef
 

docs/quickstart.md

@@ -66,8 +66,9 @@ remaining steps.
 
 ## Install DACS and an ephemeral cache
 
-The chart creates a single 2 GiB CacheServer on a Linux node. No prerequisite
-chart, local NVMe disk, or RAID setup is required.
+Install the controller, then apply a standalone Cache manifest to create a
+single 2 GiB CacheServer on a Linux node. No prerequisite chart, local NVMe
+disk, or RAID setup is required.
 
 ```bash
 helm upgrade --install dacs "$CHART_REF" \
@@ -76,15 +77,24 @@ helm upgrade --install dacs "$CHART_REF" \
   --create-namespace \
   --atomic \
   --wait \
-  --timeout 10m \
-  --set cache.enabled=true \
-  --set cache.name=cache-sample \
-  --set cache.scaling.static.numServers=1 \
-  --set cache.serverSizeInGB=2 \
-  --set cache.serverThreads=4 \
-  --set cache.storage.ephemeral=true \
-  --set cache.nodeSelectorKey=kubernetes.io/os \
-  --set cache.nodeSelectorValue=linux
+  --timeout 10m
+
+kubectl apply --namespace "$NAMESPACE" -f - <<'YAML'
+apiVersion: storage.azure.com/v1
+kind: Cache
+metadata:
+  name: cache-sample
+spec:
+  scaling:
+    static:
+      numServers: 1
+  serverSizeInGB: 2
+  serverThreads: 4
+  storage:
+    ephemeral: true
+  nodeSelectorKey: kubernetes.io/os
+  nodeSelectorValue: linux
+YAML
 ```
 
 The chart uses its `appVersion` for the controller and CacheServer image tags.