From a9af246ec6570cc01bb3862ff898457a0e93fe83 Mon Sep 17 00:00:00 2001 From: Joshua Long <44989821+jmclong@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:11:52 +0000 Subject: [PATCH] feat(helm)!: separate Cache resources from controller releases Provide standalone Cache fixtures, explicit pipeline application, and guarded controller uninstall. BREAKING CHANGE: cache.enabled=true is no longer supported. Migrate chart-owned Cache resources before upgrading and apply Cache manifests separately. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4e7adc4d-b2ae-4e92-9e76-2d857187198d --- .github/workflows/ci.yml | 8 + .pipelines/dacs-integration.yml | 14 +- .pipelines/scripts/apply-cache-fixture.sh | 65 +++++++ .pipelines/templates/Dacs.AksTest.Stages.yml | 22 ++- deploy/README.md | 18 ++ deploy/deploy.mk | 2 - docs/quickstart.md | 32 +-- .../cache-controller/charts/dacs/README.md | 184 ++++++++++++------ .../charts/dacs/examples/README.md | 43 ++++ .../cache/dynamic.yaml} | 46 +++-- .../charts/dacs/examples/cache/minikube.yaml | 47 +++++ .../cache/static.yaml} | 46 +++-- .../charts/dacs/perf-values.yaml | 9 - .../dacs/scripts/pre-delete-cache-cleanup.sh | 101 ++++++++++ .../charts/dacs/templates/NOTES.txt | 65 ++++--- .../charts/dacs/templates/cache/cache.yaml | 89 --------- .../templates/pre-delete-cache-cleanup.yaml | 24 ++- .../dacs/templates/validate-cache-values.yaml | 17 ++ .../charts/dacs/values.schema.json | 134 +------------ .../cache-controller/charts/dacs/values.yaml | 69 +------ .../internal/controller/helm_cleanup_test.go | 179 +++++++++++++++++ .../controller/helm_tolerations_test.go | 107 ---------- test/README.md | 29 +++ test/e2e/cache_cold_read_e2e_test.go | 4 +- test/e2e/cache_controller_e2e_test.go | 12 +- .../cache_short_term_node_loss_e2e_test.go | 6 +- user-guide.md | 143 +++++--------- 27 files changed, 862 insertions(+), 653 deletions(-) create mode 100644 .pipelines/scripts/apply-cache-fixture.sh create mode 100644 src/subsystems/cache-controller/charts/dacs/examples/README.md rename src/subsystems/cache-controller/charts/dacs/{aks-int-dynamic-values.yaml => examples/cache/dynamic.yaml} (53%) create mode 100644 src/subsystems/cache-controller/charts/dacs/examples/cache/minikube.yaml rename src/subsystems/cache-controller/charts/dacs/{aks-int-static-values.yaml => examples/cache/static.yaml} (52%) create mode 100644 src/subsystems/cache-controller/charts/dacs/scripts/pre-delete-cache-cleanup.sh delete mode 100644 src/subsystems/cache-controller/charts/dacs/templates/cache/cache.yaml create mode 100644 src/subsystems/cache-controller/charts/dacs/templates/validate-cache-values.yaml create mode 100644 src/subsystems/cache-controller/internal/controller/helm_cleanup_test.go delete mode 100644 src/subsystems/cache-controller/internal/controller/helm_tolerations_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c83e631c..12309a5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -319,6 +319,11 @@ jobs: make deploy CONTROLLER=new \ HELM_ARGS="--set manager.cacheServerImage.repository=cache-server --set manager.cacheServerImage.tag=local" + - name: Apply Cache fixture + run: | + bash .pipelines/scripts/apply-cache-fixture.sh \ + src/subsystems/cache-controller/charts/dacs/examples/cache/minikube.yaml + - name: Run controller and perf E2E tests run: >- make test-e2e @@ -359,6 +364,9 @@ jobs: continue-on-error: true run: | status=0 + kubectl delete --namespace cache-server \ + -f src/subsystems/cache-controller/charts/dacs/examples/cache/minikube.yaml \ + --ignore-not-found --wait=true --timeout=5m || status=$? make undeploy || status=$? ./deploy/scripts/minikube/clean.sh || status=$? exit "$status" diff --git a/.pipelines/dacs-integration.yml b/.pipelines/dacs-integration.yml index d6758105..6b58e62e 100644 --- a/.pipelines/dacs-integration.yml +++ b/.pipelines/dacs-integration.yml @@ -54,6 +54,7 @@ parameters: - name: helmUpgradeBaselineVersion displayName: Known-good immutable DACS chart version type: string + # The baseline CRD must support spec.eviction.bucketInterval in the fixtures. default: 0.1.0-alpha.0.113+5f43ac1cc927f9 - name: location displayName: AKS region @@ -184,7 +185,9 @@ extends: aksTemplate: ${{ parameters.perfAksTemplate }} helmChart: ${{ parameters.helmChart }} ${{ if eq(parameters.helmChart, 'dacs') }}: - helmArgs: --values src/subsystems/cache-controller/charts/dacs/perf-values.yaml --set cache.scaling.static.numServers=${{ parameters.cacheServerCount }} + helmArgs: --values src/subsystems/cache-controller/charts/dacs/perf-values.yaml + cacheFixture: src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml + cacheStaticServers: ${{ parameters.cacheServerCount }} ${{ else }}: helmArgs: --values helm/cache-server/perf-values.yaml --set cacheServer.numServers=${{ parameters.cacheServerCount }} extraTestArgs: >- @@ -208,7 +211,7 @@ extends: testFilter: aks && aks-upgrade testOutputFile: test-aks-upgrade.xml resourcePrefix: aks-upgrade - helmArgs: --values src/subsystems/cache-controller/charts/dacs/aks-int-${{ parameters.scalingMode }}-values.yaml + cacheFixture: src/subsystems/cache-controller/charts/dacs/examples/cache/${{ parameters.scalingMode }}.yaml condition: not(canceled()) azureServiceConnection: ${{ parameters.azureServiceConnection }} dockerRegistry: ${{ parameters.dockerRegistry }} @@ -224,7 +227,7 @@ extends: testFilter: aks && aks-scale testOutputFile: test-aks-scale.xml resourcePrefix: aks-scale - helmArgs: --values src/subsystems/cache-controller/charts/dacs/aks-int-${{ parameters.scalingMode }}-values.yaml + cacheFixture: src/subsystems/cache-controller/charts/dacs/examples/cache/${{ parameters.scalingMode }}.yaml condition: not(canceled()) azureServiceConnection: ${{ parameters.azureServiceConnection }} dockerRegistry: ${{ parameters.dockerRegistry }} @@ -240,7 +243,7 @@ extends: testFilter: aks && aks-node-crash testOutputFile: test-aks-node-crash.xml resourcePrefix: aks-node-crash - helmArgs: --values src/subsystems/cache-controller/charts/dacs/aks-int-${{ parameters.scalingMode }}-values.yaml + cacheFixture: src/subsystems/cache-controller/charts/dacs/examples/cache/${{ parameters.scalingMode }}.yaml condition: not(canceled()) azureServiceConnection: ${{ parameters.azureServiceConnection }} dockerRegistry: ${{ parameters.dockerRegistry }} @@ -256,7 +259,8 @@ extends: testFilter: aks && helm-upgrade testOutputFile: test-aks-helm-upgrade.xml resourcePrefix: aks-helm-upgrade - helmArgs: --values src/subsystems/cache-controller/charts/dacs/aks-int-${{ parameters.scalingMode }}-values.yaml --version ${{ parameters.helmUpgradeBaselineVersion }} --set manager.image.tag= --set manager.cacheServerImage.tag= + cacheFixture: src/subsystems/cache-controller/charts/dacs/examples/cache/${{ parameters.scalingMode }}.yaml + helmArgs: --version ${{ parameters.helmUpgradeBaselineVersion }} --set cache.enabled=false --set manager.image.tag= --set manager.cacheServerImage.tag= extraTestEnv: | export HELM_UPGRADE_TARGET_VERSION="$(cacheServerChartVersion)" export HELM_UPGRADE_TARGET_REGISTRY="${{ parameters.dockerRegistry }}/dacs" diff --git a/.pipelines/scripts/apply-cache-fixture.sh b/.pipelines/scripts/apply-cache-fixture.sh new file mode 100644 index 00000000..423d43b0 --- /dev/null +++ b/.pipelines/scripts/apply-cache-fixture.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Apply a Cache fixture after its admission webhook becomes ready. +set -euo pipefail + +if (( $# < 1 || $# > 2 )); then + echo "Usage: $0 FIXTURE [STATIC_SERVER_COUNT]" >&2 + exit 1 +fi + +fixture=$1 +readonly static_servers=${2:-0} +readonly max_webhook_attempts=24 + +if [[ ! -f "$fixture" ]]; then + echo "Cache fixture does not exist: $fixture" >&2 + exit 1 +fi +if [[ ! "$static_servers" =~ ^(0|[1-9][0-9]*)$ ]]; then + echo "Cache static server count must be a positive integer or zero to use the fixture default" >&2 + exit 1 +fi + +echo "Applying Cache fixture: $fixture" +if [[ "$static_servers" != 0 ]]; then + current_static_servers=$(kubectl patch --local --filename "$fixture" \ + --type=merge --patch '{}' --output='jsonpath={.spec.scaling.static.numServers}') + if [[ -z "$current_static_servers" ]]; then + echo "Cannot set cacheStaticServers on a dynamic Cache fixture; use zero to retain dynamic scaling" >&2 + exit 1 + fi + rendered_fixture=$(mktemp) + trap 'rm -f -- "$rendered_fixture"' EXIT + kubectl patch --local --filename "$fixture" --type=json \ + --patch "[{\"op\":\"replace\",\"path\":\"/spec/scaling/static/numServers\",\"value\":$static_servers}]" \ + --output=yaml > "$rendered_fixture" + fixture=$rendered_fixture +fi + +for attempt in $(seq 1 "$max_webhook_attempts"); do + if kubectl apply --dry-run=server --namespace cache-server --filename "$fixture" >/dev/null; then + break + fi + if [[ "$attempt" -eq "$max_webhook_attempts" ]]; then + echo "Cache webhook did not become ready after $max_webhook_attempts attempts" >&2 + exit 1 + fi + echo "Cache webhook is not ready; retrying admission dry-run in 5 seconds" + sleep 5 +done +kubectl apply --namespace cache-server --filename "$fixture" +kubectl wait --namespace cache-server --filename "$fixture" --for=condition=Ready --timeout=10m diff --git a/.pipelines/templates/Dacs.AksTest.Stages.yml b/.pipelines/templates/Dacs.AksTest.Stages.yml index adb1725a..c13e5e2d 100644 --- a/.pipelines/templates/Dacs.AksTest.Stages.yml +++ b/.pipelines/templates/Dacs.AksTest.Stages.yml @@ -50,6 +50,12 @@ parameters: - name: helmArgs type: string default: '' + - name: cacheFixture + type: string + default: src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml + - name: cacheStaticServers + type: number + default: 0 - name: resourcePrefix type: string default: aks-resilience @@ -205,8 +211,6 @@ stages: DACS_CACHE_CHART_REPO=dacs/helmchart/dacs \ DACS_CACHE_CHART_VERSION="$(cacheServerChartVersion)" \ HELM_ARGS="--set global.registry= --set manager.image.repository=$DOCKER_REGISTRY/dacs/cache-controller --set manager.cacheServerImage.repository=$DOCKER_REGISTRY/dacs/cache-server $EXTRA_HELM_ARGS" - echo "Waiting for all Cache CRs to be ready..." - kubectl wait --for=condition=Ready caches.storage.azure.com --all --all-namespaces --timeout=10m else make helm-install \ DOCKER_REGISTRY="$DOCKER_REGISTRY" BASE_REPO="$DOCKER_REGISTRY/dacs" \ @@ -219,6 +223,20 @@ stages: DOCKER_REGISTRY: ${{ parameters.dockerRegistry }} EXTRA_HELM_ARGS: ${{ parameters.helmArgs }} + - ${{ if and(eq(parameters.helmChart, 'dacs'), ne(parameters.cacheFixture, '')) }}: + - task: AzureCLI@2 + displayName: Apply Cache Test Fixture + inputs: + azureSubscription: ${{ parameters.azureServiceConnection }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -euo pipefail + bash .pipelines/scripts/apply-cache-fixture.sh "$CACHE_FIXTURE" "$CACHE_STATIC_SERVERS" + env: + CACHE_FIXTURE: ${{ parameters.cacheFixture }} + CACHE_STATIC_SERVERS: ${{ parameters.cacheStaticServers }} + - task: AzureCLI@2 displayName: Run Test (${{ parameters.testFilter }}) inputs: diff --git a/deploy/README.md b/deploy/README.md index 49c02b9a..38066a55 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -21,6 +21,24 @@ make undeploy make unprovision ``` +For the new controller, install the charts and then apply the desired Cache +manifest directly: + +```bash +make deploy CONTROLLER=new +kubectl apply -n cache-server \ + -f src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml +kubectl wait -n cache-server \ + -f src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml \ + --for=condition=Ready --timeout=10m +``` + +Use the dynamic example instead when needed. Cache manifests are not managed +by Make targets. The controller chart blocks uninstall while Cache resources +remain. Delete them first, or explicitly enable `cleanup.forceDeleteCaches` +on the release before uninstalling; see the +[chart uninstall instructions](../src/subsystems/cache-controller/charts/dacs/README.md#uninstall). + ## Minikube (Local Development) ```bash diff --git a/deploy/deploy.mk b/deploy/deploy.mk index 616b5033..34d3d942 100644 --- a/deploy/deploy.mk +++ b/deploy/deploy.mk @@ -267,7 +267,6 @@ $(HELM_BIN) upgrade --install dacs ./$(DACS_CACHE_CHART_PATH) \ --set manager.image.tag=$(2) \ --set manager.cacheServerImage.repository=$(LOCAL_CACHE_SERVER_IMG) \ --set manager.cacheServerImage.tag=$(LOCAL_CACHE_SERVER_TAG) \ - --set cache.enabled=true \ --wait --timeout 5m --atomic $(HELM_ARGS) endef @@ -278,7 +277,6 @@ $(HELM_BIN) upgrade --install dacs oci://$(DOCKER_REGISTRY)/$(DACS_CACHE_CHART_R --set global.registry=$(DOCKER_REGISTRY) \ --set manager.image.tag=$(IMAGE_TAG) \ --set manager.cacheServerImage.tag=$(IMAGE_TAG) \ - --set cache.enabled=true \ --wait --timeout 5m --atomic $(HELM_ARGS) endef diff --git a/docs/quickstart.md b/docs/quickstart.md index d6f9bdb5..352f299d 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -66,8 +66,9 @@ remaining steps. ## Install DACS and an ephemeral cache -The chart creates a single 2 GiB CacheServer on a Linux node. No prerequisite -chart, local NVMe disk, or RAID setup is required. +Install the controller, then apply a standalone Cache manifest to create a +single 2 GiB CacheServer on a Linux node. No prerequisite chart, local NVMe +disk, or RAID setup is required. ```bash helm upgrade --install dacs "$CHART_REF" \ @@ -76,15 +77,24 @@ helm upgrade --install dacs "$CHART_REF" \ --create-namespace \ --atomic \ --wait \ - --timeout 10m \ - --set cache.enabled=true \ - --set cache.name=cache-sample \ - --set cache.scaling.static.numServers=1 \ - --set cache.serverSizeInGB=2 \ - --set cache.serverThreads=4 \ - --set cache.storage.ephemeral=true \ - --set cache.nodeSelectorKey=kubernetes.io/os \ - --set cache.nodeSelectorValue=linux + --timeout 10m + +kubectl apply --namespace "$NAMESPACE" -f - <<'YAML' +apiVersion: storage.azure.com/v1 +kind: Cache +metadata: + name: cache-sample +spec: + scaling: + static: + numServers: 1 + serverSizeInGB: 2 + serverThreads: 4 + storage: + ephemeral: true + nodeSelectorKey: kubernetes.io/os + nodeSelectorValue: linux +YAML ``` The chart uses its `appVersion` for the controller and CacheServer image tags. diff --git a/src/subsystems/cache-controller/charts/dacs/README.md b/src/subsystems/cache-controller/charts/dacs/README.md index f5e802e4..60f54f14 100644 --- a/src/subsystems/cache-controller/charts/dacs/README.md +++ b/src/subsystems/cache-controller/charts/dacs/README.md @@ -14,21 +14,34 @@ A Kubernetes controller that manages distributed cache infrastructure through cu ```bash helm install dacs ./charts/dacs \ - --namespace dacs-system \ + --namespace cache-server \ --create-namespace \ --values ./charts/dacs/values.yaml ``` -### Install Controller + Sample Cache +### Install Controller and Example Cache ```bash helm install dacs ./charts/dacs \ - --namespace dacs-system \ + --namespace cache-server \ --create-namespace \ - --values ./charts/dacs/values.yaml \ - --set cache.enabled=true + --values ./charts/dacs/values.yaml ``` +After the controller chart succeeds, apply one of the example Cache manifests: + +```bash +kubectl apply \ + --namespace cache-server \ + -f ./charts/dacs/examples/cache/static.yaml +``` + +Use [examples/cache/dynamic.yaml](examples/cache/dynamic.yaml) instead to enable +dynamic scaling. `make deploy CONTROLLER=new` installs the controller and +prerequisites only; apply the desired Cache manifest separately. AKS test +pipelines use these same examples, with the static manifest also used for +performance tests. + ## Configuration ### Controller Configuration @@ -50,40 +63,16 @@ helm install dacs ./charts/dacs \ | `manager.resources.requests.memory` | Memory request | `64Mi` | | `manager.priorityClassName` | PriorityClass for the controller pod. Empty uses the chart-managed `-priority` PriorityClass (value `100000`); set a name to use an existing PriorityClass instead. | `""` | -### Cache Configuration +### Example Cache Configuration -| Parameter | Description | Default | -|-----------|-------------|---------| -| `cache.enabled` | Deploy a Cache CR with controller | `false` | -| `cache.name` | Name of the Cache CR (max 53 chars, RFC 1123 DNS label) | `cache-sample` | -| `cache.namespace` | Namespace for Cache CR; empty uses the release namespace | `""` | -| `cache.scaling.static.numServers` | Number of cache servers | | -| `cache.scaling.dynamic.percentageOfNodesToUse` | Percentage of nodes to run cache on (1-100) | | -| `cache.scaling.dynamic.minServers` | Minimum cache servers, 1-499 (desired below this floors to 0) | | -| `cache.scaling.dynamic.maxServers` | Maximum cache servers, 1-499 (hard cap; must be >= minServers) | | -| `cache.serverSizeInGB` | Storage per server (GB), minimum 1 | `4` | -| `cache.serverThreads` | Threads per server, range 4-128 | `80` | -| `cache.memoryLimitMB` | Optional memory limit per server, minimum 1024 MB | | -| `cache.tolerations` | CacheServer pod tolerations in the chart-managed Cache CR (`cache.enabled=true`) | `[]` | +The controller chart does not create Cache resources. Start with the static, +dynamic, or Minikube manifests in [`examples`](examples/README.md). ### CacheServer Scheduling on Tainted Nodes `manager.tolerations` applies only to the controller Deployment. CacheServer StatefulSets are generated from each `Cache` resource, not from the controller's -pod settings. For a chart-managed Cache, set: - -```yaml -cache: - enabled: true - tolerations: - - key: sku - operator: Equal - value: gpu - effect: NoSchedule -``` - -For a Cache managed separately from Helm, set the same list directly under -`spec.tolerations`: +pod settings. Set CacheServer tolerations directly under `spec.tolerations`: ```yaml apiVersion: storage.azure.com/v1 @@ -126,6 +115,14 @@ using this field. Helm does not upgrade CRDs in the chart's `crds/` directory automatically; follow the [upgrade procedure](#upgrade) using `charts/dacs/crds/storage.azure.com_caches.yaml`. +Query the installed CRD for fields omitted from the examples, including +dynamic scaling bounds, ephemeral storage, InfiniBand, and image pull policy: + +```bash +kubectl explain cache.spec --api-version=storage.azure.com/v1 --recursive +kubectl explain cache.spec.scaling.dynamic --api-version=storage.azure.com/v1 +``` + ### RBAC Configuration | Parameter | Description | Default | @@ -143,7 +140,8 @@ CRDs are always installed (required for controller to function). |-----------|-------------|---------| | `crd.keep` | Keep CRDs on uninstall | `true` | -**Warning:** Setting `crd.keep=false` will delete all Cache custom resources when uninstalling the chart. +Setting `crd.keep=false` removes the CRD on uninstall. It does not bypass the +Cache resource guard described in [Uninstall](#uninstall). ### Monitoring Configuration @@ -154,6 +152,20 @@ CRDs are always installed (required for controller to function). ## Upgrade +**Before upgrading from a chart that created a Cache:** inspect the existing +release with `helm get values dacs -n cache-server --all` +and `helm get manifest dacs -n cache-server`. Removing the +Cache template can delete a previously chart-owned Cache during upgrade. +Do not upgrade such a release until its Cache ownership has been migrated +and the migration verified. Merely removing `cache.enabled` or setting it +to `false` does not protect an existing chart-owned resource. + +Legacy `cache.enabled=true` fails rendering with a migration message. +`cache.enabled=false` is accepted for controller-only consumers; other +legacy `cache.*` settings are ignored because Cache configuration belongs in +standalone manifests. This values guard is not a substitute for inspecting +existing release resources, since old values may not be passed to the new chart. + Helm does not upgrade existing CRDs. Apply the target chart's CRDs first. ```bash @@ -162,37 +174,87 @@ kubectl apply -f ./charts/dacs/crds kubectl wait --for=condition=Established --timeout=60s -f ./charts/dacs/crds helm upgrade dacs ./charts/dacs \ - --namespace dacs-system \ + --namespace cache-server \ --set manager.image.tag=local ``` **Note:** The controller pod will be recreated during upgrade. Existing Cache CRs are not affected. Rollback retains the updated CRDs; verify compatibility with the previous controller. +Controller chart upgrades do not manage separately applied Cache manifests. +Edit and reapply the selected example when its Cache configuration changes: + +```bash +kubectl apply \ + --namespace cache-server \ + -f src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml +``` + ## Uninstall +Uninstalling the controller chart runs a pre-delete hook that blocks uninstall +while any Cache resources remain in the cluster, including resources that are +still terminating. Delete the Cache resources you own and wait for cleanup +before uninstalling the controller. For example: + ```bash -# Remove controller (CRDs are kept by default) -helm uninstall dacs -n dacs-system +kubectl delete -n cache-server \ + -f ./charts/dacs/examples/cache/static.yaml \ + --wait=true --timeout=5m +helm uninstall dacs -n cache-server +``` -# Remove CRDs manually if needed -kubectl delete crd caches.storage.azure.com +| Parameter | Description | Default | +|-----------|-------------|---------| +| `cleanup.forceDeleteCaches` | Delete all Cache resources cluster-wide; after the cleanup timeout, clear finalizers on remaining terminating Cache resources. | `false` | +| `cleanup.timeoutSeconds` | Time allowed for normal cleanup before force mode removes remaining Cache finalizers. | `300` | + +To explicitly allow deletion of **all Cache resources in the cluster**, save +`cleanup.forceDeleteCaches=true` on the release **before the first uninstall +attempt**. Helm uninstall does not accept `--set`; use the same chart version +and existing release configuration for the upgrade: + +```bash +helm upgrade dacs ./charts/dacs \ + -n cache-server --reuse-values \ + --set cleanup.forceDeleteCaches=true +helm uninstall dacs -n cache-server --timeout=10m ``` -## Namespace Architecture +This opt-in includes Cache resources created by other users or releases, even +when `crd.keep=true`. It first requests deletion without waiting, then polls +for completion. After `cleanup.timeoutSeconds`, it clears **all finalizers** +on remaining terminating Cache resources, including finalizers added by other +controllers. Uninstall proceeds only after no Cache resources remain. + +**Warning:** Removing finalizers bypasses unfinished cleanup and can leave +resources behind. Use force deletion only when this is acceptable. It does +not remove finalizers from child resources or erase cached files on nodes. +The value remains enabled on the release until changed or the release is +uninstalled. -**Recommended:** Deploy controller and cache in separate namespaces: -- Controller: `dacs-system` (watches cluster-wide) -- Cache instances: `cache-server`, `app-namespace`, etc. +If uninstall has already been blocked, delete the remaining Cache resources, +wait for their cleanup, and retry `helm uninstall`. A failed uninstall hook +can leave the release in `uninstalling` status, preventing a Helm upgrade to +enable force deletion. + +Permission or API errors fail the hook. Do not use `helm uninstall --no-hooks` +to bypass this protection: it can remove the controller while Cache resources +still depend on it. Do not uninstall if those resources must remain. + +## Namespace Architecture -**Alternative:** Deploy both in the same namespace by setting `cache.namespace=""`. +Install the controller and Cache resources in the same namespace. These +instructions use `cache-server`. If you install the Helm release in another +namespace, use that namespace for the Cache examples and subsequent commands. +The example printed by Helm uses the release namespace. ## Troubleshooting ### Controller Not Starting ```bash -kubectl logs -n dacs-system -l app.kubernetes.io/name=dacs -f +kubectl logs -n cache-server -l app.kubernetes.io/name=dacs -f ``` ### Cache Pods Pending @@ -221,29 +283,23 @@ kubectl get nodes --show-labels ```bash kubectl get crd caches.storage.azure.com # If missing, CRDs should be installed automatically. Check: -helm get manifest dacs -n dacs-system | grep CustomResourceDefinition +helm get manifest dacs -n cache-server | grep CustomResourceDefinition ``` ## Development ### Testing -Run the built-in Helm tests to verify the deployment: +With the controller installed, validate both example manifests against the +cluster without creating resources: ```bash -helm test dacs -n dacs-system -``` - -This runs three tests: -1. **Controller Test** - Verifies the controller pod is running -2. **CRD Test** - Verifies the Cache CRD is installed -3. **Cache Test** - Verifies Cache CR and pods (if `cache.enabled=true`) - -View test logs: -```bash -kubectl logs -n dacs-system dacs-test-controller -kubectl logs -n dacs-system dacs-test-crd -kubectl logs -n dacs-system dacs-test-cache # if cache enabled +kubectl apply --dry-run=server \ + --namespace cache-server \ + -f ./charts/dacs/examples/cache/static.yaml +kubectl apply --dry-run=server \ + --namespace cache-server \ + -f ./charts/dacs/examples/cache/dynamic.yaml ``` ### Linting @@ -255,9 +311,11 @@ helm lint ./charts/dacs ### Template Rendering ```bash -helm template test ./charts/dacs \ - --set cache.enabled=true \ - --debug +helm template test ./charts/dacs --debug + +kubectl apply --dry-run=server \ + --namespace cache-server \ + -f ./charts/dacs/examples/cache/static.yaml ``` ### Schema Validation diff --git a/src/subsystems/cache-controller/charts/dacs/examples/README.md b/src/subsystems/cache-controller/charts/dacs/examples/README.md new file mode 100644 index 00000000..45f5013b --- /dev/null +++ b/src/subsystems/cache-controller/charts/dacs/examples/README.md @@ -0,0 +1,43 @@ +## Cache Examples + +The standalone [static](cache/static.yaml) and [dynamic](cache/dynamic.yaml) +manifests define `cache-sample`, including its storage, resources, eviction, +node-selection, and scaling settings. + +Apply the static scaling example: + +```bash +kubectl apply \ + --namespace cache-server \ + -f src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml +``` + +Apply the dynamic scaling example: + +```bash +kubectl apply \ + --namespace cache-server \ + -f src/subsystems/cache-controller/charts/dacs/examples/cache/dynamic.yaml +``` + +Both manifests use AKS integration-test sizing and NVMe host storage. Copy +the desired manifest and edit it directly for environment-specific settings. + +For Minikube, use [minikube.yaml](cache/minikube.yaml): two servers with 4 GB +each and no explicit CPU or memory limits. + +Install the controller and Cache resources in `cache-server`. If the Helm +release uses another namespace, pass that namespace to the example commands. +For fields not shown here, run +`kubectl explain cache.spec --api-version=storage.azure.com/v1 --recursive`. + +Delete the Cache while the controller is still running so its finalizer can +complete: + +```bash +kubectl delete \ + --namespace cache-server \ + -f src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml \ + --ignore-not-found \ + --wait=true +``` diff --git a/src/subsystems/cache-controller/charts/dacs/aks-int-dynamic-values.yaml b/src/subsystems/cache-controller/charts/dacs/examples/cache/dynamic.yaml similarity index 53% rename from src/subsystems/cache-controller/charts/dacs/aks-int-dynamic-values.yaml rename to src/subsystems/cache-controller/charts/dacs/examples/cache/dynamic.yaml index 30e65093..ad4feb13 100644 --- a/src/subsystems/cache-controller/charts/dacs/aks-int-dynamic-values.yaml +++ b/src/subsystems/cache-controller/charts/dacs/examples/cache/dynamic.yaml @@ -12,26 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Shared values for AKS integration test pipelines that exercise the -# cache-controller's dynamic node-tracking scaler (e.g. aks-scale). -# -# Use with: -# helm install ... \ -# --values src/subsystems/cache-controller/charts/dacs/aks-int-dynamic-values.yaml -# -# Inherits the perf sizing (large server, NVMe host path) and configures -# dynamic scaling at 100% of nodes so the cache replica count tracks the -# nodepool 1:1. This lets the scale test drive cacheserver scaling indirectly -# by changing the AKS nodepool size. - -cache: +apiVersion: storage.azure.com/v1 +kind: Cache +metadata: + name: cache-sample + labels: + app.kubernetes.io/name: dacs + app.kubernetes.io/instance: cache-sample + app.kubernetes.io/version: example + app.kubernetes.io/component: cache + app.kubernetes.io/part-of: dacs + app.kubernetes.io/managed-by: kubectl + dacs.azure.com/example: "true" +spec: + scaling: + dynamic: + percentageOfNodesToUse: 100 serverSizeInGB: 1000 + serverThreads: 80 + port: 9065 + metricsPort: 9096 cpuLimit: 4 memoryLimitMB: 128000 + nodeSelectorKey: dacs.azure.com/processing-unit + nodeSelectorValue: cpu + eviction: + enableAccessTimeEviction: true + evictionMaintainFreeSpace: true + lastAccessThreshold: 30d + passFrequency: 15m + sizeTarget: "80" + sizeThreshold: "90" + bucketInterval: 15m storage: - ephemeral: false hostPath: nodeCacheFolderPath: /var/lib/ssd/cacheserver - scaling: - dynamic: - percentageOfNodesToUse: 100 diff --git a/src/subsystems/cache-controller/charts/dacs/examples/cache/minikube.yaml b/src/subsystems/cache-controller/charts/dacs/examples/cache/minikube.yaml new file mode 100644 index 00000000..7bdf8f0d --- /dev/null +++ b/src/subsystems/cache-controller/charts/dacs/examples/cache/minikube.yaml @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: storage.azure.com/v1 +kind: Cache +metadata: + name: cache-sample + labels: + app.kubernetes.io/name: dacs + app.kubernetes.io/instance: cache-sample + app.kubernetes.io/version: example + app.kubernetes.io/component: cache + app.kubernetes.io/part-of: dacs + app.kubernetes.io/managed-by: kubectl + dacs.azure.com/example: "true" +spec: + scaling: + static: + numServers: 2 + serverSizeInGB: 4 + serverThreads: 80 + port: 9065 + metricsPort: 9096 + nodeSelectorKey: dacs.azure.com/processing-unit + nodeSelectorValue: cpu + eviction: + enableAccessTimeEviction: true + evictionMaintainFreeSpace: true + lastAccessThreshold: 30d + passFrequency: 15m + sizeTarget: "80" + sizeThreshold: "90" + bucketInterval: 15m + storage: + hostPath: + nodeCacheFolderPath: /var/lib/ssd/cacheserver diff --git a/src/subsystems/cache-controller/charts/dacs/aks-int-static-values.yaml b/src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml similarity index 52% rename from src/subsystems/cache-controller/charts/dacs/aks-int-static-values.yaml rename to src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml index cf760cf0..80b05337 100644 --- a/src/subsystems/cache-controller/charts/dacs/aks-int-static-values.yaml +++ b/src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml @@ -12,26 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Shared values for AKS integration test pipelines that need a realistic -# multi-server cache topology (e.g. aks-upgrade, aks-restart, other resilience -# tests). -# -# Use with: -# helm install ... \ -# --values src/subsystems/cache-controller/charts/dacs/aks-int-static-values.yaml -# -# Inherits the perf sizing (large server, NVMe host path) and pins to 5 -# statically-scaled servers so resilience tests have enough nodes to exercise -# multi-server scenarios (rolling upgrades, node drains, restarts, etc.). - -cache: +apiVersion: storage.azure.com/v1 +kind: Cache +metadata: + name: cache-sample + labels: + app.kubernetes.io/name: dacs + app.kubernetes.io/instance: cache-sample + app.kubernetes.io/version: example + app.kubernetes.io/component: cache + app.kubernetes.io/part-of: dacs + app.kubernetes.io/managed-by: kubectl + dacs.azure.com/example: "true" +spec: + scaling: + static: + numServers: 5 serverSizeInGB: 1000 + serverThreads: 80 + port: 9065 + metricsPort: 9096 cpuLimit: 4 memoryLimitMB: 128000 + nodeSelectorKey: dacs.azure.com/processing-unit + nodeSelectorValue: cpu + eviction: + enableAccessTimeEviction: true + evictionMaintainFreeSpace: true + lastAccessThreshold: 30d + passFrequency: 15m + sizeTarget: "80" + sizeThreshold: "90" + bucketInterval: 15m storage: - ephemeral: false hostPath: nodeCacheFolderPath: /var/lib/ssd/cacheserver - scaling: - static: - numServers: 5 diff --git a/src/subsystems/cache-controller/charts/dacs/perf-values.yaml b/src/subsystems/cache-controller/charts/dacs/perf-values.yaml index 2fdf043c..c8acbe8f 100644 --- a/src/subsystems/cache-controller/charts/dacs/perf-values.yaml +++ b/src/subsystems/cache-controller/charts/dacs/perf-values.yaml @@ -17,12 +17,3 @@ manager: featureGates: "UseZeroCopy=true" - -cache: - serverSizeInGB: 1000 - cpuLimit: 4 - memoryLimitMB: 128000 - storage: - ephemeral: false - hostPath: - nodeCacheFolderPath: /var/lib/ssd/cacheserver diff --git a/src/subsystems/cache-controller/charts/dacs/scripts/pre-delete-cache-cleanup.sh b/src/subsystems/cache-controller/charts/dacs/scripts/pre-delete-cache-cleanup.sh new file mode 100644 index 00000000..18ce878c --- /dev/null +++ b/src/subsystems/cache-controller/charts/dacs/scripts/pre-delete-cache-cleanup.sh @@ -0,0 +1,101 @@ +#!/bin/sh +# Copyright (c) Microsoft Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +force_delete_caches=false +timeout_seconds=300 +while [ "$#" -gt 0 ]; do + case "$1" in + --force-cache-deletion) + force_delete_caches=true + shift + ;; + --timeout-seconds) + timeout_seconds=${2:?"--timeout-seconds requires a value"} + shift 2 + ;; + *) + echo "Unknown argument: $1" >&2 + exit 1 + ;; + esac +done +case "$timeout_seconds" in + *[!0-9]*|0[0-9]*) + echo "--timeout-seconds must be a non-negative integer without leading zeros." >&2 + exit 1 + ;; +esac + +crd=$(kubectl get crd caches.storage.azure.com --ignore-not-found --output=name) +if [ -z "$crd" ]; then + echo "Cache CRD not found, nothing to clean up." + exit 0 +fi + +if [ "$force_delete_caches" = false ]; then + caches=$(kubectl get caches.storage.azure.com --all-namespaces --output=name) + if [ -n "$caches" ]; then + echo "Cannot uninstall the controller while Cache resources remain:" >&2 + printf '%s\n' "$caches" >&2 + echo "Delete Cache resources and wait for cleanup, then retry uninstall." >&2 + exit 1 + fi + echo "No Cache resources remain; uninstall can proceed." + exit 0 +fi + +deadline=$(($(date +%s) + timeout_seconds)) +kubectl delete caches.storage.azure.com --all-namespaces --all \ + --wait=false \ + --ignore-not-found=true +while :; do + caches=$(kubectl get caches.storage.azure.com --all-namespaces --output=name) + if [ -z "$caches" ]; then + echo "All Cache resources deleted." + exit 0 + fi + if [ "$(date +%s)" -ge "$deadline" ]; then + break + fi + sleep 5 +done + +echo "Cache cleanup timed out; removing finalizers from remaining terminating Cache resources." >&2 +caches=$(kubectl get caches.storage.azure.com --all-namespaces \ + --output='jsonpath={range .items[?(@.metadata.deletionTimestamp)]}{.metadata.namespace}{" "}{.metadata.name}{" "}{.metadata.uid}{"\n"}{end}') +printf '%s\n' "$caches" | while read -r namespace name uid; do + if [ -z "$name" ]; then + continue + fi + if ! kubectl patch caches.storage.azure.com "$name" --namespace "$namespace" \ + --type=json \ + --patch "[{\"op\":\"test\",\"path\":\"/metadata/uid\",\"value\":\"$uid\"},{\"op\":\"add\",\"path\":\"/metadata/finalizers\",\"value\":[]}]"; then + remaining=$(kubectl get caches.storage.azure.com "$name" --namespace "$namespace" \ + --ignore-not-found --output=name) + if [ -n "$remaining" ]; then + echo "Failed to remove finalizers from Cache $namespace/$name." >&2 + exit 1 + fi + fi +done +caches=$(kubectl get caches.storage.azure.com --all-namespaces --output=name) +if [ -n "$caches" ]; then + echo "Cache resources still remain after force deletion; refusing to uninstall:" >&2 + printf '%s\n' "$caches" >&2 + exit 1 +fi +echo "All Cache resources deleted." diff --git a/src/subsystems/cache-controller/charts/dacs/templates/NOTES.txt b/src/subsystems/cache-controller/charts/dacs/templates/NOTES.txt index 61b50621..72839aa7 100644 --- a/src/subsystems/cache-controller/charts/dacs/templates/NOTES.txt +++ b/src/subsystems/cache-controller/charts/dacs/templates/NOTES.txt @@ -22,34 +22,37 @@ WEBHOOKS: kubectl get secret {{ include "chart.fullname" . }}-webhook-server-cert -n {{ .Release.Namespace }} {{- end }} -{{- if .Values.cache.enabled }} - -CACHE CR DEPLOYED: - Name: {{ .Values.cache.name }} - Namespace: {{ .Values.cache.namespace | default .Release.Namespace }} - - Check Cache status: - kubectl get cache {{ .Values.cache.name }} -n {{ .Values.cache.namespace | default .Release.Namespace }} - - Check cache servers: - kubectl get statefulset,pods -n {{ .Values.cache.namespace | default .Release.Namespace }} -l app=cacheserver - - View cache server logs: - kubectl logs -n {{ .Values.cache.namespace | default .Release.Namespace }} cacheserver-0 -f - -{{- else }} - -CACHE CR NOT DEPLOYED: - To deploy a cache, upgrade with cache.enabled=true: - helm upgrade {{ .Release.Name }} ./charts/dacs -n {{ .Release.Namespace }} --set cache.enabled=true - -{{- end }} - -{{- if .Values.cache.enabled }} -{{- if and (eq .Values.cache.nodeSelectorKey "") (eq .Values.cache.nodeSelectorValue "") }} - - WARNING: No node selector configured for Cache CR. - Cache pods may fail to schedule. Set cache.nodeSelectorKey and cache.nodeSelectorValue. - -{{- end }} -{{- end }} +CACHE EXAMPLE: + This chart does not install a Cache custom resource. + Apply a basic static Cache in the release namespace: + +kubectl apply -f - <<'EOF' +apiVersion: storage.azure.com/v1 +kind: Cache +metadata: + name: cache-sample + namespace: {{ .Release.Namespace }} +spec: + scaling: + static: + numServers: 2 + serverSizeInGB: 1000 + serverThreads: 80 + port: 9065 + metricsPort: 9096 + cpuLimit: 4 + memoryLimitMB: 128000 + nodeSelectorKey: dacs.azure.com/processing-unit + nodeSelectorValue: cpu + eviction: + enableAccessTimeEviction: true + evictionMaintainFreeSpace: true + lastAccessThreshold: 30d + passFrequency: 15m + sizeTarget: "80" + sizeThreshold: "90" + bucketInterval: 15m + storage: + hostPath: + nodeCacheFolderPath: /var/lib/ssd/cacheserver +EOF diff --git a/src/subsystems/cache-controller/charts/dacs/templates/cache/cache.yaml b/src/subsystems/cache-controller/charts/dacs/templates/cache/cache.yaml deleted file mode 100644 index 11772eee..00000000 --- a/src/subsystems/cache-controller/charts/dacs/templates/cache/cache.yaml +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - {{- if .Values.cache.enabled }} -apiVersion: storage.azure.com/v1 -kind: Cache -metadata: - name: {{ .Values.cache.name }} - namespace: {{ .Values.cache.namespace | default .Release.Namespace }} - labels: - app.kubernetes.io/name: dacs - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - scaling: - {{- if .Values.cache.scaling.dynamic }} - dynamic: - {{- with .Values.cache.scaling.dynamic }} - percentageOfNodesToUse: {{ .percentageOfNodesToUse }} - {{- if ne .minServers nil }} - minServers: {{ .minServers }} - {{- end }} - {{- if ne .maxServers nil }} - maxServers: {{ .maxServers }} - {{- end }} - {{- end }} - {{- else if .Values.cache.scaling.static }} - static: - numServers: {{ .Values.cache.scaling.static.numServers }} - {{- end }} - serverSizeInGB: {{ .Values.cache.serverSizeInGB }} - serverThreads: {{ .Values.cache.serverThreads }} - {{- with .Values.cache.port }} - port: {{ . }} - {{- end }} - {{- with .Values.cache.metricsPort }} - metricsPort: {{ . }} - {{- end }} - {{- with .Values.cache.nodeSelectorKey }} - nodeSelectorKey: {{ . | quote }} - {{- end }} - {{- with .Values.cache.nodeSelectorValue }} - nodeSelectorValue: {{ . | quote }} - {{- end }} - {{- with .Values.cache.tolerations }} - tolerations: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- if ne .Values.cache.cpuLimit nil }} - cpuLimit: {{ .Values.cache.cpuLimit }} - {{- end }} - {{- if ne .Values.cache.memoryLimitMB nil }} - memoryLimitMB: {{ .Values.cache.memoryLimitMB }} - {{- end }} - {{- with .Values.cache.egressBandwidthLimitGbps }} - egressBandwidthLimitGbps: {{ . }} - {{- end }} - eviction: - enableAccessTimeEviction: {{ .Values.cache.eviction.enableAccessTimeEviction }} - evictionMaintainFreeSpace: {{ .Values.cache.eviction.evictionMaintainFreeSpace }} - lastAccessThreshold: {{ .Values.cache.eviction.lastAccessThreshold | quote }} - passFrequency: {{ .Values.cache.eviction.passFrequency | quote }} - sizeTarget: {{ .Values.cache.eviction.sizeTarget | quote }} - sizeThreshold: {{ .Values.cache.eviction.sizeThreshold | quote }} - bucketInterval: {{ .Values.cache.eviction.bucketInterval | quote }} - storage: - {{- if .Values.cache.storage.ephemeral }} - ephemeral: true - {{- else }} - hostPath: - nodeCacheFolderPath: {{ .Values.cache.storage.hostPath.nodeCacheFolderPath | quote }} - {{- end }} - {{- /* InfiniBand resource configuration - only include when count > 0 */}} - {{- if and .Values.cache.infinibandResourceLimitName (gt (int (.Values.cache.infinibandResourceLimitCount | default 0)) 0) }} - infinibandResourceLimitCount: {{ .Values.cache.infinibandResourceLimitCount | int }} - infinibandResourceLimitName: {{ .Values.cache.infinibandResourceLimitName | quote }} - {{- end }} -{{- end }} diff --git a/src/subsystems/cache-controller/charts/dacs/templates/pre-delete-cache-cleanup.yaml b/src/subsystems/cache-controller/charts/dacs/templates/pre-delete-cache-cleanup.yaml index e9f2caba..b13cd31f 100644 --- a/src/subsystems/cache-controller/charts/dacs/templates/pre-delete-cache-cleanup.yaml +++ b/src/subsystems/cache-controller/charts/dacs/templates/pre-delete-cache-cleanup.yaml @@ -38,7 +38,13 @@ metadata: rules: - apiGroups: ["storage.azure.com"] resources: ["caches"] - verbs: ["get", "list", "delete"] + verbs: + - get + - list + {{- if .Values.cleanup.forceDeleteCaches }} + - patch + - delete + {{- end }} - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["get"] @@ -119,11 +125,11 @@ spec: - /bin/sh - -c - | - if ! kubectl get crd caches.storage.azure.com > /dev/null 2>&1; then - echo "Cache CRD not found, nothing to clean up." - exit 0 - fi - kubectl delete caches.storage.azure.com --all-namespaces --all \ - --wait=true \ - --timeout={{ .Values.cleanup.timeoutSeconds }}s \ - --ignore-not-found=true + {{- .Files.Get "scripts/pre-delete-cache-cleanup.sh" | nindent 14 }} + - pre-delete-cache-cleanup.sh + args: + {{- if .Values.cleanup.forceDeleteCaches }} + - --force-cache-deletion + {{- end }} + - --timeout-seconds + - {{ .Values.cleanup.timeoutSeconds | quote }} diff --git a/src/subsystems/cache-controller/charts/dacs/templates/validate-cache-values.yaml b/src/subsystems/cache-controller/charts/dacs/templates/validate-cache-values.yaml new file mode 100644 index 00000000..3b3b3a98 --- /dev/null +++ b/src/subsystems/cache-controller/charts/dacs/templates/validate-cache-values.yaml @@ -0,0 +1,17 @@ +# Copyright (c) Microsoft Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if (default dict .Values.cache).enabled }} +{{- fail "cache.enabled=true is no longer supported. Migrate any chart-owned Cache before upgrading, then apply Cache manifests separately. See the chart README Upgrade section." }} +{{- end }} diff --git a/src/subsystems/cache-controller/charts/dacs/values.schema.json b/src/subsystems/cache-controller/charts/dacs/values.schema.json index 610e005a..37308dc6 100644 --- a/src/subsystems/cache-controller/charts/dacs/values.schema.json +++ b/src/subsystems/cache-controller/charts/dacs/values.schema.json @@ -78,131 +78,6 @@ } } }, - "cache": { - "type": "object", - "properties": { - "enabled": { - "type": "boolean", - "description": "Deploy a Cache CR with the controller" - }, - "name": { - "type": "string", - "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", - "description": "Name of the Cache CR (must be DNS-1123 compliant)" - }, - "namespace": { - "type": "string", - "description": "Namespace for Cache CR (empty = Release.Namespace)" - }, - "serverSizeInGB": { - "type": "integer", - "minimum": 1, - "maximum": 10000 - }, - "serverThreads": { - "type": "integer", - "minimum": 4, - "maximum": 128 - }, - "tolerations": { - "type": "array", - "description": "CacheServer pod tolerations, independent of manager.tolerations; requires cache.enabled.", - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "key": { "type": "string" }, - "operator": { "type": "string", "enum": ["", "Equal", "Exists"] }, - "value": { "type": "string" }, - "effect": { "type": "string", "enum": ["", "NoSchedule", "PreferNoSchedule", "NoExecute"] }, - "tolerationSeconds": { "type": "integer" } - }, - "allOf": [ - { - "if": { "properties": { "key": { "const": "" } } }, - "then": { - "required": ["operator"], - "properties": { "operator": { "const": "Exists" } } - } - }, - { - "if": { - "required": ["operator"], - "properties": { "operator": { "const": "Exists" } } - }, - "then": { "properties": { "value": { "const": "" } } } - }, - { - "if": { "required": ["tolerationSeconds"] }, - "then": { - "required": ["effect"], - "properties": { "effect": { "const": "NoExecute" } } - } - } - ] - } - }, - "port": { - "type": "integer", - "minimum": 1, - "maximum": 65535 - }, - "metricsPort": { - "type": "integer", - "minimum": 1, - "maximum": 65535 - }, - "cpuLimit": { - "type": "integer", - "minimum": 0 - }, - "memoryLimitMB": { - "type": "integer", - "anyOf": [ - { - "const": 0 - }, - { - "minimum": 1024 - } - ] - }, - "scaling": { - "type": "object", - "properties": { - "static": { - "type": "object", - "properties": { - "numServers": { - "type": "integer", - "minimum": 1 - } - } - }, - "dynamic": { - "type": "object", - "properties": { - "minServers": { - "type": "integer", - "minimum": 1, - "maximum": 499 - }, - "maxServers": { - "type": "integer", - "minimum": 1, - "maximum": 499 - }, - "percentageOfNodesToUse": { - "type": "integer", - "minimum": 1, - "maximum": 100 - } - } - } - } - } - } - }, "rbacHelpers": { "type": "object", "properties": { @@ -221,6 +96,15 @@ } } }, + "cleanup": { + "type": "object", + "properties": { + "forceDeleteCaches": { + "type": "boolean", + "description": "Delete all Cache CRs cluster-wide before uninstall, clearing finalizers on remaining terminating Caches after cleanup.timeoutSeconds." + } + } + }, "metrics": { "type": "object", "properties": { diff --git a/src/subsystems/cache-controller/charts/dacs/values.yaml b/src/subsystems/cache-controller/charts/dacs/values.yaml index 843b5d0a..c738a971 100644 --- a/src/subsystems/cache-controller/charts/dacs/values.yaml +++ b/src/subsystems/cache-controller/charts/dacs/values.yaml @@ -136,7 +136,7 @@ rbacHelpers: # Custom Resource Definitions # CRDs are always installed (required for controller to function) -# Set keep=false to remove CRDs on chart uninstall (WARNING: deletes all Cache instances) +# Set keep=false to remove CRDs on chart uninstall; the Cache guard still applies. crd: keep: true # Keep CRDs when uninstalling @@ -173,13 +173,15 @@ prometheus: enable: false -# Pre-delete cleanup job that deletes Cache CRs before chart uninstall. -# Ensures finalizers run and pods are cleaned up gracefully. +# Pre-delete hook that blocks uninstall while Cache CRs remain. cleanup: + # Delete all Cache CRs cluster-wide; strip remaining finalizers after the timeout. + forceDeleteCaches: false image: repository: mcr.microsoft.com/oss/v2/kubernetes/kubectl tag: v1.35.9 pullPolicy: IfNotPresent + # Time allowed for normal cleanup before force mode removes Cache finalizers. timeoutSeconds: 300 resources: limits: @@ -188,64 +190,3 @@ cleanup: requests: cpu: 50m memory: 32Mi - -# Cache Custom Resource configuration -cache: - enabled: false # Set to true to deploy a sample Cache instance with the chart - name: cache-sample - namespace: "" # Namespace for Cache CR (empty = Release.Namespace) - - # CacheServer pod tolerations, written to Cache.spec.tolerations when enabled. - # For externally managed Cache resources, set spec.tolerations on the CR. - tolerations: [] - # - key: sku - # operator: Equal - # value: gpu - # effect: NoSchedule - - # Scaling configuration (use either static or dynamic, not both). - # If `dynamic` is set, it takes precedence over the default `static` below, - # so you can enable dynamic scaling without overriding `static` to null. - scaling: - static: - numServers: 2 - # dynamic: - # percentageOfNodesToUse: 50 - # minServers: 2 - # maxServers: 10 - - # Server configuration - serverSizeInGB: 4 - serverThreads: 80 - port: 9065 - metricsPort: 9096 - - # Resource limits - # cpuLimit: 4 - # memoryLimitMB: 2048 - # egressBandwidthLimitGbps: 10 - - # Node selection - nodeSelectorKey: "dacs.azure.com/processing-unit" - nodeSelectorValue: "cpu" - - # Eviction configuration - eviction: - enableAccessTimeEviction: true - evictionMaintainFreeSpace: true - lastAccessThreshold: "30d" - passFrequency: "15m" - sizeTarget: "80" - sizeThreshold: "90" - bucketInterval: "15m" - - # Storage configuration - storage: - ephemeral: false - hostPath: - nodeCacheFolderPath: /var/lib/ssd/cacheserver - - # InfiniBand resource configuration (optional) - # Set count > 0 to enable IB resources on cache server pods - infinibandResourceLimitCount: 0 - infinibandResourceLimitName: "rdma/hca_rdma_infiniband" diff --git a/src/subsystems/cache-controller/internal/controller/helm_cleanup_test.go b/src/subsystems/cache-controller/internal/controller/helm_cleanup_test.go new file mode 100644 index 00000000..594d671f --- /dev/null +++ b/src/subsystems/cache-controller/internal/controller/helm_cleanup_test.go @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "bytes" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "testing" + + batchv1 "k8s.io/api/batch/v1" + rbacv1 "k8s.io/api/rbac/v1" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" +) + +func TestHelmStandaloneCache(t *testing.T) { + helm, err := exec.LookPath("helm") + if err != nil { + t.Skip("helm is required for chart rendering tests") + } + for _, tt := range []struct { + name string + args []string + invalid bool + }{ + {name: "defaults"}, + {name: "legacy disabled", args: []string{"--set", "cache.enabled=false"}}, + {name: "obsolete settings ignored", args: []string{"--set", "cache.serverThreads=-1,cache.memoryLimitMB=1"}}, + {name: "legacy enabled rejected", args: []string{"--set", "cache.enabled=true"}, invalid: true}, + } { + t.Run(tt.name, func(t *testing.T) { + args := append([]string{"template", "standalone", "../../charts/dacs"}, tt.args...) + output, err := exec.Command(helm, args...).CombinedOutput() + if tt.invalid { + if err == nil || !bytes.Contains(output, []byte("Migrate any chart-owned Cache")) { + t.Fatalf("expected Cache ownership migration error, got %v\n%s", err, output) + } + return + } + if err != nil { + t.Fatalf("rendering controller-only chart: %v\n%s", err, output) + } + if bytes.Contains(output, []byte("\nkind: Cache\n")) || !bytes.Contains(output, []byte("\nkind: Deployment\n")) { + t.Fatal("chart must render the controller without creating a Cache") + } + }) + } +} + +func TestHelmCacheCleanup(t *testing.T) { + helm, err := exec.LookPath("helm") + if err != nil { + t.Skip("helm is required for chart rendering tests") + } + chart := "../../charts/dacs" + script, err := os.ReadFile(filepath.Join(chart, "scripts/pre-delete-cache-cleanup.sh")) + if err != nil { + t.Fatalf("reading cleanup script: %v", err) + } + for _, tt := range []struct { + name string + args []string + force bool + timeout string + invalid bool + invalidTimeout bool + }{ + {name: "default"}, + {name: "explicit false", args: []string{"--set", "cleanup.forceDeleteCaches=false"}}, + {name: "force delete", args: []string{"--set", "cleanup.forceDeleteCaches=true"}, force: true}, + {name: "custom timeout", args: []string{"--set", "cleanup.forceDeleteCaches=true,cleanup.timeoutSeconds=17"}, force: true, timeout: "17"}, + {name: "zero timeout", args: []string{"--set", "cleanup.forceDeleteCaches=true,cleanup.timeoutSeconds=0"}, force: true, timeout: "0"}, + {name: "leading zero timeout", args: []string{"--set", "cleanup.forceDeleteCaches=true", "--set-string", "cleanup.timeoutSeconds=08"}, force: true, timeout: "08", invalidTimeout: true}, + {name: "octal-looking timeout", args: []string{"--set", "cleanup.forceDeleteCaches=true", "--set-string", "cleanup.timeoutSeconds=010"}, force: true, timeout: "010", invalidTimeout: true}, + {name: "reject string flag", args: []string{"--set-string", "cleanup.forceDeleteCaches=false"}, invalid: true}, + } { + t.Run(tt.name, func(t *testing.T) { + args := append([]string{"template", "cleanup", chart, "--show-only", "templates/pre-delete-cache-cleanup.yaml"}, tt.args...) + output, err := exec.Command(helm, args...).CombinedOutput() + if tt.invalid { + if err == nil || !bytes.Contains(output, []byte("forceDeleteCaches")) { + t.Fatalf("expected cleanup flag schema error, got %v\n%s", err, output) + } + return + } + if err != nil { + t.Fatalf("rendering cleanup hook: %v\n%s", err, output) + } + decoder := yamlutil.NewYAMLOrJSONDecoder(bytes.NewReader(output), 4096) + var foundJob, foundRole bool + for { + var obj struct { + Kind string `json:"kind"` + } + var document json.RawMessage + if err := decoder.Decode(&document); err == io.EOF { + break + } else if err != nil { + t.Fatalf("decoding cleanup hook: %v", err) + } + if len(bytes.TrimSpace(document)) == 0 { + continue + } + if err := json.Unmarshal(document, &obj); err != nil { + t.Fatalf("decoding resource kind: %v", err) + } + switch obj.Kind { + case "ConfigMap": + t.Fatal("cleanup hook must not create a script ConfigMap") + case "ClusterRole": + foundRole = true + var role rbacv1.ClusterRole + if err := json.Unmarshal(document, &role); err != nil { + t.Fatalf("decoding cleanup role: %v", err) + } + wantVerbs := []string{"get", "list"} + if tt.force { + wantVerbs = append(wantVerbs, "patch", "delete") + } + if !slices.Equal(role.Rules[0].Verbs, wantVerbs) { + t.Fatalf("cleanup permissions = %v, want %v", role.Rules[0].Verbs, wantVerbs) + } + case "Job": + foundJob = true + var job batchv1.Job + if err := json.Unmarshal(document, &job); err != nil { + t.Fatalf("decoding cleanup job: %v", err) + } + container := job.Spec.Template.Spec.Containers[0] + if !slices.Equal(container.Command, []string{"/bin/sh", "-c", string(script), "pre-delete-cache-cleanup.sh"}) { + t.Fatal("cleanup command must inline the chart script unchanged and set its argument zero") + } + timeout := tt.timeout + if timeout == "" { + timeout = "300" + } + var wantArgs []string + if tt.force { + wantArgs = append(wantArgs, "--force-cache-deletion") + } + wantArgs = append(wantArgs, "--timeout-seconds", timeout) + if !slices.Equal(container.Args, wantArgs) { + t.Fatalf("cleanup arguments = %v, want %v", container.Args, wantArgs) + } + if len(container.VolumeMounts) != 0 || len(job.Spec.Template.Spec.Volumes) != 0 { + t.Fatal("inline cleanup script must not require volumes or mounts") + } + if tt.invalidTimeout { + cmd := exec.Command(container.Command[0], append(container.Command[1:], container.Args...)...) + cmd.Env = []string{"PATH=" + t.TempDir()} + output, err := cmd.CombinedOutput() + if err == nil || !bytes.Contains(output, []byte("without leading zeros")) { + t.Fatalf("expected timeout validation error before invoking kubectl, got %v\n%s", err, output) + } + } + } + } + if !foundJob || !foundRole { + t.Fatal("cleanup hook is missing its Job or ClusterRole") + } + }) + } +} diff --git a/src/subsystems/cache-controller/internal/controller/helm_tolerations_test.go b/src/subsystems/cache-controller/internal/controller/helm_tolerations_test.go deleted file mode 100644 index 1473ee84..00000000 --- a/src/subsystems/cache-controller/internal/controller/helm_tolerations_test.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package controller - -import ( - "bytes" - "io" - "os/exec" - "reflect" - "testing" - - storagev1 "github.com/Azure/dacs/cache-controller/api/v1" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - yamlutil "k8s.io/apimachinery/pkg/util/yaml" -) - -func TestHelmCacheTolerations(t *testing.T) { - helm, err := exec.LookPath("helm") - if err != nil { - t.Skip("helm is required for chart rendering tests") - } - gpu := []corev1.Toleration{{Key: "sku", Operator: corev1.TolerationOpEqual, Value: "gpu", Effect: corev1.TaintEffectNoSchedule}} - controllerDefault := []corev1.Toleration{{Key: "CriticalAddonsOnly", Operator: corev1.TolerationOpExists}} - customController := []corev1.Toleration{{Key: "controller-only", Operator: corev1.TolerationOpExists}} - for _, tt := range []struct { - name string - args []string - cache []corev1.Toleration - manager []corev1.Toleration - invalid bool - }{ - {name: "defaults", manager: controllerDefault}, - {name: "gpu", args: []string{"--set-json", `cache.tolerations=[{"key":"sku","operator":"Equal","value":"gpu","effect":"NoSchedule"}]`}, cache: gpu, manager: controllerDefault}, - {name: "independent controller", args: []string{"--set-json", `manager.tolerations=[{"key":"controller-only","operator":"Exists"}]`}, manager: customController}, - {name: "empty", args: []string{"--set-json", "cache.tolerations=[]"}, manager: controllerDefault}, - {name: "invalid operator", args: []string{"--set-json", `cache.tolerations=[{"key":"sku","operator":"Invalid"}]`}, invalid: true}, - {name: "exists with value", args: []string{"--set-json", `cache.tolerations=[{"operator":"Exists","value":"gpu"}]`}, invalid: true}, - {name: "empty key", args: []string{"--set-json", `cache.tolerations=[{"operator":"Equal","value":"gpu"}]`}, invalid: true}, - {name: "invalid effect", args: []string{"--set-json", `cache.tolerations=[{"key":"sku","effect":"Invalid"}]`}, invalid: true}, - {name: "seconds without NoExecute", args: []string{"--set-json", `cache.tolerations=[{"key":"sku","tolerationSeconds":30}]`}, invalid: true}, - } { - t.Run(tt.name, func(t *testing.T) { - args := append([]string{"template", "tolerations", "../../charts/dacs", "--set", "cache.enabled=true"}, tt.args...) - output, err := exec.Command(helm, args...).CombinedOutput() - if tt.invalid { - if err == nil { - t.Fatal("invalid tolerations passed chart schema validation") - } - if !bytes.Contains(output, []byte("cache.tolerations")) && !bytes.Contains(output, []byte("/cache/tolerations")) { - t.Fatalf("chart failed for an unrelated reason: %s", output) - } - return - } - if err != nil { - t.Fatalf("helm template: %v\n%s", err, output) - } - decoder := yamlutil.NewYAMLOrJSONDecoder(bytes.NewReader(output), 4096) - foundCache, foundManager := false, false - for { - obj := &unstructured.Unstructured{} - if err := decoder.Decode(obj); err == io.EOF { - break - } else if err != nil { - t.Fatal(err) - } - switch obj.GetKind() { - case "Cache": - foundCache = true - cache := &storagev1.Cache{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, cache); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(cache.Spec.Tolerations, tt.cache) { - t.Fatalf("cache tolerations: got %#v, want %#v", cache.Spec.Tolerations, tt.cache) - } - case "Deployment": - foundManager = true - deployment := &appsv1.Deployment{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, deployment); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(deployment.Spec.Template.Spec.Tolerations, tt.manager) { - t.Fatalf("controller tolerations changed: got %#v, want %#v", deployment.Spec.Template.Spec.Tolerations, tt.manager) - } - } - } - if !foundCache || !foundManager { - t.Fatal("rendered chart is missing Cache or controller Deployment") - } - }) - } -} diff --git a/test/README.md b/test/README.md index 08bb16b0..5e839852 100644 --- a/test/README.md +++ b/test/README.md @@ -46,6 +46,35 @@ The test framework will: ## Workflows +The new-controller Minikube job applies +`src/subsystems/cache-controller/charts/dacs/examples/cache/minikube.yaml` +after installing the controller. It defines two 4 GB servers without explicit +CPU or memory limits. Cleanup deletes the Cache and waits for it before +uninstalling the controller. + +AKS stages select `examples/cache/static.yaml` or `dynamic.yaml` using +`scalingMode`. Performance tests use the static fixture and pass +`cacheServerCount` as `cacheStaticServers`. A positive override changes only +the static server count in a temporary copy; zero keeps the fixture default. +Dynamic fixtures reject a nonzero static override. + +Both pipelines call `.pipelines/scripts/apply-cache-fixture.sh` to retry +admission dry-runs, apply once, and wait for the selected Cache to become Ready. +The shared AKS template's `cacheFixture` parameter selects the manifest; +an empty value skips fixture creation. + +For manual new-controller deployments, install the controller first and then +apply a fixture separately: + +```bash +make deploy CONTROLLER=new +kubectl apply -n cache-server \ + -f src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml +kubectl wait -n cache-server \ + -f src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml \ + --for=condition=Ready --timeout=10m +``` + ### Azure Pipeline: dacs-integration [dacs-integration](../.pipelines/dacs-integration.yml) runs on `main` every day diff --git a/test/e2e/cache_cold_read_e2e_test.go b/test/e2e/cache_cold_read_e2e_test.go index ebdd3271..386ad9b6 100644 --- a/test/e2e/cache_cold_read_e2e_test.go +++ b/test/e2e/cache_cold_read_e2e_test.go @@ -20,7 +20,9 @@ // // # Running Tests // -// make deploy CONTROLLER=new HELM_ARGS="--set cache.enabled=true" +// make deploy CONTROLLER=new +// kubectl apply -n cache-server -f src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml +// kubectl wait -n cache-server cache/cache-sample --for=condition=Ready --timeout=10m // make test-e2e TEST_FILTER=cold-read // // # Prerequisites diff --git a/test/e2e/cache_controller_e2e_test.go b/test/e2e/cache_controller_e2e_test.go index e4e4cd03..859e1a05 100644 --- a/test/e2e/cache_controller_e2e_test.go +++ b/test/e2e/cache_controller_e2e_test.go @@ -21,14 +21,16 @@ // // Controller tests: // -// make deploy CONTROLLER=new HELM_ARGS="--set cache.enabled=true" +// make deploy CONTROLLER=new +// kubectl apply -n cache-server -f src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml +// kubectl wait -n cache-server cache/cache-sample --for=condition=Ready --timeout=10m // make test-e2e TEST_FILTER=controller // // # Prerequisites // // Controller tests require: // - New controller deployed via helm (CONTROLLER=new) -// - Cache CR deployed via helm (cache.enabled=true) +// - Cache CR applied from a static or dynamic example manifest // // # Controller Test Resources // @@ -122,8 +124,8 @@ var _ = Describe("Cache Controller", Label("e2e", "controller"), Ordered, func() EnforceDefaultTimeoutsWhenUsingContexts() // Note: Controller deployment is handled by helm deploy (CONTROLLER=new). - // Cache CR is deployed via helm with cache.enabled=true. - // Cleanup is handled by helm undeploy. + // Cache CR is applied from an example manifest after the controller is ready. + // Cleanup deletes the Cache before uninstalling the controller. // Note: perf tests run BEFORE controller tests to avoid DNS propagation // issues caused by StatefulSet mutations (scaling, spec updates). @@ -134,7 +136,7 @@ var _ = Describe("Cache Controller", Label("e2e", "controller"), Ordered, func() if output, err := cmd.CombinedOutput(); err != nil { GinkgoWriter.Printf("Warning: job cleanup had issues: %v\nOutput: %s\n", err, string(output)) } - // Note: Cache CR cleanup is handled by helm undeploy + // Note: Cache CR cleanup is handled by make undeploy. }) Describe("Cache resource lifecycle", func() { diff --git a/test/e2e/cache_short_term_node_loss_e2e_test.go b/test/e2e/cache_short_term_node_loss_e2e_test.go index 8f060bad..172767d3 100644 --- a/test/e2e/cache_short_term_node_loss_e2e_test.go +++ b/test/e2e/cache_short_term_node_loss_e2e_test.go @@ -22,14 +22,16 @@ // // # Running Tests // -// make deploy CONTROLLER=new HELM_ARGS="--set cache.enabled=true" +// make deploy CONTROLLER=new +// kubectl apply -n cache-server -f src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml +// kubectl wait -n cache-server cache/cache-sample --for=condition=Ready --timeout=10m // make test-e2e TEST_FILTER=short-term-node-loss // // # Prerequisites // // - Multi-node cluster with at least one cacheserver pod on a node that // is not hosting the cache-controller pod. -// - dacs helm chart deployed (CONTROLLER=new, cache.enabled=true). +// - dacs deployed with CONTROLLER=new and a Cache CR applied separately. package e2e import ( diff --git a/user-guide.md b/user-guide.md index bef9d0b9..fba12451 100644 --- a/user-guide.md +++ b/user-guide.md @@ -83,7 +83,7 @@ Install the prerequisite chart to automatically set up RAID on each node: helm install dacs-prereq \ oci://dacstest.azurecr.io/dacs/helmchart/dacs-prereq \ --version \ - --namespace dacs-system \ + --namespace cache-server \ --create-namespace \ --wait --timeout 10m ``` @@ -112,26 +112,23 @@ infrastructure through a Cache custom resource (CR). helm install dacs \ oci://dacstest.azurecr.io/dacs/helmchart/dacs \ --version \ - --namespace dacs-system \ + --namespace cache-server \ --create-namespace \ --wait --timeout 10m ``` -### Step 3: Deploy a Cache Instance +### Step 3: Apply an Example Cache -Create a cache cluster by enabling the Cache CR: +After the controller and prerequisites are ready, apply a standalone Cache: ```sh -helm upgrade dacs \ - oci://dacstest.azurecr.io/dacs/helmchart/dacs \ - --version \ - --namespace dacs-system \ - --set cache.enabled=true \ - --set cache.scaling.static.numServers=3 \ - --set cache.serverSizeInGB=100 +kubectl apply --namespace cache-server \ + -f src/subsystems/cache-controller/charts/dacs/examples/cache/static.yaml ``` -The controller will create and manage the cache server StatefulSet automatically. +Use [dynamic.yaml](src/subsystems/cache-controller/charts/dacs/examples/cache/dynamic.yaml) +to scale with eligible nodes. Edit sizing, resources, and storage in the +manifest. The controller creates and manages the CacheServer StatefulSet. Helm chart values are documented in the [dacs README](src/subsystems/cache-controller/charts/dacs/README.md). @@ -141,7 +138,7 @@ Helm chart values are documented in the Check the controller is running: ```sh -kubectl get pods -n dacs-system +kubectl get pods -n cache-server -l app.kubernetes.io/name=dacs ``` Expected output: @@ -154,7 +151,7 @@ dacs-manager-xxxxxxx-xxxxx 1/1 Running 0 2m Check the Cache custom resource: ```sh -kubectl get cache -n dacs-system +kubectl get cache -n cache-server ``` Expected output: @@ -168,7 +165,7 @@ Check the discovery service. Cache clients use this service to discover CacheServers and connect on port `9065`: ```sh -kubectl get service cache-sample-discovery -n dacs-system +kubectl get service cache-sample-discovery -n cache-server ``` Check the cache server pods (created by the controller). Pod names are @@ -176,10 +173,10 @@ derived from the Cache CR name (default `cache-sample`): ```sh # List pods for a specific Cache CR -kubectl get pods -n dacs-system -l dacs.azure.com/cache-name=cache-sample +kubectl get pods -n cache-server -l dacs.azure.com/cache-name=cache-sample # Or list all cache server pods across all Cache CRs -kubectl get pods -n dacs-system -l app=cacheserver +kubectl get pods -n cache-server -l app=cacheserver ``` Expected output: @@ -201,84 +198,42 @@ The most common configuration options: helm upgrade dacs \ oci://dacstest.azurecr.io/dacs/helmchart/dacs \ --version \ - --namespace dacs-system \ - --set cache.enabled=true \ - --set cache.scaling.static.numServers=5 \ - --set cache.serverSizeInGB=100 \ - --set cache.serverThreads=160 \ - --set cache.cpuLimit=8 \ - --set cache.memoryLimitMB=8192 + --namespace cache-server \ + --set manager.logLevel=debug ``` -For a complete list of configuration options, see the -[dacs README](src/subsystems/cache-controller/charts/dacs/README.md) -and [values.yaml](src/subsystems/cache-controller/charts/dacs/values.yaml). - -### Using a Values File - -For complex configurations, create a `values.yaml` file: - -```yaml -global: - registry: dacstest.azurecr.io - -manager: - image: - repository: dacs/cache-controller - cacheServerImage: - repository: dacs/cache-server - -cache: - enabled: true - scaling: - static: - numServers: 5 - serverSizeInGB: 100 - serverThreads: 160 - cpuLimit: 8 - memoryLimitMB: 8192 - eviction: - enableAccessTimeEviction: true - evictionMaintainFreeSpace: true - sizeThreshold: "90" - sizeTarget: "80" - lastAccessThreshold: "30d" - passFrequency: "1h" - storage: - hostPath: - nodeCacheFolderPath: /var/lib/ssd/cacheserver -``` +For controller options, see the +[dacs README](src/subsystems/cache-controller/charts/dacs/README.md). + +### Using a Cache Manifest -Install with the values file: +Copy a manifest from +[`examples`](src/subsystems/cache-controller/charts/dacs/examples), +edit its fields, and reapply it independently of the controller Helm release: ```sh -helm upgrade --install dacs \ - oci://dacstest.azurecr.io/dacs/helmchart/dacs \ - --version \ - --namespace dacs-system \ - -f values.yaml +kubectl apply --namespace cache-server -f my-cache.yaml ``` +Before upgrading a release that used `cache.enabled=true`, read the +[migration warning](src/subsystems/cache-controller/charts/dacs/README.md#upgrade). +Removing old values alone does not preserve a chart-owned Cache. + ## Scale a cache -For a statically scaled cache, update `numServers` with the same chart version -used for the deployment: +For a statically scaled Cache, update `spec.scaling.static.numServers` in its +manifest and reapply it: ```sh -helm upgrade dacs \ - oci://dacstest.azurecr.io/dacs/helmchart/dacs \ - --version \ - --namespace dacs-system \ - --reuse-values \ - --set cache.scaling.static.numServers= +kubectl apply --namespace cache-server -f my-cache.yaml ``` Verify the requested replicas and pod readiness: ```sh -kubectl get cache cache-sample -n dacs-system +kubectl get cache cache-sample -n cache-server kubectl get pods \ - -n dacs-system \ + -n cache-server \ --selector=dacs.azure.com/cache-name=cache-sample ``` @@ -292,26 +247,26 @@ and scaling status, see ```sh # Controller logs -kubectl logs -n dacs-system -l app.kubernetes.io/name=dacs --prefix -f +kubectl logs -n cache-server -l app.kubernetes.io/name=dacs --prefix -f # Cache server logs (all cache pods) -kubectl logs -n dacs-system -l app=cacheserver --prefix --all-containers +kubectl logs -n cache-server -l app=cacheserver --prefix --all-containers # Cache server logs (specific Cache CR) -kubectl logs -n dacs-system -l dacs.azure.com/cache-name=cache-sample --prefix --all-containers +kubectl logs -n cache-server -l dacs.azure.com/cache-name=cache-sample --prefix --all-containers # Events -kubectl events -n dacs-system +kubectl events -n cache-server ``` ### Check Cache Status ```sh # View the Cache CR status -kubectl get cache -n dacs-system +kubectl get cache -n cache-server # Detailed status -kubectl describe cache -n dacs-system +kubectl describe cache -n cache-server ``` ## Troubleshooting @@ -321,7 +276,7 @@ kubectl describe cache -n dacs-system Check controller logs: ```sh -kubectl logs -n dacs-system -l app.kubernetes.io/name=dacs -f +kubectl logs -n cache-server -l app.kubernetes.io/name=dacs -f ``` ### Cache CR Not Reconciling @@ -329,13 +284,13 @@ kubectl logs -n dacs-system -l app.kubernetes.io/name=dacs -f Check the Cache resource status: ```sh -kubectl describe cache -n dacs-system +kubectl describe cache -n cache-server ``` Check controller logs for reconciliation errors: ```sh -kubectl logs -n dacs-system -l app.kubernetes.io/name=dacs -f +kubectl logs -n cache-server -l app.kubernetes.io/name=dacs -f ``` ### Pods Stuck in Pending @@ -343,7 +298,7 @@ kubectl logs -n dacs-system -l app.kubernetes.io/name=dacs -f Check pod events and node labels: ```sh -kubectl describe pod -0 -n dacs-system +kubectl describe pod -0 -n cache-server kubectl get nodes --show-labels | grep dacs.azure.com/processing-unit ``` @@ -352,7 +307,7 @@ kubectl get nodes --show-labels | grep dacs.azure.com/processing-unit Check the prereq logs: ```sh -kubectl logs -n dacs-system -l app.kubernetes.io/name=dacs-prereq -c nvme-raid-setup +kubectl logs -n cache-server -l app.kubernetes.io/name=dacs-prereq -c nvme-raid-setup ``` Common causes: @@ -365,7 +320,7 @@ Common causes: Check logs: ```sh -kubectl logs -0 -n dacs-system --previous +kubectl logs -0 -n cache-server --previous ``` ## Uninstall DACS @@ -375,7 +330,7 @@ To uninstall DACS, follow these steps: 1. Delete Cache custom resources first: ```sh - kubectl delete cache --all -n dacs-system + kubectl delete cache --all -n cache-server ``` > [!NOTE] @@ -384,13 +339,13 @@ To uninstall DACS, follow these steps: 2. Uninstall the DACS controller: ```sh - helm uninstall dacs -n dacs-system + helm uninstall dacs -n cache-server ``` 3. Uninstall the prerequisites (optional): ```sh - helm uninstall dacs-prereq -n dacs-system + helm uninstall dacs-prereq -n cache-server ``` > [!NOTE] @@ -400,7 +355,7 @@ To uninstall DACS, follow these steps: 4. Delete the namespace (optional): ```sh - kubectl delete namespace dacs-system + kubectl delete namespace cache-server ``` ## Development and contribution