From bf330fc73f2d27c20002a38896c283b09300abfe Mon Sep 17 00:00:00 2001 From: Chandan Kumar Date: Sat, 12 Sep 2026 03:47:40 +0530 Subject: [PATCH] fix(helm): make KEDA hook Job resources configurable Upgrade apply OOMs at the old 256Mi limit when kubectl plus jq load live ScaledObjects. Expose hooks.resources and raise the default limit to 1Gi while keeping the 128Mi request. Co-authored-by: Cursor --- deploy/helm/sie-cluster/README.md | 14 ++++++++++++++ deploy/helm/sie-cluster/templates/_helpers.tpl | 4 ++++ .../templates/hooks/keda-ready-test.yaml | 7 +------ .../helm/sie-cluster/templates/keda-lifecycle.yaml | 7 +------ .../sie-cluster/templates/keda-scaledobject.yaml | 7 +------ deploy/helm/sie-cluster/values.yaml | 11 +++++++++++ 6 files changed, 32 insertions(+), 18 deletions(-) diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index f0beb9ad1..ad4bd9be0 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -385,6 +385,20 @@ Kubernetes operation/hook; it exceeds the longest default 15-minute Job. With a custom `pollingInterval` above 30 seconds, also make the timeout exceed the KEDA health deadline of `3 * pollingInterval + 240` seconds. +Size `keda-apply`, `keda-cleanup`, and the KEDA ScaledObject/HPA gate with +`hooks.resources`. The default memory limit is 1Gi. Requests stay at 128Mi. + +```yaml +hooks: + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "200m" + memory: "1Gi" +``` + ### Scale-from-Zero Trigger The gateway emits `sie.gateway.pending_demand` over OTLP when requests arrive diff --git a/deploy/helm/sie-cluster/templates/_helpers.tpl b/deploy/helm/sie-cluster/templates/_helpers.tpl index 350977cda..aa436059c 100644 --- a/deploy/helm/sie-cluster/templates/_helpers.tpl +++ b/deploy/helm/sie-cluster/templates/_helpers.tpl @@ -627,6 +627,10 @@ Args (dict): base, suffix. alpine/k8s:1.29.10@sha256:a1f03afdc59b1acde5e740ed855079c7361505d6fed9d9c6069c8c3307264348 {{- end }} +{{- define "sie-cluster.hooks.resources" -}} +{{- toYaml .Values.hooks.resources -}} +{{- end }} + {{/* Explicit kubectl credentials for hook containers. */}} {{- define "sie-cluster.kubernetes.inClusterKubeconfig" -}} KUBE_SERVICE_ACCOUNT_DIR=/var/run/secrets/kubernetes.io/serviceaccount diff --git a/deploy/helm/sie-cluster/templates/hooks/keda-ready-test.yaml b/deploy/helm/sie-cluster/templates/hooks/keda-ready-test.yaml index ed49b106d..d279e4276 100644 --- a/deploy/helm/sie-cluster/templates/hooks/keda-ready-test.yaml +++ b/deploy/helm/sie-cluster/templates/hooks/keda-ready-test.yaml @@ -383,12 +383,7 @@ spec: echo "FAILED: ScaledObjects and controller-reconciled HPAs did not become current, Ready, and trigger-healthy after a complete KEDA failure window" exit 1 resources: - requests: - cpu: 10m - memory: 64Mi - limits: - cpu: 100m - memory: 256Mi + {{- include "sie-cluster.hooks.resources" . | nindent 12 }} tolerations: - key: nvidia.com/gpu operator: Exists diff --git a/deploy/helm/sie-cluster/templates/keda-lifecycle.yaml b/deploy/helm/sie-cluster/templates/keda-lifecycle.yaml index 0aee8087e..cdf6bb2d9 100644 --- a/deploy/helm/sie-cluster/templates/keda-lifecycle.yaml +++ b/deploy/helm/sie-cluster/templates/keda-lifecycle.yaml @@ -118,9 +118,4 @@ spec: done echo "Release-owned KEDA ScaledObjects removed" resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 100m - memory: 128Mi + {{- include "sie-cluster.hooks.resources" . | nindent 12 }} diff --git a/deploy/helm/sie-cluster/templates/keda-scaledobject.yaml b/deploy/helm/sie-cluster/templates/keda-scaledobject.yaml index c09a2f5ba..c750399da 100644 --- a/deploy/helm/sie-cluster/templates/keda-scaledobject.yaml +++ b/deploy/helm/sie-cluster/templates/keda-scaledobject.yaml @@ -479,12 +479,7 @@ spec: fi echo "SUCCESS: ScaledObjects applied and obsolete release objects pruned" resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 200m - memory: 256Mi + {{- include "sie-cluster.hooks.resources" . | nindent 12 }} tolerations: - key: nvidia.com/gpu operator: Exists diff --git a/deploy/helm/sie-cluster/values.yaml b/deploy/helm/sie-cluster/values.yaml index 7c1f7e5ec..191d50ae9 100644 --- a/deploy/helm/sie-cluster/values.yaml +++ b/deploy/helm/sie-cluster/values.yaml @@ -1262,6 +1262,17 @@ healthGates: # This cannot disable the mandatory autoscaling control-path gates. enabled: false +# -- Resource requests/limits for kubectl-based KEDA hook Jobs +# (keda-apply, keda-cleanup, and the KEDA ScaledObject/HPA gate). +hooks: + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "200m" + memory: "1Gi" + # -- HuggingFace token secret (optional) # Create via Helm instead of kubectl for fully declarative deploys. # Pass token at install time: --set hfToken.create=true --set hfToken.value=