From 29667fe89691e74e628b4a6d683977903c819725 Mon Sep 17 00:00:00 2001 From: Aurelio <19254254+Aureliolo@users.noreply.github.com> Date: Thu, 6 Aug 2026 18:32:40 +0200 Subject: [PATCH] feat: add upgrade_drain_timeout to drain control nodes before an upgrade termination_grace_period_seconds cannot keep jobs alive across a version upgrade. Once the first pod running the new version registers as an instance, every control node still on the old version stops its own services, because cluster_node_heartbeat shuts a node down as soon as it sees a peer reporting a higher version. That happens from inside the container, so it preempts the PreStop hook and the jobs that hook was waiting for fail with "Task was canceled due to receiving a shutdown signal." Drain before the new version is applied instead. When the application image changes, disable each control node so that newly submitted jobs stay pending rather than being scheduled onto a pod that is about to be replaced, then wait for the jobs already running there to finish. The wait sits ahead of the schema migration, so draining nodes never run old code against a migrated database. Reaching the timeout does not abort the upgrade: it proceeds, and any jobs still running fail as they do today. Opt-in. upgrade_drain_timeout defaults to 0, which keeps the current behavior. --- config/crd/bases/awx.ansible.com_awxs.yaml | 5 + .../pods-termination-grace-period.md | 34 ++++-- molecule/default/tasks/upgrade_drain_test.yml | 55 ++++++++++ .../default/templates/awx_cr_molecule.yml.j2 | 1 + molecule/default/verify.yml | 1 + roles/installer/defaults/main.yml | 5 + roles/installer/tasks/drain_control_nodes.yml | 101 ++++++++++++++++++ .../tasks/resources_configuration.yml | 4 + 8 files changed, 200 insertions(+), 6 deletions(-) create mode 100644 molecule/default/tasks/upgrade_drain_test.yml create mode 100644 roles/installer/tasks/drain_control_nodes.yml diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index 22cd4cdd..26164ec5 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -209,6 +209,11 @@ spec: description: Optional duration in seconds pods needs to terminate gracefully type: integer format: int32 + upgrade_drain_timeout: + description: Seconds to wait for the control nodes' running jobs to finish before rolling to a new application version. 0 does not wait. + type: integer + format: int32 + default: 0 service_labels: description: Additional labels to apply to the service type: string diff --git a/docs/user-guide/advanced-configuration/pods-termination-grace-period.md b/docs/user-guide/advanced-configuration/pods-termination-grace-period.md index e922e4a5..d11a84f4 100644 --- a/docs/user-guide/advanced-configuration/pods-termination-grace-period.md +++ b/docs/user-guide/advanced-configuration/pods-termination-grace-period.md @@ -29,11 +29,33 @@ following: One may want to set this value to the maximum duration they accept to wait for the affected Jobs to finish. Keeping in mind that such finishing jobs may -increase Pods termination time in such situations as `kubectl rollout restart`, -AWX upgrade by the operator, or Kubernetes [API-initiated +increase Pods termination time in such situations as `kubectl rollout restart` +or Kubernetes [API-initiated evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/). - -| Name | Description | Default | -| -------------------------------- | --------------------------------------------------------------- | ------- | -| termination_grace_period_seconds | Optional duration in seconds pods needs to terminate gracefully | not set | +#### Upgrades + +`termination_grace_period_seconds` does not cover an upgrade to a new AWX +version on its own. Once the first Pod running the new version registers itself +as an instance, every control node still running the old version stops its own +services, because AWX shuts a node down when it sees a peer reporting a higher +version. That happens from inside the container, so it preempts the `PreStop` +hook, and the jobs the hook was waiting for fail with `Task was canceled due to +receiving a shutdown signal.` + +Set `upgrade_drain_timeout` to also wait for those jobs *before* the new version +is rolled out. When the application image changes, the operator disables each +control node, so that new jobs stay pending rather than being scheduled onto a +Pod that is about to be replaced, and waits for the jobs already running there +to finish. Once they have, or once the timeout expires, the deployments are +applied as usual. The wait happens before the schema migration runs, so the +draining nodes are never running old code against a migrated database. + +A drain that reaches its timeout does not abort the upgrade: it proceeds, and +any jobs still running are failed by the rollout as they would be without this +setting. + +| Name | Description | Default | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- | +| termination_grace_period_seconds | Optional duration in seconds pods needs to terminate gracefully | not set | +| upgrade_drain_timeout | Seconds to wait for the control nodes' running jobs to finish before rolling the deployments to a new application version. 0 does not wait. | 0 | diff --git a/molecule/default/tasks/upgrade_drain_test.yml b/molecule/default/tasks/upgrade_drain_test.yml new file mode 100644 index 00000000..3d7753c2 --- /dev/null +++ b/molecule/default/tasks/upgrade_drain_test.yml @@ -0,0 +1,55 @@ +--- +# The molecule AWX spec sets upgrade_drain_timeout, but the application image +# does not change between reconciles, so the drain must not run: an ordinary +# reconcile has to leave every control node enabled and accepting jobs. +- name: Validate upgrade_drain_timeout + tags: + - upgrade_drain + block: + - name: Get the AWX resource + k8s_info: + namespace: '{{ namespace }}' + api_version: awx.ansible.com/v1beta1 + kind: AWX + name: example-awx + register: drain_awx + + - name: Assert the spec carries upgrade_drain_timeout + ansible.builtin.assert: + that: + - drain_awx.resources | length == 1 + - drain_awx.resources[0].spec.upgrade_drain_timeout | int > 0 + fail_msg: 'upgrade_drain_timeout is not set on the AWX resource' + + - name: Get the web pod + k8s_info: + namespace: '{{ namespace }}' + api_version: v1 + kind: Pod + label_selectors: + - 'app.kubernetes.io/name=example-awx-web' + field_selectors: + - status.phase=Running + register: drain_web_pods + + - name: Assert a web pod is running + ansible.builtin.assert: + that: + - drain_web_pods.resources | length + fail_msg: 'no running web pod to query the instances from' + + - name: List the instances + k8s_exec: + namespace: '{{ namespace }}' + pod: '{{ drain_web_pods.resources[0].metadata.name }}' + container: example-awx-web + command: awx-manage list_instances + register: drain_instances + changed_when: false + + # list_instances marks a disabled instance with a [DISABLED] prefix. + - name: Assert the reconcile left every control node enabled + ansible.builtin.assert: + that: + - "'[DISABLED]' not in drain_instances.stdout" + fail_msg: 'a reconcile without an image change disabled a control node: {{ drain_instances.stdout }}' diff --git a/molecule/default/templates/awx_cr_molecule.yml.j2 b/molecule/default/templates/awx_cr_molecule.yml.j2 index b2174135..31e8e537 100644 --- a/molecule/default/templates/awx_cr_molecule.yml.j2 +++ b/molecule/default/templates/awx_cr_molecule.yml.j2 @@ -38,6 +38,7 @@ spec: memory: 16M no_log: false restricted_security_context: true + upgrade_drain_timeout: 60 postgres_resource_requirements: {} redis_resource_requirements: {} additional_labels: diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index 1fec988f..ba31d8ce 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -18,6 +18,7 @@ - tasks/awx_test.yml - tasks/awx_replicas_test.yml - tasks/restricted_security_context_test.yml + - tasks/upgrade_drain_test.yml tags: - always rescue: diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index 473b31e1..f73ed063 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -460,6 +460,11 @@ postgres_security_context_settings: {} # keeps existing behavior. task_privileged and redis_capabilities still apply. restricted_security_context: false +# Seconds to wait for the control nodes' running jobs to finish before rolling +# the deployments to a new application version. Opt-in; 0 keeps existing +# behavior. See roles/installer/tasks/drain_control_nodes.yml. +upgrade_drain_timeout: 0 + # Set no_log settings on certain tasks no_log: true diff --git a/roles/installer/tasks/drain_control_nodes.yml b/roles/installer/tasks/drain_control_nodes.yml new file mode 100644 index 00000000..b56d1a7f --- /dev/null +++ b/roles/installer/tasks/drain_control_nodes.yml @@ -0,0 +1,101 @@ +--- +# Take the control nodes out of service and let their running jobs finish before +# the deployments roll to a new application version. +# +# termination_grace_period_seconds cannot cover an upgrade on its own. As soon as +# the first pod running the new version registers itself, every control node still +# on the old version stops its own services (the version check in AWX's +# cluster_node_heartbeat), which cuts short the preStop drain and fails the very +# jobs that drain was waiting for. Draining has to happen while the cluster is +# still wholly on the old version, which is here: the deployments have not been +# applied yet and the schema migration runs later still. + +- name: Get the current task deployment + k8s_info: + api_version: apps/v1 + kind: Deployment + namespace: '{{ ansible_operator_meta.namespace }}' + name: '{{ ansible_operator_meta.name }}-task' + register: _task_deployment + +- name: Set the currently deployed application image + set_fact: + _deployed_image: >- + {{ _task_deployment['resources'][0]['spec']['template']['spec']['containers'] + | selectattr('name', 'equalto', ansible_operator_meta.name ~ '-task') + | map(attribute='image') | first | default('') }} + when: _task_deployment['resources'] | length > 0 + +# Nothing to drain on a fresh install, when the application image is unchanged +# (every other reconcile), or when no web pod is available to run awx-manage in. +- name: Drain the control nodes + when: + - _deployed_image | default('') | length > 0 + - _deployed_image != _image + - awx_web_pod_name | length > 0 + block: + - name: Get the running control node pods + k8s_info: + api_version: v1 + kind: Pod + namespace: '{{ ansible_operator_meta.namespace }}' + label_selectors: + - "app.kubernetes.io/name={{ ansible_operator_meta.name }}-task" + - "app.kubernetes.io/managed-by={{ deployment_type }}-operator" + - "app.kubernetes.io/component={{ deployment_type }}" + field_selectors: + - status.phase=Running + register: _control_pods + + - name: Set the control node hostnames + set_fact: + _control_hostnames: >- + {{ _control_pods['resources'] + | rejectattr('metadata.deletionTimestamp', 'defined') + | map(attribute='metadata.name') | list }} + _drain_retries: >- + {{ [1, ((upgrade_drain_timeout | int) / 10) | round(0, 'ceil') | int] | max }} + + # Disable all of them before waiting on any of them: an instance that is still + # enabled keeps accepting work, so draining them one at a time would not + # converge. Disabling sets the instance capacity to zero, so jobs submitted + # from here on stay pending and run once the new pods register. + - name: Stop the control nodes accepting new jobs + k8s_exec: + namespace: '{{ ansible_operator_meta.namespace }}' + pod: '{{ awx_web_pod_name }}' + container: '{{ ansible_operator_meta.name }}-web' + command: awx-manage disable_instance --hostname {{ item }} + loop: '{{ _control_hostnames }}' + failed_when: false + + # --retry=1 turns the command into a probe: it exits non-zero while the + # instance still has running or waiting jobs. Retrying from here keeps the + # k8s_exec calls short instead of holding one exec stream open for the whole + # drain. + - name: Wait for the control nodes' running jobs to finish + k8s_exec: + namespace: '{{ ansible_operator_meta.namespace }}' + pod: '{{ awx_web_pod_name }}' + container: '{{ ansible_operator_meta.name }}-web' + command: awx-manage disable_instance --hostname {{ item }} --wait --retry=1 --retry_sleep=1 + loop: '{{ _control_hostnames }}' + register: _drain_result + until: _drain_result.return_code | default(1) == 0 + retries: '{{ _drain_retries | int }}' + delay: 10 + changed_when: false + ignore_errors: true + + # A drain that never finishes must not block the upgrade, so this is a + # warning rather than a failure: the jobs still running on those nodes will + # fail as they would without this feature. + - name: Report control nodes that were still running jobs + debug: + msg: >- + {{ item.item }} still had running jobs after upgrade_drain_timeout + ({{ upgrade_drain_timeout }}s); continuing with the upgrade. + loop: '{{ _drain_result.results | default([]) }}' + loop_control: + label: '{{ item.item }}' + when: item.return_code | default(1) != 0 diff --git a/roles/installer/tasks/resources_configuration.yml b/roles/installer/tasks/resources_configuration.yml index bf395392..df251bcd 100644 --- a/roles/installer/tasks/resources_configuration.yml +++ b/roles/installer/tasks/resources_configuration.yml @@ -245,6 +245,10 @@ set_fact: _redis_image: "{{ _custom_redis_image | default(lookup('env', 'RELATED_IMAGE_AWX_REDIS')) | default(_default_redis_image, true) }}" +- name: Let the control nodes' running jobs finish before rolling a new version + include_tasks: drain_control_nodes.yml + when: upgrade_drain_timeout | int > 0 + - name: Apply deployment resources k8s: apply: yes