From 843b841d974e0b86f7edecd024929078470b8a71 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 9 Aug 2026 02:31:18 +0200 Subject: [PATCH 1/4] fix(ci): date the gate-suppression control fixture relative to today The control asserted the reporter exits 0 for a single in-bound marker, but hard-coded 2026-07-26 as that marker's date. On 2026-08-09 that reached fail_after_days=14, so the reporter correctly failed and the control failed with it -- taking every platform pull request down. Use the existing in-bound relative date, as every other date assertion in this file already does. The duplicate-marker case keeps its absolute dates because it asserts failure and fails closed on duplication before age is evaluated. --- .../test-cilium-rollout-gate-suppression-signal.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/tests/test-cilium-rollout-gate-suppression-signal.sh b/scripts/tests/test-cilium-rollout-gate-suppression-signal.sh index 026bcd7a8..7f7515758 100755 --- a/scripts/tests/test-cilium-rollout-gate-suppression-signal.sh +++ b/scripts/tests/test-cilium-rollout-gate-suppression-signal.sh @@ -272,12 +272,18 @@ ok # Control for the fixture: with exactly one marker the same path succeeds, so the # assertion above is about duplication and not about the fixture being unreadable. -printf ' # %s 2026-07-26\n' "${activation_marker}" >"${fixture_kustomization}" +# +# The date is RELATIVE, like every other date in this file. An absolute one was a +# time bomb: this control asserts exit 0, so once the hard-coded date aged past +# fail_after_days the reporter correctly failed and took every pull request in the +# repository with it. The duplicate case above can keep its absolute dates because +# it asserts FAILURE — it fails closed on the duplication before age matters. +printf ' # %s %s\n' "${activation_marker}" "${inside}" >"${fixture_kustomization}" : >"${tmp}/summary" CILIUM_ROLLOUT_GATE_ACTIVE=true PLATFORM_ROOT="${fixture_root}" \ GITHUB_STEP_SUMMARY="${tmp}/summary" "${report_script}" >"${tmp}/out" 2>&1 || fail 'a single marker read from PLATFORM_ROOT must succeed' -grep -Fq '2026-07-26' "${tmp}/summary" || +grep -Fq "${inside}" "${tmp}/summary" || fail 'the fixture summary must carry the declared date' ok From 3108f61b1ba6adb7bfff15281f4e91c4747904ce Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 9 Aug 2026 14:06:26 +0200 Subject: [PATCH 2/4] chore: claim #3028 From dd4a04061b5bab54346d89ecbcccdec0e8ca1993 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 9 Aug 2026 14:15:51 +0200 Subject: [PATCH 3/4] fix(cilium): roll back the homogeneous-device gate so prod deploys resume The gate reached its 14-day hard-fail bound, which stopped the deploy's Talos machine-config sync and, through the merge queue, every merge in the repository. No agent was ever stepped, so the widened device set never ran anywhere and the rollback restores the device set the fleet is already on. Re-comments the component reference and removes the activation marker in the same diff, which is what the reference's own comment requires. The pre-publish guard has an explicit path for this and publishes without the fleet-current check that gate REMOVAL requires. Both rollout tests asserted the activated state only, and one built its fixture by copying the live overlay, so they encoded the rollout as permanent. They now construct each state and assert both. --- .../controllers/kustomization.yaml | 25 ++-- ...t-cilium-homogeneous-devices-activation.sh | 116 ++++++++++++------ ...ium-homogeneous-devices-autoscaler-gate.sh | 24 +++- 3 files changed, 112 insertions(+), 53 deletions(-) diff --git a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml index 6053f40c5..f25b4875f 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml @@ -70,13 +70,14 @@ components: # strict-mode patch so the homogeneous-devices/ successor below can # override it — parent `patches:` apply after component patches, so a value # they carry is un-overridable by any component; see the component header). - # Keep referenced during the operator-stepped activation and soak; the - # later homogeneous component wins because it is listed after this one. + # This is the device set the fleet actually runs. Keep it referenced while + # homogeneous-devices/ is activated too — that component is listed after this + # one, so its wildcards win for as long as it is enabled. - cilium/components/private-nic-devices/ - # ACTIVE homogeneous device selection (#2610): widens the device set to + # Default-off homogeneous device selection (#2610): widens the device set to # the public default-route NICs via wildcards ("en+ eth+") so # ebpf-host-routing's BPF masquerade can attach where pod egress actually - # leaves. MUST stay listed AFTER private-nic-devices/ when enabled (later + # leaves. MUST be listed AFTER private-nic-devices/ when enabled (later # component wins). The component also sets updateStrategy OnDelete, so # referencing it stages config without rolling any agent — the operator steps # the roll pod-by-pod per the component's runbook (verify each node keeps a @@ -84,12 +85,16 @@ components: # nodes roll back automatically under the restored RollingUpdate). # # While this reference is active the deploy skips `ksail cluster update` — the - # pipeline's only Talos machine-config sync. The date below is what makes that - # suppression measurable: the deploy reports it in its job summary and warns - # once it outruns the rollout's intended window. Set it when activating, and - # remove it in the same diff that re-comments the reference. - # platform.devantler.tech/rollout-gate-activated: 2026-07-26 - - cilium/components/homogeneous-devices/ + # pipeline's only Talos machine-config sync — so activating it also means + # adding a `platform.devantler.tech/rollout-gate-activated` marker line + # directly above the reference, with the activation date after its colon. + # (Deliberately written here without that colon: the reporter counts the + # marker with a plain substring match and fails when it finds more than one, + # so an example carrying the exact literal would collide with the real one.) + # That date is what makes the suppression measurable: the deploy reports it in + # its job summary, warns from 7 days and fails the deploy from 14. Set it when + # activating, and remove it in the same diff that re-comments the reference. + # - cilium/components/homogeneous-devices/ # Default-off Cilium bandwidth-manager + host-only BBR staging component # (#2607). It is not activation-ready: #2609 must prove pod-BBR datapath # prerequisites and #2610's homogeneous-devices/ must be activated first. diff --git a/scripts/tests/test-cilium-homogeneous-devices-activation.sh b/scripts/tests/test-cilium-homogeneous-devices-activation.sh index 12a044ced..23874f216 100755 --- a/scripts/tests/test-cilium-homogeneous-devices-activation.sh +++ b/scripts/tests/test-cilium-homogeneous-devices-activation.sh @@ -172,66 +172,104 @@ read -r private_line homogeneous_line < <( ' "${controllers_kustomization}" ) +# The homogeneous component is a staged rollout that is referenced only while an +# operator is stepping the fleet, so BOTH states are real production states and +# both are asserted here. Which one applies is read from the same signal the +# deploy gate reads — whether the component is referenced — so this file cannot +# drift from `scripts/guard-cilium-homogeneous-device-rollout.sh`. [[ -n "${private_line}" ]] || - fail 'the private-NIC component must remain active during the stepped rollout' -[[ -n "${homogeneous_line}" ]] || - fail 'the production controllers overlay must activate homogeneous device selection' -((private_line < homogeneous_line)) || - fail 'the homogeneous device component must follow the private-NIC component so its values win' + fail 'the private-NIC component must remain referenced in both rollout states' production_release="$(kubectl kustomize "${controllers_dir}" | extract_cilium_release)" || fail 'the production controllers render has no Cilium HelmRelease' -production_update_strategy="$(extract_top_level_block updateStrategy <<<"${production_release}")" || - fail 'the production Cilium HelmRelease has no top-level update strategy' production_encryption="$(extract_top_level_block encryption <<<"${production_release}")" || fail 'the production Cilium HelmRelease has no top-level encryption settings' -production_node_port="$(extract_top_level_block nodePort <<<"${production_release}")" || - fail 'the production Cilium HelmRelease has no top-level NodePort settings' production_upgrade="$(extract_upgrade <<<"${production_release}")" || - fail 'the production Cilium HelmRelease has no temporary upgrade handoff' + fail 'the production Cilium HelmRelease has no upgrade block' -require_pattern \ - "${production_release}" \ - "${homogeneous_devices_pattern}" \ - 'the active production render must select both public and private device families' -reject_pattern \ - "${production_release}" \ - "${private_devices_pattern}" \ - 'the active production render must not retain the private-only device pin' -readonly expected_node_port=$' nodePort:\n addresses:\n - 10.0.0.0/16' -[[ "${production_node_port}" == "${expected_node_port}" ]] || - fail 'the public-NIC device set must expose NodePort on only the private node CIDR' -require_text \ - "${production_update_strategy}" \ - 'rollingUpdate: null' \ - 'the activation must clear the chart default rollingUpdate map' -require_text \ - "${production_update_strategy}" \ - 'type: OnDelete' \ - 'the activation must clear rollingUpdate while staging an operator-stepped OnDelete rollout' -require_pattern \ - "${production_upgrade}" \ - '^[[:space:]]*disableWait:[[:space:]]*true[[:space:]]*$' \ - 'the operator-stepped rollout must not block Flux dependency convergence' +if [[ -n "${homogeneous_line}" ]]; then + ((private_line < homogeneous_line)) || + fail 'the homogeneous device component must follow the private-NIC component so its values win' + + production_update_strategy="$(extract_top_level_block updateStrategy <<<"${production_release}")" || + fail 'the production Cilium HelmRelease has no top-level update strategy' + production_node_port="$(extract_top_level_block nodePort <<<"${production_release}")" || + fail 'the production Cilium HelmRelease has no top-level NodePort settings' + + require_pattern \ + "${production_release}" \ + "${homogeneous_devices_pattern}" \ + 'the active production render must select both public and private device families' + reject_pattern \ + "${production_release}" \ + "${private_devices_pattern}" \ + 'the active production render must not retain the private-only device pin' + expected_node_port=$' nodePort:\n addresses:\n - 10.0.0.0/16' + [[ "${production_node_port}" == "${expected_node_port}" ]] || + fail 'the public-NIC device set must expose NodePort on only the private node CIDR' + require_text \ + "${production_update_strategy}" \ + 'rollingUpdate: null' \ + 'the activation must clear the chart default rollingUpdate map' + require_text \ + "${production_update_strategy}" \ + 'type: OnDelete' \ + 'the activation must clear rollingUpdate while staging an operator-stepped OnDelete rollout' + require_pattern \ + "${production_upgrade}" \ + '^[[:space:]]*disableWait:[[:space:]]*true[[:space:]]*$' \ + 'the operator-stepped rollout must not block Flux dependency convergence' + + state_summary='activates homogeneous Cilium devices behind an OnDelete rollout gate' +else + # Rolled back: the private pin is the live device set again, and BOTH gate + # fields must be gone. Leaving either behind is the dangerous half-state — + # OnDelete without the component freezes the fleet against every future + # template change while the deploy reports the gate inactive, which is exactly + # the two-week divergence #3028 was filed for. + require_pattern \ + "${production_release}" \ + "${private_devices_pattern}" \ + 'the rolled-back production render must restore the private-only device pin' + reject_pattern \ + "${production_release}" \ + "${homogeneous_devices_pattern}" \ + 'the rolled-back production render must not keep the widened device families' + if extract_top_level_block updateStrategy <<<"${production_release}" >/dev/null; then + fail 'the rolled-back render must drop the updateStrategy override so the chart default RollingUpdate resumes' + fi + if extract_top_level_block nodePort <<<"${production_release}" >/dev/null; then + fail 'the rolled-back render must drop the public-NIC NodePort restriction with its device set' + fi + reject_pattern \ + "${production_upgrade}" \ + '^[[:space:]]*disableWait:[[:space:]]*true[[:space:]]*$' \ + 'the rolled-back render must drop the temporary Helm wait handoff' + + state_summary='pins Cilium to the private NICs with the rollout gate rolled back' +fi + +# Invariant in both states: Helm remediation and WireGuard encryption survive the +# gate going up or coming down. require_pattern \ "${production_upgrade}" \ '^[[:space:]]*retries:[[:space:]]*-1[[:space:]]*$' \ - 'the temporary Helm wait handoff must preserve infinite upgrade remediation' + 'the upgrade block must preserve infinite upgrade remediation' require_pattern \ "${production_upgrade}" \ '^[[:space:]]*remediateLastFailure:[[:space:]]*true[[:space:]]*$' \ - 'the temporary Helm wait handoff must preserve last-failure remediation' + 'the upgrade block must preserve last-failure remediation' require_pattern \ "${production_encryption}" \ '^[[:space:]]*enabled:[[:space:]]*true[[:space:]]*$' \ - 'the activation must preserve enabled WireGuard encryption' + 'the render must preserve enabled WireGuard encryption' require_pattern \ "${production_encryption}" \ '^[[:space:]]*nodeEncryption:[[:space:]]*false[[:space:]]*$' \ - 'the activation must preserve the production encryption settings' + 'the render must preserve the production encryption settings' require_text \ "${production_release}" \ 'type: wireguard' \ - 'the activation must preserve WireGuard encryption' + 'the render must preserve WireGuard encryption' -printf 'PASS: production activates homogeneous Cilium devices behind an OnDelete rollout gate\n' +printf 'PASS: production %s\n' "${state_summary}" diff --git a/scripts/tests/test-cilium-homogeneous-devices-autoscaler-gate.sh b/scripts/tests/test-cilium-homogeneous-devices-autoscaler-gate.sh index dc39fe46d..7dc22c6f5 100755 --- a/scripts/tests/test-cilium-homogeneous-devices-autoscaler-gate.sh +++ b/scripts/tests/test-cilium-homogeneous-devices-autoscaler-gate.sh @@ -82,8 +82,25 @@ fixture_root="${tmp_dir}/platform" fixture_controllers="${fixture_root}/k8s/providers/hetzner/infrastructure/controllers" fixture_component="${fixture_controllers}/cilium/components/homogeneous-devices" mkdir -p "${fixture_component}" -cp "${root_dir}/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" \ - "${fixture_controllers}/kustomization.yaml" +# Every gate-ACTIVE scenario below needs the component REFERENCED, but the real +# overlay only references it while an operator is mid-rollout. Copying the live +# file made those scenarios depend on the repository's current rollout state, so +# they broke the moment the component was rolled back — a fixture that asserts +# the gate is active while the fixture itself says inactive. Construct the state +# instead of inheriting it. The rollback scenarios still build their own state by +# deleting the reference. +install_active_controllers_fixture() { + cp "${root_dir}/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" \ + "${fixture_controllers}/kustomization.yaml" + sed -i.bak -E 's|^[[:space:]]*#[[:space:]]*-[[:space:]]*cilium/components/homogeneous-devices/[[:space:]]*$| - cilium/components/homogeneous-devices/|' \ + "${fixture_controllers}/kustomization.yaml" + rm -f "${fixture_controllers}/kustomization.yaml.bak" + grep -Eq '^[[:space:]]*-[[:space:]]*cilium/components/homogeneous-devices/?[[:space:]]*(#.*)?$' \ + "${fixture_controllers}/kustomization.yaml" || + fail 'the fixture must reference the homogeneous component for a gate-active scenario' +} + +install_active_controllers_fixture cp "${root_dir}/k8s/providers/hetzner/infrastructure/controllers/cilium/components/homogeneous-devices/kustomization.yaml" \ "${fixture_component}/kustomization.yaml" @@ -290,8 +307,7 @@ run_guard --after-deploy true fail 'a reconciled component rollback must restore the owned autoscaler replica count' [[ ! -s "${state_dir}/approved-template-sha" ]] || fail 'a reconciled component rollback must clear the approved template hash' -cp "${root_dir}/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" \ - "${fixture_controllers}/kustomization.yaml" +install_active_controllers_fixture run_guard --before-publish run_guard --after-deploy true From a1b3145b4c5b1ab27b5c96c0c04794fdc24d7941 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sun, 9 Aug 2026 14:28:41 +0200 Subject: [PATCH 4/4] fix(ci): re-approve the authorization surface after the Cilium rollback The rollback moves eight non-RBAC lines in one HelmRelease, which moves the aggregate surface fingerprint the validator pins. Measured across all five roots: 519 documents both sides, membership identical, four roots byte-identical. All 67 RBAC documents and all 123 aws-bearing lines compared directly and unchanged, with a planted-change control proving the comparison is not blind. Fingerprint taken from the required job's own output. --- scripts/validate-eks-ci-role-policy/main.go | 38 +++++++++++---------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/scripts/validate-eks-ci-role-policy/main.go b/scripts/validate-eks-ci-role-policy/main.go index 4c89f717f..e33110df7 100644 --- a/scripts/validate-eks-ci-role-policy/main.go +++ b/scripts/validate-eks-ci-role-policy/main.go @@ -246,31 +246,33 @@ const ( // infrastructure/controllers} plus k8s/clusters/prod/{bootstrap,} for both // trees and diff them. // -// Measured against base 0ca55381 while re-enabling the Kubescape posture -// scanner's policy-artifact fetch: 514 rendered documents on both sides across -// all five roots (122 apps, 209 infrastructure, 176 controllers, 4 bootstrap, -// 3 prod), with membership IDENTICAL — zero added, removed, or renamed. Four of -// the five roots are byte-identical. Exactly ONE line moves in the whole -// production render: +// Measured against base 4eab6531 while rolling back the Cilium +// homogeneous-device component (#3028): 519 rendered documents on both sides +// across all five roots (123 apps, 210 infrastructure, 177 controllers, +// 5 bootstrap, 4 prod), with membership IDENTICAL — zero added, removed, or +// renamed. Four of the five roots are byte-identical; only the controllers root +// moves. EIGHT lines change in the whole production render, all of them +// `spec.values` / `spec.upgrade` fields of one HelmRelease: // -// helm.toolkit.fluxcd.io/v2 HelmRelease kubescape/kubescape -// values.capabilities.kubescapeOffline: enable -> disable +// helm.toolkit.fluxcd.io/v2 HelmRelease kube-system/cilium +// values.devices: en+ eth+ -> enp7s0 eth1 +// values.updateStrategy: {rollingUpdate: null, type: OnDelete} -> removed +// values.nodePort.addresses: [10.0.0.0/16] -> removed +// upgrade.disableWait: true -> removed // -// That field decides one thing in the chart: whether KS_OFFLINE=true is set on -// the scanner container. It reaches no identity, binding, policy document, or -// service account. Its only other chart use ORs into clusterData.keepLocal, -// which is `or (offline) (not serviceDiscovery.enabled)` — serviceDiscovery is -// false here, so that value stays true either way and does not move. The -// accompanying cilium-network-policy.yaml edit is comment-only and renders to -// nothing, which the byte-identical infrastructure root confirms. +// Those fields decide which NICs the agent attaches its eBPF programs to, how +// the DaemonSet replaces pods, which CIDR NodePort listens on, and whether Helm +// waits for the upgrade. None reaches an identity, binding, policy document, or +// service account, and none is an AWS-facing field. // // All Role / ClusterRole / RoleBinding / ClusterRoleBinding / ServiceAccount -// documents are byte-identical, as is every `aws`-bearing line — trivially so, -// since a single non-RBAC line differs across the entire surface. +// documents are byte-identical — 67 documents, 38196 bytes, compared directly +// rather than inferred from the line diff — as is every `aws`-bearing line +// (123 of them). A planted-change control confirms that comparison is not blind. // // The fingerprint itself was read from the required job's own output on the // approved renderer, because the local toolchain is refused as unapproved. -const expectedRenderedSurfaceSHA = "cb575e34b191a662da108421fdff7c67f0cc00bf3c9b2b7cf0c5a4e49b46fc52" +const expectedRenderedSurfaceSHA = "4c06f17870f9eb129abf9c49e89fef2613154f22713929a9eb5835190312177d" // authorizationOverlayPaths lists every independently reconciled production // layer where an object can grant privileges to the aws/aws service account.