diff --git a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml index 6053f40c5..f25b4875f 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml @@ -70,13 +70,14 @@ components: # strict-mode patch so the homogeneous-devices/ successor below can # override it — parent `patches:` apply after component patches, so a value # they carry is un-overridable by any component; see the component header). - # Keep referenced during the operator-stepped activation and soak; the - # later homogeneous component wins because it is listed after this one. + # This is the device set the fleet actually runs. Keep it referenced while + # homogeneous-devices/ is activated too — that component is listed after this + # one, so its wildcards win for as long as it is enabled. - cilium/components/private-nic-devices/ - # ACTIVE homogeneous device selection (#2610): widens the device set to + # Default-off homogeneous device selection (#2610): widens the device set to # the public default-route NICs via wildcards ("en+ eth+") so # ebpf-host-routing's BPF masquerade can attach where pod egress actually - # leaves. MUST stay listed AFTER private-nic-devices/ when enabled (later + # leaves. MUST be listed AFTER private-nic-devices/ when enabled (later # component wins). The component also sets updateStrategy OnDelete, so # referencing it stages config without rolling any agent — the operator steps # the roll pod-by-pod per the component's runbook (verify each node keeps a @@ -84,12 +85,16 @@ components: # nodes roll back automatically under the restored RollingUpdate). # # While this reference is active the deploy skips `ksail cluster update` — the - # pipeline's only Talos machine-config sync. The date below is what makes that - # suppression measurable: the deploy reports it in its job summary and warns - # once it outruns the rollout's intended window. Set it when activating, and - # remove it in the same diff that re-comments the reference. - # platform.devantler.tech/rollout-gate-activated: 2026-07-26 - - cilium/components/homogeneous-devices/ + # pipeline's only Talos machine-config sync — so activating it also means + # adding a `platform.devantler.tech/rollout-gate-activated` marker line + # directly above the reference, with the activation date after its colon. + # (Deliberately written here without that colon: the reporter counts the + # marker with a plain substring match and fails when it finds more than one, + # so an example carrying the exact literal would collide with the real one.) + # That date is what makes the suppression measurable: the deploy reports it in + # its job summary, warns from 7 days and fails the deploy from 14. Set it when + # activating, and remove it in the same diff that re-comments the reference. + # - cilium/components/homogeneous-devices/ # Default-off Cilium bandwidth-manager + host-only BBR staging component # (#2607). It is not activation-ready: #2609 must prove pod-BBR datapath # prerequisites and #2610's homogeneous-devices/ must be activated first. diff --git a/scripts/tests/test-cilium-homogeneous-devices-activation.sh b/scripts/tests/test-cilium-homogeneous-devices-activation.sh index 12a044ced..23874f216 100755 --- a/scripts/tests/test-cilium-homogeneous-devices-activation.sh +++ b/scripts/tests/test-cilium-homogeneous-devices-activation.sh @@ -172,66 +172,104 @@ read -r private_line homogeneous_line < <( ' "${controllers_kustomization}" ) +# The homogeneous component is a staged rollout that is referenced only while an +# operator is stepping the fleet, so BOTH states are real production states and +# both are asserted here. Which one applies is read from the same signal the +# deploy gate reads — whether the component is referenced — so this file cannot +# drift from `scripts/guard-cilium-homogeneous-device-rollout.sh`. [[ -n "${private_line}" ]] || - fail 'the private-NIC component must remain active during the stepped rollout' -[[ -n "${homogeneous_line}" ]] || - fail 'the production controllers overlay must activate homogeneous device selection' -((private_line < homogeneous_line)) || - fail 'the homogeneous device component must follow the private-NIC component so its values win' + fail 'the private-NIC component must remain referenced in both rollout states' production_release="$(kubectl kustomize "${controllers_dir}" | extract_cilium_release)" || fail 'the production controllers render has no Cilium HelmRelease' -production_update_strategy="$(extract_top_level_block updateStrategy <<<"${production_release}")" || - fail 'the production Cilium HelmRelease has no top-level update strategy' production_encryption="$(extract_top_level_block encryption <<<"${production_release}")" || fail 'the production Cilium HelmRelease has no top-level encryption settings' -production_node_port="$(extract_top_level_block nodePort <<<"${production_release}")" || - fail 'the production Cilium HelmRelease has no top-level NodePort settings' production_upgrade="$(extract_upgrade <<<"${production_release}")" || - fail 'the production Cilium HelmRelease has no temporary upgrade handoff' + fail 'the production Cilium HelmRelease has no upgrade block' -require_pattern \ - "${production_release}" \ - "${homogeneous_devices_pattern}" \ - 'the active production render must select both public and private device families' -reject_pattern \ - "${production_release}" \ - "${private_devices_pattern}" \ - 'the active production render must not retain the private-only device pin' -readonly expected_node_port=$' nodePort:\n addresses:\n - 10.0.0.0/16' -[[ "${production_node_port}" == "${expected_node_port}" ]] || - fail 'the public-NIC device set must expose NodePort on only the private node CIDR' -require_text \ - "${production_update_strategy}" \ - 'rollingUpdate: null' \ - 'the activation must clear the chart default rollingUpdate map' -require_text \ - "${production_update_strategy}" \ - 'type: OnDelete' \ - 'the activation must clear rollingUpdate while staging an operator-stepped OnDelete rollout' -require_pattern \ - "${production_upgrade}" \ - '^[[:space:]]*disableWait:[[:space:]]*true[[:space:]]*$' \ - 'the operator-stepped rollout must not block Flux dependency convergence' +if [[ -n "${homogeneous_line}" ]]; then + ((private_line < homogeneous_line)) || + fail 'the homogeneous device component must follow the private-NIC component so its values win' + + production_update_strategy="$(extract_top_level_block updateStrategy <<<"${production_release}")" || + fail 'the production Cilium HelmRelease has no top-level update strategy' + production_node_port="$(extract_top_level_block nodePort <<<"${production_release}")" || + fail 'the production Cilium HelmRelease has no top-level NodePort settings' + + require_pattern \ + "${production_release}" \ + "${homogeneous_devices_pattern}" \ + 'the active production render must select both public and private device families' + reject_pattern \ + "${production_release}" \ + "${private_devices_pattern}" \ + 'the active production render must not retain the private-only device pin' + expected_node_port=$' nodePort:\n addresses:\n - 10.0.0.0/16' + [[ "${production_node_port}" == "${expected_node_port}" ]] || + fail 'the public-NIC device set must expose NodePort on only the private node CIDR' + require_text \ + "${production_update_strategy}" \ + 'rollingUpdate: null' \ + 'the activation must clear the chart default rollingUpdate map' + require_text \ + "${production_update_strategy}" \ + 'type: OnDelete' \ + 'the activation must clear rollingUpdate while staging an operator-stepped OnDelete rollout' + require_pattern \ + "${production_upgrade}" \ + '^[[:space:]]*disableWait:[[:space:]]*true[[:space:]]*$' \ + 'the operator-stepped rollout must not block Flux dependency convergence' + + state_summary='activates homogeneous Cilium devices behind an OnDelete rollout gate' +else + # Rolled back: the private pin is the live device set again, and BOTH gate + # fields must be gone. Leaving either behind is the dangerous half-state — + # OnDelete without the component freezes the fleet against every future + # template change while the deploy reports the gate inactive, which is exactly + # the two-week divergence #3028 was filed for. + require_pattern \ + "${production_release}" \ + "${private_devices_pattern}" \ + 'the rolled-back production render must restore the private-only device pin' + reject_pattern \ + "${production_release}" \ + "${homogeneous_devices_pattern}" \ + 'the rolled-back production render must not keep the widened device families' + if extract_top_level_block updateStrategy <<<"${production_release}" >/dev/null; then + fail 'the rolled-back render must drop the updateStrategy override so the chart default RollingUpdate resumes' + fi + if extract_top_level_block nodePort <<<"${production_release}" >/dev/null; then + fail 'the rolled-back render must drop the public-NIC NodePort restriction with its device set' + fi + reject_pattern \ + "${production_upgrade}" \ + '^[[:space:]]*disableWait:[[:space:]]*true[[:space:]]*$' \ + 'the rolled-back render must drop the temporary Helm wait handoff' + + state_summary='pins Cilium to the private NICs with the rollout gate rolled back' +fi + +# Invariant in both states: Helm remediation and WireGuard encryption survive the +# gate going up or coming down. require_pattern \ "${production_upgrade}" \ '^[[:space:]]*retries:[[:space:]]*-1[[:space:]]*$' \ - 'the temporary Helm wait handoff must preserve infinite upgrade remediation' + 'the upgrade block must preserve infinite upgrade remediation' require_pattern \ "${production_upgrade}" \ '^[[:space:]]*remediateLastFailure:[[:space:]]*true[[:space:]]*$' \ - 'the temporary Helm wait handoff must preserve last-failure remediation' + 'the upgrade block must preserve last-failure remediation' require_pattern \ "${production_encryption}" \ '^[[:space:]]*enabled:[[:space:]]*true[[:space:]]*$' \ - 'the activation must preserve enabled WireGuard encryption' + 'the render must preserve enabled WireGuard encryption' require_pattern \ "${production_encryption}" \ '^[[:space:]]*nodeEncryption:[[:space:]]*false[[:space:]]*$' \ - 'the activation must preserve the production encryption settings' + 'the render must preserve the production encryption settings' require_text \ "${production_release}" \ 'type: wireguard' \ - 'the activation must preserve WireGuard encryption' + 'the render must preserve WireGuard encryption' -printf 'PASS: production activates homogeneous Cilium devices behind an OnDelete rollout gate\n' +printf 'PASS: production %s\n' "${state_summary}" diff --git a/scripts/tests/test-cilium-homogeneous-devices-autoscaler-gate.sh b/scripts/tests/test-cilium-homogeneous-devices-autoscaler-gate.sh index dc39fe46d..7dc22c6f5 100755 --- a/scripts/tests/test-cilium-homogeneous-devices-autoscaler-gate.sh +++ b/scripts/tests/test-cilium-homogeneous-devices-autoscaler-gate.sh @@ -82,8 +82,25 @@ fixture_root="${tmp_dir}/platform" fixture_controllers="${fixture_root}/k8s/providers/hetzner/infrastructure/controllers" fixture_component="${fixture_controllers}/cilium/components/homogeneous-devices" mkdir -p "${fixture_component}" -cp "${root_dir}/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" \ - "${fixture_controllers}/kustomization.yaml" +# Every gate-ACTIVE scenario below needs the component REFERENCED, but the real +# overlay only references it while an operator is mid-rollout. Copying the live +# file made those scenarios depend on the repository's current rollout state, so +# they broke the moment the component was rolled back — a fixture that asserts +# the gate is active while the fixture itself says inactive. Construct the state +# instead of inheriting it. The rollback scenarios still build their own state by +# deleting the reference. +install_active_controllers_fixture() { + cp "${root_dir}/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" \ + "${fixture_controllers}/kustomization.yaml" + sed -i.bak -E 's|^[[:space:]]*#[[:space:]]*-[[:space:]]*cilium/components/homogeneous-devices/[[:space:]]*$| - cilium/components/homogeneous-devices/|' \ + "${fixture_controllers}/kustomization.yaml" + rm -f "${fixture_controllers}/kustomization.yaml.bak" + grep -Eq '^[[:space:]]*-[[:space:]]*cilium/components/homogeneous-devices/?[[:space:]]*(#.*)?$' \ + "${fixture_controllers}/kustomization.yaml" || + fail 'the fixture must reference the homogeneous component for a gate-active scenario' +} + +install_active_controllers_fixture cp "${root_dir}/k8s/providers/hetzner/infrastructure/controllers/cilium/components/homogeneous-devices/kustomization.yaml" \ "${fixture_component}/kustomization.yaml" @@ -290,8 +307,7 @@ run_guard --after-deploy true fail 'a reconciled component rollback must restore the owned autoscaler replica count' [[ ! -s "${state_dir}/approved-template-sha" ]] || fail 'a reconciled component rollback must clear the approved template hash' -cp "${root_dir}/k8s/providers/hetzner/infrastructure/controllers/kustomization.yaml" \ - "${fixture_controllers}/kustomization.yaml" +install_active_controllers_fixture run_guard --before-publish run_guard --after-deploy true diff --git a/scripts/tests/test-cilium-rollout-gate-suppression-signal.sh b/scripts/tests/test-cilium-rollout-gate-suppression-signal.sh index 026bcd7a8..7f7515758 100755 --- a/scripts/tests/test-cilium-rollout-gate-suppression-signal.sh +++ b/scripts/tests/test-cilium-rollout-gate-suppression-signal.sh @@ -272,12 +272,18 @@ ok # Control for the fixture: with exactly one marker the same path succeeds, so the # assertion above is about duplication and not about the fixture being unreadable. -printf ' # %s 2026-07-26\n' "${activation_marker}" >"${fixture_kustomization}" +# +# The date is RELATIVE, like every other date in this file. An absolute one was a +# time bomb: this control asserts exit 0, so once the hard-coded date aged past +# fail_after_days the reporter correctly failed and took every pull request in the +# repository with it. The duplicate case above can keep its absolute dates because +# it asserts FAILURE — it fails closed on the duplication before age matters. +printf ' # %s %s\n' "${activation_marker}" "${inside}" >"${fixture_kustomization}" : >"${tmp}/summary" CILIUM_ROLLOUT_GATE_ACTIVE=true PLATFORM_ROOT="${fixture_root}" \ GITHUB_STEP_SUMMARY="${tmp}/summary" "${report_script}" >"${tmp}/out" 2>&1 || fail 'a single marker read from PLATFORM_ROOT must succeed' -grep -Fq '2026-07-26' "${tmp}/summary" || +grep -Fq "${inside}" "${tmp}/summary" || fail 'the fixture summary must carry the declared date' ok diff --git a/scripts/validate-eks-ci-role-policy/main.go b/scripts/validate-eks-ci-role-policy/main.go index 4c89f717f..e33110df7 100644 --- a/scripts/validate-eks-ci-role-policy/main.go +++ b/scripts/validate-eks-ci-role-policy/main.go @@ -246,31 +246,33 @@ const ( // infrastructure/controllers} plus k8s/clusters/prod/{bootstrap,} for both // trees and diff them. // -// Measured against base 0ca55381 while re-enabling the Kubescape posture -// scanner's policy-artifact fetch: 514 rendered documents on both sides across -// all five roots (122 apps, 209 infrastructure, 176 controllers, 4 bootstrap, -// 3 prod), with membership IDENTICAL — zero added, removed, or renamed. Four of -// the five roots are byte-identical. Exactly ONE line moves in the whole -// production render: +// Measured against base 4eab6531 while rolling back the Cilium +// homogeneous-device component (#3028): 519 rendered documents on both sides +// across all five roots (123 apps, 210 infrastructure, 177 controllers, +// 5 bootstrap, 4 prod), with membership IDENTICAL — zero added, removed, or +// renamed. Four of the five roots are byte-identical; only the controllers root +// moves. EIGHT lines change in the whole production render, all of them +// `spec.values` / `spec.upgrade` fields of one HelmRelease: // -// helm.toolkit.fluxcd.io/v2 HelmRelease kubescape/kubescape -// values.capabilities.kubescapeOffline: enable -> disable +// helm.toolkit.fluxcd.io/v2 HelmRelease kube-system/cilium +// values.devices: en+ eth+ -> enp7s0 eth1 +// values.updateStrategy: {rollingUpdate: null, type: OnDelete} -> removed +// values.nodePort.addresses: [10.0.0.0/16] -> removed +// upgrade.disableWait: true -> removed // -// That field decides one thing in the chart: whether KS_OFFLINE=true is set on -// the scanner container. It reaches no identity, binding, policy document, or -// service account. Its only other chart use ORs into clusterData.keepLocal, -// which is `or (offline) (not serviceDiscovery.enabled)` — serviceDiscovery is -// false here, so that value stays true either way and does not move. The -// accompanying cilium-network-policy.yaml edit is comment-only and renders to -// nothing, which the byte-identical infrastructure root confirms. +// Those fields decide which NICs the agent attaches its eBPF programs to, how +// the DaemonSet replaces pods, which CIDR NodePort listens on, and whether Helm +// waits for the upgrade. None reaches an identity, binding, policy document, or +// service account, and none is an AWS-facing field. // // All Role / ClusterRole / RoleBinding / ClusterRoleBinding / ServiceAccount -// documents are byte-identical, as is every `aws`-bearing line — trivially so, -// since a single non-RBAC line differs across the entire surface. +// documents are byte-identical — 67 documents, 38196 bytes, compared directly +// rather than inferred from the line diff — as is every `aws`-bearing line +// (123 of them). A planted-change control confirms that comparison is not blind. // // The fingerprint itself was read from the required job's own output on the // approved renderer, because the local toolchain is refused as unapproved. -const expectedRenderedSurfaceSHA = "cb575e34b191a662da108421fdff7c67f0cc00bf3c9b2b7cf0c5a4e49b46fc52" +const expectedRenderedSurfaceSHA = "4c06f17870f9eb129abf9c49e89fef2613154f22713929a9eb5835190312177d" // authorizationOverlayPaths lists every independently reconciled production // layer where an object can grant privileges to the aws/aws service account.