From c65c205cb0ab132c587628f1393916efb6b3dfd5 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 28 Oct 2026 09:12:38 -0700 Subject: [PATCH 1/7] fix(build): forcefully remove /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/ after configuring cgroup telemetry --- vhdbuilder/packer/install-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 28a231a6436..5f050faf8bb 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -1455,7 +1455,7 @@ fi if [ -d "/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/" ] && [ "$(ls -A /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/)" ]; then cat /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/* - rm -r /var/log/azure/Microsoft.Azure.Extensions.CustomScript || exit 1 + rm -rf /var/log/azure/Microsoft.Azure.Extensions.CustomScript || exit 1 fi capture_benchmark "${SCRIPT_NAME}_configure_telemetry" From 17c5fcd47bdb6918a5968915315bcc1fae0ea748 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 28 Oct 2026 10:40:13 -0700 Subject: [PATCH 2/7] chore: fix --- vhdbuilder/packer/install-dependencies.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 5f050faf8bb..b799d725197 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -1455,6 +1455,7 @@ fi if [ -d "/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/" ] && [ "$(ls -A /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/)" ]; then cat /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/* + rm -rf /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events || exit 1 rm -rf /var/log/azure/Microsoft.Azure.Extensions.CustomScript || exit 1 fi capture_benchmark "${SCRIPT_NAME}_configure_telemetry" From d3496acdeb51539bcc8c281ee07e15d23c95b287 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 28 Oct 2026 14:04:06 -0700 Subject: [PATCH 3/7] chore: move to nodePrep --- .../linux/cloud-init/artifacts/cse_config.sh | 17 +++ .../linux/cloud-init/artifacts/cse_helpers.sh | 2 + parts/linux/cloud-init/artifacts/cse_main.sh | 2 + .../cloud-init/artifacts/cse_config_spec.sh | 103 ++++++++++++++++++ vhdbuilder/packer/install-dependencies.sh | 20 ---- 5 files changed, 124 insertions(+), 20 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 91d0ece29f6..f30f9fd38fd 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -823,3 +823,20 @@ get_compute_sku() { fi echo "$vm_sku" } + +enableAndStartCgroupTelemetry() { + # Don't enable cgroup-memory-telemetry on AzureLinux due to incompatibility with cgroup2fs driver and absence of required azure.slice directory + if ! isMarinerOrAzureLinux "$OS"; then + systemctlEnableAndStartNoBlock cgroup-memory-telemetry.timer 30 || exit $ERR_START_CGROUP_TELEMETRY + systemctlEnableAndStartNoBlock cgroup-memory-telemetry.service || exit $ERR_START_CGROUP_TELEMETRY + fi + + CGROUP_VERSION=$(stat -fc %T /sys/fs/cgroup) + if [ "$CGROUP_VERSION" = "cgroup2fs" ]; then + systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.timer 30 || exit $ERR_START_CGROUP_TELEMETRY + systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.service || exit $ERR_START_CGROUP_TELEMETRY + + systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.timer 30 || exit $ERR_START_CGROUP_TELEMETRY + systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.service || exit $ERR_START_CGROUP_TELEMETRY + fi +} diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index b1c01fd5031..cf5ea00a14b 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -170,6 +170,8 @@ ERR_CHRONY_NTP_SYNC_FAIL=245 # Chrony could not synchronize with the configured ERR_CHRONY_CONFIG_FAIL=246 # Chrony could not be configured for Intel TDX # ----------------------------------------------------------------------------- +ERR_START_CGROUP_TELEMETRY=247 # Error enabling and starting cgroup telemetry services + # This probably wasn't launched via a login shell, so ensure the PATH is correct. [ -f /etc/profile.d/path.sh ] && . /etc/profile.d/path.sh diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index 25e750f832f..85b86070d5b 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -663,6 +663,8 @@ function nodePrep { fi fi + logs_to_events "AKS.CSE.enableAndStartCgroupTelemetry" enableAndStartCgroupTelemetry + if $REBOOTREQUIRED; then echo 'reboot required, rebooting node in 1 minute' /bin/bash -c "shutdown -r 1 &" diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh index a1a11a97a09..df37aa17b8d 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh @@ -12,6 +12,109 @@ Describe 'cse_config.sh' CSE_CONFIG_ADDONS_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_addons.sh" Include "./parts/linux/cloud-init/artifacts/cse_config.sh" Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" + Describe 'enableAndStartCgroupTelemetry' + setup_cgroup_telemetry() { + OS="UBUNTU" + OS_VARIANT="" + MOCK_CGROUP_VERSION="cgroup2fs" + FAILED_TELEMETRY_COMMAND="" + memory_commands="systemctlEnableAndStartNoBlock cgroup-memory-telemetry.timer 30 +systemctlEnableAndStartNoBlock cgroup-memory-telemetry.service" + cpu_pressure_commands="systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.timer 30 +systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.service +systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.timer 30 +systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.service" + } + + BeforeEach 'setup_cgroup_telemetry' + + stat() { + if [ "$*" != "-fc %T /sys/fs/cgroup" ]; then + echo "Unexpected stat arguments: $*" >&2 + return 1 + fi + echo "$MOCK_CGROUP_VERSION" + } + + systemctlEnableAndStartNoBlock() { + echo "systemctlEnableAndStartNoBlock $*" + [ "systemctlEnableAndStartNoBlock $*" != "$FAILED_TELEMETRY_COMMAND" ] + } + + Describe 'OS and cgroup eligibility' + Parameters + UBUNTU "" true + MARINER "" false + MARINERKATA "" false + AZURELINUX "" false + AZURELINUXKATA "" false + AZURELINUX OSGUARD false + AZURECONTAINERLINUX "" true + AZURELINUX AZURECONTAINERLINUX true + End + + It "configures eligible telemetry on cgroup v2 for $1 variant '$2'" + OS="$1" + OS_VARIANT="$2" + expected_commands="$cpu_pressure_commands" + if [ "$3" = "true" ]; then + expected_commands="$memory_commands +$cpu_pressure_commands" + fi + + When run enableAndStartCgroupTelemetry + + The status should be success + The output should equal "$expected_commands" + End + + It "configures only eligible memory telemetry on cgroup v1 for $1 variant '$2'" + OS="$1" + OS_VARIANT="$2" + MOCK_CGROUP_VERSION="tmpfs" + expected_commands="" + if [ "$3" = "true" ]; then + expected_commands="$memory_commands" + fi + + When run enableAndStartCgroupTelemetry + + The status should be success + The output should equal "$expected_commands" + End + End + + Describe 'startup failures' + Parameters + "systemctlEnableAndStartNoBlock cgroup-memory-telemetry.timer 30" 1 + "systemctlEnableAndStartNoBlock cgroup-memory-telemetry.service" 2 + "systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.timer 30" 3 + "systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.service" 4 + "systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.timer 30" 5 + "systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.service" 6 + End + + It "exits with the telemetry error and stops setup when '$1' fails" + FAILED_TELEMETRY_COMMAND="$1" + + When run enableAndStartCgroupTelemetry + + The status should equal "$ERR_START_CGROUP_TELEMETRY" + The lines of output should equal "$2" + The line "$2" of output should equal "$1" + End + End + + It 'exits with the telemetry error when memory service startup fails on cgroup v1' + MOCK_CGROUP_VERSION="tmpfs" + FAILED_TELEMETRY_COMMAND="systemctlEnableAndStartNoBlock cgroup-memory-telemetry.service" + + When run enableAndStartCgroupTelemetry + + The status should equal "$ERR_START_CGROUP_TELEMETRY" + The output should equal "$memory_commands" + End + End Describe 'configureTransparentHugePageSystemdService' setup_thp_service() { THP_ENABLED="never" diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index b799d725197..9ea1df15023 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -1435,30 +1435,10 @@ fi configureServiceExecutionTelemetryForSystemdVersion "$(systemctl --version | awk 'NR == 1 { print $2 }')" || exit $ERR_PACKER_COPY_FILE systemctl daemon-reload -# Disable cgroup-memory-telemetry on AzureLinux due to incompatibility with cgroup2fs driver and absence of required azure.slice directory -if ! isMarinerOrAzureLinux "$OS"; then - systemctlEnableAndStart cgroup-memory-telemetry.timer 30 || exit 1 - systemctl enable cgroup-memory-telemetry.service || exit 1 - systemctl restart cgroup-memory-telemetry.service -fi - -CGROUP_VERSION=$(stat -fc %T /sys/fs/cgroup) -if [ "$CGROUP_VERSION" = "cgroup2fs" ]; then - systemctlEnableAndStart cgroup-cpu-telemetry.timer 30 || exit 1 - systemctl enable cgroup-cpu-telemetry.service || exit 1 - systemctl restart cgroup-cpu-telemetry.service - - systemctlEnableAndStart cgroup-pressure-telemetry.timer 30 || exit 1 - systemctl enable cgroup-pressure-telemetry.service || exit 1 - systemctl restart cgroup-pressure-telemetry.service -fi - if [ -d "/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/" ] && [ "$(ls -A /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/)" ]; then cat /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/* - rm -rf /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events || exit 1 rm -rf /var/log/azure/Microsoft.Azure.Extensions.CustomScript || exit 1 fi -capture_benchmark "${SCRIPT_NAME}_configure_telemetry" if [ "$OS" = "$UBUNTU_OS_NAME" ]; then # remove snapd, which is not used by container stack From ff2de502944aa9c5640a143d42e9aa1d2c35372a Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 28 Oct 2026 14:11:43 -0700 Subject: [PATCH 4/7] chore: move cleanup --- vhdbuilder/packer/cleanup-vhd.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vhdbuilder/packer/cleanup-vhd.sh b/vhdbuilder/packer/cleanup-vhd.sh index 10e73fff409..25376234f65 100644 --- a/vhdbuilder/packer/cleanup-vhd.sh +++ b/vhdbuilder/packer/cleanup-vhd.sh @@ -124,3 +124,5 @@ rm -f /opt/azure/disk-usage.txt rm -f /opt/azure/containers/image-fetcher # Cleanup IMDS instance metadata cache file rm -f /opt/azure/containers/imds_instance_metadata_cache.json +# Cleanup Microsoft.Azure.Extensions.CustomScript to prevent unnecessary logs/telemetry payloads being persisted on the OS disk +rm -rf /var/log/azure/Microsoft.Azure.Extensions.CustomScript From c7c390085b4357044aed691238e67a376f806c42 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 28 Oct 2026 14:13:57 -0700 Subject: [PATCH 5/7] chore: fix --- parts/linux/cloud-init/artifacts/cse_config.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index f30f9fd38fd..322f62eca8b 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -828,15 +828,15 @@ enableAndStartCgroupTelemetry() { # Don't enable cgroup-memory-telemetry on AzureLinux due to incompatibility with cgroup2fs driver and absence of required azure.slice directory if ! isMarinerOrAzureLinux "$OS"; then systemctlEnableAndStartNoBlock cgroup-memory-telemetry.timer 30 || exit $ERR_START_CGROUP_TELEMETRY - systemctlEnableAndStartNoBlock cgroup-memory-telemetry.service || exit $ERR_START_CGROUP_TELEMETRY + systemctlEnableAndStartNoBlock cgroup-memory-telemetry.service 30 || exit $ERR_START_CGROUP_TELEMETRY fi CGROUP_VERSION=$(stat -fc %T /sys/fs/cgroup) if [ "$CGROUP_VERSION" = "cgroup2fs" ]; then systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.timer 30 || exit $ERR_START_CGROUP_TELEMETRY - systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.service || exit $ERR_START_CGROUP_TELEMETRY + systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.service 30 || exit $ERR_START_CGROUP_TELEMETRY systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.timer 30 || exit $ERR_START_CGROUP_TELEMETRY - systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.service || exit $ERR_START_CGROUP_TELEMETRY + systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.service 30 || exit $ERR_START_CGROUP_TELEMETRY fi } From 1ddfbefe9c9ce1b40a39c0b4a4098cb82bd64309 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 28 Oct 2026 14:14:17 -0700 Subject: [PATCH 6/7] chore: clean diff --- vhdbuilder/packer/install-dependencies.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 9ea1df15023..cc2c2b081ac 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -1435,11 +1435,6 @@ fi configureServiceExecutionTelemetryForSystemdVersion "$(systemctl --version | awk 'NR == 1 { print $2 }')" || exit $ERR_PACKER_COPY_FILE systemctl daemon-reload -if [ -d "/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/" ] && [ "$(ls -A /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/)" ]; then - cat /var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/* - rm -rf /var/log/azure/Microsoft.Azure.Extensions.CustomScript || exit 1 -fi - if [ "$OS" = "$UBUNTU_OS_NAME" ]; then # remove snapd, which is not used by container stack apt_get_purge 20 30 120 snapd || exit 1 From f90e630105c4f522c2a0b71584032f5d125c28e4 Mon Sep 17 00:00:00 2001 From: cameronmeissner Date: Wed, 28 Oct 2026 14:15:47 -0700 Subject: [PATCH 7/7] chore: fix tests --- .../linux/cloud-init/artifacts/cse_config_spec.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh index df37aa17b8d..e980791899f 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh @@ -19,11 +19,11 @@ Describe 'cse_config.sh' MOCK_CGROUP_VERSION="cgroup2fs" FAILED_TELEMETRY_COMMAND="" memory_commands="systemctlEnableAndStartNoBlock cgroup-memory-telemetry.timer 30 -systemctlEnableAndStartNoBlock cgroup-memory-telemetry.service" +systemctlEnableAndStartNoBlock cgroup-memory-telemetry.service 30" cpu_pressure_commands="systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.timer 30 -systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.service +systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.service 30 systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.timer 30 -systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.service" +systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.service 30" } BeforeEach 'setup_cgroup_telemetry' @@ -87,11 +87,11 @@ $cpu_pressure_commands" Describe 'startup failures' Parameters "systemctlEnableAndStartNoBlock cgroup-memory-telemetry.timer 30" 1 - "systemctlEnableAndStartNoBlock cgroup-memory-telemetry.service" 2 + "systemctlEnableAndStartNoBlock cgroup-memory-telemetry.service 30" 2 "systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.timer 30" 3 - "systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.service" 4 + "systemctlEnableAndStartNoBlock cgroup-cpu-telemetry.service 30" 4 "systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.timer 30" 5 - "systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.service" 6 + "systemctlEnableAndStartNoBlock cgroup-pressure-telemetry.service 30" 6 End It "exits with the telemetry error and stops setup when '$1' fails" @@ -107,7 +107,7 @@ $cpu_pressure_commands" It 'exits with the telemetry error when memory service startup fails on cgroup v1' MOCK_CGROUP_VERSION="tmpfs" - FAILED_TELEMETRY_COMMAND="systemctlEnableAndStartNoBlock cgroup-memory-telemetry.service" + FAILED_TELEMETRY_COMMAND="systemctlEnableAndStartNoBlock cgroup-memory-telemetry.service 30" When run enableAndStartCgroupTelemetry