From c74bbe5333b28b489c77fc2b2a7bc6bcbb13dd09 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 22 Sep 2026 10:00:42 -0700 Subject: [PATCH 01/20] feat: enable artifact streaming on ACL --- parts/common/components.json | 11 +++++ .../cloud-init/artifacts/README-COMPONENTS.md | 5 +++ .../linux/cloud-init/artifacts/cse_config.sh | 5 ++- .../linux/cloud-init/artifacts/cse_helpers.sh | 4 +- schemas/components.cue | 5 +++ .../artifacts/cse_config_gpu_spec.sh | 21 ++++++++++ .../cloud-init/artifacts/cse_helpers_spec.sh | 20 +++++++++ .../cloud-init/artifacts/test_components.json | 21 ++++++++++ vhdbuilder/packer/acl-customdata.json | 2 +- vhdbuilder/packer/acl-customdata.yaml | 10 +++++ vhdbuilder/packer/install-dependencies.sh | 41 ++++++++++++------- 11 files changed, 126 insertions(+), 19 deletions(-) diff --git a/parts/common/components.json b/parts/common/components.json index 76c81bd8237..2428faf140b 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -2526,6 +2526,17 @@ } ] } + }, + "azurecontainerlinux": { + "current": { + "versionsV2": [ + { + "renovateTag": "", + "latestVersion": "1.0.0" + } + ], + "downloadURL": "https://acrstreamingpackage.z5.web.core.windows.net/${version}/acr-mirror-azurelinux3.rpm" + } } } }, diff --git a/parts/linux/cloud-init/artifacts/README-COMPONENTS.md b/parts/linux/cloud-init/artifacts/README-COMPONENTS.md index feb4f8fbd9f..b2163dc3d51 100644 --- a/parts/linux/cloud-init/artifacts/README-COMPONENTS.md +++ b/parts/linux/cloud-init/artifacts/README-COMPONENTS.md @@ -93,6 +93,7 @@ Please refer to [components.cue](../../../../schemas/components.cue) for the mos mariner?: #MarinerOSDistro marinerkata?: #MarinerOSDistro azurelinux?: #AzureLinuxOSDistro + azurecontainerlinux?: #AzureContainerLinuxOSDistro } ``` @@ -115,6 +116,9 @@ Please refer to [components.cue](../../../../schemas/components.cue) for the mos "OSGUARD/v3.0"?: #ReleaseDownloadURI current?: #ReleaseDownloadURI } +#AzureContainerLinuxOSDistro: { + current?: #ReleaseDownloadURI +} ``` ``` @@ -139,6 +143,7 @@ Here are the explanation of the above schema. - In `UbuntuOSDistro`, we can define different OS release versions. For example, `r2404` implies release 24.04. - In `MarinerOSDistro`, we only have `current` now, which implies that single configurations will be applied to all Mariner release versions. We can distinguish them in needed. Note, we confirmed with Mariner team, Azure Linux 2.0 is reporting itself as `mariner` in the file `/etc/os-release`. So for Azure Linux 2.0 case, it will still read the package versions from `mariner` block. - In `AzureLinuxOSDistro`, `v3.0` is for Azure Linux v3.0. `current` is for otherwise but we are not using it now. Azure Linux 2.0 case is described in the `MarinerOSDistro` above. + - In `AzureContainerLinuxOSDistro`, `current` defines ACL-specific metadata. When it is absent, ACL falls back to `flatcar.current`, then `default.current`. - `DefaultOSDistro` means the default case of OS Distro. If an OSDistro metadata is not defined, it will fetch it from `default`. For example, if a node is Ubuntu 20.04, but we don't specify `ubuntu` in components.json, then it will fetch `default.current`. For another example, if only `default.current` is specified in the components.json, No matter what OSDistro is the node running, it will only fetch `default.current` because it's the default metadata. This provides flexibility while elimiating unnecessary duplication when defining the metadata. 1. The OS release version can optionally be prefixed with the OS variant. `OSGUARD/v3.0` and `DEFAULT/v3.0` are currently supported for `azurelinux`. The latter specifically matches Azure Linux 3.0 without a variant, i.e. not OS Guard. 1. In `ReleaseDownloadURI`, you can see 2 keys. diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 91d0ece29f6..419fcca12a4 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -635,7 +635,10 @@ EOF } ensureArtifactStreaming() { - waitForContainerdReady || exit $ERR_ARTIFACT_STREAMING_INSTALL + waitForContainerdReady || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + if isACL "$OS" "$OS_VARIANT"; then + systemctl unmask overlaybd-tcmu.service overlaybd-snapshotter.service || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + fi local acr_mirror_setup="${ACR_MIRROR_SETUP_SCRIPT:-/opt/acr/tools/mirror/setup.sh}" if [ -x "$acr_mirror_setup" ]; then diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index b1c01fd5031..a088652e6e6 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -1213,9 +1213,9 @@ getPackageJSON() { search=".downloadURIs.${osLowerCase}.\"${osVariant}/r${osVersion//.}\" // .downloadURIs.${osLowerCase}.\"r${osVersion//.}\" // ${search}" fi - # ACL is Flatcar-based; use flatcar download entries. + # ACL is Flatcar-based; prefer ACL-specific entries, then fall back to Flatcar. if isACL "${os}" "${osVariant}"; then - search=".downloadURIs.flatcar.current // .downloadURIs.default.current" + search=".downloadURIs.azurecontainerlinux.current // .downloadURIs.flatcar.current // .downloadURIs.default.current" fi jq -r -c "${search}" <<< "${package}" diff --git a/schemas/components.cue b/schemas/components.cue index 89a9fe9a062..d22ad74c6ec 100644 --- a/schemas/components.cue +++ b/schemas/components.cue @@ -100,6 +100,10 @@ package components current?: #ReleaseDownloadURI } +#AzureContainerLinuxOSDistro: { + current?: #ReleaseDownloadURI +} + #DownloadURIs: { default?: #DefaultOSDistro ubuntu?: #UbuntuOSDistro @@ -109,6 +113,7 @@ package components azurelinuxkata?: #AzureLinuxOSDistro windows?: #WindowsOsDistro flatcar?: #FlatcarOSDistro + azurecontainerlinux?: #AzureContainerLinuxOSDistro } #Package: { diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh index 6651ec586fa..17d72ad96de 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh @@ -140,6 +140,8 @@ Describe 'cse_config_gpu.sh' TEST_ACR_DIR="$(mktemp -d)" ACR_MIRROR_SETUP_SCRIPT="${TEST_ACR_DIR}/setup.sh" ACR_CONFIG_BIN="${TEST_ACR_DIR}/acr-config" + OS="$UBUNTU_OS_NAME" + OS_VARIANT="" } cleanup_streaming() { rm -rf "${TEST_ACR_DIR}" @@ -185,6 +187,25 @@ Describe 'cse_config_gpu.sh' The status should be success End + It 'unmasks the OverlayBD services on Azure Container Linux' + OS="$ACL_OS_NAME" + install_setup_sh + When run ensureArtifactStreaming + The output should include "systemctl unmask overlaybd-tcmu.service overlaybd-snapshotter.service" + The output should include "setup.sh aks" + The status should be success + End + + It 'unmasks the OverlayBD services for the Azure Linux ACL variant' + OS="$AZURELINUX_OS_NAME" + OS_VARIANT="$ACL_OS_VARIANT" + install_setup_sh + When run ensureArtifactStreaming + The output should include "systemctl unmask overlaybd-tcmu.service overlaybd-snapshotter.service" + The output should include "setup.sh aks" + The status should be success + End + It 'fails fast when enabling the streaming services fails' RETRYCMD_RC=1 install_setup_sh diff --git a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh index 905f47c78cf..55978630ea0 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh @@ -85,6 +85,21 @@ Describe 'cse_helpers.sh' When call updatePackageVersions "$package" "MARINER" "current" The variable PACKAGE_VERSIONS[@] should equal "dummyVersion5 dummyVersion6.1 dummyVersion6.0" End + It 'returns the ACL-specific version for Azure Container Linux' + package=$(readPackage "pkgVersionsV2") + When call updatePackageVersions "$package" "AZURECONTAINERLINUX" "3.0" + The variable PACKAGE_VERSIONS[@] should equal "dummyVersionACL" + End + It 'returns the ACL-specific version for the Azure Linux ACL variant' + package=$(readPackage "pkgVersionsV2") + When call updatePackageVersions "$package" "AZURELINUX" "3.0" "AZURECONTAINERLINUX" + The variable PACKAGE_VERSIONS[@] should equal "dummyVersionACL" + End + It 'falls back to the Flatcar version when no ACL-specific version exists' + package=$(readPackage "pkgVersionsV2" | jq 'del(.downloadURIs.azurecontainerlinux)') + When call updatePackageVersions "$package" "AZURECONTAINERLINUX" "3.0" + The variable PACKAGE_VERSIONS[@] should equal "dummyVersionFlatcar" + End It 'returns if there is a in latestVersion' package=$(readPackage "pkgVersionsV2") When call updatePackageVersions "$package" "MARINERKATA" "current" @@ -123,6 +138,11 @@ Describe 'cse_helpers.sh' When call updatePackageDownloadURL "$package" "UBUNTU" "dummy_release" The variable PACKAGE_DOWNLOAD_URL should equal "https://dummydefaultcurrentpath/v\${version}/dummy_\${version}_linux_\${CPU_ARCH}.tar.gz" End + It 'returns the ACL-specific download URL for Azure Container Linux' + package=$(readPackage "pkgVersionsV2") + When call updatePackageDownloadURL "$package" "AZURECONTAINERLINUX" "3.0" + The variable PACKAGE_DOWNLOAD_URL should equal "https://dummypath/dummy-acl.rpm" + End End Describe 'evalPackageDownloadURL' diff --git a/spec/parts/linux/cloud-init/artifacts/test_components.json b/spec/parts/linux/cloud-init/artifacts/test_components.json index 01389f78a93..26f22d10a16 100644 --- a/spec/parts/linux/cloud-init/artifacts/test_components.json +++ b/spec/parts/linux/cloud-init/artifacts/test_components.json @@ -108,6 +108,27 @@ ] } }, + "azurecontainerlinux": { + "current": { + "versionsV2": [ + { + "renovateTag": "", + "latestVersion": "dummyVersionACL" + } + ], + "downloadURL": "https://dummypath/dummy-acl.rpm" + } + }, + "flatcar": { + "current": { + "versionsV2": [ + { + "renovateTag": "", + "latestVersion": "dummyVersionFlatcar" + } + ] + } + }, "default": { "current": { "versionsV2": [ diff --git a/vhdbuilder/packer/acl-customdata.json b/vhdbuilder/packer/acl-customdata.json index 8783d77f65c..3e7ba8a237c 100644 --- a/vhdbuilder/packer/acl-customdata.json +++ b/vhdbuilder/packer/acl-customdata.json @@ -1 +1 @@ -{"ignition":{"version":"3.4.0"},"kernelArguments":{"shouldNotExist":["flatcar.autologin"]},"storage":{"directories":[{"path":"/opt/aks-sysext-rw/usr/local/bin"}],"files":[{"overwrite":true,"path":"/opt/aks-sysext/usr/lib/extension-release.d/extension-release.aks-sysext","contents":{"compression":"","source":"data:,ID%3Dazurelinux%0ASYSEXT_LEVEL%3D1.0%0A"}},{"overwrite":true,"path":"/etc/flatcar/update.conf","contents":{"compression":"","source":"data:,SERVER%3Ddisabled%0A"}},{"overwrite":true,"path":"/etc/systemd/system/containerd.service.d/50-default-config.conf","contents":{"compression":"","source":"data:,%5BService%5D%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0A"}}],"links":[{"path":"/etc/extensions/aks-sysext","hard":false,"target":"/opt/aks-sysext"},{"path":"/opt/aks-sysext/usr/local/bin","hard":false,"target":"/opt/aks-sysext-rw/usr/local/bin"},{"path":"/etc/containerd/config.toml","hard":false,"target":"/usr/share/containerd/config.toml"}]}} +{"ignition":{"version":"3.4.0"},"kernelArguments":{"shouldNotExist":["flatcar.autologin"]},"storage":{"directories":[{"path":"/opt/aks-sysext-rw/usr/local/bin"}],"files":[{"overwrite":true,"path":"/opt/aks-sysext/usr/lib/extension-release.d/extension-release.aks-sysext","contents":{"compression":"","source":"data:,ID%3Dazurelinux%0ASYSEXT_LEVEL%3D1.0%0A"}},{"path":"/etc/flatcar/enabled-sysext.conf","contents":{"compression":"","source":"data:,overlaybd%0A"}},{"overwrite":true,"path":"/etc/flatcar/update.conf","contents":{"compression":"","source":"data:,SERVER%3Ddisabled%0A"}},{"overwrite":true,"path":"/etc/systemd/system/containerd.service.d/50-default-config.conf","contents":{"compression":"","source":"data:,%5BService%5D%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0A"}}],"links":[{"path":"/etc/extensions/aks-sysext","hard":false,"target":"/opt/aks-sysext"},{"path":"/opt/aks-sysext/usr/local/bin","hard":false,"target":"/opt/aks-sysext-rw/usr/local/bin"},{"path":"/etc/containerd/config.toml","hard":false,"target":"/usr/share/containerd/config.toml"}]},"systemd":{"units":[{"mask":true,"name":"overlaybd-tcmu.service"},{"mask":true,"name":"overlaybd-snapshotter.service"}]}} diff --git a/vhdbuilder/packer/acl-customdata.yaml b/vhdbuilder/packer/acl-customdata.yaml index 903069da068..4dbf1618916 100644 --- a/vhdbuilder/packer/acl-customdata.yaml +++ b/vhdbuilder/packer/acl-customdata.yaml @@ -13,6 +13,10 @@ storage: inline: | ID=azurelinux SYSEXT_LEVEL=1.0 + - path: /etc/flatcar/enabled-sysext.conf + contents: + inline: | + overlaybd - path: /etc/flatcar/update.conf overwrite: true contents: @@ -42,3 +46,9 @@ storage: # NOTE: ACL does NOT have update-ca-certificates.service (Flatcar-specific). # ACL uses update-ca-trust (Azure Linux style) which is invoked on-demand by CSE scripts. # The Flatcar customdata has a dropin for update-ca-certificates.service that is omitted here. +systemd: + units: + - name: overlaybd-tcmu.service + mask: true + - name: overlaybd-snapshotter.service + mask: true diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 28a231a6436..e5bf8a9a2b5 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -398,23 +398,34 @@ installAndConfigureArtifactStreaming() { downloadURL="${downloadURL%.*}-arm64.${downloadURL##*.}" fi local MIRROR_DOWNLOAD_PATH="./$(basename "${downloadURL}")" - retrycmd_curl_file 10 5 60 "$MIRROR_DOWNLOAD_PATH" "$downloadURL" || exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} - case "$downloadURL" in - *.deb) - apt_get_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD - ;; - *.rpm) - dnf_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD - ;; - *) - echo "Unsupported acr-mirror package extension in URL: ${downloadURL}" >&2 - exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} - ;; - esac + retrycmd_curl_file 10 5 60 "$MIRROR_DOWNLOAD_PATH" "$downloadURL" || exit "${ERR_ARTIFACT_STREAMING_DOWNLOAD}" + + if isACL "$OS" "$OS_VARIANT"; then + bsdtar -C / -xf "$MIRROR_DOWNLOAD_PATH" opt/ || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + bsdtar -Oxf "$MIRROR_DOWNLOAD_PATH" usr/lib/systemd/system/acr-mirror.service | + install -m 0644 /dev/stdin /etc/systemd/system/acr-mirror.service || + exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + env -C /opt/acr/bin ./acr init --min-init || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + else + case "$downloadURL" in + *.deb) + apt_get_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + ;; + *.rpm) + dnf_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + ;; + *) + echo "Unsupported acr-mirror package extension in URL: ${downloadURL}" >&2 + exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + ;; + esac + fi rm "$MIRROR_DOWNLOAD_PATH" - /opt/acr/tools/overlaybd/install.sh - # Remove the bundled overlaybd installer packages (~55-58 MB); install.sh already installed them and they're unused at runtime. + if ! isACL "$OS" "$OS_VARIANT"; then + /opt/acr/tools/overlaybd/install.sh + fi + # Remove the bundled OverlayBD installer packages (~55-58 MB); they're unused at runtime. rm -f /opt/acr/tools/overlaybd/bin/*.deb /opt/acr/tools/overlaybd/bin/*.rpm echo " - acr-mirror version ${version}" >> ${VHD_LOGS_FILEPATH} } From 69e9a48622cd1f42f5c1edec110e0c2cbc018fdc Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 22 Sep 2026 14:33:15 -0700 Subject: [PATCH 02/20] feat: enable ACL artifact streaming in ANC Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../pkg/nodeconfigutils/utils.go | 36 ++++++++++- .../pkg/nodeconfigutils/utils_test.go | 60 +++++++++++++++++++ 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/aks-node-controller/pkg/nodeconfigutils/utils.go b/aks-node-controller/pkg/nodeconfigutils/utils.go index e7338bc0326..6e04247fbb4 100644 --- a/aks-node-controller/pkg/nodeconfigutils/utils.go +++ b/aks-node-controller/pkg/nodeconfigutils/utils.go @@ -64,15 +64,41 @@ if [ -s /sys/class/dmi/id/product_uuid ]; then fi` flatcarTemplate = `{ - "ignition": { "version": "3.4.0" }, + "ignition": { "version": "3.4.0" }%[2]s, "storage": { "files": [{ "path": "/opt/azure/containers/aks-node-controller-config.json", "mode": 384, - "contents": { "source": "data:;base64,%s" } + "contents": { "source": "data:;base64,%[1]s" } }] } }` + + artifactStreamingSystemd = `, + "systemd": { + "units": [ + { + "name": "acr-mirror.service", + "enabled": true, + "mask": false + }, + { + "name": "overlaybd-tcmu.service", + "enabled": true, + "mask": false + }, + { + "name": "overlaybd-snapshotter.service", + "enabled": true, + "mask": false + }, + { + "name": "aks-artifact-streaming.service", + "enabled": true, + "contents": "[Unit]\nDescription=Configure AKS artifact streaming\nRequires=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service\nAfter=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service\nBefore=aks-node-controller.service\n\n[Service]\nType=oneshot\nExecStart=/opt/acr/bin/acr-config --enable-containerd azurecr.io\nRemainAfterExit=yes\n\n[Install]\nWantedBy=multi-user.target\n" + } + ] + }` ) // CustomData builds base64-encoded VM custom data for cloud-init. The boothook writes the node @@ -137,7 +163,11 @@ func CustomDataFlatcar(cfg *aksnodeconfigv1.Configuration) (string, error) { } encodedAksNodeConfigJSON := base64.StdEncoding.EncodeToString(aksNodeConfigJSON) - customDataYAML := fmt.Sprintf(flatcarTemplate, encodedAksNodeConfigJSON) + var artifactStreamingConfig string + if cfg.GetEnableArtifactStreaming() { + artifactStreamingConfig = artifactStreamingSystemd + } + customDataYAML := fmt.Sprintf(flatcarTemplate, encodedAksNodeConfigJSON, artifactStreamingConfig) return base64.StdEncoding.EncodeToString([]byte(customDataYAML)), nil } diff --git a/aks-node-controller/pkg/nodeconfigutils/utils_test.go b/aks-node-controller/pkg/nodeconfigutils/utils_test.go index 5bea3f89bdc..c2be19f3087 100644 --- a/aks-node-controller/pkg/nodeconfigutils/utils_test.go +++ b/aks-node-controller/pkg/nodeconfigutils/utils_test.go @@ -2,6 +2,7 @@ package nodeconfigutils import ( "encoding/base64" + "encoding/json" "io" "mime" "mime/multipart" @@ -44,6 +45,18 @@ func TestContainerdImagePullPresenceRoundTrip(t *testing.T) { } } +func decodeFlatcarCustomData(t *testing.T, cfg *aksnodeconfigv1.Configuration) map[string]any { + t.Helper() + customData, err := CustomDataFlatcar(cfg) + require.NoError(t, err) + decoded, err := base64.StdEncoding.DecodeString(customData) + require.NoError(t, err) + + var ignition map[string]any + require.NoError(t, json.Unmarshal(decoded, &ignition)) + return ignition +} + func TestUnmarshalConfigurationV1(t *testing.T) { tests := []struct { name string @@ -300,6 +313,53 @@ func TestCustomDataUsesMultipartBoothookAndCloudConfig(t *testing.T) { require.ErrorIs(t, err, io.EOF) } +func TestCustomDataFlatcarOmitsArtifactStreamingWhenDisabled(t *testing.T) { + ignition := decodeFlatcarCustomData(t, &aksnodeconfigv1.Configuration{}) + + _, found := ignition["systemd"] + require.False(t, found) +} + +func TestCustomDataFlatcarEnablesArtifactStreaming(t *testing.T) { + ignition := decodeFlatcarCustomData(t, &aksnodeconfigv1.Configuration{EnableArtifactStreaming: true}) + + systemd, ok := ignition["systemd"].(map[string]any) + require.True(t, ok) + units, ok := systemd["units"].([]any) + require.True(t, ok) + + unitsByName := make(map[string]map[string]any, len(units)) + for _, item := range units { + unit, ok := item.(map[string]any) + require.True(t, ok) + name, ok := unit["name"].(string) + require.True(t, ok) + unitsByName[name] = unit + } + + for _, name := range []string{ + "acr-mirror.service", + "overlaybd-tcmu.service", + "overlaybd-snapshotter.service", + } { + unit, found := unitsByName[name] + require.True(t, found, "expected %s in Ignition systemd units", name) + require.Equal(t, true, unit["enabled"]) + require.Equal(t, false, unit["mask"]) + } + + setupUnit, found := unitsByName["aks-artifact-streaming.service"] + require.True(t, found) + require.Equal(t, true, setupUnit["enabled"]) + contents, ok := setupUnit["contents"].(string) + require.True(t, ok) + require.Contains(t, contents, "Requires=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service") + require.Contains(t, contents, "After=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service") + require.Contains(t, contents, "Before=aks-node-controller.service") + require.Contains(t, contents, "ExecStart=/opt/acr/bin/acr-config --enable-containerd azurecr.io") + require.Contains(t, contents, "WantedBy=multi-user.target") +} + // decodeBoothook extracts the decoded cloud-boothook part from the base64-encoded custom data. func decodeBoothook(t *testing.T, cfg *aksnodeconfigv1.Configuration) string { t.Helper() From 175c5e2725a8cc47f033499c022cf1be7369ad54 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 22 Sep 2026 14:52:29 -0700 Subject: [PATCH 03/20] test: validate ACL overlaybd containerd config Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aks-node-controller/parser/helper_test.go | 38 +++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 97ab2f8e957..eb23ec140f8 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -939,6 +939,44 @@ oom_score = -999 } } +func TestACLArtifactStreamingContainerdConfig(t *testing.T) { + tests := []struct { + name string + containerdVersion string + noGPU bool + }{ + {name: "containerd v1", containerdVersion: "1.7.22"}, + {name: "containerd v1 no GPU", containerdVersion: "1.7.22", noGPU: true}, + {name: "containerd v2", containerdVersion: "2.0.0"}, + {name: "containerd v2 no GPU", containerdVersion: "2.0.0", noGPU: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config, err := containerdConfigFromAKSNodeConfig(&aksnodeconfigv1.Configuration{ + EnableArtifactStreaming: true, + }, tt.noGPU, tt.containerdVersion) + if err != nil { + t.Fatalf("containerdConfigFromAKSNodeConfig() error = %v", err) + } + + if count := strings.Count(config, `snapshotter = "overlaybd"`); count != 1 { + t.Errorf("expected exactly one overlaybd snapshotter, got %d in:\n%s", count, config) + } + for _, expected := range []string{ + `disable_snapshot_annotations = false`, + `[proxy_plugins.overlaybd]`, + `type = "snapshot"`, + `address = "/run/overlaybd-snapshotter/overlaybd.sock"`, + } { + if !strings.Contains(config, expected) { + t.Errorf("expected rendered config to contain %q:\n%s", expected, config) + } + } + }) + } +} + func Test_getKubenetTemplate(t *testing.T) { tests := []struct { name string From be96e729bf59f0bb24f05026f9e9bcead3927260 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 22 Sep 2026 16:04:19 -0700 Subject: [PATCH 04/20] test: scenario, shellcheck --- e2e/scenario/scenario.go | 31 ++++++ .../packer/install_dependencies_spec.sh | 104 ++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 spec/vhdbuilder/packer/install_dependencies_spec.sh diff --git a/e2e/scenario/scenario.go b/e2e/scenario/scenario.go index c92611b1167..b9b44c6ccd9 100755 --- a/e2e/scenario/scenario.go +++ b/e2e/scenario/scenario.go @@ -78,6 +78,37 @@ var _ = Register(&Scenario{ }, }) +// Artifact streaming changes both the node bootstrap input and the services activated on the +// node, so this requires a separate node from the baseline ACL scenario. +var _ = Register(&Scenario{ + Name: "ACL_ArtifactStreaming", + Description: "Tests that an ACL node with artifact streaming enabled activates OverlayBD and acr-mirror", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDACLGen2TL, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.EnableArtifactStreaming = true + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.EnableArtifactStreaming = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + }, + Validator: func(ctx context.Context, s *Scenario) error { + return errors.Join( + ValidateFileHasContent(ctx, s, "/etc/os-release", "ID=azurelinux"), + ValidateFileHasContent(ctx, s, "/etc/os-release", "VARIANT_ID=azurecontainerlinux"), + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd"), + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service"), + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service"), + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service"), + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service"), + ) + }, + }, +}) + var _ = Register(&Scenario{ Name: "ACL_CustomCA", Description: "Tests that a node using an ACL VHD can be properly bootstrapped and custom CA was correctly added", diff --git a/spec/vhdbuilder/packer/install_dependencies_spec.sh b/spec/vhdbuilder/packer/install_dependencies_spec.sh new file mode 100644 index 00000000000..7c8f02fe557 --- /dev/null +++ b/spec/vhdbuilder/packer/install_dependencies_spec.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +eval "$(sed -n '/^installAndConfigureArtifactStreaming() {/,/^}/p' vhdbuilder/packer/install-dependencies.sh)" + +Describe 'installAndConfigureArtifactStreaming' + setup_artifact_streaming() { + ARTIFACT_STREAMING_CALLS="$(mktemp)" + VHD_LOGS_FILEPATH="$(mktemp)" + ERR_ARTIFACT_STREAMING_DOWNLOAD=149 + OS="UBUNTU" + OS_VARIANT="" + } + cleanup_artifact_streaming() { + command rm -f ./acr-mirror.rpm ./acr-mirror-arm64.rpm + rm -f "$ARTIFACT_STREAMING_CALLS" "$VHD_LOGS_FILEPATH" + } + BeforeEach 'setup_artifact_streaming' + AfterEach 'cleanup_artifact_streaming' + + isARM64() { + echo "${TEST_ARM64:-0}" + } + isACL() { + [ "$OS" = "AZURECONTAINERLINUX" ] || + { [ "$OS" = "AZURELINUX" ] && [ "$OS_VARIANT" = "AZURECONTAINERLINUX" ]; } + } + retrycmd_curl_file() { + echo "retrycmd_curl_file $*" >> "$ARTIFACT_STREAMING_CALLS" + : > "$4" + } + apt_get_install() { + echo "apt_get_install $*" >> "$ARTIFACT_STREAMING_CALLS" + } + dnf_install() { + echo "dnf_install $*" >> "$ARTIFACT_STREAMING_CALLS" + } + bsdtar() { + echo "bsdtar $*" >> "$ARTIFACT_STREAMING_CALLS" + [ "${BSDTAR_RC:-0}" -eq 0 ] || return "$BSDTAR_RC" + if [ "$1" = "-Oxf" ]; then + echo "[Unit]" + fi + } + install() { + cat > /dev/null + echo "install $*" >> "$ARTIFACT_STREAMING_CALLS" + } + env() { + echo "env $*" >> "$ARTIFACT_STREAMING_CALLS" + return "${ACR_INIT_RC:-0}" + } + rm() { + echo "rm $*" >> "$ARTIFACT_STREAMING_CALLS" + command rm "$@" + } + + It 'extracts the rpm payload and initializes acr on Azure Container Linux' + OS="AZURECONTAINERLINUX" + When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "bsdtar -C / -xf ./acr-mirror.rpm opt/" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "bsdtar -Oxf ./acr-mirror.rpm usr/lib/systemd/system/acr-mirror.service" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "install -m 0644 /dev/stdin /etc/systemd/system/acr-mirror.service" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "env -C /opt/acr/bin ./acr init --min-init" + The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "dnf_install" + The status should be success + End + + It 'uses the arm64 package name on Azure Container Linux ARM64' + OS="AZURECONTAINERLINUX" + TEST_ARM64=1 + When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_curl_file 10 5 60 ./acr-mirror-arm64.rpm https://example/acr-mirror-arm64.rpm" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "bsdtar -C / -xf ./acr-mirror-arm64.rpm opt/" + The status should be success + End + + It 'extracts the rpm payload for the Azure Linux ACL variant' + OS="AZURELINUX" + OS_VARIANT="AZURECONTAINERLINUX" + When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "bsdtar -C / -xf ./acr-mirror.rpm opt/" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "env -C /opt/acr/bin ./acr init --min-init" + The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "dnf_install" + The status should be success + End + + It 'fails when the ACL rpm payload cannot be extracted' + OS="AZURECONTAINERLINUX" + BSDTAR_RC=127 + When run installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "bsdtar -C / -xf ./acr-mirror.rpm opt/" + The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "acr init --min-init" + The status should equal "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + End + + It 'fails when acr initialization fails on ACL' + OS="AZURECONTAINERLINUX" + ACR_INIT_RC=1 + When run installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "env -C /opt/acr/bin ./acr init --min-init" + The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "rm ./acr-mirror.rpm" + The status should equal "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + End +End From 701ef3d33a70362cb52a771f94c1297e422c7e72 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 22 Sep 2026 16:24:05 -0700 Subject: [PATCH 05/20] fix: extract ACL acr-mirror without bsdtar Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../packer/install_dependencies_spec.sh | 42 ++++++++++++------- vhdbuilder/packer/install-dependencies.sh | 19 +++++++-- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/spec/vhdbuilder/packer/install_dependencies_spec.sh b/spec/vhdbuilder/packer/install_dependencies_spec.sh index 7c8f02fe557..e1eb38d344e 100644 --- a/spec/vhdbuilder/packer/install_dependencies_spec.sh +++ b/spec/vhdbuilder/packer/install_dependencies_spec.sh @@ -1,4 +1,5 @@ #!/bin/bash +set -o pipefail eval "$(sed -n '/^installAndConfigureArtifactStreaming() {/,/^}/p' vhdbuilder/packer/install-dependencies.sh)" @@ -34,16 +35,24 @@ Describe 'installAndConfigureArtifactStreaming' dnf_install() { echo "dnf_install $*" >> "$ARTIFACT_STREAMING_CALLS" } - bsdtar() { - echo "bsdtar $*" >> "$ARTIFACT_STREAMING_CALLS" - [ "${BSDTAR_RC:-0}" -eq 0 ] || return "$BSDTAR_RC" - if [ "$1" = "-Oxf" ]; then - echo "[Unit]" - fi + rpm2cpio() { + echo "rpm2cpio $*" >> "$ARTIFACT_STREAMING_CALLS" + return "${RPM2CPIO_RC:-0}" + } + cpio() { + echo "cpio $*" >> "$ARTIFACT_STREAMING_CALLS" + [ "${CPIO_RC:-0}" -eq 0 ] || return "$CPIO_RC" + mkdir -p opt/acr/bin usr/lib/systemd/system + : > opt/acr/bin/acr + : > usr/lib/systemd/system/acr-mirror.service + } + cp() { + echo "cp $*" >> "$ARTIFACT_STREAMING_CALLS" + return "${CP_RC:-0}" } install() { - cat > /dev/null echo "install $*" >> "$ARTIFACT_STREAMING_CALLS" + return "${INSTALL_RC:-0}" } env() { echo "env $*" >> "$ARTIFACT_STREAMING_CALLS" @@ -57,9 +66,12 @@ Describe 'installAndConfigureArtifactStreaming' It 'extracts the rpm payload and initializes acr on Azure Container Linux' OS="AZURECONTAINERLINUX" When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "bsdtar -C / -xf ./acr-mirror.rpm opt/" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "bsdtar -Oxf ./acr-mirror.rpm usr/lib/systemd/system/acr-mirror.service" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "install -m 0644 /dev/stdin /etc/systemd/system/acr-mirror.service" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "cpio -idm" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "cp -a" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/opt/. /opt/" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "install -m 0644" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/usr/lib/systemd/system/acr-mirror.service /etc/systemd/system/acr-mirror.service" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "env -C /opt/acr/bin ./acr init --min-init" The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "dnf_install" The status should be success @@ -70,7 +82,8 @@ Describe 'installAndConfigureArtifactStreaming' TEST_ARM64=1 When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_curl_file 10 5 60 ./acr-mirror-arm64.rpm https://example/acr-mirror-arm64.rpm" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "bsdtar -C / -xf ./acr-mirror-arm64.rpm opt/" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/acr-mirror-arm64.rpm" The status should be success End @@ -78,7 +91,8 @@ Describe 'installAndConfigureArtifactStreaming' OS="AZURELINUX" OS_VARIANT="AZURECONTAINERLINUX" When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "bsdtar -C / -xf ./acr-mirror.rpm opt/" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "cpio -idm" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "env -C /opt/acr/bin ./acr init --min-init" The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "dnf_install" The status should be success @@ -86,9 +100,9 @@ Describe 'installAndConfigureArtifactStreaming' It 'fails when the ACL rpm payload cannot be extracted' OS="AZURECONTAINERLINUX" - BSDTAR_RC=127 + RPM2CPIO_RC=127 When run installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "bsdtar -C / -xf ./acr-mirror.rpm opt/" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "acr init --min-init" The status should equal "$ERR_ARTIFACT_STREAMING_DOWNLOAD" End diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index e5bf8a9a2b5..8dc1f088112 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -401,10 +401,23 @@ installAndConfigureArtifactStreaming() { retrycmd_curl_file 10 5 60 "$MIRROR_DOWNLOAD_PATH" "$downloadURL" || exit "${ERR_ARTIFACT_STREAMING_DOWNLOAD}" if isACL "$OS" "$OS_VARIANT"; then - bsdtar -C / -xf "$MIRROR_DOWNLOAD_PATH" opt/ || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" - bsdtar -Oxf "$MIRROR_DOWNLOAD_PATH" usr/lib/systemd/system/acr-mirror.service | - install -m 0644 /dev/stdin /etc/systemd/system/acr-mirror.service || + local extract_dir + local mirror_download_path_absolute + extract_dir="$(mktemp -d)" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + mirror_download_path_absolute="$(readlink -f "$MIRROR_DOWNLOAD_PATH")" + if ! (cd "$extract_dir" && rpm2cpio "$mirror_download_path_absolute" | cpio -idm); then + rm -rf "$extract_dir" exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + fi + cp -a "$extract_dir/opt/." /opt/ || { + rm -rf "$extract_dir" + exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + } + install -m 0644 "$extract_dir/usr/lib/systemd/system/acr-mirror.service" /etc/systemd/system/acr-mirror.service || { + rm -rf "$extract_dir" + exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + } + rm -rf "$extract_dir" env -C /opt/acr/bin ./acr init --min-init || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" else case "$downloadURL" in From 357f9958b0bac68136499ef8ea9bb7b3bf7cd985 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 22 Sep 2026 16:29:48 -0700 Subject: [PATCH 06/20] fix: extract ACL RPM with native zstd Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../packer/install_dependencies_spec.sh | 18 ++++--- vhdbuilder/packer/install-dependencies.sh | 54 ++++++++++++++++++- 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/spec/vhdbuilder/packer/install_dependencies_spec.sh b/spec/vhdbuilder/packer/install_dependencies_spec.sh index e1eb38d344e..8f46318dcdb 100644 --- a/spec/vhdbuilder/packer/install_dependencies_spec.sh +++ b/spec/vhdbuilder/packer/install_dependencies_spec.sh @@ -1,6 +1,7 @@ #!/bin/bash set -o pipefail +eval "$(sed -n '/^extractZstdRPMPayload() {/,/^}/p' vhdbuilder/packer/install-dependencies.sh)" eval "$(sed -n '/^installAndConfigureArtifactStreaming() {/,/^}/p' vhdbuilder/packer/install-dependencies.sh)" Describe 'installAndConfigureArtifactStreaming' @@ -35,9 +36,10 @@ Describe 'installAndConfigureArtifactStreaming' dnf_install() { echo "dnf_install $*" >> "$ARTIFACT_STREAMING_CALLS" } - rpm2cpio() { - echo "rpm2cpio $*" >> "$ARTIFACT_STREAMING_CALLS" - return "${RPM2CPIO_RC:-0}" + python3() { + cat > /dev/null + echo "python3 $*" >> "$ARTIFACT_STREAMING_CALLS" + return "${PYTHON3_RC:-0}" } cpio() { echo "cpio $*" >> "$ARTIFACT_STREAMING_CALLS" @@ -66,7 +68,7 @@ Describe 'installAndConfigureArtifactStreaming' It 'extracts the rpm payload and initializes acr on Azure Container Linux' OS="AZURECONTAINERLINUX" When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "python3 -" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "cpio -idm" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "cp -a" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/opt/. /opt/" @@ -82,7 +84,7 @@ Describe 'installAndConfigureArtifactStreaming' TEST_ARM64=1 When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_curl_file 10 5 60 ./acr-mirror-arm64.rpm https://example/acr-mirror-arm64.rpm" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "python3 -" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/acr-mirror-arm64.rpm" The status should be success End @@ -91,7 +93,7 @@ Describe 'installAndConfigureArtifactStreaming' OS="AZURELINUX" OS_VARIANT="AZURECONTAINERLINUX" When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "python3 -" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "cpio -idm" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "env -C /opt/acr/bin ./acr init --min-init" The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "dnf_install" @@ -100,9 +102,9 @@ Describe 'installAndConfigureArtifactStreaming' It 'fails when the ACL rpm payload cannot be extracted' OS="AZURECONTAINERLINUX" - RPM2CPIO_RC=127 + PYTHON3_RC=1 When run installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "python3 -" The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "acr init --min-init" The status should equal "$ERR_ARTIFACT_STREAMING_DOWNLOAD" End diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 8dc1f088112..475853da89a 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -389,6 +389,58 @@ downloadAndInstallCriTools() { installCriCtlPackage "${version}" } +extractZstdRPMPayload() { + local rpm_path="$1" + python3 - "$rpm_path" <<'PY' +import ctypes +import struct +import sys + +MAX_DECOMPRESSED_SIZE = 512 * 1024 * 1024 +RPM_HEADER_MAGIC = b"\x8e\xad\xe8\x01" +RPM_LEAD_MAGIC = b"\xed\xab\xee\xdb" + +with open(sys.argv[1], "rb") as rpm: + if rpm.read(4) != RPM_LEAD_MAGIC: + raise SystemExit("invalid RPM lead") + rpm.seek(96) + for padded in (True, False): + header = rpm.read(16) + if len(header) != 16 or header[:4] != RPM_HEADER_MAGIC: + raise SystemExit("invalid RPM header") + index_count, data_length = struct.unpack(">II", header[8:16]) + rpm.seek(index_count * 16 + data_length, 1) + if padded: + rpm.seek((-rpm.tell()) % 8, 1) + payload = rpm.read() + +libzstd = ctypes.CDLL("libzstd.so.1") +libzstd.ZSTD_decompressBound.argtypes = [ctypes.c_void_p, ctypes.c_size_t] +libzstd.ZSTD_decompressBound.restype = ctypes.c_ulonglong +libzstd.ZSTD_decompress.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t] +libzstd.ZSTD_decompress.restype = ctypes.c_size_t +libzstd.ZSTD_isError.argtypes = [ctypes.c_size_t] +libzstd.ZSTD_isError.restype = ctypes.c_uint +libzstd.ZSTD_getErrorName.argtypes = [ctypes.c_size_t] +libzstd.ZSTD_getErrorName.restype = ctypes.c_char_p + +source = ctypes.create_string_buffer(payload) +bound = libzstd.ZSTD_decompressBound(source, len(payload)) +if libzstd.ZSTD_isError(bound): + error = libzstd.ZSTD_getErrorName(bound).decode() + raise SystemExit(f"failed to inspect RPM payload: {error}") +if bound == 0 or bound > MAX_DECOMPRESSED_SIZE: + raise SystemExit(f"invalid RPM payload decompression bound: {bound}") + +destination = ctypes.create_string_buffer(bound) +size = libzstd.ZSTD_decompress(destination, bound, source, len(payload)) +if libzstd.ZSTD_isError(size): + error = libzstd.ZSTD_getErrorName(size).decode() + raise SystemExit(f"failed to decompress RPM payload: {error}") +sys.stdout.buffer.write(destination.raw[:size]) +PY +} + installAndConfigureArtifactStreaming() { local downloadURL="$1" local version="$2" @@ -405,7 +457,7 @@ installAndConfigureArtifactStreaming() { local mirror_download_path_absolute extract_dir="$(mktemp -d)" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" mirror_download_path_absolute="$(readlink -f "$MIRROR_DOWNLOAD_PATH")" - if ! (cd "$extract_dir" && rpm2cpio "$mirror_download_path_absolute" | cpio -idm); then + if ! (cd "$extract_dir" && extractZstdRPMPayload "$mirror_download_path_absolute" | cpio -idm); then rm -rf "$extract_dir" exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" fi From 9a0d995db27770ba68532b2051b2631c8b370d42 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 22 Sep 2026 17:02:22 -0700 Subject: [PATCH 07/20] fix: configure ACL artifact streaming at node boot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../pkg/nodeconfigutils/utils.go | 19 +++++++++++++++++-- .../pkg/nodeconfigutils/utils_test.go | 18 +++++++++--------- .../packer/install_dependencies_spec.sh | 17 +++++------------ vhdbuilder/packer/install-dependencies.sh | 1 - 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/aks-node-controller/pkg/nodeconfigutils/utils.go b/aks-node-controller/pkg/nodeconfigutils/utils.go index 6e04247fbb4..be0bbe218f7 100644 --- a/aks-node-controller/pkg/nodeconfigutils/utils.go +++ b/aks-node-controller/pkg/nodeconfigutils/utils.go @@ -74,6 +74,21 @@ fi` } }` + artifactStreamingUnit = `[Unit] +Description=Configure AKS artifact streaming +Requires=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service +After=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service +Before=aks-node-controller.service + +[Service] +Type=oneshot +ExecStart=/opt/acr/tools/mirror/setup.sh aks +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +` + artifactStreamingSystemd = `, "systemd": { "units": [ @@ -95,7 +110,7 @@ fi` { "name": "aks-artifact-streaming.service", "enabled": true, - "contents": "[Unit]\nDescription=Configure AKS artifact streaming\nRequires=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service\nAfter=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service\nBefore=aks-node-controller.service\n\n[Service]\nType=oneshot\nExecStart=/opt/acr/bin/acr-config --enable-containerd azurecr.io\nRemainAfterExit=yes\n\n[Install]\nWantedBy=multi-user.target\n" + "contents": %q } ] }` @@ -165,7 +180,7 @@ func CustomDataFlatcar(cfg *aksnodeconfigv1.Configuration) (string, error) { encodedAksNodeConfigJSON := base64.StdEncoding.EncodeToString(aksNodeConfigJSON) var artifactStreamingConfig string if cfg.GetEnableArtifactStreaming() { - artifactStreamingConfig = artifactStreamingSystemd + artifactStreamingConfig = fmt.Sprintf(artifactStreamingSystemd, artifactStreamingUnit) } customDataYAML := fmt.Sprintf(flatcarTemplate, encodedAksNodeConfigJSON, artifactStreamingConfig) return base64.StdEncoding.EncodeToString([]byte(customDataYAML)), nil diff --git a/aks-node-controller/pkg/nodeconfigutils/utils_test.go b/aks-node-controller/pkg/nodeconfigutils/utils_test.go index c2be19f3087..f8acb64e6a9 100644 --- a/aks-node-controller/pkg/nodeconfigutils/utils_test.go +++ b/aks-node-controller/pkg/nodeconfigutils/utils_test.go @@ -323,17 +323,17 @@ func TestCustomDataFlatcarOmitsArtifactStreamingWhenDisabled(t *testing.T) { func TestCustomDataFlatcarEnablesArtifactStreaming(t *testing.T) { ignition := decodeFlatcarCustomData(t, &aksnodeconfigv1.Configuration{EnableArtifactStreaming: true}) - systemd, ok := ignition["systemd"].(map[string]any) - require.True(t, ok) - units, ok := systemd["units"].([]any) - require.True(t, ok) + systemd, systemdOK := ignition["systemd"].(map[string]any) + require.True(t, systemdOK) + units, unitsOK := systemd["units"].([]any) + require.True(t, unitsOK) unitsByName := make(map[string]map[string]any, len(units)) for _, item := range units { - unit, ok := item.(map[string]any) - require.True(t, ok) - name, ok := unit["name"].(string) - require.True(t, ok) + unit, unitOK := item.(map[string]any) + require.True(t, unitOK) + name, nameOK := unit["name"].(string) + require.True(t, nameOK) unitsByName[name] = unit } @@ -356,7 +356,7 @@ func TestCustomDataFlatcarEnablesArtifactStreaming(t *testing.T) { require.Contains(t, contents, "Requires=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service") require.Contains(t, contents, "After=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service") require.Contains(t, contents, "Before=aks-node-controller.service") - require.Contains(t, contents, "ExecStart=/opt/acr/bin/acr-config --enable-containerd azurecr.io") + require.Contains(t, contents, "ExecStart=/opt/acr/tools/mirror/setup.sh aks") require.Contains(t, contents, "WantedBy=multi-user.target") } diff --git a/spec/vhdbuilder/packer/install_dependencies_spec.sh b/spec/vhdbuilder/packer/install_dependencies_spec.sh index 8f46318dcdb..71317585d65 100644 --- a/spec/vhdbuilder/packer/install_dependencies_spec.sh +++ b/spec/vhdbuilder/packer/install_dependencies_spec.sh @@ -56,16 +56,12 @@ Describe 'installAndConfigureArtifactStreaming' echo "install $*" >> "$ARTIFACT_STREAMING_CALLS" return "${INSTALL_RC:-0}" } - env() { - echo "env $*" >> "$ARTIFACT_STREAMING_CALLS" - return "${ACR_INIT_RC:-0}" - } rm() { echo "rm $*" >> "$ARTIFACT_STREAMING_CALLS" command rm "$@" } - It 'extracts the rpm payload and initializes acr on Azure Container Linux' + It 'extracts the rpm payload on Azure Container Linux' OS="AZURECONTAINERLINUX" When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "python3 -" @@ -74,7 +70,6 @@ Describe 'installAndConfigureArtifactStreaming' The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/opt/. /opt/" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "install -m 0644" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/usr/lib/systemd/system/acr-mirror.service /etc/systemd/system/acr-mirror.service" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "env -C /opt/acr/bin ./acr init --min-init" The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "dnf_install" The status should be success End @@ -95,7 +90,6 @@ Describe 'installAndConfigureArtifactStreaming' When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "python3 -" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "cpio -idm" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "env -C /opt/acr/bin ./acr init --min-init" The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "dnf_install" The status should be success End @@ -105,16 +99,15 @@ Describe 'installAndConfigureArtifactStreaming' PYTHON3_RC=1 When run installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "python3 -" - The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "acr init --min-init" The status should equal "$ERR_ARTIFACT_STREAMING_DOWNLOAD" End - It 'fails when acr initialization fails on ACL' + It 'fails when the acr-mirror service cannot be installed on ACL' OS="AZURECONTAINERLINUX" - ACR_INIT_RC=1 + INSTALL_RC=1 When run installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "env -C /opt/acr/bin ./acr init --min-init" - The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "rm ./acr-mirror.rpm" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "install -m 0644" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/usr/lib/systemd/system/acr-mirror.service /etc/systemd/system/acr-mirror.service" The status should equal "$ERR_ARTIFACT_STREAMING_DOWNLOAD" End End diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 475853da89a..ace2ede6cc4 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -470,7 +470,6 @@ installAndConfigureArtifactStreaming() { exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" } rm -rf "$extract_dir" - env -C /opt/acr/bin ./acr init --min-init || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" else case "$downloadURL" in *.deb) From e7ee95c3b99a3c4a5a29186ac3fa27e69bab3c82 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 22 Sep 2026 19:27:46 -0700 Subject: [PATCH 08/20] fix: use rpm2cpio for ACL acr-mirror Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../packer/install_dependencies_spec.sh | 18 +++---- vhdbuilder/packer/install-dependencies.sh | 54 +------------------ 2 files changed, 9 insertions(+), 63 deletions(-) diff --git a/spec/vhdbuilder/packer/install_dependencies_spec.sh b/spec/vhdbuilder/packer/install_dependencies_spec.sh index 71317585d65..0f48a04a1f1 100644 --- a/spec/vhdbuilder/packer/install_dependencies_spec.sh +++ b/spec/vhdbuilder/packer/install_dependencies_spec.sh @@ -1,7 +1,6 @@ #!/bin/bash set -o pipefail -eval "$(sed -n '/^extractZstdRPMPayload() {/,/^}/p' vhdbuilder/packer/install-dependencies.sh)" eval "$(sed -n '/^installAndConfigureArtifactStreaming() {/,/^}/p' vhdbuilder/packer/install-dependencies.sh)" Describe 'installAndConfigureArtifactStreaming' @@ -36,10 +35,9 @@ Describe 'installAndConfigureArtifactStreaming' dnf_install() { echo "dnf_install $*" >> "$ARTIFACT_STREAMING_CALLS" } - python3() { - cat > /dev/null - echo "python3 $*" >> "$ARTIFACT_STREAMING_CALLS" - return "${PYTHON3_RC:-0}" + rpm2cpio() { + echo "rpm2cpio $*" >> "$ARTIFACT_STREAMING_CALLS" + return "${RPM2CPIO_RC:-0}" } cpio() { echo "cpio $*" >> "$ARTIFACT_STREAMING_CALLS" @@ -64,7 +62,7 @@ Describe 'installAndConfigureArtifactStreaming' It 'extracts the rpm payload on Azure Container Linux' OS="AZURECONTAINERLINUX" When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "python3 -" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "cpio -idm" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "cp -a" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/opt/. /opt/" @@ -79,7 +77,7 @@ Describe 'installAndConfigureArtifactStreaming' TEST_ARM64=1 When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_curl_file 10 5 60 ./acr-mirror-arm64.rpm https://example/acr-mirror-arm64.rpm" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "python3 -" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/acr-mirror-arm64.rpm" The status should be success End @@ -88,7 +86,7 @@ Describe 'installAndConfigureArtifactStreaming' OS="AZURELINUX" OS_VARIANT="AZURECONTAINERLINUX" When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "python3 -" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "cpio -idm" The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "dnf_install" The status should be success @@ -96,9 +94,9 @@ Describe 'installAndConfigureArtifactStreaming' It 'fails when the ACL rpm payload cannot be extracted' OS="AZURECONTAINERLINUX" - PYTHON3_RC=1 + RPM2CPIO_RC=1 When run installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "python3 -" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" The status should equal "$ERR_ARTIFACT_STREAMING_DOWNLOAD" End diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index ace2ede6cc4..608010eda94 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -389,58 +389,6 @@ downloadAndInstallCriTools() { installCriCtlPackage "${version}" } -extractZstdRPMPayload() { - local rpm_path="$1" - python3 - "$rpm_path" <<'PY' -import ctypes -import struct -import sys - -MAX_DECOMPRESSED_SIZE = 512 * 1024 * 1024 -RPM_HEADER_MAGIC = b"\x8e\xad\xe8\x01" -RPM_LEAD_MAGIC = b"\xed\xab\xee\xdb" - -with open(sys.argv[1], "rb") as rpm: - if rpm.read(4) != RPM_LEAD_MAGIC: - raise SystemExit("invalid RPM lead") - rpm.seek(96) - for padded in (True, False): - header = rpm.read(16) - if len(header) != 16 or header[:4] != RPM_HEADER_MAGIC: - raise SystemExit("invalid RPM header") - index_count, data_length = struct.unpack(">II", header[8:16]) - rpm.seek(index_count * 16 + data_length, 1) - if padded: - rpm.seek((-rpm.tell()) % 8, 1) - payload = rpm.read() - -libzstd = ctypes.CDLL("libzstd.so.1") -libzstd.ZSTD_decompressBound.argtypes = [ctypes.c_void_p, ctypes.c_size_t] -libzstd.ZSTD_decompressBound.restype = ctypes.c_ulonglong -libzstd.ZSTD_decompress.argtypes = [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t] -libzstd.ZSTD_decompress.restype = ctypes.c_size_t -libzstd.ZSTD_isError.argtypes = [ctypes.c_size_t] -libzstd.ZSTD_isError.restype = ctypes.c_uint -libzstd.ZSTD_getErrorName.argtypes = [ctypes.c_size_t] -libzstd.ZSTD_getErrorName.restype = ctypes.c_char_p - -source = ctypes.create_string_buffer(payload) -bound = libzstd.ZSTD_decompressBound(source, len(payload)) -if libzstd.ZSTD_isError(bound): - error = libzstd.ZSTD_getErrorName(bound).decode() - raise SystemExit(f"failed to inspect RPM payload: {error}") -if bound == 0 or bound > MAX_DECOMPRESSED_SIZE: - raise SystemExit(f"invalid RPM payload decompression bound: {bound}") - -destination = ctypes.create_string_buffer(bound) -size = libzstd.ZSTD_decompress(destination, bound, source, len(payload)) -if libzstd.ZSTD_isError(size): - error = libzstd.ZSTD_getErrorName(size).decode() - raise SystemExit(f"failed to decompress RPM payload: {error}") -sys.stdout.buffer.write(destination.raw[:size]) -PY -} - installAndConfigureArtifactStreaming() { local downloadURL="$1" local version="$2" @@ -457,7 +405,7 @@ installAndConfigureArtifactStreaming() { local mirror_download_path_absolute extract_dir="$(mktemp -d)" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" mirror_download_path_absolute="$(readlink -f "$MIRROR_DOWNLOAD_PATH")" - if ! (cd "$extract_dir" && extractZstdRPMPayload "$mirror_download_path_absolute" | cpio -idm); then + if ! (cd "$extract_dir" && rpm2cpio "$mirror_download_path_absolute" | cpio -idm); then rm -rf "$extract_dir" exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" fi From 358060c45a669c435ae5c07e95a3ebfcf4416187 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 23 Sep 2026 16:31:45 -0700 Subject: [PATCH 09/20] fix: install ACL acr-mirror with dnf Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../packer/install_dependencies_spec.sh | 53 ++++--------------- vhdbuilder/packer/install-dependencies.sh | 18 +------ 2 files changed, 12 insertions(+), 59 deletions(-) diff --git a/spec/vhdbuilder/packer/install_dependencies_spec.sh b/spec/vhdbuilder/packer/install_dependencies_spec.sh index 0f48a04a1f1..59bc84e918d 100644 --- a/spec/vhdbuilder/packer/install_dependencies_spec.sh +++ b/spec/vhdbuilder/packer/install_dependencies_spec.sh @@ -35,39 +35,19 @@ Describe 'installAndConfigureArtifactStreaming' dnf_install() { echo "dnf_install $*" >> "$ARTIFACT_STREAMING_CALLS" } - rpm2cpio() { - echo "rpm2cpio $*" >> "$ARTIFACT_STREAMING_CALLS" - return "${RPM2CPIO_RC:-0}" - } - cpio() { - echo "cpio $*" >> "$ARTIFACT_STREAMING_CALLS" - [ "${CPIO_RC:-0}" -eq 0 ] || return "$CPIO_RC" - mkdir -p opt/acr/bin usr/lib/systemd/system - : > opt/acr/bin/acr - : > usr/lib/systemd/system/acr-mirror.service - } - cp() { - echo "cp $*" >> "$ARTIFACT_STREAMING_CALLS" - return "${CP_RC:-0}" - } - install() { - echo "install $*" >> "$ARTIFACT_STREAMING_CALLS" - return "${INSTALL_RC:-0}" + retrycmd_if_failure() { + echo "retrycmd_if_failure $*" >> "$ARTIFACT_STREAMING_CALLS" + return "${DNF_RC:-0}" } rm() { echo "rm $*" >> "$ARTIFACT_STREAMING_CALLS" command rm "$@" } - It 'extracts the rpm payload on Azure Container Linux' + It 'installs the rpm with dnf on Azure Container Linux' OS="AZURECONTAINERLINUX" When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "cpio -idm" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "cp -a" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/opt/. /opt/" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "install -m 0644" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/usr/lib/systemd/system/acr-mirror.service /etc/systemd/system/acr-mirror.service" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_if_failure 10 2 120 dnf install -y ./acr-mirror.rpm" The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "dnf_install" The status should be success End @@ -77,35 +57,24 @@ Describe 'installAndConfigureArtifactStreaming' TEST_ARM64=1 When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_curl_file 10 5 60 ./acr-mirror-arm64.rpm https://example/acr-mirror-arm64.rpm" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/acr-mirror-arm64.rpm" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_if_failure 10 2 120 dnf install -y ./acr-mirror-arm64.rpm" The status should be success End - It 'extracts the rpm payload for the Azure Linux ACL variant' + It 'installs the rpm with dnf for the Azure Linux ACL variant' OS="AZURELINUX" OS_VARIANT="AZURECONTAINERLINUX" When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "cpio -idm" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_if_failure 10 2 120 dnf install -y ./acr-mirror.rpm" The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "dnf_install" The status should be success End - It 'fails when the ACL rpm payload cannot be extracted' - OS="AZURECONTAINERLINUX" - RPM2CPIO_RC=1 - When run installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "rpm2cpio" - The status should equal "$ERR_ARTIFACT_STREAMING_DOWNLOAD" - End - - It 'fails when the acr-mirror service cannot be installed on ACL' + It 'fails when dnf cannot install the ACL rpm' OS="AZURECONTAINERLINUX" - INSTALL_RC=1 + DNF_RC=1 When run installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "install -m 0644" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "/usr/lib/systemd/system/acr-mirror.service /etc/systemd/system/acr-mirror.service" + The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_if_failure 10 2 120 dnf install -y ./acr-mirror.rpm" The status should equal "$ERR_ARTIFACT_STREAMING_DOWNLOAD" End End diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 608010eda94..78bc9b43e33 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -401,23 +401,7 @@ installAndConfigureArtifactStreaming() { retrycmd_curl_file 10 5 60 "$MIRROR_DOWNLOAD_PATH" "$downloadURL" || exit "${ERR_ARTIFACT_STREAMING_DOWNLOAD}" if isACL "$OS" "$OS_VARIANT"; then - local extract_dir - local mirror_download_path_absolute - extract_dir="$(mktemp -d)" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" - mirror_download_path_absolute="$(readlink -f "$MIRROR_DOWNLOAD_PATH")" - if ! (cd "$extract_dir" && rpm2cpio "$mirror_download_path_absolute" | cpio -idm); then - rm -rf "$extract_dir" - exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" - fi - cp -a "$extract_dir/opt/." /opt/ || { - rm -rf "$extract_dir" - exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" - } - install -m 0644 "$extract_dir/usr/lib/systemd/system/acr-mirror.service" /etc/systemd/system/acr-mirror.service || { - rm -rf "$extract_dir" - exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" - } - rm -rf "$extract_dir" + retrycmd_if_failure 10 2 120 dnf install -y "$MIRROR_DOWNLOAD_PATH" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" else case "$downloadURL" in *.deb) From ce65176be91ba251d51e96b4044280d3f7fe3cb1 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 28 Oct 2026 11:24:08 -0700 Subject: [PATCH 10/20] fix: use the combined artifact streaming sysext on ACL Cache mcr.microsoft.com/acr/v2/artifact-streaming-sysext instead of installing the ACR RPM or enabling a separate OverlayBD extension. Keep exact DALEC revision and architecture pins for local and network-isolated delivery. Defer activation to nodePrep for both CSE and node-controller. Initialize writable paths and ACR configuration before unmasking consumers, preserve GPU containerd settings when materializing its symlink, and fail provisioning before kubelet starts if setup fails. Keep other OS package paths unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0d5fc628-3611-41e9-b6d6-55b92081296b --- .../pkg/nodeconfigutils/utils.go | 49 +---------- .../pkg/nodeconfigutils/utils_test.go | 57 +++---------- e2e/scenario/scenario.go | 11 +++ parts/common/components.json | 6 +- .../artifacts/acl/cse_install_acl.sh | 24 ++++++ .../linux/cloud-init/artifacts/cse_config.sh | 23 ++++- parts/linux/cloud-init/artifacts/cse_main.sh | 7 +- .../artifacts/cse_config_gpu_spec.sh | 76 +++++++++++++++-- .../artifacts/cse_install_acl_spec.sh | 85 +++++++++++++++++++ .../packer/install_dependencies_spec.sh | 56 ++++++------ vhdbuilder/packer/acl-customdata.json | 2 +- vhdbuilder/packer/acl-customdata.yaml | 8 +- vhdbuilder/packer/install-dependencies.sh | 39 ++++----- 13 files changed, 285 insertions(+), 158 deletions(-) diff --git a/aks-node-controller/pkg/nodeconfigutils/utils.go b/aks-node-controller/pkg/nodeconfigutils/utils.go index be0bbe218f7..6798519f344 100644 --- a/aks-node-controller/pkg/nodeconfigutils/utils.go +++ b/aks-node-controller/pkg/nodeconfigutils/utils.go @@ -64,7 +64,7 @@ if [ -s /sys/class/dmi/id/product_uuid ]; then fi` flatcarTemplate = `{ - "ignition": { "version": "3.4.0" }%[2]s, + "ignition": { "version": "3.4.0" }, "storage": { "files": [{ "path": "/opt/azure/containers/aks-node-controller-config.json", @@ -73,47 +73,6 @@ fi` }] } }` - - artifactStreamingUnit = `[Unit] -Description=Configure AKS artifact streaming -Requires=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service -After=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service -Before=aks-node-controller.service - -[Service] -Type=oneshot -ExecStart=/opt/acr/tools/mirror/setup.sh aks -RemainAfterExit=yes - -[Install] -WantedBy=multi-user.target -` - - artifactStreamingSystemd = `, - "systemd": { - "units": [ - { - "name": "acr-mirror.service", - "enabled": true, - "mask": false - }, - { - "name": "overlaybd-tcmu.service", - "enabled": true, - "mask": false - }, - { - "name": "overlaybd-snapshotter.service", - "enabled": true, - "mask": false - }, - { - "name": "aks-artifact-streaming.service", - "enabled": true, - "contents": %q - } - ] - }` ) // CustomData builds base64-encoded VM custom data for cloud-init. The boothook writes the node @@ -178,11 +137,7 @@ func CustomDataFlatcar(cfg *aksnodeconfigv1.Configuration) (string, error) { } encodedAksNodeConfigJSON := base64.StdEncoding.EncodeToString(aksNodeConfigJSON) - var artifactStreamingConfig string - if cfg.GetEnableArtifactStreaming() { - artifactStreamingConfig = fmt.Sprintf(artifactStreamingSystemd, artifactStreamingUnit) - } - customDataYAML := fmt.Sprintf(flatcarTemplate, encodedAksNodeConfigJSON, artifactStreamingConfig) + customDataYAML := fmt.Sprintf(flatcarTemplate, encodedAksNodeConfigJSON) return base64.StdEncoding.EncodeToString([]byte(customDataYAML)), nil } diff --git a/aks-node-controller/pkg/nodeconfigutils/utils_test.go b/aks-node-controller/pkg/nodeconfigutils/utils_test.go index f8acb64e6a9..edb90212e2a 100644 --- a/aks-node-controller/pkg/nodeconfigutils/utils_test.go +++ b/aks-node-controller/pkg/nodeconfigutils/utils_test.go @@ -313,51 +313,20 @@ func TestCustomDataUsesMultipartBoothookAndCloudConfig(t *testing.T) { require.ErrorIs(t, err, io.EOF) } -func TestCustomDataFlatcarOmitsArtifactStreamingWhenDisabled(t *testing.T) { - ignition := decodeFlatcarCustomData(t, &aksnodeconfigv1.Configuration{}) - - _, found := ignition["systemd"] - require.False(t, found) -} - -func TestCustomDataFlatcarEnablesArtifactStreaming(t *testing.T) { - ignition := decodeFlatcarCustomData(t, &aksnodeconfigv1.Configuration{EnableArtifactStreaming: true}) - - systemd, systemdOK := ignition["systemd"].(map[string]any) - require.True(t, systemdOK) - units, unitsOK := systemd["units"].([]any) - require.True(t, unitsOK) - - unitsByName := make(map[string]map[string]any, len(units)) - for _, item := range units { - unit, unitOK := item.(map[string]any) - require.True(t, unitOK) - name, nameOK := unit["name"].(string) - require.True(t, nameOK) - unitsByName[name] = unit - } - - for _, name := range []string{ - "acr-mirror.service", - "overlaybd-tcmu.service", - "overlaybd-snapshotter.service", - } { - unit, found := unitsByName[name] - require.True(t, found, "expected %s in Ignition systemd units", name) - require.Equal(t, true, unit["enabled"]) - require.Equal(t, false, unit["mask"]) +func TestCustomDataFlatcarDefersArtifactStreamingToProvisioning(t *testing.T) { + for _, enabled := range []bool{false, true} { + cfg := &aksnodeconfigv1.Configuration{EnableArtifactStreaming: enabled} + ignition := decodeFlatcarCustomData(t, cfg) + require.NotContains(t, ignition, "systemd", "Ignition must not start streaming before containerd is configured") + + files := ignition["storage"].(map[string]any)["files"].([]any) + source := files[0].(map[string]any)["contents"].(map[string]any)["source"].(string) + configJSON, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(source, "data:;base64,")) + require.NoError(t, err) + decoded, err := UnmarshalConfigurationV1(configJSON) + require.NoError(t, err) + require.Equal(t, enabled, decoded.GetEnableArtifactStreaming()) } - - setupUnit, found := unitsByName["aks-artifact-streaming.service"] - require.True(t, found) - require.Equal(t, true, setupUnit["enabled"]) - contents, ok := setupUnit["contents"].(string) - require.True(t, ok) - require.Contains(t, contents, "Requires=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service") - require.Contains(t, contents, "After=containerd.service acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service") - require.Contains(t, contents, "Before=aks-node-controller.service") - require.Contains(t, contents, "ExecStart=/opt/acr/tools/mirror/setup.sh aks") - require.Contains(t, contents, "WantedBy=multi-user.target") } // decodeBoothook extracts the decoded cloud-boothook part from the base64-encoded custom data. diff --git a/e2e/scenario/scenario.go b/e2e/scenario/scenario.go index b9b44c6ccd9..a2267fa34e7 100755 --- a/e2e/scenario/scenario.go +++ b/e2e/scenario/scenario.go @@ -73,6 +73,11 @@ var _ = Register(&Scenario{ return errors.Join( ValidateFileHasContent(ctx, s, "/etc/os-release", "ID=azurelinux"), ValidateFileHasContent(ctx, s, "/etc/os-release", "VARIANT_ID=azurecontainerlinux"), + ValidateFileDoesNotExist(ctx, s, "/etc/extensions/artifact-streaming.raw"), + ValidateSystemdUnitIsNotRunning(ctx, s, "artifact-streaming-setup.service"), + ValidateSystemdUnitIsNotRunning(ctx, s, "acr-mirror.service"), + ValidateSystemdUnitIsNotRunning(ctx, s, "overlaybd-tcmu.service"), + ValidateSystemdUnitIsNotRunning(ctx, s, "overlaybd-snapshotter.service"), ) }, }, @@ -99,7 +104,13 @@ var _ = Register(&Scenario{ return errors.Join( ValidateFileHasContent(ctx, s, "/etc/os-release", "ID=azurelinux"), ValidateFileHasContent(ctx, s, "/etc/os-release", "VARIANT_ID=azurecontainerlinux"), + ValidateFileExists(ctx, s, "/etc/extensions/artifact-streaming.raw"), + ValidateFileDoesNotExist(ctx, s, "/etc/extensions/overlaybd.raw"), + ValidateFileIsRegularFile(ctx, s, "/etc/containerd/config.toml"), + ValidateFileExists(ctx, s, "/etc/acr/acr.toml"), ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd"), + ValidateFileExists(ctx, s, "/etc/overlaybd-snapshotter/config.json"), + ValidateSystemdUnitIsRunning(ctx, s, "artifact-streaming-setup.service"), ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service"), ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service"), ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service"), diff --git a/parts/common/components.json b/parts/common/components.json index 2428faf140b..68c20202c90 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -2531,11 +2531,11 @@ "current": { "versionsV2": [ { - "renovateTag": "", - "latestVersion": "1.0.0" + "renovateTag": "OCI_registry=https://mcr.microsoft.com, name=acr/v2/artifact-streaming-sysext", + "latestVersion": "v1.0.0-2-azlinux3" } ], - "downloadURL": "https://acrstreamingpackage.z5.web.core.windows.net/${version}/acr-mirror-azurelinux3.rpm" + "downloadURL": "mcr.microsoft.com/acr/v2/artifact-streaming-sysext:${version}-${SYSTEMD_ARCH}" } } } diff --git a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh index f3db5805620..48a4b9c6d50 100644 --- a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh +++ b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh @@ -22,6 +22,13 @@ downloadSysextFromVersion() { matchLocalSysext() { local seName=$1 desiredVer=$2 seArch=$3 local downloadDir="/opt/${seName}/downloads" + # A fully pinned DALEC version must not select a different revision. + case "${desiredVer}" in + *-azlinux3) + find "${downloadDir}" -maxdepth 2 -name "${seName}-v${desiredVer}-${seArch}.raw" -type f 2>/dev/null | head -n1 + return + ;; + esac # Try arch-specific versioned filename first (kubelet-style: name-vVER.X-arch.raw) local match match=$(find "${downloadDir}" -maxdepth 2 -name "${seName}-v${desiredVer}*-${seArch}.raw" -type f 2>/dev/null | sort -V | tail -n1) @@ -38,6 +45,13 @@ matchLocalSysext() { matchRemoteSysext() { local seURL=$1 desiredVer=$2 seArch=$3 + # Pinned tags also work with isolated registries that cannot enumerate tags. + case "${desiredVer}" in + *-azlinux3) + echo "v${desiredVer}-${seArch}" + return 0 + ;; + esac if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then # For network isolated cluster, acr cache rule does not support oras repo tags. # return fixed renovateTag '-1' as workaround @@ -127,6 +141,16 @@ installCredentialProviderPackageFromBootstrapProfileRegistry() { installCredentialProviderFromPkg "$2" "$1" } +installArtifactStreamingSysext() { + local version + version=$(jq -er '.Packages[] | select(.name == "acr-mirror") | + .downloadURIs.azurecontainerlinux.current.versionsV2[0].latestVersion | + select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+-[0-9]+-azlinux3$"))' \ + "${COMPONENTS_FILEPATH:-/opt/azure/components.json}") || return "$ERR_ORAS_PULL_SYSEXT_FAIL" + local registry="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER:-${MCR_REPOSITORY_BASE:-mcr.microsoft.com}}" + mergeSysexts artifact-streaming "${registry%/}/acr/v2/artifact-streaming-sysext" "${version#v}" +} + # Only called at build-time, unlike kubelet or credential provider installation. installSecureTLSBootstrapClientSysext() { local version=$1 diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 419fcca12a4..afcc2ac7e81 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -635,12 +635,29 @@ EOF } ensureArtifactStreaming() { - waitForContainerdReady || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + local acr_mirror_setup="${ACR_MIRROR_SETUP_SCRIPT:-/opt/acr/tools/mirror/setup.sh}" if isACL "$OS" "$OS_VARIANT"; then - systemctl unmask overlaybd-tcmu.service overlaybd-snapshotter.service || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + local services=(acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service) + # DALEC's Upholds= starts unmasked units as soon as the manager reloads. + systemctl mask "${services[@]}" artifact-streaming-setup.service || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + installArtifactStreamingSysext || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + systemctl daemon-reload || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + systemctl unmask artifact-streaming-setup.service || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + systemctl start artifact-streaming-setup.service || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + # acr-config rejects ACL's absolute symlink, including a preserved GPU config. + if test -L /etc/containerd/config.toml; then + cp --remove-destination "$(readlink -f /etc/containerd/config.toml)" /etc/containerd/config.toml || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + fi + "$acr_mirror_setup" aks || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + systemctl unmask "${services[@]}" || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + systemctl enable "${services[@]}" || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + retrycmd_if_failure 120 5 25 systemctl restart "${services[@]}" || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + systemctl restart containerd || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + waitForContainerdReady || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + return fi + waitForContainerdReady || exit "$ERR_ARTIFACT_STREAMING_INSTALL" - local acr_mirror_setup="${ACR_MIRROR_SETUP_SCRIPT:-/opt/acr/tools/mirror/setup.sh}" if [ -x "$acr_mirror_setup" ]; then "$acr_mirror_setup" aks || exit $ERR_ARTIFACT_STREAMING_INSTALL else diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index 25e750f832f..b90b13dc51f 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -398,7 +398,7 @@ EOF fi fi - if [ "${ARTIFACT_STREAMING_ENABLED}" = "true" ]; then + if [ "${ARTIFACT_STREAMING_ENABLED}" = "true" ] && ! isACL "$OS" "$OS_VARIANT"; then logs_to_events "AKS.CSE.ensureContainerd.ensureArtifactStreaming" ensureArtifactStreaming || exit $ERR_ARTIFACT_STREAMING_INSTALL fi @@ -623,6 +623,11 @@ function nodePrep { addKubeletNodeLabel "kubernetes.azure.com/localdns-exporter=enabled" fi + # Keep ACL sysexts masked during PIS image preparation; configure them on the real node. + if [ "${ARTIFACT_STREAMING_ENABLED}" = "true" ] && isACL "$OS" "$OS_VARIANT"; then + logs_to_events "AKS.CSE.ensureContainerd.ensureArtifactStreaming" ensureArtifactStreaming || exit $ERR_ARTIFACT_STREAMING_INSTALL + fi + logs_to_events "AKS.CSE.ensureKubelet" ensureKubelet # Configure localdns metrics exporter socket after ensureKubelet. diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh index 17d72ad96de..32c923e95db 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh @@ -153,8 +153,15 @@ Describe 'cse_config_gpu.sh' return "${CONTAINERD_READY_RC:-0}" } systemctl() { + if [ "${FAIL_SYSTEMCTL:-}" = "$*" ]; then + return 1 + fi echo "systemctl $@" } + installArtifactStreamingSysext() { + echo "installArtifactStreamingSysext" + return "${SYSEXT_RC:-0}" + } retrycmd_if_failure() { echo "retrycmd_if_failure $@" return "${RETRYCMD_RC:-0}" @@ -187,23 +194,78 @@ Describe 'cse_config_gpu.sh' The status should be success End - It 'unmasks the OverlayBD services on Azure Container Linux' + Describe 'initializes the ACL sysext before unmasking consumers' + Parameters + "AZURECONTAINERLINUX" "" + "AZURELINUX" "AZURECONTAINERLINUX" + End + It "orders initialization on $1 $2" + OS="$1" + OS_VARIANT="$2" + install_setup_sh + When run ensureArtifactStreaming + The output should equal "$(cat <<'EXPECTED' +systemctl mask acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service artifact-streaming-setup.service +installArtifactStreamingSysext +systemctl daemon-reload +systemctl unmask artifact-streaming-setup.service +systemctl start artifact-streaming-setup.service +setup.sh aks +systemctl unmask acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service +systemctl enable acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service +retrycmd_if_failure 120 5 25 systemctl restart acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service +systemctl restart containerd +EXPECTED +)" + The status should be success + End + End + + It 'materializes a preserved ACL containerd config before configuring ACR' OS="$ACL_OS_NAME" install_setup_sh + test() { + if [ "$*" = "-L /etc/containerd/config.toml" ]; then + return 0 + fi + builtin test "$@" + } + readlink() { echo "/usr/share/containerd/config.toml"; } + cp() { echo "cp $*"; } When run ensureArtifactStreaming - The output should include "systemctl unmask overlaybd-tcmu.service overlaybd-snapshotter.service" + The output should include "cp --remove-destination /usr/share/containerd/config.toml /etc/containerd/config.toml" The output should include "setup.sh aks" The status should be success End - It 'unmasks the OverlayBD services for the Azure Linux ACL variant' - OS="$AZURELINUX_OS_NAME" - OS_VARIANT="$ACL_OS_VARIANT" + It 'leaves consumers masked when sysext initialization fails' + OS="$ACL_OS_NAME" + FAIL_SYSTEMCTL="start artifact-streaming-setup.service" + install_setup_sh + When run ensureArtifactStreaming + The output should not include "setup.sh aks" + The output should not include "systemctl unmask acr-mirror.service" + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + End + + It 'leaves consumers masked when ACR configuration fails' + OS="$ACL_OS_NAME" + export SETUP_RC=1 install_setup_sh When run ensureArtifactStreaming - The output should include "systemctl unmask overlaybd-tcmu.service overlaybd-snapshotter.service" The output should include "setup.sh aks" - The status should be success + The output should not include "systemctl unmask acr-mirror.service" + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + End + + It 'fails before initialization if the sysext cannot be merged' + OS="$ACL_OS_NAME" + SYSEXT_RC=1 + install_setup_sh + When run ensureArtifactStreaming + The output should not include "systemctl daemon-reload" + The output should not include "setup.sh aks" + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" End It 'fails fast when enabling the streaming services fails' diff --git a/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh index 76770aeb04c..6d58644968a 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh @@ -44,6 +44,91 @@ Describe 'cse_install_acl.sh' Include "./parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh" Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" + Describe 'pinned DALEC sysext versions' + Describe 'remote resolution' + Parameters + "" "x86-64" + "" "arm64" + "cache.azurecr.io/aks-managed-repository" "x86-64" + "cache.azurecr.io/aks-managed-repository" "arm64" + End + It "retains revision 2 for $2 with registry '$1' without listing tags" + BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="$1" + retrycmd_silent() { echo "unexpected tag listing" >&2; return 1; } + When call matchRemoteSysext "mcr.microsoft.com/acr/v2/artifact-streaming-sysext" "1.0.0-2-azlinux3" "$2" + The output should equal "v1.0.0-2-azlinux3-$2" + The status should be success + End + End + + Describe 'local resolution' + setup_cache() { + SYSEXT_CACHE=$(mktemp -d) + touch "$SYSEXT_CACHE/artifact-streaming-v1.0.0-2-azlinux3-x86-64.raw" + touch "$SYSEXT_CACHE/artifact-streaming-v1.0.0-20-azlinux3-x86-64.raw" + touch "$SYSEXT_CACHE/artifact-streaming.raw" + } + cleanup_cache() { + rm -f "$SYSEXT_CACHE/artifact-streaming-v1.0.0-2-azlinux3-x86-64.raw" \ + "$SYSEXT_CACHE/artifact-streaming-v1.0.0-20-azlinux3-x86-64.raw" \ + "$SYSEXT_CACHE/artifact-streaming.raw" + rmdir "$SYSEXT_CACHE" + } + BeforeEach 'setup_cache' + AfterEach 'cleanup_cache' + find() { + shift + command find "$SYSEXT_CACHE" "$@" + } + + It 'selects the exact cached revision rather than a later prefix match' + When call matchLocalSysext artifact-streaming "1.0.0-2-azlinux3" x86-64 + The output should equal "$SYSEXT_CACHE/artifact-streaming-v1.0.0-2-azlinux3-x86-64.raw" + The status should be success + End + + It 'does not use an unversioned or different-revision image for a missing pin' + When call matchLocalSysext artifact-streaming "1.0.0-3-azlinux3" x86-64 + The output should equal "" + The status should be success + End + End + End + + Describe 'installArtifactStreamingSysext' + COMPONENTS_FILEPATH="parts/common/components.json" + mergeSysexts() { echo "mergeSysexts $*"; return "${MERGE_RC:-0}"; } + Describe 'registry selection' + Parameters + "" "" "mcr.microsoft.com" + "" "mcr.azure.cn/" "mcr.azure.cn" + "cache.azurecr.io/aks-managed-repository" "mcr.microsoft.com" "cache.azurecr.io/aks-managed-repository" + End + It "uses $3 and the version pinned in the VHD" + BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="$1" + MCR_REPOSITORY_BASE="$2" + version=$(jq -r '.Packages[] | select(.name == "acr-mirror") | .downloadURIs.azurecontainerlinux.current.versionsV2[0].latestVersion' "$COMPONENTS_FILEPATH") + When call installArtifactStreamingSysext + The output should equal "mergeSysexts artifact-streaming $3/acr/v2/artifact-streaming-sysext ${version#v}" + The status should be success + End + End + + It 'fails when the VHD does not declare a sysext version' + COMPONENTS_FILEPATH="spec/parts/linux/cloud-init/artifacts/test_components.json" + When call installArtifactStreamingSysext + The output should equal "" + The status should equal "$ERR_ORAS_PULL_SYSEXT_FAIL" + End + + It 'propagates activation failure' + MERGE_RC=1 + When call installArtifactStreamingSysext + The output should include "mergeSysexts artifact-streaming" + The status should be failure + End + End + Describe 'installKubeletKubectlFromPkg' It 'masks the kubelet sysext Upholds drop-in before activating the sysext' MASK_CREATED=false diff --git a/spec/vhdbuilder/packer/install_dependencies_spec.sh b/spec/vhdbuilder/packer/install_dependencies_spec.sh index 59bc84e918d..d5bf32a214c 100644 --- a/spec/vhdbuilder/packer/install_dependencies_spec.sh +++ b/spec/vhdbuilder/packer/install_dependencies_spec.sh @@ -35,46 +35,40 @@ Describe 'installAndConfigureArtifactStreaming' dnf_install() { echo "dnf_install $*" >> "$ARTIFACT_STREAMING_CALLS" } - retrycmd_if_failure() { - echo "retrycmd_if_failure $*" >> "$ARTIFACT_STREAMING_CALLS" - return "${DNF_RC:-0}" + downloadSysextFromVersion() { + echo "downloadSysextFromVersion $*" >> "$ARTIFACT_STREAMING_CALLS" + return "${DOWNLOAD_RC:-0}" } rm() { echo "rm $*" >> "$ARTIFACT_STREAMING_CALLS" command rm "$@" } - It 'installs the rpm with dnf on Azure Container Linux' - OS="AZURECONTAINERLINUX" - When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_if_failure 10 2 120 dnf install -y ./acr-mirror.rpm" - The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "dnf_install" - The status should be success - End - - It 'uses the arm64 package name on Azure Container Linux ARM64' - OS="AZURECONTAINERLINUX" - TEST_ARM64=1 - When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_curl_file 10 5 60 ./acr-mirror-arm64.rpm https://example/acr-mirror-arm64.rpm" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_if_failure 10 2 120 dnf install -y ./acr-mirror-arm64.rpm" - The status should be success - End - - It 'installs the rpm with dnf for the Azure Linux ACL variant' - OS="AZURELINUX" - OS_VARIANT="AZURECONTAINERLINUX" - When call installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_if_failure 10 2 120 dnf install -y ./acr-mirror.rpm" - The contents of file "$ARTIFACT_STREAMING_CALLS" should not include "dnf_install" - The status should be success + Describe 'caches the combined sysext without installing RPMs or rewriting OCI tags' + Parameters + "AZURECONTAINERLINUX" "" "x86-64" 0 + "AZURECONTAINERLINUX" "" "arm64" 1 + "AZURELINUX" "AZURECONTAINERLINUX" "x86-64" 0 + "AZURELINUX" "AZURECONTAINERLINUX" "arm64" 1 + End + It "caches the $3 artifact for $1 $2" + OS="$1" + OS_VARIANT="$2" + TEST_ARM64="$4" + version=$(jq -r '.Packages[] | select(.name == "acr-mirror") | .downloadURIs.azurecontainerlinux.current.versionsV2[0].latestVersion' parts/common/components.json) + url=$(jq -r '.Packages[] | select(.name == "acr-mirror") | .downloadURIs.azurecontainerlinux.current.downloadURL' parts/common/components.json) + url=${url//\$\{version\}/$version} + url=${url//\$\{SYSTEMD_ARCH\}/$3} + When call installAndConfigureArtifactStreaming "$url" "$version" + The contents of file "$ARTIFACT_STREAMING_CALLS" should equal "downloadSysextFromVersion artifact-streaming mcr.microsoft.com/acr/v2/artifact-streaming-sysext:${version}-$3" + The status should be success + End End - It 'fails when dnf cannot install the ACL rpm' + It 'fails when the combined sysext cannot be cached' OS="AZURECONTAINERLINUX" - DNF_RC=1 - When run installAndConfigureArtifactStreaming "https://example/acr-mirror.rpm" "1.0.0" - The contents of file "$ARTIFACT_STREAMING_CALLS" should include "retrycmd_if_failure 10 2 120 dnf install -y ./acr-mirror.rpm" + DOWNLOAD_RC=1 + When run installAndConfigureArtifactStreaming "mcr.microsoft.com/acr/v2/artifact-streaming-sysext:v1.0.0-2-azlinux3-x86-64" "v1.0.0-2-azlinux3" The status should equal "$ERR_ARTIFACT_STREAMING_DOWNLOAD" End End diff --git a/vhdbuilder/packer/acl-customdata.json b/vhdbuilder/packer/acl-customdata.json index 3e7ba8a237c..05ddcb19e6e 100644 --- a/vhdbuilder/packer/acl-customdata.json +++ b/vhdbuilder/packer/acl-customdata.json @@ -1 +1 @@ -{"ignition":{"version":"3.4.0"},"kernelArguments":{"shouldNotExist":["flatcar.autologin"]},"storage":{"directories":[{"path":"/opt/aks-sysext-rw/usr/local/bin"}],"files":[{"overwrite":true,"path":"/opt/aks-sysext/usr/lib/extension-release.d/extension-release.aks-sysext","contents":{"compression":"","source":"data:,ID%3Dazurelinux%0ASYSEXT_LEVEL%3D1.0%0A"}},{"path":"/etc/flatcar/enabled-sysext.conf","contents":{"compression":"","source":"data:,overlaybd%0A"}},{"overwrite":true,"path":"/etc/flatcar/update.conf","contents":{"compression":"","source":"data:,SERVER%3Ddisabled%0A"}},{"overwrite":true,"path":"/etc/systemd/system/containerd.service.d/50-default-config.conf","contents":{"compression":"","source":"data:,%5BService%5D%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0A"}}],"links":[{"path":"/etc/extensions/aks-sysext","hard":false,"target":"/opt/aks-sysext"},{"path":"/opt/aks-sysext/usr/local/bin","hard":false,"target":"/opt/aks-sysext-rw/usr/local/bin"},{"path":"/etc/containerd/config.toml","hard":false,"target":"/usr/share/containerd/config.toml"}]},"systemd":{"units":[{"mask":true,"name":"overlaybd-tcmu.service"},{"mask":true,"name":"overlaybd-snapshotter.service"}]}} +{"ignition":{"version":"3.4.0"},"kernelArguments":{"shouldNotExist":["flatcar.autologin"]},"storage":{"directories":[{"path":"/opt/aks-sysext-rw/usr/local/bin"}],"files":[{"overwrite":true,"path":"/opt/aks-sysext/usr/lib/extension-release.d/extension-release.aks-sysext","contents":{"compression":"","source":"data:,ID%3Dazurelinux%0ASYSEXT_LEVEL%3D1.0%0A"}},{"overwrite":true,"path":"/etc/flatcar/enabled-sysext.conf","contents":{"compression":"","source":"data:,"}},{"overwrite":true,"path":"/etc/flatcar/update.conf","contents":{"compression":"","source":"data:,SERVER%3Ddisabled%0A"}},{"overwrite":true,"path":"/etc/systemd/system/containerd.service.d/50-default-config.conf","contents":{"compression":"","source":"data:,%5BService%5D%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0A"}}],"links":[{"path":"/etc/extensions/aks-sysext","hard":false,"target":"/opt/aks-sysext"},{"path":"/opt/aks-sysext/usr/local/bin","hard":false,"target":"/opt/aks-sysext-rw/usr/local/bin"},{"path":"/etc/containerd/config.toml","hard":false,"target":"/usr/share/containerd/config.toml"}]},"systemd":{"units":[{"mask":true,"name":"artifact-streaming-setup.service"},{"mask":true,"name":"acr-mirror.service"},{"mask":true,"name":"overlaybd-tcmu.service"},{"mask":true,"name":"overlaybd-snapshotter.service"}]}} diff --git a/vhdbuilder/packer/acl-customdata.yaml b/vhdbuilder/packer/acl-customdata.yaml index 4dbf1618916..f560c47fc9e 100644 --- a/vhdbuilder/packer/acl-customdata.yaml +++ b/vhdbuilder/packer/acl-customdata.yaml @@ -14,9 +14,9 @@ storage: ID=azurelinux SYSEXT_LEVEL=1.0 - path: /etc/flatcar/enabled-sysext.conf + overwrite: true contents: - inline: | - overlaybd + inline: "" - path: /etc/flatcar/update.conf overwrite: true contents: @@ -48,6 +48,10 @@ storage: # The Flatcar customdata has a dropin for update-ca-certificates.service that is omitted here. systemd: units: + - name: artifact-streaming-setup.service + mask: true + - name: acr-mirror.service + mask: true - name: overlaybd-tcmu.service mask: true - name: overlaybd-snapshotter.service diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 78bc9b43e33..c40ea912129 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -392,6 +392,13 @@ downloadAndInstallCriTools() { installAndConfigureArtifactStreaming() { local downloadURL="$1" local version="$2" + if isACL "$OS" "$OS_VARIANT"; then + # Cache only: activation is feature-gated during node provisioning. + downloadSysextFromVersion artifact-streaming "$downloadURL" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + echo " - artifact-streaming sysext version ${version}" >> "${VHD_LOGS_FILEPATH}" + return + fi + # The arm64 packages have "-arm64" inserted before the file extension, # e.g. acr-mirror-2204-arm64.deb instead of acr-mirror-2204.deb if [ "$(isARM64)" -eq 1 ]; then @@ -400,27 +407,21 @@ installAndConfigureArtifactStreaming() { local MIRROR_DOWNLOAD_PATH="./$(basename "${downloadURL}")" retrycmd_curl_file 10 5 60 "$MIRROR_DOWNLOAD_PATH" "$downloadURL" || exit "${ERR_ARTIFACT_STREAMING_DOWNLOAD}" - if isACL "$OS" "$OS_VARIANT"; then - retrycmd_if_failure 10 2 120 dnf install -y "$MIRROR_DOWNLOAD_PATH" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" - else - case "$downloadURL" in - *.deb) - apt_get_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" - ;; - *.rpm) - dnf_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" - ;; - *) - echo "Unsupported acr-mirror package extension in URL: ${downloadURL}" >&2 - exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" - ;; - esac - fi + case "$downloadURL" in + *.deb) + apt_get_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + ;; + *.rpm) + dnf_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + ;; + *) + echo "Unsupported acr-mirror package extension in URL: ${downloadURL}" >&2 + exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + ;; + esac rm "$MIRROR_DOWNLOAD_PATH" - if ! isACL "$OS" "$OS_VARIANT"; then - /opt/acr/tools/overlaybd/install.sh - fi + /opt/acr/tools/overlaybd/install.sh # Remove the bundled OverlayBD installer packages (~55-58 MB); they're unused at runtime. rm -f /opt/acr/tools/overlaybd/bin/*.deb /opt/acr/tools/overlaybd/bin/*.rpm echo " - acr-mirror version ${version}" >> ${VHD_LOGS_FILEPATH} From eb4345d604383825639ed18b56e8a110d02e781e Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 28 Oct 2026 13:17:02 -0700 Subject: [PATCH 11/20] fix: check Ignition type assertions in artifact streaming test Check each decoded JSON type assertion and guard the file list before indexing it, satisfying errcheck without suppressing lint findings. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0d5fc628-3611-41e9-b6d6-55b92081296b --- .../pkg/nodeconfigutils/utils_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/aks-node-controller/pkg/nodeconfigutils/utils_test.go b/aks-node-controller/pkg/nodeconfigutils/utils_test.go index edb90212e2a..2653c3bac71 100644 --- a/aks-node-controller/pkg/nodeconfigutils/utils_test.go +++ b/aks-node-controller/pkg/nodeconfigutils/utils_test.go @@ -319,8 +319,17 @@ func TestCustomDataFlatcarDefersArtifactStreamingToProvisioning(t *testing.T) { ignition := decodeFlatcarCustomData(t, cfg) require.NotContains(t, ignition, "systemd", "Ignition must not start streaming before containerd is configured") - files := ignition["storage"].(map[string]any)["files"].([]any) - source := files[0].(map[string]any)["contents"].(map[string]any)["source"].(string) + storage, ok := ignition["storage"].(map[string]any) + require.True(t, ok) + files, ok := storage["files"].([]any) + require.True(t, ok) + require.NotEmpty(t, files) + file, ok := files[0].(map[string]any) + require.True(t, ok) + contents, ok := file["contents"].(map[string]any) + require.True(t, ok) + source, ok := contents["source"].(string) + require.True(t, ok) configJSON, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(source, "data:;base64,")) require.NoError(t, err) decoded, err := UnmarshalConfigurationV1(configJSON) From 05ae0648aa8784aa73bb8799d80db548f1b075bd Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 28 Oct 2026 13:22:04 -0700 Subject: [PATCH 12/20] fix: update artifact streaming sysext registry namespace Use azurecontainerregistry/v2/artifact-streaming-sysext for VHD caching, Renovate metadata, and node provisioning, retaining the pinned version and registry overrides. Update the matching ShellSpec expectations. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0d5fc628-3611-41e9-b6d6-55b92081296b --- parts/common/components.json | 4 ++-- parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh | 2 +- spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh | 4 ++-- spec/vhdbuilder/packer/install_dependencies_spec.sh | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/parts/common/components.json b/parts/common/components.json index 68c20202c90..441f729eda1 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -2531,11 +2531,11 @@ "current": { "versionsV2": [ { - "renovateTag": "OCI_registry=https://mcr.microsoft.com, name=acr/v2/artifact-streaming-sysext", + "renovateTag": "OCI_registry=https://mcr.microsoft.com, name=azurecontainerregistry/v2/artifact-streaming-sysext", "latestVersion": "v1.0.0-2-azlinux3" } ], - "downloadURL": "mcr.microsoft.com/acr/v2/artifact-streaming-sysext:${version}-${SYSTEMD_ARCH}" + "downloadURL": "mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext:${version}-${SYSTEMD_ARCH}" } } } diff --git a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh index 48a4b9c6d50..10619e4a95f 100644 --- a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh +++ b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh @@ -148,7 +148,7 @@ installArtifactStreamingSysext() { select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+-[0-9]+-azlinux3$"))' \ "${COMPONENTS_FILEPATH:-/opt/azure/components.json}") || return "$ERR_ORAS_PULL_SYSEXT_FAIL" local registry="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER:-${MCR_REPOSITORY_BASE:-mcr.microsoft.com}}" - mergeSysexts artifact-streaming "${registry%/}/acr/v2/artifact-streaming-sysext" "${version#v}" + mergeSysexts artifact-streaming "${registry%/}/azurecontainerregistry/v2/artifact-streaming-sysext" "${version#v}" } # Only called at build-time, unlike kubelet or credential provider installation. diff --git a/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh index 6d58644968a..6e8cedfa464 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh @@ -55,7 +55,7 @@ Describe 'cse_install_acl.sh' It "retains revision 2 for $2 with registry '$1' without listing tags" BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="$1" retrycmd_silent() { echo "unexpected tag listing" >&2; return 1; } - When call matchRemoteSysext "mcr.microsoft.com/acr/v2/artifact-streaming-sysext" "1.0.0-2-azlinux3" "$2" + When call matchRemoteSysext "mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext" "1.0.0-2-azlinux3" "$2" The output should equal "v1.0.0-2-azlinux3-$2" The status should be success End @@ -109,7 +109,7 @@ Describe 'cse_install_acl.sh' MCR_REPOSITORY_BASE="$2" version=$(jq -r '.Packages[] | select(.name == "acr-mirror") | .downloadURIs.azurecontainerlinux.current.versionsV2[0].latestVersion' "$COMPONENTS_FILEPATH") When call installArtifactStreamingSysext - The output should equal "mergeSysexts artifact-streaming $3/acr/v2/artifact-streaming-sysext ${version#v}" + The output should equal "mergeSysexts artifact-streaming $3/azurecontainerregistry/v2/artifact-streaming-sysext ${version#v}" The status should be success End End diff --git a/spec/vhdbuilder/packer/install_dependencies_spec.sh b/spec/vhdbuilder/packer/install_dependencies_spec.sh index d5bf32a214c..a6d03e7226f 100644 --- a/spec/vhdbuilder/packer/install_dependencies_spec.sh +++ b/spec/vhdbuilder/packer/install_dependencies_spec.sh @@ -60,7 +60,7 @@ Describe 'installAndConfigureArtifactStreaming' url=${url//\$\{version\}/$version} url=${url//\$\{SYSTEMD_ARCH\}/$3} When call installAndConfigureArtifactStreaming "$url" "$version" - The contents of file "$ARTIFACT_STREAMING_CALLS" should equal "downloadSysextFromVersion artifact-streaming mcr.microsoft.com/acr/v2/artifact-streaming-sysext:${version}-$3" + The contents of file "$ARTIFACT_STREAMING_CALLS" should equal "downloadSysextFromVersion artifact-streaming mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext:${version}-$3" The status should be success End End @@ -68,7 +68,7 @@ Describe 'installAndConfigureArtifactStreaming' It 'fails when the combined sysext cannot be cached' OS="AZURECONTAINERLINUX" DOWNLOAD_RC=1 - When run installAndConfigureArtifactStreaming "mcr.microsoft.com/acr/v2/artifact-streaming-sysext:v1.0.0-2-azlinux3-x86-64" "v1.0.0-2-azlinux3" + When run installAndConfigureArtifactStreaming "mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext:v1.0.0-2-azlinux3-x86-64" "v1.0.0-2-azlinux3" The status should equal "$ERR_ARTIFACT_STREAMING_DOWNLOAD" End End From 9872a5fbce2ada51bde39d66bebf61f5077a896f Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 28 Oct 2026 13:28:35 -0700 Subject: [PATCH 13/20] docs: explain ACL artifact streaming deferral in basePrep Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 0d5fc628-3611-41e9-b6d6-55b92081296b --- parts/linux/cloud-init/artifacts/cse_main.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index b90b13dc51f..110fa21526c 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -398,6 +398,8 @@ EOF fi fi + # Defer ACL artifact streaming to nodePrep: keep sysext services masked + # during PIS image preparation and activate them on the real node. if [ "${ARTIFACT_STREAMING_ENABLED}" = "true" ] && ! isACL "$OS" "$OS_VARIANT"; then logs_to_events "AKS.CSE.ensureContainerd.ensureArtifactStreaming" ensureArtifactStreaming || exit $ERR_ARTIFACT_STREAMING_INSTALL fi From dc286d4cd901ddb1a459731f54d516c3642ed212 Mon Sep 17 00:00:00 2001 From: rguptar <69279773+rguptar@users.noreply.github.com> Date: Wed, 28 Oct 2026 14:18:49 -0700 Subject: [PATCH 14/20] Align ACL artifact-streaming pin with initial DALEC revision Use v1.0.0-1-azlinux3 and update the exact-revision resolution and VHD caching tests to match the initial component publication. Validated: 25 focused ShellSpec examples, 0 failures; JSON pin, shell syntax, and whitespace checks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: da9d0569-0965-44eb-9e2b-1d2ac995e638 --- parts/common/components.json | 2 +- .../cloud-init/artifacts/cse_install_acl_spec.sh | 14 +++++++------- .../vhdbuilder/packer/install_dependencies_spec.sh | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/parts/common/components.json b/parts/common/components.json index 441f729eda1..6c68d8eb5b6 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -2532,7 +2532,7 @@ "versionsV2": [ { "renovateTag": "OCI_registry=https://mcr.microsoft.com, name=azurecontainerregistry/v2/artifact-streaming-sysext", - "latestVersion": "v1.0.0-2-azlinux3" + "latestVersion": "v1.0.0-1-azlinux3" } ], "downloadURL": "mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext:${version}-${SYSTEMD_ARCH}" diff --git a/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh index 6e8cedfa464..6c705178b5e 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh @@ -52,11 +52,11 @@ Describe 'cse_install_acl.sh' "cache.azurecr.io/aks-managed-repository" "x86-64" "cache.azurecr.io/aks-managed-repository" "arm64" End - It "retains revision 2 for $2 with registry '$1' without listing tags" + It "retains revision 1 for $2 with registry '$1' without listing tags" BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="$1" retrycmd_silent() { echo "unexpected tag listing" >&2; return 1; } - When call matchRemoteSysext "mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext" "1.0.0-2-azlinux3" "$2" - The output should equal "v1.0.0-2-azlinux3-$2" + When call matchRemoteSysext "mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext" "1.0.0-1-azlinux3" "$2" + The output should equal "v1.0.0-1-azlinux3-$2" The status should be success End End @@ -64,12 +64,12 @@ Describe 'cse_install_acl.sh' Describe 'local resolution' setup_cache() { SYSEXT_CACHE=$(mktemp -d) - touch "$SYSEXT_CACHE/artifact-streaming-v1.0.0-2-azlinux3-x86-64.raw" + touch "$SYSEXT_CACHE/artifact-streaming-v1.0.0-1-azlinux3-x86-64.raw" touch "$SYSEXT_CACHE/artifact-streaming-v1.0.0-20-azlinux3-x86-64.raw" touch "$SYSEXT_CACHE/artifact-streaming.raw" } cleanup_cache() { - rm -f "$SYSEXT_CACHE/artifact-streaming-v1.0.0-2-azlinux3-x86-64.raw" \ + rm -f "$SYSEXT_CACHE/artifact-streaming-v1.0.0-1-azlinux3-x86-64.raw" \ "$SYSEXT_CACHE/artifact-streaming-v1.0.0-20-azlinux3-x86-64.raw" \ "$SYSEXT_CACHE/artifact-streaming.raw" rmdir "$SYSEXT_CACHE" @@ -82,8 +82,8 @@ Describe 'cse_install_acl.sh' } It 'selects the exact cached revision rather than a later prefix match' - When call matchLocalSysext artifact-streaming "1.0.0-2-azlinux3" x86-64 - The output should equal "$SYSEXT_CACHE/artifact-streaming-v1.0.0-2-azlinux3-x86-64.raw" + When call matchLocalSysext artifact-streaming "1.0.0-1-azlinux3" x86-64 + The output should equal "$SYSEXT_CACHE/artifact-streaming-v1.0.0-1-azlinux3-x86-64.raw" The status should be success End diff --git a/spec/vhdbuilder/packer/install_dependencies_spec.sh b/spec/vhdbuilder/packer/install_dependencies_spec.sh index a6d03e7226f..557da1258e4 100644 --- a/spec/vhdbuilder/packer/install_dependencies_spec.sh +++ b/spec/vhdbuilder/packer/install_dependencies_spec.sh @@ -68,7 +68,7 @@ Describe 'installAndConfigureArtifactStreaming' It 'fails when the combined sysext cannot be cached' OS="AZURECONTAINERLINUX" DOWNLOAD_RC=1 - When run installAndConfigureArtifactStreaming "mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext:v1.0.0-2-azlinux3-x86-64" "v1.0.0-2-azlinux3" + When run installAndConfigureArtifactStreaming "mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext:v1.0.0-1-azlinux3-x86-64" "v1.0.0-1-azlinux3" The status should equal "$ERR_ARTIFACT_STREAMING_DOWNLOAD" End End From 93408b9a468f2366d39c25828f68b3f09af83701 Mon Sep 17 00:00:00 2001 From: rguptar <69279773+rguptar@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:35:36 -0700 Subject: [PATCH 15/20] Remove Flatcar sysext configuration from ACL custom data Remove the unverified enabled-sysext.conf entry from ACL custom data and regenerate Ignition JSON using strict Butane validation. Preserve all remaining configuration and streaming service masks. Validated: strict Butane regeneration, Ignition configuration equivalence apart from the removed entry, 28 focused ShellSpec examples, and whitespace checks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: da9d0569-0965-44eb-9e2b-1d2ac995e638 --- vhdbuilder/packer/acl-customdata.json | 2 +- vhdbuilder/packer/acl-customdata.yaml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/vhdbuilder/packer/acl-customdata.json b/vhdbuilder/packer/acl-customdata.json index 05ddcb19e6e..cccf7120d85 100644 --- a/vhdbuilder/packer/acl-customdata.json +++ b/vhdbuilder/packer/acl-customdata.json @@ -1 +1 @@ -{"ignition":{"version":"3.4.0"},"kernelArguments":{"shouldNotExist":["flatcar.autologin"]},"storage":{"directories":[{"path":"/opt/aks-sysext-rw/usr/local/bin"}],"files":[{"overwrite":true,"path":"/opt/aks-sysext/usr/lib/extension-release.d/extension-release.aks-sysext","contents":{"compression":"","source":"data:,ID%3Dazurelinux%0ASYSEXT_LEVEL%3D1.0%0A"}},{"overwrite":true,"path":"/etc/flatcar/enabled-sysext.conf","contents":{"compression":"","source":"data:,"}},{"overwrite":true,"path":"/etc/flatcar/update.conf","contents":{"compression":"","source":"data:,SERVER%3Ddisabled%0A"}},{"overwrite":true,"path":"/etc/systemd/system/containerd.service.d/50-default-config.conf","contents":{"compression":"","source":"data:,%5BService%5D%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0A"}}],"links":[{"path":"/etc/extensions/aks-sysext","hard":false,"target":"/opt/aks-sysext"},{"path":"/opt/aks-sysext/usr/local/bin","hard":false,"target":"/opt/aks-sysext-rw/usr/local/bin"},{"path":"/etc/containerd/config.toml","hard":false,"target":"/usr/share/containerd/config.toml"}]},"systemd":{"units":[{"mask":true,"name":"artifact-streaming-setup.service"},{"mask":true,"name":"acr-mirror.service"},{"mask":true,"name":"overlaybd-tcmu.service"},{"mask":true,"name":"overlaybd-snapshotter.service"}]}} +{"ignition":{"version":"3.4.0"},"kernelArguments":{"shouldNotExist":["flatcar.autologin"]},"storage":{"directories":[{"path":"/opt/aks-sysext-rw/usr/local/bin"}],"files":[{"overwrite":true,"path":"/opt/aks-sysext/usr/lib/extension-release.d/extension-release.aks-sysext","contents":{"compression":"","source":"data:,ID%3Dazurelinux%0ASYSEXT_LEVEL%3D1.0%0A"}},{"overwrite":true,"path":"/etc/flatcar/update.conf","contents":{"compression":"","source":"data:,SERVER%3Ddisabled%0A"}},{"overwrite":true,"path":"/etc/systemd/system/containerd.service.d/50-default-config.conf","contents":{"compression":"","source":"data:,%5BService%5D%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0A"}}],"links":[{"path":"/etc/extensions/aks-sysext","hard":false,"target":"/opt/aks-sysext"},{"path":"/opt/aks-sysext/usr/local/bin","hard":false,"target":"/opt/aks-sysext-rw/usr/local/bin"},{"path":"/etc/containerd/config.toml","hard":false,"target":"/usr/share/containerd/config.toml"}]},"systemd":{"units":[{"mask":true,"name":"artifact-streaming-setup.service"},{"mask":true,"name":"acr-mirror.service"},{"mask":true,"name":"overlaybd-tcmu.service"},{"mask":true,"name":"overlaybd-snapshotter.service"}]}} diff --git a/vhdbuilder/packer/acl-customdata.yaml b/vhdbuilder/packer/acl-customdata.yaml index f560c47fc9e..409a04446ea 100644 --- a/vhdbuilder/packer/acl-customdata.yaml +++ b/vhdbuilder/packer/acl-customdata.yaml @@ -13,10 +13,6 @@ storage: inline: | ID=azurelinux SYSEXT_LEVEL=1.0 - - path: /etc/flatcar/enabled-sysext.conf - overwrite: true - contents: - inline: "" - path: /etc/flatcar/update.conf overwrite: true contents: From 340d19697897ab9056c3c130738e8e26a03e37ba Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Thu, 29 Oct 2026 09:05:17 -0700 Subject: [PATCH 16/20] test(e2e): skip ACL artifact streaming on main-built images ACL e2e provisions with the CSE scripts baked into the VHD, so the ACL artifact streaming path is absent on main-built ACL VHDs and the scenario fails with ERR_ARTIFACT_STREAMING_INSTALL (153). Skip it when the lane resolves a main-built image, matching the existing kubelet disk gate. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb6a904c-2cd4-4514-b330-b363fe5ea1e1 --- e2e/scenario/scenario.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/e2e/scenario/scenario.go b/e2e/scenario/scenario.go index a2267fa34e7..8805bbc221d 100755 --- a/e2e/scenario/scenario.go +++ b/e2e/scenario/scenario.go @@ -88,6 +88,7 @@ var _ = Register(&Scenario{ var _ = Register(&Scenario{ Name: "ACL_ArtifactStreaming", Description: "Tests that an ACL node with artifact streaming enabled activates OverlayBD and acr-mirror", + SkipIf: skipACLArtifactStreamingOnMainBuiltImage, Config: Config{ Cluster: ClusterKubenet, VHD: config.VHDACLGen2TL, @@ -120,6 +121,19 @@ var _ = Register(&Scenario{ }, }) +// ACL e2e provisions with the CSE scripts baked into the VHD rather than the branch's scripts, +// so the ACL artifact streaming path only exists on VHDs built from a branch that contains it. +func skipACLArtifactStreamingOnMainBuiltImage(context.Context) string { + if laneResolvedMainBuiltImage() { + return fmt.Sprintf( + "this lane resolved a main-built image (%s=%s), which predates ACL artifact streaming support baked into the VHD; run against the PR's VHD build to exercise it", + config.Config.SIGVersionTagName, + config.Config.SIGVersionTagValue, + ) + } + return "" +} + var _ = Register(&Scenario{ Name: "ACL_CustomCA", Description: "Tests that a node using an ACL VHD can be properly bootstrapped and custom CA was correctly added", From 9e45ddcbfa4e760caa8cf321086b7c50d48e352b Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Thu, 29 Oct 2026 13:44:17 -0700 Subject: [PATCH 17/20] test(e2e): use ACL security profile helper for artifact streaming scenario Use aclVMSSSecurityProfile so Secure Boot is disabled when ACL_BASE_IMAGE_SIGNED=false, matching other ACL scenarios. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb6a904c-2cd4-4514-b330-b363fe5ea1e1 --- e2e/scenario/scenario.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/scenario/scenario.go b/e2e/scenario/scenario.go index 8805bbc221d..a2246f1bcfb 100755 --- a/e2e/scenario/scenario.go +++ b/e2e/scenario/scenario.go @@ -99,7 +99,7 @@ var _ = Register(&Scenario{ config.EnableArtifactStreaming = true }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) }, Validator: func(ctx context.Context, s *Scenario) error { return errors.Join( From 4dc720b9baeea3c5e835e3a38a682928dbc2c116 Mon Sep 17 00:00:00 2001 From: aadhar-agarwal <108542189+aadhar-agarwal@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:56:28 -0700 Subject: [PATCH 18/20] fix: check ACL IPE compatibility with best-effort combined sources Keep a single compatibility guard in ensureArtifactStreaming. Reject streaming if the boot-time cached profile requests ipe=audit/enforcing OR any kernel IPE policy is active. A disabled tag never overrides active kernel state; inactive kernel state never overrides an explicit IPE request. Missing, unreadable, or invalid sources produce warnings, not provisioning failures. Continue to kernel state after an inconclusive tag check and continue scanning remaining policies after individual read failures. Do not query IMDS, mount securityfs, or alter IPE/snapshotter settings in the guard. Name the customer-facing acl-node-security-profile=ipe=audit setting explicitly in conflict errors. Report the actual requested mode for cached-tag conflicts; describe audit as an example for kernel-only conflicts. Explain how to disable IPE without overwriting other profile entries and reboot before provisioning. This is a provisioning-time guard, not a reboot-time interlock. Preserve VHD caching and non-ACL paths. Based directly on artifact-streaming PR head 340d19697897ab9056c3c130738e8e26a03e37ba. Verification: 26 focused compatibility cases passed after the message amendment; repository-configured ShellCheck, syntax checks, and patch applicability passed. The earlier combined implementation passed 128 focused examples. Document the provisioning-only stopgap and link the EROFS artifact-streaming follow-up: https://dev.azure.com/mariner-org/ACL/_workitems/edit/24753. Comment-only amendment; syntax and repository-configured ShellCheck passed. Signed-off-by: Aadhar Agarwal --- .../linux/cloud-init/artifacts/cse_config.sh | 76 ++++++ .../artifacts/cse_config_gpu_spec.sh | 222 ++++++++++++++++++ 2 files changed, 298 insertions(+) diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index afcc2ac7e81..32f1750f70e 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -634,9 +634,85 @@ configureContainerdLegacyMooncakeMcrHost() { EOF } +# Temporary provisioning-only guard until EROFS supports artifact streaming. +# Fail only for detected IPE conflicts; unavailable checks warn and continue. +# Later IPE enablement is not covered. +# Tracking: https://dev.azure.com/mariner-org/ACL/_workitems/edit/24753 +validateACLArtifactStreamingSecurityProfile() { + local profile_cache="${ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE:-/run/acl/node-security-profile}" + if [ ! -e "$profile_cache" ]; then + echo "WARNING: No cached ACL security profile; skipping the IPE tag check." >&2 + return 0 + fi + + local profile pair key mode + local -a pairs + if ! profile="$(cat "$profile_cache")"; then + echo "WARNING: Cannot read ${profile_cache}; skipping the IPE tag check." >&2 + return 0 + fi + # Match ACL's initrd parser: comma-separated pairs, first matching key wins. + IFS=',' read -ra pairs <<< "$profile" + for pair in "${pairs[@]}"; do + IFS='=' read -r key mode <<< "$pair" + [ "$key" = "ipe" ] || continue + case "$mode" in + audit|enforcing) + echo "Artifact streaming requires OverlayBD, but the boot-time tag requests acl-node-security-profile=ipe=${mode}. Disable artifact streaming or set ipe=disabled in acl-node-security-profile (preserving other entries), then reboot before provisioning." >&2 + return 1 + ;; + disabled|off|"") ;; + *) + echo "WARNING: Unrecognized cached IPE mode '${mode}'; skipping the IPE tag check." >&2 + ;; + esac + break + done + return 0 +} + +validateACLArtifactStreamingIPECompatibility() { + validateACLArtifactStreamingSecurityProfile || return 1 + + local securityfs_root="${ARTIFACT_STREAMING_SECURITYFS_ROOT:-/sys/kernel/security}" + if ! findmnt -n -t securityfs --mountpoint "$securityfs_root" >/dev/null || + [ ! -r "$securityfs_root" ] || [ ! -x "$securityfs_root" ]; then + echo "WARNING: Cannot inspect securityfs at ${securityfs_root}; skipping the kernel IPE check." >&2 + return 0 + fi + [ -d "${securityfs_root}/ipe" ] || return 0 + + local policies_dir="${securityfs_root}/ipe/policies" + if [ ! -d "$policies_dir" ] || [ ! -r "$policies_dir" ] || [ ! -x "$policies_dir" ]; then + echo "WARNING: Cannot enumerate ${policies_dir}; skipping the kernel IPE check." >&2 + return 0 + fi + local policy active + # Audit mode still selects EROFS. Include policies with dot-prefixed names. + for policy in "$policies_dir"/* "$policies_dir"/.[!.]* "$policies_dir"/..?*; do + [ -e "$policy" ] || continue + if ! active="$(cat "${policy}/active")"; then + echo "WARNING: Cannot read ${policy}/active; skipping this IPE policy." >&2 + continue + fi + case "$active" in + 0) ;; + 1) + echo "Artifact streaming requires OverlayBD, but ACL IPE is active and requires EROFS (for example, enabled with acl-node-security-profile=ipe=audit). Disable artifact streaming or set ipe=disabled in acl-node-security-profile (preserving other entries), then reboot before provisioning." >&2 + return 1 + ;; + *) + echo "WARNING: Invalid IPE policy state '${active}' in ${policy}/active; skipping this IPE policy." >&2 + ;; + esac + done + return 0 +} + ensureArtifactStreaming() { local acr_mirror_setup="${ACR_MIRROR_SETUP_SCRIPT:-/opt/acr/tools/mirror/setup.sh}" if isACL "$OS" "$OS_VARIANT"; then + validateACLArtifactStreamingIPECompatibility || exit "$ERR_ARTIFACT_STREAMING_INSTALL" local services=(acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service) # DALEC's Upholds= starts unmasked units as soon as the manager reloads. systemctl mask "${services[@]}" artifact-streaming-setup.service || exit "$ERR_ARTIFACT_STREAMING_INSTALL" diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh index 32c923e95db..dc3b16a13ff 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh @@ -140,6 +140,10 @@ Describe 'cse_config_gpu.sh' TEST_ACR_DIR="$(mktemp -d)" ACR_MIRROR_SETUP_SCRIPT="${TEST_ACR_DIR}/setup.sh" ACR_CONFIG_BIN="${TEST_ACR_DIR}/acr-config" + ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE="${TEST_ACR_DIR}/node-security-profile" + ARTIFACT_STREAMING_SECURITYFS_ROOT="${TEST_ACR_DIR}/security" + touch "$ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE" + mkdir -p "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/policies" OS="$UBUNTU_OS_NAME" OS_VARIANT="" } @@ -152,6 +156,10 @@ Describe 'cse_config_gpu.sh' waitForContainerdReady() { return "${CONTAINERD_READY_RC:-0}" } + findmnt() { + [ "$*" = "-n -t securityfs --mountpoint ${ARTIFACT_STREAMING_SECURITYFS_ROOT}" ] || return 1 + return "${FINDMNT_RC:-0}" + } systemctl() { if [ "${FAIL_SYSTEMCTL:-}" = "$*" ]; then return 1 @@ -176,6 +184,220 @@ Describe 'cse_config_gpu.sh' chmod +x "${ACR_CONFIG_BIN}" } + Describe 'best-effort ACL IPE compatibility' + BeforeEach 'install_setup_sh' + + set_cached_profile() { + case "$1" in + missing) rm "$ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE" ;; + unreadable) PROFILE_READ_FAIL=true ;; + *) printf '%s' "$1" > "$ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE" ;; + esac + } + cat() { + if [ "$1" = "$ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE" ] && [ "${PROFILE_READ_FAIL:-false}" = true ]; then + return 1 + fi + command cat "$@" + } + write_ipe_policy() { + mkdir -p "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/policies/$1" + printf '%s\n' "$2" > "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/policies/$1/active" + } + + Describe 'cached intent conflicts' + Parameters + "$ACL_OS_NAME" "" "ipe=audit" 0 audit + "$ACL_OS_NAME" "" "selinux=enforcing,ipe=enforcing" 1 enforcing + "$AZURELINUX_OS_NAME" "$ACL_OS_VARIANT" "ipe=audit,selinux=permissive" 1 audit + End + + It "rejects '$3' before side effects even with inactive or unavailable kernel state" + OS="$1" + OS_VARIANT="$2" + set_cached_profile "$3" + FINDMNT_RC="$4" + write_ipe_policy "inactive" 0 + + When run ensureArtifactStreaming + + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + The output should be blank + The stderr should include "boot-time tag requests acl-node-security-profile=ipe=$5" + The stderr should include "set ipe=disabled in acl-node-security-profile (preserving other entries)" + End + End + + Describe 'active kernel policies' + Parameters + "ipe=disabled" 0 + "ipe=off" 1 + missing 0 + unreadable 1 + "ipe=unknown" 0 + End + + It "rejects active IPE with cache='$1' and enforce=$2" + OS="$ACL_OS_NAME" + set_cached_profile "$1" + printf '%s\n' "$2" > "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/enforce" + write_ipe_policy "a_inactive" 0 + write_ipe_policy ".active" 1 + + When run ensureArtifactStreaming + + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + The output should be blank + The stderr should include "ACL IPE is active and requires EROFS" + The stderr should include "for example, enabled with acl-node-security-profile=ipe=audit" + The stderr should include "set ipe=disabled in acl-node-security-profile (preserving other entries)" + End + End + + Describe 'available nonconflicting state' + Parameters + "ipe=disabled" + "selinux=enforcing,ipe=off" + "" + "selinux=enforcing" + "ipe=off,ipe=audit" + End + + It "allows '$1' with loaded but inactive IPE" + OS="$ACL_OS_NAME" + set_cached_profile "$1" + printf '1\n' > "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/enforce" + write_ipe_policy "inactive" 0 + + When run ensureArtifactStreaming + + The status should be success + The output should include "setup.sh aks" + The stderr should be blank + End + End + + Describe 'unavailable or unknown sources' + Parameters + missing 0 "No cached ACL security profile" + missing 1 "skipping the kernel IPE check" + unreadable 0 "skipping the IPE tag check" + unreadable 1 "skipping the kernel IPE check" + "ipe=disabled" 1 "skipping the kernel IPE check" + "ipe=unknown" 0 "Unrecognized cached IPE mode" + End + + It "warns and continues for cache='$1' and kernel lookup status=$2" + OS="$ACL_OS_NAME" + set_cached_profile "$1" + FINDMNT_RC="$2" + + When run ensureArtifactStreaming + + The status should be success + The output should include "setup.sh aks" + The stderr should include "WARNING:" + The stderr should include "$3" + End + End + + It 'warns and continues when policies cannot be enumerated' + OS="$ACL_OS_NAME" + rmdir "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/policies" + + When run ensureArtifactStreaming + + The status should be success + The output should include "setup.sh aks" + The stderr should include "WARNING: Cannot enumerate" + End + + It 'allows older kernels without IPE on mounted securityfs' + OS="$ACL_OS_NAME" + rmdir "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/policies" "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe" + + When run ensureArtifactStreaming + + The status should be success + The output should include "setup.sh aks" + The stderr should be blank + End + + Describe 'individual policy read failures' + Parameters + 0 0 + 1 "$ERR_ARTIFACT_STREAMING_INSTALL" + End + + It "continues past unreadable and invalid entries to a later policy with active=$1" + OS="$ACL_OS_NAME" + mkdir -p "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/policies/a_unreadable" + write_ipe_policy "b_invalid" unexpected + write_ipe_policy ".later" "$1" + + When run ensureArtifactStreaming + + The status should equal "$2" + The stderr should include "WARNING: Cannot read" + The stderr should include "WARNING: Invalid IPE policy state" + if [ "$1" = 1 ]; then + The output should be blank + The stderr should include "ACL IPE is active and requires EROFS" + else + The output should include "setup.sh aks" + fi + End + End + + It 'does not apply ACL checks to Ubuntu' + set_cached_profile "ipe=audit" + write_ipe_policy "active" 1 + + When run ensureArtifactStreaming + + The status should be success + The output should include "setup.sh aks" + The stderr should be blank + End + + Describe 'nodePrep activation on PIS nodes' + logs_to_events() { + shift + "$@" + } + ensureKubelet() { echo "ensureKubelet"; } + run_node_prep_streaming() { + eval "$(sed -n '/# Keep ACL sysexts masked during PIS image preparation/,/logs_to_events "AKS.CSE.ensureKubelet" ensureKubelet/p' \ + ./parts/linux/cloud-init/artifacts/cse_main.sh)" + } + + It 'propagates a detected conflict before kubelet starts' + OS="$ACL_OS_NAME" + ARTIFACT_STREAMING_ENABLED=true + write_ipe_policy "active" 1 + + When run run_node_prep_streaming + + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + The output should be blank + The stderr should include "ACL IPE is active and requires EROFS" + End + + It 'does not reject IPE when streaming is disabled' + OS="$ACL_OS_NAME" + ARTIFACT_STREAMING_ENABLED=false + set_cached_profile "ipe=audit" + write_ipe_policy "active" 1 + + When run run_node_prep_streaming + + The status should be success + The output should equal "ensureKubelet" + The stderr should be blank + End + End + End + It 'uses setup.sh aks when acr-mirror 1.0.0+ is installed' install_setup_sh When run ensureArtifactStreaming From 6e43412bfd7adc879df6004313469902acea8eda Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Thu, 29 Oct 2026 14:28:39 -0700 Subject: [PATCH 19/20] refactor: move ACL IPE artifact streaming guard to ACL helpers The guard is ACL-only but lived in cse_config.sh, which is embedded in every distro's CustomData and pushed traditional CustomData past MaxCustomDataLength (88032 > 87380). Move it unchanged to cse_helpers_acl.sh, which ships only for ACL and is sourced before provisioning. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb6a904c-2cd4-4514-b330-b363fe5ea1e1 --- .../artifacts/acl/cse_helpers_acl.sh | 74 ++++++++++++++++++ .../linux/cloud-init/artifacts/cse_config.sh | 75 ------------------- .../artifacts/cse_config_gpu_spec.sh | 1 + 3 files changed, 75 insertions(+), 75 deletions(-) diff --git a/parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh b/parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh index 0bf141f7cb2..24cec587b3d 100644 --- a/parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh +++ b/parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh @@ -21,3 +21,77 @@ apt_get_update() { apt_get_dist_upgrade() { stub } + +# Temporary provisioning-only guard until EROFS supports artifact streaming. +# Fail only for detected IPE conflicts; unavailable checks warn and continue. +# Later IPE enablement is not covered. +validateACLArtifactStreamingSecurityProfile() { + local profile_cache="${ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE:-/run/acl/node-security-profile}" + if [ ! -e "$profile_cache" ]; then + echo "WARNING: No cached ACL security profile; skipping the IPE tag check." >&2 + return 0 + fi + + local profile pair key mode + local -a pairs + if ! profile="$(cat "$profile_cache")"; then + echo "WARNING: Cannot read ${profile_cache}; skipping the IPE tag check." >&2 + return 0 + fi + # Match ACL's initrd parser: comma-separated pairs, first matching key wins. + IFS=',' read -ra pairs <<< "$profile" + for pair in "${pairs[@]}"; do + IFS='=' read -r key mode <<< "$pair" + [ "$key" = "ipe" ] || continue + case "$mode" in + audit|enforcing) + echo "Artifact streaming requires OverlayBD, but the boot-time tag requests acl-node-security-profile=ipe=${mode}. Disable artifact streaming or set ipe=disabled in acl-node-security-profile (preserving other entries), then reboot before provisioning." >&2 + return 1 + ;; + disabled|off|"") ;; + *) + echo "WARNING: Unrecognized cached IPE mode '${mode}'; skipping the IPE tag check." >&2 + ;; + esac + break + done + return 0 +} + +validateACLArtifactStreamingIPECompatibility() { + validateACLArtifactStreamingSecurityProfile || return 1 + + local securityfs_root="${ARTIFACT_STREAMING_SECURITYFS_ROOT:-/sys/kernel/security}" + if ! findmnt -n -t securityfs --mountpoint "$securityfs_root" >/dev/null || + [ ! -r "$securityfs_root" ] || [ ! -x "$securityfs_root" ]; then + echo "WARNING: Cannot inspect securityfs at ${securityfs_root}; skipping the kernel IPE check." >&2 + return 0 + fi + [ -d "${securityfs_root}/ipe" ] || return 0 + + local policies_dir="${securityfs_root}/ipe/policies" + if [ ! -d "$policies_dir" ] || [ ! -r "$policies_dir" ] || [ ! -x "$policies_dir" ]; then + echo "WARNING: Cannot enumerate ${policies_dir}; skipping the kernel IPE check." >&2 + return 0 + fi + local policy active + # Audit mode still selects EROFS. Include policies with dot-prefixed names. + for policy in "$policies_dir"/* "$policies_dir"/.[!.]* "$policies_dir"/..?*; do + [ -e "$policy" ] || continue + if ! active="$(cat "${policy}/active")"; then + echo "WARNING: Cannot read ${policy}/active; skipping this IPE policy." >&2 + continue + fi + case "$active" in + 0) ;; + 1) + echo "Artifact streaming requires OverlayBD, but ACL IPE is active and requires EROFS (for example, enabled with acl-node-security-profile=ipe=audit). Disable artifact streaming or set ipe=disabled in acl-node-security-profile (preserving other entries), then reboot before provisioning." >&2 + return 1 + ;; + *) + echo "WARNING: Invalid IPE policy state '${active}' in ${policy}/active; skipping this IPE policy." >&2 + ;; + esac + done + return 0 +} diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 32f1750f70e..fbaf41ed09a 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -634,81 +634,6 @@ configureContainerdLegacyMooncakeMcrHost() { EOF } -# Temporary provisioning-only guard until EROFS supports artifact streaming. -# Fail only for detected IPE conflicts; unavailable checks warn and continue. -# Later IPE enablement is not covered. -# Tracking: https://dev.azure.com/mariner-org/ACL/_workitems/edit/24753 -validateACLArtifactStreamingSecurityProfile() { - local profile_cache="${ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE:-/run/acl/node-security-profile}" - if [ ! -e "$profile_cache" ]; then - echo "WARNING: No cached ACL security profile; skipping the IPE tag check." >&2 - return 0 - fi - - local profile pair key mode - local -a pairs - if ! profile="$(cat "$profile_cache")"; then - echo "WARNING: Cannot read ${profile_cache}; skipping the IPE tag check." >&2 - return 0 - fi - # Match ACL's initrd parser: comma-separated pairs, first matching key wins. - IFS=',' read -ra pairs <<< "$profile" - for pair in "${pairs[@]}"; do - IFS='=' read -r key mode <<< "$pair" - [ "$key" = "ipe" ] || continue - case "$mode" in - audit|enforcing) - echo "Artifact streaming requires OverlayBD, but the boot-time tag requests acl-node-security-profile=ipe=${mode}. Disable artifact streaming or set ipe=disabled in acl-node-security-profile (preserving other entries), then reboot before provisioning." >&2 - return 1 - ;; - disabled|off|"") ;; - *) - echo "WARNING: Unrecognized cached IPE mode '${mode}'; skipping the IPE tag check." >&2 - ;; - esac - break - done - return 0 -} - -validateACLArtifactStreamingIPECompatibility() { - validateACLArtifactStreamingSecurityProfile || return 1 - - local securityfs_root="${ARTIFACT_STREAMING_SECURITYFS_ROOT:-/sys/kernel/security}" - if ! findmnt -n -t securityfs --mountpoint "$securityfs_root" >/dev/null || - [ ! -r "$securityfs_root" ] || [ ! -x "$securityfs_root" ]; then - echo "WARNING: Cannot inspect securityfs at ${securityfs_root}; skipping the kernel IPE check." >&2 - return 0 - fi - [ -d "${securityfs_root}/ipe" ] || return 0 - - local policies_dir="${securityfs_root}/ipe/policies" - if [ ! -d "$policies_dir" ] || [ ! -r "$policies_dir" ] || [ ! -x "$policies_dir" ]; then - echo "WARNING: Cannot enumerate ${policies_dir}; skipping the kernel IPE check." >&2 - return 0 - fi - local policy active - # Audit mode still selects EROFS. Include policies with dot-prefixed names. - for policy in "$policies_dir"/* "$policies_dir"/.[!.]* "$policies_dir"/..?*; do - [ -e "$policy" ] || continue - if ! active="$(cat "${policy}/active")"; then - echo "WARNING: Cannot read ${policy}/active; skipping this IPE policy." >&2 - continue - fi - case "$active" in - 0) ;; - 1) - echo "Artifact streaming requires OverlayBD, but ACL IPE is active and requires EROFS (for example, enabled with acl-node-security-profile=ipe=audit). Disable artifact streaming or set ipe=disabled in acl-node-security-profile (preserving other entries), then reboot before provisioning." >&2 - return 1 - ;; - *) - echo "WARNING: Invalid IPE policy state '${active}' in ${policy}/active; skipping this IPE policy." >&2 - ;; - esac - done - return 0 -} - ensureArtifactStreaming() { local acr_mirror_setup="${ACR_MIRROR_SETUP_SCRIPT:-/opt/acr/tools/mirror/setup.sh}" if isACL "$OS" "$OS_VARIANT"; then diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh index dc3b16a13ff..1353c67d5d3 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh @@ -130,6 +130,7 @@ Describe 'cse_config_gpu.sh' End End Describe 'ensureArtifactStreaming' + Include "./parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh" # ensureArtifactStreaming runs the version-appropriate enablement path # before enabling the acr-mirror/overlaybd services: # - acr-mirror 1.0.0+ -> setup.sh aks From 2f5c6ee93bf122f77145325dd58e3becbbbd1f1e Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Thu, 29 Oct 2026 17:44:14 -0700 Subject: [PATCH 20/20] test(spec): cover ACL artifact streaming PIS phase gate Assert ACL streaming activates only in nodePrep (basePrep keeps non-ACL streaming), runs before ensureKubelet, and fails provisioning on error. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: eb6a904c-2cd4-4514-b330-b363fe5ea1e1 --- .../artifacts/cse_main_pis_reconcile_spec.sh | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/spec/parts/linux/cloud-init/artifacts/cse_main_pis_reconcile_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_main_pis_reconcile_spec.sh index 85934425ff9..530a242a099 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_main_pis_reconcile_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_main_pis_reconcile_spec.sh @@ -177,6 +177,44 @@ Describe 'cse_main.sh PIS-safe configuration' End End + Describe 'artifact streaming' + streaming_call='AKS.CSE.ensureContainerd.ensureArtifactStreaming' + + It 'activates ACL streaming only in nodePrep so PIS images keep sysexts masked' + streaming_gates() { + phase_count "basePrep" "${streaming_call}" + phase_count "nodePrep" "${streaming_call}" + script_count "${streaming_call}" + phase_body "basePrep" | code_lines | grep -B1 -- "${streaming_call}" | head -1 + phase_body "nodePrep" | code_lines | grep -B1 -- "${streaming_call}" | head -1 + } + When call streaming_gates + The line 1 of output should equal "1" + The line 2 of output should equal "1" + The line 3 of output should equal "2" + The line 4 of output should equal ' if [ "${ARTIFACT_STREAMING_ENABLED}" = "true" ] && ! isACL "$OS" "$OS_VARIANT"; then' + The line 5 of output should equal ' if [ "${ARTIFACT_STREAMING_ENABLED}" = "true" ] && isACL "$OS" "$OS_VARIANT"; then' + End + + It 'activates ACL streaming before kubelet startup' + streaming_order() { + phase_body "nodePrep" | code_lines | grep -E "${streaming_call}|AKS.CSE.ensureKubelet\"" + } + When call streaming_order + The line 1 of output should include "${streaming_call}" + The line 2 of output should include 'AKS.CSE.ensureKubelet' + The lines of output should equal 2 + End + + It 'fails provisioning when ACL streaming cannot be activated' + streaming_call_line() { + phase_body "nodePrep" | code_lines | grep -- "${streaming_call}" + } + When call streaming_call_line + The output should include 'ensureArtifactStreaming || exit $ERR_ARTIFACT_STREAMING_INSTALL' + End + End + Describe 'stage gate' It 'configures Chrony before either provisioning stage' When call chrony_dispatch_calls