feat: enable artifact streaming on ACL

aks-node-controller/parser/helper_test.go

@@ -939,6 +939,44 @@ oom_score = -999
 	}
 }
 
+func TestACLArtifactStreamingContainerdConfig(t *testing.T) {
+	tests := []struct {
+		name              string
+		containerdVersion string
+		noGPU             bool
+	}{
+		{name: "containerd v1", containerdVersion: "1.7.22"},
+		{name: "containerd v1 no GPU", containerdVersion: "1.7.22", noGPU: true},
+		{name: "containerd v2", containerdVersion: "2.0.0"},
+		{name: "containerd v2 no GPU", containerdVersion: "2.0.0", noGPU: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			config, err := containerdConfigFromAKSNodeConfig(&aksnodeconfigv1.Configuration{
+				EnableArtifactStreaming: true,
+			}, tt.noGPU, tt.containerdVersion)
+			if err != nil {
+				t.Fatalf("containerdConfigFromAKSNodeConfig() error = %v", err)
+			}
+
+			if count := strings.Count(config, `snapshotter = "overlaybd"`); count != 1 {
+				t.Errorf("expected exactly one overlaybd snapshotter, got %d in:\n%s", count, config)
+			}
+			for _, expected := range []string{
+				`disable_snapshot_annotations = false`,
+				`[proxy_plugins.overlaybd]`,
+				`type = "snapshot"`,
+				`address = "/run/overlaybd-snapshotter/overlaybd.sock"`,
+			} {
+				if !strings.Contains(config, expected) {
+					t.Errorf("expected rendered config to contain %q:\n%s", expected, config)
+				}
+			}
+		})
+	}
+}
+
 func Test_getKubenetTemplate(t *testing.T) {
 	tests := []struct {
 		name string

aks-node-controller/pkg/nodeconfigutils/utils.go

@@ -69,7 +69,7 @@ fi`
        "files": [{
          "path": "/opt/azure/containers/aks-node-controller-config.json",
          "mode": 384,
-         "contents": { "source": "data:;base64,%s" }
+         "contents": { "source": "data:;base64,%[1]s" }
        }]
      }
     }`

aks-node-controller/pkg/nodeconfigutils/utils_test.go

@@ -2,6 +2,7 @@ package nodeconfigutils
 
 import (
 	"encoding/base64"
+	"encoding/json"
 	"io"
 	"mime"
 	"mime/multipart"
@@ -44,6 +45,18 @@ func TestContainerdImagePullPresenceRoundTrip(t *testing.T) {
 	}
 }
 
+func decodeFlatcarCustomData(t *testing.T, cfg *aksnodeconfigv1.Configuration) map[string]any {
+	t.Helper()
+	customData, err := CustomDataFlatcar(cfg)
+	require.NoError(t, err)
+	decoded, err := base64.StdEncoding.DecodeString(customData)
+	require.NoError(t, err)
+
+	var ignition map[string]any
+	require.NoError(t, json.Unmarshal(decoded, &ignition))
+	return ignition
+}
+
 func TestUnmarshalConfigurationV1(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -300,6 +313,31 @@ func TestCustomDataUsesMultipartBoothookAndCloudConfig(t *testing.T) {
 	require.ErrorIs(t, err, io.EOF)
 }
 
+func TestCustomDataFlatcarDefersArtifactStreamingToProvisioning(t *testing.T) {
+	for _, enabled := range []bool{false, true} {
+		cfg := &aksnodeconfigv1.Configuration{EnableArtifactStreaming: enabled}
+		ignition := decodeFlatcarCustomData(t, cfg)
+		require.NotContains(t, ignition, "systemd", "Ignition must not start streaming before containerd is configured")
+
+		storage, ok := ignition["storage"].(map[string]any)
+		require.True(t, ok)
+		files, ok := storage["files"].([]any)
+		require.True(t, ok)
+		require.NotEmpty(t, files)
+		file, ok := files[0].(map[string]any)
+		require.True(t, ok)
+		contents, ok := file["contents"].(map[string]any)
+		require.True(t, ok)
+		source, ok := contents["source"].(string)
+		require.True(t, ok)
+		configJSON, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(source, "data:;base64,"))
+		require.NoError(t, err)
+		decoded, err := UnmarshalConfigurationV1(configJSON)
+		require.NoError(t, err)
+		require.Equal(t, enabled, decoded.GetEnableArtifactStreaming())
+	}
+}
+
 // decodeBoothook extracts the decoded cloud-boothook part from the base64-encoded custom data.
 func decodeBoothook(t *testing.T, cfg *aksnodeconfigv1.Configuration) string {
 	t.Helper()

e2e/scenario/scenario.go

@@ -73,11 +73,67 @@ var _ = Register(&Scenario{
 			return errors.Join(
 				ValidateFileHasContent(ctx, s, "/etc/os-release", "ID=azurelinux"),
 				ValidateFileHasContent(ctx, s, "/etc/os-release", "VARIANT_ID=azurecontainerlinux"),
+				ValidateFileDoesNotExist(ctx, s, "/etc/extensions/artifact-streaming.raw"),
+				ValidateSystemdUnitIsNotRunning(ctx, s, "artifact-streaming-setup.service"),
+				ValidateSystemdUnitIsNotRunning(ctx, s, "acr-mirror.service"),
+				ValidateSystemdUnitIsNotRunning(ctx, s, "overlaybd-tcmu.service"),
+				ValidateSystemdUnitIsNotRunning(ctx, s, "overlaybd-snapshotter.service"),
 			)
 		},
 	},
 })
 
+// Artifact streaming changes both the node bootstrap input and the services activated on the
+// node, so this requires a separate node from the baseline ACL scenario.
+var _ = Register(&Scenario{
+	Name:        "ACL_ArtifactStreaming",
+	Description: "Tests that an ACL node with artifact streaming enabled activates OverlayBD and acr-mirror",
+	SkipIf:      skipACLArtifactStreamingOnMainBuiltImage,
+	Config: Config{
+		Cluster: ClusterKubenet,
+		VHD:     config.VHDACLGen2TL,
+		BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) {
+			nbc.EnableArtifactStreaming = true
+		},
+		AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) {
+			config.EnableArtifactStreaming = true
+		},
+		VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) {
+			vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned)
+		},
+		Validator: func(ctx context.Context, s *Scenario) error {
+			return errors.Join(
+				ValidateFileHasContent(ctx, s, "/etc/os-release", "ID=azurelinux"),
+				ValidateFileHasContent(ctx, s, "/etc/os-release", "VARIANT_ID=azurecontainerlinux"),
+				ValidateFileExists(ctx, s, "/etc/extensions/artifact-streaming.raw"),
+				ValidateFileDoesNotExist(ctx, s, "/etc/extensions/overlaybd.raw"),
+				ValidateFileIsRegularFile(ctx, s, "/etc/containerd/config.toml"),
+				ValidateFileExists(ctx, s, "/etc/acr/acr.toml"),
+				ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd"),
+				ValidateFileExists(ctx, s, "/etc/overlaybd-snapshotter/config.json"),
+				ValidateSystemdUnitIsRunning(ctx, s, "artifact-streaming-setup.service"),
+				ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service"),
+				ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service"),
+				ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service"),
+				ValidateSystemdUnitIsRunning(ctx, s, "containerd.service"),
+			)
+		},
+	},
+})
+
+// ACL e2e provisions with the CSE scripts baked into the VHD rather than the branch's scripts,
+// so the ACL artifact streaming path only exists on VHDs built from a branch that contains it.
+func skipACLArtifactStreamingOnMainBuiltImage(context.Context) string {
+	if laneResolvedMainBuiltImage() {
+		return fmt.Sprintf(
+			"this lane resolved a main-built image (%s=%s), which predates ACL artifact streaming support baked into the VHD; run against the PR's VHD build to exercise it",
+			config.Config.SIGVersionTagName,
+			config.Config.SIGVersionTagValue,
+		)
+	}
+	return ""
+}
+
 var _ = Register(&Scenario{
 	Name:        "ACL_CustomCA",
 	Description: "Tests that a node using an ACL VHD can be properly bootstrapped and custom CA was correctly added",

parts/common/components.json

@@ -2526,6 +2526,17 @@
               }
             ]
           }
+        },
+        "azurecontainerlinux": {
+          "current": {
+            "versionsV2": [
+              {
+                "renovateTag": "OCI_registry=https://mcr.microsoft.com, name=azurecontainerregistry/v2/artifact-streaming-sysext",
+                "latestVersion": "v1.0.0-1-azlinux3"
+              }
+            ],
+            "downloadURL": "mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext:${version}-${SYSTEMD_ARCH}"
+          }
         }
       }
     },

parts/linux/cloud-init/artifacts/README-COMPONENTS.md

@@ -93,6 +93,7 @@ Please refer to [components.cue](../../../../schemas/components.cue) for the mos
 	mariner?:      #MarinerOSDistro
 	marinerkata?:  #MarinerOSDistro
 	azurelinux?:   #AzureLinuxOSDistro
+	azurecontainerlinux?: #AzureContainerLinuxOSDistro
 }
 ```
 
@@ -115,6 +116,9 @@ Please refer to [components.cue](../../../../schemas/components.cue) for the mos
 	"OSGUARD/v3.0"?: #ReleaseDownloadURI
 	current?: #ReleaseDownloadURI
 }
+#AzureContainerLinuxOSDistro: {
+	current?: #ReleaseDownloadURI
+}
 ```
 
 ```
@@ -139,6 +143,7 @@ Here are the explanation of the above schema.
     - In `UbuntuOSDistro`, we can define different OS release versions. For example, `r2404` implies release 24.04.
      - In `MarinerOSDistro`, we only have `current` now, which implies that single configurations will be applied to all Mariner release versions. We can distinguish them in needed. Note, we confirmed with Mariner team, Azure Linux 2.0 is reporting itself as `mariner` in the file `/etc/os-release`. So for Azure Linux 2.0 case, it will still read the package versions from `mariner` block.
     - In `AzureLinuxOSDistro`, `v3.0` is for Azure Linux v3.0. `current` is for otherwise but we are not using it now. Azure Linux 2.0 case is described in the `MarinerOSDistro` above. 
+    - In `AzureContainerLinuxOSDistro`, `current` defines ACL-specific metadata. When it is absent, ACL falls back to `flatcar.current`, then `default.current`.
     - `DefaultOSDistro` means the default case of OS Distro. If an OSDistro metadata is not defined, it will fetch it from `default`. For example, if a node is Ubuntu 20.04, but we don't specify `ubuntu` in components.json, then it will fetch `default.current`. For another example, if only `default.current` is specified in the components.json, No matter what OSDistro is the node running, it will only fetch `default.current` because it's the default metadata. This provides flexibility while elimiating unnecessary duplication when defining the metadata.
 1. The OS release version can optionally be prefixed with the OS variant. `OSGUARD/v3.0` and `DEFAULT/v3.0` are currently supported for `azurelinux`. The latter specifically matches Azure Linux 3.0 without a variant, i.e. not OS Guard.
 1. In `ReleaseDownloadURI`, you can see 2 keys.

parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh

@@ -21,3 +21,77 @@ apt_get_update() {
 apt_get_dist_upgrade() {
     stub
 }
+
+# Temporary provisioning-only guard until EROFS supports artifact streaming.
+# Fail only for detected IPE conflicts; unavailable checks warn and continue.
+# Later IPE enablement is not covered.
+validateACLArtifactStreamingSecurityProfile() {
+  local profile_cache="${ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE:-/run/acl/node-security-profile}"
+  if [ ! -e "$profile_cache" ]; then
+    echo "WARNING: No cached ACL security profile; skipping the IPE tag check." >&2
+    return 0
+  fi
+
+  local profile pair key mode
+  local -a pairs
+  if ! profile="$(cat "$profile_cache")"; then
+    echo "WARNING: Cannot read ${profile_cache}; skipping the IPE tag check." >&2
+    return 0
+  fi
+  # Match ACL's initrd parser: comma-separated pairs, first matching key wins.
+  IFS=',' read -ra pairs <<< "$profile"
+  for pair in "${pairs[@]}"; do
+    IFS='=' read -r key mode <<< "$pair"
+    [ "$key" = "ipe" ] || continue
+    case "$mode" in
+      audit|enforcing)
+        echo "Artifact streaming requires OverlayBD, but the boot-time tag requests acl-node-security-profile=ipe=${mode}. Disable artifact streaming or set ipe=disabled in acl-node-security-profile (preserving other entries), then reboot before provisioning." >&2
+        return 1
+        ;;
+      disabled|off|"") ;;
+      *)
+        echo "WARNING: Unrecognized cached IPE mode '${mode}'; skipping the IPE tag check." >&2
+        ;;
+    esac
+    break
+  done
+  return 0
+}
+
+validateACLArtifactStreamingIPECompatibility() {
+  validateACLArtifactStreamingSecurityProfile || return 1
+
+  local securityfs_root="${ARTIFACT_STREAMING_SECURITYFS_ROOT:-/sys/kernel/security}"
+  if ! findmnt -n -t securityfs --mountpoint "$securityfs_root" >/dev/null ||
+      [ ! -r "$securityfs_root" ] || [ ! -x "$securityfs_root" ]; then
+    echo "WARNING: Cannot inspect securityfs at ${securityfs_root}; skipping the kernel IPE check." >&2
+    return 0
+  fi
+  [ -d "${securityfs_root}/ipe" ] || return 0
+
+  local policies_dir="${securityfs_root}/ipe/policies"
+  if [ ! -d "$policies_dir" ] || [ ! -r "$policies_dir" ] || [ ! -x "$policies_dir" ]; then
+    echo "WARNING: Cannot enumerate ${policies_dir}; skipping the kernel IPE check." >&2
+    return 0
+  fi
+  local policy active
+  # Audit mode still selects EROFS. Include policies with dot-prefixed names.
+  for policy in "$policies_dir"/* "$policies_dir"/.[!.]* "$policies_dir"/..?*; do
+    [ -e "$policy" ] || continue
+    if ! active="$(cat "${policy}/active")"; then
+      echo "WARNING: Cannot read ${policy}/active; skipping this IPE policy." >&2
+      continue
+    fi
+    case "$active" in
+      0) ;;
+      1)
+        echo "Artifact streaming requires OverlayBD, but ACL IPE is active and requires EROFS (for example, enabled with acl-node-security-profile=ipe=audit). Disable artifact streaming or set ipe=disabled in acl-node-security-profile (preserving other entries), then reboot before provisioning." >&2
+        return 1
+        ;;
+      *)
+        echo "WARNING: Invalid IPE policy state '${active}' in ${policy}/active; skipping this IPE policy." >&2
+        ;;
+    esac
+  done
+  return 0
+}

parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh

@@ -22,6 +22,13 @@ downloadSysextFromVersion() {
 matchLocalSysext() {
     local seName=$1 desiredVer=$2 seArch=$3
     local downloadDir="/opt/${seName}/downloads"
+    # A fully pinned DALEC version must not select a different revision.
+    case "${desiredVer}" in
+        *-azlinux3)
+            find "${downloadDir}" -maxdepth 2 -name "${seName}-v${desiredVer}-${seArch}.raw" -type f 2>/dev/null | head -n1
+            return
+            ;;
+    esac
     # Try arch-specific versioned filename first (kubelet-style: name-vVER.X-arch.raw)
     local match
     match=$(find "${downloadDir}" -maxdepth 2 -name "${seName}-v${desiredVer}*-${seArch}.raw" -type f 2>/dev/null | sort -V | tail -n1)
@@ -38,6 +45,13 @@ matchLocalSysext() {
 
 matchRemoteSysext() {
     local seURL=$1 desiredVer=$2 seArch=$3
+    # Pinned tags also work with isolated registries that cannot enumerate tags.
+    case "${desiredVer}" in
+        *-azlinux3)
+            echo "v${desiredVer}-${seArch}"
+            return 0
+            ;;
+    esac
     if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then
         # For network isolated cluster, acr cache rule does not support oras repo tags.
         # return fixed renovateTag '-1' as workaround
@@ -127,6 +141,16 @@ installCredentialProviderPackageFromBootstrapProfileRegistry() {
     installCredentialProviderFromPkg "$2" "$1"
 }
 
+installArtifactStreamingSysext() {
+    local version
+    version=$(jq -er '.Packages[] | select(.name == "acr-mirror") |
+        .downloadURIs.azurecontainerlinux.current.versionsV2[0].latestVersion |
+        select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+-[0-9]+-azlinux3$"))' \
+        "${COMPONENTS_FILEPATH:-/opt/azure/components.json}") || return "$ERR_ORAS_PULL_SYSEXT_FAIL"
+    local registry="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER:-${MCR_REPOSITORY_BASE:-mcr.microsoft.com}}"
+    mergeSysexts artifact-streaming "${registry%/}/azurecontainerregistry/v2/artifact-streaming-sysext" "${version#v}"
+}
+
 # Only called at build-time, unlike kubelet or credential provider installation.
 installSecureTLSBootstrapClientSysext() {
     local version=$1

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -635,9 +635,30 @@ EOF
 }
 
 ensureArtifactStreaming() {
-  waitForContainerdReady || exit $ERR_ARTIFACT_STREAMING_INSTALL
-
   local acr_mirror_setup="${ACR_MIRROR_SETUP_SCRIPT:-/opt/acr/tools/mirror/setup.sh}"
+  if isACL "$OS" "$OS_VARIANT"; then
+    validateACLArtifactStreamingIPECompatibility || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+    local services=(acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service)
+    # DALEC's Upholds= starts unmasked units as soon as the manager reloads.
+    systemctl mask "${services[@]}" artifact-streaming-setup.service || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+    installArtifactStreamingSysext || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+    systemctl daemon-reload || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+    systemctl unmask artifact-streaming-setup.service || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+    systemctl start artifact-streaming-setup.service || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+    # acr-config rejects ACL's absolute symlink, including a preserved GPU config.
+    if test -L /etc/containerd/config.toml; then
+      cp --remove-destination "$(readlink -f /etc/containerd/config.toml)" /etc/containerd/config.toml || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+    fi
+    "$acr_mirror_setup" aks || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+    systemctl unmask "${services[@]}" || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+    systemctl enable "${services[@]}" || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+    retrycmd_if_failure 120 5 25 systemctl restart "${services[@]}" || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+    systemctl restart containerd || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+    waitForContainerdReady || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+    return
+  fi
+  waitForContainerdReady || exit "$ERR_ARTIFACT_STREAMING_INSTALL"
+
   if [ -x "$acr_mirror_setup" ]; then
     "$acr_mirror_setup" aks || exit $ERR_ARTIFACT_STREAMING_INSTALL
   else

parts/linux/cloud-init/artifacts/cse_helpers.sh

@@ -1213,9 +1213,9 @@ getPackageJSON() {
         search=".downloadURIs.${osLowerCase}.\"${osVariant}/r${osVersion//.}\" // .downloadURIs.${osLowerCase}.\"r${osVersion//.}\" // ${search}"
     fi
 
-    # ACL is Flatcar-based; use flatcar download entries.
+    # ACL is Flatcar-based; prefer ACL-specific entries, then fall back to Flatcar.
     if isACL "${os}" "${osVariant}"; then
-        search=".downloadURIs.flatcar.current // .downloadURIs.default.current"
+        search=".downloadURIs.azurecontainerlinux.current // .downloadURIs.flatcar.current // .downloadURIs.default.current"
     fi
 
     jq -r -c "${search}" <<< "${package}"