diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index 97ab2f8e957..eb23ec140f8 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -939,6 +939,44 @@ oom_score = -999 } } +func TestACLArtifactStreamingContainerdConfig(t *testing.T) { + tests := []struct { + name string + containerdVersion string + noGPU bool + }{ + {name: "containerd v1", containerdVersion: "1.7.22"}, + {name: "containerd v1 no GPU", containerdVersion: "1.7.22", noGPU: true}, + {name: "containerd v2", containerdVersion: "2.0.0"}, + {name: "containerd v2 no GPU", containerdVersion: "2.0.0", noGPU: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config, err := containerdConfigFromAKSNodeConfig(&aksnodeconfigv1.Configuration{ + EnableArtifactStreaming: true, + }, tt.noGPU, tt.containerdVersion) + if err != nil { + t.Fatalf("containerdConfigFromAKSNodeConfig() error = %v", err) + } + + if count := strings.Count(config, `snapshotter = "overlaybd"`); count != 1 { + t.Errorf("expected exactly one overlaybd snapshotter, got %d in:\n%s", count, config) + } + for _, expected := range []string{ + `disable_snapshot_annotations = false`, + `[proxy_plugins.overlaybd]`, + `type = "snapshot"`, + `address = "/run/overlaybd-snapshotter/overlaybd.sock"`, + } { + if !strings.Contains(config, expected) { + t.Errorf("expected rendered config to contain %q:\n%s", expected, config) + } + } + }) + } +} + func Test_getKubenetTemplate(t *testing.T) { tests := []struct { name string diff --git a/aks-node-controller/pkg/nodeconfigutils/utils.go b/aks-node-controller/pkg/nodeconfigutils/utils.go index e7338bc0326..6798519f344 100644 --- a/aks-node-controller/pkg/nodeconfigutils/utils.go +++ b/aks-node-controller/pkg/nodeconfigutils/utils.go @@ -69,7 +69,7 @@ fi` "files": [{ "path": "/opt/azure/containers/aks-node-controller-config.json", "mode": 384, - "contents": { "source": "data:;base64,%s" } + "contents": { "source": "data:;base64,%[1]s" } }] } }` diff --git a/aks-node-controller/pkg/nodeconfigutils/utils_test.go b/aks-node-controller/pkg/nodeconfigutils/utils_test.go index 5bea3f89bdc..2653c3bac71 100644 --- a/aks-node-controller/pkg/nodeconfigutils/utils_test.go +++ b/aks-node-controller/pkg/nodeconfigutils/utils_test.go @@ -2,6 +2,7 @@ package nodeconfigutils import ( "encoding/base64" + "encoding/json" "io" "mime" "mime/multipart" @@ -44,6 +45,18 @@ func TestContainerdImagePullPresenceRoundTrip(t *testing.T) { } } +func decodeFlatcarCustomData(t *testing.T, cfg *aksnodeconfigv1.Configuration) map[string]any { + t.Helper() + customData, err := CustomDataFlatcar(cfg) + require.NoError(t, err) + decoded, err := base64.StdEncoding.DecodeString(customData) + require.NoError(t, err) + + var ignition map[string]any + require.NoError(t, json.Unmarshal(decoded, &ignition)) + return ignition +} + func TestUnmarshalConfigurationV1(t *testing.T) { tests := []struct { name string @@ -300,6 +313,31 @@ func TestCustomDataUsesMultipartBoothookAndCloudConfig(t *testing.T) { require.ErrorIs(t, err, io.EOF) } +func TestCustomDataFlatcarDefersArtifactStreamingToProvisioning(t *testing.T) { + for _, enabled := range []bool{false, true} { + cfg := &aksnodeconfigv1.Configuration{EnableArtifactStreaming: enabled} + ignition := decodeFlatcarCustomData(t, cfg) + require.NotContains(t, ignition, "systemd", "Ignition must not start streaming before containerd is configured") + + storage, ok := ignition["storage"].(map[string]any) + require.True(t, ok) + files, ok := storage["files"].([]any) + require.True(t, ok) + require.NotEmpty(t, files) + file, ok := files[0].(map[string]any) + require.True(t, ok) + contents, ok := file["contents"].(map[string]any) + require.True(t, ok) + source, ok := contents["source"].(string) + require.True(t, ok) + configJSON, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(source, "data:;base64,")) + require.NoError(t, err) + decoded, err := UnmarshalConfigurationV1(configJSON) + require.NoError(t, err) + require.Equal(t, enabled, decoded.GetEnableArtifactStreaming()) + } +} + // decodeBoothook extracts the decoded cloud-boothook part from the base64-encoded custom data. func decodeBoothook(t *testing.T, cfg *aksnodeconfigv1.Configuration) string { t.Helper() diff --git a/e2e/scenario/scenario.go b/e2e/scenario/scenario.go index c92611b1167..a2246f1bcfb 100755 --- a/e2e/scenario/scenario.go +++ b/e2e/scenario/scenario.go @@ -73,11 +73,67 @@ var _ = Register(&Scenario{ return errors.Join( ValidateFileHasContent(ctx, s, "/etc/os-release", "ID=azurelinux"), ValidateFileHasContent(ctx, s, "/etc/os-release", "VARIANT_ID=azurecontainerlinux"), + ValidateFileDoesNotExist(ctx, s, "/etc/extensions/artifact-streaming.raw"), + ValidateSystemdUnitIsNotRunning(ctx, s, "artifact-streaming-setup.service"), + ValidateSystemdUnitIsNotRunning(ctx, s, "acr-mirror.service"), + ValidateSystemdUnitIsNotRunning(ctx, s, "overlaybd-tcmu.service"), + ValidateSystemdUnitIsNotRunning(ctx, s, "overlaybd-snapshotter.service"), ) }, }, }) +// Artifact streaming changes both the node bootstrap input and the services activated on the +// node, so this requires a separate node from the baseline ACL scenario. +var _ = Register(&Scenario{ + Name: "ACL_ArtifactStreaming", + Description: "Tests that an ACL node with artifact streaming enabled activates OverlayBD and acr-mirror", + SkipIf: skipACLArtifactStreamingOnMainBuiltImage, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDACLGen2TL, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.EnableArtifactStreaming = true + }, + AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { + config.EnableArtifactStreaming = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) + }, + Validator: func(ctx context.Context, s *Scenario) error { + return errors.Join( + ValidateFileHasContent(ctx, s, "/etc/os-release", "ID=azurelinux"), + ValidateFileHasContent(ctx, s, "/etc/os-release", "VARIANT_ID=azurecontainerlinux"), + ValidateFileExists(ctx, s, "/etc/extensions/artifact-streaming.raw"), + ValidateFileDoesNotExist(ctx, s, "/etc/extensions/overlaybd.raw"), + ValidateFileIsRegularFile(ctx, s, "/etc/containerd/config.toml"), + ValidateFileExists(ctx, s, "/etc/acr/acr.toml"), + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd"), + ValidateFileExists(ctx, s, "/etc/overlaybd-snapshotter/config.json"), + ValidateSystemdUnitIsRunning(ctx, s, "artifact-streaming-setup.service"), + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service"), + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service"), + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service"), + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service"), + ) + }, + }, +}) + +// ACL e2e provisions with the CSE scripts baked into the VHD rather than the branch's scripts, +// so the ACL artifact streaming path only exists on VHDs built from a branch that contains it. +func skipACLArtifactStreamingOnMainBuiltImage(context.Context) string { + if laneResolvedMainBuiltImage() { + return fmt.Sprintf( + "this lane resolved a main-built image (%s=%s), which predates ACL artifact streaming support baked into the VHD; run against the PR's VHD build to exercise it", + config.Config.SIGVersionTagName, + config.Config.SIGVersionTagValue, + ) + } + return "" +} + var _ = Register(&Scenario{ Name: "ACL_CustomCA", Description: "Tests that a node using an ACL VHD can be properly bootstrapped and custom CA was correctly added", diff --git a/parts/common/components.json b/parts/common/components.json index 76c81bd8237..6c68d8eb5b6 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -2526,6 +2526,17 @@ } ] } + }, + "azurecontainerlinux": { + "current": { + "versionsV2": [ + { + "renovateTag": "OCI_registry=https://mcr.microsoft.com, name=azurecontainerregistry/v2/artifact-streaming-sysext", + "latestVersion": "v1.0.0-1-azlinux3" + } + ], + "downloadURL": "mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext:${version}-${SYSTEMD_ARCH}" + } } } }, diff --git a/parts/linux/cloud-init/artifacts/README-COMPONENTS.md b/parts/linux/cloud-init/artifacts/README-COMPONENTS.md index feb4f8fbd9f..b2163dc3d51 100644 --- a/parts/linux/cloud-init/artifacts/README-COMPONENTS.md +++ b/parts/linux/cloud-init/artifacts/README-COMPONENTS.md @@ -93,6 +93,7 @@ Please refer to [components.cue](../../../../schemas/components.cue) for the mos mariner?: #MarinerOSDistro marinerkata?: #MarinerOSDistro azurelinux?: #AzureLinuxOSDistro + azurecontainerlinux?: #AzureContainerLinuxOSDistro } ``` @@ -115,6 +116,9 @@ Please refer to [components.cue](../../../../schemas/components.cue) for the mos "OSGUARD/v3.0"?: #ReleaseDownloadURI current?: #ReleaseDownloadURI } +#AzureContainerLinuxOSDistro: { + current?: #ReleaseDownloadURI +} ``` ``` @@ -139,6 +143,7 @@ Here are the explanation of the above schema. - In `UbuntuOSDistro`, we can define different OS release versions. For example, `r2404` implies release 24.04. - In `MarinerOSDistro`, we only have `current` now, which implies that single configurations will be applied to all Mariner release versions. We can distinguish them in needed. Note, we confirmed with Mariner team, Azure Linux 2.0 is reporting itself as `mariner` in the file `/etc/os-release`. So for Azure Linux 2.0 case, it will still read the package versions from `mariner` block. - In `AzureLinuxOSDistro`, `v3.0` is for Azure Linux v3.0. `current` is for otherwise but we are not using it now. Azure Linux 2.0 case is described in the `MarinerOSDistro` above. + - In `AzureContainerLinuxOSDistro`, `current` defines ACL-specific metadata. When it is absent, ACL falls back to `flatcar.current`, then `default.current`. - `DefaultOSDistro` means the default case of OS Distro. If an OSDistro metadata is not defined, it will fetch it from `default`. For example, if a node is Ubuntu 20.04, but we don't specify `ubuntu` in components.json, then it will fetch `default.current`. For another example, if only `default.current` is specified in the components.json, No matter what OSDistro is the node running, it will only fetch `default.current` because it's the default metadata. This provides flexibility while elimiating unnecessary duplication when defining the metadata. 1. The OS release version can optionally be prefixed with the OS variant. `OSGUARD/v3.0` and `DEFAULT/v3.0` are currently supported for `azurelinux`. The latter specifically matches Azure Linux 3.0 without a variant, i.e. not OS Guard. 1. In `ReleaseDownloadURI`, you can see 2 keys. diff --git a/parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh b/parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh index 0bf141f7cb2..24cec587b3d 100644 --- a/parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh +++ b/parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh @@ -21,3 +21,77 @@ apt_get_update() { apt_get_dist_upgrade() { stub } + +# Temporary provisioning-only guard until EROFS supports artifact streaming. +# Fail only for detected IPE conflicts; unavailable checks warn and continue. +# Later IPE enablement is not covered. +validateACLArtifactStreamingSecurityProfile() { + local profile_cache="${ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE:-/run/acl/node-security-profile}" + if [ ! -e "$profile_cache" ]; then + echo "WARNING: No cached ACL security profile; skipping the IPE tag check." >&2 + return 0 + fi + + local profile pair key mode + local -a pairs + if ! profile="$(cat "$profile_cache")"; then + echo "WARNING: Cannot read ${profile_cache}; skipping the IPE tag check." >&2 + return 0 + fi + # Match ACL's initrd parser: comma-separated pairs, first matching key wins. + IFS=',' read -ra pairs <<< "$profile" + for pair in "${pairs[@]}"; do + IFS='=' read -r key mode <<< "$pair" + [ "$key" = "ipe" ] || continue + case "$mode" in + audit|enforcing) + echo "Artifact streaming requires OverlayBD, but the boot-time tag requests acl-node-security-profile=ipe=${mode}. Disable artifact streaming or set ipe=disabled in acl-node-security-profile (preserving other entries), then reboot before provisioning." >&2 + return 1 + ;; + disabled|off|"") ;; + *) + echo "WARNING: Unrecognized cached IPE mode '${mode}'; skipping the IPE tag check." >&2 + ;; + esac + break + done + return 0 +} + +validateACLArtifactStreamingIPECompatibility() { + validateACLArtifactStreamingSecurityProfile || return 1 + + local securityfs_root="${ARTIFACT_STREAMING_SECURITYFS_ROOT:-/sys/kernel/security}" + if ! findmnt -n -t securityfs --mountpoint "$securityfs_root" >/dev/null || + [ ! -r "$securityfs_root" ] || [ ! -x "$securityfs_root" ]; then + echo "WARNING: Cannot inspect securityfs at ${securityfs_root}; skipping the kernel IPE check." >&2 + return 0 + fi + [ -d "${securityfs_root}/ipe" ] || return 0 + + local policies_dir="${securityfs_root}/ipe/policies" + if [ ! -d "$policies_dir" ] || [ ! -r "$policies_dir" ] || [ ! -x "$policies_dir" ]; then + echo "WARNING: Cannot enumerate ${policies_dir}; skipping the kernel IPE check." >&2 + return 0 + fi + local policy active + # Audit mode still selects EROFS. Include policies with dot-prefixed names. + for policy in "$policies_dir"/* "$policies_dir"/.[!.]* "$policies_dir"/..?*; do + [ -e "$policy" ] || continue + if ! active="$(cat "${policy}/active")"; then + echo "WARNING: Cannot read ${policy}/active; skipping this IPE policy." >&2 + continue + fi + case "$active" in + 0) ;; + 1) + echo "Artifact streaming requires OverlayBD, but ACL IPE is active and requires EROFS (for example, enabled with acl-node-security-profile=ipe=audit). Disable artifact streaming or set ipe=disabled in acl-node-security-profile (preserving other entries), then reboot before provisioning." >&2 + return 1 + ;; + *) + echo "WARNING: Invalid IPE policy state '${active}' in ${policy}/active; skipping this IPE policy." >&2 + ;; + esac + done + return 0 +} diff --git a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh index f3db5805620..10619e4a95f 100644 --- a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh +++ b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh @@ -22,6 +22,13 @@ downloadSysextFromVersion() { matchLocalSysext() { local seName=$1 desiredVer=$2 seArch=$3 local downloadDir="/opt/${seName}/downloads" + # A fully pinned DALEC version must not select a different revision. + case "${desiredVer}" in + *-azlinux3) + find "${downloadDir}" -maxdepth 2 -name "${seName}-v${desiredVer}-${seArch}.raw" -type f 2>/dev/null | head -n1 + return + ;; + esac # Try arch-specific versioned filename first (kubelet-style: name-vVER.X-arch.raw) local match match=$(find "${downloadDir}" -maxdepth 2 -name "${seName}-v${desiredVer}*-${seArch}.raw" -type f 2>/dev/null | sort -V | tail -n1) @@ -38,6 +45,13 @@ matchLocalSysext() { matchRemoteSysext() { local seURL=$1 desiredVer=$2 seArch=$3 + # Pinned tags also work with isolated registries that cannot enumerate tags. + case "${desiredVer}" in + *-azlinux3) + echo "v${desiredVer}-${seArch}" + return 0 + ;; + esac if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then # For network isolated cluster, acr cache rule does not support oras repo tags. # return fixed renovateTag '-1' as workaround @@ -127,6 +141,16 @@ installCredentialProviderPackageFromBootstrapProfileRegistry() { installCredentialProviderFromPkg "$2" "$1" } +installArtifactStreamingSysext() { + local version + version=$(jq -er '.Packages[] | select(.name == "acr-mirror") | + .downloadURIs.azurecontainerlinux.current.versionsV2[0].latestVersion | + select(test("^v[0-9]+\\.[0-9]+\\.[0-9]+-[0-9]+-azlinux3$"))' \ + "${COMPONENTS_FILEPATH:-/opt/azure/components.json}") || return "$ERR_ORAS_PULL_SYSEXT_FAIL" + local registry="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER:-${MCR_REPOSITORY_BASE:-mcr.microsoft.com}}" + mergeSysexts artifact-streaming "${registry%/}/azurecontainerregistry/v2/artifact-streaming-sysext" "${version#v}" +} + # Only called at build-time, unlike kubelet or credential provider installation. installSecureTLSBootstrapClientSysext() { local version=$1 diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 91d0ece29f6..fbaf41ed09a 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -635,9 +635,30 @@ EOF } ensureArtifactStreaming() { - waitForContainerdReady || exit $ERR_ARTIFACT_STREAMING_INSTALL - local acr_mirror_setup="${ACR_MIRROR_SETUP_SCRIPT:-/opt/acr/tools/mirror/setup.sh}" + if isACL "$OS" "$OS_VARIANT"; then + validateACLArtifactStreamingIPECompatibility || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + local services=(acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service) + # DALEC's Upholds= starts unmasked units as soon as the manager reloads. + systemctl mask "${services[@]}" artifact-streaming-setup.service || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + installArtifactStreamingSysext || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + systemctl daemon-reload || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + systemctl unmask artifact-streaming-setup.service || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + systemctl start artifact-streaming-setup.service || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + # acr-config rejects ACL's absolute symlink, including a preserved GPU config. + if test -L /etc/containerd/config.toml; then + cp --remove-destination "$(readlink -f /etc/containerd/config.toml)" /etc/containerd/config.toml || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + fi + "$acr_mirror_setup" aks || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + systemctl unmask "${services[@]}" || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + systemctl enable "${services[@]}" || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + retrycmd_if_failure 120 5 25 systemctl restart "${services[@]}" || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + systemctl restart containerd || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + waitForContainerdReady || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + return + fi + waitForContainerdReady || exit "$ERR_ARTIFACT_STREAMING_INSTALL" + if [ -x "$acr_mirror_setup" ]; then "$acr_mirror_setup" aks || exit $ERR_ARTIFACT_STREAMING_INSTALL else diff --git a/parts/linux/cloud-init/artifacts/cse_helpers.sh b/parts/linux/cloud-init/artifacts/cse_helpers.sh index b1c01fd5031..a088652e6e6 100755 --- a/parts/linux/cloud-init/artifacts/cse_helpers.sh +++ b/parts/linux/cloud-init/artifacts/cse_helpers.sh @@ -1213,9 +1213,9 @@ getPackageJSON() { search=".downloadURIs.${osLowerCase}.\"${osVariant}/r${osVersion//.}\" // .downloadURIs.${osLowerCase}.\"r${osVersion//.}\" // ${search}" fi - # ACL is Flatcar-based; use flatcar download entries. + # ACL is Flatcar-based; prefer ACL-specific entries, then fall back to Flatcar. if isACL "${os}" "${osVariant}"; then - search=".downloadURIs.flatcar.current // .downloadURIs.default.current" + search=".downloadURIs.azurecontainerlinux.current // .downloadURIs.flatcar.current // .downloadURIs.default.current" fi jq -r -c "${search}" <<< "${package}" diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index 25e750f832f..110fa21526c 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -398,7 +398,9 @@ EOF fi fi - if [ "${ARTIFACT_STREAMING_ENABLED}" = "true" ]; then + # Defer ACL artifact streaming to nodePrep: keep sysext services masked + # during PIS image preparation and activate them on the real node. + if [ "${ARTIFACT_STREAMING_ENABLED}" = "true" ] && ! isACL "$OS" "$OS_VARIANT"; then logs_to_events "AKS.CSE.ensureContainerd.ensureArtifactStreaming" ensureArtifactStreaming || exit $ERR_ARTIFACT_STREAMING_INSTALL fi @@ -623,6 +625,11 @@ function nodePrep { addKubeletNodeLabel "kubernetes.azure.com/localdns-exporter=enabled" fi + # Keep ACL sysexts masked during PIS image preparation; configure them on the real node. + if [ "${ARTIFACT_STREAMING_ENABLED}" = "true" ] && isACL "$OS" "$OS_VARIANT"; then + logs_to_events "AKS.CSE.ensureContainerd.ensureArtifactStreaming" ensureArtifactStreaming || exit $ERR_ARTIFACT_STREAMING_INSTALL + fi + logs_to_events "AKS.CSE.ensureKubelet" ensureKubelet # Configure localdns metrics exporter socket after ensureKubelet. diff --git a/schemas/components.cue b/schemas/components.cue index 89a9fe9a062..d22ad74c6ec 100644 --- a/schemas/components.cue +++ b/schemas/components.cue @@ -100,6 +100,10 @@ package components current?: #ReleaseDownloadURI } +#AzureContainerLinuxOSDistro: { + current?: #ReleaseDownloadURI +} + #DownloadURIs: { default?: #DefaultOSDistro ubuntu?: #UbuntuOSDistro @@ -109,6 +113,7 @@ package components azurelinuxkata?: #AzureLinuxOSDistro windows?: #WindowsOsDistro flatcar?: #FlatcarOSDistro + azurecontainerlinux?: #AzureContainerLinuxOSDistro } #Package: { diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh index 6651ec586fa..1353c67d5d3 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh @@ -130,6 +130,7 @@ Describe 'cse_config_gpu.sh' End End Describe 'ensureArtifactStreaming' + Include "./parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh" # ensureArtifactStreaming runs the version-appropriate enablement path # before enabling the acr-mirror/overlaybd services: # - acr-mirror 1.0.0+ -> setup.sh aks @@ -140,6 +141,12 @@ Describe 'cse_config_gpu.sh' TEST_ACR_DIR="$(mktemp -d)" ACR_MIRROR_SETUP_SCRIPT="${TEST_ACR_DIR}/setup.sh" ACR_CONFIG_BIN="${TEST_ACR_DIR}/acr-config" + ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE="${TEST_ACR_DIR}/node-security-profile" + ARTIFACT_STREAMING_SECURITYFS_ROOT="${TEST_ACR_DIR}/security" + touch "$ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE" + mkdir -p "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/policies" + OS="$UBUNTU_OS_NAME" + OS_VARIANT="" } cleanup_streaming() { rm -rf "${TEST_ACR_DIR}" @@ -150,9 +157,20 @@ Describe 'cse_config_gpu.sh' waitForContainerdReady() { return "${CONTAINERD_READY_RC:-0}" } + findmnt() { + [ "$*" = "-n -t securityfs --mountpoint ${ARTIFACT_STREAMING_SECURITYFS_ROOT}" ] || return 1 + return "${FINDMNT_RC:-0}" + } systemctl() { + if [ "${FAIL_SYSTEMCTL:-}" = "$*" ]; then + return 1 + fi echo "systemctl $@" } + installArtifactStreamingSysext() { + echo "installArtifactStreamingSysext" + return "${SYSEXT_RC:-0}" + } retrycmd_if_failure() { echo "retrycmd_if_failure $@" return "${RETRYCMD_RC:-0}" @@ -167,6 +185,220 @@ Describe 'cse_config_gpu.sh' chmod +x "${ACR_CONFIG_BIN}" } + Describe 'best-effort ACL IPE compatibility' + BeforeEach 'install_setup_sh' + + set_cached_profile() { + case "$1" in + missing) rm "$ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE" ;; + unreadable) PROFILE_READ_FAIL=true ;; + *) printf '%s' "$1" > "$ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE" ;; + esac + } + cat() { + if [ "$1" = "$ARTIFACT_STREAMING_SECURITY_PROFILE_CACHE" ] && [ "${PROFILE_READ_FAIL:-false}" = true ]; then + return 1 + fi + command cat "$@" + } + write_ipe_policy() { + mkdir -p "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/policies/$1" + printf '%s\n' "$2" > "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/policies/$1/active" + } + + Describe 'cached intent conflicts' + Parameters + "$ACL_OS_NAME" "" "ipe=audit" 0 audit + "$ACL_OS_NAME" "" "selinux=enforcing,ipe=enforcing" 1 enforcing + "$AZURELINUX_OS_NAME" "$ACL_OS_VARIANT" "ipe=audit,selinux=permissive" 1 audit + End + + It "rejects '$3' before side effects even with inactive or unavailable kernel state" + OS="$1" + OS_VARIANT="$2" + set_cached_profile "$3" + FINDMNT_RC="$4" + write_ipe_policy "inactive" 0 + + When run ensureArtifactStreaming + + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + The output should be blank + The stderr should include "boot-time tag requests acl-node-security-profile=ipe=$5" + The stderr should include "set ipe=disabled in acl-node-security-profile (preserving other entries)" + End + End + + Describe 'active kernel policies' + Parameters + "ipe=disabled" 0 + "ipe=off" 1 + missing 0 + unreadable 1 + "ipe=unknown" 0 + End + + It "rejects active IPE with cache='$1' and enforce=$2" + OS="$ACL_OS_NAME" + set_cached_profile "$1" + printf '%s\n' "$2" > "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/enforce" + write_ipe_policy "a_inactive" 0 + write_ipe_policy ".active" 1 + + When run ensureArtifactStreaming + + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + The output should be blank + The stderr should include "ACL IPE is active and requires EROFS" + The stderr should include "for example, enabled with acl-node-security-profile=ipe=audit" + The stderr should include "set ipe=disabled in acl-node-security-profile (preserving other entries)" + End + End + + Describe 'available nonconflicting state' + Parameters + "ipe=disabled" + "selinux=enforcing,ipe=off" + "" + "selinux=enforcing" + "ipe=off,ipe=audit" + End + + It "allows '$1' with loaded but inactive IPE" + OS="$ACL_OS_NAME" + set_cached_profile "$1" + printf '1\n' > "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/enforce" + write_ipe_policy "inactive" 0 + + When run ensureArtifactStreaming + + The status should be success + The output should include "setup.sh aks" + The stderr should be blank + End + End + + Describe 'unavailable or unknown sources' + Parameters + missing 0 "No cached ACL security profile" + missing 1 "skipping the kernel IPE check" + unreadable 0 "skipping the IPE tag check" + unreadable 1 "skipping the kernel IPE check" + "ipe=disabled" 1 "skipping the kernel IPE check" + "ipe=unknown" 0 "Unrecognized cached IPE mode" + End + + It "warns and continues for cache='$1' and kernel lookup status=$2" + OS="$ACL_OS_NAME" + set_cached_profile "$1" + FINDMNT_RC="$2" + + When run ensureArtifactStreaming + + The status should be success + The output should include "setup.sh aks" + The stderr should include "WARNING:" + The stderr should include "$3" + End + End + + It 'warns and continues when policies cannot be enumerated' + OS="$ACL_OS_NAME" + rmdir "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/policies" + + When run ensureArtifactStreaming + + The status should be success + The output should include "setup.sh aks" + The stderr should include "WARNING: Cannot enumerate" + End + + It 'allows older kernels without IPE on mounted securityfs' + OS="$ACL_OS_NAME" + rmdir "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/policies" "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe" + + When run ensureArtifactStreaming + + The status should be success + The output should include "setup.sh aks" + The stderr should be blank + End + + Describe 'individual policy read failures' + Parameters + 0 0 + 1 "$ERR_ARTIFACT_STREAMING_INSTALL" + End + + It "continues past unreadable and invalid entries to a later policy with active=$1" + OS="$ACL_OS_NAME" + mkdir -p "${ARTIFACT_STREAMING_SECURITYFS_ROOT}/ipe/policies/a_unreadable" + write_ipe_policy "b_invalid" unexpected + write_ipe_policy ".later" "$1" + + When run ensureArtifactStreaming + + The status should equal "$2" + The stderr should include "WARNING: Cannot read" + The stderr should include "WARNING: Invalid IPE policy state" + if [ "$1" = 1 ]; then + The output should be blank + The stderr should include "ACL IPE is active and requires EROFS" + else + The output should include "setup.sh aks" + fi + End + End + + It 'does not apply ACL checks to Ubuntu' + set_cached_profile "ipe=audit" + write_ipe_policy "active" 1 + + When run ensureArtifactStreaming + + The status should be success + The output should include "setup.sh aks" + The stderr should be blank + End + + Describe 'nodePrep activation on PIS nodes' + logs_to_events() { + shift + "$@" + } + ensureKubelet() { echo "ensureKubelet"; } + run_node_prep_streaming() { + eval "$(sed -n '/# Keep ACL sysexts masked during PIS image preparation/,/logs_to_events "AKS.CSE.ensureKubelet" ensureKubelet/p' \ + ./parts/linux/cloud-init/artifacts/cse_main.sh)" + } + + It 'propagates a detected conflict before kubelet starts' + OS="$ACL_OS_NAME" + ARTIFACT_STREAMING_ENABLED=true + write_ipe_policy "active" 1 + + When run run_node_prep_streaming + + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + The output should be blank + The stderr should include "ACL IPE is active and requires EROFS" + End + + It 'does not reject IPE when streaming is disabled' + OS="$ACL_OS_NAME" + ARTIFACT_STREAMING_ENABLED=false + set_cached_profile "ipe=audit" + write_ipe_policy "active" 1 + + When run run_node_prep_streaming + + The status should be success + The output should equal "ensureKubelet" + The stderr should be blank + End + End + End + It 'uses setup.sh aks when acr-mirror 1.0.0+ is installed' install_setup_sh When run ensureArtifactStreaming @@ -185,6 +417,80 @@ Describe 'cse_config_gpu.sh' The status should be success End + Describe 'initializes the ACL sysext before unmasking consumers' + Parameters + "AZURECONTAINERLINUX" "" + "AZURELINUX" "AZURECONTAINERLINUX" + End + It "orders initialization on $1 $2" + OS="$1" + OS_VARIANT="$2" + install_setup_sh + When run ensureArtifactStreaming + The output should equal "$(cat <<'EXPECTED' +systemctl mask acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service artifact-streaming-setup.service +installArtifactStreamingSysext +systemctl daemon-reload +systemctl unmask artifact-streaming-setup.service +systemctl start artifact-streaming-setup.service +setup.sh aks +systemctl unmask acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service +systemctl enable acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service +retrycmd_if_failure 120 5 25 systemctl restart acr-mirror.service overlaybd-tcmu.service overlaybd-snapshotter.service +systemctl restart containerd +EXPECTED +)" + The status should be success + End + End + + It 'materializes a preserved ACL containerd config before configuring ACR' + OS="$ACL_OS_NAME" + install_setup_sh + test() { + if [ "$*" = "-L /etc/containerd/config.toml" ]; then + return 0 + fi + builtin test "$@" + } + readlink() { echo "/usr/share/containerd/config.toml"; } + cp() { echo "cp $*"; } + When run ensureArtifactStreaming + The output should include "cp --remove-destination /usr/share/containerd/config.toml /etc/containerd/config.toml" + The output should include "setup.sh aks" + The status should be success + End + + It 'leaves consumers masked when sysext initialization fails' + OS="$ACL_OS_NAME" + FAIL_SYSTEMCTL="start artifact-streaming-setup.service" + install_setup_sh + When run ensureArtifactStreaming + The output should not include "setup.sh aks" + The output should not include "systemctl unmask acr-mirror.service" + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + End + + It 'leaves consumers masked when ACR configuration fails' + OS="$ACL_OS_NAME" + export SETUP_RC=1 + install_setup_sh + When run ensureArtifactStreaming + The output should include "setup.sh aks" + The output should not include "systemctl unmask acr-mirror.service" + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + End + + It 'fails before initialization if the sysext cannot be merged' + OS="$ACL_OS_NAME" + SYSEXT_RC=1 + install_setup_sh + When run ensureArtifactStreaming + The output should not include "systemctl daemon-reload" + The output should not include "setup.sh aks" + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + End + It 'fails fast when enabling the streaming services fails' RETRYCMD_RC=1 install_setup_sh diff --git a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh index 905f47c78cf..55978630ea0 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh @@ -85,6 +85,21 @@ Describe 'cse_helpers.sh' When call updatePackageVersions "$package" "MARINER" "current" The variable PACKAGE_VERSIONS[@] should equal "dummyVersion5 dummyVersion6.1 dummyVersion6.0" End + It 'returns the ACL-specific version for Azure Container Linux' + package=$(readPackage "pkgVersionsV2") + When call updatePackageVersions "$package" "AZURECONTAINERLINUX" "3.0" + The variable PACKAGE_VERSIONS[@] should equal "dummyVersionACL" + End + It 'returns the ACL-specific version for the Azure Linux ACL variant' + package=$(readPackage "pkgVersionsV2") + When call updatePackageVersions "$package" "AZURELINUX" "3.0" "AZURECONTAINERLINUX" + The variable PACKAGE_VERSIONS[@] should equal "dummyVersionACL" + End + It 'falls back to the Flatcar version when no ACL-specific version exists' + package=$(readPackage "pkgVersionsV2" | jq 'del(.downloadURIs.azurecontainerlinux)') + When call updatePackageVersions "$package" "AZURECONTAINERLINUX" "3.0" + The variable PACKAGE_VERSIONS[@] should equal "dummyVersionFlatcar" + End It 'returns if there is a in latestVersion' package=$(readPackage "pkgVersionsV2") When call updatePackageVersions "$package" "MARINERKATA" "current" @@ -123,6 +138,11 @@ Describe 'cse_helpers.sh' When call updatePackageDownloadURL "$package" "UBUNTU" "dummy_release" The variable PACKAGE_DOWNLOAD_URL should equal "https://dummydefaultcurrentpath/v\${version}/dummy_\${version}_linux_\${CPU_ARCH}.tar.gz" End + It 'returns the ACL-specific download URL for Azure Container Linux' + package=$(readPackage "pkgVersionsV2") + When call updatePackageDownloadURL "$package" "AZURECONTAINERLINUX" "3.0" + The variable PACKAGE_DOWNLOAD_URL should equal "https://dummypath/dummy-acl.rpm" + End End Describe 'evalPackageDownloadURL' diff --git a/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh index 76770aeb04c..6c705178b5e 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_install_acl_spec.sh @@ -44,6 +44,91 @@ Describe 'cse_install_acl.sh' Include "./parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh" Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" + Describe 'pinned DALEC sysext versions' + Describe 'remote resolution' + Parameters + "" "x86-64" + "" "arm64" + "cache.azurecr.io/aks-managed-repository" "x86-64" + "cache.azurecr.io/aks-managed-repository" "arm64" + End + It "retains revision 1 for $2 with registry '$1' without listing tags" + BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="$1" + retrycmd_silent() { echo "unexpected tag listing" >&2; return 1; } + When call matchRemoteSysext "mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext" "1.0.0-1-azlinux3" "$2" + The output should equal "v1.0.0-1-azlinux3-$2" + The status should be success + End + End + + Describe 'local resolution' + setup_cache() { + SYSEXT_CACHE=$(mktemp -d) + touch "$SYSEXT_CACHE/artifact-streaming-v1.0.0-1-azlinux3-x86-64.raw" + touch "$SYSEXT_CACHE/artifact-streaming-v1.0.0-20-azlinux3-x86-64.raw" + touch "$SYSEXT_CACHE/artifact-streaming.raw" + } + cleanup_cache() { + rm -f "$SYSEXT_CACHE/artifact-streaming-v1.0.0-1-azlinux3-x86-64.raw" \ + "$SYSEXT_CACHE/artifact-streaming-v1.0.0-20-azlinux3-x86-64.raw" \ + "$SYSEXT_CACHE/artifact-streaming.raw" + rmdir "$SYSEXT_CACHE" + } + BeforeEach 'setup_cache' + AfterEach 'cleanup_cache' + find() { + shift + command find "$SYSEXT_CACHE" "$@" + } + + It 'selects the exact cached revision rather than a later prefix match' + When call matchLocalSysext artifact-streaming "1.0.0-1-azlinux3" x86-64 + The output should equal "$SYSEXT_CACHE/artifact-streaming-v1.0.0-1-azlinux3-x86-64.raw" + The status should be success + End + + It 'does not use an unversioned or different-revision image for a missing pin' + When call matchLocalSysext artifact-streaming "1.0.0-3-azlinux3" x86-64 + The output should equal "" + The status should be success + End + End + End + + Describe 'installArtifactStreamingSysext' + COMPONENTS_FILEPATH="parts/common/components.json" + mergeSysexts() { echo "mergeSysexts $*"; return "${MERGE_RC:-0}"; } + Describe 'registry selection' + Parameters + "" "" "mcr.microsoft.com" + "" "mcr.azure.cn/" "mcr.azure.cn" + "cache.azurecr.io/aks-managed-repository" "mcr.microsoft.com" "cache.azurecr.io/aks-managed-repository" + End + It "uses $3 and the version pinned in the VHD" + BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="$1" + MCR_REPOSITORY_BASE="$2" + version=$(jq -r '.Packages[] | select(.name == "acr-mirror") | .downloadURIs.azurecontainerlinux.current.versionsV2[0].latestVersion' "$COMPONENTS_FILEPATH") + When call installArtifactStreamingSysext + The output should equal "mergeSysexts artifact-streaming $3/azurecontainerregistry/v2/artifact-streaming-sysext ${version#v}" + The status should be success + End + End + + It 'fails when the VHD does not declare a sysext version' + COMPONENTS_FILEPATH="spec/parts/linux/cloud-init/artifacts/test_components.json" + When call installArtifactStreamingSysext + The output should equal "" + The status should equal "$ERR_ORAS_PULL_SYSEXT_FAIL" + End + + It 'propagates activation failure' + MERGE_RC=1 + When call installArtifactStreamingSysext + The output should include "mergeSysexts artifact-streaming" + The status should be failure + End + End + Describe 'installKubeletKubectlFromPkg' It 'masks the kubelet sysext Upholds drop-in before activating the sysext' MASK_CREATED=false diff --git a/spec/parts/linux/cloud-init/artifacts/cse_main_pis_reconcile_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_main_pis_reconcile_spec.sh index 85934425ff9..530a242a099 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_main_pis_reconcile_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_main_pis_reconcile_spec.sh @@ -177,6 +177,44 @@ Describe 'cse_main.sh PIS-safe configuration' End End + Describe 'artifact streaming' + streaming_call='AKS.CSE.ensureContainerd.ensureArtifactStreaming' + + It 'activates ACL streaming only in nodePrep so PIS images keep sysexts masked' + streaming_gates() { + phase_count "basePrep" "${streaming_call}" + phase_count "nodePrep" "${streaming_call}" + script_count "${streaming_call}" + phase_body "basePrep" | code_lines | grep -B1 -- "${streaming_call}" | head -1 + phase_body "nodePrep" | code_lines | grep -B1 -- "${streaming_call}" | head -1 + } + When call streaming_gates + The line 1 of output should equal "1" + The line 2 of output should equal "1" + The line 3 of output should equal "2" + The line 4 of output should equal ' if [ "${ARTIFACT_STREAMING_ENABLED}" = "true" ] && ! isACL "$OS" "$OS_VARIANT"; then' + The line 5 of output should equal ' if [ "${ARTIFACT_STREAMING_ENABLED}" = "true" ] && isACL "$OS" "$OS_VARIANT"; then' + End + + It 'activates ACL streaming before kubelet startup' + streaming_order() { + phase_body "nodePrep" | code_lines | grep -E "${streaming_call}|AKS.CSE.ensureKubelet\"" + } + When call streaming_order + The line 1 of output should include "${streaming_call}" + The line 2 of output should include 'AKS.CSE.ensureKubelet' + The lines of output should equal 2 + End + + It 'fails provisioning when ACL streaming cannot be activated' + streaming_call_line() { + phase_body "nodePrep" | code_lines | grep -- "${streaming_call}" + } + When call streaming_call_line + The output should include 'ensureArtifactStreaming || exit $ERR_ARTIFACT_STREAMING_INSTALL' + End + End + Describe 'stage gate' It 'configures Chrony before either provisioning stage' When call chrony_dispatch_calls diff --git a/spec/parts/linux/cloud-init/artifacts/test_components.json b/spec/parts/linux/cloud-init/artifacts/test_components.json index 01389f78a93..26f22d10a16 100644 --- a/spec/parts/linux/cloud-init/artifacts/test_components.json +++ b/spec/parts/linux/cloud-init/artifacts/test_components.json @@ -108,6 +108,27 @@ ] } }, + "azurecontainerlinux": { + "current": { + "versionsV2": [ + { + "renovateTag": "", + "latestVersion": "dummyVersionACL" + } + ], + "downloadURL": "https://dummypath/dummy-acl.rpm" + } + }, + "flatcar": { + "current": { + "versionsV2": [ + { + "renovateTag": "", + "latestVersion": "dummyVersionFlatcar" + } + ] + } + }, "default": { "current": { "versionsV2": [ diff --git a/spec/vhdbuilder/packer/install_dependencies_spec.sh b/spec/vhdbuilder/packer/install_dependencies_spec.sh new file mode 100644 index 00000000000..557da1258e4 --- /dev/null +++ b/spec/vhdbuilder/packer/install_dependencies_spec.sh @@ -0,0 +1,74 @@ +#!/bin/bash +set -o pipefail + +eval "$(sed -n '/^installAndConfigureArtifactStreaming() {/,/^}/p' vhdbuilder/packer/install-dependencies.sh)" + +Describe 'installAndConfigureArtifactStreaming' + setup_artifact_streaming() { + ARTIFACT_STREAMING_CALLS="$(mktemp)" + VHD_LOGS_FILEPATH="$(mktemp)" + ERR_ARTIFACT_STREAMING_DOWNLOAD=149 + OS="UBUNTU" + OS_VARIANT="" + } + cleanup_artifact_streaming() { + command rm -f ./acr-mirror.rpm ./acr-mirror-arm64.rpm + rm -f "$ARTIFACT_STREAMING_CALLS" "$VHD_LOGS_FILEPATH" + } + BeforeEach 'setup_artifact_streaming' + AfterEach 'cleanup_artifact_streaming' + + isARM64() { + echo "${TEST_ARM64:-0}" + } + isACL() { + [ "$OS" = "AZURECONTAINERLINUX" ] || + { [ "$OS" = "AZURELINUX" ] && [ "$OS_VARIANT" = "AZURECONTAINERLINUX" ]; } + } + retrycmd_curl_file() { + echo "retrycmd_curl_file $*" >> "$ARTIFACT_STREAMING_CALLS" + : > "$4" + } + apt_get_install() { + echo "apt_get_install $*" >> "$ARTIFACT_STREAMING_CALLS" + } + dnf_install() { + echo "dnf_install $*" >> "$ARTIFACT_STREAMING_CALLS" + } + downloadSysextFromVersion() { + echo "downloadSysextFromVersion $*" >> "$ARTIFACT_STREAMING_CALLS" + return "${DOWNLOAD_RC:-0}" + } + rm() { + echo "rm $*" >> "$ARTIFACT_STREAMING_CALLS" + command rm "$@" + } + + Describe 'caches the combined sysext without installing RPMs or rewriting OCI tags' + Parameters + "AZURECONTAINERLINUX" "" "x86-64" 0 + "AZURECONTAINERLINUX" "" "arm64" 1 + "AZURELINUX" "AZURECONTAINERLINUX" "x86-64" 0 + "AZURELINUX" "AZURECONTAINERLINUX" "arm64" 1 + End + It "caches the $3 artifact for $1 $2" + OS="$1" + OS_VARIANT="$2" + TEST_ARM64="$4" + version=$(jq -r '.Packages[] | select(.name == "acr-mirror") | .downloadURIs.azurecontainerlinux.current.versionsV2[0].latestVersion' parts/common/components.json) + url=$(jq -r '.Packages[] | select(.name == "acr-mirror") | .downloadURIs.azurecontainerlinux.current.downloadURL' parts/common/components.json) + url=${url//\$\{version\}/$version} + url=${url//\$\{SYSTEMD_ARCH\}/$3} + When call installAndConfigureArtifactStreaming "$url" "$version" + The contents of file "$ARTIFACT_STREAMING_CALLS" should equal "downloadSysextFromVersion artifact-streaming mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext:${version}-$3" + The status should be success + End + End + + It 'fails when the combined sysext cannot be cached' + OS="AZURECONTAINERLINUX" + DOWNLOAD_RC=1 + When run installAndConfigureArtifactStreaming "mcr.microsoft.com/azurecontainerregistry/v2/artifact-streaming-sysext:v1.0.0-1-azlinux3-x86-64" "v1.0.0-1-azlinux3" + The status should equal "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + End +End diff --git a/vhdbuilder/packer/acl-customdata.json b/vhdbuilder/packer/acl-customdata.json index 8783d77f65c..cccf7120d85 100644 --- a/vhdbuilder/packer/acl-customdata.json +++ b/vhdbuilder/packer/acl-customdata.json @@ -1 +1 @@ -{"ignition":{"version":"3.4.0"},"kernelArguments":{"shouldNotExist":["flatcar.autologin"]},"storage":{"directories":[{"path":"/opt/aks-sysext-rw/usr/local/bin"}],"files":[{"overwrite":true,"path":"/opt/aks-sysext/usr/lib/extension-release.d/extension-release.aks-sysext","contents":{"compression":"","source":"data:,ID%3Dazurelinux%0ASYSEXT_LEVEL%3D1.0%0A"}},{"overwrite":true,"path":"/etc/flatcar/update.conf","contents":{"compression":"","source":"data:,SERVER%3Ddisabled%0A"}},{"overwrite":true,"path":"/etc/systemd/system/containerd.service.d/50-default-config.conf","contents":{"compression":"","source":"data:,%5BService%5D%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0A"}}],"links":[{"path":"/etc/extensions/aks-sysext","hard":false,"target":"/opt/aks-sysext"},{"path":"/opt/aks-sysext/usr/local/bin","hard":false,"target":"/opt/aks-sysext-rw/usr/local/bin"},{"path":"/etc/containerd/config.toml","hard":false,"target":"/usr/share/containerd/config.toml"}]}} +{"ignition":{"version":"3.4.0"},"kernelArguments":{"shouldNotExist":["flatcar.autologin"]},"storage":{"directories":[{"path":"/opt/aks-sysext-rw/usr/local/bin"}],"files":[{"overwrite":true,"path":"/opt/aks-sysext/usr/lib/extension-release.d/extension-release.aks-sysext","contents":{"compression":"","source":"data:,ID%3Dazurelinux%0ASYSEXT_LEVEL%3D1.0%0A"}},{"overwrite":true,"path":"/etc/flatcar/update.conf","contents":{"compression":"","source":"data:,SERVER%3Ddisabled%0A"}},{"overwrite":true,"path":"/etc/systemd/system/containerd.service.d/50-default-config.conf","contents":{"compression":"","source":"data:,%5BService%5D%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0A"}}],"links":[{"path":"/etc/extensions/aks-sysext","hard":false,"target":"/opt/aks-sysext"},{"path":"/opt/aks-sysext/usr/local/bin","hard":false,"target":"/opt/aks-sysext-rw/usr/local/bin"},{"path":"/etc/containerd/config.toml","hard":false,"target":"/usr/share/containerd/config.toml"}]},"systemd":{"units":[{"mask":true,"name":"artifact-streaming-setup.service"},{"mask":true,"name":"acr-mirror.service"},{"mask":true,"name":"overlaybd-tcmu.service"},{"mask":true,"name":"overlaybd-snapshotter.service"}]}} diff --git a/vhdbuilder/packer/acl-customdata.yaml b/vhdbuilder/packer/acl-customdata.yaml index 903069da068..409a04446ea 100644 --- a/vhdbuilder/packer/acl-customdata.yaml +++ b/vhdbuilder/packer/acl-customdata.yaml @@ -42,3 +42,13 @@ storage: # NOTE: ACL does NOT have update-ca-certificates.service (Flatcar-specific). # ACL uses update-ca-trust (Azure Linux style) which is invoked on-demand by CSE scripts. # The Flatcar customdata has a dropin for update-ca-certificates.service that is omitted here. +systemd: + units: + - name: artifact-streaming-setup.service + mask: true + - name: acr-mirror.service + mask: true + - name: overlaybd-tcmu.service + mask: true + - name: overlaybd-snapshotter.service + mask: true diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 28a231a6436..c40ea912129 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -392,29 +392,37 @@ downloadAndInstallCriTools() { installAndConfigureArtifactStreaming() { local downloadURL="$1" local version="$2" + if isACL "$OS" "$OS_VARIANT"; then + # Cache only: activation is feature-gated during node provisioning. + downloadSysextFromVersion artifact-streaming "$downloadURL" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" + echo " - artifact-streaming sysext version ${version}" >> "${VHD_LOGS_FILEPATH}" + return + fi + # The arm64 packages have "-arm64" inserted before the file extension, # e.g. acr-mirror-2204-arm64.deb instead of acr-mirror-2204.deb if [ "$(isARM64)" -eq 1 ]; then downloadURL="${downloadURL%.*}-arm64.${downloadURL##*.}" fi local MIRROR_DOWNLOAD_PATH="./$(basename "${downloadURL}")" - retrycmd_curl_file 10 5 60 "$MIRROR_DOWNLOAD_PATH" "$downloadURL" || exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} + retrycmd_curl_file 10 5 60 "$MIRROR_DOWNLOAD_PATH" "$downloadURL" || exit "${ERR_ARTIFACT_STREAMING_DOWNLOAD}" + case "$downloadURL" in *.deb) - apt_get_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD + apt_get_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" ;; *.rpm) - dnf_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit $ERR_ARTIFACT_STREAMING_DOWNLOAD + dnf_install 10 2 120 "$MIRROR_DOWNLOAD_PATH" || exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" ;; *) echo "Unsupported acr-mirror package extension in URL: ${downloadURL}" >&2 - exit ${ERR_ARTIFACT_STREAMING_DOWNLOAD} + exit "$ERR_ARTIFACT_STREAMING_DOWNLOAD" ;; esac rm "$MIRROR_DOWNLOAD_PATH" /opt/acr/tools/overlaybd/install.sh - # Remove the bundled overlaybd installer packages (~55-58 MB); install.sh already installed them and they're unused at runtime. + # Remove the bundled OverlayBD installer packages (~55-58 MB); they're unused at runtime. rm -f /opt/acr/tools/overlaybd/bin/*.deb /opt/acr/tools/overlaybd/bin/*.rpm echo " - acr-mirror version ${version}" >> ${VHD_LOGS_FILEPATH} }