feat(aks-node-controller): support optional containerd image-pull settings

aks-node-controller/parser/helper_test.go

@@ -21,6 +21,7 @@ import (
 	_ "embed"
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"math"
 	"reflect"
 	"regexp"
@@ -546,6 +547,68 @@ func Test_createSortedKeyValueInt32Pairs(t *testing.T) {
 	}
 }
 
+func TestContainerdImagePullTemplates(t *testing.T) {
+	for _, version := range []string{"1.7.22", "2.2.4"} {
+		for _, noGPU := range []bool{false, true} {
+			for _, tc := range []struct {
+				name     string
+				settings *aksnodeconfigv1.ContainerdConfig
+			}{
+				{"absent", nil},
+				{"empty", &aksnodeconfigv1.ContainerdConfig{}},
+				{"false", &aksnodeconfigv1.ContainerdConfig{ImagePullWithSyncFs: proto.Bool(false)}},
+				{"true", &aksnodeconfigv1.ContainerdConfig{ImagePullWithSyncFs: proto.Bool(true)}},
+				{"zero timeout", &aksnodeconfigv1.ContainerdConfig{ImagePullProgressTimeout: proto.String("0s")}},
+				{"both", &aksnodeconfigv1.ContainerdConfig{ImagePullWithSyncFs: proto.Bool(true), ImagePullProgressTimeout: proto.String("1m30s")}},
+			} {
+				t.Run(fmt.Sprintf("%s/noGPU=%t/%s", version, noGPU, tc.name), func(t *testing.T) {
+					assertContainerdImagePullTemplate(t, tc.settings, noGPU, version)
+				})
+			}
+		}
+	}
+}
+
+func assertContainerdImagePullTemplate(t *testing.T, settings *aksnodeconfigv1.ContainerdConfig, noGPU bool, version string) {
+	t.Helper()
+	cfg := &aksnodeconfigv1.Configuration{
+		KubernetesVersion: "1.34.0",
+		GpuConfig:         &aksnodeconfigv1.GpuConfig{EnableNvidia: proto.Bool(true)},
+		ContainerdConfig:  settings,
+	}
+	rendered, err := containerdConfigFromAKSNodeConfig(cfg, noGPU, version)
+	require.NoError(t, err)
+	plugin := "io.containerd.grpc.v1.cri"
+	if isContainerdV2(version) {
+		plugin = "io.containerd.cri.v1.images"
+	}
+	header := `[plugins."` + plugin + `"]` + "\n"
+	start := strings.Index(rendered, header)
+	require.NotEqual(t, -1, start)
+	section := rendered[start+len(header):]
+	if next := regexp.MustCompile(`(?m)^[ \t]*\[`).FindStringIndex(section); next != nil {
+		section = section[:next[0]]
+	}
+	if settings != nil && settings.ImagePullWithSyncFs != nil {
+		require.Contains(t, section, fmt.Sprintf("  image_pull_with_sync_fs = %t\n", settings.GetImagePullWithSyncFs()))
+		require.Equal(t, 1, strings.Count(rendered, "image_pull_with_sync_fs"))
+	} else {
+		require.NotContains(t, rendered, "image_pull_with_sync_fs")
+	}
+	if settings != nil && settings.ImagePullProgressTimeout != nil {
+		require.Contains(t, section, fmt.Sprintf("  image_pull_progress_timeout = %q\n", settings.GetImagePullProgressTimeout()))
+		require.Equal(t, 1, strings.Count(rendered, "image_pull_progress_timeout"))
+	} else {
+		require.NotContains(t, rendered, "image_pull_progress_timeout")
+	}
+	if settings == nil || (settings.ImagePullWithSyncFs == nil && settings.ImagePullProgressTimeout == nil) {
+		cfg.ContainerdConfig = nil
+		baseline, err := containerdConfigFromAKSNodeConfig(cfg, noGPU, version)
+		require.NoError(t, err)
+		require.Equal(t, baseline, rendered)
+	}
+}
+
 func Test_getContainerdConfig(t *testing.T) {
 	type args struct {
 		aksnodeconfig *aksnodeconfigv1.Configuration

aks-node-controller/parser/parser_test.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -17,8 +18,38 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
 )
 
+func TestBuildCSECmdContainerdImagePullSettings(t *testing.T) {
+	for _, version := range []string{"1.7.22", "2.2.4"} {
+		t.Run(version, func(t *testing.T) {
+			dir := t.TempDir()
+			require.NoError(t, os.WriteFile(filepath.Join(dir, "containerd"), []byte(fmt.Sprintf(`#!/bin/sh
+[ "$1" = "--version" ] || exit 1
+echo "containerd containerd.io %s test"
+`, version)), 0755))
+			t.Setenv("PATH", dir)
+			cfg := &aksnodeconfigv1.Configuration{ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{
+				ImagePullWithSyncFs:      proto.Bool(false),
+				ImagePullProgressTimeout: proto.String("0s"),
+			}}
+			cmd, err := BuildCSECmd(context.Background(), cfg, nil)
+			require.NoError(t, err)
+			env := environToMap(cmd.Env)
+			for _, key := range []string{"CONTAINERD_CONFIG_CONTENT", "CONTAINERD_CONFIG_NO_GPU_CONTENT"} {
+				content, err := base64.StdEncoding.DecodeString(env[key])
+				require.NoError(t, err)
+				require.Contains(t, string(content), "image_pull_with_sync_fs = false")
+				require.Contains(t, string(content), `image_pull_progress_timeout = "0s"`)
+			}
+			require.NotContains(t, env, "ANC_CONFIGURE_CONTAINERD_BINARY")
+			require.NotContains(t, env, "CONTAINERD_IMAGE_PULL_WITH_SYNC_FS")
+			require.NotContains(t, env, "CONTAINERD_IMAGE_PULL_PROGRESS_TIMEOUT")
+		})
+	}
+}
+
 func TestBuildCSECmd(t *testing.T) {
 	tests := []struct {
 		name                 string

aks-node-controller/parser/templates/containerd.toml.gtpl

@@ -15,6 +15,14 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
   enable_tar_index = false
 {{- end}}
 [plugins."io.containerd.grpc.v1.cri"]
+{{- with .GetContainerdConfig }}
+  {{- if ne .ImagePullWithSyncFs nil }}
+  image_pull_with_sync_fs = {{ .GetImagePullWithSyncFs }}
+  {{- end }}
+  {{- if ne .ImagePullProgressTimeout nil }}
+  image_pull_progress_timeout = "{{ .GetImagePullProgressTimeout }}"
+  {{- end }}
+{{- end }}
   sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}"
   enable_cdi = true
   [plugins."io.containerd.grpc.v1.cri".containerd]

aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl

@@ -15,6 +15,14 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
   enable_tar_index = false
 {{- end}}
 [plugins."io.containerd.grpc.v1.cri"]
+{{- with .GetContainerdConfig }}
+  {{- if ne .ImagePullWithSyncFs nil }}
+  image_pull_with_sync_fs = {{ .GetImagePullWithSyncFs }}
+  {{- end }}
+  {{- if ne .ImagePullProgressTimeout nil }}
+  image_pull_progress_timeout = "{{ .GetImagePullProgressTimeout }}"
+  {{- end }}
+{{- end }}
   sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}"
   [plugins."io.containerd.grpc.v1.cri".containerd]
     {{- if .GetIsKata }}

aks-node-controller/parser/templates/containerd_v2.toml.gtpl

@@ -15,6 +15,14 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
   enable_tar_index = false
 {{- end}}
 [plugins."io.containerd.cri.v1.images"]
+{{- with .GetContainerdConfig }}
+  {{- if ne .ImagePullWithSyncFs nil }}
+  image_pull_with_sync_fs = {{ .GetImagePullWithSyncFs }}
+  {{- end }}
+  {{- if ne .ImagePullProgressTimeout nil }}
+  image_pull_progress_timeout = "{{ .GetImagePullProgressTimeout }}"
+  {{- end }}
+{{- end }}
 {{- if .GetEnableArtifactStreaming }}
   snapshotter = "overlaybd"
   disable_snapshot_annotations = false

aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl

@@ -15,6 +15,14 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}}
   enable_tar_index = false
 {{- end}}
 [plugins."io.containerd.cri.v1.images"]
+{{- with .GetContainerdConfig }}
+  {{- if ne .ImagePullWithSyncFs nil }}
+  image_pull_with_sync_fs = {{ .GetImagePullWithSyncFs }}
+  {{- end }}
+  {{- if ne .ImagePullProgressTimeout nil }}
+  image_pull_progress_timeout = "{{ .GetImagePullProgressTimeout }}"
+  {{- end }}
+{{- end }}
 {{- if .GetEnableArtifactStreaming }}
   snapshotter = "overlaybd"
   disable_snapshot_annotations = false

aks-node-controller/pkg/nodeconfigutils/utils_test.go

@@ -16,6 +16,34 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+func TestContainerdImagePullPresenceRoundTrip(t *testing.T) {
+	for _, raw := range []string{
+		`{"containerdConfig":{"imagePullWithSyncFs":false,"imagePullProgressTimeout":"0s"}}`,
+		`{"containerd_config":{"image_pull_with_sync_fs":true,"image_pull_progress_timeout":"30s"}}`,
+		`{"containerd_config":{}}`,
+	} {
+		cfg, err := UnmarshalConfigurationV1([]byte(raw))
+		require.NoError(t, err)
+		encoded, err := MarshalConfigurationV1(cfg)
+		require.NoError(t, err)
+		restored, err := UnmarshalConfigurationV1(encoded)
+		require.NoError(t, err)
+		require.True(t, proto.Equal(cfg, restored))
+		wire, err := proto.Marshal(cfg)
+		require.NoError(t, err)
+		decoded := &aksnodeconfigv1.Configuration{}
+		require.NoError(t, proto.Unmarshal(wire, decoded))
+		require.True(t, proto.Equal(cfg, decoded))
+		if cfg.ContainerdConfig.ImagePullWithSyncFs != nil {
+			require.Contains(t, string(encoded), `"image_pull_with_sync_fs"`)
+			require.NotNil(t, decoded.ContainerdConfig.ImagePullWithSyncFs)
+			require.NotNil(t, decoded.ContainerdConfig.ImagePullProgressTimeout)
+		} else {
+			require.NotContains(t, string(encoded), "image_pull_")
+		}
+	}
+}
+
 func TestUnmarshalConfigurationV1(t *testing.T) {
 	tests := []struct {
 		name    string

aks-node-controller/pkg/nodeconfigutils/validation.go

@@ -6,6 +6,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1"
 	"github.com/Masterminds/semver/v3"
@@ -22,6 +23,9 @@ func validateAndNormalize(cfg *aksnodeconfigv1.Configuration) error {
 	if err := validateCustomLinuxOSConfig(cfg.GetCustomLinuxOsConfig()); err != nil {
 		return err
 	}
+	if err := validateContainerdConfig(cfg.GetContainerdConfig()); err != nil {
+		return err
+	}
 
 	kubelet := cfg.GetKubeletConfig()
 	if kubelet == nil {
@@ -57,6 +61,19 @@ func validateAndNormalize(cfg *aksnodeconfigv1.Configuration) error {
 	return nil
 }
 
+func validateContainerdConfig(cfg *aksnodeconfigv1.ContainerdConfig) error {
+	if cfg != nil && cfg.ImagePullProgressTimeout != nil {
+		duration, err := time.ParseDuration(cfg.GetImagePullProgressTimeout())
+		if err != nil {
+			return fmt.Errorf("containerd_config.image_pull_progress_timeout: %w", err)
+		}
+		if duration < 0 {
+			return fmt.Errorf("containerd_config.image_pull_progress_timeout must not be negative")
+		}
+	}
+	return nil
+}
+
 func validateCustomLinuxOSConfig(osConfig *aksnodeconfigv1.CustomLinuxOsConfig) error {
 	for _, field := range []struct {
 		name, value string

aks-node-controller/pkg/nodeconfigutils/validation_test.go

@@ -8,6 +8,29 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+func TestContainerdImagePullValidationWithoutKubelet(t *testing.T) {
+	for _, timeout := range []string{"0", "0s", "30s", "5m", "1m30s"} {
+		cfg := &aksnodeconfigv1.Configuration{
+			Version: "v1",
+			ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{
+				ImagePullWithSyncFs:      proto.Bool(false),
+				ImagePullProgressTimeout: proto.String(timeout),
+			},
+		}
+		require.NoError(t, validateAndNormalize(cfg))
+		require.NoError(t, validateAndNormalize(cfg))
+		require.Equal(t, timeout, cfg.ContainerdConfig.GetImagePullProgressTimeout())
+	}
+	for _, timeout := range []string{"", "-1s", "invalid", "5", "999999999999999999999h", "5m\n"} {
+		cfg := &aksnodeconfigv1.Configuration{
+			Version:          "v1",
+			ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ImagePullProgressTimeout: proto.String(timeout)},
+		}
+		require.ErrorContains(t, validateAndNormalize(cfg), "containerd_config.image_pull_progress_timeout")
+	}
+	require.NoError(t, validateAndNormalize(&aksnodeconfigv1.Configuration{Version: "v1"}))
+}
+
 func TestValidateAndNormalize(t *testing.T) {
 	for _, version := range []string{"1.10.0", "1.11.0", "1.20.0", "1.24.0", "1.25.0", "1.34.0", "1.34.0-rc.1"} {
 		t.Run(version, func(t *testing.T) {

aks-node-controller/proto/aksnodeconfig/v1/containerd_config.proto

@@ -12,4 +12,12 @@ message ContainerdConfig {
 
   // The URL for downloading the containerd package.
   string containerd_package_url = 3;
+
+  // Force filesystem synchronization during image unpacking. Absence preserves
+  // the existing setting. Enabling this may select local pulls on containerd 2.x.
+  optional bool image_pull_with_sync_fs = 4;
+
+  // Maximum time without image download progress, in Go duration format.
+  // Zero disables the timeout. Absence preserves the existing setting.
+  optional string image_pull_progress_timeout = 5;
 }