diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index 29e3fbdaa73..7b3805ad8c9 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -93,6 +93,13 @@ type App struct { // checkHotfixFetcher overrides the real LPS hotfix-pointer GET for testing, letting // unit tests inject a canned pointer body or errors without real networking. checkHotfixFetcher func(ctx context.Context) ([]byte, error) + // networkingImageCacheFetcher overrides the LPS networking image config fetch for tests. + networkingImageCacheFetcher func(ctx context.Context) ([]byte, error) + networkingImagePuller func(ctx context.Context, reference string) (networkingImagePullOutput, error) + networkingImageRetryWait func(ctx context.Context) error + // imageFetcherPath and networkingImageCacheStatusPath override node paths for tests. + imageFetcherPath string + networkingImageCacheStatusPath string // fetchAttestedToken overrides retrieval of the IMDS attested-data token used as the // Authorization header for the check-hotfix LPS fetch. When nil, the real IMDS endpoint // is queried. @@ -217,6 +224,16 @@ func (a *App) Run(ctx context.Context, args []string) int { return a.runCheckHotfixCommand(ctx) }, }, + { + Name: "prepull-networking-images", + Usage: "Pull and unpack the synth-rendered Cilium/CNS images before kubelet starts (fail-open)", + Action: func(ctx context.Context, cmd *cli.Command) error { + if extra := cmd.Args().Slice(); len(extra) > 0 { + slog.Warn("ignoring unexpected prepull-networking-images arguments", "args", strings.Join(extra, " ")) + } + return a.runPrepullNetworkingImagesCommand(ctx) + }, + }, }, } diff --git a/aks-node-controller/checkhotfix_grpc.go b/aks-node-controller/checkhotfix_grpc.go index ebd78d49241..e6daa5f2693 100644 --- a/aks-node-controller/checkhotfix_grpc.go +++ b/aks-node-controller/checkhotfix_grpc.go @@ -7,6 +7,7 @@ import ( "fmt" "log/slog" "net" + "time" lpsv1 "github.com/Azure/agentbaker/aks-live-patching/pkg/gen/akslivepatching/v1" "google.golang.org/grpc" @@ -64,6 +65,10 @@ func (e *lpsGRPCStatusError) Error() string { // returns the opaque response bytes for the shared parse/stage path. // The gRPC status is mapped onto the benign-vs-fatal taxonomy so handleFetchError is unchanged. func (a *App) fetchHotfixOverGRPC(ctx context.Context) ([]byte, error) { + return a.fetchLPSComponentOverGRPC(ctx, ancComponentName, lpsFetchTimeout) +} + +func (a *App) fetchLPSComponentOverGRPC(ctx context.Context, componentName string, timeout time.Duration) ([]byte, error) { fqdn, caPEM, err := a.lpsTargetFromNodeConfig() if err != nil { return nil, fmt.Errorf("resolving LPS endpoint: %w", err) @@ -87,11 +92,11 @@ func (a *App) fetchHotfixOverGRPC(ctx context.Context) ([]byte, error) { return nil, fmt.Errorf("dialing LPS gRPC: %w", err) } defer conn.Close() - slog.Info("check-hotfix LPS gRPC dial", "dialHost", fqdn, "alpn", lpsALPNProto, "component", ancComponentName) + slog.Info("LPS gRPC dial", "dialHost", fqdn, "alpn", lpsALPNProto, "component", componentName) // Bound the whole round-trip (cold connect + RPC); on expiry we fail open to the cold-start // pointer. See the lpsFetchTimeout const doc for the deadline tradeoff. - ctx, cancel := context.WithTimeout(ctx, lpsFetchTimeout) + ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() // Carry the attested-data document that authenticates the node in request metadata, plus @@ -102,7 +107,7 @@ func (a *App) fetchHotfixOverGRPC(ctx context.Context) ([]byte, error) { } client := lpsv1.NewLivePatchingServiceClient(conn) - resp, err := client.GetComponentConfig(ctx, &lpsv1.GetComponentConfigRequest{ComponentName: ancComponentName}) + resp, err := client.GetComponentConfig(ctx, &lpsv1.GetComponentConfigRequest{ComponentName: componentName}) if err != nil { return nil, mapGRPCError(err) } diff --git a/aks-node-controller/networkingimagecache.go b/aks-node-controller/networkingimagecache.go new file mode 100644 index 00000000000..441b35714e0 --- /dev/null +++ b/aks-node-controller/networkingimagecache.go @@ -0,0 +1,408 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/Azure/agentbaker/aks-node-controller/helpers" + "github.com/Azure/agentbaker/aks-node-controller/pkg/nodeconfigutils" +) + +const ( + networkingImageCacheComponentName = "networkingImageCache" + networkingImageCacheSchemaVersion = 2 + defaultImageFetcherPath = "/opt/azure/containers/image-fetcher" + defaultNetworkingStatusPath = "/opt/azure/containers/networking-image-cache-status.json" + agentPoolLabelKey = "kubernetes.azure.com/agentpool" + maxNetworkingImages = 128 + maxImageReferenceLength = 512 + networkingLPSFetchTimeout = 10 * time.Second + networkingPullTimeout = 2 * time.Minute + networkingOverallTimeout = 5 * time.Minute + networkingPullConcurrency = 4 + networkingLPSFetchAttempts = 3 + networkingLPSRetryDelay = 2 * time.Second +) + +type networkingImageCacheConfig struct { + SchemaVersion int + AgentPool string + Images []networkingImageEntry + Fingerprint string +} + +type networkingImageCachePayload struct { + SchemaVersion int `json:"schemaVersion"` + AgentPools map[string]json.RawMessage `json:"agentPools"` +} + +type networkingImageCacheAgentPool struct { + Images []networkingImageEntry `json:"images"` +} + +type networkingImageEntry struct { + Reference string `json:"reference"` + Workload string `json:"workload,omitempty"` + Container string `json:"container,omitempty"` +} + +type networkingImageResult struct { + Reference string `json:"reference"` + Digest string `json:"digest,omitempty"` + Snapshotter string `json:"snapshotter,omitempty"` + Outcome string `json:"outcome"` + Duration time.Duration `json:"duration"` + Error string `json:"error,omitempty"` +} + +type networkingImagePullOutput struct { + Reference string `json:"reference"` + Digest string `json:"digest"` + Snapshotter string `json:"snapshotter"` + Outcome string `json:"outcome"` +} + +type networkingImageCacheStatus struct { + SchemaVersion int `json:"schemaVersion"` + ComponentFingerprint string `json:"componentFingerprint,omitempty"` + AgentPool string `json:"agentPool,omitempty"` + Outcome string `json:"outcome"` + StartedAt time.Time `json:"startedAt"` + FinishedAt time.Time `json:"finishedAt"` + Images []networkingImageResult `json:"images,omitempty"` + Error string `json:"error,omitempty"` +} + +func (a *App) runPrepullNetworkingImagesCommand(ctx context.Context) (err error) { + started := time.Now().UTC() + status := networkingImageCacheStatus{StartedAt: started, Outcome: "failed"} + defer func() { + status.FinishedAt = time.Now().UTC() + if err != nil { + status.Error = err.Error() + slog.Warn("networking image pre-pull completed with error (fail-open)", "error", err) + } else { + slog.Info("networking image pre-pull completed", "outcome", status.Outcome, "images", len(status.Images)) + } + if writeErr := a.writeNetworkingImageCacheStatus(status); writeErr != nil { + slog.Warn("failed to write networking image cache status", "error", writeErr) + } + if a.eventLogger != nil { + level := helpers.EventLevelInformational + if err != nil { + level = helpers.EventLevelError + } + message := fmt.Sprintf("prepull-networking-images outcome=%s images=%d", status.Outcome, len(status.Images)) + if err != nil { + message += " error=" + err.Error() + } + a.eventLogger.LogEvent("PrepullNetworkingImages", message, level, started, status.FinishedAt) + } + err = nil + }() + + ctx, cancel := context.WithTimeout(ctx, networkingOverallTimeout) + defer cancel() + + localPool, poolErr := a.agentPoolFromNodeConfig() + if poolErr != nil { + status.Outcome = "skipped" + return poolErr + } + status.AgentPool = localPool + if localPool == "" { + status.Outcome = "skipped" + return errors.New("node config has no kubernetes.azure.com/agentpool label") + } + + config, fetchErr := a.fetchNetworkingImageCacheConfig(ctx, localPool) + if fetchErr != nil { + status.Outcome = "skipped" + return fetchErr + } + status.SchemaVersion = config.SchemaVersion + status.ComponentFingerprint = config.Fingerprint + + results, pullErr := a.pullNetworkingImages(ctx, config.Images) + status.Images = results + status.Outcome = "complete" + if pullErr != nil { + status.Outcome = "failed" + for _, result := range results { + if result.Outcome != "failed" { + status.Outcome = "partial" + break + } + } + return pullErr + } + return nil +} + +func (a *App) fetchNetworkingImageCacheConfig(ctx context.Context, localPool string) (*networkingImageCacheConfig, error) { + var ( + data []byte + err error + ) + for attempt := 1; attempt <= networkingLPSFetchAttempts; attempt++ { + if a.networkingImageCacheFetcher != nil { + data, err = a.networkingImageCacheFetcher(ctx) + } else { + data, err = a.fetchLPSComponentOverGRPC(ctx, networkingImageCacheComponentName, networkingLPSFetchTimeout) + } + if err == nil { + break + } + if attempt == networkingLPSFetchAttempts { + return nil, fmt.Errorf("fetching %s from LPS after %d attempts: %w", networkingImageCacheComponentName, attempt, err) + } + if err := a.waitForNetworkingImageRetry(ctx); err != nil { + return nil, err + } + } + + var schema struct { + SchemaVersion int `json:"schemaVersion"` + } + schemaDecoder := json.NewDecoder(strings.NewReader(string(data))) + if err := schemaDecoder.Decode(&schema); err != nil { + return nil, fmt.Errorf("parsing networking image cache config: %w", err) + } + if err := schemaDecoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return nil, errors.New("parsing networking image cache config: trailing JSON data") + } + if schema.SchemaVersion != networkingImageCacheSchemaVersion { + return nil, fmt.Errorf("unsupported networking image cache schema version %d", schema.SchemaVersion) + } + + var payload networkingImageCachePayload + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&payload); err != nil { + return nil, fmt.Errorf("parsing networking image cache config: %w", err) + } + localPool = strings.TrimSpace(localPool) + if localPool == "" { + return nil, errors.New("local agent pool must not be empty") + } + rawPool, ok := payload.AgentPools[localPool] + if !ok { + return nil, fmt.Errorf("networking image cache config has no inventory for local agent pool %q", localPool) + } + + var pool networkingImageCacheAgentPool + poolDecoder := json.NewDecoder(strings.NewReader(string(rawPool))) + poolDecoder.DisallowUnknownFields() + if err := poolDecoder.Decode(&pool); err != nil { + return nil, fmt.Errorf("parsing networking image cache config for local agent pool %q: %w", localPool, err) + } + if err := poolDecoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("parsing networking image cache config for local agent pool %q: trailing JSON data", localPool) + } + if len(pool.Images) == 0 { + return nil, fmt.Errorf("networking image cache config has no images for local agent pool %q", localPool) + } + if len(pool.Images) > maxNetworkingImages { + return nil, fmt.Errorf("networking image count %d for local agent pool %q exceeds limit %d", len(pool.Images), localPool, maxNetworkingImages) + } + + deduped := make(map[string]networkingImageEntry, len(pool.Images)) + for _, image := range pool.Images { + image.Reference = strings.TrimSpace(image.Reference) + if err := validateNetworkingImageReference(image.Reference); err != nil { + return nil, err + } + existing, exists := deduped[image.Reference] + if !exists || image.Workload < existing.Workload || + (image.Workload == existing.Workload && image.Container < existing.Container) { + deduped[image.Reference] = image + } + } + config := &networkingImageCacheConfig{ + SchemaVersion: payload.SchemaVersion, + AgentPool: localPool, + Images: make([]networkingImageEntry, 0, len(deduped)), + Fingerprint: fmt.Sprintf("%x", sha256.Sum256(data)), + } + for _, image := range deduped { + config.Images = append(config.Images, image) + } + sort.Slice(config.Images, func(i, j int) bool { + return config.Images[i].Reference < config.Images[j].Reference + }) + return config, nil +} + +func (a *App) waitForNetworkingImageRetry(ctx context.Context) error { + if a.networkingImageRetryWait != nil { + return a.networkingImageRetryWait(ctx) + } + timer := time.NewTimer(networkingLPSRetryDelay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func validateNetworkingImageReference(reference string) error { + if reference == "" { + return errors.New("networking image reference must not be empty") + } + if len(reference) > maxImageReferenceLength { + return fmt.Errorf("networking image reference exceeds %d bytes", maxImageReferenceLength) + } + if strings.IndexFunc(reference, func(r rune) bool { return r == ' ' || r == '\t' || r == '\n' || r == '\r' }) >= 0 { + return fmt.Errorf("networking image reference %q contains whitespace", reference) + } + lastSlash := strings.LastIndex(reference, "/") + if lastSlash <= 0 || (!strings.Contains(reference[lastSlash+1:], ":") && !strings.Contains(reference, "@")) { + return fmt.Errorf("networking image reference %q must include registry, repository, and tag or digest", reference) + } + return nil +} + +func (a *App) pullNetworkingImages(ctx context.Context, images []networkingImageEntry) ([]networkingImageResult, error) { + if len(images) == 0 { + return nil, nil + } + path := a.imageFetcherPath + if path == "" { + path = defaultImageFetcherPath + } + + results := make([]networkingImageResult, len(images)) + jobs := make(chan int) + var wg sync.WaitGroup + workers := networkingPullConcurrency + if len(images) < workers { + workers = len(images) + } + for range workers { + wg.Add(1) + go func() { + defer wg.Done() + for index := range jobs { + image := images[index] + started := time.Now() + result := networkingImageResult{Reference: image.Reference, Outcome: "failed"} + imageCtx, cancel := context.WithTimeout(ctx, networkingPullTimeout) + output, err := a.pullNetworkingImage(imageCtx, path, image.Reference) + if err != nil { + result.Error = err.Error() + } else { + result.Digest = output.Digest + result.Snapshotter = output.Snapshotter + result.Outcome = output.Outcome + } + cancel() + result.Duration = time.Since(started) + results[index] = result + } + }() + } + for index := range images { + select { + case jobs <- index: + case <-ctx.Done(): + close(jobs) + wg.Wait() + return results, ctx.Err() + } + } + close(jobs) + wg.Wait() + + var errs []error + for _, result := range results { + if result.Error != "" { + errs = append(errs, fmt.Errorf("%s: %s", result.Reference, result.Error)) + } + } + return results, errors.Join(errs...) +} + +func (a *App) pullNetworkingImage(ctx context.Context, path, reference string) (networkingImagePullOutput, error) { + if a.networkingImagePuller != nil { + return a.networkingImagePuller(ctx, reference) + } + cmd := exec.CommandContext(ctx, path, "pull-image", "--image", reference) + output, err := cmd.CombinedOutput() + if err != nil { + return networkingImagePullOutput{}, fmt.Errorf("image-fetcher failed: %w: %s", err, strings.TrimSpace(string(output))) + } + var result networkingImagePullOutput + if err := json.Unmarshal(output, &result); err != nil { + return networkingImagePullOutput{}, fmt.Errorf("parsing image-fetcher result: %w", err) + } + if result.Reference != reference { + return networkingImagePullOutput{}, fmt.Errorf("image-fetcher returned reference %q for %q", result.Reference, reference) + } + if result.Digest == "" || result.Snapshotter == "" { + return networkingImagePullOutput{}, errors.New("image-fetcher result is missing digest or snapshotter") + } + switch result.Outcome { + case "cache-hit", "unpacked", "pulled": + return result, nil + default: + return networkingImagePullOutput{}, fmt.Errorf("image-fetcher returned unsupported outcome %q", result.Outcome) + } +} + +func (a *App) agentPoolFromNodeConfig() (string, error) { + raw, err := os.ReadFile(a.getNodeConfigPath()) + if err != nil { + return "", fmt.Errorf("reading node config for agent pool: %w", err) + } + config, parseErr := nodeconfigutils.UnmarshalConfigurationV1(raw) + if config == nil { + return "", fmt.Errorf("parsing node config for agent pool: %w", parseErr) + } + return strings.TrimSpace(config.GetKubeletConfig().GetKubeletNodeLabels()[agentPoolLabelKey]), nil +} + +func (a *App) writeNetworkingImageCacheStatus(status networkingImageCacheStatus) error { + path := a.networkingImageCacheStatusPath + if path == "" { + path = defaultNetworkingStatusPath + } + data, err := json.MarshalIndent(status, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(path), ".networking-image-cache-*.tmp") + if err != nil { + return err + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + if err := tmp.Chmod(0o600); err != nil { + tmp.Close() + return err + } + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmpPath, path) +} diff --git a/aks-node-controller/networkingimagecache_test.go b/aks-node-controller/networkingimagecache_test.go new file mode 100644 index 00000000000..65e27302d81 --- /dev/null +++ b/aks-node-controller/networkingimagecache_test.go @@ -0,0 +1,478 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFetchNetworkingImageCacheConfig(t *testing.T) { + payload := []byte(`{ + "schemaVersion": 2, + "agentPools": { + "pool1": { + "images": [ + {"reference":"mcr.microsoft.com/z:v1","workload":"cilium"}, + {"reference":"mcr.microsoft.com/a:v2","workload":"azure-cns"}, + {"reference":"mcr.microsoft.com/z:v1","workload":"duplicate"} + ] + } + } + }`) + app := &App{ + networkingImageCacheFetcher: func(context.Context) ([]byte, error) { + return payload, nil + }, + } + config, err := app.fetchNetworkingImageCacheConfig(context.Background(), "pool1") + require.NoError(t, err) + assert.Equal(t, "pool1", config.AgentPool) + require.Len(t, config.Images, 2) + assert.Equal(t, "mcr.microsoft.com/a:v2", config.Images[0].Reference) + assert.Equal(t, "mcr.microsoft.com/z:v1", config.Images[1].Reference) + assert.Equal(t, fmt.Sprintf("%x", sha256.Sum256(payload)), config.Fingerprint) +} + +func TestFetchNetworkingImageCacheConfigValidatesOnlyLocalPool(t *testing.T) { + app := &App{ + networkingImageCacheFetcher: func(context.Context) ([]byte, error) { + return []byte(`{ + "schemaVersion":2, + "agentPools":{ + "pool1":{"images":[{"reference":"mcr.microsoft.com/local:v1"}]}, + "pool2":{"futureField":"ignored because this pool is not local"} + } + }`), nil + }, + } + config, err := app.fetchNetworkingImageCacheConfig(context.Background(), "pool1") + require.NoError(t, err) + require.Len(t, config.Images, 1) + assert.Equal(t, "mcr.microsoft.com/local:v1", config.Images[0].Reference) +} + +func TestFetchNetworkingImageCacheConfigRejectsIncompletePayloads(t *testing.T) { + tests := []struct { + name string + payload string + want string + }{ + { + name: "missing local pool", + payload: `{"schemaVersion":2,"agentPools":{"pool2":{"images":[{"reference":"mcr.microsoft.com/a:v1"}]}}}`, + want: `no inventory for local agent pool "pool1"`, + }, + { + name: "empty local images", + payload: `{"schemaVersion":2,"agentPools":{"pool1":{"images":[]}}}`, + want: `no images for local agent pool "pool1"`, + }, + { + name: "trailing JSON", + payload: `{"schemaVersion":2,"agentPools":{"pool1":{"images":[{"reference":"mcr.microsoft.com/a:v1"}]}}} {}`, + want: "trailing JSON data", + }, + { + name: "unknown selected pool field", + payload: `{"schemaVersion":2,"agentPools":{"pool1":{"images":[{"reference":"mcr.microsoft.com/a:v1"}],"unknown":true}}}`, + want: `unknown field "unknown"`, + }, + { + name: "unknown top-level field", + payload: `{"schemaVersion":2,"agentPools":{"pool1":{"images":[{"reference":"mcr.microsoft.com/a:v1"}]}},"unknown":true}`, + want: `unknown field "unknown"`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + app := &App{ + networkingImageCacheFetcher: func(context.Context) ([]byte, error) { + return []byte(tt.payload), nil + }, + } + _, err := app.fetchNetworkingImageCacheConfig(context.Background(), "pool1") + require.Error(t, err) + assert.Contains(t, err.Error(), tt.want) + }) + } +} + +func TestFetchNetworkingImageCacheConfigRejectsMalformedOversizedAndSchemaMismatches(t *testing.T) { + oversizedImages := make([]networkingImageEntry, maxNetworkingImages+1) + for i := range oversizedImages { + oversizedImages[i].Reference = fmt.Sprintf("mcr.microsoft.com/image-%d:v1", i) + } + oversizedPayload, err := json.Marshal(map[string]any{ + "schemaVersion": 2, + "agentPools": map[string]any{ + "pool1": map[string]any{"images": oversizedImages}, + }, + }) + require.NoError(t, err) + + tests := []struct { + name string + payload []byte + want string + }{ + { + name: "malformed JSON", + payload: []byte(`{"schemaVersion":2,"agentPools":`), + want: "unexpected EOF", + }, + { + name: "too many local images", + payload: oversizedPayload, + want: "exceeds limit", + }, + { + name: "oversized reference", + payload: []byte(fmt.Sprintf( + `{"schemaVersion":2,"agentPools":{"pool1":{"images":[{"reference":"mcr.microsoft.com/%s:v1"}]}}}`, + strings.Repeat("a", maxImageReferenceLength), + )), + want: "exceeds 512 bytes", + }, + { + name: "invalid reference", + payload: []byte(`{"schemaVersion":2,"agentPools":{"pool1":{"images":[{"reference":"not-a-full-reference"}]}}}`), + want: "registry, repository, and tag or digest", + }, + { + name: "unsupported schema", + payload: []byte(`{"schemaVersion":3,"agentPools":{"pool1":{"images":[{"reference":"mcr.microsoft.com/a:v1"}]}}}`), + want: "unsupported networking image cache schema version 3", + }, + { + name: "old v1 payload", + payload: []byte(`{"schemaVersion":1,"agentPool":"pool1","images":[{"reference":"mcr.microsoft.com/a:v1"}]}`), + want: "unsupported networking image cache schema version 1", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + app := &App{ + networkingImageCacheFetcher: func(context.Context) ([]byte, error) { + return tt.payload, nil + }, + } + _, err := app.fetchNetworkingImageCacheConfig(context.Background(), "pool1") + require.Error(t, err) + assert.Contains(t, err.Error(), tt.want) + }) + } +} + +func TestFetchNetworkingImageCacheConfigRetriesTransientFailures(t *testing.T) { + var attempts atomic.Int32 + var waits atomic.Int32 + app := &App{ + networkingImageCacheFetcher: func(context.Context) ([]byte, error) { + if attempts.Add(1) < networkingLPSFetchAttempts { + return nil, errors.New("not ready") + } + return []byte(`{"schemaVersion":2,"agentPools":{"pool1":{"images":[{"reference":"mcr.microsoft.com/a:v1"}]}}}`), nil + }, + networkingImageRetryWait: func(context.Context) error { + waits.Add(1) + return nil + }, + } + config, err := app.fetchNetworkingImageCacheConfig(context.Background(), "pool1") + require.NoError(t, err) + assert.EqualValues(t, networkingLPSFetchAttempts, attempts.Load()) + assert.EqualValues(t, networkingLPSFetchAttempts-1, waits.Load()) + require.Len(t, config.Images, 1) +} + +func TestFetchNetworkingImageCacheConfigStopsRetryOnCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + app := &App{ + networkingImageCacheFetcher: func(context.Context) ([]byte, error) { + return nil, errors.New("not ready") + }, + networkingImageRetryWait: func(context.Context) error { + cancel() + return ctx.Err() + }, + } + _, err := app.fetchNetworkingImageCacheConfig(ctx, "pool1") + require.ErrorIs(t, err, context.Canceled) +} + +func TestFetchNetworkingImageCacheConfigSelectsDeterministicDuplicateMetadata(t *testing.T) { + app := &App{ + networkingImageCacheFetcher: func(context.Context) ([]byte, error) { + return []byte(`{ + "schemaVersion":2, + "agentPools":{"pool1":{"images":[ + {"reference":"mcr.microsoft.com/a:v1","workload":"z-workload","container":"a"}, + {"reference":"mcr.microsoft.com/a:v1","workload":"a-workload","container":"z"}, + {"reference":"mcr.microsoft.com/a:v1","workload":"a-workload","container":"a"} + ]}} + }`), nil + }, + } + config, err := app.fetchNetworkingImageCacheConfig(context.Background(), "pool1") + require.NoError(t, err) + require.Len(t, config.Images, 1) + assert.Equal(t, "a-workload", config.Images[0].Workload) + assert.Equal(t, "a", config.Images[0].Container) +} + +func TestPullNetworkingImagesInvokesPullImage(t *testing.T) { + var mu sync.Mutex + var references []string + app := &App{ + networkingImagePuller: func(_ context.Context, reference string) (networkingImagePullOutput, error) { + mu.Lock() + defer mu.Unlock() + references = append(references, reference) + return networkingImagePullOutput{ + Reference: reference, + Digest: "sha256:test", + Snapshotter: "overlayfs", + Outcome: "pulled", + }, nil + }, + } + images := []networkingImageEntry{ + {Reference: "mcr.microsoft.com/a:v1"}, + {Reference: "mcr.microsoft.com/b:v2"}, + } + results, err := app.pullNetworkingImages(context.Background(), images) + require.NoError(t, err) + require.Len(t, results, 2) + assert.ElementsMatch(t, []string{"mcr.microsoft.com/a:v1", "mcr.microsoft.com/b:v2"}, references) + assert.Equal(t, "sha256:test", results[0].Digest) +} + +func TestPullNetworkingImageValidatesImageFetcherOutput(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "image-fetcher") + require.NoError(t, os.WriteFile(path, []byte(`#!/bin/sh +printf '%s' '{"reference":"mcr.microsoft.com/wrong:v1","digest":"sha256:test","snapshotter":"overlayfs","outcome":"pulled"}' +`), 0o700)) + + app := &App{} + _, err := app.pullNetworkingImage(context.Background(), path, "mcr.microsoft.com/a:v1") + require.Error(t, err) + assert.Contains(t, err.Error(), `returned reference "mcr.microsoft.com/wrong:v1"`) +} + +func TestRunPrepullNetworkingImagesCommandIsFailOpenAndWritesStatus(t *testing.T) { + dir := t.TempDir() + nodeConfigPath := filepath.Join(dir, "node-config.json") + require.NoError(t, os.WriteFile(nodeConfigPath, []byte(`{ + "version":"v1", + "kubeletConfig":{"kubeletNodeLabels":{"kubernetes.azure.com/agentpool":"pool1"}} + }`), 0o600)) + statusPath := filepath.Join(dir, "status.json") + + app := &App{ + networkingImagePuller: func(context.Context, string) (networkingImagePullOutput, error) { + return networkingImagePullOutput{}, assert.AnError + }, + nodeConfigPath: nodeConfigPath, + networkingImageCacheStatusPath: statusPath, + networkingImageCacheFetcher: func(context.Context) ([]byte, error) { + return []byte(`{"schemaVersion":2,"agentPools":{"pool1":{"images":[{"reference":"mcr.microsoft.com/a:v1"}]}}}`), nil + }, + } + require.NoError(t, app.runPrepullNetworkingImagesCommand(context.Background())) + + raw, err := os.ReadFile(statusPath) + require.NoError(t, err) + var status networkingImageCacheStatus + require.NoError(t, json.Unmarshal(raw, &status)) + assert.Equal(t, "failed", status.Outcome) + assert.Equal(t, "pool1", status.AgentPool) + assert.NotEmpty(t, status.ComponentFingerprint) + require.Len(t, status.Images, 1) + assert.Equal(t, "failed", status.Images[0].Outcome) +} + +func TestRunPrepullNetworkingImagesCommandReportsPartialSuccess(t *testing.T) { + dir := t.TempDir() + nodeConfigPath := filepath.Join(dir, "node-config.json") + require.NoError(t, os.WriteFile(nodeConfigPath, []byte(`{ + "version":"v1", + "kubeletConfig":{"kubeletNodeLabels":{"kubernetes.azure.com/agentpool":"pool1"}} + }`), 0o600)) + statusPath := filepath.Join(dir, "status.json") + + app := &App{ + networkingImagePuller: func(_ context.Context, reference string) (networkingImagePullOutput, error) { + if reference == "mcr.microsoft.com/b:v1" { + return networkingImagePullOutput{}, assert.AnError + } + return networkingImagePullOutput{ + Reference: reference, + Digest: "sha256:test", + Snapshotter: "overlayfs", + Outcome: "pulled", + }, nil + }, + nodeConfigPath: nodeConfigPath, + networkingImageCacheStatusPath: statusPath, + networkingImageCacheFetcher: func(context.Context) ([]byte, error) { + return []byte(`{ + "schemaVersion":2, + "agentPools":{"pool1":{"images":[ + {"reference":"mcr.microsoft.com/a:v1"}, + {"reference":"mcr.microsoft.com/b:v1"} + ]}} + }`), nil + }, + } + require.NoError(t, app.runPrepullNetworkingImagesCommand(context.Background())) + + raw, err := os.ReadFile(statusPath) + require.NoError(t, err) + var status networkingImageCacheStatus + require.NoError(t, json.Unmarshal(raw, &status)) + assert.Equal(t, "partial", status.Outcome) + require.Len(t, status.Images, 2) + assert.Equal(t, "pulled", status.Images[0].Outcome) + assert.Equal(t, "failed", status.Images[1].Outcome) +} + +func TestRunPrepullNetworkingImagesCommandSelectsOnlyLocalPoolAndReportsComplete(t *testing.T) { + dir := t.TempDir() + nodeConfigPath := filepath.Join(dir, "node-config.json") + require.NoError(t, os.WriteFile(nodeConfigPath, []byte(`{ + "version":"v1", + "kubeletConfig":{"kubeletNodeLabels":{"kubernetes.azure.com/agentpool":"pool1"}} + }`), 0o600)) + statusPath := filepath.Join(dir, "status.json") + + var pulled []string + app := &App{ + networkingImagePuller: func(_ context.Context, reference string) (networkingImagePullOutput, error) { + pulled = append(pulled, reference) + return networkingImagePullOutput{ + Reference: reference, + Digest: "sha256:test", + Snapshotter: "overlayfs", + Outcome: "cache-hit", + }, nil + }, + nodeConfigPath: nodeConfigPath, + networkingImageCacheStatusPath: statusPath, + networkingImageCacheFetcher: func(context.Context) ([]byte, error) { + return []byte(`{ + "schemaVersion":2, + "agentPools":{ + "pool2":{"images":[{"reference":"mcr.microsoft.com/other:v1"}]}, + "pool1":{"images":[{"reference":"mcr.microsoft.com/local:v1"}]} + } + }`), nil + }, + } + require.NoError(t, app.runPrepullNetworkingImagesCommand(context.Background())) + assert.Equal(t, []string{"mcr.microsoft.com/local:v1"}, pulled) + + raw, err := os.ReadFile(statusPath) + require.NoError(t, err) + var status networkingImageCacheStatus + require.NoError(t, json.Unmarshal(raw, &status)) + assert.Equal(t, networkingImageCacheSchemaVersion, status.SchemaVersion) + assert.Equal(t, "pool1", status.AgentPool) + assert.Equal(t, "complete", status.Outcome) + require.Len(t, status.Images, 1) + assert.Equal(t, "cache-hit", status.Images[0].Outcome) +} + +func TestRunPrepullNetworkingImagesCommandSkipsMissingOrEmptyLocalInventory(t *testing.T) { + tests := []struct { + name string + payload string + wantErr string + }{ + { + name: "missing local pool", + payload: `{"schemaVersion":2,"agentPools":{"pool2":{"images":[{"reference":"mcr.microsoft.com/other:v1"}]}}}`, + wantErr: `no inventory for local agent pool "pool1"`, + }, + { + name: "empty local inventory", + payload: `{"schemaVersion":2,"agentPools":{"pool1":{"images":[]}}}`, + wantErr: `no images for local agent pool "pool1"`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + nodeConfigPath := filepath.Join(dir, "node-config.json") + require.NoError(t, os.WriteFile(nodeConfigPath, []byte(`{ + "version":"v1", + "kubeletConfig":{"kubeletNodeLabels":{"kubernetes.azure.com/agentpool":"pool1"}} + }`), 0o600)) + statusPath := filepath.Join(dir, "status.json") + var pullCalls atomic.Int32 + app := &App{ + networkingImagePuller: func(context.Context, string) (networkingImagePullOutput, error) { + pullCalls.Add(1) + return networkingImagePullOutput{}, nil + }, + nodeConfigPath: nodeConfigPath, + networkingImageCacheStatusPath: statusPath, + networkingImageCacheFetcher: func(context.Context) ([]byte, error) { + return []byte(tt.payload), nil + }, + } + + require.NoError(t, app.runPrepullNetworkingImagesCommand(context.Background())) + assert.Zero(t, pullCalls.Load()) + + raw, err := os.ReadFile(statusPath) + require.NoError(t, err) + var status networkingImageCacheStatus + require.NoError(t, json.Unmarshal(raw, &status)) + assert.Equal(t, "pool1", status.AgentPool) + assert.Equal(t, "skipped", status.Outcome) + assert.Contains(t, status.Error, tt.wantErr) + assert.Empty(t, status.Images) + }) + } +} + +func TestRunPrepullNetworkingImagesCommandSkipsMissingLocalPoolLabel(t *testing.T) { + dir := t.TempDir() + nodeConfigPath := filepath.Join(dir, "node-config.json") + require.NoError(t, os.WriteFile(nodeConfigPath, []byte(`{ + "version":"v1", + "kubeletConfig":{"kubeletNodeLabels":{}} + }`), 0o600)) + statusPath := filepath.Join(dir, "status.json") + var fetchCalls atomic.Int32 + app := &App{ + nodeConfigPath: nodeConfigPath, + networkingImageCacheStatusPath: statusPath, + networkingImageCacheFetcher: func(context.Context) ([]byte, error) { + fetchCalls.Add(1) + return nil, assert.AnError + }, + } + + require.NoError(t, app.runPrepullNetworkingImagesCommand(context.Background())) + assert.Zero(t, fetchCalls.Load()) + + raw, err := os.ReadFile(statusPath) + require.NoError(t, err) + var status networkingImageCacheStatus + require.NoError(t, json.Unmarshal(raw, &status)) + assert.Equal(t, "skipped", status.Outcome) + assert.Contains(t, status.Error, "node config has no kubernetes.azure.com/agentpool label") +} diff --git a/image-fetcher/cli.go b/image-fetcher/cli.go index 2d61e9bb949..7d388c6a391 100644 --- a/image-fetcher/cli.go +++ b/image-fetcher/cli.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "io" @@ -45,6 +46,29 @@ func newCommand() *cli.Command { }) }, }, + { + Name: "pull-image", + Usage: "Ensure one image is fully pulled and unpacked for CRI", + Flags: []cli.Flag{ + &cli.StringFlag{Name: "image", Required: true, Usage: "Image reference to pull and unpack"}, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + if cmd.Args().Len() != 0 { + return errors.New("pull-image accepts no positional arguments; use --image") + } + if strings.TrimSpace(cmd.String("image")) == "" { + return errors.New("--image must not be empty") + } + return withContainerd(ctx, func(ctx context.Context, client *containerd.Client) error { + ref := cmd.String("image") + result, err := pullAndUnpackImage(ctx, client, ref) + if err != nil { + return fmt.Errorf("%s: %w", ref, err) + } + return json.NewEncoder(os.Stdout).Encode(result) + }) + }, + }, { Name: "cache-components", Usage: "Cache the Linux container images declared in components.json", diff --git a/image-fetcher/image.go b/image-fetcher/image.go index 4e7d338cde3..a790e940c74 100644 --- a/image-fetcher/image.go +++ b/image-fetcher/image.go @@ -1,10 +1,13 @@ package main import ( + "bufio" "context" "fmt" "os" "runtime" + "strconv" + "strings" containerd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/core/images" @@ -13,13 +16,130 @@ import ( ) const ( - defaultSocket = "/run/containerd/containerd.sock" - defaultNS = "k8s.io" + defaultSocket = "/run/containerd/containerd.sock" + defaultNS = "k8s.io" + defaultSnapshotter = "overlayfs" + containerdConfig = "/etc/containerd/config.toml" // images with compressed content size below this threshold are // unpacked after fetch, effectively turning the operation into a // full pull (~200 MiB compressed ≈ ~400 MiB unpacked). pullSizeThreshold = 200 * 1024 * 1024 // 200 MiB ) +type pullImageResult struct { + Reference string `json:"reference"` + Digest string `json:"digest"` + Snapshotter string `json:"snapshotter"` + Outcome string `json:"outcome"` +} + +func pullAndUnpackImage(ctx context.Context, client *containerd.Client, ref string) (pullImageResult, error) { + platform := fmt.Sprintf("linux/%s", runtime.GOARCH) + p, err := platforms.Parse(platform) + if err != nil { + return pullImageResult{}, fmt.Errorf("parse platform %s: %w", platform, err) + } + platformMatcher := platforms.OnlyStrict(p) + snapshotter := detectSnapshotter(containerdConfig) + + if imageMeta, err := client.GetImage(ctx, ref); err == nil { + image := containerd.NewImageWithPlatform(client, imageMeta.Metadata(), platformMatcher) + if err := validateImageContentAndPlatform(ctx, image, p); err == nil { + unpacked, unpackErr := image.IsUnpacked(ctx, snapshotter) + if unpackErr == nil && unpacked { + return pullImageResult{ + Reference: imageMeta.Name(), + Digest: imageMeta.Target().Digest.String(), + Snapshotter: snapshotter, + Outcome: "cache-hit", + }, nil + } + if unpackErr == nil { + if err := image.Unpack(ctx, snapshotter); err == nil { + return pullImageResult{ + Reference: imageMeta.Name(), + Digest: imageMeta.Target().Digest.String(), + Snapshotter: snapshotter, + Outcome: "unpacked", + }, nil + } + } + } + } + + image, err := client.Pull(ctx, ref, + containerd.WithPlatformMatcher(platformMatcher), + containerd.WithPullSnapshotter(snapshotter), + containerd.WithPullUnpack, + ) + if err != nil { + return pullImageResult{}, fmt.Errorf("pull and unpack failed: %w", err) + } + if err := validateImageContentAndPlatform(ctx, image, p); err != nil { + return pullImageResult{}, err + } + unpacked, err := image.IsUnpacked(ctx, snapshotter) + if err != nil { + return pullImageResult{}, fmt.Errorf("checking unpacked image: %w", err) + } + if !unpacked { + return pullImageResult{}, fmt.Errorf("image is not unpacked in snapshotter %s", snapshotter) + } + return pullImageResult{ + Reference: image.Name(), + Digest: image.Target().Digest.String(), + Snapshotter: snapshotter, + Outcome: "pulled", + }, nil +} + +func validateImageContentAndPlatform(ctx context.Context, image containerd.Image, expected ocispec.Platform) error { + if err := validateImagePlatform(ctx, image, expected); err != nil { + return err + } + if _, err := image.Size(ctx); err != nil { + return fmt.Errorf("validating image content: %w", err) + } + return nil +} + +func detectSnapshotter(path string) string { + if snapshotter := strings.TrimSpace(os.Getenv("CONTAINERD_SNAPSHOTTER")); snapshotter != "" { + return snapshotter + } + file, err := os.Open(path) + if err != nil { + return defaultSnapshotter + } + defer file.Close() + + section := "" + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(strings.SplitN(scanner.Text(), "#", 2)[0]) + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + section = line + continue + } + isCRISection := strings.Contains(section, "io.containerd.cri.v1.images") || + (strings.Contains(section, "io.containerd.grpc.v1.cri") && strings.Contains(section, ".containerd")) + if !isCRISection || !strings.HasPrefix(line, "snapshotter") { + continue + } + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + continue + } + value := strings.TrimSpace(parts[1]) + if unquoted, err := strconv.Unquote(value); err == nil && unquoted != "" { + return unquoted + } + if len(value) >= 2 && value[0] == '\'' && value[len(value)-1] == '\'' { + return value[1 : len(value)-1] + } + } + return defaultSnapshotter +} + // fetchImage uses client.Fetch() which: // - Downloads all blobs (manifest, config, layers) into the content store // - Creates an image record in the metadata database diff --git a/image-fetcher/image_test.go b/image-fetcher/image_test.go new file mode 100644 index 00000000000..d77b55bf39a --- /dev/null +++ b/image-fetcher/image_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestDetectSnapshotter(t *testing.T) { + t.Setenv("CONTAINERD_SNAPSHOTTER", "") + path := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(path, []byte(` +[plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "overlaybd" +`), 0o600); err != nil { + t.Fatal(err) + } + if got := detectSnapshotter(path); got != "overlaybd" { + t.Fatalf("detectSnapshotter() = %q, want overlaybd", got) + } +} + +func TestDetectSnapshotterV2Config(t *testing.T) { + t.Setenv("CONTAINERD_SNAPSHOTTER", "") + path := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(path, []byte(` +[plugins.'io.containerd.cri.v1.images'] + snapshotter = 'erofs' +`), 0o600); err != nil { + t.Fatal(err) + } + if got := detectSnapshotter(path); got != "erofs" { + t.Fatalf("detectSnapshotter() = %q, want erofs", got) + } +} + +func TestDetectSnapshotterOverrideAndFallback(t *testing.T) { + t.Setenv("CONTAINERD_SNAPSHOTTER", "native") + if got := detectSnapshotter(filepath.Join(t.TempDir(), "missing")); got != "native" { + t.Fatalf("detectSnapshotter() = %q, want native", got) + } + + t.Setenv("CONTAINERD_SNAPSHOTTER", "") + if got := detectSnapshotter(filepath.Join(t.TempDir(), "missing")); got != defaultSnapshotter { + t.Fatalf("detectSnapshotter() = %q, want %q", got, defaultSnapshotter) + } +} diff --git a/parts/linux/cloud-init/artifacts/aks-networking-image-cache.sh b/parts/linux/cloud-init/artifacts/aks-networking-image-cache.sh new file mode 100644 index 00000000000..da906009cc4 --- /dev/null +++ b/parts/linux/cloud-init/artifacts/aks-networking-image-cache.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -uo pipefail + +FEATURES_PATH="${FEATURES_PATH:-/opt/azure/containers/enabled_features.sh}" +LAUNCHER_PATH="${LAUNCHER_PATH:-/opt/azure/containers/aks-node-controller-launcher.sh}" +grep -qx 'ENABLE_NETWORKING_IMAGE_CACHE=true' "$FEATURES_PATH" 2>/dev/null || exit 0 +exec "$LAUNCHER_PATH" prepull-networking-images diff --git a/parts/linux/cloud-init/artifacts/aks-node-controller-launcher.sh b/parts/linux/cloud-init/artifacts/aks-node-controller-launcher.sh index e65e16e5f8d..66556f5348c 100644 --- a/parts/linux/cloud-init/artifacts/aks-node-controller-launcher.sh +++ b/parts/linux/cloud-init/artifacts/aks-node-controller-launcher.sh @@ -39,6 +39,11 @@ source "$(dirname "${BASH_SOURCE[0]}")/aks-node-controller-hotfix.sh" # this is to ensure that shellspec won't interpret any further lines below ${__SOURCED__:+return} +if [ "${1:-}" = "prepull-networking-images" ]; then + anc_run_hotfix_flow "$BIN_PATH" "$HOTFIX_BIN" "$HOTFIX_JSON" "$FEATURES_PATH" + exec "$ANC_HOTFIX_SELECTED_BIN" prepull-networking-images +fi + if [ ! -f "$CONFIG_PATH" ] && [ ! -f "$NBC_CMD_PATH" ]; then log "Gracefully exit aks-node-controller without provision config or nbc cmd" exit 0 diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index 25e750f832f..9a4db7a5855 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -609,7 +609,7 @@ function nodePrep { exit $VALIDATION_ERR fi - checkServiceHealth containerd || exit $ERR_SYSTEMCTL_START_FAIL + checkServiceHealth containerd;nic || exit $ERR_SYSTEMCTL_START_FAIL if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" = "true" ]; then checkServiceHealth secure-tls-bootstrap || true fi diff --git a/parts/linux/cloud-init/nodecustomdata.yml b/parts/linux/cloud-init/nodecustomdata.yml index 200eeb0ce5f..8fef733532f 100644 --- a/parts/linux/cloud-init/nodecustomdata.yml +++ b/parts/linux/cloud-init/nodecustomdata.yml @@ -35,6 +35,15 @@ write_files: {{end}} {{- else }} +{{if GetVariableProperty "cloudInitData" "enabledFeatures"}} +- path: /opt/azure/containers/enabled_features.sh + permissions: "0600" + encoding: gzip + owner: root + content: !!binary | + {{GetVariableProperty "cloudInitData" "enabledFeatures"}} + +{{end}} - path: {{GetCSEHelpersScriptFilepath}} permissions: "0744" encoding: gzip diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 91eacc7f9fb..1816e0674d4 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -322,6 +322,14 @@ func renderEnabledFeatures(features map[string]string) string { return b.String() } +func getBase64EncodedGzippedEnabledFeatures(features map[string]string) string { + content := renderEnabledFeatures(features) + if content == "" { + return "" + } + return getBase64EncodedGzippedCustomScriptFromStr(content) +} + // featureKeyRe matches a valid shell identifier ([a-zA-Z_][a-zA-Z0-9_]*) - the same set the // aks-node-controller wrapper parses out of enabled_features.sh. var featureKeyRe = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index d0d34b9e0cd..0dae5301de6 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -1366,6 +1366,30 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { Expect(string(decodedPayload)).To(ContainSubstring(encodedEnabledFeatures)) }) + It("should embed the enabled_features file in classic cloud-init when EnabledFeatures is set", func() { + templateGenerator := InitializeTemplateGenerator() + config := newConfig(false) + config.EnabledFeatures = map[string]string{ + "ZED_FEATURE": "1", + "ENABLE_PROVISIONING_HOTFIX": "true", + } + + customData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(config)) + + encodedEnabledFeatures := getBase64EncodedGzippedCustomScriptFromStr("ENABLE_PROVISIONING_HOTFIX=true\nZED_FEATURE=1\n") + Expect(customData).To(ContainSubstring(enabledFeaturesFilepath)) + Expect(customData).To(ContainSubstring(encodedEnabledFeatures)) + }) + + It("should not embed the enabled_features file in classic cloud-init when EnabledFeatures is empty", func() { + templateGenerator := InitializeTemplateGenerator() + config := newConfig(false) + + customData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(config)) + + Expect(customData).NotTo(ContainSubstring(enabledFeaturesFilepath)) + }) + It("should render multiple enabled features as sorted KEY=VALUE lines", func() { templateGenerator := InitializeTemplateGenerator() config := newConfig(false) diff --git a/pkg/agent/variables.go b/pkg/agent/variables.go index 7efc595a410..9566ab4cbbd 100644 --- a/pkg/agent/variables.go +++ b/pkg/agent/variables.go @@ -15,6 +15,7 @@ func getCustomDataVariables(config *datamodel.NodeBootstrappingConfiguration) pa cs := config.ContainerService cloudInitFiles := map[string]interface{}{ "cloudInitData": paramsMap{ + "enabledFeatures": getBase64EncodedGzippedEnabledFeatures(config.EnabledFeatures), "provisionStartScript": getBase64EncodedGzippedCustomScript(kubernetesCSEStartScript, config), "provisionScript": getBase64EncodedGzippedCustomScript(kubernetesCSEMainScript, config), "provisionSource": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScript, config), diff --git a/spec/parts/linux/cloud-init/artifacts/aks_networking_image_cache_spec.sh b/spec/parts/linux/cloud-init/artifacts/aks_networking_image_cache_spec.sh new file mode 100644 index 00000000000..017dd740b2f --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/aks_networking_image_cache_spec.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env shellspec + +Describe 'aks-networking-image-cache.sh' + SCRIPT="./parts/linux/cloud-init/artifacts/aks-networking-image-cache.sh" + + setup_networking_image_cache_test() { + TEST_DIR="${SHELLSPEC_WORKDIR}/aks-networking-image-cache" + mkdir -p "$TEST_DIR/bin" + export TEST_DIR + export FEATURES_PATH="${TEST_DIR}/enabled_features.sh" + export LAUNCHER_PATH="${TEST_DIR}/launcher" + cat >"${TEST_DIR}/launcher" <<'EOF' +#!/bin/bash +printf '%s\n' "$@" >"${TEST_DIR}/args" +EOF + chmod +x "${TEST_DIR}/launcher" + } + + cleanup_networking_image_cache_test() { + rm -rf "$TEST_DIR" + unset TEST_DIR FEATURES_PATH LAUNCHER_PATH + } + + BeforeEach setup_networking_image_cache_test + AfterEach cleanup_networking_image_cache_test + + It 'is a no-op when the feature is disabled' + When run bash "$SCRIPT" + The status should be success + The path "${TEST_DIR}/args" should not be exist + End + + It 'invokes ANC prefetch when the feature is enabled' + printf 'ENABLE_NETWORKING_IMAGE_CACHE=true\n' >"$FEATURES_PATH" + + When run bash "$SCRIPT" + The status should be success + args=$(cat "${TEST_DIR}/args") + The variable args should eq "prepull-networking-images" + End +End diff --git a/spec/parts/linux/cloud-init/artifacts/aks_node_controller_launcher_spec.sh b/spec/parts/linux/cloud-init/artifacts/aks_node_controller_launcher_spec.sh index 28c8c196846..9bae6f430c0 100644 --- a/spec/parts/linux/cloud-init/artifacts/aks_node_controller_launcher_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/aks_node_controller_launcher_spec.sh @@ -151,6 +151,16 @@ EOF The output should not include "Spawned aks-node-controller" End + It 'pre-pulls networking images without requiring provision config' + create_fake_aks_node_controller + + When run bash "$SCRIPT" prepull-networking-images + The status should be success + The output should include "Using VHD-baked binary" + firstArg=$(sed -n '1p' "${TEST_DIR}/args") + The variable firstArg should eq "prepull-networking-images" + End + It 'passes both provision config and nbc cmd when both files are present' touch "$CONFIG_PATH" "$NBC_CMD_PATH" create_fake_aks_node_controller diff --git a/spec/parts/linux/cloud-init/artifacts/cse_main_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_main_spec.sh index 2f8f24d1b33..63d052a318b 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_main_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_main_spec.sh @@ -135,6 +135,19 @@ Describe 'NVIDIA Fabric Manager startup timeout' End End +Describe 'networking image pre-pull ordering' + It 'runs after containerd is healthy and before kubelet starts' + ordering_is_valid() { + prepull_line="$(grep -n 'checkServiceHealth containerd;nic' parts/linux/cloud-init/artifacts/cse_main.sh | cut -d: -f1)" + kubelet_line="$(grep -n 'logs_to_events "AKS.CSE.ensureKubelet"' parts/linux/cloud-init/artifacts/cse_main.sh | cut -d: -f1)" + [ -n "${prepull_line}" ] && [ "${prepull_line}" -lt "${kubelet_line}" ] + } + + When call ordering_is_valid + The status should be success + End +End + Describe 'proxy environment exports' setup() { unset HTTP_PROXY http_proxy HTTPS_PROXY https_proxy NO_PROXY no_proxy diff --git a/vhdbuilder/packer/cleanup-vhd.sh b/vhdbuilder/packer/cleanup-vhd.sh index 10e73fff409..acd02a6085c 100644 --- a/vhdbuilder/packer/cleanup-vhd.sh +++ b/vhdbuilder/packer/cleanup-vhd.sh @@ -120,7 +120,6 @@ if [ -f /boot/acl/uki-addons/firstboot.addon.efi ]; then fi # Cleanup disk usage diagnostics file (created by generate-disk-usage.sh) rm -f /opt/azure/disk-usage.txt -# remove image-fetcher binary from the image since it's only needed during build and is not expected to be present on the final image -rm -f /opt/azure/containers/image-fetcher +# image-fetcher remains on the node for the pre-kubelet networking image cache path. # Cleanup IMDS instance metadata cache file rm -f /opt/azure/containers/imds_instance_metadata_cache.json diff --git a/vhdbuilder/packer/packer_source.sh b/vhdbuilder/packer/packer_source.sh index 3b088e23268..9106682e922 100644 --- a/vhdbuilder/packer/packer_source.sh +++ b/vhdbuilder/packer/packer_source.sh @@ -147,6 +147,8 @@ copyPackerFiles() { AKS_CHECK_NETWORK_SERVICE_DEST=/etc/systemd/system/aks-check-network.service AKS_NODE_CONTROLLER_LAUNCHER_SRC=/home/packer/aks-node-controller-launcher.sh AKS_NODE_CONTROLLER_LAUNCHER_DEST=/opt/azure/containers/aks-node-controller-launcher.sh + AKS_NETWORKING_IMAGE_CACHE_SRC=/home/packer/aks-networking-image-cache.sh + AKS_NETWORKING_IMAGE_CACHE_DEST=/opt/azure/containers/aks-networking-image-cache.sh AKS_NODE_CONTROLLER_HOTFIX_SRC=/home/packer/aks-node-controller-hotfix.sh AKS_NODE_CONTROLLER_HOTFIX_DEST=/opt/azure/containers/aks-node-controller-hotfix.sh BLOCK_WIRESERVER_SRC=/home/packer/block_wireserver.sh @@ -337,6 +339,8 @@ copyPackerFiles() { AKS_NODE_CONTROLLER_DEST=/opt/azure/containers/aks-node-controller cpAndMode $AKS_NODE_CONTROLLER_SRC $AKS_NODE_CONTROLLER_DEST 755 cpAndMode $AKS_NODE_CONTROLLER_LAUNCHER_SRC $AKS_NODE_CONTROLLER_LAUNCHER_DEST 0755 + cpAndMode $AKS_NETWORKING_IMAGE_CACHE_SRC $AKS_NETWORKING_IMAGE_CACHE_DEST 0755 + ln -sf $AKS_NETWORKING_IMAGE_CACHE_DEST /usr/local/bin/nic cpAndMode $AKS_NODE_CONTROLLER_HOTFIX_SRC $AKS_NODE_CONTROLLER_HOTFIX_DEST 0755 AKS_NODE_CONTROLLER_SERVICE_SRC=/home/packer/aks-node-controller.service diff --git a/vhdbuilder/packer/vhd-image-builder-acl-arm64.json b/vhdbuilder/packer/vhd-image-builder-acl-arm64.json index 615ef010b8b..4e5c505e430 100644 --- a/vhdbuilder/packer/vhd-image-builder-acl-arm64.json +++ b/vhdbuilder/packer/vhd-image-builder-acl-arm64.json @@ -108,6 +108,11 @@ "source": "parts/linux/cloud-init/artifacts/aks-node-controller-launcher.sh", "destination": "/home/packer/aks-node-controller-launcher.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/aks-networking-image-cache.sh", + "destination": "/home/packer/aks-networking-image-cache.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-acl.json b/vhdbuilder/packer/vhd-image-builder-acl.json index d4eca8a7cd0..a0fd3df7f24 100644 --- a/vhdbuilder/packer/vhd-image-builder-acl.json +++ b/vhdbuilder/packer/vhd-image-builder-acl.json @@ -108,6 +108,11 @@ "source": "parts/linux/cloud-init/artifacts/aks-node-controller-launcher.sh", "destination": "/home/packer/aks-node-controller-launcher.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/aks-networking-image-cache.sh", + "destination": "/home/packer/aks-networking-image-cache.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json index a74db09e3c7..bd9e081247c 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gb.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gb.json @@ -109,6 +109,11 @@ "source": "parts/linux/cloud-init/artifacts/aks-node-controller-launcher.sh", "destination": "/home/packer/aks-node-controller-launcher.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/aks-networking-image-cache.sh", + "destination": "/home/packer/aks-networking-image-cache.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json index d60eba4569f..1db9562377e 100644 --- a/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json +++ b/vhdbuilder/packer/vhd-image-builder-arm64-gen2.json @@ -109,6 +109,11 @@ "source": "parts/linux/cloud-init/artifacts/aks-node-controller-launcher.sh", "destination": "/home/packer/aks-node-controller-launcher.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/aks-networking-image-cache.sh", + "destination": "/home/packer/aks-networking-image-cache.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-base.json b/vhdbuilder/packer/vhd-image-builder-base.json index c1a7cfe1c23..7c7d0c2c389 100644 --- a/vhdbuilder/packer/vhd-image-builder-base.json +++ b/vhdbuilder/packer/vhd-image-builder-base.json @@ -112,6 +112,11 @@ "source": "parts/linux/cloud-init/artifacts/aks-node-controller-launcher.sh", "destination": "/home/packer/aks-node-controller-launcher.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/aks-networking-image-cache.sh", + "destination": "/home/packer/aks-networking-image-cache.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-cvm.json b/vhdbuilder/packer/vhd-image-builder-cvm.json index 228b2187700..45cb8cd7429 100644 --- a/vhdbuilder/packer/vhd-image-builder-cvm.json +++ b/vhdbuilder/packer/vhd-image-builder-cvm.json @@ -115,6 +115,11 @@ "source": "parts/linux/cloud-init/artifacts/aks-node-controller-launcher.sh", "destination": "/home/packer/aks-node-controller-launcher.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/aks-networking-image-cache.sh", + "destination": "/home/packer/aks-networking-image-cache.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json b/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json index 533ed2fe46b..6120880d266 100644 --- a/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json +++ b/vhdbuilder/packer/vhd-image-builder-mariner-arm64.json @@ -108,6 +108,11 @@ "source": "parts/linux/cloud-init/artifacts/aks-node-controller-launcher.sh", "destination": "/home/packer/aks-node-controller-launcher.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/aks-networking-image-cache.sh", + "destination": "/home/packer/aks-networking-image-cache.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json b/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json index 333522d9efc..ef2f31014c1 100644 --- a/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json +++ b/vhdbuilder/packer/vhd-image-builder-mariner-cvm.json @@ -118,6 +118,11 @@ "source": "parts/linux/cloud-init/artifacts/aks-node-controller-launcher.sh", "destination": "/home/packer/aks-node-controller-launcher.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/aks-networking-image-cache.sh", + "destination": "/home/packer/aks-networking-image-cache.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.sh", diff --git a/vhdbuilder/packer/vhd-image-builder-mariner.json b/vhdbuilder/packer/vhd-image-builder-mariner.json index 22351f79469..d19e26deebf 100644 --- a/vhdbuilder/packer/vhd-image-builder-mariner.json +++ b/vhdbuilder/packer/vhd-image-builder-mariner.json @@ -113,6 +113,11 @@ "source": "parts/linux/cloud-init/artifacts/aks-node-controller-launcher.sh", "destination": "/home/packer/aks-node-controller-launcher.sh" }, + { + "type": "file", + "source": "parts/linux/cloud-init/artifacts/aks-networking-image-cache.sh", + "destination": "/home/packer/aks-networking-image-cache.sh" + }, { "type": "file", "source": "parts/linux/cloud-init/artifacts/aks-node-controller-hotfix.sh",