From 97de2198c58390b8614c455788367f18ee49305c Mon Sep 17 00:00:00 2001 From: Abigailliang Date: Mon, 26 Oct 2026 14:53:54 -0700 Subject: [PATCH] feat: add native RepoDepot initialization [2/3] Split repository configuration and package-cache refresh from #9891 on top of the certificate/runtime layer. Keep provisioning on its existing shell path, preserve explicit cache and cancellation errors, and include direct source/keyring, retry, and endpoint coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: be499bac-1283-498e-b5b1-59d8c91047a4 --- aks-node-controller/README.md | 16 ++ aks-node-controller/cloud_repositories.go | 268 ++++++++++++++++++ .../cloud_repositories_configuration_test.go | 89 ++++++ .../cloud_repositories_test.go | 81 ++++++ 4 files changed, 454 insertions(+) create mode 100644 aks-node-controller/cloud_repositories.go create mode 100644 aks-node-controller/cloud_repositories_configuration_test.go create mode 100644 aks-node-controller/cloud_repositories_test.go diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index 4a8532fee1f..ab7da4ee2ec 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -81,6 +81,22 @@ This command does not configure repositories or install refresh schedules. Existing provisioning still uses `init-aks-cloud.sh`; native cloud initialization is not yet connected to the provisioning entry point. +### Native RepoDepot capabilities + +The Go cloud runtime includes RepoDepot configuration for Ubuntu, Mariner, and +Azure Linux. Ubuntu sources and Microsoft keyrings, Mariner mirrors, and Azure +Linux repositories retain their existing roles. Empty endpoints and immutable +Flatcar/ACL images skip package-repository configuration. + +Cache operations retry up to ten times and return a classified failure when +retries are exhausted. Cancellation remains a context error, distinct from an +ordinary cache-refresh failure. Repository configuration failures also remain +errors; the eventual provisioning caller is responsible for the shell-compatible +best-effort cache policy. + +These capabilities are not yet invoked by provisioning. The existing shell +initialization path and refresh schedules remain unchanged. + ### Validating and normalizing AKSNodeConfig Before serializing a typed config, producers such as RP should call: diff --git a/aks-node-controller/cloud_repositories.go b/aks-node-controller/cloud_repositories.go new file mode 100644 index 00000000000..a899d082325 --- /dev/null +++ b/aks-node-controller/cloud_repositories.go @@ -0,0 +1,268 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + + "github.com/Azure/agentbaker/aks-node-controller/common" +) + +const cloudRepoSchemeHTTP = "http" + +const cloudRepoSchemeHTTPS = "https" + +const ubuntuCloudSource = ` +Types: deb +URIs: %s/ubuntu +Suites: %s %s-updates %s-backports %s-security +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +` + +const microsoftCloudSource = ` +Types: deb +URIs: %s +Suites: %s +Components: main +Architectures: %s +Signed-By: %s +` + +const azureLinuxCloudRepo = `[azurelinux-official-%s] +name=Azure Linux Official %s $releasever $basearch +baseurl=%s/azurelinux/$releasever/prod/%s/$basearch +gpgkey=file:///etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY +gpgcheck=1 +repo_gpgcheck=1 +enabled=1 +skip_if_unavailable=True +sslverify=1 +` + +var errCloudRepositoryCacheRefresh = errors.New("refresh RepoDepot package cache") + +func (c *cloudInitializer) initializeRepositories(ctx context.Context, platform cloudPlatform, endpoint string) error { + if endpoint == "" { + slog.Info("RepoDepot endpoint is empty, skipping repository initialization") + return nil + } + endpoint = strings.TrimSuffix(strings.TrimRight(endpoint, "/"), "/ubuntu") + parsed, err := url.Parse(endpoint) + if err != nil || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || + (parsed.Scheme != cloudRepoSchemeHTTP && parsed.Scheme != cloudRepoSchemeHTTPS) || strings.ContainsAny(endpoint, "\r\n\t ") { + return errors.New("invalid RepoDepot endpoint") + } + switch platform.ID { + case osReleaseIDUbuntu: + if err := c.initializeUbuntuRepositories(ctx, platform, endpoint); err != nil { + return err + } + return c.updateRepositoryCache(ctx, "apt-get", "update") + case osReleaseIDMariner: + if err := c.initializeMarinerRepositories(endpoint); err != nil { + return err + } + return c.updateRepositoryCache(ctx, "dnf", "makecache", "-y") + case osReleaseIDAzureLinux: + if err := c.initializeAzureLinuxRepositories(endpoint); err != nil { + return err + } + return c.updateRepositoryCache(ctx, "dnf", "makecache", "-y") + default: + slog.Info("RepoDepot initialization is not needed on immutable OS", "OS", platform.ID) + return nil + } +} + +func (c *cloudInitializer) repositoryGET(ctx context.Context, endpoint string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + resp, err := c.repoClient.Do(req) + if err != nil { + return nil, err + } + body, err := readCloudResponse(resp) + if err != nil { + return nil, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("RepoDepot request returned HTTP %d", resp.StatusCode) + } + return body, nil +} + +func (c *cloudInitializer) initializeUbuntuRepositories(ctx context.Context, platform cloudPlatform, endpoint string) error { + if platform.Version == "" || platform.Codename == "" || + strings.ContainsAny(platform.Version+platform.Codename, "\r\n\t ") { + return errors.New("missing valid Ubuntu VERSION_ID or VERSION_CODENAME") + } + if err := c.copyOpenSSLBundle(); err != nil { + return fmt.Errorf("copy Ubuntu CA bundle: %w", err) + } + if err := c.backupUbuntuSources(); err != nil { + return err + } + mainSource := fmt.Sprintf(ubuntuCloudSource, + endpoint, platform.Codename, platform.Codename, platform.Codename, platform.Codename) + if err := c.writeFile(defaultAptSourcesDir+"/ubuntu.sources", []byte(mainSource)); err != nil { + return err + } + microsoftSource := endpoint + "/microsoft/ubuntu/" + platform.Version + "/prod" + if _, err := c.repositoryGET(ctx, microsoftSource); err != nil { + return fmt.Errorf("check Ubuntu Microsoft repository: %w", err) + } + keyPaths, err := c.ubuntuRepositoryKeyrings(ctx, endpoint) + if err != nil { + return err + } + microsoftConfig := fmt.Sprintf(microsoftCloudSource, + microsoftSource, platform.Codename, runtime.GOARCH, strings.Join(keyPaths, " ")) + for _, name := range []string{"microsoft-prod", "microsoft-prod-testing"} { + if err := c.writeFile(defaultAptSourcesDir+"/"+name+".sources", []byte(microsoftConfig)); err != nil { + return err + } + } + return nil +} + +func (c *cloudInitializer) backupUbuntuSources() error { + sourcesDir, backupDir := c.path(defaultAptSourcesDir), c.path("/etc/apt/backup") + if err := os.MkdirAll(sourcesDir, 0o755); err != nil { + return err + } + if err := os.MkdirAll(backupDir, 0o755); err != nil { + return err + } + sources, err := filepath.Glob(filepath.Join(sourcesDir, "*")) + if err != nil { + return err + } + sources = append(sources, c.path("/etc/apt/sources.list")) + for _, source := range sources { + info, err := os.Stat(source) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return err + } + if info.Mode().IsRegular() { + if err := os.Rename(source, filepath.Join(backupDir, filepath.Base(source))); err != nil { + return err + } + } + } + return nil +} + +func (c *cloudInitializer) ubuntuRepositoryKeyrings(ctx context.Context, endpoint string) ([]string, error) { + var keyPaths []string + for _, key := range []string{"microsoft.asc", "msopentech.asc"} { + body, err := c.repositoryGET(ctx, endpoint+"/keys/"+key) + if err != nil { + return nil, fmt.Errorf("download RepoDepot key %s: %w", key, err) + } + keyring, err := c.command(ctx, body, "gpg", "--dearmor") + if err != nil { + return nil, err + } + if len(keyring) == 0 { + return nil, fmt.Errorf("RepoDepot key %s produced an empty keyring", key) + } + path := "/etc/apt/keyrings/" + key + ".gpg" + if err := c.writeFile(path, keyring); err != nil { + return nil, err + } + keyPaths = append(keyPaths, path) + } + return keyPaths, nil +} + +func (c *cloudInitializer) initializeMarinerRepositories(endpoint string) error { + source, err := os.ReadFile(c.path(defaultYumReposDir + "/mariner-extras.repo")) + if err != nil { + return err + } + for _, repo := range []struct{ id, title string }{ + {"extended", "Extended"}, {"nvidia", "Nvidia"}, {"cloud-native", "Cloud-Native"}, + } { + content := strings.ReplaceAll(string(source), "extras", repo.id) + content = strings.ReplaceAll(content, "Extras", repo.title) + if writeErr := c.writeFile(defaultYumReposDir+"/mariner-"+repo.id+".repo", []byte(content)); writeErr != nil { + return writeErr + } + } + paths, err := filepath.Glob(c.path(defaultYumReposDir + "/*.repo")) + if err != nil { + return err + } + for _, path := range paths { + content, err := os.ReadFile(path) + if err != nil { + return err + } + content = []byte(strings.ReplaceAll(string(content), "https://packages.microsoft.com", endpoint+"/mariner/packages.microsoft.com")) + if err := writeCloudFile(path, content); err != nil { + return err + } + } + return nil +} + +func (c *cloudInitializer) initializeAzureLinuxRepositories(endpoint string) error { + paths, err := filepath.Glob(c.path(defaultYumReposDir + "/azurelinux*")) + if err != nil { + return err + } + for _, path := range paths { + if err := os.Remove(path); err != nil { + return err + } + } + for _, repo := range []string{"amd", "base", "cloud-native", "extended", "ms-non-oss", "ms-oss", "nvidia"} { + content := fmt.Sprintf(azureLinuxCloudRepo, + repo, repo, endpoint, repo) + if err := c.writeFile(defaultYumReposDir+"/azurelinux-"+repo+".repo", []byte(content)); err != nil { + return err + } + } + return nil +} + +func (c *cloudInitializer) updateRepositoryCache(ctx context.Context, name string, args ...string) error { + attempt := 0 + dnfError := regexp.MustCompile(`(?m)^([WE]:.*)|([eE]rr.*)$`) + _, err := common.RetryFetch(ctx, cloudRequestAttempts, nil, func(ctx context.Context) (struct{}, error) { + if attempt > 0 { + if err := c.wait(ctx, 5*time.Second); err != nil { + return struct{}{}, err + } + } + attempt++ + output, err := c.command(ctx, nil, name, args...) + slog.Info("refreshing RepoDepot package cache", "command", name, "attempt", attempt, "output", string(output)) + if err == nil && name == "dnf" && dnfError.Match(output) { + err = errors.New("dnf reported repository metadata errors") + } + return struct{}{}, err + }) + if err != nil { + if contextErr := ctx.Err(); contextErr != nil { + return fmt.Errorf("refresh RepoDepot package cache: %w", contextErr) + } + return fmt.Errorf("%w: %w", errCloudRepositoryCacheRefresh, err) + } + return nil +} diff --git a/aks-node-controller/cloud_repositories_configuration_test.go b/aks-node-controller/cloud_repositories_configuration_test.go new file mode 100644 index 00000000000..4cf546d3569 --- /dev/null +++ b/aks-node-controller/cloud_repositories_configuration_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "os" + "os/exec" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCloudUbuntuRepositoryConfiguration(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + serveCloudTestResponses(t, c, nil, `{"IsOptedInForRootCerts":false}`) + require.NoError(t, c.writeFile("/etc/apt/sources.list", []byte("original ubuntu sources"))) + require.NoError(t, c.writeFile(defaultAptSourcesDir+"/original.sources", []byte("original extra sources"))) + var commands []string + run := c.run + c.run = func(cmd *exec.Cmd) error { + commands = append(commands, strings.Join(cmd.Args, " ")) + return run(cmd) + } + platform, err := c.platform() + require.NoError(t, err) + require.NoError(t, c.initializeRepositories(t.Context(), platform, c.wireEndpoint+"/ubuntu/")) + require.Equal(t, []string{"gpg --dearmor", "gpg --dearmor", "apt-get update"}, commands) + backup, err := os.ReadFile(c.path("/etc/apt/backup/sources.list")) + require.NoError(t, err) + require.Equal(t, "original ubuntu sources", string(backup)) + backup, err = os.ReadFile(c.path("/etc/apt/backup/original.sources")) + require.NoError(t, err) + require.Equal(t, "original extra sources", string(backup)) + source, err := os.ReadFile(c.path(defaultAptSourcesDir + "/ubuntu.sources")) + require.NoError(t, err) + require.Contains(t, string(source), "URIs: "+c.wireEndpoint+"/ubuntu\n") + require.Contains(t, string(source), "Suites: jammy jammy-updates jammy-backports jammy-security") + for _, name := range []string{"microsoft-prod", "microsoft-prod-testing"} { + source, err := os.ReadFile(c.path(defaultAptSourcesDir + "/" + name + ".sources")) + require.NoError(t, err) + require.Contains(t, string(source), c.wireEndpoint+"/microsoft/ubuntu/22.04/prod") + require.Contains(t, string(source), "Architectures: "+runtime.GOARCH) + require.Contains(t, string(source), "Signed-By: /etc/apt/keyrings/microsoft.asc.gpg /etc/apt/keyrings/msopentech.asc.gpg") + } + for _, name := range []string{"microsoft.asc", "msopentech.asc"} { + key, err := os.ReadFile(c.path("/etc/apt/keyrings/" + name + ".gpg")) + require.NoError(t, err) + require.Equal(t, "dearmored-key", string(key)) + } +} + +func TestCloudRPMRepositoryConfiguration(t *testing.T) { + for _, id := range []string{osReleaseIDMariner, osReleaseIDAzureLinux} { + t.Run(id, func(t *testing.T) { + c := newTestCloudInitializer(t, "ID="+id+"\n") + require.NoError(t, c.writeFile(defaultYumReposDir+"/keep.repo", []byte("keep this repository"))) + require.NoError(t, c.writeFile(defaultYumReposDir+"/azurelinux-old.repo", []byte("obsolete"))) + require.NoError(t, c.writeFile(defaultYumReposDir+"/mariner-extras.repo", + []byte("[mariner-extras]\nname=Mariner Extras\nbaseurl=https://packages.microsoft.com/cbl-mariner/$releasever/extras/$basearch\n"))) + var commands []string + c.run = func(cmd *exec.Cmd) error { + commands = append(commands, strings.Join(cmd.Args, " ")) + return nil + } + require.NoError(t, c.initializeRepositories(t.Context(), cloudPlatform{ID: id}, "https://repo.example/ubuntu")) + require.Equal(t, []string{"dnf makecache -y"}, commands) + if id == osReleaseIDMariner { + for _, repo := range []string{"extras", "extended", "nvidia", "cloud-native"} { + source, err := os.ReadFile(c.path(defaultYumReposDir + "/mariner-" + repo + ".repo")) + require.NoError(t, err) + require.Contains(t, string(source), "https://repo.example/mariner/packages.microsoft.com/") + require.Contains(t, string(source), "/"+repo+"/") + } + } else { + _, err := os.Stat(c.path(defaultYumReposDir + "/azurelinux-old.repo")) + require.ErrorIs(t, err, os.ErrNotExist) + for _, repo := range []string{"amd", "base", "cloud-native", "extended", "ms-non-oss", "ms-oss", "nvidia"} { + source, err := os.ReadFile(c.path(defaultYumReposDir + "/azurelinux-" + repo + ".repo")) + require.NoError(t, err) + require.Contains(t, string(source), "https://repo.example/azurelinux/$releasever/prod/"+repo+"/$basearch") + require.Contains(t, string(source), "gpgcheck=1\nrepo_gpgcheck=1") + } + } + keep, err := os.ReadFile(c.path(defaultYumReposDir + "/keep.repo")) + require.NoError(t, err) + require.Equal(t, "keep this repository", string(keep)) + }) + } +} diff --git a/aks-node-controller/cloud_repositories_test.go b/aks-node-controller/cloud_repositories_test.go new file mode 100644 index 00000000000..e71d1358a11 --- /dev/null +++ b/aks-node-controller/cloud_repositories_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + "errors" + "io" + "os/exec" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCloudRepositoriesSkipEmptyEndpointAndImmutableOS(t *testing.T) { + for _, tc := range []struct{ id, endpoint string }{ + {osReleaseIDUbuntu, ""}, {"mariner", ""}, {osReleaseIDAzureLinux, ""}, + {osReleaseIDFlatcar, "https://repo.example"}, {osReleaseIDAzureContainerLinux, "https://repo.example"}, + } { + t.Run(tc.id+"/"+tc.endpoint, func(t *testing.T) { + c := newTestCloudInitializer(t, "ID="+tc.id+"\n") + c.run = func(*exec.Cmd) error { + t.Fatal("no repository commands expected") + return nil + } + require.NoError(t, c.initializeRepositories(t.Context(), cloudPlatform{ID: tc.id}, tc.endpoint)) + }) + } +} + +func TestCloudRepositoryEndpointValidation(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + for _, endpoint := range []string{ + "not-a-url", "file:///etc/apt", "https://repo.example\nSigned-By: bad", "https://user:password@repo.example", + "https://repo.example?query=1", "https://repo.example/#fragment", + } { + require.ErrorContains(t, c.initializeRepositories(t.Context(), cloudPlatform{ID: osReleaseIDUbuntu}, endpoint), "invalid RepoDepot endpoint") + } +} + +func TestCloudRepositoryCacheRetriesAndFailsExplicitly(t *testing.T) { + for _, name := range []string{"apt-get", "dnf"} { + for _, recoverAfter := range []int{2, cloudRequestAttempts + 1} { + t.Run(name+"/"+strconv.Itoa(recoverAfter), func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + attempts := 0 + c.run = func(cmd *exec.Cmd) error { + attempts++ + if attempts < recoverAfter { + if name == "dnf" { + _, _ = io.WriteString(cmd.Stderr, "Error: repository metadata is unavailable\n") + return nil + } + return errors.New("cache refresh failed") + } + return nil + } + err := c.updateRepositoryCache(t.Context(), name, "update") + if recoverAfter > cloudRequestAttempts { + require.ErrorContains(t, err, "refresh RepoDepot package cache") + require.ErrorIs(t, err, errCloudRepositoryCacheRefresh) + require.Equal(t, cloudRequestAttempts, attempts) + } else { + require.NoError(t, err) + require.Equal(t, recoverAfter, attempts) + } + }) + } + } + t.Run("canceled", func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + c.run = func(*exec.Cmd) error { + t.Fatal("canceled context must not execute a command") + return nil + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + err := c.updateRepositoryCache(ctx, "apt-get", "update") + require.ErrorIs(t, err, context.Canceled) + require.NotErrorIs(t, err, errCloudRepositoryCacheRefresh) + }) +}