From c50263760b77f7ca3cef1753d2c9fd81d29f0108 Mon Sep 17 00:00:00 2001 From: Abigailliang Date: Mon, 26 Oct 2026 14:50:50 -0700 Subject: [PATCH] feat: add native cloud certificate capabilities [1/3] Split certificate retrieval, trust-store installation, shared runtime, and the standalone refresh command from #9891. Preserve certificate failure semantics while keeping existing provisioning on the shell path. Include corresponding unit coverage and direct refresh tests for all supported OS families. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: be499bac-1283-498e-b5b1-59d8c91047a4 --- aks-node-controller/README.md | 27 ++ aks-node-controller/app.go | 15 + aks-node-controller/cloud_certificates.go | 342 ++++++++++++++ .../cloud_certificates_test.go | 417 ++++++++++++++++++ aks-node-controller/cloud_refresh.go | 65 +++ aks-node-controller/cloud_refresh_test.go | 48 ++ aks-node-controller/cloud_runtime.go | 185 ++++++++ aks-node-controller/cloud_runtime_test.go | 58 +++ .../cloud_test_helpers_test.go | 126 ++++++ aks-node-controller/embeddednodecustomdata.go | 3 +- 10 files changed, 1285 insertions(+), 1 deletion(-) create mode 100644 aks-node-controller/cloud_certificates.go create mode 100644 aks-node-controller/cloud_certificates_test.go create mode 100644 aks-node-controller/cloud_refresh.go create mode 100644 aks-node-controller/cloud_refresh_test.go create mode 100644 aks-node-controller/cloud_runtime.go create mode 100644 aks-node-controller/cloud_runtime_test.go create mode 100644 aks-node-controller/cloud_test_helpers_test.go diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index 0539665d43b..4a8532fee1f 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -54,6 +54,33 @@ the provisioning config in CustomData; the automatic size fallback uses the same config-only payload. Leaving the new flag false preserves legacy, Phase 2, and Phase 2.5 behavior. +### Azure cloud certificate refresh + +ANC provides a standalone command for refreshing Azure cloud CA certificates: + +```sh +/opt/azure/containers/aks-node-controller refresh-cloud-certificates --location westus2 +``` + +Regions starting with `ussec` or `usnat` use the legacy certificate endpoint. +Other regions check WireServer's root-certificate opt-in before retrieving root +and intermediate certificates. Only the JSON boolean `true` enables opt-in; +nonempty malformed or non-true responses skip installation. Failed or empty +opt-in requests remain errors. Individual operation-certificate download failures +are logged and skipped, and legacy trust-store installation failures remain +best-effort unless the context is canceled. + +Downloaded contents are passed to the system trust-store tools without additional +PEM/X.509 validation. Refresh clears the flat staging cache, including hidden +files, without removing existing system trust-store certificates. Raw responses +are never logged; `AKS_CLOUD_LOG_CERTIFICATES=true` enables public-certificate-only +diagnostics. The refresh command has a 15-minute budget and emits +`CloudCertificateRefresh` guest-agent events. + +This command does not configure repositories or install refresh schedules. +Existing provisioning still uses `init-aks-cloud.sh`; native cloud initialization +is not yet connected to the provisioning entry point. + ### Validating and normalizing AKSNodeConfig Before serializing a typed config, producers such as RP should call: diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index 71942951b3e..5e6882268fb 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -102,6 +102,8 @@ type App struct { // grpcDialContext overrides how the gRPC LPS client dials, letting tests point the client at // an in-process (bufconn) server. When nil, the real TLS dial to the apiserver front is used. grpcDialContext func(ctx context.Context, target string) (net.Conn, error) + // refreshCloudCertificates overrides standalone cloud certificate refresh for testing. + refreshCloudCertificates func(context.Context, string) error } // provision.json values are emitted as strings by the shell jq invocation. @@ -181,6 +183,19 @@ func (a *App) Run(ctx context.Context, args []string) int { return nil }, }, + { + Name: "refresh-cloud-certificates", + Usage: "Refresh Azure cloud CA certificates without configuring repositories", + Flags: []cli.Flag{ + &cli.StringFlag{Name: "location", Required: true, Usage: "Azure region used to select the certificate endpoint"}, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + if cmd.Args().Len() != 0 { + return fmt.Errorf("unexpected refresh-cloud-certificates arguments: %s", strings.Join(cmd.Args().Slice(), " ")) + } + return a.runCloudCertificateRefresh(ctx, cmd.String("location")) + }, + }, { Name: "download-hotfix", Usage: "Download the requested hotfix binary", diff --git a/aks-node-controller/cloud_certificates.go b/aks-node-controller/cloud_certificates.go new file mode 100644 index 00000000000..6dffa5563c3 --- /dev/null +++ b/aks-node-controller/cloud_certificates.go @@ -0,0 +1,342 @@ +package main + +import ( + "bytes" + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "log/slog" + "maps" + "net/http" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/Azure/agentbaker/aks-node-controller/common" +) + +func (c *cloudInitializer) requestCertificates(ctx context.Context, path string) ([]byte, error) { + attempt := 0 + delay, nextDelay := 3*time.Second, time.Duration(0) + return common.RetryFetch(ctx, cloudRequestAttempts, nil, func(ctx context.Context) ([]byte, error) { + if attempt > 0 { + if err := c.wait(ctx, nextDelay); err != nil { + return nil, err + } + } + attempt++ + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.wireEndpoint+path, nil) + if err != nil { + return nil, err + } + nextDelay = delay + resp, err := c.wireClient.Do(req) + if err != nil { + return nil, err + } + body, err := readCloudResponse(resp) + if err != nil { + return nil, err + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return body, nil + } + if resp.StatusCode == http.StatusForbidden && bytes.Contains(body, []byte("RequestRateLimitExceeded")) { + delay *= 2 + } + return nil, fmt.Errorf("WireServer certificate request %s returned HTTP %d", path, resp.StatusCode) + }) +} + +func (c *cloudInitializer) refresh(ctx context.Context, platform cloudPlatform, location string) (bool, error) { + c.clearCertificateCache() + legacy := strings.HasPrefix(location, "ussec") || strings.HasPrefix(location, "usnat") + slog.Info("selecting Azure cloud certificate endpoint", "legacy", legacy, "location", location) + if !legacy { + body, err := c.requestCertificates(ctx, "/acms/isOptedInForRootCerts") + if err != nil { + return false, fmt.Errorf("check Azure cloud root certificate opt-in: %w", err) + } + if len(bytes.TrimRight(body, "\n")) == 0 { + return false, errors.New("cloud root certificate opt-in response is empty") + } + var response map[string]json.RawMessage + if err := json.Unmarshal(body, &response); err != nil { + slog.Warn("could not decode Azure cloud root certificate opt-in, skipping certificate installation", "error", err) + return false, nil + } + if !bytes.Equal(bytes.TrimSpace(response["IsOptedInForRootCerts"]), []byte("true")) { + slog.Info("Azure cloud root certificates are not opted in, skipping certificate installation") + return false, nil + } + } + var certificates map[string][]byte + var err error + if legacy { + certificates, err = c.legacyCertificates(ctx) + } else { + certificates, err = c.operationCertificates(ctx) + } + if err != nil { + return false, err + } + if err := c.installCertificates(ctx, platform, certificates); err != nil { + if ctx.Err() != nil { + return false, ctx.Err() + } + if legacy { + slog.Warn("legacy Azure cloud CA certificate installation failed, continuing cloud initialization", "error", err) + return true, nil + } + return false, fmt.Errorf("install Azure cloud CA certificates: %w", err) + } + slog.Info("installed Azure cloud CA certificates") + return true, nil +} + +func (c *cloudInitializer) clearCertificateCache() { + directory := c.path("/root/AzureCACertificates") + if err := os.MkdirAll(directory, 0o755); err != nil { + slog.Warn("could not create Azure cloud certificate cache", "error", err) + return + } + entries, err := os.ReadDir(directory) + if err != nil { + slog.Warn("could not read Azure cloud certificate cache", "error", err) + return + } + // Certificate downloads are written directly here as files, without subdirectories. + for _, entry := range entries { + if err := os.Remove(filepath.Join(directory, entry.Name())); err != nil { + slog.Warn("could not remove cached Azure cloud certificate", "name", entry.Name(), "error", err) + } + } +} + +func certificatePackagePath(kind, extension string) string { + return "/machine?" + url.Values{ + "comp": {"acmspackage"}, + "type": {kind}, + "ext": {extension}, + }.Encode() +} + +func validCloudCertificateName(name string) bool { + return name != "" && name != "." && name != ".." && filepath.Base(name) == name && + !strings.ContainsRune(name, 0) +} + +func (c *cloudInitializer) legacyCertificates(ctx context.Context) (map[string][]byte, error) { + body, err := c.requestCertificates(ctx, certificatePackagePath("cacertificates", "json")) + if err != nil { + return nil, fmt.Errorf("retrieve legacy Azure cloud certificates: %w", err) + } + var response struct { + Certificates []struct { + Name json.RawMessage + Body json.RawMessage `json:"CertBody"` + } + } + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("decode legacy Azure cloud certificates: %w", err) + } + if response.Certificates == nil { + return nil, errors.New("legacy Azure cloud response is missing a Certificates array") + } + certificates := make(map[string][]byte) + for _, cert := range response.Certificates { + var name string + if err := json.Unmarshal(cert.Name, &name); err != nil || !validCloudCertificateName(name) { + slog.Warn("rejecting invalid Azure cloud certificate filename") + continue + } + var certBody *string + if err := json.Unmarshal(cert.Body, &certBody); err != nil { + return nil, fmt.Errorf("decode legacy Azure cloud certificate body %s: %w", name, err) + } + if certBody == nil { + return nil, fmt.Errorf("legacy Azure cloud certificate %s is missing CertBody", name) + } + // Shell command substitution trims trailing LFs before CRLF normalization. + data := strings.ReplaceAll(strings.TrimRight(*certBody, "\n"), "\r\n", "\n") + "\n" + name = strings.ReplaceAll(name, ".cer", ".crt") + if err := c.writeFile("/root/AzureCACertificates/"+name, []byte(data)); err != nil { + return nil, fmt.Errorf("cache legacy Azure cloud certificate %s: %w", name, err) + } + certificates[name] = []byte(data) + } + return certificates, nil +} + +func (c *cloudInitializer) operationCertificates(ctx context.Context) (map[string][]byte, error) { + certificates := make(map[string][]byte) + for _, kind := range []string{"operationrequestsroot", "operationrequestsintermediate"} { + group, err := c.operationCertificateGroup(ctx, kind) + if err != nil { + return nil, err + } + maps.Copy(certificates, group) + } + return certificates, nil +} + +func (c *cloudInitializer) operationCertificateGroup(ctx context.Context, kind string) (map[string][]byte, error) { + body, err := c.requestCertificates(ctx, certificatePackagePath(kind, "json")) + if err != nil { + return nil, fmt.Errorf("retrieve Azure cloud %s: %w", kind, err) + } + var response struct { + Operations []struct { + // ResouceFileName is the spelling used by the WireServer protocol. + Name json.RawMessage `json:"ResouceFileName"` + } `json:"OperationsInfo"` + } + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("decode Azure cloud %s: %w", kind, err) + } + certificates := make(map[string][]byte) + found := false + for _, operation := range response.Operations { + var name string + if err := json.Unmarshal(operation.Name, &name); err != nil || name == "" { + slog.Warn("ignoring missing or non-string Azure cloud certificate filename") + continue + } + found = true + if !validCloudCertificateName(name) { + slog.Warn("rejecting invalid Azure cloud certificate filename") + continue + } + data, err := c.operationCertificate(ctx, name) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + slog.Warn("could not retrieve Azure cloud certificate, continuing with remaining certificates", "name", name, "error", err) + continue + } + if err := c.writeFile("/root/AzureCACertificates/"+name, data); err != nil { + return nil, fmt.Errorf("cache Azure cloud certificate %s: %w", name, err) + } + certificates[name] = data + } + if !found { + return nil, fmt.Errorf("cloud %s contains no certificate filenames", kind) + } + return certificates, nil +} + +func (c *cloudInitializer) operationCertificate(ctx context.Context, filename string) ([]byte, error) { + name, extension := filename, filename + if suffix := filepath.Ext(filename); suffix != "" { + name = strings.TrimSuffix(filename, suffix) + extension = strings.TrimPrefix(suffix, ".") + } + body, err := c.requestCertificates(ctx, certificatePackagePath(name, extension)) + if err != nil { + return nil, fmt.Errorf("retrieve Azure cloud certificate %s: %w", filename, err) + } + body = bytes.TrimRight(body, "\n") + if len(body) == 0 { + return nil, fmt.Errorf("cloud certificate %s response is empty", filename) + } + return append(body, '\n'), nil +} + +func publicCloudCertificates(data []byte) ([]byte, error) { + var result bytes.Buffer + for { + block, rest := pem.Decode(data) + if block == nil { + break + } + data = rest + if block.Type != "CERTIFICATE" { + continue + } + if _, err := x509.ParseCertificate(block.Bytes); err != nil { + return nil, fmt.Errorf("invalid CA certificate: %w", err) + } + if err := pem.Encode(&result, &pem.Block{Type: "CERTIFICATE", Bytes: block.Bytes}); err != nil { + return nil, err + } + } + if result.Len() == 0 { + return nil, errors.New("response contains no PEM certificates") + } + return result.Bytes(), nil +} + +func (c *cloudInitializer) installCertificates(ctx context.Context, platform cloudPlatform, certificates map[string][]byte) error { + destination, update := "/usr/local/share/ca-certificates", "update-ca-certificates" + switch platform.ID { + case osReleaseIDAzureLinux, osReleaseIDMariner, osReleaseIDAzureContainerLinux: + destination, update = "/etc/pki/ca-trust/source/anchors", "update-ca-trust" + case osReleaseIDFlatcar: + destination = "/etc/ssl/certs" + } + names := make([]string, 0, len(certificates)) + for name := range certificates { + names = append(names, name) + } + sort.Strings(names) + installed := 0 + for _, name := range names { + data := certificates[name] + if !strings.HasSuffix(name, ".crt") { + continue + } + targetName := name + if platform.ID == osReleaseIDFlatcar { + targetName = strings.TrimSuffix(name, ".crt") + ".pem" + } + if err := c.writeFile(filepath.Join(destination, targetName), data); err != nil { + return err + } + installed++ + if os.Getenv("AKS_CLOUD_LOG_CERTIFICATES") == "true" { + public, err := publicCloudCertificates(data) + if err != nil { + slog.Warn("could not decode public Azure cloud CA certificates for logging", "name", name, "error", err) + continue + } + slog.Info("Azure cloud public CA certificate", "name", name, "certificate", string(public)) + } + } + if installed == 0 { + return errors.New("no *.crt files in Azure cloud certificate cache to install") + } + if _, err := c.command(ctx, nil, update); err != nil { + return err + } + if platform.ID == osReleaseIDUbuntu { + return c.copyOpenSSLBundle() + } + return nil +} + +func (c *cloudInitializer) copyOpenSSLBundle() error { + source, target := c.path("/etc/ssl/certs/ca-certificates.crt"), c.path("/usr/lib/ssl/cert.pem") + sourceInfo, err := os.Stat(source) + if err != nil { + return err + } + targetInfo, err := os.Stat(target) + if err == nil && os.SameFile(sourceInfo, targetInfo) { + return nil + } + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + data, err := os.ReadFile(source) + if err != nil { + return err + } + return c.writeFile("/usr/lib/ssl/cert.pem", data) +} diff --git a/aks-node-controller/cloud_certificates_test.go b/aks-node-controller/cloud_certificates_test.go new file mode 100644 index 00000000000..9f454cf7f4e --- /dev/null +++ b/aks-node-controller/cloud_certificates_test.go @@ -0,0 +1,417 @@ +package main + +import ( + "context" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCloudCertificateRefreshModes(t *testing.T) { + cert := cloudTestCertificate(t) + for _, platform := range []struct{ name, release, directory, filename, command string }{ + {"Ubuntu", cloudTestUbuntu, "/usr/local/share/ca-certificates", "root.crt", "update-ca-certificates"}, + {"Mariner", "ID=mariner\n", "/etc/pki/ca-trust/source/anchors", "root.crt", "update-ca-trust"}, + {"AzureLinux", "ID=azurelinux\n", "/etc/pki/ca-trust/source/anchors", "root.crt", "update-ca-trust"}, + {"Flatcar", "ID=flatcar\n", "/etc/ssl/certs", "root.pem", "update-ca-certificates"}, + {"ACL", "ID=azurecontainerlinux\n", "/etc/pki/ca-trust/source/anchors", "root.crt", "update-ca-trust"}, + } { + for _, mode := range []struct{ name, location, optIn string }{ + {"opt-out", "westus2", `{"IsOptedInForRootCerts":false}`}, + {"rcv1p", "westus2", `{"IsOptedInForRootCerts":true}`}, + {"legacy", " US NAT East ", "not used"}, + } { + t.Run(platform.name+"/"+mode.name, func(t *testing.T) { + c := newTestCloudInitializer(t, platform.release) + serveCloudTestResponses(t, c, cert, mode.optIn) + var commands []string + c.run = func(cmd *exec.Cmd) error { + commands = append(commands, cmd.Args[0]) + return nil + } + require.NoError(t, c.refreshCertificates(t.Context(), mode.location)) + if mode.name == "opt-out" { + require.Empty(t, commands) + return + } + require.Equal(t, []string{platform.command}, commands) + installed, err := os.ReadFile(c.path(filepath.Join(platform.directory, platform.filename))) + require.NoError(t, err) + expected := cert + if mode.name == "legacy" { + expected = []byte(strings.TrimSuffix(string(cert), "\n") + "\r\n") + } + require.Equal(t, expected, installed) + }) + } + } +} + +func TestCloudCertificateRefreshNonTrueOptInResponsesSkip(t *testing.T) { + for _, response := range []string{ + "not json", " ", `{}`, `null`, `[]`, `{"IsOptedInForRootCerts":false}`, + `{"IsOptedInForRootCerts":null}`, `{"IsOptedInForRootCerts":"false"}`, + `{"IsOptedInForRootCerts":"true"}`, `{"IsOptedInForRootCerts":1}`, + `{"isOptedInForRootCerts":true}`, + } { + t.Run(response, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + requests := serveCloudTestResponses(t, c, nil, response) + c.run = func(*exec.Cmd) error { + t.Fatal("non-true opt-in must not update the trust store") + return nil + } + require.NoError(t, c.refreshCertificates(t.Context(), "westus2")) + require.Len(t, requests, 1) + }) + } +} + +func TestCloudLegacyCertificateRefreshTrustFailureIsBestEffort(t *testing.T) { + c := newTestCloudInitializer(t, "ID=azurelinux\n") + serveCloudTestResponses(t, c, cloudTestCertificate(t), "not used") + c.run = func(cmd *exec.Cmd) error { + require.Equal(t, "update-ca-trust", cmd.Args[0]) + return errors.New("trust store update failed") + } + require.NoError(t, c.refreshCertificates(t.Context(), "usnateast")) +} + +func TestCloudCertificateOptInEmptyResponsesFail(t *testing.T) { + for _, response := range []string{"", "\n", "\n\n"} { + t.Run(fmt.Sprintf("%q", response), func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + serveCloudTestResponses(t, c, nil, response) + require.ErrorContains(t, c.refreshCertificates(t.Context(), "westus2"), "response is empty") + }) + } +} + +func TestCloudCertificateRefreshClearsCachedFiles(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + serveCloudTestResponses(t, c, nil, `{"IsOptedInForRootCerts":false}`) + for _, name := range []string{"stale.crt", ".stale.crt", "certificate-without-extension"} { + require.NoError(t, c.writeFile("/root/AzureCACertificates/"+name, []byte("stale"))) + } + require.NoError(t, c.writeFile("/usr/local/share/ca-certificates/existing.crt", []byte("keep"))) + require.NoError(t, c.refreshCertificates(t.Context(), "westus2")) + entries, err := os.ReadDir(c.path("/root/AzureCACertificates")) + require.NoError(t, err) + require.Empty(t, entries) + data, err := os.ReadFile(c.path("/usr/local/share/ca-certificates/existing.crt")) + require.NoError(t, err) + require.Equal(t, "keep", string(data)) +} + +func TestCloudCertificatePayloadsArePassedToTrustStoreTools(t *testing.T) { + for _, tc := range []struct{ name, location, want string }{ + {"legacy", "usnateast", "not a PEM certificate\r\n"}, + {"rcv1p", "westus2", "not a PEM certificate\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("AKS_CLOUD_LOG_CERTIFICATES", "true") + logs := installLogCapturer(t) + c := newTestCloudInitializer(t, cloudTestUbuntu) + serveCloudTestResponses(t, c, []byte("not a PEM certificate\n"), `{"IsOptedInForRootCerts":true}`) + var commands []string + c.run = func(cmd *exec.Cmd) error { + commands = append(commands, cmd.Args[0]) + return nil + } + require.NoError(t, c.refreshCertificates(t.Context(), tc.location)) + got, err := os.ReadFile(c.path("/usr/local/share/ca-certificates/root.crt")) + require.NoError(t, err) + require.Equal(t, tc.want, string(got)) + require.Equal(t, []string{"update-ca-certificates"}, commands) + for _, record := range logs.getRecords() { + for _, value := range record.Attrs { + require.NotContains(t, value, "not a PEM certificate") + } + } + }) + } +} + +func TestCloudCertificateOperationDownloadFailures(t *testing.T) { + for _, tc := range []struct { + name string + roots []string + intermediates []string + wantInstalled []string + wantError bool + }{ + {"partial group", []string{"missing.crt", "root.crt"}, []string{"intermediate.crt"}, []string{"root.crt", "intermediate.crt"}, false}, + {"empty root group after failures", []string{"missing.crt"}, []string{"intermediate.crt"}, []string{"intermediate.crt"}, false}, + {"empty certificate response", []string{"empty.crt"}, []string{"intermediate.crt"}, []string{"intermediate.crt"}, false}, + {"non-crt root", []string{"root.cer"}, []string{"intermediate.crt"}, []string{"intermediate.crt"}, false}, + {"all downloads fail", []string{"missing.crt"}, []string{"missing.crt"}, nil, true}, + {"no crt files", []string{"root.cer"}, []string{"intermediate.cer"}, nil, true}, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, "ID=azurelinux\n") + serveCloudOperationTestResponses(t, c, tc.roots, tc.intermediates) + var commands []string + c.run = func(cmd *exec.Cmd) error { + commands = append(commands, cmd.Args[0]) + return nil + } + err := c.refreshCertificates(t.Context(), "westus2") + if tc.wantError { + require.ErrorContains(t, err, "no *.crt files") + require.Empty(t, commands) + return + } + require.NoError(t, err) + require.Equal(t, []string{"update-ca-trust"}, commands) + for _, name := range tc.wantInstalled { + data, readErr := os.ReadFile(c.path("/etc/pki/ca-trust/source/anchors/" + name)) + require.NoError(t, readErr) + require.Equal(t, "certificate payload\n", string(data)) + } + for _, name := range []string{"missing.crt", "empty.crt", "root.cer"} { + _, err = os.Stat(c.path("/etc/pki/ca-trust/source/anchors/" + name)) + require.ErrorIs(t, err, os.ErrNotExist) + } + }) + } +} + +func TestCloudLegacyCertificateInstallationDoesNotIgnoreCancellation(t *testing.T) { + c := newTestCloudInitializer(t, "ID=azurelinux\n") + serveCloudTestResponses(t, c, cloudTestCertificate(t), "not used") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + c.run = func(*exec.Cmd) error { + cancel() + return ctx.Err() + } + require.ErrorIs(t, c.refreshCertificates(ctx, "usnateast"), context.Canceled) +} + +func TestCloudCertificateRequestRetryAndCancellation(t *testing.T) { + for _, tc := range []struct { + name string + status int + body string + failures int32 + wantWaits []time.Duration + }{ + {"transient", http.StatusServiceUnavailable, "do-not-log-body", 2, []time.Duration{3 * time.Second, 3 * time.Second}}, + {"rate limited", http.StatusForbidden, "RequestRateLimitExceeded", 2, []time.Duration{3 * time.Second, 6 * time.Second}}, + {"exhausted", http.StatusServiceUnavailable, "do-not-log-body", cloudRequestAttempts, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if attempts.Add(1) <= tc.failures { + w.WriteHeader(tc.status) + _, _ = io.WriteString(w, tc.body) + return + } + _, _ = io.WriteString(w, "success") + })) + defer server.Close() + c.wireEndpoint = server.URL + var waits []time.Duration + c.wait = func(_ context.Context, delay time.Duration) error { + waits = append(waits, delay) + return nil + } + body, err := c.requestCertificates(t.Context(), "/test") + if tc.failures == cloudRequestAttempts { + require.Error(t, err) + require.NotContains(t, err.Error(), tc.body) + require.EqualValues(t, cloudRequestAttempts, attempts.Load()) + require.Len(t, waits, cloudRequestAttempts-1) + } else { + require.NoError(t, err) + require.Equal(t, "success", string(body)) + require.Equal(t, tc.wantWaits, waits) + } + }) + } + t.Run("cancellation during backoff", func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + c.wireEndpoint = server.URL + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + c.wait = func(ctx context.Context, _ time.Duration) error { + cancel() + return ctx.Err() + } + _, err := c.requestCertificates(ctx, "/test") + require.ErrorIs(t, err, context.Canceled) + require.EqualValues(t, 1, attempts.Load()) + }) +} + +func TestCloudCertificateRequestsDoNotFollowRedirects(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var redirected atomic.Int32 + target := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { redirected.Add(1) })) + defer target.Close() + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer source.Close() + c.wireEndpoint = source.URL + _, err := c.requestCertificates(t.Context(), "/test") + require.ErrorContains(t, err, "HTTP 302") + require.Zero(t, redirected.Load()) +} + +func TestCloudCertificateResourceNamesAreURLEncoded(t *testing.T) { + c := newTestCloudInitializer(t, "ID=azurelinux\n") + cert := cloudTestCertificate(t) + var fetched atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch kind := r.URL.Query().Get("type"); { + case r.URL.Path == "/acms/isOptedInForRootCerts": + _, _ = io.WriteString(w, `{"IsOptedInForRootCerts":true}`) + case kind == "operationrequestsroot": + _, _ = io.WriteString(w, `{"OperationsInfo":[{"ResouceFileName":"root+ca.crt"}]}`) + case kind == "operationrequestsintermediate": + _, _ = io.WriteString(w, `{"OperationsInfo":[{"ResouceFileName":"intermediate.crt"}]}`) + case kind == "root+ca", kind == "intermediate": + fetched.Store(true) + _, _ = w.Write(cert) + default: + t.Errorf("incorrectly encoded certificate resource: %s", r.URL) + w.WriteHeader(http.StatusBadRequest) + } + })) + defer server.Close() + c.wireEndpoint = server.URL + require.NoError(t, c.refreshCertificates(t.Context(), "westus2")) + require.True(t, fetched.Load()) + installed, err := os.ReadFile(c.path("/etc/pki/ca-trust/source/anchors/root+ca.crt")) + require.NoError(t, err) + require.Equal(t, cert, installed) +} + +func TestCloudCertificateResponseLimits(t *testing.T) { + _, err := readCloudResponse(&http.Response{ + Body: io.NopCloser(strings.NewReader(strings.Repeat("x", cloudResponseLimit+1))), + }) + require.ErrorContains(t, err, "size limit") +} + +func TestCloudCertificateNamesAndPublicContent(t *testing.T) { + for _, name := range []string{"", ".", "..", "../root.crt", "/root.crt", "root\x00.crt"} { + require.False(t, validCloudCertificateName(name), name) + } + for _, name := range []string{"root+ca.crt", `..\root.crt`, "root\n.crt"} { + require.True(t, validCloudCertificateName(name), name) + } + cert := cloudTestCertificate(t) + private := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte("not-a-real-private-key")}) + got, err := publicCloudCertificates(append(private, cert...)) + require.NoError(t, err) + require.Equal(t, cert, got) + for _, bad := range [][]byte{nil, private, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte("invalid")})} { + _, err = publicCloudCertificates(bad) + require.Error(t, err) + } +} + +func TestCloudCertificateLoggingIsOptInAndPublicOnly(t *testing.T) { + for _, enabled := range []string{"false", "true"} { + t.Run(enabled, func(t *testing.T) { + t.Setenv("AKS_CLOUD_LOG_CERTIFICATES", enabled) + logs := installLogCapturer(t) + c := newTestCloudInitializer(t, "ID=azurelinux\n") + cert := cloudTestCertificate(t) + payload := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte("not-a-real-private-key")}) + payload = append(payload, cert...) + serveCloudTestResponses(t, c, payload, `{"IsOptedInForRootCerts":true}`) + require.NoError(t, c.refreshCertificates(t.Context(), "westus2")) + found := false + for _, record := range logs.getRecords() { + for _, value := range record.Attrs { + require.NotContains(t, value, "PRIVATE KEY") + if strings.Contains(value, "BEGIN CERTIFICATE") { + found = true + } + } + } + require.Equal(t, enabled == "true", found) + got, err := os.ReadFile(c.path("/etc/pki/ca-trust/source/anchors/root.crt")) + require.NoError(t, err) + require.Equal(t, payload, got) + }) + } +} + +func TestCloudCertificateInvalidOperationsDoNotInstall(t *testing.T) { + for _, response := range []string{ + `{}`, `{"OperationsInfo":[]}`, `{"OperationsInfo":[{"ResouceFileName":"../escape.crt"}]}`, + `{"OperationsInfo":[{"ResouceFileName":"/escape.crt"}]}`, `{"OperationsInfo":[{"ResouceFileName":"missing-extension"}]}`, + } { + t.Run(response, func(t *testing.T) { + c := newTestCloudInitializer(t, "ID=azurelinux\n") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/acms/isOptedInForRootCerts" { + _, _ = io.WriteString(w, `{"IsOptedInForRootCerts":true}`) + } else { + _, _ = io.WriteString(w, response) + } + })) + defer server.Close() + c.wireEndpoint = server.URL + c.run = func(*exec.Cmd) error { + t.Fatal("invalid operations must not install certificates") + return nil + } + require.Error(t, c.refreshCertificates(t.Context(), "westus2")) + _, err := os.Stat(c.path("/etc/pki/ca-trust/source/anchors")) + require.ErrorIs(t, err, os.ErrNotExist) + }) + } +} + +func TestCloudCertificateUbuntuBundleSymlink(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + require.NoError(t, os.MkdirAll(c.path("/usr/lib/ssl"), 0o755)) + require.NoError(t, os.Symlink(c.path("/etc/ssl/certs/ca-certificates.crt"), c.path("/usr/lib/ssl/cert.pem"))) + require.NoError(t, c.copyOpenSSLBundle()) + info, err := os.Lstat(c.path("/usr/lib/ssl/cert.pem")) + require.NoError(t, err) + require.NotZero(t, info.Mode()&os.ModeSymlink) +} + +func TestCloudPeriodicRefreshDoesNotInitializeRepositoriesOrSchedules(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + requests := serveCloudTestResponses(t, c, cloudTestCertificate(t), `{"IsOptedInForRootCerts":true}`) + var commands []string + c.run = func(cmd *exec.Cmd) error { + commands = append(commands, strings.Join(cmd.Args, " ")) + return nil + } + require.NoError(t, c.refreshCertificates(t.Context(), "westus2")) + require.Equal(t, []string{"update-ca-certificates"}, commands) + require.Len(t, requests, 5) + _, err := os.Stat(c.path(defaultAptSourcesDir)) + require.ErrorIs(t, err, os.ErrNotExist) + events := c.eventLogger.Events() + require.Len(t, events, 1) + require.Equal(t, "AKS.AKSNodeController.CloudCertificateRefresh", events[0].TaskName) +} diff --git a/aks-node-controller/cloud_refresh.go b/aks-node-controller/cloud_refresh.go new file mode 100644 index 00000000000..5351dc5afb9 --- /dev/null +++ b/aks-node-controller/cloud_refresh.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "errors" + "log/slog" + "strings" + "time" + "unicode" +) + +const cloudRefreshTimeout = 15 * time.Minute + +func (a *App) runCloudCertificateRefresh(ctx context.Context, location string) error { + if strings.TrimSpace(location) == "" { + return errors.New("cloud certificate refresh requires a location") + } + var err error + if a.refreshCloudCertificates != nil { + err = a.refreshCloudCertificates(ctx, location) + } else { + initializer := newCloudInitializer(a) + defer initializer.close() + err = initializer.refreshCertificates(ctx, location) + } + if err != nil { + slog.Error("Azure cloud certificate refresh failed", "error", err) + } + return err +} + +func (c *cloudInitializer) refreshCertificates(ctx context.Context, location string) error { + refreshCtx, cancel := context.WithTimeout(ctx, cloudRefreshTimeout) + defer cancel() + return c.withEvent("CloudCertificateRefresh", func() error { + normalizedLocation, err := normalizeCloudLocation(location) + if err != nil { + return err + } + platform, err := c.platform() + if err != nil { + return err + } + _, err = c.refresh(refreshCtx, platform, normalizedLocation) + return err + }) +} + +func normalizeCloudLocation(location string) (string, error) { + location = strings.ToLower(strings.Map(func(r rune) rune { + if unicode.IsSpace(r) { + return -1 + } + return r + }, location)) + if strings.IndexFunc(location, func(r rune) bool { + return !strings.ContainsRune("abcdefghijklmnopqrstuvwxyz0123456789-", r) + }) != -1 { + return "", errors.New("cloud location contains unsupported characters") + } + if location == "" { + slog.Warn("Azure cloud location is empty, using rcv1p certificate endpoints") + } + return location, nil +} diff --git a/aks-node-controller/cloud_refresh_test.go b/aks-node-controller/cloud_refresh_test.go new file mode 100644 index 00000000000..06c98447237 --- /dev/null +++ b/aks-node-controller/cloud_refresh_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNormalizeCloudLocation(t *testing.T) { + location, err := normalizeCloudLocation(" US Sec East \t\n") + require.NoError(t, err) + require.Equal(t, "usseceast", location) + for _, value := range []string{`westus";echo bad`, "westus%bad", "westus\\bad"} { + _, err = normalizeCloudLocation(value) + require.Error(t, err) + } +} + +func TestCloudCertificateRefreshCommand(t *testing.T) { + for _, tc := range []struct { + name string + args []string + err error + wantCall bool + wantError bool + }{ + {"valid", []string{"--location", "usnateast"}, nil, true, false}, + {"refresh failure", []string{"--location", "usnateast"}, errors.New("refresh failed"), true, true}, + {"missing location", nil, nil, false, true}, + {"empty location", []string{"--location", " "}, nil, false, true}, + {"unexpected argument", []string{"--location", "usnateast", "extra"}, nil, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + called := false + tt.App.refreshCloudCertificates = func(_ context.Context, location string) error { + called = true + require.Equal(t, "usnateast", location) + return tc.err + } + code := tt.App.Run(t.Context(), append([]string{"aks-node-controller", "refresh-cloud-certificates"}, tc.args...)) + require.Equal(t, tc.wantCall, called) + require.Equal(t, tc.wantError, code != 0) + }) + } +} diff --git a/aks-node-controller/cloud_runtime.go b/aks-node-controller/cloud_runtime.go new file mode 100644 index 00000000000..30f0038e5d4 --- /dev/null +++ b/aks-node-controller/cloud_runtime.go @@ -0,0 +1,185 @@ +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/Azure/agentbaker/aks-node-controller/common" + "github.com/Azure/agentbaker/aks-node-controller/helpers" +) + +const cloudRequestAttempts = 10 + +const cloudResponseLimit = 8 << 20 + +const cloudCommandTimeout = 2 * time.Minute + +type cloudPlatform struct { + ID, Version, Codename string +} + +type cloudInitializer struct { + run func(*exec.Cmd) error + wait func(context.Context, time.Duration) error + wireClient *http.Client + repoClient *http.Client + wireEndpoint string + eventLogger *helpers.EventLogger + rootDir string + osReleasePath string + executable string +} + +func newCloudInitializer(a *App) *cloudInitializer { + transport := common.NewBaseTransport(common.HTTPTransportOptions{ + DialTimeout: 10 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: 30 * time.Second, + }) + repoTransport := transport.Clone() + repoTransport.Proxy = http.ProxyFromEnvironment + return &cloudInitializer{ + run: a.cmdRun, + wait: waitCloudRetry, + wireEndpoint: wireServerEndpoint, + wireClient: &http.Client{ + Transport: transport, + Timeout: 30 * time.Second, + // WireServer responses must not redirect to another endpoint. + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }, + repoClient: &http.Client{Transport: repoTransport, Timeout: 30 * time.Second}, + eventLogger: a.eventLogger, + osReleasePath: a.osReleasePath, + } +} + +func (c *cloudInitializer) close() { + c.wireClient.CloseIdleConnections() + c.repoClient.CloseIdleConnections() +} + +func (c *cloudInitializer) platform() (cloudPlatform, error) { + path := c.osReleasePath + if path == "" { + path = c.path(defaultOSReleasePath) + } + data, err := os.ReadFile(path) + if err != nil { + return cloudPlatform{}, fmt.Errorf("read cloud initialization OS release: %w", err) + } + values := parseNodeCustomDataOSRelease(data) + id := strings.ToLower(values["ID"]) + switch { + case strings.Contains(values["NAME"], "Ubuntu"), id == osReleaseIDUbuntu: + id = osReleaseIDUbuntu + case strings.Contains(id, osReleaseIDFlatcar): + id = osReleaseIDFlatcar + case id == osReleaseIDAzureContainerLinux, + id == osReleaseIDAzureLinux && values["VARIANT_ID"] == osReleaseIDAzureContainerLinux: + id = osReleaseIDAzureContainerLinux + case strings.Contains(values["NAME"], "Mariner"), id == osReleaseIDMariner: + id = osReleaseIDMariner + case strings.Contains(values["NAME"], "Microsoft Azure Linux"), id == osReleaseIDAzureLinux: + id = osReleaseIDAzureLinux + default: + return cloudPlatform{}, fmt.Errorf("unsupported cloud initialization OS %q", id) + } + return cloudPlatform{ID: id, Version: values["VERSION_ID"], Codename: values["VERSION_CODENAME"]}, nil +} + +func (c *cloudInitializer) path(path string) string { + if c.rootDir == "" { + return path + } + return filepath.Join(c.rootDir, strings.TrimPrefix(path, "/")) +} + +func (c *cloudInitializer) writeFile(path string, data []byte) error { + return writeCloudFile(c.path(path), data) +} + +func writeCloudFile(path string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + root, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + return err + } + defer root.Close() + // #nosec G306 -- Public CAs, package keys, and repository metadata must be readable by unprivileged clients. + return root.WriteFile(filepath.Base(path), data, 0o644) +} + +func (c *cloudInitializer) withEvent(task string, action func() error) error { + start := time.Now() + err := action() + level, message := helpers.EventLevelInformational, "Completed" + if err != nil { + level, message = helpers.EventLevelError, err.Error() + slog.Error("Azure cloud operation failed", "task", task, "error", err) + } + if c.eventLogger != nil { + c.eventLogger.LogEvent(task, message, level, start, time.Now()) + } + return err +} + +func (c *cloudInitializer) command(ctx context.Context, input []byte, name string, args ...string) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, cloudCommandTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, name, args...) + cmd.Env = append(os.Environ(), "LC_ALL=C") + cmd.Stdin = bytes.NewReader(input) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + run := c.run + if run == nil { + run = cmdRunner + } + if err := run(cmd); err != nil { + return stdout.Bytes(), fmt.Errorf("%s: %w: %s", name, err, strings.TrimSpace(stderr.String())) + } + if name == "dnf" { + // DNF can report unavailable repositories on stderr despite exiting zero. + _, _ = stdout.Write(stderr.Bytes()) + } + return stdout.Bytes(), nil +} + +func waitCloudRetry(ctx context.Context, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func readCloudResponse(resp *http.Response) ([]byte, error) { + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, cloudResponseLimit+1)) + if err != nil { + return nil, err + } + if len(body) > cloudResponseLimit { + return nil, errors.New("cloud response exceeds size limit") + } + return body, nil +} diff --git a/aks-node-controller/cloud_runtime_test.go b/aks-node-controller/cloud_runtime_test.go new file mode 100644 index 00000000000..0fb8ace9c89 --- /dev/null +++ b/aks-node-controller/cloud_runtime_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCloudFileWrites(t *testing.T) { + t.Run("create and replace", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "certificates", "root.crt") + require.NoError(t, writeCloudFile(path, []byte("original"))) + require.NoError(t, writeCloudFile(path, []byte("replacement"))) + data, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, "replacement", string(data)) + }) + t.Run("reject symlinks outside the destination directory", func(t *testing.T) { + directory := t.TempDir() + outside := filepath.Join(t.TempDir(), "outside.crt") + require.NoError(t, os.WriteFile(outside, []byte("original"), 0o600)) + path := filepath.Join(directory, "root.crt") + require.NoError(t, os.Symlink(outside, path)) + require.Error(t, writeCloudFile(path, []byte("replacement"))) + data, err := os.ReadFile(outside) + require.NoError(t, err) + require.Equal(t, "original", string(data)) + }) +} + +func TestCloudPlatform(t *testing.T) { + for _, tc := range []struct { + name, release, want string + }{ + {"Ubuntu", cloudTestUbuntu, osReleaseIDUbuntu}, + {"Mariner", "ID=mariner\nNAME=CBL-Mariner\nVERSION_ID=2.0\n", "mariner"}, + {"AzureLinux", "ID=azurelinux\nNAME=\"Microsoft Azure Linux\"\nVERSION_ID=3.0\n", osReleaseIDAzureLinux}, + {"OSGuard", "ID=azurelinux\nVARIANT_ID=osguard\nVERSION_ID=3.0\n", osReleaseIDAzureLinux}, + {"Flatcar", "ID=flatcar\n", osReleaseIDFlatcar}, + {"ACL", "ID=azurecontainerlinux\n", osReleaseIDAzureContainerLinux}, + {"ACL variant", "ID=azurelinux\nVARIANT_ID=azurecontainerlinux\n", osReleaseIDAzureContainerLinux}, + {"unsupported", "ID=debian\n", ""}, + {"missing ID", "", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, tc.release) + platform, err := c.platform() + if tc.want == "" { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tc.want, platform.ID) + } + }) + } +} diff --git a/aks-node-controller/cloud_test_helpers_test.go b/aks-node-controller/cloud_test_helpers_test.go new file mode 100644 index 00000000000..7e005495b32 --- /dev/null +++ b/aks-node-controller/cloud_test_helpers_test.go @@ -0,0 +1,126 @@ +package main + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "io" + "math/big" + "net/http" + "net/http/httptest" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Azure/agentbaker/aks-node-controller/helpers" + "github.com/stretchr/testify/require" +) + +const cloudTestUbuntu = "ID=ubuntu\nNAME=Ubuntu\nVERSION_ID=22.04\nVERSION_CODENAME=jammy\n" + +func newTestCloudInitializer(t *testing.T, release string) *cloudInitializer { + t.Helper() + c := newCloudInitializer(&App{ + cmdRun: func(cmd *exec.Cmd) error { + if filepath.Base(cmd.Path) == "gpg" { + _, _ = io.WriteString(cmd.Stdout, "dearmored-key") + } + return nil + }, + eventLogger: helpers.NewEventLogger(t.TempDir()), + }) + c.rootDir = t.TempDir() + c.executable = hotfixBinaryPath + c.wait = func(ctx context.Context, _ time.Duration) error { return ctx.Err() } + require.NoError(t, c.writeFile(defaultOSReleasePath, []byte(release))) + require.NoError(t, c.writeFile("/etc/ssl/certs/ca-certificates.crt", []byte("system-ca-bundle"))) + t.Cleanup(c.close) + return c +} + +func cloudTestCertificate(t *testing.T) []byte { + t.Helper() + public, private, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "ANC cloud initialization test CA"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, public, private) + require.NoError(t, err) + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +func serveCloudTestResponses(t *testing.T, c *cloudInitializer, cert []byte, optIn string) <-chan string { + t.Helper() + requests := make(chan string, 100) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests <- r.URL.Path + "?" + r.URL.RawQuery + switch kind := r.URL.Query().Get("type"); { + case r.URL.Path == "/acms/isOptedInForRootCerts": + _, _ = io.WriteString(w, optIn) + case kind == "cacertificates": + _ = json.NewEncoder(w).Encode(map[string]any{"Certificates": []map[string]string{ + {"Name": "root.cer", "CertBody": strings.ReplaceAll(string(cert), "\n", "\r\n")}, + }}) + case kind == "operationrequestsroot", kind == "operationrequestsintermediate": + name := "root.crt" + if kind == "operationrequestsintermediate" { + name = "intermediate.crt" + } + _ = json.NewEncoder(w).Encode(map[string]any{"OperationsInfo": []map[string]string{{"ResouceFileName": name}}}) + case kind == "root", kind == "intermediate": + _, _ = w.Write(cert) + case strings.HasPrefix(r.URL.Path, "/keys/"): + _, _ = io.WriteString(w, "armored-public-package-key") + case strings.HasPrefix(r.URL.Path, "/microsoft/ubuntu/"): + _, _ = io.WriteString(w, "repository available") + default: + t.Errorf("unexpected cloud request: %s", r.URL) + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + c.wireEndpoint = server.URL + c.wireClient, c.repoClient = server.Client(), server.Client() + return requests +} + +func serveCloudOperationTestResponses(t *testing.T, c *cloudInitializer, roots, intermediates []string) { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch kind := r.URL.Query().Get("type"); { + case r.URL.Path == "/acms/isOptedInForRootCerts": + _, _ = io.WriteString(w, `{"IsOptedInForRootCerts":true}`) + case kind == "operationrequestsroot", kind == "operationrequestsintermediate": + names := roots + if kind == "operationrequestsintermediate" { + names = intermediates + } + operations := make([]map[string]string, 0, len(names)) + for _, name := range names { + operations = append(operations, map[string]string{"ResouceFileName": name}) + } + _ = json.NewEncoder(w).Encode(map[string]any{"OperationsInfo": operations}) + case kind == "missing": + w.WriteHeader(http.StatusServiceUnavailable) + case kind == "empty": + _, _ = io.WriteString(w, "\n\n") + default: + _, _ = io.WriteString(w, "certificate payload\n\n") + } + })) + t.Cleanup(server.Close) + c.wireEndpoint = server.URL +} diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go index 67615d54fcb..1f1ee70c0e5 100644 --- a/aks-node-controller/embeddednodecustomdata.go +++ b/aks-node-controller/embeddednodecustomdata.go @@ -20,6 +20,7 @@ const ( osReleaseIDAzureLinux = "azurelinux" osReleaseIDFlatcar = "flatcar" osReleaseIDUbuntu = "ubuntu" + osReleaseIDMariner = "mariner" osVariantIDOSGuard = "osguard" ) @@ -27,7 +28,7 @@ type nodeCustomDataPlatform string const ( nodeCustomDataPlatformUbuntu nodeCustomDataPlatform = "ubuntu" - nodeCustomDataPlatformMariner nodeCustomDataPlatform = "mariner" + nodeCustomDataPlatformMariner nodeCustomDataPlatform = osReleaseIDMariner nodeCustomDataPlatformUnsupported nodeCustomDataPlatform = "unsupported" )