diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index f803ffa5a7e..b33eda10fc6 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -71,9 +71,18 @@ recursively deleting directories or removing installed system-trust certificates `AKS_CLOUD_LOG_CERTIFICATES=true` enables public-certificate-only diagnostics; parsing for logging does not gate installation. -These are internal capabilities only. No certificate-retrieval flow, standalone -refresh command, refresh schedule, or native provisioning hook is enabled here. -Existing provisioning still uses `init-aks-cloud.sh`. +These are internal capabilities only. Existing provisioning still uses +`init-aks-cloud.sh`. + +### Azure cloud certificate retrieval capabilities + +WireServer helpers retrieve legacy certificate packages and RCV1P root/intermediate +operation groups. They preserve resource-name encoding, bounded requests/retries, +flat download caching, and per-certificate download skipping. + +Retrieval does not install certificates or make opt-in decisions. Refresh +orchestration, the standalone refresh command, scheduling, and native provisioning +activation are supplied by later layers; they are not enabled here. ### Validating and normalizing AKSNodeConfig diff --git a/aks-node-controller/cloud_certificate_transport.go b/aks-node-controller/cloud_certificate_transport.go new file mode 100644 index 00000000000..06c04baa38c --- /dev/null +++ b/aks-node-controller/cloud_certificate_transport.go @@ -0,0 +1,183 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "maps" + "net/http" + "net/url" + "path/filepath" + "strings" + "time" + + "github.com/Azure/agentbaker/aks-node-controller/common" +) + +const cloudRequestAttempts = 10 + +func (c *cloudInitializer) requestCertificates(ctx context.Context, path string) ([]byte, error) { + attempt := 0 + delay, nextDelay := 3*time.Second, time.Duration(0) + return common.RetryFetch(ctx, cloudRequestAttempts, nil, func(ctx context.Context) ([]byte, error) { + if attempt > 0 { + if err := c.wait(ctx, nextDelay); err != nil { + return nil, err + } + } + attempt++ + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.wireEndpoint+path, nil) + if err != nil { + return nil, err + } + nextDelay = delay + resp, err := c.wireClient.Do(req) + if err != nil { + return nil, err + } + body, err := readCloudResponse(resp) + if err != nil { + return nil, err + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return body, nil + } + if resp.StatusCode == http.StatusForbidden && bytes.Contains(body, []byte("RequestRateLimitExceeded")) { + delay *= 2 + } + return nil, fmt.Errorf("WireServer certificate request %s returned HTTP %d", path, resp.StatusCode) + }) +} + +func certificatePackagePath(kind, extension string) string { + return "/machine?" + url.Values{ + "comp": {"acmspackage"}, + "type": {kind}, + "ext": {extension}, + }.Encode() +} + +func validCloudCertificateName(name string) bool { + return name != "" && name != "." && name != ".." && filepath.Base(name) == name && + !strings.ContainsRune(name, 0) +} + +func (c *cloudInitializer) legacyCertificates(ctx context.Context) (map[string][]byte, error) { + body, err := c.requestCertificates(ctx, certificatePackagePath("cacertificates", "json")) + if err != nil { + return nil, fmt.Errorf("retrieve legacy Azure cloud certificates: %w", err) + } + var response struct { + Certificates []struct { + Name json.RawMessage + Body json.RawMessage `json:"CertBody"` + } + } + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("decode legacy Azure cloud certificates: %w", err) + } + if response.Certificates == nil { + return nil, errors.New("legacy Azure cloud response is missing a Certificates array") + } + certificates := make(map[string][]byte) + for _, cert := range response.Certificates { + var name string + if err := json.Unmarshal(cert.Name, &name); err != nil || !validCloudCertificateName(name) { + slog.Warn("rejecting invalid Azure cloud certificate filename") + continue + } + var certBody *string + if err := json.Unmarshal(cert.Body, &certBody); err != nil { + return nil, fmt.Errorf("decode legacy Azure cloud certificate body %s: %w", name, err) + } + if certBody == nil { + return nil, fmt.Errorf("legacy Azure cloud certificate %s is missing CertBody", name) + } + // Shell command substitution trims trailing LFs before CRLF normalization. + data := strings.ReplaceAll(strings.TrimRight(*certBody, "\n"), "\r\n", "\n") + "\n" + name = strings.ReplaceAll(name, ".cer", ".crt") + if err := c.writeFile("/root/AzureCACertificates/"+name, []byte(data)); err != nil { + return nil, fmt.Errorf("cache legacy Azure cloud certificate %s: %w", name, err) + } + certificates[name] = []byte(data) + } + return certificates, nil +} + +func (c *cloudInitializer) operationCertificates(ctx context.Context) (map[string][]byte, error) { + certificates := make(map[string][]byte) + for _, kind := range []string{"operationrequestsroot", "operationrequestsintermediate"} { + group, err := c.operationCertificateGroup(ctx, kind) + if err != nil { + return nil, err + } + maps.Copy(certificates, group) + } + return certificates, nil +} + +func (c *cloudInitializer) operationCertificateGroup(ctx context.Context, kind string) (map[string][]byte, error) { + body, err := c.requestCertificates(ctx, certificatePackagePath(kind, "json")) + if err != nil { + return nil, fmt.Errorf("retrieve Azure cloud %s: %w", kind, err) + } + var response struct { + Operations []struct { + // ResouceFileName is the spelling used by the WireServer protocol. + Name json.RawMessage `json:"ResouceFileName"` + } `json:"OperationsInfo"` + } + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("decode Azure cloud %s: %w", kind, err) + } + certificates := make(map[string][]byte) + found := false + for _, operation := range response.Operations { + var name string + if err := json.Unmarshal(operation.Name, &name); err != nil || name == "" { + slog.Warn("ignoring missing or non-string Azure cloud certificate filename") + continue + } + found = true + if !validCloudCertificateName(name) { + slog.Warn("rejecting invalid Azure cloud certificate filename") + continue + } + data, err := c.operationCertificate(ctx, name) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + slog.Warn("could not retrieve Azure cloud certificate, continuing with remaining certificates", "name", name, "error", err) + continue + } + if err := c.writeFile("/root/AzureCACertificates/"+name, data); err != nil { + return nil, fmt.Errorf("cache Azure cloud certificate %s: %w", name, err) + } + certificates[name] = data + } + if !found { + return nil, fmt.Errorf("cloud %s contains no certificate filenames", kind) + } + return certificates, nil +} + +func (c *cloudInitializer) operationCertificate(ctx context.Context, filename string) ([]byte, error) { + name, extension := filename, filename + if suffix := filepath.Ext(filename); suffix != "" { + name = strings.TrimSuffix(filename, suffix) + extension = strings.TrimPrefix(suffix, ".") + } + body, err := c.requestCertificates(ctx, certificatePackagePath(name, extension)) + if err != nil { + return nil, fmt.Errorf("retrieve Azure cloud certificate %s: %w", filename, err) + } + body = bytes.TrimRight(body, "\n") + if len(body) == 0 { + return nil, fmt.Errorf("cloud certificate %s response is empty", filename) + } + return append(body, '\n'), nil +} diff --git a/aks-node-controller/cloud_certificate_transport_behavior_test.go b/aks-node-controller/cloud_certificate_transport_behavior_test.go new file mode 100644 index 00000000000..d5f5c94d8ef --- /dev/null +++ b/aks-node-controller/cloud_certificate_transport_behavior_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCloudLegacyCertificateTransport(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + c.run = func(*exec.Cmd) error { + t.Fatal("certificate retrieval must not invoke trust-store tools") + return nil + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + if r.URL.Path != "/machine" || query.Get("comp") != "acmspackage" || + query.Get("type") != "cacertificates" || query.Get("ext") != "json" { + t.Errorf("unexpected legacy certificate request: %s", r.URL) + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = io.WriteString(w, `{"Certificates":[{"Name":"root.cer","CertBody":"first\r\nsecond\r\n"},{"Name":"../escape.cer","CertBody":"ignored"}]}`) + })) + defer server.Close() + c.wireEndpoint = server.URL + certificates, err := c.legacyCertificates(t.Context()) + require.NoError(t, err) + payload := []byte("first\nsecond\r\n") + require.Equal(t, map[string][]byte{"root.crt": payload}, certificates) + cached, err := os.ReadFile(c.path("/root/AzureCACertificates/root.crt")) + require.NoError(t, err) + require.Equal(t, payload, cached) + _, err = os.Stat(c.path("/root/escape.crt")) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestCloudOperationCertificateTransport(t *testing.T) { + c := newTestCloudInitializer(t, "ID=azurelinux\n") + c.run = func(*exec.Cmd) error { + t.Fatal("certificate retrieval must not invoke trust-store tools") + return nil + } + serveCloudOperationTestResponses(t, c, + []string{"missing.crt", "empty.crt", "root+ca.crt", "unfiltered.cer"}, + []string{"intermediate.crt"}) + certificates, err := c.operationCertificates(t.Context()) + require.NoError(t, err) + payload := []byte("certificate payload\n") + require.Equal(t, map[string][]byte{ + "root+ca.crt": payload, + "unfiltered.cer": payload, + "intermediate.crt": payload, + }, certificates) + for name, want := range certificates { + cached, readErr := os.ReadFile(c.path("/root/AzureCACertificates/" + name)) + require.NoError(t, readErr) + require.Equal(t, want, cached) + } + _, err = os.Stat(c.path("/etc/pki/ca-trust/source/anchors")) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestCloudCertificateTransportEncodesResourceNames(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + if query.Get("type") != "root+ca" || query.Get("ext") != "crt" { + t.Errorf("incorrectly encoded certificate resource: %s", r.URL) + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = io.WriteString(w, "raw certificate payload\n\n") + })) + defer server.Close() + c.wireEndpoint = server.URL + payload, err := c.operationCertificate(t.Context(), "root+ca.crt") + require.NoError(t, err) + require.Equal(t, "raw certificate payload\n", string(payload)) +} diff --git a/aks-node-controller/cloud_certificate_transport_test.go b/aks-node-controller/cloud_certificate_transport_test.go new file mode 100644 index 00000000000..207c1667930 --- /dev/null +++ b/aks-node-controller/cloud_certificate_transport_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "context" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCloudCertificateRequestRetryAndCancellation(t *testing.T) { + for _, tc := range []struct { + name string + status int + body string + failures int32 + wantWaits []time.Duration + }{ + {"transient", http.StatusServiceUnavailable, "do-not-log-body", 2, []time.Duration{3 * time.Second, 3 * time.Second}}, + {"rate limited", http.StatusForbidden, "RequestRateLimitExceeded", 2, []time.Duration{3 * time.Second, 6 * time.Second}}, + {"exhausted", http.StatusServiceUnavailable, "do-not-log-body", cloudRequestAttempts, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if attempts.Add(1) <= tc.failures { + w.WriteHeader(tc.status) + _, _ = io.WriteString(w, tc.body) + return + } + _, _ = io.WriteString(w, "success") + })) + defer server.Close() + c.wireEndpoint = server.URL + var waits []time.Duration + c.wait = func(_ context.Context, delay time.Duration) error { + waits = append(waits, delay) + return nil + } + body, err := c.requestCertificates(t.Context(), "/test") + if tc.failures == cloudRequestAttempts { + require.Error(t, err) + require.NotContains(t, err.Error(), tc.body) + require.EqualValues(t, cloudRequestAttempts, attempts.Load()) + require.Len(t, waits, cloudRequestAttempts-1) + } else { + require.NoError(t, err) + require.Equal(t, "success", string(body)) + require.Equal(t, tc.wantWaits, waits) + } + }) + } + t.Run("cancellation during backoff", func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + c.wireEndpoint = server.URL + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + c.wait = func(ctx context.Context, _ time.Duration) error { + cancel() + return ctx.Err() + } + _, err := c.requestCertificates(ctx, "/test") + require.ErrorIs(t, err, context.Canceled) + require.EqualValues(t, 1, attempts.Load()) + }) +} + +func TestCloudCertificateRequestsDoNotFollowRedirects(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var redirected atomic.Int32 + target := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { redirected.Add(1) })) + defer target.Close() + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer source.Close() + c.wireEndpoint = source.URL + _, err := c.requestCertificates(t.Context(), "/test") + require.ErrorContains(t, err, "HTTP 302") + require.Zero(t, redirected.Load()) +} + +func TestCloudCertificateNamesAndPublicContent(t *testing.T) { + for _, name := range []string{"", ".", "..", "../root.crt", "/root.crt", "root\x00.crt"} { + require.False(t, validCloudCertificateName(name), name) + } + for _, name := range []string{"root+ca.crt", `..\root.crt`, "root\n.crt"} { + require.True(t, validCloudCertificateName(name), name) + } + cert := cloudTestCertificate(t) + private := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte("not-a-real-private-key")}) + got, err := publicCloudCertificates(append(private, cert...)) + require.NoError(t, err) + require.Equal(t, cert, got) + for _, bad := range [][]byte{nil, private, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte("invalid")})} { + _, err = publicCloudCertificates(bad) + require.Error(t, err) + } +} diff --git a/aks-node-controller/cloud_test_helpers_test.go b/aks-node-controller/cloud_test_helpers_test.go index d99bac48b43..0efb19cc093 100644 --- a/aks-node-controller/cloud_test_helpers_test.go +++ b/aks-node-controller/cloud_test_helpers_test.go @@ -6,9 +6,12 @@ import ( "crypto/rand" "crypto/x509" "crypto/x509/pkix" + "encoding/json" "encoding/pem" "io" "math/big" + "net/http" + "net/http/httptest" "os/exec" "path/filepath" "testing" @@ -57,3 +60,31 @@ func cloudTestCertificate(t *testing.T) []byte { require.NoError(t, err) return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) } + +func serveCloudOperationTestResponses(t *testing.T, c *cloudInitializer, roots, intermediates []string) { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch kind := r.URL.Query().Get("type"); { + case r.URL.Path == "/acms/isOptedInForRootCerts": + _, _ = io.WriteString(w, `{"IsOptedInForRootCerts":true}`) + case kind == "operationrequestsroot", kind == "operationrequestsintermediate": + names := roots + if kind == "operationrequestsintermediate" { + names = intermediates + } + operations := make([]map[string]string, 0, len(names)) + for _, name := range names { + operations = append(operations, map[string]string{"ResouceFileName": name}) + } + _ = json.NewEncoder(w).Encode(map[string]any{"OperationsInfo": operations}) + case kind == "missing": + w.WriteHeader(http.StatusServiceUnavailable) + case kind == "empty": + _, _ = io.WriteString(w, "\n\n") + default: + _, _ = io.WriteString(w, "certificate payload\n\n") + } + })) + t.Cleanup(server.Close) + c.wireEndpoint = server.URL +}