Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions aks-node-controller/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,27 @@ the provisioning config in CustomData; the automatic size fallback uses the same
config-only payload. Leaving the new flag false preserves legacy, Phase 2, and
Phase 2.5 behavior.

### Azure cloud trust-store capabilities

The shared cloud runtime provides platform detection, bounded HTTP and command
execution, public-file writes, and guest-agent events. The trust-store helpers
write downloaded contents without PEM/X.509 filtering, update the platform's
existing CA tools, and preserve Ubuntu's OpenSSL bundle/symlink behavior.

Cloud commands run in their own process groups with a two-minute timeout, or
the caller's earlier cancellation. Cancellation kills the entire group. A
five-second `WaitDelay` bounds output draining after cancellation or command
exit, including when a descendant leaves the group but retains an output pipe.

Cache cleanup removes flat staging files, including hidden files, without
recursively deleting directories or removing installed system-trust certificates.
`AKS_CLOUD_LOG_CERTIFICATES=true` enables public-certificate-only diagnostics;
parsing for logging does not gate installation.

These are internal capabilities only. No certificate-retrieval flow, standalone
refresh command, refresh schedule, or native provisioning hook is enabled here.
Existing provisioning still uses `init-aks-cloud.sh`.

### Validating and normalizing AKSNodeConfig

Before serializing a typed config, producers such as RP should call:
Expand Down
126 changes: 126 additions & 0 deletions aks-node-controller/cloud_certificate_store.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
package main

import (
"bytes"
"context"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"log/slog"
"os"
"path/filepath"
"sort"
"strings"
)

func (c *cloudInitializer) clearCertificateCache() {
directory := c.path("/root/AzureCACertificates")
if err := os.MkdirAll(directory, 0o755); err != nil {
slog.Warn("could not create Azure cloud certificate cache", "error", err)
return
}
entries, err := os.ReadDir(directory)
if err != nil {
slog.Warn("could not read Azure cloud certificate cache", "error", err)
return
}
// Certificate downloads are written directly here as files, without subdirectories.
for _, entry := range entries {
if err := os.Remove(filepath.Join(directory, entry.Name())); err != nil {
slog.Warn("could not remove cached Azure cloud certificate", "name", entry.Name(), "error", err)
}
}
}

func publicCloudCertificates(data []byte) ([]byte, error) {
var result bytes.Buffer
for {
block, rest := pem.Decode(data)
if block == nil {
break
}
data = rest
if block.Type != "CERTIFICATE" {
continue
}
if _, err := x509.ParseCertificate(block.Bytes); err != nil {
return nil, fmt.Errorf("invalid CA certificate: %w", err)
}
if err := pem.Encode(&result, &pem.Block{Type: "CERTIFICATE", Bytes: block.Bytes}); err != nil {
return nil, err
}
}
if result.Len() == 0 {
return nil, errors.New("response contains no PEM certificates")
}
return result.Bytes(), nil
}

func (c *cloudInitializer) installCertificates(ctx context.Context, platform cloudPlatform, certificates map[string][]byte) error {
destination, update := "/usr/local/share/ca-certificates", "update-ca-certificates"
switch platform.ID {
case osReleaseIDAzureLinux, osReleaseIDMariner, osReleaseIDAzureContainerLinux:
destination, update = "/etc/pki/ca-trust/source/anchors", "update-ca-trust"
case osReleaseIDFlatcar:
destination = "/etc/ssl/certs"
}
names := make([]string, 0, len(certificates))
for name := range certificates {
names = append(names, name)
}
sort.Strings(names)
installed := 0
for _, name := range names {
data := certificates[name]
if !strings.HasSuffix(name, ".crt") {
continue
}
targetName := name
if platform.ID == osReleaseIDFlatcar {
targetName = strings.TrimSuffix(name, ".crt") + ".pem"
}
if err := c.writeFile(filepath.Join(destination, targetName), data); err != nil {
return err
}
installed++
if os.Getenv("AKS_CLOUD_LOG_CERTIFICATES") == "true" {
public, err := publicCloudCertificates(data)
if err != nil {
slog.Warn("could not decode public Azure cloud CA certificates for logging", "name", name, "error", err)
continue
}
slog.Info("Azure cloud public CA certificate", "name", name, "certificate", string(public))
}
}
if installed == 0 {
return errors.New("no *.crt files in Azure cloud certificate cache to install")
}
if _, err := c.command(ctx, nil, update); err != nil {
return err
}
if platform.ID == osReleaseIDUbuntu {
return c.copyOpenSSLBundle()
}
return nil
}

func (c *cloudInitializer) copyOpenSSLBundle() error {
source, target := c.path("/etc/ssl/certs/ca-certificates.crt"), c.path("/usr/lib/ssl/cert.pem")
sourceInfo, err := os.Stat(source)
if err != nil {
return err
}
targetInfo, err := os.Stat(target)
if err == nil && os.SameFile(sourceInfo, targetInfo) {
return nil
}
if err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
data, err := os.ReadFile(source)
if err != nil {
return err
}
return c.writeFile("/usr/lib/ssl/cert.pem", data)
}
103 changes: 103 additions & 0 deletions aks-node-controller/cloud_certificate_store_behavior_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package main

import (
"encoding/pem"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"

"github.com/stretchr/testify/require"
)

func TestCloudCertificateStorePlatformInstallation(t *testing.T) {
for _, tc := range []struct{ id, target, command string }{
{osReleaseIDUbuntu, "/usr/local/share/ca-certificates/root.crt", "update-ca-certificates"},
{osReleaseIDMariner, "/etc/pki/ca-trust/source/anchors/root.crt", "update-ca-trust"},
{osReleaseIDAzureLinux, "/etc/pki/ca-trust/source/anchors/root.crt", "update-ca-trust"},
{osReleaseIDAzureContainerLinux, "/etc/pki/ca-trust/source/anchors/root.crt", "update-ca-trust"},
{osReleaseIDFlatcar, "/etc/ssl/certs/root.pem", "update-ca-certificates"},
} {
t.Run(tc.id, func(t *testing.T) {
c := newTestCloudInitializer(t, "ID="+tc.id+"\n")
platform, err := c.platform()
require.NoError(t, err)
var commands []string
c.run = func(cmd *exec.Cmd) error {
commands = append(commands, cmd.Args[0])
return nil
}
payload := []byte("unparsed certificate payload\n")
require.NoError(t, c.installCertificates(t.Context(), platform, map[string][]byte{
"root.crt": payload,
"skip.cer": []byte("ignored"),
}))
got, err := os.ReadFile(c.path(tc.target))
require.NoError(t, err)
require.Equal(t, payload, got)
require.Equal(t, []string{tc.command}, commands)
_, err = os.Stat(c.path(filepath.Join(filepath.Dir(tc.target), "skip.cer")))
require.ErrorIs(t, err, os.ErrNotExist)
if tc.id == osReleaseIDUbuntu {
bundle, err := os.ReadFile(c.path("/usr/lib/ssl/cert.pem"))
require.NoError(t, err)
require.Equal(t, "system-ca-bundle", string(bundle))
}
})
}
}

func TestCloudCertificateStoreRejectsMissingCRTFiles(t *testing.T) {
c := newTestCloudInitializer(t, cloudTestUbuntu)
c.run = func(*exec.Cmd) error {
t.Fatal("empty certificate sets must not invoke trust-store tools")
return nil
}
err := c.installCertificates(t.Context(), cloudPlatform{ID: osReleaseIDUbuntu}, map[string][]byte{
"root.cer": []byte("ignored"),
})
require.ErrorContains(t, err, "no *.crt files")
}

func TestCloudCertificateStoreClearsOnlyFlatStagingFiles(t *testing.T) {
c := newTestCloudInitializer(t, cloudTestUbuntu)
for _, name := range []string{"root.crt", ".hidden.crt", "nested/keep.crt"} {
require.NoError(t, c.writeFile("/root/AzureCACertificates/"+name, []byte(name)))
}
installed := "/usr/local/share/ca-certificates/installed.crt"
require.NoError(t, c.writeFile(installed, []byte("installed CA")))
c.clearCertificateCache()
for _, name := range []string{"root.crt", ".hidden.crt"} {
_, err := os.Stat(c.path("/root/AzureCACertificates/" + name))
require.ErrorIs(t, err, os.ErrNotExist)
}
for _, name := range []string{"/root/AzureCACertificates/nested/keep.crt", installed} {
_, err := os.Stat(c.path(name))
require.NoError(t, err)
}
}

func TestCloudCertificateStoreLoggingPreservesRawPayload(t *testing.T) {
t.Setenv("AKS_CLOUD_LOG_CERTIFICATES", "true")
logs := installLogCapturer(t)
c := newTestCloudInitializer(t, "ID=azurelinux\n")
payload := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte("not-a-real-private-key")})
payload = append(payload, cloudTestCertificate(t)...)
require.NoError(t, c.installCertificates(t.Context(), cloudPlatform{ID: osReleaseIDAzureLinux}, map[string][]byte{
"root.crt": payload,
}))
got, err := os.ReadFile(c.path("/etc/pki/ca-trust/source/anchors/root.crt"))
require.NoError(t, err)
require.Equal(t, payload, got)
found := false
for _, record := range logs.getRecords() {
for _, value := range record.Attrs {
require.NotContains(t, value, "PRIVATE KEY")
if strings.Contains(value, "BEGIN CERTIFICATE") {
found = true
}
}
}
require.True(t, found)
}
18 changes: 18 additions & 0 deletions aks-node-controller/cloud_certificate_store_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package main

import (
"os"
"testing"

"github.com/stretchr/testify/require"
)

func TestCloudCertificateUbuntuBundleSymlink(t *testing.T) {
c := newTestCloudInitializer(t, cloudTestUbuntu)
require.NoError(t, os.MkdirAll(c.path("/usr/lib/ssl"), 0o755))
require.NoError(t, os.Symlink(c.path("/etc/ssl/certs/ca-certificates.crt"), c.path("/usr/lib/ssl/cert.pem")))
require.NoError(t, c.copyOpenSSLBundle())
info, err := os.Lstat(c.path("/usr/lib/ssl/cert.pem"))
require.NoError(t, err)
require.NotZero(t, info.Mode()&os.ModeSymlink)
}
Loading
Loading