From cfeae50716d3f129c15441452d3e151b2430b951 Mon Sep 17 00:00:00 2001 From: sinmentis Date: Tue, 27 Oct 2026 11:54:09 +1300 Subject: [PATCH 1/4] fix: reduce traditional Linux custom data encoding overhead Avoid per-file gzip/Base64 when the complete payload is already compressed. Use YAML-native quoting and omit the plain-text encoding key. Preserve file bytes and alternate delivery modes. AB#40305782 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 162f68f8-110a-4d60-b4fe-b9fb42253542 --- hotfix/hotfix_generate_test.py | 3 +- hotfix/render-nodecustomdata/main_test.go | 16 +- parts/linux/cloud-init/nodecustomdata.yml | 115 +++++------- pkg/agent/baker.go | 60 +++++- pkg/agent/baker_test.go | 11 +- pkg/agent/cloud_init_encoding_test.go | 172 ++++++++++++++++++ pkg/agent/live_patching_delivery_test.go | 4 +- .../mariner_live_patching_delivery_test.go | 4 +- pkg/agent/utils.go | 6 +- pkg/agent/utils_test.go | 13 +- pkg/agent/variables.go | 114 ++++++------ 11 files changed, 369 insertions(+), 149 deletions(-) create mode 100644 pkg/agent/cloud_init_encoding_test.go diff --git a/hotfix/hotfix_generate_test.py b/hotfix/hotfix_generate_test.py index 7626a0863c7..b321195994b 100644 --- a/hotfix/hotfix_generate_test.py +++ b/hotfix/hotfix_generate_test.py @@ -161,7 +161,8 @@ def test_config_modules_are_independently_hotfixable(self): self.assertEqual({variable}, blocks[0][0]) self.assertIn(f"- path: {{{{{path_function}}}}}", template) self.assertIn('permissions: "0744"', template) - self.assertIn("encoding: gzip", template) + self.assertIn("{{GetCloudInitFileEncodingLine}}", template) + self.assertIn("| GetCloudInitFileContent}}", template) self.assertNotIn("{{if", template) def test_mariner_package_update_is_not_embedded_in_custom_data(self): diff --git a/hotfix/render-nodecustomdata/main_test.go b/hotfix/render-nodecustomdata/main_test.go index d490001e6c4..55102a76c80 100644 --- a/hotfix/render-nodecustomdata/main_test.go +++ b/hotfix/render-nodecustomdata/main_test.go @@ -18,18 +18,16 @@ func TestRenderSelectedHotfixScripts(t *testing.T) { const initCloudBlock = `{{if IsAKSCustomCloud}} - path: {{GetInitAKSCloudFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "initAKSCloud"}} + content: {{GetVariableProperty "cloudInitData" "initAKSCloud" | GetCloudInitFileContent}} {{end}} ` const helperBlock = `- path: {{GetCSEHelpersScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSource"}} + content: {{GetVariableProperty "cloudInitData" "provisionSource" | GetCloudInitFileContent}} ` for _, distro := range []datamodel.Distro{ datamodel.AKSUbuntuContainerd2204Gen2, @@ -47,8 +45,9 @@ func TestRenderSelectedHotfixScripts(t *testing.T) { require.NoError(t, err) var payload struct { WriteFiles []struct { - Path string `yaml:"path"` - Content string `yaml:"content"` + Path string `yaml:"path"` + Encoding string `yaml:"encoding"` + Content string `yaml:"content"` } `yaml:"write_files"` } require.NoError(t, yaml.Unmarshal([]byte(rendered), &payload)) @@ -62,6 +61,7 @@ func TestRenderSelectedHotfixScripts(t *testing.T) { require.Len(t, payload.WriteFiles, 1+wantInitCount) for _, file := range payload.WriteFiles { require.NotEmpty(t, file.Content) + require.Empty(t, file.Encoding) } }) } diff --git a/parts/linux/cloud-init/nodecustomdata.yml b/parts/linux/cloud-init/nodecustomdata.yml index ece63879f98..6c59344a200 100644 --- a/parts/linux/cloud-init/nodecustomdata.yml +++ b/parts/linux/cloud-init/nodecustomdata.yml @@ -28,174 +28,151 @@ write_files: {{if IsAKSCustomCloud}} - path: {{GetInitAKSCloudFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "initAKSCloud"}} + content: {{GetVariableProperty "cloudInitData" "initAKSCloud" | GetCloudInitFileContent}} {{end}} {{- else }} - path: {{GetCSEHelpersScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSource"}} + content: {{GetVariableProperty "cloudInitData" "provisionSource" | GetCloudInitFileContent}} {{if IsACL }} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceACL"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceACL" | GetCloudInitFileContent}} {{- else if IsAzlOSGuard}} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceAzlOSGuard"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceAzlOSGuard" | GetCloudInitFileContent}} {{- else if IsMariner}} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceMariner"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceMariner" | GetCloudInitFileContent}} {{- else if IsFlatcar }} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceFlatcar"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceFlatcar" | GetCloudInitFileContent}} {{- else }} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceUbuntu"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceUbuntu" | GetCloudInitFileContent}} {{end}} {{ if not IsCustomImage -}} - path: /opt/azure/containers/provision_start.sh permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionStartScript"}} + content: {{GetVariableProperty "cloudInitData" "provisionStartScript" | GetCloudInitFileContent}} {{- end }} - path: /opt/azure/containers/provision.sh permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionScript"}} + content: {{GetVariableProperty "cloudInitData" "provisionScript" | GetCloudInitFileContent}} - path: {{GetCSEInstallScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstalls"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstalls" | GetCloudInitFileContent}} {{if IsACL }} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsACL"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsACL" | GetCloudInitFileContent}} {{- else if IsAzlOSGuard}} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsAzlOSGuard"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsAzlOSGuard" | GetCloudInitFileContent}} {{- else if IsMariner}} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsMariner"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsMariner" | GetCloudInitFileContent}} {{- else if IsFlatcar }} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsFlatcar"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsFlatcar" | GetCloudInitFileContent}} {{- else }} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsUbuntu"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsUbuntu" | GetCloudInitFileContent}} {{end}} - path: {{GetCSEConfigScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigs"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigs" | GetCloudInitFileContent}} - path: {{GetCSEConfigGPUScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsGPU"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsGPU" | GetCloudInitFileContent}} - path: {{GetCSEConfigLocalDNSScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsLocalDNS"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsLocalDNS" | GetCloudInitFileContent}} - path: {{GetCSEConfigKubeletScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsKubelet"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsKubelet" | GetCloudInitFileContent}} - path: {{GetCSEConfigNetworkScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsNetwork"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsNetwork" | GetCloudInitFileContent}} - path: {{GetCSEConfigChronyScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsChrony"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsChrony" | GetCloudInitFileContent}} {{if IsAKSCustomCloud}} - path: {{GetInitAKSCloudFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "initAKSCloud"}} + content: {{GetVariableProperty "cloudInitData" "initAKSCloud" | GetCloudInitFileContent}} {{end}} - path: /etc/systemd/system/kubelet.service permissions: "0600" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "kubeletSystemdService"}} + content: {{GetVariableProperty "cloudInitData" "kubeletSystemdService" | GetCloudInitFileContent}} {{- end }} diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 6a910755dfb..3789be41292 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "text/template" + "unicode/utf8" "github.com/Azure/agentbaker/parts" "github.com/Azure/agentbaker/pkg/agent/datamodel" @@ -346,17 +347,20 @@ func buildScriptlessCustomData(cloudInitTemplate, fileListTemplate, separator st // GetLinuxNodeCustomDataJSONObject returns Linux customData JSON object in the form. // { "customData": "" }. func (t *TemplateGenerator) getLinuxNodeCustomDataJSONObject(config *datamodel.NodeBootstrappingConfiguration) string { - // get parameters - parameters := getParameters(config) - // get variable cloudInit - variables := getCustomDataVariables(config) - str, e := t.getSingleLineForTemplate(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, getBakerFuncMap(config, parameters, variables), true) - - if e != nil { - panic(e) + b, err := parts.Templates.ReadFile(kubernetesNodeCustomDataYaml) + if err != nil { + panic(err) + } + str, err := RenderLinuxNodeCustomDataTemplate(b, config) + if err != nil { + panic(err) } - return fmt.Sprintf("{\"customData\": \"%s\"}", str) + customData, err := json.Marshal(map[string]string{"customData": str}) + if err != nil { + panic(err) + } + return string(customData) } const ( @@ -674,6 +678,8 @@ func (t *TemplateGenerator) getSingleLine(textFilename string, profile interface // same variables and functions used by the production AgentBaker path. // Callers must supply a complete configuration, as required by the production // rendering helpers. +// File entries must use GetCloudInitFileEncodingLine and GetCloudInitFileContent +// to select the encoding for the delivery mode. func RenderLinuxNodeCustomDataTemplate(templateContent []byte, config *datamodel.NodeBootstrappingConfiguration) (string, error) { parameters := getParameters(config) variables := getCustomDataVariables(config) @@ -691,10 +697,46 @@ func RenderLinuxNodeCustomDataTemplate(templateContent []byte, config *datamodel return buffer.String(), nil } +// Traditional custom data is gzipped as a whole, so its files need no inner encoding. +func usePlainCloudInit(config *datamodel.NodeBootstrappingConfiguration) bool { + return !config.EnableScriptlessCSECmd && !config.IsFlatcar() && !config.IsACL() +} + +func getCloudInitFileEncodingLine(config *datamodel.NodeBootstrappingConfiguration) string { + if usePlainCloudInit(config) { + return "" + } + return "encoding: " + encodingGZIP +} + +func getCloudInitFileContent(config *datamodel.NodeBootstrappingConfiguration, content string) (string, error) { + if usePlainCloudInit(config) { + if !utf8.ValidString(content) { + return "", fmt.Errorf("cloud-init file content contains invalid UTF-8") + } + // YAML quoting preserves Unicode line breaks that JSON leaves unescaped. + encoded, err := yaml.Marshal(&yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Style: yaml.DoubleQuotedStyle, + Value: content, + }) + return strings.TrimSuffix(string(encoded), "\n"), err + } + return "!!binary |\n " + content, nil +} + // getTemplateFuncMap returns the general purpose template func map from getContainerServiceFuncMap. func getBakerFuncMap(config *datamodel.NodeBootstrappingConfiguration, params paramsMap, variables paramsMap) template.FuncMap { funcMap := getContainerServiceFuncMap(config) + funcMap["GetCloudInitFileEncodingLine"] = func() string { + return getCloudInitFileEncodingLine(config) + } + funcMap["GetCloudInitFileContent"] = func(content string) (string, error) { + return getCloudInitFileContent(config, content) + } + funcMap["GetParameter"] = func(s string) interface{} { if v, ok := params[s].(paramsMap); ok && v != nil { if v["value"] == nil { diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index ae71d3a0cf8..746ac9c1eed 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/gomega" "github.com/stretchr/testify/require" "github.com/vincent-petithory/dataurl" + "gopkg.in/yaml.v3" ) // this regex looks for groups of the following forms, returning KEY and VALUE as submatches. @@ -1606,8 +1607,14 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { expectedCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(config)) Expect(string(decompressedPayload)).To(Equal(expectedCustomData)) - Expect(string(decompressedPayload)).NotTo(ContainSubstring(aksNodeCustomDataFilepath)) - Expect(string(decompressedPayload)).NotTo(ContainSubstring(aksNbcCmdFilepath)) + // Script contents can reference these paths; only delivery as write_files + // entries would indicate that the scriptless path was used. + var cloudConfig cloudInit + Expect(yaml.Unmarshal(decompressedPayload, &cloudConfig)).To(Succeed()) + for _, file := range cloudConfig.WriteFiles { + Expect(file.Path).NotTo(Equal(aksNodeCustomDataFilepath)) + Expect(file.Path).NotTo(Equal(aksNbcCmdFilepath)) + } }) }) diff --git a/pkg/agent/cloud_init_encoding_test.go b/pkg/agent/cloud_init_encoding_test.go new file mode 100644 index 00000000000..cef7d2ffc9f --- /dev/null +++ b/pkg/agent/cloud_init_encoding_test.go @@ -0,0 +1,172 @@ +package agent + +import ( + "encoding/base64" + "fmt" + "strings" + "testing" + "text/template" + + "github.com/Azure/agentbaker/parts" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestTraditionalCloudInitUsesSingleCompression(t *testing.T) { + template, err := parts.Templates.ReadFile(kubernetesNodeCustomDataYaml) + require.NoError(t, err) + for _, distro := range []datamodel.Distro{datamodel.AKSUbuntuContainerd2204Gen2, datamodel.AKSAzureLinuxV2Gen2, datamodel.AKSAzureLinuxV3Gen2} { + for _, custom := range []bool{false, true} { + for _, dns := range []bool{false, true} { + for _, preProvision := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/custom=%t/localdns=%t/preProvision=%t", distro, custom, dns, preProvision), func(t *testing.T) { + config := newNodeCustomDataRenderConfig(distro) + config.PreProvisionOnly = preProvision + config.AgentPoolProfile.LocalDNSProfile = &datamodel.LocalDNSProfile{EnableLocalDNS: dns} + if custom { + config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{Name: "akscustom"} + } + assertTraditionalCloudInitPayload(t, template, config) + }) + } + } + } + } +} + +func assertTraditionalCloudInitPayload(t *testing.T, template []byte, config *datamodel.NodeBootstrappingConfiguration) { + t.Helper() + generator := InitializeTemplateGenerator() + original, err := generator.getSingleLine(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, legacyCloudInitFuncMap(config), true) + require.NoError(t, err) + payload := generator.getLinuxNodeBootstrappingPayload(config) + compressed, err := base64.StdEncoding.Strict().DecodeString(payload) + require.NoError(t, err) + decoded, err := getGzipDecodedValue(compressed) + require.NoError(t, err) + require.True(t, strings.HasPrefix(string(decoded), "#cloud-config\n")) + rendered, err := RenderLinuxNodeCustomDataTemplate(template, config) + require.NoError(t, err) + require.Equal(t, rendered, string(decoded)) + var before, after cloudInit + require.NoError(t, yaml.Unmarshal([]byte(original), &before)) + require.NoError(t, yaml.Unmarshal(decoded, &after)) + var document struct { + WriteFiles []map[string]any `yaml:"write_files"` + } + require.NoError(t, yaml.Unmarshal(decoded, &document)) + for _, file := range document.WriteFiles { + _, hasEncoding := file["encoding"] + require.False(t, hasEncoding, "plain files must use cloud-init's default, not an invalid empty encoding") + } + beforeFiles, err := buildIgnitionTarEntries(before) + require.NoError(t, err) + afterFiles, err := buildIgnitionTarEntries(after) + require.NoError(t, err) + require.Equal(t, beforeFiles, afterFiles) + for _, file := range after.WriteFiles { + require.Empty(t, file.Encoding) + } + require.Less(t, len(payload), len(getBase64EncodedGzippedCustomScriptFromStr(original))) + t.Logf("customData: %d encoded characters", len(payload)) + require.LessOrEqual(t, len(payload), MaxCustomDataLength) +} + +func TestCloudInitEncodingLeavesOtherModesUnchanged(t *testing.T) { + template, err := parts.Templates.ReadFile(kubernetesNodeCustomDataYaml) + require.NoError(t, err) + generator := InitializeTemplateGenerator() + for _, distro := range []datamodel.Distro{datamodel.AKSUbuntuContainerd2204Gen2, datamodel.AKSAzureLinuxV3Gen2, datamodel.AKSFlatcarGen2, datamodel.AKSACLGen2TL} { + for _, scriptless := range []bool{false, true} { + if !scriptless && distro != datamodel.AKSFlatcarGen2 && distro != datamodel.AKSACLGen2TL { + continue + } + t.Run(fmt.Sprintf("%s/scriptless=%t", distro, scriptless), func(t *testing.T) { + config := newNodeCustomDataRenderConfig(distro) + config.EnableScriptlessCSECmd = scriptless + config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{Name: "akscustom"} + original, err := generator.getSingleLine(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, legacyCloudInitFuncMap(config), true) + require.NoError(t, err) + rendered, err := RenderLinuxNodeCustomDataTemplate(template, config) + require.NoError(t, err) + require.Equal(t, original, rendered) + require.Equal(t, original, getCustomDataFromJSON(generator.getLinuxNodeCustomDataJSONObject(config))) + }) + } + } +} + +func legacyCloudInitFuncMap(config *datamodel.NodeBootstrappingConfiguration) template.FuncMap { + funcs := getBakerFuncMap(config, getParameters(config), getCustomDataVariablesWithEncoder(config, false)) + funcs["GetCloudInitFileEncodingLine"] = func() string { return "encoding: " + encodingGZIP } + funcs["GetCloudInitFileContent"] = func(content string) string { return "!!binary |\n " + content } + return funcs +} + +func TestCloudInitRenderingPreservesAdditionalFields(t *testing.T) { + input := []byte("#cloud-config\nruncmd:\n- echo hello\nwrite_files:\n- path: /opt/example\n append: true\n defer: true\n content: \"a\\tb\\n\\n\"\n") + output, err := RenderLinuxNodeCustomDataTemplate(input, newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2)) + require.NoError(t, err) + require.Equal(t, string(input), output) +} + +func TestPlainCloudInitFileContentPreservesWhitespace(t *testing.T) { + input := []byte(`#cloud-config +write_files: +- path: /opt/example + {{GetCloudInitFileEncodingLine}} + content: {{.Name | GetCloudInitFileContent}} +`) + for _, content := range []string{ + "", + "tab\tquote\"backslash\\", + "trailing newline\n\n", + "\u00e9\r\n", + "next\u0085line\u2028paragraph\u2029end", + "nul\x00escape\x1b", + "echo \"$HOME\"; printf '%s\\n' 'a: # & < >'\n", + strings.Repeat("long line: # \"\\ ", 1024), + } { + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) + config.AgentPoolProfile.Name = content + output, err := RenderLinuxNodeCustomDataTemplate(input, config) + require.NoError(t, err) + var parsed cloudInit + require.NoError(t, yaml.Unmarshal([]byte(output), &parsed)) + require.Len(t, parsed.WriteFiles, 1) + require.Empty(t, parsed.WriteFiles[0].Encoding) + require.Equal(t, content, parsed.WriteFiles[0].Content) + } +} + +func TestPlainCloudInitFileContentRejectsInvalidUTF8(t *testing.T) { + input := []byte("#cloud-config\nwrite_files:\n- path: /opt/example\n content: {{.Name | GetCloudInitFileContent}}\n") + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) + config.AgentPoolProfile.Name = string([]byte{'a', 0xff, 'b'}) + output, err := RenderLinuxNodeCustomDataTemplate(input, config) + require.ErrorContains(t, err, "invalid UTF-8") + require.Empty(t, output) +} + +func BenchmarkTraditionalCloudInitEncoding(b *testing.B) { + for _, old := range []bool{true, false} { + b.Run(fmt.Sprintf("legacy=%t", old), func(b *testing.B) { + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) + generator := InitializeTemplateGenerator() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if old { + rendered, err := generator.getSingleLineForTemplate(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, legacyCloudInitFuncMap(config), true) + if err != nil { + b.Fatal(err) + } + customData := getCustomDataFromJSON(fmt.Sprintf("{\"customData\": \"%s\"}", rendered)) + _ = getBase64EncodedGzippedCustomScriptFromStr(customData) + } else { + _ = generator.getLinuxNodeBootstrappingPayload(config) + } + } + }) + } +} diff --git a/pkg/agent/live_patching_delivery_test.go b/pkg/agent/live_patching_delivery_test.go index b616b2bcf0e..04ca8b846c2 100644 --- a/pkg/agent/live_patching_delivery_test.go +++ b/pkg/agent/live_patching_delivery_test.go @@ -18,7 +18,9 @@ func TestLivePatchingUsesVHDInsteadOfCustomData(t *testing.T) { require.NotContains(t, rendered, "/opt/azure/containers/ubuntu-snapshot-update.sh") require.NotContains(t, rendered, "/opt/azure/containers/security-update.sh") require.NotContains(t, rendered, "/opt/azure/containers/npd-update.sh") - size := len(getBase64EncodedGzippedCustomScriptFromStr(rendered)) + payload := InitializeTemplateGenerator().getLinuxNodeBootstrappingPayload(config) + require.Equal(t, getBase64EncodedGzippedCustomScriptFromStr(rendered), payload) + size := len(payload) t.Logf("traditional custom data: %d encoded characters", size) require.Less(t, size, MaxCustomDataLength) } diff --git a/pkg/agent/mariner_live_patching_delivery_test.go b/pkg/agent/mariner_live_patching_delivery_test.go index e7cf41fc494..7b9cb9293f3 100644 --- a/pkg/agent/mariner_live_patching_delivery_test.go +++ b/pkg/agent/mariner_live_patching_delivery_test.go @@ -20,7 +20,9 @@ func TestMarinerLivePatchingUsesVHDInsteadOfCustomData(t *testing.T) { require.NotContains(t, rendered, "/opt/azure/containers/mariner-package-update.sh") require.NotContains(t, rendered, "/opt/azure/containers/security-update.sh") require.NotContains(t, rendered, "/opt/azure/containers/npd-update.sh") - size := len(getBase64EncodedGzippedCustomScriptFromStr(rendered)) + payload := InitializeTemplateGenerator().getLinuxNodeBootstrappingPayload(config) + require.Equal(t, getBase64EncodedGzippedCustomScriptFromStr(rendered), payload) + size := len(payload) t.Logf("traditional custom data: %d encoded characters", size) require.Less(t, size, MaxCustomDataLength) }) diff --git a/pkg/agent/utils.go b/pkg/agent/utils.go index 1bc6c2d8fc1..67a251a7b6a 100644 --- a/pkg/agent/utils.go +++ b/pkg/agent/utils.go @@ -182,6 +182,10 @@ func escapeSingleLine(escapedStr string) string { // getBase64EncodedGzippedCustomScript will return a base64 of the CSE. func getBase64EncodedGzippedCustomScript(csFilename string, config *datamodel.NodeBootstrappingConfiguration) string { + return getBase64EncodedGzippedCustomScriptFromStr(getCustomScript(csFilename, config)) +} + +func getCustomScript(csFilename string, config *datamodel.NodeBootstrappingConfiguration) string { b, err := parts.Templates.ReadFile(csFilename) if err != nil { // this should never happen and this is a bug. @@ -209,7 +213,7 @@ func getBase64EncodedGzippedCustomScript(csFilename string, config *datamodel.No panic(fmt.Sprintf("BUG: failed to minify %s: %v", csFilename, err)) } } - return getBase64EncodedGzippedCustomScriptFromStr(csStr) + return csStr } func minifyShellScript(name, script string) (string, error) { diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index 44108764db4..e442ced785a 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -1348,6 +1348,11 @@ func TestCSEScriptRoundTrip(t *testing.T) { require.NoError(t, err) decoded, err := getGzipDecodedValue(compressed) require.NoError(t, err) + plain := getCustomScript(script, config) + minified, err := minifyShellScript(script, plain) + require.NoError(t, err) + require.Equal(t, minified, plain, "plain delivery must not bypass the minifier") + require.Equal(t, plain, string(decoded)) if script == kubernetesCSEHelpersScript { require.True(t, strings.HasSuffix(string(decoded), "#HELPERSEOF\n")) } @@ -1357,21 +1362,21 @@ func TestCSEScriptRoundTrip(t *testing.T) { } // discoverCSEScripts parses variables.go to find all constant names passed to -// getBase64EncodedGzippedCustomScript(), then resolves those constants to file +// encodeScript(), then resolves those constants to file // paths from const.go, filtering to .sh files only. func discoverCSEScripts(t *testing.T) []string { t.Helper() root := repoRoot() - // Step 1: Read variables.go and extract constant names from getBase64EncodedGzippedCustomScript() calls + // Step 1: Read variables.go and extract constant names from encodeScript() calls variablesPath := filepath.Join(root, "pkg", "agent", "variables.go") variablesBytes, err := os.ReadFile(variablesPath) if err != nil { t.Fatalf("failed to read variables.go: %v", err) } - // Match: getBase64EncodedGzippedCustomScript(constantName, config) - callRe := regexp.MustCompile(`getBase64EncodedGzippedCustomScript\((\w+),`) + // Match: encodeScript(constantName, config) + callRe := regexp.MustCompile(`encodeScript\((\w+),`) matches := callRe.FindAllStringSubmatch(string(variablesBytes), -1) constNames := make(map[string]bool) for _, m := range matches { diff --git a/pkg/agent/variables.go b/pkg/agent/variables.go index e83f5e51292..c0e254b57a0 100644 --- a/pkg/agent/variables.go +++ b/pkg/agent/variables.go @@ -12,72 +12,80 @@ import ( // getCustomDataVariables returns cloudinit data used by Linux. func getCustomDataVariables(config *datamodel.NodeBootstrappingConfiguration) paramsMap { + return getCustomDataVariablesWithEncoder(config, usePlainCloudInit(config)) +} + +func getCustomDataVariablesWithEncoder(config *datamodel.NodeBootstrappingConfiguration, plain bool) paramsMap { + encodeScript := getBase64EncodedGzippedCustomScript + if plain { + encodeScript = getCustomScript + } cs := config.ContainerService cloudInitFiles := map[string]interface{}{ "cloudInitData": paramsMap{ - "provisionStartScript": getBase64EncodedGzippedCustomScript(kubernetesCSEStartScript, config), - "provisionScript": getBase64EncodedGzippedCustomScript(kubernetesCSEMainScript, config), - "provisionSource": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScript, config), - "provisionSourceUbuntu": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptUbuntu, config), - "provisionSourceMariner": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptMariner, config), - "provisionSourceAzlOSGuard": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptAzlOSGuard, config), - "provisionSourceFlatcar": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptFlatcar, config), - "provisionSourceACL": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptACL, config), - "provisionInstalls": getBase64EncodedGzippedCustomScript(kubernetesCSEInstall, config), - "provisionInstallsUbuntu": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallUbuntu, config), - "provisionInstallsMariner": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallMariner, config), - "provisionInstallsAzlOSGuard": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallAzlOSGuard, config), - "provisionInstallsFlatcar": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallFlatcar, config), - "provisionInstallsACL": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallACL, config), - "provisionConfigs": getBase64EncodedGzippedCustomScript(kubernetesCSEConfig, config), - "provisionConfigsGPU": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigGPU, config), - "provisionConfigsLocalDNS": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigLocalDNS, config), - "provisionConfigsKubelet": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigKubelet, config), - "provisionConfigsNetwork": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigNetwork, config), - "provisionConfigsChrony": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigChrony, config), - "provisionSendLogs": getBase64EncodedGzippedCustomScript(kubernetesCSESendLogs, config), - "provisionRedactCloudConfig": getBase64EncodedGzippedCustomScript(kubernetesCSERedactCloudConfig, config), - "customSearchDomainsScript": getBase64EncodedGzippedCustomScript(kubernetesCustomSearchDomainsScript, config), - "dhcpv6SystemdService": getBase64EncodedGzippedCustomScript(dhcpv6SystemdService, config), - "dhcpv6ConfigurationScript": getBase64EncodedGzippedCustomScript(dhcpv6ConfigurationScript, config), - "kubeletSystemdService": getBase64EncodedGzippedCustomScript(kubeletSystemdService, config), - "reconcilePrivateHostsScript": getBase64EncodedGzippedCustomScript(reconcilePrivateHostsScript, config), - "reconcilePrivateHostsService": getBase64EncodedGzippedCustomScript(reconcilePrivateHostsService, config), - "ensureNoDupEbtablesScript": getBase64EncodedGzippedCustomScript(ensureNoDupEbtablesScript, config), - "ensureNoDupEbtablesService": getBase64EncodedGzippedCustomScript(ensureNoDupEbtablesService, config), - "bindMountScript": getBase64EncodedGzippedCustomScript(bindMountScript, config), - "bindMountSystemdService": getBase64EncodedGzippedCustomScript(bindMountSystemdService, config), - "migPartitionSystemdService": getBase64EncodedGzippedCustomScript(migPartitionSystemdService, config), - "migPartitionScript": getBase64EncodedGzippedCustomScript(migPartitionScript, config), - "ensureIMDSRestrictionScript": getBase64EncodedGzippedCustomScript(ensureIMDSRestrictionScript, config), - "snapshotUpdateScript": getBase64EncodedGzippedCustomScript(snapshotUpdateScript, config), - "snapshotUpdateService": getBase64EncodedGzippedCustomScript(snapshotUpdateSystemdService, config), - "snapshotUpdateTimer": getBase64EncodedGzippedCustomScript(snapshotUpdateSystemdTimer, config), - "packageUpdateScriptMariner": getBase64EncodedGzippedCustomScript(packageUpdateScriptMariner, config), - "packageUpdateServiceMariner": getBase64EncodedGzippedCustomScript(packageUpdateSystemdServiceMariner, config), - "packageUpdateTimerMariner": getBase64EncodedGzippedCustomScript(packageUpdateSystemdTimerMariner, config), - "componentManifestFile": getBase64EncodedGzippedCustomScript(componentManifestFile, config), - "validateKubeletCredentialsScript": getBase64EncodedGzippedCustomScript(validateKubeletCredentialsScript, config), - "secureTLSBootstrapService": getBase64EncodedGzippedCustomScript(secureTLSBootstrapService, config), - "cloudInitStatusCheckScript": getBase64EncodedGzippedCustomScript(cloudInitStatusCheckScript, config), - "measureTLSBootstrappingLatencyScript": getBase64EncodedGzippedCustomScript(measureTLSBootstrappingLatencyScript, config), - "measureTLSBootstrappingLatencyService": getBase64EncodedGzippedCustomScript(measureTLSBootstrappingLatencyService, config), - "configureAzureNetworkScript": getBase64EncodedGzippedCustomScript(configureAzureNetworkScript, config), - "azureNetworkUdevRule": getBase64EncodedGzippedCustomScript(azureNetworkUdevRule, config), + "provisionStartScript": encodeScript(kubernetesCSEStartScript, config), + "provisionScript": encodeScript(kubernetesCSEMainScript, config), + "provisionSource": encodeScript(kubernetesCSEHelpersScript, config), + "provisionSourceUbuntu": encodeScript(kubernetesCSEHelpersScriptUbuntu, config), + "provisionSourceMariner": encodeScript(kubernetesCSEHelpersScriptMariner, config), + "provisionSourceAzlOSGuard": encodeScript(kubernetesCSEHelpersScriptAzlOSGuard, config), + "provisionSourceFlatcar": encodeScript(kubernetesCSEHelpersScriptFlatcar, config), + "provisionSourceACL": encodeScript(kubernetesCSEHelpersScriptACL, config), + "provisionInstalls": encodeScript(kubernetesCSEInstall, config), + "provisionInstallsUbuntu": encodeScript(kubernetesCSEInstallUbuntu, config), + "provisionInstallsMariner": encodeScript(kubernetesCSEInstallMariner, config), + "provisionInstallsAzlOSGuard": encodeScript(kubernetesCSEInstallAzlOSGuard, config), + "provisionInstallsFlatcar": encodeScript(kubernetesCSEInstallFlatcar, config), + "provisionInstallsACL": encodeScript(kubernetesCSEInstallACL, config), + "provisionConfigs": encodeScript(kubernetesCSEConfig, config), + "provisionConfigsGPU": encodeScript(kubernetesCSEConfigGPU, config), + "provisionConfigsLocalDNS": encodeScript(kubernetesCSEConfigLocalDNS, config), + "provisionConfigsKubelet": encodeScript(kubernetesCSEConfigKubelet, config), + "provisionConfigsNetwork": encodeScript(kubernetesCSEConfigNetwork, config), + "provisionConfigsChrony": encodeScript(kubernetesCSEConfigChrony, config), + "provisionSendLogs": encodeScript(kubernetesCSESendLogs, config), + "provisionRedactCloudConfig": encodeScript(kubernetesCSERedactCloudConfig, config), + "customSearchDomainsScript": encodeScript(kubernetesCustomSearchDomainsScript, config), + "dhcpv6SystemdService": encodeScript(dhcpv6SystemdService, config), + "dhcpv6ConfigurationScript": encodeScript(dhcpv6ConfigurationScript, config), + "kubeletSystemdService": encodeScript(kubeletSystemdService, config), + "reconcilePrivateHostsScript": encodeScript(reconcilePrivateHostsScript, config), + "reconcilePrivateHostsService": encodeScript(reconcilePrivateHostsService, config), + "ensureNoDupEbtablesScript": encodeScript(ensureNoDupEbtablesScript, config), + "ensureNoDupEbtablesService": encodeScript(ensureNoDupEbtablesService, config), + "bindMountScript": encodeScript(bindMountScript, config), + "bindMountSystemdService": encodeScript(bindMountSystemdService, config), + "migPartitionSystemdService": encodeScript(migPartitionSystemdService, config), + "migPartitionScript": encodeScript(migPartitionScript, config), + "ensureIMDSRestrictionScript": encodeScript(ensureIMDSRestrictionScript, config), + "snapshotUpdateScript": encodeScript(snapshotUpdateScript, config), + "snapshotUpdateService": encodeScript(snapshotUpdateSystemdService, config), + "snapshotUpdateTimer": encodeScript(snapshotUpdateSystemdTimer, config), + "packageUpdateScriptMariner": encodeScript(packageUpdateScriptMariner, config), + "packageUpdateServiceMariner": encodeScript(packageUpdateSystemdServiceMariner, config), + "packageUpdateTimerMariner": encodeScript(packageUpdateSystemdTimerMariner, config), + "componentManifestFile": encodeScript(componentManifestFile, config), + "validateKubeletCredentialsScript": encodeScript(validateKubeletCredentialsScript, config), + "secureTLSBootstrapService": encodeScript(secureTLSBootstrapService, config), + "cloudInitStatusCheckScript": encodeScript(cloudInitStatusCheckScript, config), + "measureTLSBootstrappingLatencyScript": encodeScript(measureTLSBootstrappingLatencyScript, config), + "measureTLSBootstrappingLatencyService": encodeScript(measureTLSBootstrappingLatencyService, config), + "configureAzureNetworkScript": encodeScript(configureAzureNetworkScript, config), + "azureNetworkUdevRule": encodeScript(azureNetworkUdevRule, config), }, } cloudInitData := cloudInitFiles["cloudInitData"].(paramsMap) //nolint:errcheck // no error is actually here - cloudInitData["initAKSCloud"] = getBase64EncodedGzippedCustomScript(initAKSCloudScript, config) + cloudInitData["initAKSCloud"] = encodeScript(initAKSCloudScript, config) if config.IsFlatcar() || config.IsACL() { cloudInitData["provisionRedactCloudConfig"] = "" // Flatcar and ACL do not have cloud-init } if !cs.Properties.IsVHDDistroForAllNodes() { - cloudInitData["kmsSystemdService"] = getBase64EncodedGzippedCustomScript(kmsSystemdService, config) - cloudInitData["aptPreferences"] = getBase64EncodedGzippedCustomScript(aptPreferences, config) - cloudInitData["dockerClearMountPropagationFlags"] = getBase64EncodedGzippedCustomScript(dockerClearMountPropagationFlags, config) + cloudInitData["kmsSystemdService"] = encodeScript(kmsSystemdService, config) + cloudInitData["aptPreferences"] = encodeScript(aptPreferences, config) + cloudInitData["dockerClearMountPropagationFlags"] = encodeScript(dockerClearMountPropagationFlags, config) } return cloudInitFiles From d1d7649d33d70e5dc21afdc2bd21c3c618f623b2 Mon Sep 17 00:00:00 2001 From: sinmentis Date: Tue, 27 Oct 2026 22:54:57 +1300 Subject: [PATCH 2/4] fix: verify traditional Linux custom data with CRC Check delivered file contents and lengths before initialization. Keep expected checksums in the CSE command so CustomData remains unchanged. AB#40305782 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 162f68f8-110a-4d60-b4fe-b9fb42253542 --- parts/linux/cloud-init/artifacts/cse_cmd.sh | 9 ++ pkg/agent/baker.go | 60 +++++++++ pkg/agent/cloud_init_encoding_test.go | 120 ++++++++++++++++++ .../cloud-init/artifacts/cse_cmd_spec.sh | 50 ++++++++ 4 files changed, 239 insertions(+) create mode 100644 spec/parts/linux/cloud-init/artifacts/cse_cmd_spec.sh diff --git a/parts/linux/cloud-init/artifacts/cse_cmd.sh b/parts/linux/cloud-init/artifacts/cse_cmd.sh index a08d3b35d6d..a9cb450cda6 100644 --- a/parts/linux/cloud-init/artifacts/cse_cmd.sh +++ b/parts/linux/cloud-init/artifacts/cse_cmd.sh @@ -15,6 +15,15 @@ else echo "cloud-init failed with exit code ${cloudInitExitCode}" >> ${PROVISION_OUTPUT}; exit ${cloudInitExitCode}; fi; +{{with GetCloudInitFileChecksums -}} +for checksum in {{.}}; do + file="${checksum##* }"; + if ! actual=$(cksum -- "${file}" 2>> "${PROVISION_OUTPUT}") || [ "${actual}" != "${checksum}" ]; then + echo "CustomData checksum verification failed: ${file}" | tee -a "${PROVISION_OUTPUT}"; + exit 1; + fi; +done; +{{end -}} {{end}} INIT_AKS_CLOUD_FILEPATH="{{GetInitAKSCloudFilepath}}"; if [ -f "${INIT_AKS_CLOUD_FILEPATH}" ]; then diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 3789be41292..aab6571fe19 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -726,6 +726,63 @@ func getCloudInitFileContent(config *datamodel.NodeBootstrappingConfiguration, c return "!!binary |\n " + content, nil } +// POSIX cksum uses an MSB-first CRC-32 and includes the byte length. +func posixCksum(content string) uint32 { + var table [256]uint32 + for i := range table { + crc := uint32(i) << 24 + for bit := 0; bit < 8; bit++ { + if crc&0x80000000 != 0 { + crc = crc<<1 ^ 0x04c11db7 + } else { + crc <<= 1 + } + } + table[i] = crc + } + var crc uint32 + for i := 0; i < len(content); i++ { + crc = crc<<8 ^ table[byte(crc>>24)^content[i]] + } + for size := len(content); size > 0; size >>= 8 { + crc = crc<<8 ^ table[byte(crc>>24)^byte(size)] + } + return ^crc +} + +var cloudInitChecksumPattern = regexp.MustCompile(`^[0-9]+ [0-9]+ /[A-Za-z0-9._/-]+$`) + +func getCloudInitFileChecksums(config *datamodel.NodeBootstrappingConfiguration) (string, error) { + if config.DisableCustomData || !usePlainCloudInit(config) { + return "", nil + } + funcs := getBakerFuncMap(config, getParameters(config), getCustomDataVariables(config)) + // Render only source checksums, so YAML never processes the bytes being checked. + funcs["GetCloudInitFileContent"] = func(content string) string { + return fmt.Sprintf("%d %d", posixCksum(content), len(content)) + } + rendered, err := InitializeTemplateGenerator().getSingleLine(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, funcs, true) + if err != nil { + return "", err + } + var cloud cloudInit + if err := yaml.Unmarshal([]byte(rendered), &cloud); err != nil { + return "", fmt.Errorf("failed to parse cloud-init file checksums: %w", err) + } + if len(cloud.WriteFiles) == 0 { + return "", fmt.Errorf("cloud-init file checksums are empty") + } + checksums := make([]string, 0, len(cloud.WriteFiles)) + for _, file := range cloud.WriteFiles { + line := file.Content + " " + file.Path + if !cloudInitChecksumPattern.MatchString(line) { + return "", fmt.Errorf("invalid cloud-init checksum entry for %q", file.Path) + } + checksums = append(checksums, shellQuote(line)) + } + return strings.Join(checksums, " "), nil +} + // getTemplateFuncMap returns the general purpose template func map from getContainerServiceFuncMap. func getBakerFuncMap(config *datamodel.NodeBootstrappingConfiguration, params paramsMap, variables paramsMap) template.FuncMap { funcMap := getContainerServiceFuncMap(config) @@ -736,6 +793,9 @@ func getBakerFuncMap(config *datamodel.NodeBootstrappingConfiguration, params pa funcMap["GetCloudInitFileContent"] = func(content string) (string, error) { return getCloudInitFileContent(config, content) } + funcMap["GetCloudInitFileChecksums"] = func() (string, error) { + return getCloudInitFileChecksums(config) + } funcMap["GetParameter"] = func(s string) interface{} { if v, ok := params[s].(paramsMap); ok && v != nil { diff --git a/pkg/agent/cloud_init_encoding_test.go b/pkg/agent/cloud_init_encoding_test.go index cef7d2ffc9f..5c5410f1118 100644 --- a/pkg/agent/cloud_init_encoding_test.go +++ b/pkg/agent/cloud_init_encoding_test.go @@ -3,6 +3,9 @@ package agent import ( "encoding/base64" "fmt" + "os" + "os/exec" + "path/filepath" "strings" "testing" "text/template" @@ -13,6 +16,123 @@ import ( "gopkg.in/yaml.v3" ) +func TestPOSIXCksumMatchesSystem(t *testing.T) { + if _, err := exec.LookPath("cksum"); err != nil { + t.Skip("cksum is not installed") + } + for _, content := range []string{ + "", + "#!/bin/bash\nprintf 'hello\\n'\n", + "\x00\xff\xc3\xa9\r\n\u0085\u2028\u2029", + strings.Repeat("x", 255), + strings.Repeat("x", 256), + strings.Repeat("x", 65536), + } { + cmd := exec.Command("cksum") + cmd.Stdin = strings.NewReader(content) + output, err := cmd.Output() + require.NoError(t, err) + require.Equal(t, string(output), fmt.Sprintf("%d %d\n", posixCksum(content), len(content))) + } +} + +func TestCloudInitChecksumsMatchDeliveredFiles(t *testing.T) { + for _, distro := range []datamodel.Distro{ + datamodel.AKSUbuntuContainerd2204Gen2, + datamodel.AKSUbuntuContainerd2404Gen2, + datamodel.AKSAzureLinuxV2Gen2, + datamodel.AKSAzureLinuxV3Gen2, + datamodel.AKSAzureLinuxV3OSGuardGen2FIPSTL, + datamodel.CustomizedImage, + datamodel.CustomizedImageLinuxGuard, + datamodel.CustomizedImageTrustedLaunch, + } { + for _, custom := range []bool{false, true} { + for _, preProvision := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/custom=%t/preProvision=%t", distro, custom, preProvision), func(t *testing.T) { + config := newNodeCustomDataRenderConfig(distro) + config.PreProvisionOnly = preProvision + if custom { + config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{Name: "akscustom"} + } + generator := InitializeTemplateGenerator() + payload := generator.getLinuxNodeBootstrappingPayload(config) + compressed, err := base64.StdEncoding.DecodeString(payload) + require.NoError(t, err) + decoded, err := getGzipDecodedValue(compressed) + require.NoError(t, err) + var cloud cloudInit + require.NoError(t, yaml.Unmarshal(decoded, &cloud)) + var expected []string + for _, file := range cloud.WriteFiles { + expected = append(expected, shellQuote(fmt.Sprintf("%d %d %s", posixCksum(file.Content), len(file.Content), file.Path))) + } + checksums, err := getCloudInitFileChecksums(config) + require.NoError(t, err) + require.NotEmpty(t, checksums) + require.Equal(t, strings.Join(expected, " "), checksums) + assertCloudInitChecksumCommand(t, generator.getLinuxNodeCSECommand(config), checksums, cloud) + }) + } + } + } +} + +func assertCloudInitChecksumCommand(t *testing.T, command, checksums string, cloud cloudInit) { + t.Helper() + require.Contains(t, command, "for checksum in "+checksums+"; do") + require.Less(t, strings.Index(command, "for checksum in "), strings.Index(command, "INIT_AKS_CLOUD_FILEPATH=")) + if _, err := exec.LookPath("sh"); err != nil { + t.Log("sh is not installed; skipping shell execution") + return + } + output, err := exec.Command("sh", "-n", "-c", command).CombinedOutput() + require.NoError(t, err, "%s", output) + if _, lookupErr := exec.LookPath("cksum"); lookupErr != nil { + t.Log("cksum is not installed; skipping checksum command execution") + return + } + _, check, found := strings.Cut(command, "for checksum in ") + require.True(t, found) + check, _, found = strings.Cut(check, "done;") + require.True(t, found) + check = "for checksum in " + check + "done;" + root := t.TempDir() + for _, file := range cloud.WriteFiles { + target := filepath.Join(root, strings.TrimPrefix(file.Path, "/")) + require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o700)) + require.NoError(t, os.WriteFile(target, []byte(file.Content), 0o600)) + check = strings.ReplaceAll(check, file.Path, target) + } + cmd := exec.Command("sh", "-c", check) + cmd.Env = append(os.Environ(), "PROVISION_OUTPUT="+filepath.Join(root, "provision.log")) + output, err = cmd.CombinedOutput() + require.NoError(t, err, "%s", output) + require.Empty(t, output) +} + +func TestCloudInitChecksumsSkipOtherModes(t *testing.T) { + for _, mode := range []string{"disabled", "scriptless", "flatcar", "acl"} { + t.Run(mode, func(t *testing.T) { + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) + switch mode { + case "disabled": + config.DisableCustomData = true + case "scriptless": + config.EnableScriptlessCSECmd = true + case "flatcar": + config.AgentPoolProfile.Distro = datamodel.AKSFlatcarGen2 + case "acl": + config.AgentPoolProfile.Distro = datamodel.AKSACLGen2TL + } + checksums, err := getCloudInitFileChecksums(config) + require.NoError(t, err) + require.Empty(t, checksums) + require.NotContains(t, InitializeTemplateGenerator().getLinuxNodeCSECommand(config), "for checksum in ") + }) + } +} + func TestTraditionalCloudInitUsesSingleCompression(t *testing.T) { template, err := parts.Templates.ReadFile(kubernetesNodeCustomDataYaml) require.NoError(t, err) diff --git a/spec/parts/linux/cloud-init/artifacts/cse_cmd_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_cmd_spec.sh new file mode 100644 index 00000000000..6361bb79e62 --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/cse_cmd_spec.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env shellspec + +Describe 'CSE CustomData checksum verification' + setup() { + TEST_DIR="${SHELLSPEC_TMPBASE}/cse-checksums" + mkdir -p "${TEST_DIR}" + printf 'first\n' > "${TEST_DIR}/first" + printf 'second\n' > "${TEST_DIR}/second" + CHECKSUMS="'$(cksum "${TEST_DIR}/first")' '$(cksum "${TEST_DIR}/second")'" + PROVISION_OUTPUT="${TEST_DIR}/provision.log" + export PROVISION_OUTPUT + : > "${PROVISION_OUTPUT}" + } + + run_check() { + { + printf 'for checksum in %s; do\n' "${CHECKSUMS}" + sed -n '/^for checksum in /,/^done;/p' parts/linux/cloud-init/artifacts/cse_cmd.sh | sed '1d' + printf 'echo continued\n' + } > "${TEST_DIR}/check.sh" + sh "${TEST_DIR}/check.sh" + } + + BeforeEach 'setup' + + It 'continues when every file matches' + When run run_check + The status should be success + The output should eq 'continued' + The contents of file "${PROVISION_OUTPUT}" should eq '' + End + + It 'reports the changed file and stops even when its size is unchanged' + printf 'seconx\n' > "${TEST_DIR}/second" + When run run_check + The status should eq 1 + The output should include "CustomData checksum verification failed: ${TEST_DIR}/second" + The output should not include 'continued' + The contents of file "${PROVISION_OUTPUT}" should include "${TEST_DIR}/second" + End + + It 'reports a missing file and stops' + mv "${TEST_DIR}/first" "${TEST_DIR}/missing" + When run run_check + The status should eq 1 + The output should include "CustomData checksum verification failed: ${TEST_DIR}/first" + The output should not include 'continued' + The contents of file "${PROVISION_OUTPUT}" should include "${TEST_DIR}/first" + End +End From f2750481377945103cb2138576f004b5b5cbbdd7 Mon Sep 17 00:00:00 2001 From: sinmentis Date: Thu, 29 Oct 2026 10:41:46 +1300 Subject: [PATCH 3/4] fix: revert per-file CRC verification Revert 89864be9ec8f051f84ecc71d3245a10584c17a9f. The checksum gate can reject valid files when the CSE command and delivered scripts differ across provisioning attempts. Keep the CustomData compression improvement. AB#40305782 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 162f68f8-110a-4d60-b4fe-b9fb42253542 --- parts/linux/cloud-init/artifacts/cse_cmd.sh | 9 -- pkg/agent/baker.go | 60 --------- pkg/agent/cloud_init_encoding_test.go | 120 ------------------ .../cloud-init/artifacts/cse_cmd_spec.sh | 50 -------- 4 files changed, 239 deletions(-) delete mode 100644 spec/parts/linux/cloud-init/artifacts/cse_cmd_spec.sh diff --git a/parts/linux/cloud-init/artifacts/cse_cmd.sh b/parts/linux/cloud-init/artifacts/cse_cmd.sh index a9cb450cda6..a08d3b35d6d 100644 --- a/parts/linux/cloud-init/artifacts/cse_cmd.sh +++ b/parts/linux/cloud-init/artifacts/cse_cmd.sh @@ -15,15 +15,6 @@ else echo "cloud-init failed with exit code ${cloudInitExitCode}" >> ${PROVISION_OUTPUT}; exit ${cloudInitExitCode}; fi; -{{with GetCloudInitFileChecksums -}} -for checksum in {{.}}; do - file="${checksum##* }"; - if ! actual=$(cksum -- "${file}" 2>> "${PROVISION_OUTPUT}") || [ "${actual}" != "${checksum}" ]; then - echo "CustomData checksum verification failed: ${file}" | tee -a "${PROVISION_OUTPUT}"; - exit 1; - fi; -done; -{{end -}} {{end}} INIT_AKS_CLOUD_FILEPATH="{{GetInitAKSCloudFilepath}}"; if [ -f "${INIT_AKS_CLOUD_FILEPATH}" ]; then diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index aab6571fe19..3789be41292 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -726,63 +726,6 @@ func getCloudInitFileContent(config *datamodel.NodeBootstrappingConfiguration, c return "!!binary |\n " + content, nil } -// POSIX cksum uses an MSB-first CRC-32 and includes the byte length. -func posixCksum(content string) uint32 { - var table [256]uint32 - for i := range table { - crc := uint32(i) << 24 - for bit := 0; bit < 8; bit++ { - if crc&0x80000000 != 0 { - crc = crc<<1 ^ 0x04c11db7 - } else { - crc <<= 1 - } - } - table[i] = crc - } - var crc uint32 - for i := 0; i < len(content); i++ { - crc = crc<<8 ^ table[byte(crc>>24)^content[i]] - } - for size := len(content); size > 0; size >>= 8 { - crc = crc<<8 ^ table[byte(crc>>24)^byte(size)] - } - return ^crc -} - -var cloudInitChecksumPattern = regexp.MustCompile(`^[0-9]+ [0-9]+ /[A-Za-z0-9._/-]+$`) - -func getCloudInitFileChecksums(config *datamodel.NodeBootstrappingConfiguration) (string, error) { - if config.DisableCustomData || !usePlainCloudInit(config) { - return "", nil - } - funcs := getBakerFuncMap(config, getParameters(config), getCustomDataVariables(config)) - // Render only source checksums, so YAML never processes the bytes being checked. - funcs["GetCloudInitFileContent"] = func(content string) string { - return fmt.Sprintf("%d %d", posixCksum(content), len(content)) - } - rendered, err := InitializeTemplateGenerator().getSingleLine(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, funcs, true) - if err != nil { - return "", err - } - var cloud cloudInit - if err := yaml.Unmarshal([]byte(rendered), &cloud); err != nil { - return "", fmt.Errorf("failed to parse cloud-init file checksums: %w", err) - } - if len(cloud.WriteFiles) == 0 { - return "", fmt.Errorf("cloud-init file checksums are empty") - } - checksums := make([]string, 0, len(cloud.WriteFiles)) - for _, file := range cloud.WriteFiles { - line := file.Content + " " + file.Path - if !cloudInitChecksumPattern.MatchString(line) { - return "", fmt.Errorf("invalid cloud-init checksum entry for %q", file.Path) - } - checksums = append(checksums, shellQuote(line)) - } - return strings.Join(checksums, " "), nil -} - // getTemplateFuncMap returns the general purpose template func map from getContainerServiceFuncMap. func getBakerFuncMap(config *datamodel.NodeBootstrappingConfiguration, params paramsMap, variables paramsMap) template.FuncMap { funcMap := getContainerServiceFuncMap(config) @@ -793,9 +736,6 @@ func getBakerFuncMap(config *datamodel.NodeBootstrappingConfiguration, params pa funcMap["GetCloudInitFileContent"] = func(content string) (string, error) { return getCloudInitFileContent(config, content) } - funcMap["GetCloudInitFileChecksums"] = func() (string, error) { - return getCloudInitFileChecksums(config) - } funcMap["GetParameter"] = func(s string) interface{} { if v, ok := params[s].(paramsMap); ok && v != nil { diff --git a/pkg/agent/cloud_init_encoding_test.go b/pkg/agent/cloud_init_encoding_test.go index 5c5410f1118..cef7d2ffc9f 100644 --- a/pkg/agent/cloud_init_encoding_test.go +++ b/pkg/agent/cloud_init_encoding_test.go @@ -3,9 +3,6 @@ package agent import ( "encoding/base64" "fmt" - "os" - "os/exec" - "path/filepath" "strings" "testing" "text/template" @@ -16,123 +13,6 @@ import ( "gopkg.in/yaml.v3" ) -func TestPOSIXCksumMatchesSystem(t *testing.T) { - if _, err := exec.LookPath("cksum"); err != nil { - t.Skip("cksum is not installed") - } - for _, content := range []string{ - "", - "#!/bin/bash\nprintf 'hello\\n'\n", - "\x00\xff\xc3\xa9\r\n\u0085\u2028\u2029", - strings.Repeat("x", 255), - strings.Repeat("x", 256), - strings.Repeat("x", 65536), - } { - cmd := exec.Command("cksum") - cmd.Stdin = strings.NewReader(content) - output, err := cmd.Output() - require.NoError(t, err) - require.Equal(t, string(output), fmt.Sprintf("%d %d\n", posixCksum(content), len(content))) - } -} - -func TestCloudInitChecksumsMatchDeliveredFiles(t *testing.T) { - for _, distro := range []datamodel.Distro{ - datamodel.AKSUbuntuContainerd2204Gen2, - datamodel.AKSUbuntuContainerd2404Gen2, - datamodel.AKSAzureLinuxV2Gen2, - datamodel.AKSAzureLinuxV3Gen2, - datamodel.AKSAzureLinuxV3OSGuardGen2FIPSTL, - datamodel.CustomizedImage, - datamodel.CustomizedImageLinuxGuard, - datamodel.CustomizedImageTrustedLaunch, - } { - for _, custom := range []bool{false, true} { - for _, preProvision := range []bool{false, true} { - t.Run(fmt.Sprintf("%s/custom=%t/preProvision=%t", distro, custom, preProvision), func(t *testing.T) { - config := newNodeCustomDataRenderConfig(distro) - config.PreProvisionOnly = preProvision - if custom { - config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{Name: "akscustom"} - } - generator := InitializeTemplateGenerator() - payload := generator.getLinuxNodeBootstrappingPayload(config) - compressed, err := base64.StdEncoding.DecodeString(payload) - require.NoError(t, err) - decoded, err := getGzipDecodedValue(compressed) - require.NoError(t, err) - var cloud cloudInit - require.NoError(t, yaml.Unmarshal(decoded, &cloud)) - var expected []string - for _, file := range cloud.WriteFiles { - expected = append(expected, shellQuote(fmt.Sprintf("%d %d %s", posixCksum(file.Content), len(file.Content), file.Path))) - } - checksums, err := getCloudInitFileChecksums(config) - require.NoError(t, err) - require.NotEmpty(t, checksums) - require.Equal(t, strings.Join(expected, " "), checksums) - assertCloudInitChecksumCommand(t, generator.getLinuxNodeCSECommand(config), checksums, cloud) - }) - } - } - } -} - -func assertCloudInitChecksumCommand(t *testing.T, command, checksums string, cloud cloudInit) { - t.Helper() - require.Contains(t, command, "for checksum in "+checksums+"; do") - require.Less(t, strings.Index(command, "for checksum in "), strings.Index(command, "INIT_AKS_CLOUD_FILEPATH=")) - if _, err := exec.LookPath("sh"); err != nil { - t.Log("sh is not installed; skipping shell execution") - return - } - output, err := exec.Command("sh", "-n", "-c", command).CombinedOutput() - require.NoError(t, err, "%s", output) - if _, lookupErr := exec.LookPath("cksum"); lookupErr != nil { - t.Log("cksum is not installed; skipping checksum command execution") - return - } - _, check, found := strings.Cut(command, "for checksum in ") - require.True(t, found) - check, _, found = strings.Cut(check, "done;") - require.True(t, found) - check = "for checksum in " + check + "done;" - root := t.TempDir() - for _, file := range cloud.WriteFiles { - target := filepath.Join(root, strings.TrimPrefix(file.Path, "/")) - require.NoError(t, os.MkdirAll(filepath.Dir(target), 0o700)) - require.NoError(t, os.WriteFile(target, []byte(file.Content), 0o600)) - check = strings.ReplaceAll(check, file.Path, target) - } - cmd := exec.Command("sh", "-c", check) - cmd.Env = append(os.Environ(), "PROVISION_OUTPUT="+filepath.Join(root, "provision.log")) - output, err = cmd.CombinedOutput() - require.NoError(t, err, "%s", output) - require.Empty(t, output) -} - -func TestCloudInitChecksumsSkipOtherModes(t *testing.T) { - for _, mode := range []string{"disabled", "scriptless", "flatcar", "acl"} { - t.Run(mode, func(t *testing.T) { - config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) - switch mode { - case "disabled": - config.DisableCustomData = true - case "scriptless": - config.EnableScriptlessCSECmd = true - case "flatcar": - config.AgentPoolProfile.Distro = datamodel.AKSFlatcarGen2 - case "acl": - config.AgentPoolProfile.Distro = datamodel.AKSACLGen2TL - } - checksums, err := getCloudInitFileChecksums(config) - require.NoError(t, err) - require.Empty(t, checksums) - require.NotContains(t, InitializeTemplateGenerator().getLinuxNodeCSECommand(config), "for checksum in ") - }) - } -} - func TestTraditionalCloudInitUsesSingleCompression(t *testing.T) { template, err := parts.Templates.ReadFile(kubernetesNodeCustomDataYaml) require.NoError(t, err) diff --git a/spec/parts/linux/cloud-init/artifacts/cse_cmd_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_cmd_spec.sh deleted file mode 100644 index 6361bb79e62..00000000000 --- a/spec/parts/linux/cloud-init/artifacts/cse_cmd_spec.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env shellspec - -Describe 'CSE CustomData checksum verification' - setup() { - TEST_DIR="${SHELLSPEC_TMPBASE}/cse-checksums" - mkdir -p "${TEST_DIR}" - printf 'first\n' > "${TEST_DIR}/first" - printf 'second\n' > "${TEST_DIR}/second" - CHECKSUMS="'$(cksum "${TEST_DIR}/first")' '$(cksum "${TEST_DIR}/second")'" - PROVISION_OUTPUT="${TEST_DIR}/provision.log" - export PROVISION_OUTPUT - : > "${PROVISION_OUTPUT}" - } - - run_check() { - { - printf 'for checksum in %s; do\n' "${CHECKSUMS}" - sed -n '/^for checksum in /,/^done;/p' parts/linux/cloud-init/artifacts/cse_cmd.sh | sed '1d' - printf 'echo continued\n' - } > "${TEST_DIR}/check.sh" - sh "${TEST_DIR}/check.sh" - } - - BeforeEach 'setup' - - It 'continues when every file matches' - When run run_check - The status should be success - The output should eq 'continued' - The contents of file "${PROVISION_OUTPUT}" should eq '' - End - - It 'reports the changed file and stops even when its size is unchanged' - printf 'seconx\n' > "${TEST_DIR}/second" - When run run_check - The status should eq 1 - The output should include "CustomData checksum verification failed: ${TEST_DIR}/second" - The output should not include 'continued' - The contents of file "${PROVISION_OUTPUT}" should include "${TEST_DIR}/second" - End - - It 'reports a missing file and stops' - mv "${TEST_DIR}/first" "${TEST_DIR}/missing" - When run run_check - The status should eq 1 - The output should include "CustomData checksum verification failed: ${TEST_DIR}/first" - The output should not include 'continued' - The contents of file "${PROVISION_OUTPUT}" should include "${TEST_DIR}/first" - End -End From 61f82bf8331e3bc59660e448e8a14f6c627fec56 Mon Sep 17 00:00:00 2001 From: sinmentis Date: Thu, 29 Oct 2026 11:02:40 +1300 Subject: [PATCH 4/4] refactor: simplify Linux custom data rendering Keep rendered file contents unencoded until template output and return Linux YAML directly without a JSON round trip. AB#40305782 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 162f68f8-110a-4d60-b4fe-b9fb42253542 --- pkg/agent/baker.go | 18 ++-- pkg/agent/baker_test.go | 12 +-- pkg/agent/cloud_init_encoding_test.go | 26 +++++- pkg/agent/utils_test.go | 8 +- pkg/agent/variables.go | 116 ++++++++++++-------------- 5 files changed, 92 insertions(+), 88 deletions(-) diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 3789be41292..b01d4553f7b 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -165,14 +165,12 @@ func (t *TemplateGenerator) getLinuxNodeBootstrappingPayload(config *datamodel.N return t.getScriptlessBoothook(config) } - // this might seem strange that we're encoding the custom data to a JSON string and then extracting it, but without that serialisation and deserialisation - // lots of tests fail. var encoded string if config.IsFlatcar() || config.IsACL() { customData := getCustomDataFromJSON(t.getFlatcarLinuxNodeCustomDataJSONObject(config)) encoded = base64.StdEncoding.EncodeToString([]byte(customData)) } else { - customData := getCustomDataFromJSON(t.getLinuxNodeCustomDataJSONObject(config)) + customData := t.getLinuxNodeCustomData(config) encoded = getBase64EncodedGzippedCustomScriptFromStr(customData) } return encoded @@ -189,7 +187,7 @@ type encodedFile struct { func (t *TemplateGenerator) getScriptlessBoothook(config *datamodel.NodeBootstrappingConfiguration) string { config.DisableCustomData = true config.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(t.getLinuxNodeCustomDataJSONObject(config)) + nodeCustomData := t.getLinuxNodeCustomData(config) encodedNodeCustomData := getBase64EncodedGzippedCustomScriptFromStr(nodeCustomData) // hotfixJSONFile is optional: only VHDs that bake a static default hotfix @@ -344,9 +342,7 @@ func buildScriptlessCustomData(cloudInitTemplate, fileListTemplate, separator st return fmt.Sprintf(cloudInitTemplate, strings.Join(fileList, separator)) } -// GetLinuxNodeCustomDataJSONObject returns Linux customData JSON object in the form. -// { "customData": "" }. -func (t *TemplateGenerator) getLinuxNodeCustomDataJSONObject(config *datamodel.NodeBootstrappingConfiguration) string { +func (t *TemplateGenerator) getLinuxNodeCustomData(config *datamodel.NodeBootstrappingConfiguration) string { b, err := parts.Templates.ReadFile(kubernetesNodeCustomDataYaml) if err != nil { panic(err) @@ -356,11 +352,7 @@ func (t *TemplateGenerator) getLinuxNodeCustomDataJSONObject(config *datamodel.N panic(err) } - customData, err := json.Marshal(map[string]string{"customData": str}) - if err != nil { - panic(err) - } - return string(customData) + return str } const ( @@ -723,7 +715,7 @@ func getCloudInitFileContent(config *datamodel.NodeBootstrappingConfiguration, c }) return strings.TrimSuffix(string(encoded), "\n"), err } - return "!!binary |\n " + content, nil + return "!!binary |\n " + getBase64EncodedGzippedCustomScriptFromStr(content), nil } // getTemplateFuncMap returns the general purpose template func map from getContainerServiceFuncMap. diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 746ac9c1eed..9b7acc68439 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -1320,7 +1320,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) encodedNodeCustomData := getBase64EncodedGzippedCustomScriptFromStr(nodeCustomData) Expect(string(decodedPayload)).To(ContainSubstring(aksNodeCustomDataFilepath)) @@ -1534,7 +1534,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) Expect(nodeCustomData).To(ContainSubstring(initAKSCloudFilepath)) Expect(nodeCustomData).To(ContainSubstring("permissions: \"0744\"")) @@ -1551,7 +1551,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) Expect(nodeCustomData).To(ContainSubstring(initAKSCloudFilepath)) Expect(nodeCustomData).To(ContainSubstring("permissions: \"0744\"")) @@ -1568,7 +1568,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) Expect(nodeCustomData).To(ContainSubstring(initAKSCloudFilepath)) Expect(nodeCustomData).To(ContainSubstring("permissions: \"0744\"")) @@ -1586,7 +1586,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) Expect(nodeCustomData).To(ContainSubstring(initAKSCloudFilepath)) Expect(nodeCustomData).To(ContainSubstring("permissions: \"0744\"")) @@ -1604,7 +1604,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { decompressedPayload, err := getGzipDecodedValue(decodedPayload) Expect(err).NotTo(HaveOccurred()) - expectedCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(config)) + expectedCustomData := templateGenerator.getLinuxNodeCustomData(config) Expect(string(decompressedPayload)).To(Equal(expectedCustomData)) // Script contents can reference these paths; only delivery as write_files diff --git a/pkg/agent/cloud_init_encoding_test.go b/pkg/agent/cloud_init_encoding_test.go index cef7d2ffc9f..310213cb683 100644 --- a/pkg/agent/cloud_init_encoding_test.go +++ b/pkg/agent/cloud_init_encoding_test.go @@ -91,16 +91,18 @@ func TestCloudInitEncodingLeavesOtherModesUnchanged(t *testing.T) { rendered, err := RenderLinuxNodeCustomDataTemplate(template, config) require.NoError(t, err) require.Equal(t, original, rendered) - require.Equal(t, original, getCustomDataFromJSON(generator.getLinuxNodeCustomDataJSONObject(config))) + require.Equal(t, original, generator.getLinuxNodeCustomData(config)) }) } } } func legacyCloudInitFuncMap(config *datamodel.NodeBootstrappingConfiguration) template.FuncMap { - funcs := getBakerFuncMap(config, getParameters(config), getCustomDataVariablesWithEncoder(config, false)) + funcs := getBakerFuncMap(config, getParameters(config), getCustomDataVariables(config)) funcs["GetCloudInitFileEncodingLine"] = func() string { return "encoding: " + encodingGZIP } - funcs["GetCloudInitFileContent"] = func(content string) string { return "!!binary |\n " + content } + funcs["GetCloudInitFileContent"] = func(content string) string { + return "!!binary |\n " + getBase64EncodedGzippedCustomScriptFromStr(content) + } return funcs } @@ -149,6 +151,24 @@ func TestPlainCloudInitFileContentRejectsInvalidUTF8(t *testing.T) { require.Empty(t, output) } +func TestEncodedCloudInitFileContentUsesRawBytes(t *testing.T) { + for _, distro := range []datamodel.Distro{datamodel.AKSUbuntuContainerd2204Gen2, datamodel.AKSFlatcarGen2, datamodel.AKSACLGen2TL} { + for i, content := range []string{"", "#!/bin/bash\nprintf 'hello\\n'\n", "\x00\xff\r\n\u0085\u2028\u2029"} { + t.Run(fmt.Sprintf("%s/%d", distro, i), func(t *testing.T) { + config := newNodeCustomDataRenderConfig(distro) + config.EnableScriptlessCSECmd = true + encoded, err := getCloudInitFileContent(config, content) + require.NoError(t, err) + var file cloudInitWriteFile + require.NoError(t, yaml.Unmarshal([]byte("content: "+encoded+"\n"), &file)) + decoded, err := getGzipDecodedValue([]byte(file.Content)) + require.NoError(t, err) + require.Equal(t, []byte(content), decoded) + }) + } + } +} + func BenchmarkTraditionalCloudInitEncoding(b *testing.B) { for _, old := range []bool{true, false} { b.Run(fmt.Sprintf("legacy=%t", old), func(b *testing.B) { diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index e442ced785a..ca46f7d69b4 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -1362,21 +1362,21 @@ func TestCSEScriptRoundTrip(t *testing.T) { } // discoverCSEScripts parses variables.go to find all constant names passed to -// encodeScript(), then resolves those constants to file +// getCustomScript(), then resolves those constants to file // paths from const.go, filtering to .sh files only. func discoverCSEScripts(t *testing.T) []string { t.Helper() root := repoRoot() - // Step 1: Read variables.go and extract constant names from encodeScript() calls + // Step 1: Read variables.go and extract constant names from getCustomScript() calls variablesPath := filepath.Join(root, "pkg", "agent", "variables.go") variablesBytes, err := os.ReadFile(variablesPath) if err != nil { t.Fatalf("failed to read variables.go: %v", err) } - // Match: encodeScript(constantName, config) - callRe := regexp.MustCompile(`encodeScript\((\w+),`) + // Match: getCustomScript(constantName, config) + callRe := regexp.MustCompile(`getCustomScript\((\w+),`) matches := callRe.FindAllStringSubmatch(string(variablesBytes), -1) constNames := make(map[string]bool) for _, m := range matches { diff --git a/pkg/agent/variables.go b/pkg/agent/variables.go index c0e254b57a0..ed282bcaa09 100644 --- a/pkg/agent/variables.go +++ b/pkg/agent/variables.go @@ -10,82 +10,74 @@ import ( "github.com/Azure/agentbaker/pkg/agent/datamodel" ) -// getCustomDataVariables returns cloudinit data used by Linux. +// getCustomDataVariables returns rendered, unencoded file contents used by Linux. func getCustomDataVariables(config *datamodel.NodeBootstrappingConfiguration) paramsMap { - return getCustomDataVariablesWithEncoder(config, usePlainCloudInit(config)) -} - -func getCustomDataVariablesWithEncoder(config *datamodel.NodeBootstrappingConfiguration, plain bool) paramsMap { - encodeScript := getBase64EncodedGzippedCustomScript - if plain { - encodeScript = getCustomScript - } cs := config.ContainerService cloudInitFiles := map[string]interface{}{ "cloudInitData": paramsMap{ - "provisionStartScript": encodeScript(kubernetesCSEStartScript, config), - "provisionScript": encodeScript(kubernetesCSEMainScript, config), - "provisionSource": encodeScript(kubernetesCSEHelpersScript, config), - "provisionSourceUbuntu": encodeScript(kubernetesCSEHelpersScriptUbuntu, config), - "provisionSourceMariner": encodeScript(kubernetesCSEHelpersScriptMariner, config), - "provisionSourceAzlOSGuard": encodeScript(kubernetesCSEHelpersScriptAzlOSGuard, config), - "provisionSourceFlatcar": encodeScript(kubernetesCSEHelpersScriptFlatcar, config), - "provisionSourceACL": encodeScript(kubernetesCSEHelpersScriptACL, config), - "provisionInstalls": encodeScript(kubernetesCSEInstall, config), - "provisionInstallsUbuntu": encodeScript(kubernetesCSEInstallUbuntu, config), - "provisionInstallsMariner": encodeScript(kubernetesCSEInstallMariner, config), - "provisionInstallsAzlOSGuard": encodeScript(kubernetesCSEInstallAzlOSGuard, config), - "provisionInstallsFlatcar": encodeScript(kubernetesCSEInstallFlatcar, config), - "provisionInstallsACL": encodeScript(kubernetesCSEInstallACL, config), - "provisionConfigs": encodeScript(kubernetesCSEConfig, config), - "provisionConfigsGPU": encodeScript(kubernetesCSEConfigGPU, config), - "provisionConfigsLocalDNS": encodeScript(kubernetesCSEConfigLocalDNS, config), - "provisionConfigsKubelet": encodeScript(kubernetesCSEConfigKubelet, config), - "provisionConfigsNetwork": encodeScript(kubernetesCSEConfigNetwork, config), - "provisionConfigsChrony": encodeScript(kubernetesCSEConfigChrony, config), - "provisionSendLogs": encodeScript(kubernetesCSESendLogs, config), - "provisionRedactCloudConfig": encodeScript(kubernetesCSERedactCloudConfig, config), - "customSearchDomainsScript": encodeScript(kubernetesCustomSearchDomainsScript, config), - "dhcpv6SystemdService": encodeScript(dhcpv6SystemdService, config), - "dhcpv6ConfigurationScript": encodeScript(dhcpv6ConfigurationScript, config), - "kubeletSystemdService": encodeScript(kubeletSystemdService, config), - "reconcilePrivateHostsScript": encodeScript(reconcilePrivateHostsScript, config), - "reconcilePrivateHostsService": encodeScript(reconcilePrivateHostsService, config), - "ensureNoDupEbtablesScript": encodeScript(ensureNoDupEbtablesScript, config), - "ensureNoDupEbtablesService": encodeScript(ensureNoDupEbtablesService, config), - "bindMountScript": encodeScript(bindMountScript, config), - "bindMountSystemdService": encodeScript(bindMountSystemdService, config), - "migPartitionSystemdService": encodeScript(migPartitionSystemdService, config), - "migPartitionScript": encodeScript(migPartitionScript, config), - "ensureIMDSRestrictionScript": encodeScript(ensureIMDSRestrictionScript, config), - "snapshotUpdateScript": encodeScript(snapshotUpdateScript, config), - "snapshotUpdateService": encodeScript(snapshotUpdateSystemdService, config), - "snapshotUpdateTimer": encodeScript(snapshotUpdateSystemdTimer, config), - "packageUpdateScriptMariner": encodeScript(packageUpdateScriptMariner, config), - "packageUpdateServiceMariner": encodeScript(packageUpdateSystemdServiceMariner, config), - "packageUpdateTimerMariner": encodeScript(packageUpdateSystemdTimerMariner, config), - "componentManifestFile": encodeScript(componentManifestFile, config), - "validateKubeletCredentialsScript": encodeScript(validateKubeletCredentialsScript, config), - "secureTLSBootstrapService": encodeScript(secureTLSBootstrapService, config), - "cloudInitStatusCheckScript": encodeScript(cloudInitStatusCheckScript, config), - "measureTLSBootstrappingLatencyScript": encodeScript(measureTLSBootstrappingLatencyScript, config), - "measureTLSBootstrappingLatencyService": encodeScript(measureTLSBootstrappingLatencyService, config), - "configureAzureNetworkScript": encodeScript(configureAzureNetworkScript, config), - "azureNetworkUdevRule": encodeScript(azureNetworkUdevRule, config), + "provisionStartScript": getCustomScript(kubernetesCSEStartScript, config), + "provisionScript": getCustomScript(kubernetesCSEMainScript, config), + "provisionSource": getCustomScript(kubernetesCSEHelpersScript, config), + "provisionSourceUbuntu": getCustomScript(kubernetesCSEHelpersScriptUbuntu, config), + "provisionSourceMariner": getCustomScript(kubernetesCSEHelpersScriptMariner, config), + "provisionSourceAzlOSGuard": getCustomScript(kubernetesCSEHelpersScriptAzlOSGuard, config), + "provisionSourceFlatcar": getCustomScript(kubernetesCSEHelpersScriptFlatcar, config), + "provisionSourceACL": getCustomScript(kubernetesCSEHelpersScriptACL, config), + "provisionInstalls": getCustomScript(kubernetesCSEInstall, config), + "provisionInstallsUbuntu": getCustomScript(kubernetesCSEInstallUbuntu, config), + "provisionInstallsMariner": getCustomScript(kubernetesCSEInstallMariner, config), + "provisionInstallsAzlOSGuard": getCustomScript(kubernetesCSEInstallAzlOSGuard, config), + "provisionInstallsFlatcar": getCustomScript(kubernetesCSEInstallFlatcar, config), + "provisionInstallsACL": getCustomScript(kubernetesCSEInstallACL, config), + "provisionConfigs": getCustomScript(kubernetesCSEConfig, config), + "provisionConfigsGPU": getCustomScript(kubernetesCSEConfigGPU, config), + "provisionConfigsLocalDNS": getCustomScript(kubernetesCSEConfigLocalDNS, config), + "provisionConfigsKubelet": getCustomScript(kubernetesCSEConfigKubelet, config), + "provisionConfigsNetwork": getCustomScript(kubernetesCSEConfigNetwork, config), + "provisionConfigsChrony": getCustomScript(kubernetesCSEConfigChrony, config), + "provisionSendLogs": getCustomScript(kubernetesCSESendLogs, config), + "provisionRedactCloudConfig": getCustomScript(kubernetesCSERedactCloudConfig, config), + "customSearchDomainsScript": getCustomScript(kubernetesCustomSearchDomainsScript, config), + "dhcpv6SystemdService": getCustomScript(dhcpv6SystemdService, config), + "dhcpv6ConfigurationScript": getCustomScript(dhcpv6ConfigurationScript, config), + "kubeletSystemdService": getCustomScript(kubeletSystemdService, config), + "reconcilePrivateHostsScript": getCustomScript(reconcilePrivateHostsScript, config), + "reconcilePrivateHostsService": getCustomScript(reconcilePrivateHostsService, config), + "ensureNoDupEbtablesScript": getCustomScript(ensureNoDupEbtablesScript, config), + "ensureNoDupEbtablesService": getCustomScript(ensureNoDupEbtablesService, config), + "bindMountScript": getCustomScript(bindMountScript, config), + "bindMountSystemdService": getCustomScript(bindMountSystemdService, config), + "migPartitionSystemdService": getCustomScript(migPartitionSystemdService, config), + "migPartitionScript": getCustomScript(migPartitionScript, config), + "ensureIMDSRestrictionScript": getCustomScript(ensureIMDSRestrictionScript, config), + "snapshotUpdateScript": getCustomScript(snapshotUpdateScript, config), + "snapshotUpdateService": getCustomScript(snapshotUpdateSystemdService, config), + "snapshotUpdateTimer": getCustomScript(snapshotUpdateSystemdTimer, config), + "packageUpdateScriptMariner": getCustomScript(packageUpdateScriptMariner, config), + "packageUpdateServiceMariner": getCustomScript(packageUpdateSystemdServiceMariner, config), + "packageUpdateTimerMariner": getCustomScript(packageUpdateSystemdTimerMariner, config), + "componentManifestFile": getCustomScript(componentManifestFile, config), + "validateKubeletCredentialsScript": getCustomScript(validateKubeletCredentialsScript, config), + "secureTLSBootstrapService": getCustomScript(secureTLSBootstrapService, config), + "cloudInitStatusCheckScript": getCustomScript(cloudInitStatusCheckScript, config), + "measureTLSBootstrappingLatencyScript": getCustomScript(measureTLSBootstrappingLatencyScript, config), + "measureTLSBootstrappingLatencyService": getCustomScript(measureTLSBootstrappingLatencyService, config), + "configureAzureNetworkScript": getCustomScript(configureAzureNetworkScript, config), + "azureNetworkUdevRule": getCustomScript(azureNetworkUdevRule, config), }, } cloudInitData := cloudInitFiles["cloudInitData"].(paramsMap) //nolint:errcheck // no error is actually here - cloudInitData["initAKSCloud"] = encodeScript(initAKSCloudScript, config) + cloudInitData["initAKSCloud"] = getCustomScript(initAKSCloudScript, config) if config.IsFlatcar() || config.IsACL() { cloudInitData["provisionRedactCloudConfig"] = "" // Flatcar and ACL do not have cloud-init } if !cs.Properties.IsVHDDistroForAllNodes() { - cloudInitData["kmsSystemdService"] = encodeScript(kmsSystemdService, config) - cloudInitData["aptPreferences"] = encodeScript(aptPreferences, config) - cloudInitData["dockerClearMountPropagationFlags"] = encodeScript(dockerClearMountPropagationFlags, config) + cloudInitData["kmsSystemdService"] = getCustomScript(kmsSystemdService, config) + cloudInitData["aptPreferences"] = getCustomScript(aptPreferences, config) + cloudInitData["dockerClearMountPropagationFlags"] = getCustomScript(dockerClearMountPropagationFlags, config) } return cloudInitFiles