diff --git a/hotfix/hotfix_generate_test.py b/hotfix/hotfix_generate_test.py index 7626a0863c7..b321195994b 100644 --- a/hotfix/hotfix_generate_test.py +++ b/hotfix/hotfix_generate_test.py @@ -161,7 +161,8 @@ def test_config_modules_are_independently_hotfixable(self): self.assertEqual({variable}, blocks[0][0]) self.assertIn(f"- path: {{{{{path_function}}}}}", template) self.assertIn('permissions: "0744"', template) - self.assertIn("encoding: gzip", template) + self.assertIn("{{GetCloudInitFileEncodingLine}}", template) + self.assertIn("| GetCloudInitFileContent}}", template) self.assertNotIn("{{if", template) def test_mariner_package_update_is_not_embedded_in_custom_data(self): diff --git a/hotfix/render-nodecustomdata/main_test.go b/hotfix/render-nodecustomdata/main_test.go index d490001e6c4..55102a76c80 100644 --- a/hotfix/render-nodecustomdata/main_test.go +++ b/hotfix/render-nodecustomdata/main_test.go @@ -18,18 +18,16 @@ func TestRenderSelectedHotfixScripts(t *testing.T) { const initCloudBlock = `{{if IsAKSCustomCloud}} - path: {{GetInitAKSCloudFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "initAKSCloud"}} + content: {{GetVariableProperty "cloudInitData" "initAKSCloud" | GetCloudInitFileContent}} {{end}} ` const helperBlock = `- path: {{GetCSEHelpersScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSource"}} + content: {{GetVariableProperty "cloudInitData" "provisionSource" | GetCloudInitFileContent}} ` for _, distro := range []datamodel.Distro{ datamodel.AKSUbuntuContainerd2204Gen2, @@ -47,8 +45,9 @@ func TestRenderSelectedHotfixScripts(t *testing.T) { require.NoError(t, err) var payload struct { WriteFiles []struct { - Path string `yaml:"path"` - Content string `yaml:"content"` + Path string `yaml:"path"` + Encoding string `yaml:"encoding"` + Content string `yaml:"content"` } `yaml:"write_files"` } require.NoError(t, yaml.Unmarshal([]byte(rendered), &payload)) @@ -62,6 +61,7 @@ func TestRenderSelectedHotfixScripts(t *testing.T) { require.Len(t, payload.WriteFiles, 1+wantInitCount) for _, file := range payload.WriteFiles { require.NotEmpty(t, file.Content) + require.Empty(t, file.Encoding) } }) } diff --git a/parts/linux/cloud-init/nodecustomdata.yml b/parts/linux/cloud-init/nodecustomdata.yml index ece63879f98..6c59344a200 100644 --- a/parts/linux/cloud-init/nodecustomdata.yml +++ b/parts/linux/cloud-init/nodecustomdata.yml @@ -28,174 +28,151 @@ write_files: {{if IsAKSCustomCloud}} - path: {{GetInitAKSCloudFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "initAKSCloud"}} + content: {{GetVariableProperty "cloudInitData" "initAKSCloud" | GetCloudInitFileContent}} {{end}} {{- else }} - path: {{GetCSEHelpersScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSource"}} + content: {{GetVariableProperty "cloudInitData" "provisionSource" | GetCloudInitFileContent}} {{if IsACL }} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceACL"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceACL" | GetCloudInitFileContent}} {{- else if IsAzlOSGuard}} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceAzlOSGuard"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceAzlOSGuard" | GetCloudInitFileContent}} {{- else if IsMariner}} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceMariner"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceMariner" | GetCloudInitFileContent}} {{- else if IsFlatcar }} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceFlatcar"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceFlatcar" | GetCloudInitFileContent}} {{- else }} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceUbuntu"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceUbuntu" | GetCloudInitFileContent}} {{end}} {{ if not IsCustomImage -}} - path: /opt/azure/containers/provision_start.sh permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionStartScript"}} + content: {{GetVariableProperty "cloudInitData" "provisionStartScript" | GetCloudInitFileContent}} {{- end }} - path: /opt/azure/containers/provision.sh permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionScript"}} + content: {{GetVariableProperty "cloudInitData" "provisionScript" | GetCloudInitFileContent}} - path: {{GetCSEInstallScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstalls"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstalls" | GetCloudInitFileContent}} {{if IsACL }} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsACL"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsACL" | GetCloudInitFileContent}} {{- else if IsAzlOSGuard}} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsAzlOSGuard"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsAzlOSGuard" | GetCloudInitFileContent}} {{- else if IsMariner}} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsMariner"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsMariner" | GetCloudInitFileContent}} {{- else if IsFlatcar }} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsFlatcar"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsFlatcar" | GetCloudInitFileContent}} {{- else }} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsUbuntu"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsUbuntu" | GetCloudInitFileContent}} {{end}} - path: {{GetCSEConfigScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigs"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigs" | GetCloudInitFileContent}} - path: {{GetCSEConfigGPUScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsGPU"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsGPU" | GetCloudInitFileContent}} - path: {{GetCSEConfigLocalDNSScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsLocalDNS"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsLocalDNS" | GetCloudInitFileContent}} - path: {{GetCSEConfigKubeletScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsKubelet"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsKubelet" | GetCloudInitFileContent}} - path: {{GetCSEConfigNetworkScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsNetwork"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsNetwork" | GetCloudInitFileContent}} - path: {{GetCSEConfigChronyScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsChrony"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsChrony" | GetCloudInitFileContent}} {{if IsAKSCustomCloud}} - path: {{GetInitAKSCloudFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "initAKSCloud"}} + content: {{GetVariableProperty "cloudInitData" "initAKSCloud" | GetCloudInitFileContent}} {{end}} - path: /etc/systemd/system/kubelet.service permissions: "0600" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "kubeletSystemdService"}} + content: {{GetVariableProperty "cloudInitData" "kubeletSystemdService" | GetCloudInitFileContent}} {{- end }} diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 6a910755dfb..b01d4553f7b 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "text/template" + "unicode/utf8" "github.com/Azure/agentbaker/parts" "github.com/Azure/agentbaker/pkg/agent/datamodel" @@ -164,14 +165,12 @@ func (t *TemplateGenerator) getLinuxNodeBootstrappingPayload(config *datamodel.N return t.getScriptlessBoothook(config) } - // this might seem strange that we're encoding the custom data to a JSON string and then extracting it, but without that serialisation and deserialisation - // lots of tests fail. var encoded string if config.IsFlatcar() || config.IsACL() { customData := getCustomDataFromJSON(t.getFlatcarLinuxNodeCustomDataJSONObject(config)) encoded = base64.StdEncoding.EncodeToString([]byte(customData)) } else { - customData := getCustomDataFromJSON(t.getLinuxNodeCustomDataJSONObject(config)) + customData := t.getLinuxNodeCustomData(config) encoded = getBase64EncodedGzippedCustomScriptFromStr(customData) } return encoded @@ -188,7 +187,7 @@ type encodedFile struct { func (t *TemplateGenerator) getScriptlessBoothook(config *datamodel.NodeBootstrappingConfiguration) string { config.DisableCustomData = true config.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(t.getLinuxNodeCustomDataJSONObject(config)) + nodeCustomData := t.getLinuxNodeCustomData(config) encodedNodeCustomData := getBase64EncodedGzippedCustomScriptFromStr(nodeCustomData) // hotfixJSONFile is optional: only VHDs that bake a static default hotfix @@ -343,20 +342,17 @@ func buildScriptlessCustomData(cloudInitTemplate, fileListTemplate, separator st return fmt.Sprintf(cloudInitTemplate, strings.Join(fileList, separator)) } -// GetLinuxNodeCustomDataJSONObject returns Linux customData JSON object in the form. -// { "customData": "" }. -func (t *TemplateGenerator) getLinuxNodeCustomDataJSONObject(config *datamodel.NodeBootstrappingConfiguration) string { - // get parameters - parameters := getParameters(config) - // get variable cloudInit - variables := getCustomDataVariables(config) - str, e := t.getSingleLineForTemplate(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, getBakerFuncMap(config, parameters, variables), true) - - if e != nil { - panic(e) +func (t *TemplateGenerator) getLinuxNodeCustomData(config *datamodel.NodeBootstrappingConfiguration) string { + b, err := parts.Templates.ReadFile(kubernetesNodeCustomDataYaml) + if err != nil { + panic(err) + } + str, err := RenderLinuxNodeCustomDataTemplate(b, config) + if err != nil { + panic(err) } - return fmt.Sprintf("{\"customData\": \"%s\"}", str) + return str } const ( @@ -674,6 +670,8 @@ func (t *TemplateGenerator) getSingleLine(textFilename string, profile interface // same variables and functions used by the production AgentBaker path. // Callers must supply a complete configuration, as required by the production // rendering helpers. +// File entries must use GetCloudInitFileEncodingLine and GetCloudInitFileContent +// to select the encoding for the delivery mode. func RenderLinuxNodeCustomDataTemplate(templateContent []byte, config *datamodel.NodeBootstrappingConfiguration) (string, error) { parameters := getParameters(config) variables := getCustomDataVariables(config) @@ -691,10 +689,46 @@ func RenderLinuxNodeCustomDataTemplate(templateContent []byte, config *datamodel return buffer.String(), nil } +// Traditional custom data is gzipped as a whole, so its files need no inner encoding. +func usePlainCloudInit(config *datamodel.NodeBootstrappingConfiguration) bool { + return !config.EnableScriptlessCSECmd && !config.IsFlatcar() && !config.IsACL() +} + +func getCloudInitFileEncodingLine(config *datamodel.NodeBootstrappingConfiguration) string { + if usePlainCloudInit(config) { + return "" + } + return "encoding: " + encodingGZIP +} + +func getCloudInitFileContent(config *datamodel.NodeBootstrappingConfiguration, content string) (string, error) { + if usePlainCloudInit(config) { + if !utf8.ValidString(content) { + return "", fmt.Errorf("cloud-init file content contains invalid UTF-8") + } + // YAML quoting preserves Unicode line breaks that JSON leaves unescaped. + encoded, err := yaml.Marshal(&yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Style: yaml.DoubleQuotedStyle, + Value: content, + }) + return strings.TrimSuffix(string(encoded), "\n"), err + } + return "!!binary |\n " + getBase64EncodedGzippedCustomScriptFromStr(content), nil +} + // getTemplateFuncMap returns the general purpose template func map from getContainerServiceFuncMap. func getBakerFuncMap(config *datamodel.NodeBootstrappingConfiguration, params paramsMap, variables paramsMap) template.FuncMap { funcMap := getContainerServiceFuncMap(config) + funcMap["GetCloudInitFileEncodingLine"] = func() string { + return getCloudInitFileEncodingLine(config) + } + funcMap["GetCloudInitFileContent"] = func(content string) (string, error) { + return getCloudInitFileContent(config, content) + } + funcMap["GetParameter"] = func(s string) interface{} { if v, ok := params[s].(paramsMap); ok && v != nil { if v["value"] == nil { diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index ae71d3a0cf8..9b7acc68439 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/gomega" "github.com/stretchr/testify/require" "github.com/vincent-petithory/dataurl" + "gopkg.in/yaml.v3" ) // this regex looks for groups of the following forms, returning KEY and VALUE as submatches. @@ -1319,7 +1320,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) encodedNodeCustomData := getBase64EncodedGzippedCustomScriptFromStr(nodeCustomData) Expect(string(decodedPayload)).To(ContainSubstring(aksNodeCustomDataFilepath)) @@ -1533,7 +1534,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) Expect(nodeCustomData).To(ContainSubstring(initAKSCloudFilepath)) Expect(nodeCustomData).To(ContainSubstring("permissions: \"0744\"")) @@ -1550,7 +1551,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) Expect(nodeCustomData).To(ContainSubstring(initAKSCloudFilepath)) Expect(nodeCustomData).To(ContainSubstring("permissions: \"0744\"")) @@ -1567,7 +1568,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) Expect(nodeCustomData).To(ContainSubstring(initAKSCloudFilepath)) Expect(nodeCustomData).To(ContainSubstring("permissions: \"0744\"")) @@ -1585,7 +1586,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) Expect(nodeCustomData).To(ContainSubstring(initAKSCloudFilepath)) Expect(nodeCustomData).To(ContainSubstring("permissions: \"0744\"")) @@ -1603,11 +1604,17 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { decompressedPayload, err := getGzipDecodedValue(decodedPayload) Expect(err).NotTo(HaveOccurred()) - expectedCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(config)) + expectedCustomData := templateGenerator.getLinuxNodeCustomData(config) Expect(string(decompressedPayload)).To(Equal(expectedCustomData)) - Expect(string(decompressedPayload)).NotTo(ContainSubstring(aksNodeCustomDataFilepath)) - Expect(string(decompressedPayload)).NotTo(ContainSubstring(aksNbcCmdFilepath)) + // Script contents can reference these paths; only delivery as write_files + // entries would indicate that the scriptless path was used. + var cloudConfig cloudInit + Expect(yaml.Unmarshal(decompressedPayload, &cloudConfig)).To(Succeed()) + for _, file := range cloudConfig.WriteFiles { + Expect(file.Path).NotTo(Equal(aksNodeCustomDataFilepath)) + Expect(file.Path).NotTo(Equal(aksNbcCmdFilepath)) + } }) }) diff --git a/pkg/agent/cloud_init_encoding_test.go b/pkg/agent/cloud_init_encoding_test.go new file mode 100644 index 00000000000..310213cb683 --- /dev/null +++ b/pkg/agent/cloud_init_encoding_test.go @@ -0,0 +1,192 @@ +package agent + +import ( + "encoding/base64" + "fmt" + "strings" + "testing" + "text/template" + + "github.com/Azure/agentbaker/parts" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestTraditionalCloudInitUsesSingleCompression(t *testing.T) { + template, err := parts.Templates.ReadFile(kubernetesNodeCustomDataYaml) + require.NoError(t, err) + for _, distro := range []datamodel.Distro{datamodel.AKSUbuntuContainerd2204Gen2, datamodel.AKSAzureLinuxV2Gen2, datamodel.AKSAzureLinuxV3Gen2} { + for _, custom := range []bool{false, true} { + for _, dns := range []bool{false, true} { + for _, preProvision := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/custom=%t/localdns=%t/preProvision=%t", distro, custom, dns, preProvision), func(t *testing.T) { + config := newNodeCustomDataRenderConfig(distro) + config.PreProvisionOnly = preProvision + config.AgentPoolProfile.LocalDNSProfile = &datamodel.LocalDNSProfile{EnableLocalDNS: dns} + if custom { + config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{Name: "akscustom"} + } + assertTraditionalCloudInitPayload(t, template, config) + }) + } + } + } + } +} + +func assertTraditionalCloudInitPayload(t *testing.T, template []byte, config *datamodel.NodeBootstrappingConfiguration) { + t.Helper() + generator := InitializeTemplateGenerator() + original, err := generator.getSingleLine(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, legacyCloudInitFuncMap(config), true) + require.NoError(t, err) + payload := generator.getLinuxNodeBootstrappingPayload(config) + compressed, err := base64.StdEncoding.Strict().DecodeString(payload) + require.NoError(t, err) + decoded, err := getGzipDecodedValue(compressed) + require.NoError(t, err) + require.True(t, strings.HasPrefix(string(decoded), "#cloud-config\n")) + rendered, err := RenderLinuxNodeCustomDataTemplate(template, config) + require.NoError(t, err) + require.Equal(t, rendered, string(decoded)) + var before, after cloudInit + require.NoError(t, yaml.Unmarshal([]byte(original), &before)) + require.NoError(t, yaml.Unmarshal(decoded, &after)) + var document struct { + WriteFiles []map[string]any `yaml:"write_files"` + } + require.NoError(t, yaml.Unmarshal(decoded, &document)) + for _, file := range document.WriteFiles { + _, hasEncoding := file["encoding"] + require.False(t, hasEncoding, "plain files must use cloud-init's default, not an invalid empty encoding") + } + beforeFiles, err := buildIgnitionTarEntries(before) + require.NoError(t, err) + afterFiles, err := buildIgnitionTarEntries(after) + require.NoError(t, err) + require.Equal(t, beforeFiles, afterFiles) + for _, file := range after.WriteFiles { + require.Empty(t, file.Encoding) + } + require.Less(t, len(payload), len(getBase64EncodedGzippedCustomScriptFromStr(original))) + t.Logf("customData: %d encoded characters", len(payload)) + require.LessOrEqual(t, len(payload), MaxCustomDataLength) +} + +func TestCloudInitEncodingLeavesOtherModesUnchanged(t *testing.T) { + template, err := parts.Templates.ReadFile(kubernetesNodeCustomDataYaml) + require.NoError(t, err) + generator := InitializeTemplateGenerator() + for _, distro := range []datamodel.Distro{datamodel.AKSUbuntuContainerd2204Gen2, datamodel.AKSAzureLinuxV3Gen2, datamodel.AKSFlatcarGen2, datamodel.AKSACLGen2TL} { + for _, scriptless := range []bool{false, true} { + if !scriptless && distro != datamodel.AKSFlatcarGen2 && distro != datamodel.AKSACLGen2TL { + continue + } + t.Run(fmt.Sprintf("%s/scriptless=%t", distro, scriptless), func(t *testing.T) { + config := newNodeCustomDataRenderConfig(distro) + config.EnableScriptlessCSECmd = scriptless + config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{Name: "akscustom"} + original, err := generator.getSingleLine(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, legacyCloudInitFuncMap(config), true) + require.NoError(t, err) + rendered, err := RenderLinuxNodeCustomDataTemplate(template, config) + require.NoError(t, err) + require.Equal(t, original, rendered) + require.Equal(t, original, generator.getLinuxNodeCustomData(config)) + }) + } + } +} + +func legacyCloudInitFuncMap(config *datamodel.NodeBootstrappingConfiguration) template.FuncMap { + funcs := getBakerFuncMap(config, getParameters(config), getCustomDataVariables(config)) + funcs["GetCloudInitFileEncodingLine"] = func() string { return "encoding: " + encodingGZIP } + funcs["GetCloudInitFileContent"] = func(content string) string { + return "!!binary |\n " + getBase64EncodedGzippedCustomScriptFromStr(content) + } + return funcs +} + +func TestCloudInitRenderingPreservesAdditionalFields(t *testing.T) { + input := []byte("#cloud-config\nruncmd:\n- echo hello\nwrite_files:\n- path: /opt/example\n append: true\n defer: true\n content: \"a\\tb\\n\\n\"\n") + output, err := RenderLinuxNodeCustomDataTemplate(input, newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2)) + require.NoError(t, err) + require.Equal(t, string(input), output) +} + +func TestPlainCloudInitFileContentPreservesWhitespace(t *testing.T) { + input := []byte(`#cloud-config +write_files: +- path: /opt/example + {{GetCloudInitFileEncodingLine}} + content: {{.Name | GetCloudInitFileContent}} +`) + for _, content := range []string{ + "", + "tab\tquote\"backslash\\", + "trailing newline\n\n", + "\u00e9\r\n", + "next\u0085line\u2028paragraph\u2029end", + "nul\x00escape\x1b", + "echo \"$HOME\"; printf '%s\\n' 'a: # & < >'\n", + strings.Repeat("long line: # \"\\ ", 1024), + } { + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) + config.AgentPoolProfile.Name = content + output, err := RenderLinuxNodeCustomDataTemplate(input, config) + require.NoError(t, err) + var parsed cloudInit + require.NoError(t, yaml.Unmarshal([]byte(output), &parsed)) + require.Len(t, parsed.WriteFiles, 1) + require.Empty(t, parsed.WriteFiles[0].Encoding) + require.Equal(t, content, parsed.WriteFiles[0].Content) + } +} + +func TestPlainCloudInitFileContentRejectsInvalidUTF8(t *testing.T) { + input := []byte("#cloud-config\nwrite_files:\n- path: /opt/example\n content: {{.Name | GetCloudInitFileContent}}\n") + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) + config.AgentPoolProfile.Name = string([]byte{'a', 0xff, 'b'}) + output, err := RenderLinuxNodeCustomDataTemplate(input, config) + require.ErrorContains(t, err, "invalid UTF-8") + require.Empty(t, output) +} + +func TestEncodedCloudInitFileContentUsesRawBytes(t *testing.T) { + for _, distro := range []datamodel.Distro{datamodel.AKSUbuntuContainerd2204Gen2, datamodel.AKSFlatcarGen2, datamodel.AKSACLGen2TL} { + for i, content := range []string{"", "#!/bin/bash\nprintf 'hello\\n'\n", "\x00\xff\r\n\u0085\u2028\u2029"} { + t.Run(fmt.Sprintf("%s/%d", distro, i), func(t *testing.T) { + config := newNodeCustomDataRenderConfig(distro) + config.EnableScriptlessCSECmd = true + encoded, err := getCloudInitFileContent(config, content) + require.NoError(t, err) + var file cloudInitWriteFile + require.NoError(t, yaml.Unmarshal([]byte("content: "+encoded+"\n"), &file)) + decoded, err := getGzipDecodedValue([]byte(file.Content)) + require.NoError(t, err) + require.Equal(t, []byte(content), decoded) + }) + } + } +} + +func BenchmarkTraditionalCloudInitEncoding(b *testing.B) { + for _, old := range []bool{true, false} { + b.Run(fmt.Sprintf("legacy=%t", old), func(b *testing.B) { + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) + generator := InitializeTemplateGenerator() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if old { + rendered, err := generator.getSingleLineForTemplate(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, legacyCloudInitFuncMap(config), true) + if err != nil { + b.Fatal(err) + } + customData := getCustomDataFromJSON(fmt.Sprintf("{\"customData\": \"%s\"}", rendered)) + _ = getBase64EncodedGzippedCustomScriptFromStr(customData) + } else { + _ = generator.getLinuxNodeBootstrappingPayload(config) + } + } + }) + } +} diff --git a/pkg/agent/live_patching_delivery_test.go b/pkg/agent/live_patching_delivery_test.go index b616b2bcf0e..04ca8b846c2 100644 --- a/pkg/agent/live_patching_delivery_test.go +++ b/pkg/agent/live_patching_delivery_test.go @@ -18,7 +18,9 @@ func TestLivePatchingUsesVHDInsteadOfCustomData(t *testing.T) { require.NotContains(t, rendered, "/opt/azure/containers/ubuntu-snapshot-update.sh") require.NotContains(t, rendered, "/opt/azure/containers/security-update.sh") require.NotContains(t, rendered, "/opt/azure/containers/npd-update.sh") - size := len(getBase64EncodedGzippedCustomScriptFromStr(rendered)) + payload := InitializeTemplateGenerator().getLinuxNodeBootstrappingPayload(config) + require.Equal(t, getBase64EncodedGzippedCustomScriptFromStr(rendered), payload) + size := len(payload) t.Logf("traditional custom data: %d encoded characters", size) require.Less(t, size, MaxCustomDataLength) } diff --git a/pkg/agent/mariner_live_patching_delivery_test.go b/pkg/agent/mariner_live_patching_delivery_test.go index e7cf41fc494..7b9cb9293f3 100644 --- a/pkg/agent/mariner_live_patching_delivery_test.go +++ b/pkg/agent/mariner_live_patching_delivery_test.go @@ -20,7 +20,9 @@ func TestMarinerLivePatchingUsesVHDInsteadOfCustomData(t *testing.T) { require.NotContains(t, rendered, "/opt/azure/containers/mariner-package-update.sh") require.NotContains(t, rendered, "/opt/azure/containers/security-update.sh") require.NotContains(t, rendered, "/opt/azure/containers/npd-update.sh") - size := len(getBase64EncodedGzippedCustomScriptFromStr(rendered)) + payload := InitializeTemplateGenerator().getLinuxNodeBootstrappingPayload(config) + require.Equal(t, getBase64EncodedGzippedCustomScriptFromStr(rendered), payload) + size := len(payload) t.Logf("traditional custom data: %d encoded characters", size) require.Less(t, size, MaxCustomDataLength) }) diff --git a/pkg/agent/utils.go b/pkg/agent/utils.go index 1bc6c2d8fc1..67a251a7b6a 100644 --- a/pkg/agent/utils.go +++ b/pkg/agent/utils.go @@ -182,6 +182,10 @@ func escapeSingleLine(escapedStr string) string { // getBase64EncodedGzippedCustomScript will return a base64 of the CSE. func getBase64EncodedGzippedCustomScript(csFilename string, config *datamodel.NodeBootstrappingConfiguration) string { + return getBase64EncodedGzippedCustomScriptFromStr(getCustomScript(csFilename, config)) +} + +func getCustomScript(csFilename string, config *datamodel.NodeBootstrappingConfiguration) string { b, err := parts.Templates.ReadFile(csFilename) if err != nil { // this should never happen and this is a bug. @@ -209,7 +213,7 @@ func getBase64EncodedGzippedCustomScript(csFilename string, config *datamodel.No panic(fmt.Sprintf("BUG: failed to minify %s: %v", csFilename, err)) } } - return getBase64EncodedGzippedCustomScriptFromStr(csStr) + return csStr } func minifyShellScript(name, script string) (string, error) { diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index 44108764db4..ca46f7d69b4 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -1348,6 +1348,11 @@ func TestCSEScriptRoundTrip(t *testing.T) { require.NoError(t, err) decoded, err := getGzipDecodedValue(compressed) require.NoError(t, err) + plain := getCustomScript(script, config) + minified, err := minifyShellScript(script, plain) + require.NoError(t, err) + require.Equal(t, minified, plain, "plain delivery must not bypass the minifier") + require.Equal(t, plain, string(decoded)) if script == kubernetesCSEHelpersScript { require.True(t, strings.HasSuffix(string(decoded), "#HELPERSEOF\n")) } @@ -1357,21 +1362,21 @@ func TestCSEScriptRoundTrip(t *testing.T) { } // discoverCSEScripts parses variables.go to find all constant names passed to -// getBase64EncodedGzippedCustomScript(), then resolves those constants to file +// getCustomScript(), then resolves those constants to file // paths from const.go, filtering to .sh files only. func discoverCSEScripts(t *testing.T) []string { t.Helper() root := repoRoot() - // Step 1: Read variables.go and extract constant names from getBase64EncodedGzippedCustomScript() calls + // Step 1: Read variables.go and extract constant names from getCustomScript() calls variablesPath := filepath.Join(root, "pkg", "agent", "variables.go") variablesBytes, err := os.ReadFile(variablesPath) if err != nil { t.Fatalf("failed to read variables.go: %v", err) } - // Match: getBase64EncodedGzippedCustomScript(constantName, config) - callRe := regexp.MustCompile(`getBase64EncodedGzippedCustomScript\((\w+),`) + // Match: getCustomScript(constantName, config) + callRe := regexp.MustCompile(`getCustomScript\((\w+),`) matches := callRe.FindAllStringSubmatch(string(variablesBytes), -1) constNames := make(map[string]bool) for _, m := range matches { diff --git a/pkg/agent/variables.go b/pkg/agent/variables.go index e83f5e51292..ed282bcaa09 100644 --- a/pkg/agent/variables.go +++ b/pkg/agent/variables.go @@ -10,74 +10,74 @@ import ( "github.com/Azure/agentbaker/pkg/agent/datamodel" ) -// getCustomDataVariables returns cloudinit data used by Linux. +// getCustomDataVariables returns rendered, unencoded file contents used by Linux. func getCustomDataVariables(config *datamodel.NodeBootstrappingConfiguration) paramsMap { cs := config.ContainerService cloudInitFiles := map[string]interface{}{ "cloudInitData": paramsMap{ - "provisionStartScript": getBase64EncodedGzippedCustomScript(kubernetesCSEStartScript, config), - "provisionScript": getBase64EncodedGzippedCustomScript(kubernetesCSEMainScript, config), - "provisionSource": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScript, config), - "provisionSourceUbuntu": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptUbuntu, config), - "provisionSourceMariner": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptMariner, config), - "provisionSourceAzlOSGuard": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptAzlOSGuard, config), - "provisionSourceFlatcar": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptFlatcar, config), - "provisionSourceACL": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptACL, config), - "provisionInstalls": getBase64EncodedGzippedCustomScript(kubernetesCSEInstall, config), - "provisionInstallsUbuntu": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallUbuntu, config), - "provisionInstallsMariner": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallMariner, config), - "provisionInstallsAzlOSGuard": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallAzlOSGuard, config), - "provisionInstallsFlatcar": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallFlatcar, config), - "provisionInstallsACL": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallACL, config), - "provisionConfigs": getBase64EncodedGzippedCustomScript(kubernetesCSEConfig, config), - "provisionConfigsGPU": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigGPU, config), - "provisionConfigsLocalDNS": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigLocalDNS, config), - "provisionConfigsKubelet": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigKubelet, config), - "provisionConfigsNetwork": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigNetwork, config), - "provisionConfigsChrony": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigChrony, config), - "provisionSendLogs": getBase64EncodedGzippedCustomScript(kubernetesCSESendLogs, config), - "provisionRedactCloudConfig": getBase64EncodedGzippedCustomScript(kubernetesCSERedactCloudConfig, config), - "customSearchDomainsScript": getBase64EncodedGzippedCustomScript(kubernetesCustomSearchDomainsScript, config), - "dhcpv6SystemdService": getBase64EncodedGzippedCustomScript(dhcpv6SystemdService, config), - "dhcpv6ConfigurationScript": getBase64EncodedGzippedCustomScript(dhcpv6ConfigurationScript, config), - "kubeletSystemdService": getBase64EncodedGzippedCustomScript(kubeletSystemdService, config), - "reconcilePrivateHostsScript": getBase64EncodedGzippedCustomScript(reconcilePrivateHostsScript, config), - "reconcilePrivateHostsService": getBase64EncodedGzippedCustomScript(reconcilePrivateHostsService, config), - "ensureNoDupEbtablesScript": getBase64EncodedGzippedCustomScript(ensureNoDupEbtablesScript, config), - "ensureNoDupEbtablesService": getBase64EncodedGzippedCustomScript(ensureNoDupEbtablesService, config), - "bindMountScript": getBase64EncodedGzippedCustomScript(bindMountScript, config), - "bindMountSystemdService": getBase64EncodedGzippedCustomScript(bindMountSystemdService, config), - "migPartitionSystemdService": getBase64EncodedGzippedCustomScript(migPartitionSystemdService, config), - "migPartitionScript": getBase64EncodedGzippedCustomScript(migPartitionScript, config), - "ensureIMDSRestrictionScript": getBase64EncodedGzippedCustomScript(ensureIMDSRestrictionScript, config), - "snapshotUpdateScript": getBase64EncodedGzippedCustomScript(snapshotUpdateScript, config), - "snapshotUpdateService": getBase64EncodedGzippedCustomScript(snapshotUpdateSystemdService, config), - "snapshotUpdateTimer": getBase64EncodedGzippedCustomScript(snapshotUpdateSystemdTimer, config), - "packageUpdateScriptMariner": getBase64EncodedGzippedCustomScript(packageUpdateScriptMariner, config), - "packageUpdateServiceMariner": getBase64EncodedGzippedCustomScript(packageUpdateSystemdServiceMariner, config), - "packageUpdateTimerMariner": getBase64EncodedGzippedCustomScript(packageUpdateSystemdTimerMariner, config), - "componentManifestFile": getBase64EncodedGzippedCustomScript(componentManifestFile, config), - "validateKubeletCredentialsScript": getBase64EncodedGzippedCustomScript(validateKubeletCredentialsScript, config), - "secureTLSBootstrapService": getBase64EncodedGzippedCustomScript(secureTLSBootstrapService, config), - "cloudInitStatusCheckScript": getBase64EncodedGzippedCustomScript(cloudInitStatusCheckScript, config), - "measureTLSBootstrappingLatencyScript": getBase64EncodedGzippedCustomScript(measureTLSBootstrappingLatencyScript, config), - "measureTLSBootstrappingLatencyService": getBase64EncodedGzippedCustomScript(measureTLSBootstrappingLatencyService, config), - "configureAzureNetworkScript": getBase64EncodedGzippedCustomScript(configureAzureNetworkScript, config), - "azureNetworkUdevRule": getBase64EncodedGzippedCustomScript(azureNetworkUdevRule, config), + "provisionStartScript": getCustomScript(kubernetesCSEStartScript, config), + "provisionScript": getCustomScript(kubernetesCSEMainScript, config), + "provisionSource": getCustomScript(kubernetesCSEHelpersScript, config), + "provisionSourceUbuntu": getCustomScript(kubernetesCSEHelpersScriptUbuntu, config), + "provisionSourceMariner": getCustomScript(kubernetesCSEHelpersScriptMariner, config), + "provisionSourceAzlOSGuard": getCustomScript(kubernetesCSEHelpersScriptAzlOSGuard, config), + "provisionSourceFlatcar": getCustomScript(kubernetesCSEHelpersScriptFlatcar, config), + "provisionSourceACL": getCustomScript(kubernetesCSEHelpersScriptACL, config), + "provisionInstalls": getCustomScript(kubernetesCSEInstall, config), + "provisionInstallsUbuntu": getCustomScript(kubernetesCSEInstallUbuntu, config), + "provisionInstallsMariner": getCustomScript(kubernetesCSEInstallMariner, config), + "provisionInstallsAzlOSGuard": getCustomScript(kubernetesCSEInstallAzlOSGuard, config), + "provisionInstallsFlatcar": getCustomScript(kubernetesCSEInstallFlatcar, config), + "provisionInstallsACL": getCustomScript(kubernetesCSEInstallACL, config), + "provisionConfigs": getCustomScript(kubernetesCSEConfig, config), + "provisionConfigsGPU": getCustomScript(kubernetesCSEConfigGPU, config), + "provisionConfigsLocalDNS": getCustomScript(kubernetesCSEConfigLocalDNS, config), + "provisionConfigsKubelet": getCustomScript(kubernetesCSEConfigKubelet, config), + "provisionConfigsNetwork": getCustomScript(kubernetesCSEConfigNetwork, config), + "provisionConfigsChrony": getCustomScript(kubernetesCSEConfigChrony, config), + "provisionSendLogs": getCustomScript(kubernetesCSESendLogs, config), + "provisionRedactCloudConfig": getCustomScript(kubernetesCSERedactCloudConfig, config), + "customSearchDomainsScript": getCustomScript(kubernetesCustomSearchDomainsScript, config), + "dhcpv6SystemdService": getCustomScript(dhcpv6SystemdService, config), + "dhcpv6ConfigurationScript": getCustomScript(dhcpv6ConfigurationScript, config), + "kubeletSystemdService": getCustomScript(kubeletSystemdService, config), + "reconcilePrivateHostsScript": getCustomScript(reconcilePrivateHostsScript, config), + "reconcilePrivateHostsService": getCustomScript(reconcilePrivateHostsService, config), + "ensureNoDupEbtablesScript": getCustomScript(ensureNoDupEbtablesScript, config), + "ensureNoDupEbtablesService": getCustomScript(ensureNoDupEbtablesService, config), + "bindMountScript": getCustomScript(bindMountScript, config), + "bindMountSystemdService": getCustomScript(bindMountSystemdService, config), + "migPartitionSystemdService": getCustomScript(migPartitionSystemdService, config), + "migPartitionScript": getCustomScript(migPartitionScript, config), + "ensureIMDSRestrictionScript": getCustomScript(ensureIMDSRestrictionScript, config), + "snapshotUpdateScript": getCustomScript(snapshotUpdateScript, config), + "snapshotUpdateService": getCustomScript(snapshotUpdateSystemdService, config), + "snapshotUpdateTimer": getCustomScript(snapshotUpdateSystemdTimer, config), + "packageUpdateScriptMariner": getCustomScript(packageUpdateScriptMariner, config), + "packageUpdateServiceMariner": getCustomScript(packageUpdateSystemdServiceMariner, config), + "packageUpdateTimerMariner": getCustomScript(packageUpdateSystemdTimerMariner, config), + "componentManifestFile": getCustomScript(componentManifestFile, config), + "validateKubeletCredentialsScript": getCustomScript(validateKubeletCredentialsScript, config), + "secureTLSBootstrapService": getCustomScript(secureTLSBootstrapService, config), + "cloudInitStatusCheckScript": getCustomScript(cloudInitStatusCheckScript, config), + "measureTLSBootstrappingLatencyScript": getCustomScript(measureTLSBootstrappingLatencyScript, config), + "measureTLSBootstrappingLatencyService": getCustomScript(measureTLSBootstrappingLatencyService, config), + "configureAzureNetworkScript": getCustomScript(configureAzureNetworkScript, config), + "azureNetworkUdevRule": getCustomScript(azureNetworkUdevRule, config), }, } cloudInitData := cloudInitFiles["cloudInitData"].(paramsMap) //nolint:errcheck // no error is actually here - cloudInitData["initAKSCloud"] = getBase64EncodedGzippedCustomScript(initAKSCloudScript, config) + cloudInitData["initAKSCloud"] = getCustomScript(initAKSCloudScript, config) if config.IsFlatcar() || config.IsACL() { cloudInitData["provisionRedactCloudConfig"] = "" // Flatcar and ACL do not have cloud-init } if !cs.Properties.IsVHDDistroForAllNodes() { - cloudInitData["kmsSystemdService"] = getBase64EncodedGzippedCustomScript(kmsSystemdService, config) - cloudInitData["aptPreferences"] = getBase64EncodedGzippedCustomScript(aptPreferences, config) - cloudInitData["dockerClearMountPropagationFlags"] = getBase64EncodedGzippedCustomScript(dockerClearMountPropagationFlags, config) + cloudInitData["kmsSystemdService"] = getCustomScript(kmsSystemdService, config) + cloudInitData["aptPreferences"] = getCustomScript(aptPreferences, config) + cloudInitData["dockerClearMountPropagationFlags"] = getCustomScript(dockerClearMountPropagationFlags, config) } return cloudInitFiles