From e97c2600f52497a08932f27319bb3caae9f37103 Mon Sep 17 00:00:00 2001 From: Ben Brady <86134744+benjamin-brady@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:36:50 +1300 Subject: [PATCH 1/3] fix(cloud-init): avoid nested script compression Keep scripts literal inside the outer compressed CustomData payload to stay within Azure VMSS limits. Preserve gzip write_files for Flatcar, ACL and scriptless paths. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 29247196-6c9e-475a-8892-77fc9c3316d4 --- hotfix/hotfix_generate.py | 22 +++- hotfix/hotfix_generate_test.py | 95 ++++++++++++++- parts/linux/cloud-init/nodecustomdata.yml | 108 +++++------------ pkg/agent/baker.go | 54 ++++++++- pkg/agent/baker_test.go | 78 +++++++++++- pkg/agent/bakerapi_test.go | 138 ++++++++++++++++++++++ pkg/agent/utils.go | 14 ++- pkg/agent/variables.go | 65 ++++++---- 8 files changed, 455 insertions(+), 119 deletions(-) diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index 2623604bac3..080e0f20434 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -55,8 +55,8 @@ SCRIPTS_BEGIN = "# ---- hotfix-scripts: auto-generated ----" SCRIPTS_END = "# ---- end hotfix-scripts ----" -# Map from source file paths (relative to artifacts/) to the GetVariableProperty -# keys used in nodecustomdata.yml. Only scripts that appear as write_files entries +# Map from source file paths (relative to artifacts/) to the template variable +# keys used in nodecustomdata.yml. Only files that appear as write_files entries # in the traditional section are included. SOURCE_TO_VARKEY = { # CSE helpers — base (non-distro) @@ -380,7 +380,9 @@ def parse_write_files_blocks(traditional_lines): """Parse write_files blocks from the traditional section. Each block is either a simple '- path:' entry or an entire conditional - block (e.g., {{if IsAzlOSGuard}}...{{end}}) treated as a single unit. + block (e.g., {{if IsAzlOSGuard}}...{{end}}) treated as a single unit. File + keys may use either the legacy GetVariableProperty form or the literal-file + GetCloudInitFileProperties form. Returns a list of (varkeys_set, lines_list) tuples. """ @@ -421,9 +423,17 @@ def parse_write_files_blocks(traditional_lines): if in_block: current_block.append(line) - match = re.search(r'GetVariableProperty\s+"cloudInitData"\s+"(\w+)"', stripped) - if match: - current_varkeys.add(match.group(1)) + matches = ( + re.search( + r'GetVariableProperty\s+"cloudInitData"\s+"(\w+)"', + stripped, + ), + re.search(r'GetCloudInitFileProperties\s+"(\w+)"', stripped), + ) + for match in matches: + if match: + current_varkeys.add(match.group(1)) + break if current_block and current_varkeys: blocks.append((current_varkeys.copy(), list(current_block))) diff --git a/hotfix/hotfix_generate_test.py b/hotfix/hotfix_generate_test.py index 23b5ab5b11e..e73ff3ea888 100644 --- a/hotfix/hotfix_generate_test.py +++ b/hotfix/hotfix_generate_test.py @@ -54,6 +54,16 @@ {{end}} """ +MIXED_ENCODING_TEMPLATE = """- path: /literal.sh + owner: root +{{GetCloudInitFileProperties "literalFile"}} +- path: /legacy.sh + owner: root + encoding: gzip + content: !!binary | + {{GetVariableProperty "cloudInitData" "legacyFile"}} +""" + class HotfixGenerateTests(unittest.TestCase): def test_config_modules_are_independently_hotfixable(self): @@ -114,7 +124,10 @@ def test_config_modules_are_independently_hotfixable(self): self.assertEqual({variable}, blocks[0][0]) self.assertIn(f"- path: {{{{{path_function}}}}}", template) self.assertIn('permissions: "0744"', template) - self.assertIn("encoding: gzip", template) + self.assertTrue( + f'GetCloudInitFileProperties "{variable}"' in template + or f'GetVariableProperty "cloudInitData" "{variable}"' in template + ) self.assertNotIn("{{if", template) def test_mariner_package_update_is_not_embedded_in_custom_data(self): @@ -187,6 +200,86 @@ def test_parse_write_files_blocks_keeps_distro_chain_together(self): blocks[1][0], ) + def test_parse_write_files_blocks_accepts_literal_and_legacy_syntax(self): + blocks = hotfix_generate.parse_write_files_blocks( + MIXED_ENCODING_TEMPLATE.splitlines(keepends=True) + ) + + self.assertEqual( + [ + ({"literalFile"}, [ + "- path: /literal.sh\n", + " owner: root\n", + '{{GetCloudInitFileProperties "literalFile"}}\n', + ]), + ({"legacyFile"}, [ + "- path: /legacy.sh\n", + " owner: root\n", + " encoding: gzip\n", + " content: !!binary |\n", + ' {{GetVariableProperty "cloudInitData" "legacyFile"}}\n', + ]), + ], + blocks, + ) + + def test_current_template_literal_entries_are_injected_and_rendered(self): + repository = Path(__file__).resolve().parents[1] + template_lines = (repository / hotfix_generate.TEMPLATE).read_text().splitlines( + keepends=True + ) + _, outer_else, end = hotfix_generate.find_block_boundaries(template_lines) + self.assertIsNotNone(outer_else) + self.assertIsNotNone(end) + traditional = template_lines[outer_else + 1:end] + available = set().union( + *(keys for keys, _ in hotfix_generate.parse_write_files_blocks(traditional)) + ) + selected = {"provisionConfigsGPU"} + self.assertIn("provisionConfigsGPU", available) + + hotfix_template = hotfix_generate.build_hotfix_template(selected, traditional) + self.assertIn( + '{{GetCloudInitFileProperties "provisionConfigsGPU"}}', + hotfix_template, + ) + + with tempfile.TemporaryDirectory() as temp_dir: + injected_template = Path(temp_dir) / "nodecustomdata.yml" + injected_template.write_text("".join(template_lines)) + generated_dir = Path(temp_dir) / "generated" + with mock.patch.object( + hotfix_generate, "TEMPLATE", str(injected_template) + ), mock.patch.object( + hotfix_generate, "GENERATED_DIR", str(generated_dir) + ): + hotfix_generate.update_nodecustomdata(selected) + injected = injected_template.read_text() + marked_block = injected.split(hotfix_generate.SCRIPTS_BEGIN, 1)[1].split( + hotfix_generate.SCRIPTS_END, 1 + )[0] + self.assertIn( + '{{GetCloudInitFileProperties "provisionConfigsGPU"}}', + marked_block, + ) + + hotfix_generate.write_rendered_payload(selected, traditional) + + for platform in ("ubuntu", "mariner"): + rendered = ( + generated_dir / f"rendered_nodecustomdata_{platform}.yml" + ).read_text() + self.assertIn( + "- path: /opt/azure/containers/provision_configs_gpu.sh", + rendered, + ) + self.assertIn('permissions: "0744"', rendered) + self.assertIn(r'content: "#!/bin/bash\n', rendered) + self.assertIn("configGPUDrivers() {", rendered) + self.assertNotIn("GetCloudInitFileProperties", rendered) + self.assertNotIn("{{", rendered) + self.assertNotIn("encoding: gzip", rendered) + def test_build_hotfix_template_selects_only_requested_blocks(self): rendered = hotfix_generate.build_hotfix_template( {"provisionSource"}, diff --git a/parts/linux/cloud-init/nodecustomdata.yml b/parts/linux/cloud-init/nodecustomdata.yml index 200eeb0ce5f..46ca391d300 100644 --- a/parts/linux/cloud-init/nodecustomdata.yml +++ b/parts/linux/cloud-init/nodecustomdata.yml @@ -28,201 +28,147 @@ write_files: {{if IsAKSCustomCloud}} - path: {{GetInitAKSCloudFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "initAKSCloud"}} +{{GetCloudInitFileProperties "initAKSCloud"}} {{end}} {{- else }} - path: {{GetCSEHelpersScriptFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSource"}} +{{GetCloudInitFileProperties "provisionSource"}} {{if IsACL }} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceACL"}} +{{GetCloudInitFileProperties "provisionSourceACL"}} {{- else if IsAzlOSGuard}} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceAzlOSGuard"}} +{{GetCloudInitFileProperties "provisionSourceAzlOSGuard"}} {{- else if IsMariner}} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceMariner"}} +{{GetCloudInitFileProperties "provisionSourceMariner"}} {{- else if IsFlatcar }} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceFlatcar"}} +{{GetCloudInitFileProperties "provisionSourceFlatcar"}} {{- else }} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceUbuntu"}} +{{GetCloudInitFileProperties "provisionSourceUbuntu"}} {{end}} {{ if not IsCustomImage -}} - path: /opt/azure/containers/provision_start.sh permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionStartScript"}} +{{GetCloudInitFileProperties "provisionStartScript"}} {{- end }} - path: /opt/azure/containers/provision.sh permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionScript"}} +{{GetCloudInitFileProperties "provisionScript"}} - path: {{GetCSEInstallScriptFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstalls"}} +{{GetCloudInitFileProperties "provisionInstalls"}} {{if IsACL }} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsACL"}} +{{GetCloudInitFileProperties "provisionInstallsACL"}} {{- else if IsAzlOSGuard}} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsAzlOSGuard"}} +{{GetCloudInitFileProperties "provisionInstallsAzlOSGuard"}} {{- else if IsMariner}} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsMariner"}} +{{GetCloudInitFileProperties "provisionInstallsMariner"}} {{- else if IsFlatcar }} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsFlatcar"}} +{{GetCloudInitFileProperties "provisionInstallsFlatcar"}} {{- else }} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsUbuntu"}} +{{GetCloudInitFileProperties "provisionInstallsUbuntu"}} {{end}} - path: {{GetCSEConfigScriptFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigs"}} +{{GetCloudInitFileProperties "provisionConfigs"}} - path: {{GetCSEConfigGPUScriptFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsGPU"}} +{{GetCloudInitFileProperties "provisionConfigsGPU"}} - path: {{GetCSEConfigLocalDNSScriptFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsLocalDNS"}} +{{GetCloudInitFileProperties "provisionConfigsLocalDNS"}} - path: {{GetCSEConfigKubeletScriptFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsKubelet"}} +{{GetCloudInitFileProperties "provisionConfigsKubelet"}} - path: {{GetCSEConfigNetworkScriptFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsNetwork"}} +{{GetCloudInitFileProperties "provisionConfigsNetwork"}} - path: {{GetCSEConfigAddonsScriptFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsAddons"}} +{{GetCloudInitFileProperties "provisionConfigsAddons"}} - path: {{GetCSEConfigChronyScriptFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsChrony"}} +{{GetCloudInitFileProperties "provisionConfigsChrony"}} {{if IsAKSCustomCloud}} - path: {{GetInitAKSCloudFilepath}} permissions: "0744" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "initAKSCloud"}} +{{GetCloudInitFileProperties "initAKSCloud"}} {{end}} - path: /etc/systemd/system/reconcile-private-hosts.service permissions: "0644" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "reconcilePrivateHostsService"}} +{{GetCloudInitFileProperties "reconcilePrivateHostsService"}} - path: /etc/systemd/system/kubelet.service permissions: "0600" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "kubeletSystemdService"}} +{{GetCloudInitFileProperties "kubeletSystemdService"}} - path: /opt/azure-network/configure-azure-network.sh permissions: "0755" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "configureAzureNetworkScript"}} +{{GetCloudInitFileProperties "configureAzureNetworkScript"}} - path: /etc/udev/rules.d/99-azure-network.rules permissions: "0644" - encoding: gzip owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "azureNetworkUdevRule"}} +{{GetCloudInitFileProperties "azureNetworkUdevRule"}} {{- end }} diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index dfa337aff80..8d06ca51420 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -314,13 +314,13 @@ func (t *TemplateGenerator) getLinuxNodeCustomDataJSONObject(config *datamodel.N parameters := getParameters(config) // get variable cloudInit variables := getCustomDataVariables(config) - str, e := t.getSingleLineForTemplate(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, getBakerFuncMap(config, parameters, variables), true) + str, e := t.getSingleLine(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, getBakerFuncMap(config, parameters, variables), true) if e != nil { panic(e) } - return fmt.Sprintf("{\"customData\": \"%s\"}", str) + return getCustomDataJSONObject(str) } const ( @@ -659,6 +659,10 @@ func RenderLinuxNodeCustomDataTemplate(templateContent []byte, config *datamodel func getBakerFuncMap(config *datamodel.NodeBootstrappingConfiguration, params paramsMap, variables paramsMap) template.FuncMap { funcMap := getContainerServiceFuncMap(config) + funcMap["GetCloudInitFileProperties"] = func(name string) (string, error) { + return getCloudInitFileProperties(variables, name) + } + funcMap["GetParameter"] = func(s string) interface{} { if v, ok := params[s].(paramsMap); ok && v != nil { if v["value"] == nil { @@ -706,6 +710,52 @@ func getBakerFuncMap(config *datamodel.NodeBootstrappingConfiguration, params pa return funcMap } +func getCloudInitFileProperties(variables paramsMap, name string) (string, error) { + if files, ok := variables["cloudInitFileData"].(paramsMap); ok { + if content, ok := files[name].(string); ok { + return renderCloudInitFileContent(content) + } + } + if cloudInitData, ok := variables["cloudInitData"].(paramsMap); ok { + if encoded, ok := cloudInitData[name].(string); ok { + return fmt.Sprintf(" encoding: gzip\n content: !!binary |\n %s\n", encoded), nil + } + } + return "", fmt.Errorf("cloud-init file %q is unavailable", name) +} + +func renderCloudInitFileContent(content string) (string, error) { + style := yaml.LiteralStyle + for _, line := range strings.Split(content, "\n") { + if strings.HasPrefix(line, "\t") { + style = yaml.DoubleQuotedStyle + break + } + } + document := &yaml.Node{ + Kind: yaml.MappingNode, + Tag: "!!map", + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Tag: "!!str", Value: "content"}, + {Kind: yaml.ScalarNode, Tag: "!!str", Value: content, Style: style}, + }, + } + var buffer bytes.Buffer + encoder := yaml.NewEncoder(&buffer) + encoder.SetIndent(2) + if err := encoder.Encode(document); err != nil { + return "", fmt.Errorf("encode cloud-init file content: %w", err) + } + if err := encoder.Close(); err != nil { + return "", fmt.Errorf("close cloud-init file content encoder: %w", err) + } + lines := strings.Split(strings.TrimSuffix(buffer.String(), "\n"), "\n") + for index := range lines { + lines[index] = " " + lines[index] + } + return strings.Join(lines, "\n") + "\n", nil +} + /* normalizeResourceGroupNameForLabel normalizes resource group name to be used as a label, similar to what the ARM template used to do. diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index c3d5bf3b11f..7a20ddd82d3 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -26,6 +26,7 @@ import ( . "github.com/onsi/gomega" "github.com/stretchr/testify/require" "github.com/vincent-petithory/dataurl" + "gopkg.in/yaml.v3" ) // this regex looks for groups of the following forms, returning KEY and VALUE as submatches. @@ -76,6 +77,72 @@ write_files: } } +func TestRenderCloudInitFileContentRoundTrips(t *testing.T) { + testCases := []struct { + name string + content string + }{ + {name: "empty", content: ""}, + {name: "no trailing newline", content: "echo ready"}, + {name: "one trailing newline", content: "echo ready\n"}, + {name: "multiple trailing newlines", content: "echo ready\n\n"}, + {name: "YAML-significant text", content: "---\n# shell text\nkey: value\n"}, + {name: "tabs and indentation", content: "\tcommand\n indented\n"}, + {name: "Unicode", content: "printf 'héllo 🌍'\n"}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + content, err := renderCloudInitFileContent(testCase.content) + require.NoError(t, err) + + customData := "#cloud-config\nwrite_files:\n- path: /opt/script.sh\n owner: root\n" + content + var parsed cloudInit + require.NoError(t, yaml.Unmarshal([]byte(customData), &parsed)) + require.Len(t, parsed.WriteFiles, 1) + require.Equal(t, testCase.content, parsed.WriteFiles[0].Content) + }) + } +} + +func TestGetCloudInitFilePropertiesKeepsGzipFallback(t *testing.T) { + testCases := []struct { + name string + distro datamodel.Distro + enableScriptlessCSECmd bool + }{ + {name: "Flatcar", distro: datamodel.AKSFlatcarGen2}, + {name: "ACL", distro: datamodel.AKSACLGen2TL}, + {name: "scriptless", distro: datamodel.AKSUbuntuContainerd2204Gen2, enableScriptlessCSECmd: true}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + config := newNodeCustomDataRenderConfig(testCase.distro) + config.EnableScriptlessCSECmd = testCase.enableScriptlessCSECmd + variables := getCustomDataVariables(config) + require.NotContains(t, variables, "cloudInitFileData") + + funcMap := getBakerFuncMap(config, getParameters(config), variables) + getProperties, ok := funcMap["GetCloudInitFileProperties"].(func(string) (string, error)) + require.True(t, ok) + properties, err := getProperties("provisionSource") + require.NoError(t, err) + require.Contains(t, properties, "encoding: gzip") + + customData := "#cloud-config\nwrite_files:\n- path: /opt/script.sh\n owner: root\n" + properties + var parsed cloudInit + require.NoError(t, yaml.Unmarshal([]byte(customData), &parsed)) + require.Len(t, parsed.WriteFiles, 1) + require.Equal(t, "gzip", parsed.WriteFiles[0].Encoding) + + actual, err := getGzipDecodedValue([]byte(parsed.WriteFiles[0].Content)) + require.NoError(t, err) + require.Equal(t, getCustomScriptContent(kubernetesCSEHelpersScript, config), string(actual)) + }) + } +} + func newNodeCustomDataRenderConfig(distro datamodel.Distro) *datamodel.NodeBootstrappingConfiguration { profile := &datamodel.AgentPoolProfile{ Name: "hotfix-render-test", @@ -1368,8 +1435,15 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { expectedCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(config)) Expect(string(decompressedPayload)).To(Equal(expectedCustomData)) - Expect(string(decompressedPayload)).NotTo(ContainSubstring(aksNodeCustomDataFilepath)) - Expect(string(decompressedPayload)).NotTo(ContainSubstring(aksNbcCmdFilepath)) + var parsedCustomData cloudInit + Expect(yaml.Unmarshal(decompressedPayload, &parsedCustomData)).To(Succeed()) + for _, file := range parsedCustomData.WriteFiles { + Expect(file.Path).NotTo(Equal(aksNodeCustomDataFilepath)) + Expect(file.Path).NotTo(Equal(aksNbcCmdFilepath)) + } + for _, command := range parsedCustomData.BootCommands { + Expect(command).NotTo(ContainSubstring(aksNbcCmdFilepath)) + } }) }) diff --git a/pkg/agent/bakerapi_test.go b/pkg/agent/bakerapi_test.go index 32998201e16..c155c824f65 100644 --- a/pkg/agent/bakerapi_test.go +++ b/pkg/agent/bakerapi_test.go @@ -1,13 +1,19 @@ package agent import ( + "bytes" + "compress/gzip" "context" + "encoding/base64" + "fmt" + "io" "github.com/Azure/agentbaker/pkg/agent/datamodel" agenttoggles "github.com/Azure/agentbaker/pkg/agent/toggles" "github.com/barkimedes/go-deepcopy" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" ) type testToggles struct { @@ -179,6 +185,138 @@ var _ = Describe("AgentBaker API implementation tests", func() { Expect(nodeBootStrapping.SigImageConfig.Definition).To(Equal("2204gen2containerd")) }) + It("should render Linux cloud-init files as literal write_files content without changing their bytes", func() { + testCases := []struct { + name string + distro datamodel.Distro + }{ + {name: "Ubuntu 22.04", distro: datamodel.AKSUbuntuContainerd2204Gen2}, + {name: "Ubuntu 24.04", distro: datamodel.AKSUbuntuContainerd2404Gen2}, + {name: "Azure Linux 3", distro: datamodel.AKSAzureLinuxV3Gen2}, + } + + for _, testCase := range testCases { + for _, preProvisionOnly := range []bool{false, true} { + name := testCase.name + if preProvisionOnly { + name += " PIS" + } else { + name += " ordinary" + } + By(name) + + configCopy, err := deepcopy.Anything(config) + Expect(err).NotTo(HaveOccurred()) + testConfig, ok := configCopy.(*datamodel.NodeBootstrappingConfiguration) + Expect(ok).To(BeTrue()) + testConfig.AgentPoolProfile.Distro = testCase.distro + testConfig.AgentPoolProfile.VMSize = "Standard_NC4as_T4_v3" + testConfig.PreProvisionOnly = preProvisionOnly + testConfig.EnableScriptlessCSECmd = false + testConfig.ConfigGPUDriverIfNeeded = false + testConfig.EnableGPUDevicePluginIfNeeded = false + testConfig.EnableNvidia = false + + payload := InitializeTemplateGenerator().getLinuxNodeBootstrappingPayload(testConfig) + Expect(len(payload)).To(BeNumerically("<", MaxCustomDataLength)) + encoded, err := base64.StdEncoding.Strict().DecodeString(payload) + Expect(err).NotTo(HaveOccurred()) + Expect(len(encoded)).To(BeNumerically("<=", 65535)) + reader, err := gzip.NewReader(bytes.NewReader(encoded)) + Expect(err).NotTo(HaveOccurred()) + customDataBytes, err := io.ReadAll(reader) + Expect(err).NotTo(HaveOccurred()) + Expect(reader.Close()).To(Succeed()) + var customData cloudInit + Expect(yaml.Unmarshal(customDataBytes, &customData)).To(Succeed()) + fmt.Fprintf( + GinkgoWriter, + "PIS_CUSTOMDATA distro=%s preProvisionOnly=%t base64=%d gzip=%d yaml=%d\n", + testCase.distro, + preProvisionOnly, + len(payload), + len(encoded), + len(customDataBytes), + ) + + customDataVariables := getCustomDataVariables(testConfig) + variables, ok := customDataVariables["cloudInitFileData"].(paramsMap) + Expect(ok).To(BeTrue()) + expectedScripts := []struct { + path string + key string + }{ + {cseHelpersScriptFilepath, "provisionSource"}, + {cseHelpersScriptDistroFilepath, "provisionSourceUbuntu"}, + {"/opt/azure/containers/provision_start.sh", "provisionStartScript"}, + {"/opt/azure/containers/provision.sh", "provisionScript"}, + {cseInstallScriptFilepath, "provisionInstalls"}, + {cseInstallScriptDistroFilepath, "provisionInstallsUbuntu"}, + {cseConfigScriptFilepath, "provisionConfigs"}, + {cseConfigGPUScriptFilepath, "provisionConfigsGPU"}, + {cseConfigLocalDNSScriptFilepath, "provisionConfigsLocalDNS"}, + {cseConfigKubeletScriptFilepath, "provisionConfigsKubelet"}, + {cseConfigNetworkScriptFilepath, "provisionConfigsNetwork"}, + {cseConfigAddonsScriptFilepath, "provisionConfigsAddons"}, + {cseConfigChronyScriptFilepath, "provisionConfigsChrony"}, + } + if testCase.distro == datamodel.AKSAzureLinuxV3Gen2 { + expectedScripts[1].key = "provisionSourceMariner" + expectedScripts[5].key = "provisionInstallsMariner" + } + + expectedPaths := make([]string, 0, len(customData.WriteFiles)) + for _, script := range expectedScripts { + expectedPaths = append(expectedPaths, script.path) + } + expectedPaths = append(expectedPaths, + "/etc/systemd/system/reconcile-private-hosts.service", + "/etc/systemd/system/kubelet.service", + "/opt/azure-network/configure-azure-network.sh", + "/etc/udev/rules.d/99-azure-network.rules", + ) + actualPaths := make([]string, 0, len(customData.WriteFiles)) + for _, file := range customData.WriteFiles { + actualPaths = append(actualPaths, file.Path) + } + Expect(actualPaths).To(Equal(expectedPaths)) + + filesByPath := make(map[string]cloudInitWriteFile, len(customData.WriteFiles)) + for _, file := range customData.WriteFiles { + filesByPath[file.Path] = file + Expect(file.Owner).To(Equal("root")) + } + for _, script := range expectedScripts { + file := filesByPath[script.path] + Expect(file.Permissions).To(Equal("0744")) + Expect(file.Encoding).To(BeEmpty()) + expectedContent, ok := variables[script.key].(string) + Expect(ok).To(BeTrue()) + Expect(file.Content).To(Equal(expectedContent), script.path) + } + expectedLiteralFiles := []struct { + path string + key string + permissions string + }{ + {"/etc/systemd/system/reconcile-private-hosts.service", "reconcilePrivateHostsService", "0644"}, + {"/etc/systemd/system/kubelet.service", "kubeletSystemdService", "0600"}, + {"/opt/azure-network/configure-azure-network.sh", "configureAzureNetworkScript", "0755"}, + {"/etc/udev/rules.d/99-azure-network.rules", "azureNetworkUdevRule", "0644"}, + } + for _, expected := range expectedLiteralFiles { + file := filesByPath[expected.path] + Expect(file.Permissions).To(Equal(expected.permissions)) + Expect(file.Encoding).To(BeEmpty()) + expectedContent, ok := variables[expected.key].(string) + Expect(ok).To(BeTrue()) + Expect(file.Content).To(Equal(expectedContent), expected.path) + } + By(fmt.Sprintf("%s customData size: encoded=%d decoded=%d", name, len(payload), len(customDataBytes))) + } + } + }) + It("should accept supported custom Linux transparent huge page values", func() { config.AgentPoolProfile.CustomLinuxOSConfig = &datamodel.CustomLinuxOSConfig{ TransparentHugePageEnabled: "never", diff --git a/pkg/agent/utils.go b/pkg/agent/utils.go index 7b6d4072fa9..d67838fd8cb 100644 --- a/pkg/agent/utils.go +++ b/pkg/agent/utils.go @@ -181,6 +181,10 @@ func escapeSingleLine(escapedStr string) string { // getBase64EncodedGzippedCustomScript will return a base64 of the CSE. func getBase64EncodedGzippedCustomScript(csFilename string, config *datamodel.NodeBootstrappingConfiguration) string { + return getBase64EncodedGzippedCustomScriptFromStr(getCustomScriptContent(csFilename, config)) +} + +func getCustomScriptContent(csFilename string, config *datamodel.NodeBootstrappingConfiguration) string { b, err := parts.Templates.ReadFile(csFilename) if err != nil { // this should never happen and this is a bug. @@ -202,7 +206,7 @@ func getBase64EncodedGzippedCustomScript(csFilename string, config *datamodel.No } csStr := buffer.String() csStr = strings.ReplaceAll(csStr, "\r\n", "\n") - return getBase64EncodedGzippedCustomScriptFromStr(csStr) + return csStr } // This is "best-effort" - removes MOST of the comments with obvious formats, to lower the space required by CustomData component. @@ -392,6 +396,14 @@ func getCustomDataFromJSON(jsonStr string) string { return customDataObj["customData"] } +func getCustomDataJSONObject(customData string) string { + customDataJSON, err := json.Marshal(map[string]string{"customData": customData}) + if err != nil { + panic(fmt.Errorf("failed to marshal customData: %w", err)) + } + return string(customDataJSON) +} + // GetOrderedKubeletConfigFlagString returns an ordered string of key/val pairs. // copied from AKS-Engine and filter out flags that already translated to config file. func GetOrderedKubeletConfigFlagString(config *datamodel.NodeBootstrappingConfiguration) string { diff --git a/pkg/agent/variables.go b/pkg/agent/variables.go index cfc83855285..9e701385c6b 100644 --- a/pkg/agent/variables.go +++ b/pkg/agent/variables.go @@ -13,37 +13,47 @@ import ( // getCustomDataVariables returns cloudinit data used by Linux. func getCustomDataVariables(config *datamodel.NodeBootstrappingConfiguration) paramsMap { cs := config.ContainerService + useCloudInitLiteralFiles := !config.IsFlatcar() && !config.IsACL() && !config.EnableScriptlessCSECmd + cloudInitFileData := paramsMap{} + getCloudInitFile := func(name, filename string) string { + content := getCustomScriptContent(filename, config) + if useCloudInitLiteralFiles { + cloudInitFileData[name] = content + return "" + } + return getBase64EncodedGzippedCustomScriptFromStr(content) + } cloudInitFiles := map[string]interface{}{ "cloudInitData": paramsMap{ - "provisionStartScript": getBase64EncodedGzippedCustomScript(kubernetesCSEStartScript, config), - "provisionScript": getBase64EncodedGzippedCustomScript(kubernetesCSEMainScript, config), - "provisionSource": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScript, config), - "provisionSourceUbuntu": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptUbuntu, config), - "provisionSourceMariner": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptMariner, config), - "provisionSourceAzlOSGuard": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptAzlOSGuard, config), - "provisionSourceFlatcar": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptFlatcar, config), - "provisionSourceACL": getBase64EncodedGzippedCustomScript(kubernetesCSEHelpersScriptACL, config), - "provisionInstalls": getBase64EncodedGzippedCustomScript(kubernetesCSEInstall, config), - "provisionInstallsUbuntu": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallUbuntu, config), - "provisionInstallsMariner": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallMariner, config), - "provisionInstallsAzlOSGuard": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallAzlOSGuard, config), - "provisionInstallsFlatcar": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallFlatcar, config), - "provisionInstallsACL": getBase64EncodedGzippedCustomScript(kubernetesCSEInstallACL, config), - "provisionConfigs": getBase64EncodedGzippedCustomScript(kubernetesCSEConfig, config), - "provisionConfigsGPU": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigGPU, config), - "provisionConfigsLocalDNS": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigLocalDNS, config), - "provisionConfigsKubelet": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigKubelet, config), - "provisionConfigsNetwork": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigNetwork, config), - "provisionConfigsAddons": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigAddons, config), - "provisionConfigsChrony": getBase64EncodedGzippedCustomScript(kubernetesCSEConfigChrony, config), + "provisionStartScript": getCloudInitFile("provisionStartScript", kubernetesCSEStartScript), + "provisionScript": getCloudInitFile("provisionScript", kubernetesCSEMainScript), + "provisionSource": getCloudInitFile("provisionSource", kubernetesCSEHelpersScript), + "provisionSourceUbuntu": getCloudInitFile("provisionSourceUbuntu", kubernetesCSEHelpersScriptUbuntu), + "provisionSourceMariner": getCloudInitFile("provisionSourceMariner", kubernetesCSEHelpersScriptMariner), + "provisionSourceAzlOSGuard": getCloudInitFile("provisionSourceAzlOSGuard", kubernetesCSEHelpersScriptAzlOSGuard), + "provisionSourceFlatcar": getCloudInitFile("provisionSourceFlatcar", kubernetesCSEHelpersScriptFlatcar), + "provisionSourceACL": getCloudInitFile("provisionSourceACL", kubernetesCSEHelpersScriptACL), + "provisionInstalls": getCloudInitFile("provisionInstalls", kubernetesCSEInstall), + "provisionInstallsUbuntu": getCloudInitFile("provisionInstallsUbuntu", kubernetesCSEInstallUbuntu), + "provisionInstallsMariner": getCloudInitFile("provisionInstallsMariner", kubernetesCSEInstallMariner), + "provisionInstallsAzlOSGuard": getCloudInitFile("provisionInstallsAzlOSGuard", kubernetesCSEInstallAzlOSGuard), + "provisionInstallsFlatcar": getCloudInitFile("provisionInstallsFlatcar", kubernetesCSEInstallFlatcar), + "provisionInstallsACL": getCloudInitFile("provisionInstallsACL", kubernetesCSEInstallACL), + "provisionConfigs": getCloudInitFile("provisionConfigs", kubernetesCSEConfig), + "provisionConfigsGPU": getCloudInitFile("provisionConfigsGPU", kubernetesCSEConfigGPU), + "provisionConfigsLocalDNS": getCloudInitFile("provisionConfigsLocalDNS", kubernetesCSEConfigLocalDNS), + "provisionConfigsKubelet": getCloudInitFile("provisionConfigsKubelet", kubernetesCSEConfigKubelet), + "provisionConfigsNetwork": getCloudInitFile("provisionConfigsNetwork", kubernetesCSEConfigNetwork), + "provisionConfigsAddons": getCloudInitFile("provisionConfigsAddons", kubernetesCSEConfigAddons), + "provisionConfigsChrony": getCloudInitFile("provisionConfigsChrony", kubernetesCSEConfigChrony), "provisionSendLogs": getBase64EncodedGzippedCustomScript(kubernetesCSESendLogs, config), "provisionRedactCloudConfig": getBase64EncodedGzippedCustomScript(kubernetesCSERedactCloudConfig, config), "customSearchDomainsScript": getBase64EncodedGzippedCustomScript(kubernetesCustomSearchDomainsScript, config), "dhcpv6SystemdService": getBase64EncodedGzippedCustomScript(dhcpv6SystemdService, config), "dhcpv6ConfigurationScript": getBase64EncodedGzippedCustomScript(dhcpv6ConfigurationScript, config), - "kubeletSystemdService": getBase64EncodedGzippedCustomScript(kubeletSystemdService, config), + "kubeletSystemdService": getCloudInitFile("kubeletSystemdService", kubeletSystemdService), "reconcilePrivateHostsScript": getBase64EncodedGzippedCustomScript(reconcilePrivateHostsScript, config), - "reconcilePrivateHostsService": getBase64EncodedGzippedCustomScript(reconcilePrivateHostsService, config), + "reconcilePrivateHostsService": getCloudInitFile("reconcilePrivateHostsService", reconcilePrivateHostsService), "ensureNoDupEbtablesScript": getBase64EncodedGzippedCustomScript(ensureNoDupEbtablesScript, config), "ensureNoDupEbtablesService": getBase64EncodedGzippedCustomScript(ensureNoDupEbtablesService, config), "bindMountScript": getBase64EncodedGzippedCustomScript(bindMountScript, config), @@ -63,13 +73,16 @@ func getCustomDataVariables(config *datamodel.NodeBootstrappingConfiguration) pa "cloudInitStatusCheckScript": getBase64EncodedGzippedCustomScript(cloudInitStatusCheckScript, config), "measureTLSBootstrappingLatencyScript": getBase64EncodedGzippedCustomScript(measureTLSBootstrappingLatencyScript, config), "measureTLSBootstrappingLatencyService": getBase64EncodedGzippedCustomScript(measureTLSBootstrappingLatencyService, config), - "configureAzureNetworkScript": getBase64EncodedGzippedCustomScript(configureAzureNetworkScript, config), - "azureNetworkUdevRule": getBase64EncodedGzippedCustomScript(azureNetworkUdevRule, config), + "configureAzureNetworkScript": getCloudInitFile("configureAzureNetworkScript", configureAzureNetworkScript), + "azureNetworkUdevRule": getCloudInitFile("azureNetworkUdevRule", azureNetworkUdevRule), }, } cloudInitData := cloudInitFiles["cloudInitData"].(paramsMap) //nolint:errcheck // no error is actually here - cloudInitData["initAKSCloud"] = getBase64EncodedGzippedCustomScript(initAKSCloudScript, config) + cloudInitData["initAKSCloud"] = getCloudInitFile("initAKSCloud", initAKSCloudScript) + if useCloudInitLiteralFiles { + cloudInitFiles["cloudInitFileData"] = cloudInitFileData + } if config.IsFlatcar() || config.IsACL() { cloudInitData["provisionRedactCloudConfig"] = "" // Flatcar and ACL do not have cloud-init From 894b94ebd3d9aeddf4cdef97491014d7590fa8bd Mon Sep 17 00:00:00 2001 From: Ben Brady <86134744+benjamin-brady@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:02:43 +1300 Subject: [PATCH 2/3] test(cloud-init): discover literal CSE scripts Keep transformed-script round-trip and bash syntax checks on scripts routed through getCloudInitFile. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 29247196-6c9e-475a-8892-77fc9c3316d4 --- pkg/agent/utils_test.go | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/pkg/agent/utils_test.go b/pkg/agent/utils_test.go index bf884dad2c1..b2fd4efb329 100644 --- a/pkg/agent/utils_test.go +++ b/pkg/agent/utils_test.go @@ -1284,8 +1284,9 @@ func TestRemoveComments_ShellPatterns(t *testing.T) { // syntax error after stripping. // // The script list is dynamically derived by parsing variables.go and const.go source -// to find all .sh files passed to getBase64EncodedGzippedCustomScript(). If a new script -// is added to the CSE pipeline, it is automatically covered by this test. +// to find all .sh files passed to getBase64EncodedGzippedCustomScript() or +// getCloudInitFile(). If a new script is added to the CSE pipeline, it is +// automatically covered by this test. func TestCSEScriptRoundTrip(t *testing.T) { cseScripts := discoverCSEScripts(t) if len(cseScripts) == 0 { @@ -1303,14 +1304,28 @@ func TestCSEScriptRoundTrip(t *testing.T) { } } +func TestDiscoverCSEScriptsIncludesLiteralAndGzipFiles(t *testing.T) { + scripts := discoverCSEScripts(t) + for _, path := range []string{ + kubernetesCSEMainScript, + kubernetesCSEHelpersScript, + kubernetesCSEInstall, + kubernetesCSEConfigGPU, + kubernetesCSEConfigChrony, + kubernetesCustomSearchDomainsScript, + } { + assert.Contains(t, scripts, path) + } +} + // discoverCSEScripts parses variables.go to find all constant names passed to -// getBase64EncodedGzippedCustomScript(), then resolves those constants to file -// paths from const.go, filtering to .sh files only. +// getBase64EncodedGzippedCustomScript() or getCloudInitFile(), then resolves +// those constants to file paths from const.go, filtering to .sh files only. func discoverCSEScripts(t *testing.T) []string { t.Helper() root := repoRoot() - // Step 1: Read variables.go and extract constant names from getBase64EncodedGzippedCustomScript() calls + // Step 1: Read variables.go and extract constant names from both file helpers variablesPath := filepath.Join(root, "pkg", "agent", "variables.go") variablesBytes, err := os.ReadFile(variablesPath) if err != nil { @@ -1318,7 +1333,8 @@ func discoverCSEScripts(t *testing.T) []string { } // Match: getBase64EncodedGzippedCustomScript(constantName, config) - callRe := regexp.MustCompile(`getBase64EncodedGzippedCustomScript\((\w+),`) + // or getCloudInitFile("key", constantName) + callRe := regexp.MustCompile(`(?:getBase64EncodedGzippedCustomScript\(\s*|getCloudInitFile\(\s*"[^"]+"\s*,\s*)(\w+)\s*[,)]`) matches := callRe.FindAllStringSubmatch(string(variablesBytes), -1) constNames := make(map[string]bool) for _, m := range matches { From 487af9ac03817b0ff6462b6e0b0b200547ba5152 Mon Sep 17 00:00:00 2001 From: Ben Brady <86134744+benjamin-brady@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:55:40 +1300 Subject: [PATCH 3/3] fix(cloud-init): preserve legacy gzip lookup Only compress a raw script when a legacy template requests cloudInitData. Keep the literal CustomData path free of nested compression. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 29247196-6c9e-475a-8892-77fc9c3316d4 --- pkg/agent/baker.go | 26 ++++++++++++++++++-------- pkg/agent/baker_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 65570512085..1f272043feb 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -713,19 +713,29 @@ func getBakerFuncMap(config *datamodel.NodeBootstrappingConfiguration, params pa } funcMap["GetVariableProperty"] = func(v, p string) interface{} { - if v, ok := variables[v].(paramsMap); ok && v != nil { - if v[p] == nil { - // return empty string so we don't get from go template - return "" - } - return v[p] - } - return "" + return getVariableProperty(variables, v, p) } return funcMap } +func getVariableProperty(variables paramsMap, variable, name string) interface{} { + if variable == "cloudInitData" { + if files, ok := variables["cloudInitFileData"].(paramsMap); ok { + if content, ok := files[name].(string); ok { + return getBase64EncodedGzippedCustomScriptFromStr(content) + } + } + } + if values, ok := variables[variable].(paramsMap); ok && values != nil { + if values[name] == nil { + return "" + } + return values[name] + } + return "" +} + func getCloudInitFileProperties(variables paramsMap, name string) (string, error) { if files, ok := variables["cloudInitFileData"].(paramsMap); ok { if content, ok := files[name].(string); ok { diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index 4d521a71974..5d1ce6531f0 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -131,6 +131,40 @@ func TestRenderCloudInitFileContentRoundTrips(t *testing.T) { } } +func TestRenderLinuxNodeCustomDataTemplateLegacyGzip(t *testing.T) { + for _, testCase := range []struct { + name string + distro datamodel.Distro + key string + filename string + }{ + {"Ubuntu", datamodel.AKSUbuntuContainerd2204Gen2, "provisionSourceUbuntu", kubernetesCSEHelpersScriptUbuntu}, + {"Azure Linux", datamodel.AKSAzureLinuxV3Gen2, "provisionSourceMariner", kubernetesCSEHelpersScriptMariner}, + } { + t.Run(testCase.name, func(t *testing.T) { + config := newNodeCustomDataRenderConfig(testCase.distro) + template := fmt.Appendf(nil, `#cloud-config +write_files: +- path: /opt/script.sh + owner: root + encoding: gzip + content: !!binary | + {{GetVariableProperty "cloudInitData" %q}} +`, testCase.key) + rendered, err := RenderLinuxNodeCustomDataTemplate(template, config) + require.NoError(t, err) + + var parsed cloudInit + require.NoError(t, yaml.Unmarshal([]byte(rendered), &parsed)) + require.Len(t, parsed.WriteFiles, 1) + require.Equal(t, "gzip", parsed.WriteFiles[0].Encoding) + decoded, err := getGzipDecodedValue([]byte(parsed.WriteFiles[0].Content)) + require.NoError(t, err) + require.Equal(t, getCustomScriptContent(testCase.filename, config), string(decoded)) + }) + } +} + func TestGetCloudInitFilePropertiesKeepsGzipFallback(t *testing.T) { testCases := []struct { name string