fix: reduce legacy Linux custom data encoding overhead

hotfix/hotfix_generate_test.py

@@ -161,7 +161,8 @@ def test_config_modules_are_independently_hotfixable(self):
                 self.assertEqual({variable}, blocks[0][0])
                 self.assertIn(f"- path: {{{{{path_function}}}}}", template)
                 self.assertIn('permissions: "0744"', template)
-                self.assertIn("encoding: gzip", template)
+                self.assertIn("{{GetCloudInitFileEncodingLine}}", template)
+                self.assertIn("| GetCloudInitFileContent}}", template)
                 self.assertNotIn("{{if", template)
 
     def test_mariner_package_update_is_not_embedded_in_custom_data(self):

hotfix/render-nodecustomdata/main_test.go

@@ -18,18 +18,16 @@ func TestRenderSelectedHotfixScripts(t *testing.T) {
 	const initCloudBlock = `{{if IsAKSCustomCloud}}
 - path: {{GetInitAKSCloudFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "initAKSCloud"}}
+  content: {{GetVariableProperty "cloudInitData" "initAKSCloud" | GetCloudInitFileContent}}
 {{end}}
 `
 	const helperBlock = `- path: {{GetCSEHelpersScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSource"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionSource" | GetCloudInitFileContent}}
 `
 	for _, distro := range []datamodel.Distro{
 		datamodel.AKSUbuntuContainerd2204Gen2,
@@ -47,8 +45,9 @@ func TestRenderSelectedHotfixScripts(t *testing.T) {
 				require.NoError(t, err)
 				var payload struct {
 					WriteFiles []struct {
-						Path    string `yaml:"path"`
-						Content string `yaml:"content"`
+						Path     string `yaml:"path"`
+						Encoding string `yaml:"encoding"`
+						Content  string `yaml:"content"`
 					} `yaml:"write_files"`
 				}
 				require.NoError(t, yaml.Unmarshal([]byte(rendered), &payload))
@@ -62,6 +61,7 @@ func TestRenderSelectedHotfixScripts(t *testing.T) {
 				require.Len(t, payload.WriteFiles, 1+wantInitCount)
 				for _, file := range payload.WriteFiles {
 					require.NotEmpty(t, file.Content)
+					require.Empty(t, file.Encoding)
 				}
 			})
 		}

parts/linux/cloud-init/nodecustomdata.yml

@@ -28,174 +28,151 @@ write_files:
 {{if IsAKSCustomCloud}}
 - path: {{GetInitAKSCloudFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "initAKSCloud"}}
+  content: {{GetVariableProperty "cloudInitData" "initAKSCloud" | GetCloudInitFileContent}}
 {{end}}
 
 {{- else }}
 - path: {{GetCSEHelpersScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSource"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionSource" | GetCloudInitFileContent}}
 
 
 {{if IsACL }}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceACL"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionSourceACL" | GetCloudInitFileContent}}
 {{- else if IsAzlOSGuard}}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceAzlOSGuard"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionSourceAzlOSGuard" | GetCloudInitFileContent}}
 {{- else if IsMariner}}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceMariner"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionSourceMariner" | GetCloudInitFileContent}}
 {{- else if IsFlatcar }}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceFlatcar"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionSourceFlatcar" | GetCloudInitFileContent}}
 {{- else }}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceUbuntu"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionSourceUbuntu" | GetCloudInitFileContent}}
 {{end}}
 
 {{ if not IsCustomImage -}}
 - path: /opt/azure/containers/provision_start.sh
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionStartScript"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionStartScript" | GetCloudInitFileContent}}
 {{- end }}
 
 - path: /opt/azure/containers/provision.sh
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionScript"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionScript" | GetCloudInitFileContent}}
 
 - path: {{GetCSEInstallScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstalls"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionInstalls" | GetCloudInitFileContent}}
 
 {{if IsACL }}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsACL"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionInstallsACL" | GetCloudInitFileContent}}
 {{- else if IsAzlOSGuard}}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsAzlOSGuard"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionInstallsAzlOSGuard" | GetCloudInitFileContent}}
 {{- else if IsMariner}}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsMariner"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionInstallsMariner" | GetCloudInitFileContent}}
 {{- else if IsFlatcar }}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsFlatcar"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionInstallsFlatcar" | GetCloudInitFileContent}}
 {{- else }}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsUbuntu"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionInstallsUbuntu" | GetCloudInitFileContent}}
 {{end}}
 
 - path: {{GetCSEConfigScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionConfigs"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionConfigs" | GetCloudInitFileContent}}
 
 - path: {{GetCSEConfigGPUScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionConfigsGPU"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionConfigsGPU" | GetCloudInitFileContent}}
 
 - path: {{GetCSEConfigLocalDNSScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionConfigsLocalDNS"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionConfigsLocalDNS" | GetCloudInitFileContent}}
 
 - path: {{GetCSEConfigKubeletScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionConfigsKubelet"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionConfigsKubelet" | GetCloudInitFileContent}}
 
 - path: {{GetCSEConfigNetworkScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionConfigsNetwork"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionConfigsNetwork" | GetCloudInitFileContent}}
 
 - path: {{GetCSEConfigChronyScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionConfigsChrony"}}
+  content: {{GetVariableProperty "cloudInitData" "provisionConfigsChrony" | GetCloudInitFileContent}}
 
 {{if IsAKSCustomCloud}}
 - path: {{GetInitAKSCloudFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "initAKSCloud"}}
+  content: {{GetVariableProperty "cloudInitData" "initAKSCloud" | GetCloudInitFileContent}}
 {{end}}
 
 - path: /etc/systemd/system/kubelet.service
   permissions: "0600"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "kubeletSystemdService"}}
+  content: {{GetVariableProperty "cloudInitData" "kubeletSystemdService" | GetCloudInitFileContent}}
 
 {{- end }}

pkg/agent/baker.go

@@ -17,6 +17,7 @@ import (
 	"strconv"
 	"strings"
 	"text/template"
+	"unicode/utf8"
 
 	"github.com/Azure/agentbaker/parts"
 	"github.com/Azure/agentbaker/pkg/agent/datamodel"
@@ -164,14 +165,12 @@ func (t *TemplateGenerator) getLinuxNodeBootstrappingPayload(config *datamodel.N
 		return t.getScriptlessBoothook(config)
 	}
 
-	// this might seem strange that we're encoding the custom data to a JSON string and then extracting it, but without that serialisation and deserialisation
-	// lots of tests fail.
 	var encoded string
 	if config.IsFlatcar() || config.IsACL() {
 		customData := getCustomDataFromJSON(t.getFlatcarLinuxNodeCustomDataJSONObject(config))
 		encoded = base64.StdEncoding.EncodeToString([]byte(customData))
 	} else {
-		customData := getCustomDataFromJSON(t.getLinuxNodeCustomDataJSONObject(config))
+		customData := t.getLinuxNodeCustomData(config)
 		encoded = getBase64EncodedGzippedCustomScriptFromStr(customData)
 	}
 	return encoded
@@ -188,7 +187,7 @@ type encodedFile struct {
 func (t *TemplateGenerator) getScriptlessBoothook(config *datamodel.NodeBootstrappingConfiguration) string {
 	config.DisableCustomData = true
 	config.EnableScriptlessCSECmd = true
-	nodeCustomData := getCustomDataFromJSON(t.getLinuxNodeCustomDataJSONObject(config))
+	nodeCustomData := t.getLinuxNodeCustomData(config)
 	encodedNodeCustomData := getBase64EncodedGzippedCustomScriptFromStr(nodeCustomData)
 
 	// hotfixJSONFile is optional: only VHDs that bake a static default hotfix
@@ -343,20 +342,17 @@ func buildScriptlessCustomData(cloudInitTemplate, fileListTemplate, separator st
 	return fmt.Sprintf(cloudInitTemplate, strings.Join(fileList, separator))
 }
 
-// GetLinuxNodeCustomDataJSONObject returns Linux customData JSON object in the form.
-// { "customData": "<customData string>" }.
-func (t *TemplateGenerator) getLinuxNodeCustomDataJSONObject(config *datamodel.NodeBootstrappingConfiguration) string {
-	// get parameters
-	parameters := getParameters(config)
-	// get variable cloudInit
-	variables := getCustomDataVariables(config)
-	str, e := t.getSingleLineForTemplate(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, getBakerFuncMap(config, parameters, variables), true)
-
-	if e != nil {
-		panic(e)
+func (t *TemplateGenerator) getLinuxNodeCustomData(config *datamodel.NodeBootstrappingConfiguration) string {
+	b, err := parts.Templates.ReadFile(kubernetesNodeCustomDataYaml)
+	if err != nil {
+		panic(err)
+	}
+	str, err := RenderLinuxNodeCustomDataTemplate(b, config)
+	if err != nil {
+		panic(err)
 	}
 
-	return fmt.Sprintf("{\"customData\": \"%s\"}", str)
+	return str
 }
 
 const (
@@ -674,6 +670,8 @@ func (t *TemplateGenerator) getSingleLine(textFilename string, profile interface
 // same variables and functions used by the production AgentBaker path.
 // Callers must supply a complete configuration, as required by the production
 // rendering helpers.
+// File entries must use GetCloudInitFileEncodingLine and GetCloudInitFileContent
+// to select the encoding for the delivery mode.
 func RenderLinuxNodeCustomDataTemplate(templateContent []byte, config *datamodel.NodeBootstrappingConfiguration) (string, error) {
 	parameters := getParameters(config)
 	variables := getCustomDataVariables(config)
@@ -691,10 +689,46 @@ func RenderLinuxNodeCustomDataTemplate(templateContent []byte, config *datamodel
 	return buffer.String(), nil
 }
 
+// Traditional custom data is gzipped as a whole, so its files need no inner encoding.
+func usePlainCloudInit(config *datamodel.NodeBootstrappingConfiguration) bool {
+	return !config.EnableScriptlessCSECmd && !config.IsFlatcar() && !config.IsACL()
+}
+
+func getCloudInitFileEncodingLine(config *datamodel.NodeBootstrappingConfiguration) string {
+	if usePlainCloudInit(config) {
+		return ""
+	}
+	return "encoding: " + encodingGZIP
+}
+
+func getCloudInitFileContent(config *datamodel.NodeBootstrappingConfiguration, content string) (string, error) {
+	if usePlainCloudInit(config) {
+		if !utf8.ValidString(content) {
+			return "", fmt.Errorf("cloud-init file content contains invalid UTF-8")
+		}
+		// YAML quoting preserves Unicode line breaks that JSON leaves unescaped.
+		encoded, err := yaml.Marshal(&yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Style: yaml.DoubleQuotedStyle,
+			Value: content,
+		})
+		return strings.TrimSuffix(string(encoded), "\n"), err
+	}
+	return "!!binary |\n    " + getBase64EncodedGzippedCustomScriptFromStr(content), nil
+}
+
 // getTemplateFuncMap returns the general purpose template func map from getContainerServiceFuncMap.
 func getBakerFuncMap(config *datamodel.NodeBootstrappingConfiguration, params paramsMap, variables paramsMap) template.FuncMap {
 	funcMap := getContainerServiceFuncMap(config)
 
+	funcMap["GetCloudInitFileEncodingLine"] = func() string {
+		return getCloudInitFileEncodingLine(config)
+	}
+	funcMap["GetCloudInitFileContent"] = func(content string) (string, error) {
+		return getCloudInitFileContent(config, content)
+	}
+
 	funcMap["GetParameter"] = func(s string) interface{} {
 		if v, ok := params[s].(paramsMap); ok && v != nil {
 			if v["value"] == nil {