fix: reduce Linux custom data encoding overhead

hotfix/hotfix_generate.py

@@ -421,7 +421,10 @@ def parse_write_files_blocks(traditional_lines):
 
         if in_block:
             current_block.append(line)
-            match = re.search(r'GetVariableProperty\s+"cloudInitData"\s+"(\w+)"', stripped)
+            match = re.search(
+                r'GetVariableProperty\s+"cloudInit(?:File)?Data"\s+"(\w+)"',
+                stripped,
+            )
             if match:
                 current_varkeys.add(match.group(1))
 

hotfix/hotfix_generate_test.py

@@ -65,7 +65,7 @@ def test_production_renderer_omits_empty_platform_payloads(self):
         cases = [
             ("custom cloud only", hotfix_generate.build_hotfix_template(
                 {"initAKSCloud"}, lines[outer_else + 1:end]
-            ), set()),
+            ), {"ubuntu", "mariner"}),
             ("missing", "#cloud-config\n", set()),
             ("null", "write_files:\n", set()),
             ("empty list", "write_files: []\n", set()),
@@ -93,7 +93,7 @@ def test_production_renderer_omits_empty_platform_payloads(self):
                     f"rendered_nodecustomdata_{platform}.yml" for platform in platforms
                 } or {"README"}
                 self.assertEqual(expected, {path.name for path in output.iterdir()})
-                if name == "custom cloud only":
+                if name == "missing":
                     probe = Path(directory) / "main.go"
                     probe.write_text('package main\nimport "embed"\n'
                                      '//go:embed generated\nvar files embed.FS\n'
@@ -161,7 +161,8 @@ def test_config_modules_are_independently_hotfixable(self):
                 self.assertEqual({variable}, blocks[0][0])
                 self.assertIn(f"- path: {{{{{path_function}}}}}", template)
                 self.assertIn('permissions: "0744"', template)
-                self.assertIn("encoding: gzip", template)
+                self.assertIn("{{GetCloudInitFileEncodingLine}}", template)
+                self.assertIn("| GetCloudInitFileContent}}", template)
                 self.assertNotIn("{{if", template)
 
     def test_mariner_package_update_is_not_embedded_in_custom_data(self):
@@ -244,6 +245,22 @@ def test_build_hotfix_template_selects_only_requested_blocks(self):
         self.assertIn("provisionSource", rendered)
         self.assertNotIn("provisionSourceUbuntu", rendered)
 
+    def test_parse_write_files_blocks_accepts_legacy_and_current_maps(self):
+        for variable_map in ("cloudInitData", "cloudInitFileData"):
+            with self.subTest(variable_map=variable_map):
+                source = TRADITIONAL_TEMPLATE.replace("cloudInitData", variable_map)
+                blocks = hotfix_generate.parse_write_files_blocks(
+                    source.splitlines(keepends=True)
+                )
+                self.assertEqual(2, len(blocks))
+                self.assertEqual({"provisionSource"}, blocks[0][0])
+                self.assertIn("provisionSourceUbuntu", blocks[1][0])
+                selected = hotfix_generate.build_hotfix_template(
+                    {"provisionSource"}, source.splitlines(keepends=True)
+                )
+                self.assertIn(f'"{variable_map}" "provisionSource"', selected)
+                self.assertNotIn("provisionSourceUbuntu", selected)
+
     def test_detect_changed_varkeys_expands_distro_group(self):
         with tempfile.TemporaryDirectory() as temp_dir:
             artifacts = Path(temp_dir)

parts/linux/cloud-init/artifacts/README.MD

@@ -3,6 +3,19 @@ For tutorial for the descriptions of components.json, please refer to this [doc]
 For CA refresh logging and Linux opt-in certificate diagnostics, see
 [CA refresh diagnostics](../../../../doc/ca-refresh.md).
 
+## CustomData file encoding
+
+Traditional Linux CustomData is gzip-compressed once around the whole
+cloud-config document. Its `write_files` content uses YAML double-quoted
+scalars to preserve script bytes, including whitespace and Unicode line
+separators. Flatcar, ACL, and scriptless delivery retain per-file gzip encoding.
+This changes packaging only; no new VHD binary or guest-side checksum gate is
+required.
+
+Template entries use `cloudInitFileData` with `GetCloudInitFileEncodingLine`
+and `GetCloudInitFileContent`. Legacy `cloudInitData` lookups still return
+gzip/base64 content, including when mixed with new entries in a hotfix template.
+
 ## SSH-disable telemetry
 
 When `DISABLE_SSH=true` and `basePrep` runs, `disableSSH` emits structured

parts/linux/cloud-init/nodecustomdata.yml

@@ -28,201 +28,174 @@ write_files:
 {{if IsAKSCustomCloud}}
 - path: {{GetInitAKSCloudFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "initAKSCloud"}}
+  content: {{GetVariableProperty "cloudInitFileData" "initAKSCloud" | GetCloudInitFileContent}}
 {{end}}
 
 {{- else }}
 - path: {{GetCSEHelpersScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSource"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionSource" | GetCloudInitFileContent}}
 
 
 {{if IsACL }}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceACL"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionSourceACL" | GetCloudInitFileContent}}
 {{- else if IsAzlOSGuard}}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceAzlOSGuard"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionSourceAzlOSGuard" | GetCloudInitFileContent}}
 {{- else if IsMariner}}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceMariner"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionSourceMariner" | GetCloudInitFileContent}}
 {{- else if IsFlatcar }}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceFlatcar"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionSourceFlatcar" | GetCloudInitFileContent}}
 {{- else }}
 - path: {{GetCSEHelpersScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionSourceUbuntu"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionSourceUbuntu" | GetCloudInitFileContent}}
 {{end}}
 
 {{ if not IsCustomImage -}}
 - path: /opt/azure/containers/provision_start.sh
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionStartScript"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionStartScript" | GetCloudInitFileContent}}
 {{- end }}
 
 - path: /opt/azure/containers/provision.sh
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionScript"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionScript" | GetCloudInitFileContent}}
 
 - path: {{GetCSEInstallScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstalls"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionInstalls" | GetCloudInitFileContent}}
 
 {{if IsACL }}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsACL"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionInstallsACL" | GetCloudInitFileContent}}
 {{- else if IsAzlOSGuard}}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsAzlOSGuard"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionInstallsAzlOSGuard" | GetCloudInitFileContent}}
 {{- else if IsMariner}}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsMariner"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionInstallsMariner" | GetCloudInitFileContent}}
 {{- else if IsFlatcar }}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsFlatcar"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionInstallsFlatcar" | GetCloudInitFileContent}}
 {{- else }}
 - path: {{GetCSEInstallScriptDistroFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionInstallsUbuntu"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionInstallsUbuntu" | GetCloudInitFileContent}}
 {{end}}
 
 - path: {{GetCSEConfigScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionConfigs"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionConfigs" | GetCloudInitFileContent}}
 
 - path: {{GetCSEConfigGPUScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionConfigsGPU"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionConfigsGPU" | GetCloudInitFileContent}}
 
 - path: {{GetCSEConfigLocalDNSScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionConfigsLocalDNS"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionConfigsLocalDNS" | GetCloudInitFileContent}}
 
 - path: {{GetCSEConfigKubeletScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionConfigsKubelet"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionConfigsKubelet" | GetCloudInitFileContent}}
 
 - path: {{GetCSEConfigNetworkScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionConfigsNetwork"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionConfigsNetwork" | GetCloudInitFileContent}}
 
 - path: {{GetCSEConfigAddonsScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionConfigsAddons"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionConfigsAddons" | GetCloudInitFileContent}}
 
 - path: {{GetCSEConfigChronyScriptFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "provisionConfigsChrony"}}
+  content: {{GetVariableProperty "cloudInitFileData" "provisionConfigsChrony" | GetCloudInitFileContent}}
 
 {{if IsAKSCustomCloud}}
 - path: {{GetInitAKSCloudFilepath}}
   permissions: "0744"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "initAKSCloud"}}
+  content: {{GetVariableProperty "cloudInitFileData" "initAKSCloud" | GetCloudInitFileContent}}
 {{end}}
 
 - path: /etc/systemd/system/reconcile-private-hosts.service
   permissions: "0644"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "reconcilePrivateHostsService"}}
+  content: {{GetVariableProperty "cloudInitFileData" "reconcilePrivateHostsService" | GetCloudInitFileContent}}
 
 - path: /etc/systemd/system/kubelet.service
   permissions: "0600"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "kubeletSystemdService"}}
+  content: {{GetVariableProperty "cloudInitFileData" "kubeletSystemdService" | GetCloudInitFileContent}}
 
 - path: /opt/azure-network/configure-azure-network.sh
   permissions: "0755"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "configureAzureNetworkScript"}}
+  content: {{GetVariableProperty "cloudInitFileData" "configureAzureNetworkScript" | GetCloudInitFileContent}}
 
 - path: /etc/udev/rules.d/99-azure-network.rules
   permissions: "0644"
-  encoding: gzip
+  {{GetCloudInitFileEncodingLine}}
   owner: root
-  content: !!binary |
-    {{GetVariableProperty "cloudInitData" "azureNetworkUdevRule"}}
+  content: {{GetVariableProperty "cloudInitFileData" "azureNetworkUdevRule" | GetCloudInitFileContent}}
 {{- end }}