diff --git a/.github/README-RENOVATE.md b/.github/README-RENOVATE.md index cc5710a9f9e..c4b58a893b6 100644 --- a/.github/README-RENOVATE.md +++ b/.github/README-RENOVATE.md @@ -204,7 +204,7 @@ We have some custom data sources in the renovate.json now. Let's walk through an "defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/24.04/prod/dists/noble/main/binary-amd64/Packages", "format": "plain", "transformTemplates": [ - "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" + "{\"releases\": [$split($replace($join(releases.version, \"\\n\"), /^\\n+/, \"\"), /\\n{2,}/)[$substringBefore($, \"\\n\") = \"Package: {{packageName}}\"].{\"version\": $substringAfter($split($, \"\\n\")[1], \"Version: \")}]}" ] } ``` @@ -286,6 +286,10 @@ Renovate will actually convert this plain text file to a JSON file which looks l ``` So the JSONata query you are working on, should take this as inputs, instead of the raw content. +For the Ubuntu PMC and NVIDIA Debian indexes, the transform joins the trimmed lines once and splits them into package stanzas at one or more blank lines. Leading blank lines are removed before splitting. It matches the exact `Package:` header, then reads the following `Version:` line within each matching stanza, preserving feed order. This relies on the same `Package:`-then-`Version:` arrangement as the previous transform. The outer array keeps `releases` an array even for zero or one matching version; no matches produce `{"releases": []}`. + +Avoid looking up `releases[$i + 1]` for every matching package line: JSONata repeatedly traverses the full line sequence for these positional lookups, making large indexes expensive. Keep version extraction local to the matching stanza instead. Renovate's plain fetcher trims each line (including CRLF line endings) and preserves blank lines, so whitespace-only separators also become stanza boundaries. + How do I know that? By reading the Renovate trace carefully you will find it's actually trying to parse this JSON file. If the raw content from your `transformTemplates` is not in plain text but other format, I would still highly recommend you enable the Renovate trace so that you can see what is happening behing the scene. diff --git a/.github/renovate.json b/.github/renovate.json index 9cbb2b47314..d08c9ad0686 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -1028,42 +1028,42 @@ "defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/20.04/prod/dists/focal/main/binary-amd64/Packages", "format": "plain", "transformTemplates": [ - "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" + "{\"releases\": [$split($replace($join(releases.version, \"\\n\"), /^\\n+/, \"\"), /\\n{2,}/)[$substringBefore($, \"\\n\") = \"Package: {{packageName}}\"].{\"version\": $substringAfter($split($, \"\\n\")[1], \"Version: \")}]}" ] }, "deb2204": { "defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/22.04/prod/dists/jammy/main/binary-amd64/Packages", "format": "plain", "transformTemplates": [ - "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" + "{\"releases\": [$split($replace($join(releases.version, \"\\n\"), /^\\n+/, \"\"), /\\n{2,}/)[$substringBefore($, \"\\n\") = \"Package: {{packageName}}\"].{\"version\": $substringAfter($split($, \"\\n\")[1], \"Version: \")}]}" ] }, "nvidia-deb2204": { "defaultRegistryUrlTemplate": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/Packages", "format": "plain", "transformTemplates": [ - "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" + "{\"releases\": [$split($replace($join(releases.version, \"\\n\"), /^\\n+/, \"\"), /\\n{2,}/)[$substringBefore($, \"\\n\") = \"Package: {{packageName}}\"].{\"version\": $substringAfter($split($, \"\\n\")[1], \"Version: \")}]}" ] }, "deb2404": { "defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/24.04/prod/dists/noble/main/binary-amd64/Packages", "format": "plain", "transformTemplates": [ - "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" + "{\"releases\": [$split($replace($join(releases.version, \"\\n\"), /^\\n+/, \"\"), /\\n{2,}/)[$substringBefore($, \"\\n\") = \"Package: {{packageName}}\"].{\"version\": $substringAfter($split($, \"\\n\")[1], \"Version: \")}]}" ] }, "nvidia-deb2404": { "defaultRegistryUrlTemplate": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/Packages", "format": "plain", "transformTemplates": [ - "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" + "{\"releases\": [$split($replace($join(releases.version, \"\\n\"), /^\\n+/, \"\"), /\\n{2,}/)[$substringBefore($, \"\\n\") = \"Package: {{packageName}}\"].{\"version\": $substringAfter($split($, \"\\n\")[1], \"Version: \")}]}" ] }, "nvidia-deb2604": { "defaultRegistryUrlTemplate": "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2604/x86_64/Packages", "format": "plain", "transformTemplates": [ - "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" + "{\"releases\": [$split($replace($join(releases.version, \"\\n\"), /^\\n+/, \"\"), /\\n{2,}/)[$substringBefore($, \"\\n\") = \"Package: {{packageName}}\"].{\"version\": $substringAfter($split($, \"\\n\")[1], \"Version: \")}]}" ] }, "nvidia-rpm-azl3": { @@ -1077,14 +1077,14 @@ "defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/24.04/prod/dists/testing/main/binary-amd64/Packages", "format": "plain", "transformTemplates": [ - "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" + "{\"releases\": [$split($replace($join(releases.version, \"\\n\"), /^\\n+/, \"\"), /\\n{2,}/)[$substringBefore($, \"\\n\") = \"Package: {{packageName}}\"].{\"version\": $substringAfter($split($, \"\\n\")[1], \"Version: \")}]}" ] }, "deb2604": { "defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/26.04/prod/dists/resolute/main/binary-amd64/Packages", "format": "plain", "transformTemplates": [ - "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $substringAfter(releases[$i + 1].version, \"Version: \") })), function($v) { {\"version\": $v} })[]}" + "{\"releases\": [$split($replace($join(releases.version, \"\\n\"), /^\\n+/, \"\"), /\\n{2,}/)[$substringBefore($, \"\\n\") = \"Package: {{packageName}}\"].{\"version\": $substringAfter($split($, \"\\n\")[1], \"Version: \")}]}" ] } } diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index a64e0ce17d9..5ba3ae4596b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -50,7 +50,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -64,7 +64,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/autobuild@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 # ℹī¸ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -77,4 +77,4 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 + uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 30ddd043a62..ee175c7a279 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -25,6 +25,8 @@ jobs: with: go-version: '1.27' - name: Run unit tests for go code in the repository (with coverage) + env: + CSE_SCRIPT_DIFF_DIR: ${{ runner.temp }}/cse-minification-diffs # Single instrumented pass instead of `make test` followed by a second coverage run: # this builds and tests each module once, mirroring what `make test` does, while also # producing coverage profiles. @@ -36,6 +38,15 @@ jobs: go test $(go list ./... | grep -v e2e) -coverprofile=coverage/agentbaker.out -covermode=atomic (cd aks-node-controller && go build -mod=readonly ./... && go test ./... -coverprofile=../coverage/aks-node-controller.out -covermode=atomic) (cd aks-live-patching && go build -mod=readonly ./... && go test ./... -coverprofile=../coverage/aks-live-patching.out -covermode=atomic) + (cd vhdbuilder/cosi-uploader && go build -mod=readonly ./... && go test ./... -coverprofile=../../coverage/cosi-uploader.out -covermode=atomic) + - name: Upload CSE minification diffs + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + if: (!cancelled()) + with: + name: cse-minification-diffs + path: ${{ runner.temp }}/cse-minification-diffs + if-no-files-found: error + retention-days: 5 - name: Strip generated/mock code from coverage profiles run: | set -ex diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index af34812dc6a..c4ce46bc59a 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - dirs: [".", "aks-node-controller", "aks-live-patching"] + dirs: [".", "aks-node-controller", "aks-live-patching", "vhdbuilder/cosi-uploader"] steps: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: diff --git a/.github/workflows/tidy.yaml b/.github/workflows/tidy.yaml index bc236871e01..887c48c4318 100644 --- a/.github/workflows/tidy.yaml +++ b/.github/workflows/tidy.yaml @@ -11,6 +11,7 @@ on: # zizmor: ignore[dangerous-triggers] intentional: pull_request_target is req - e2e/go.mod - aks-node-controller/go.mod - aks-live-patching/go.mod + - vhdbuilder/cosi-uploader/go.mod - .github/workflows/tidy.yaml permissions: @@ -56,7 +57,7 @@ jobs: - name: Run go mod tidy run: | set -e - for mod_dir in ./ e2e/ aks-node-controller/ aks-live-patching/; do + for mod_dir in ./ e2e/ aks-node-controller/ aks-live-patching/ vhdbuilder/cosi-uploader/; do echo "running go mod tidy in ${mod_dir}" pushd "$mod_dir" go mod tidy @@ -74,7 +75,7 @@ jobs: name: Remove toolchain lines run: | set -e - for mod_dir in ./ e2e/ aks-node-controller/ aks-live-patching/; do + for mod_dir in ./ e2e/ aks-node-controller/ aks-live-patching/ vhdbuilder/cosi-uploader/; do echo "removing any toolchain lines from go.mod in ${mod_dir}" pushd "$mod_dir" # this pains me greatly, though adding the toolchain line seems to break our other workflows, diff --git a/.github/workflows/yamllint.yml b/.github/workflows/yamllint.yml index 9e1cc1a124f..03695389d9a 100644 --- a/.github/workflows/yamllint.yml +++ b/.github/workflows/yamllint.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: '3.x' - name: Install yamllint diff --git a/.gitignore b/.gitignore index 85d467e7468..55ecf39fff9 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,9 @@ e2e/scenario-logs/ # vhdbuilder generated prefetch scripts vhdbuilder/packer/prefetch.sh +# default `go build` output inside the COSI uploader module; packer.mk builds into bin/ +vhdbuilder/cosi-uploader/cosi-uploader + # ignore mise configs (https://mise.jdx.dev/) .mise diff --git a/.pipelines/.vsts-e2e-garbage-collection.yaml b/.pipelines/.vsts-e2e-garbage-collection.yaml new file mode 100644 index 00000000000..cce06449863 --- /dev/null +++ b/.pipelines/.vsts-e2e-garbage-collection.yaml @@ -0,0 +1,38 @@ +trigger: none +pr: none + +parameters: + - name: DRY_RUN + displayName: Run in dry-run mode + type: boolean + default: false + - name: NO_WAIT + displayName: Wait for resource deletion + type: boolean + default: true + +pool: + name: $(POOL_NAME) + +stages: + - stage: e2e_gc_msft + displayName: MSFT Tenant + dependsOn: [] + jobs: + - template: ./templates/e2e-gc-template.yaml + parameters: + ARM_SERVICE_CONNECTION: $(MSFT_ARM_SERVICE_CONNECTION) + SUBSCRIPTION_ID: $(MSFT_E2E_SUBSCRIPTION_ID) + DRY_RUN: ${{ parameters.DRY_RUN }} + NO_WAIT: ${{ parameters.NO_WAIT }} + + - stage: e2e_gc_tme + displayName: TME Tenant + dependsOn: [] + jobs: + - template: ./templates/e2e-gc-template.yaml + parameters: + ARM_SERVICE_CONNECTION: $(TME_ARM_SERVICE_CONNECTION) + SUBSCRIPTION_ID: $(TME_E2E_SUBSCRIPTION_ID) + DRY_RUN: ${{ parameters.DRY_RUN }} + NO_WAIT: ${{ parameters.NO_WAIT }} diff --git a/.pipelines/.vsts-vhd-builder-pr-windows.yaml b/.pipelines/.vsts-vhd-builder-pr-windows.yaml index 591414b307a..95917727941 100644 --- a/.pipelines/.vsts-vhd-builder-pr-windows.yaml +++ b/.pipelines/.vsts-vhd-builder-pr-windows.yaml @@ -72,9 +72,9 @@ stages: useOverrides: False enableBackfillCleanup: True buildVmSize: Standard_D16ds_v5 - # 2022 gen1 is the default, so build that for PRs + # Keep the default Gen1 image and cover both security modes on Gen2. build2022containerd: True - build2022containerdgen2: False + build2022containerdgen2: True # 2025 gen2 is the default, so build that for PRs build2025: False build2025gen2: True diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index b4e213baf34..b7c7cd66d37 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -5,6 +5,27 @@ pool: name: $(POOL_NAME) parameters: + - name: ACL_BASE_IMAGE_SOURCE + type: string + default: acg + values: + - acg + - acldevel + # The Run pipeline form marks a string parameter whose default is "" as required, + # so these default to a single space. Leave the space to keep the pinned versions; + # values are trimmed where they're used. + - name: ACL_BASE_IMAGE_VERSION + displayName: ACL base image version override - shared fallback + type: string + default: " " + - name: ACL_BASE_IMAGE_VERSION_AMD64 + displayName: ACL base image version override - AMD64 + type: string + default: " " + - name: ACL_BASE_IMAGE_VERSION_ARM64 + displayName: ACL base image version override - ARM64 + type: string + default: " " - name: useOverrides displayName: Use component overrides type: boolean @@ -144,7 +165,7 @@ parameters: - name: buildaclcvmfipstlgen2 displayName: Build Azure Container Linux CVM FIPS Gen2 type: boolean - default: false + default: true - name: buildaclarm64tlgen2 displayName: Build Azure Container Linux ARM64 TL Gen2 type: boolean @@ -157,20 +178,9 @@ parameters: displayName: Build Azure Container Linux ARM64 FIPS TL Gen2 type: boolean default: true - - name: aclSigSourceGallery - displayName: ACL SIG source gallery unique name - type: string - default: 'ae6ab47f-1fc9-4b67-8f82-4636119119c9-ACL' - - name: aclSigSourceImageName - displayName: ACL SIG source image definition (amd64) - type: string - default: 'acl-3.0-amd64' - - name: aclImageVersion - displayName: ACL SIG source image version - type: string - default: '20261016.1226753.0' variables: + - template: ./templates/acl-image-versions.yaml - name: MODE value: "linuxVhdMode" - group: aks-vuln-to-kusto @@ -919,9 +929,6 @@ stages: - bash: | echo '##vso[task.setvariable variable=OS_SKU]AzureContainerLinux' echo '##vso[task.setvariable variable=OS_VERSION]acl' - echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]ae6ab47f-1fc9-4b67-8f82-4636119119c9-ACL' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acl-3.0-amd64' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]20261022.1233080.0' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' @@ -931,11 +938,17 @@ stages: echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' displayName: Setup Build Variables + - template: ./templates/.acl-base-image-vars.yaml + parameters: + ACL_BASE_IMAGE_SOURCE: ${{ parameters.ACL_BASE_IMAGE_SOURCE }} + VARIANT: amd64 + SIG_SOURCE_IMAGE_VERSION_OVERRIDE: ${{ iif(ne(trim(parameters.ACL_BASE_IMAGE_VERSION_AMD64), ''), trim(parameters.ACL_BASE_IMAGE_VERSION_AMD64), trim(parameters.ACL_BASE_IMAGE_VERSION)) }} - template: ./templates/.builder-release-template.yaml parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: acl-tl-gen2 + ENABLE_COSI_PUBLISHING: true - job: buildaclcvmgen2 condition: eq('${{ parameters.buildaclcvmgen2 }}', true) dependsOn: [ ] @@ -944,9 +957,6 @@ stages: - bash: | echo '##vso[task.setvariable variable=OS_SKU]AzureContainerLinux' echo '##vso[task.setvariable variable=OS_VERSION]acl' - echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]ae6ab47f-1fc9-4b67-8f82-4636119119c9-ACL' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acl-3.0-amd64' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]20261022.1233080.0' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D4ads_v5' echo '##vso[task.setvariable variable=CVM_TEST_VM_SIZE]Standard_DC4es_v6' @@ -956,6 +966,10 @@ stages: echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' displayName: Setup Build Variables + - template: ./templates/.acl-base-image-vars.yaml + parameters: + ACL_BASE_IMAGE_SOURCE: acg + VARIANT: amd64 - template: ./templates/.builder-release-template.yaml parameters: useOverrides: ${{ parameters.useOverrides }} @@ -969,9 +983,9 @@ stages: - bash: | echo '##vso[task.setvariable variable=OS_SKU]AzureContainerLinux' echo '##vso[task.setvariable variable=OS_VERSION]acl' - echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]${{ parameters.aclSigSourceGallery }}' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]${{ parameters.aclSigSourceImageName }}' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]${{ parameters.aclImageVersion }}' + echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]ae6ab47f-1fc9-4b67-8f82-4636119119c9-ACL' + echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acl-3.0-amd64' + echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]$(ACL_AMD64_IMAGE_VERSION)' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D4ads_v5' echo '##vso[task.setvariable variable=CVM_TEST_VM_SIZE]Standard_DC8ads_v5' @@ -994,9 +1008,6 @@ stages: - bash: | echo '##vso[task.setvariable variable=OS_SKU]AzureContainerLinux' echo '##vso[task.setvariable variable=OS_VERSION]acl' - echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]ae6ab47f-1fc9-4b67-8f82-4636119119c9-ACL' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acl-3.0-arm64' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]20261022.1233079.0' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16pds_v6' echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' @@ -1006,11 +1017,17 @@ stages: echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' displayName: Setup Build Variables + - template: ./templates/.acl-base-image-vars.yaml + parameters: + ACL_BASE_IMAGE_SOURCE: ${{ parameters.ACL_BASE_IMAGE_SOURCE }} + VARIANT: arm64 + SIG_SOURCE_IMAGE_VERSION_OVERRIDE: ${{ iif(ne(trim(parameters.ACL_BASE_IMAGE_VERSION_ARM64), ''), trim(parameters.ACL_BASE_IMAGE_VERSION_ARM64), trim(parameters.ACL_BASE_IMAGE_VERSION)) }} - template: ./templates/.builder-release-template.yaml parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: acl-arm64-tl-gen2 + ENABLE_COSI_PUBLISHING: true - job: buildaclfipstlgen2 condition: eq('${{ parameters.buildaclfipstlgen2 }}', true) dependsOn: [ ] @@ -1019,9 +1036,6 @@ stages: - bash: | echo '##vso[task.setvariable variable=OS_SKU]AzureContainerLinux' echo '##vso[task.setvariable variable=OS_VERSION]acl' - echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]ae6ab47f-1fc9-4b67-8f82-4636119119c9-ACL' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acl-3.0-amd64' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]20261022.1233080.0' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' @@ -1031,11 +1045,17 @@ stages: echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' displayName: Setup Build Variables + - template: ./templates/.acl-base-image-vars.yaml + parameters: + ACL_BASE_IMAGE_SOURCE: ${{ parameters.ACL_BASE_IMAGE_SOURCE }} + VARIANT: amd64 + SIG_SOURCE_IMAGE_VERSION_OVERRIDE: ${{ iif(ne(trim(parameters.ACL_BASE_IMAGE_VERSION_AMD64), ''), trim(parameters.ACL_BASE_IMAGE_VERSION_AMD64), trim(parameters.ACL_BASE_IMAGE_VERSION)) }} - template: ./templates/.builder-release-template.yaml parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: acl-fips-tl-gen2 + ENABLE_COSI_PUBLISHING: true - job: buildaclarm64fipstlgen2 condition: eq('${{ parameters.buildaclarm64fipstlgen2 }}', true) dependsOn: [ ] @@ -1044,9 +1064,6 @@ stages: - bash: | echo '##vso[task.setvariable variable=OS_SKU]AzureContainerLinux' echo '##vso[task.setvariable variable=OS_VERSION]acl' - echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]ae6ab47f-1fc9-4b67-8f82-4636119119c9-ACL' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acl-3.0-arm64' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]20261022.1233079.0' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16pds_v6' echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' @@ -1056,11 +1073,49 @@ stages: echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' displayName: Setup Build Variables + - template: ./templates/.acl-base-image-vars.yaml + parameters: + ACL_BASE_IMAGE_SOURCE: ${{ parameters.ACL_BASE_IMAGE_SOURCE }} + VARIANT: arm64 + SIG_SOURCE_IMAGE_VERSION_OVERRIDE: ${{ iif(ne(trim(parameters.ACL_BASE_IMAGE_VERSION_ARM64), ''), trim(parameters.ACL_BASE_IMAGE_VERSION_ARM64), trim(parameters.ACL_BASE_IMAGE_VERSION)) }} - template: ./templates/.builder-release-template.yaml parameters: useOverrides: ${{ parameters.useOverrides }} overrideBranch: ${{ parameters.overrideBranch }} artifactName: acl-arm64-fips-tl-gen2 + ENABLE_COSI_PUBLISHING: true + - template: ./templates/.builder-cosi-convert-job-template.yaml + parameters: + JOB_NAME: convertacltlgen2 + DEPENDS_ON_JOB: buildacltlgen2 + ARCHITECTURE: X86_64 + IMG_SKU: azure-linux-3-acl + ENABLE_FIPS: false + ARTIFACT_NAME: acl-tl-gen2 + - template: ./templates/.builder-cosi-convert-job-template.yaml + parameters: + JOB_NAME: convertaclarm64tlgen2 + DEPENDS_ON_JOB: buildaclarm64tlgen2 + ARCHITECTURE: ARM64 + IMG_SKU: azure-linux-3-arm64-gen2-acl + ENABLE_FIPS: false + ARTIFACT_NAME: acl-arm64-tl-gen2 + - template: ./templates/.builder-cosi-convert-job-template.yaml + parameters: + JOB_NAME: convertaclfipstlgen2 + DEPENDS_ON_JOB: buildaclfipstlgen2 + ARCHITECTURE: X86_64 + IMG_SKU: azure-linux-3-acl + ENABLE_FIPS: true + ARTIFACT_NAME: acl-fips-tl-gen2 + - template: ./templates/.builder-cosi-convert-job-template.yaml + parameters: + JOB_NAME: convertaclarm64fipstlgen2 + DEPENDS_ON_JOB: buildaclarm64fipstlgen2 + ARCHITECTURE: ARM64 + IMG_SKU: azure-linux-3-arm64-gen2-acl + ENABLE_FIPS: true + ARTIFACT_NAME: acl-arm64-fips-tl-gen2 - job: build2404arm64gbgen2containerd condition: eq('${{ parameters.build2404arm64gbgen2containerd }}', true) dependsOn: [ ] @@ -1099,3 +1154,13 @@ stages: name: All Linux IgnoreScenariosWithMissingVhd: true useVhdMetadataArtifacts: true + ACL_BASE_IMAGE_SOURCE: ${{ parameters.ACL_BASE_IMAGE_SOURCE }} + COSI_ARTIFACT_NAMES: + - ${{ if eq(parameters.buildacltlgen2, true) }}: + - "cosi-publishing-info-acl-tl-gen2" + - ${{ if eq(parameters.buildaclarm64tlgen2, true) }}: + - "cosi-publishing-info-acl-arm64-tl-gen2" + - ${{ if eq(parameters.buildaclfipstlgen2, true) }}: + - "cosi-publishing-info-acl-fips-tl-gen2" + - ${{ if eq(parameters.buildaclarm64fipstlgen2, true) }}: + - "cosi-publishing-info-acl-arm64-fips-tl-gen2" diff --git a/.pipelines/.vsts-vhd-builder.yaml b/.pipelines/.vsts-vhd-builder.yaml index d0574aebae9..10417cade1a 100644 --- a/.pipelines/.vsts-vhd-builder.yaml +++ b/.pipelines/.vsts-vhd-builder.yaml @@ -12,6 +12,7 @@ pr: - vhdbuilder/packer - vhdbuilder/scripts/linux - .pipelines/.vsts-vhd-builder.yaml + - .pipelines/templates/acl-image-versions.yaml - .pipelines/templates/.builder-release-template.yaml - .pipelines/templates/.template-override-components-json.yaml - .pipelines/templates/.template-copy-file.yaml @@ -38,7 +39,31 @@ pr: pool: name: $(POOL_NAME) +parameters: + - name: ACL_BASE_IMAGE_SOURCE + type: string + default: acg + values: + - acg + - acldevel + # The Run pipeline form marks a string parameter whose default is "" as required, + # so these default to a single space. Leave the space to keep the pinned versions; + # values are trimmed where they're used. + - name: ACL_BASE_IMAGE_VERSION + displayName: ACL base image version override - shared fallback + type: string + default: " " + - name: ACL_BASE_IMAGE_VERSION_AMD64 + displayName: ACL base image version override - AMD64 + type: string + default: " " + - name: ACL_BASE_IMAGE_VERSION_ARM64 + displayName: ACL base image version override - ARM64 + type: string + default: " " + variables: + - template: ./templates/acl-image-versions.yaml - name: MODE value: "linuxVhdMode" - name: USE_SHALLOW_REPLICATION @@ -242,9 +267,6 @@ stages: - bash: | echo '##vso[task.setvariable variable=OS_SKU]AzureContainerLinux' echo '##vso[task.setvariable variable=OS_VERSION]acl' - echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]ae6ab47f-1fc9-4b67-8f82-4636119119c9-ACL' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acl-3.0-amd64' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]20261022.1233080.0' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' @@ -254,6 +276,11 @@ stages: echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' displayName: Setup Build Variables + - template: ./templates/.acl-base-image-vars.yaml + parameters: + ACL_BASE_IMAGE_SOURCE: ${{ parameters.ACL_BASE_IMAGE_SOURCE }} + VARIANT: amd64 + SIG_SOURCE_IMAGE_VERSION_OVERRIDE: ${{ iif(ne(trim(parameters.ACL_BASE_IMAGE_VERSION_AMD64), ''), trim(parameters.ACL_BASE_IMAGE_VERSION_AMD64), trim(parameters.ACL_BASE_IMAGE_VERSION)) }} - template: ./templates/.builder-release-template.yaml parameters: artifactName: acl-tl-gen2 @@ -264,9 +291,6 @@ stages: - bash: | echo '##vso[task.setvariable variable=OS_SKU]AzureContainerLinux' echo '##vso[task.setvariable variable=OS_VERSION]acl' - echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]ae6ab47f-1fc9-4b67-8f82-4636119119c9-ACL' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acl-3.0-arm64' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]20261022.1233079.0' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16pds_v6' echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' @@ -276,6 +300,11 @@ stages: echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' displayName: Setup Build Variables + - template: ./templates/.acl-base-image-vars.yaml + parameters: + ACL_BASE_IMAGE_SOURCE: ${{ parameters.ACL_BASE_IMAGE_SOURCE }} + VARIANT: arm64 + SIG_SOURCE_IMAGE_VERSION_OVERRIDE: ${{ iif(ne(trim(parameters.ACL_BASE_IMAGE_VERSION_ARM64), ''), trim(parameters.ACL_BASE_IMAGE_VERSION_ARM64), trim(parameters.ACL_BASE_IMAGE_VERSION)) }} - template: ./templates/.builder-release-template.yaml parameters: artifactName: acl-arm64-tl-gen2 @@ -286,9 +315,6 @@ stages: - bash: | echo '##vso[task.setvariable variable=OS_SKU]AzureContainerLinux' echo '##vso[task.setvariable variable=OS_VERSION]acl' - echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]ae6ab47f-1fc9-4b67-8f82-4636119119c9-ACL' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acl-3.0-amd64' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]20261022.1233080.0' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' @@ -298,6 +324,11 @@ stages: echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' displayName: Setup Build Variables + - template: ./templates/.acl-base-image-vars.yaml + parameters: + ACL_BASE_IMAGE_SOURCE: ${{ parameters.ACL_BASE_IMAGE_SOURCE }} + VARIANT: amd64 + SIG_SOURCE_IMAGE_VERSION_OVERRIDE: ${{ iif(ne(trim(parameters.ACL_BASE_IMAGE_VERSION_AMD64), ''), trim(parameters.ACL_BASE_IMAGE_VERSION_AMD64), trim(parameters.ACL_BASE_IMAGE_VERSION)) }} - template: ./templates/.builder-release-template.yaml parameters: artifactName: acl-fips-tl-gen2 @@ -308,9 +339,6 @@ stages: - bash: | echo '##vso[task.setvariable variable=OS_SKU]AzureContainerLinux' echo '##vso[task.setvariable variable=OS_VERSION]acl' - echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]ae6ab47f-1fc9-4b67-8f82-4636119119c9-ACL' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acl-3.0-arm64' - echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]20261022.1233079.0' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16pds_v6' echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' @@ -320,6 +348,11 @@ stages: echo '##vso[task.setvariable variable=TRUSTED_LAUNCH_SUPPORTED]True' echo '##vso[task.setvariable variable=ENABLE_CGROUPV2]True' displayName: Setup Build Variables + - template: ./templates/.acl-base-image-vars.yaml + parameters: + ACL_BASE_IMAGE_SOURCE: ${{ parameters.ACL_BASE_IMAGE_SOURCE }} + VARIANT: arm64 + SIG_SOURCE_IMAGE_VERSION_OVERRIDE: ${{ iif(ne(trim(parameters.ACL_BASE_IMAGE_VERSION_ARM64), ''), trim(parameters.ACL_BASE_IMAGE_VERSION_ARM64), trim(parameters.ACL_BASE_IMAGE_VERSION)) }} - template: ./templates/.builder-release-template.yaml parameters: artifactName: acl-arm64-fips-tl-gen2 @@ -336,3 +369,4 @@ stages: parameters: IgnoreScenariosWithMissingVhd: true useVhdMetadataArtifacts: true + ACL_BASE_IMAGE_SOURCE: ${{ parameters.ACL_BASE_IMAGE_SOURCE }} diff --git a/.pipelines/scripts/e2e_gc.sh b/.pipelines/scripts/e2e_gc.sh new file mode 100755 index 00000000000..8c379a88263 --- /dev/null +++ b/.pipelines/scripts/e2e_gc.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -euo pipefail + +[ -z "${SUBSCRIPTION_ID:-}" ] && echo "SUBSCRIPTION_ID must be set" && exit 1 + +DRY_RUN="${DRY_RUN:-false}" +NO_WAIT="${NO_WAIT:-false}" + +main() { + az account set -s "$SUBSCRIPTION_ID" + + echo "attempting to garbage collect AgentBaker E2E resource groups in subscription: $SUBSCRIPTION_ID" + garbage_collect || exit $? +} + +garbage_collect() { + groups=$(az group list 2>/dev/null | jq -r '.[] | select(.name | startswith("abe2e")) | .name' | tr -d '\"' || "") + if [ -z "$groups" ]; then + echo "no AgentBaker E2E resource groups found in subscription $SUBSCRIPTION_ID, nothing to garbage collect" + return 0 + fi + + local -a no_wait_args=() + if [ "${NO_WAIT,,}" = "true" ]; then + no_wait_args+=(--no-wait) + fi + + for group in $groups; do + echo "will delete resource group: $group" + if [ "${DRY_RUN,,}" = "true" ]; then + echo "DRY_RUN: az group delete -g $group --yes ${no_wait_args[*]}" + continue + fi + if ! az group delete -g "$group" --yes "${no_wait_args[@]}"; then + echo "failed to delete resource group: $group, continuing..." + fi + done +} + +main "$@" diff --git a/.pipelines/scripts/e2e_run.sh b/.pipelines/scripts/e2e_run.sh index 9a7c11e3f90..5ee8a1b1b73 100644 --- a/.pipelines/scripts/e2e_run.sh +++ b/.pipelines/scripts/e2e_run.sh @@ -96,7 +96,7 @@ fi az extension add --name bastion cd e2e -go test -count=1 ./... +go test -count=1 -timeout "${E2E_GO_TEST_TIMEOUT}" ./... go run . run \ --parallel 60 \ diff --git a/.pipelines/templates/.acl-base-image-signed-vars.yaml b/.pipelines/templates/.acl-base-image-signed-vars.yaml new file mode 100644 index 00000000000..fe7cfb197c5 --- /dev/null +++ b/.pipelines/templates/.acl-base-image-signed-vars.yaml @@ -0,0 +1,36 @@ +# Sets ACL_BASE_IMAGE_SIGNED from ACL_BASE_IMAGE_SOURCE alone. Deliberately +# independent of variant (amd64/arm64): the e2e stage runs a single job +# across all variants of a given source, so no per-variant value could ever +# be selected correctly there. Every base-image source must therefore set +# (or leave) ACL_BASE_IMAGE_SIGNED the same for all variants it supports -- +# if a future source's signed status ever needs to vary by variant, this +# template (and the single-flag Config.ACLBaseImageSigned it feeds) will +# need a larger redesign, not just a parameter added here. +# +# ACL_BASE_IMAGE_SOURCE selects where the ACL base image comes from: +# acg - the ACL prod pipeline's Azure Compute Gallery image. Default +# when not specified. Signed. +# acldevel - the acldevel dev pipeline's unsigned image (see PR #9458). +# Sets ACL_BASE_IMAGE_SIGNED=false because acldevel images +# are unsigned and cannot pass UEFI Secure Boot signature +# verification; read by all ACL e2e scenarios to select a VM +# config with Secure Boot disabled. +# +# Included by .acl-base-image-vars.yaml (for build jobs) and invoked +# directly by e2e-template.yaml (which only needs the signed flag, not the +# variant-specific image-pinning vars). +parameters: + - name: ACL_BASE_IMAGE_SOURCE + type: string + default: acg + values: + - acg + - acldevel + +steps: + - ${{ if eq(parameters.ACL_BASE_IMAGE_SOURCE, 'acldevel') }}: + - bash: echo '##vso[task.setvariable variable=ACL_BASE_IMAGE_SIGNED]false' + displayName: Set ACL_BASE_IMAGE_SIGNED (acldevel, unsigned) + - ${{ else }}: + - bash: echo '##vso[task.setvariable variable=ACL_BASE_IMAGE_SIGNED]true' + displayName: Set ACL_BASE_IMAGE_SIGNED (signed) diff --git a/.pipelines/templates/.acl-base-image-vars.yaml b/.pipelines/templates/.acl-base-image-vars.yaml new file mode 100644 index 00000000000..8ede0483f08 --- /dev/null +++ b/.pipelines/templates/.acl-base-image-vars.yaml @@ -0,0 +1,91 @@ +# Sets the ACL build variables shared by all ACL build jobs: the +# image-customizer container pin, and the base-image source variables +# selected by ACL_BASE_IMAGE_SOURCE. ACL_BASE_IMAGE_SIGNED is set by +# .acl-base-image-signed-vars.yaml (included below), which callers who only +# need the signed flag (e.g. the e2e stage) can invoke directly instead. +# +# ACL_BASE_IMAGE_SOURCE selects where the ACL base image comes from: +# acg - the ACL prod pipeline's Azure Compute Gallery image. Default +# when not specified. +# acldevel - the acldevel dev pipeline's unsigned image (see PR #9458). +# See .acl-base-image-signed-vars.yaml for how this affects +# ACL_BASE_IMAGE_SIGNED. +# +# SIG_SOURCE_IMAGE_VERSION_OVERRIDE optionally overrides the default +# SIG_SOURCE_IMAGE_VERSION for the acg/acldevel sources (e.g. to pick up a +# newer acldevel build without editing this file). Leave empty to use the +# default: for acg, ACL_AMD64_IMAGE_VERSION / ACL_ARM64_IMAGE_VERSION from +# acl-image-versions.yaml (callers must include that variables template); for +# acldevel, the pin below. +# +# IMG_CUSTOMIZER_CONTAINER defaults to the pin every ACL job currently uses, +# but is exposed as a parameter so a caller can override it (e.g. to test a +# newer image-customizer version) without editing this file. +parameters: + - name: ACL_BASE_IMAGE_SOURCE + type: string + default: acg + values: + - acg + - acldevel + - name: VARIANT + type: string + values: + - amd64 + - arm64 + - name: SIG_SOURCE_IMAGE_VERSION_OVERRIDE + type: string + default: "" + - name: IMG_CUSTOMIZER_CONTAINER + type: string + default: mcr.microsoft.com/azurelinux/imagecustomizer:1.5@sha256:b1a46447f67d73dce1a76163d3f8dbcbe12988b65c51393621529c52af160729 + +steps: + - template: ./.acl-base-image-signed-vars.yaml + parameters: + ACL_BASE_IMAGE_SOURCE: ${{ parameters.ACL_BASE_IMAGE_SOURCE }} + + - bash: echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER]${{ parameters.IMG_CUSTOMIZER_CONTAINER }}' + displayName: Set ACL image-customizer container + + - ${{ if eq(parameters.ACL_BASE_IMAGE_SOURCE, 'acg') }}: + - ${{ if eq(parameters.VARIANT, 'amd64') }}: + - bash: | + echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]ae6ab47f-1fc9-4b67-8f82-4636119119c9-ACL' + echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acl-3.0-amd64' + displayName: Set ACL base image vars (acg, amd64) + - ${{ if eq(parameters.VARIANT, 'arm64') }}: + - bash: | + echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]ae6ab47f-1fc9-4b67-8f82-4636119119c9-ACL' + echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acl-3.0-arm64' + displayName: Set ACL base image vars (acg, arm64) + - ${{ if and(eq(parameters.VARIANT, 'amd64'), eq(parameters.SIG_SOURCE_IMAGE_VERSION_OVERRIDE, '')) }}: + - bash: echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]$(ACL_AMD64_IMAGE_VERSION)' + displayName: Set SIG_SOURCE_IMAGE_VERSION (acg, amd64, default) + - ${{ if and(eq(parameters.VARIANT, 'arm64'), eq(parameters.SIG_SOURCE_IMAGE_VERSION_OVERRIDE, '')) }}: + - bash: echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]$(ACL_ARM64_IMAGE_VERSION)' + displayName: Set SIG_SOURCE_IMAGE_VERSION (acg, arm64, default) + - ${{ if ne(parameters.SIG_SOURCE_IMAGE_VERSION_OVERRIDE, '') }}: + - bash: echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]${{ parameters.SIG_SOURCE_IMAGE_VERSION_OVERRIDE }}' + displayName: Set SIG_SOURCE_IMAGE_VERSION (acg, override) + + - ${{ if eq(parameters.ACL_BASE_IMAGE_SOURCE, 'acldevel') }}: + - ${{ if eq(parameters.VARIANT, 'amd64') }}: + - bash: | + echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]035db282-f1c8-4ce7-b78f-2a7265d5398c-ACLDEVEL' + echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acldevel' + displayName: Set ACL base image vars (acldevel, amd64) + - ${{ if eq(parameters.VARIANT, 'arm64') }}: + - bash: | + echo '##vso[task.setvariable variable=SIG_SOURCE_GALLERY_UNIQUE_NAME]035db282-f1c8-4ce7-b78f-2a7265d5398c-ACLDEVEL' + echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_NAME]acldevel-arm64' + displayName: Set ACL base image vars (acldevel, arm64) + - ${{ if and(eq(parameters.VARIANT, 'amd64'), eq(parameters.SIG_SOURCE_IMAGE_VERSION_OVERRIDE, '')) }}: + - bash: echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]0.20261023.1234357' + displayName: Set SIG_SOURCE_IMAGE_VERSION (acldevel, amd64, default) + - ${{ if and(eq(parameters.VARIANT, 'arm64'), eq(parameters.SIG_SOURCE_IMAGE_VERSION_OVERRIDE, '')) }}: + - bash: echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]0.20261023.1234357' + displayName: Set SIG_SOURCE_IMAGE_VERSION (acldevel, arm64, default) + - ${{ if ne(parameters.SIG_SOURCE_IMAGE_VERSION_OVERRIDE, '') }}: + - bash: echo '##vso[task.setvariable variable=SIG_SOURCE_IMAGE_VERSION]${{ parameters.SIG_SOURCE_IMAGE_VERSION_OVERRIDE }}' + displayName: Set SIG_SOURCE_IMAGE_VERSION (acldevel, override) diff --git a/.pipelines/templates/.build-and-test-windows-vhd-template.yaml b/.pipelines/templates/.build-and-test-windows-vhd-template.yaml index 8a85f6a009a..81b3174c5ba 100644 --- a/.pipelines/templates/.build-and-test-windows-vhd-template.yaml +++ b/.pipelines/templates/.build-and-test-windows-vhd-template.yaml @@ -36,10 +36,6 @@ parameters: type: boolean displayName: Enable Trusted Launch on the SIG image definition default: False - - name: trustedLaunchSupported - type: boolean - displayName: Determines if the image is capable of supporting Trusted Launch - default: False - name: skipExtensionCheck displayName: Skip Extension Check type: boolean @@ -91,7 +87,7 @@ stages: hyperVGeneration: ${{ parameters.hyperVGeneration }} architecture: ${{ parameters.architecture }} enableTrustedLaunch: ${{ parameters.enableTrustedLaunch }} - trustedLaunchSupported: ${{ parameters.trustedLaunchSupported }} + trustedLaunchSupported: ${{ eq(parameters.hyperVGeneration, 'V2') }} buildVmSize: ${{ parameters.buildVmSize }} build: ${{ parameters.build }} vhddebug: ${{ parameters.vhddebug }} diff --git a/.pipelines/templates/.build-and-test-windows-vhds-template.yaml b/.pipelines/templates/.build-and-test-windows-vhds-template.yaml index c0bcb8e24e6..7dd37e9a771 100644 --- a/.pipelines/templates/.build-and-test-windows-vhds-template.yaml +++ b/.pipelines/templates/.build-and-test-windows-vhds-template.yaml @@ -174,9 +174,8 @@ stages: windowsSku: 2025-gen2-tl hyperVGeneration: V2 architecture: X86_64 - # TODO: replace enableTrustedLaunch with trustedLaunchSupported when / as able - by default we shouldn't need to build dedicated trusted launch images + # TODO: retire the dedicated image after Trusted Launch moves to the shared Gen2 image. enableTrustedLaunch: True - trustedLaunchSupported: True buildVmSize: ${{ parameters.buildVmSize }} build: ${{ parameters.build2025gen2tl }} vhddebug: ${{ parameters.vhddebug }} diff --git a/.pipelines/templates/.builder-cosi-convert-job-template.yaml b/.pipelines/templates/.builder-cosi-convert-job-template.yaml new file mode 100644 index 00000000000..32ed6c14c44 --- /dev/null +++ b/.pipelines/templates/.builder-cosi-convert-job-template.yaml @@ -0,0 +1,120 @@ +# Converts a build job's ACL VHD to COSI, uploads it to PMC, and publishes the +# COSI publishing info. Conversion runs natively; the source build job owns +# immutable VHD promotion and supplies the VHD location, version, and source bundle. +parameters: + - name: JOB_NAME + type: string + - name: DEPENDS_ON_JOB + type: string + - name: ARCHITECTURE + type: string + values: + - X86_64 + - ARM64 + - name: ARTIFACT_NAME + type: string + - name: IMG_SKU + type: string + - name: ENABLE_FIPS + type: boolean + default: false + +jobs: + - job: ${{ parameters.JOB_NAME }} + condition: and(succeeded(), in(dependencies.${{ parameters.DEPENDS_ON_JOB }}.result, 'Succeeded', 'SucceededWithIssues'), eq(variables.ENVIRONMENT, 'tme'), eq(variables.GENERATE_PUBLISHING_INFO, 'True')) + dependsOn: ${{ parameters.DEPENDS_ON_JOB }} + timeoutInMinutes: 120 + pool: + ${{ if eq(parameters.ARCHITECTURE, 'ARM64') }}: + name: $(NODE_SIG_TME_POOL_ARM64_NAME) + ${{ else }}: + name: $(POOL_NAME) + variables: + CAPTURED_SIG_VERSION: $[ dependencies.${{ parameters.DEPENDS_ON_JOB }}.outputs['exportCapturedSigVersion.CAPTURED_SIG_VERSION'] ] + IMAGE_VERSION: $[ dependencies.${{ parameters.DEPENDS_ON_JOB }}.outputs['exportImageVersion.IMAGE_VERSION'] ] + DESTINATION_STORAGE_CONTAINER: $[ dependencies.${{ parameters.DEPENDS_ON_JOB }}.outputs['exportDestinationStorageContainer.DESTINATION_STORAGE_CONTAINER'] ] + steps: + - checkout: none + + - bash: | + echo '##vso[task.setvariable variable=OS_SKU]AzureContainerLinux' + echo '##vso[task.setvariable variable=OS_VERSION]acl' + echo '##vso[task.setvariable variable=IMG_SKU]${{ parameters.IMG_SKU }}' + echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' + echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' + echo '##vso[task.setvariable variable=ARCHITECTURE]${{ parameters.ARCHITECTURE }}' + echo '##vso[task.setvariable variable=ENABLE_FIPS]${{ parameters.ENABLE_FIPS }}' + echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]True' + echo '##vso[task.setvariable variable=IMG_CUSTOMIZER_CONTAINER]mcr.microsoft.com/azurelinux/imagecustomizer:1.5@sha256:b1a46447f67d73dce1a76163d3f8dbcbe12988b65c51393621529c52af160729' + displayName: Setup Build Variables + + - task: DownloadPipelineArtifact@2 + displayName: Download AgentBaker source bundle + inputs: + artifact: agentbaker-sources-${{ parameters.ARTIFACT_NAME }} + path: $(Pipeline.Workspace)/agentbaker-sources + + # tar is present on both convert pool images, so the source bundle can be + # extracted before the remaining tools are bootstrapped from it. + - bash: | + set -euo pipefail + mkdir -p "$(Build.SourcesDirectory)" + tar -xzf "$(Pipeline.Workspace)/agentbaker-sources/agentbaker-sources.tar.gz" -C "$(Build.SourcesDirectory)" + ls "$(Build.SourcesDirectory)" + displayName: Extract AgentBaker source bundle + + - bash: ./vhdbuilder/packer/imagecustomizer/scripts/bootstrap-cosi-convert-tools.sh + displayName: Bootstrap COSI convert tooling + + - bash: make -f packer.mk build-cosi-uploader + displayName: Build COSI uploader with Microsoft system crypto + condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True')) + + - bash: | + SKU_NAME=${OS_VERSION} && \ + if [[ "${HYPERV_GENERATION,,}" == "v2" ]]; then SKU_NAME="${SKU_NAME}gen2"; fi && \ + if [[ "${ARCHITECTURE,,}" == "arm64" ]]; then SKU_NAME="${SKU_NAME}arm64"; fi && \ + if grep -q "NVIDIA_GB" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}gb"; fi && \ + if [[ "${ENABLE_FIPS,,}" == "true" ]]; then SKU_NAME="${SKU_NAME}fips"; fi && \ + if grep -q "cvm" <<< "$FEATURE_FLAGS"; then SKU_NAME="${SKU_NAME}CVM"; fi && \ + if [[ "${IMG_SKU}" == *"minimal"* ]]; then SKU_NAME="${SKU_NAME}minimal"; fi && \ + if [[ "${ENABLE_TRUSTED_LAUNCH}" == "True" ]]; then SKU_NAME="${SKU_NAME}TL"; fi && \ + SKU_NAME=$(echo ${SKU_NAME} | tr -d '.') && \ + echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME" + echo "Set SKU_NAME to $SKU_NAME" + displayName: Set SKU Name + + - task: AzureCLI@2 + inputs: + azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + make -f packer.mk convert-vhd-to-cosi + condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True')) + displayName: Convert ACL VHD to COSI + env: + RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) + OFFER_NAME: $(OS_SKU) + AFD_DOWNLOAD_HOSTNAME: $(AFD_DOWNLOAD_HOSTNAME) + COSI_CONTAINER: $(COSI_CONTAINER) + + - task: AzureCLI@2 + inputs: + azureSubscription: $(COSI_PMC_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + bash ./vhdbuilder/packer/imagecustomizer/scripts/upload-cosi-to-pmc.sh + condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True')) + displayName: Upload COSI to PMC + env: + AFD_UPLOAD_ENDPOINT: $(AFD_UPLOAD_ENDPOINT) + COSI_CONTAINER: $(COSI_CONTAINER) + + - task: PublishPipelineArtifact@1 + condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True')) + displayName: Publish COSI Publishing Info + inputs: + artifactName: 'cosi-publishing-info-${{ parameters.ARTIFACT_NAME }}' + targetPath: 'cosi-publishing-info.json' diff --git a/.pipelines/templates/.builder-release-template.yaml b/.pipelines/templates/.builder-release-template.yaml index 9c72fc48166..4881adc1b5a 100644 --- a/.pipelines/templates/.builder-release-template.yaml +++ b/.pipelines/templates/.builder-release-template.yaml @@ -8,16 +8,44 @@ parameters: - name: overrideBranch type: string default: master + - name: ENABLE_COSI_PUBLISHING + type: boolean + default: false - name: kataCC type: boolean default: false steps: + # Keep the baked version and publishing metadata identical across UTC midnight. + - bash: | + set -euo pipefail + image_version="${IMAGE_VERSION:-$(date -u +%Y%m.%d.0)}" + echo "##vso[task.setvariable variable=IMAGE_VERSION]${image_version}" + displayName: Set IMAGE_VERSION + - template: ./.template-override-components-json.yaml parameters: overrideBranch: ${{ parameters.overrideBranch }} useOverrides: ${{ parameters.useOverrides }} + # Bundle before downloading scanner binaries into the source tree. + - ${{ if parameters.ENABLE_COSI_PUBLISHING }}: + - bash: | + set -euo pipefail + cd "$(Build.SourcesDirectory)" + tar --exclude='./.git' \ + -czf "$(Build.ArtifactStagingDirectory)/agentbaker-sources.tar.gz" . + ls -lh "$(Build.ArtifactStagingDirectory)/agentbaker-sources.tar.gz" + condition: and(succeeded(), eq(variables.ENVIRONMENT, 'tme'), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq(variables['OS_SKU'], 'AzureContainerLinux')) + displayName: Bundle source for COSI convert + + - task: PublishPipelineArtifact@1 + condition: and(succeeded(), eq(variables.ENVIRONMENT, 'tme'), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq(variables['OS_SKU'], 'AzureContainerLinux')) + displayName: Publish source bundle for COSI convert + inputs: + artifactName: agentbaker-sources-${{ parameters.artifactName }} + targetPath: $(Build.ArtifactStagingDirectory)/agentbaker-sources.tar.gz + - task: UniversalPackages@0 displayName: Download CIS scanner inputs: @@ -118,6 +146,7 @@ steps: popd cue export ./schemas/manifest.cue > ./parts/linux/cloud-init/artifacts/manifest.json displayName: Setup Cue + retryCountOnTaskFailure: 3 - bash: | SKU_NAME=${OS_VERSION} && \ @@ -158,6 +187,7 @@ steps: UA_TOKEN: $(ua-token) LOCAL_DOCA_REPO_URL: $(LOCAL_DOCA_REPO_URL) CONTINUE_ON_LOCAL_REPO_DOWNLOAD_ERROR: $(CONTINUE_ON_LOCAL_REPO_DOWNLOAD_ERROR) + IMAGE_VERSION: $(IMAGE_VERSION) - task: AzureCLI@2 inputs: @@ -215,6 +245,19 @@ steps: condition: always() displayName: Set Post-Build Variables + - ${{ if parameters.ENABLE_COSI_PUBLISHING }}: + - bash: | + echo "##vso[task.setvariable variable=CAPTURED_SIG_VERSION;isOutput=true]${CAPTURED_SIG_VERSION}" + name: exportCapturedSigVersion + displayName: Export Captured SIG Version for COSI convert + condition: and(succeeded(), eq(variables.ENVIRONMENT, 'tme'), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq(variables['OS_SKU'], 'AzureContainerLinux')) + + - bash: | + echo "##vso[task.setvariable variable=IMAGE_VERSION;isOutput=true]${IMAGE_VERSION}" + name: exportImageVersion + displayName: Export Image Version for COSI convert + condition: and(succeeded(), eq(variables.ENVIRONMENT, 'tme'), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq(variables['OS_SKU'], 'AzureContainerLinux')) + - bash: | set -euo pipefail if [ "${BUILDER:-}" = "imagecustomizer" ]; then @@ -470,6 +513,7 @@ steps: succeeded(), eq(variables.ENVIRONMENT, 'tme'), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), + not(and(eq('${{ parameters.ENABLE_COSI_PUBLISHING }}', true), eq(variables['OS_SKU'], 'AzureContainerLinux'))), or( eq(variables.RUN_PREFETCH_OPTIMIZATION, 'False'), eq(variables.PREFETCH_COMPATIBLE, 'False') @@ -477,6 +521,32 @@ steps: ) displayName: Copy to Immutable Storage Container + # The COSI convert jobs read the ACL VHD from the immutable container. + - ${{ if parameters.ENABLE_COSI_PUBLISHING }}: + - task: AzureCLI@2 + inputs: + azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -euo pipefail + bash ./vhdbuilder/packer/imagecustomizer/scripts/copy-vhd-to-immutable.sh + echo "##vso[task.setvariable variable=DESTINATION_STORAGE_CONTAINER]${CLASSIC_BLOB}" + condition: | + and( + succeeded(), + eq(variables.ENVIRONMENT, 'tme'), + eq(variables.GENERATE_PUBLISHING_INFO, 'True'), + eq(variables['OS_SKU'], 'AzureContainerLinux') + ) + displayName: Copy ACL VHD to Immutable Storage Container + + - bash: | + echo "##vso[task.setvariable variable=DESTINATION_STORAGE_CONTAINER;isOutput=true]${DESTINATION_STORAGE_CONTAINER}" + name: exportDestinationStorageContainer + displayName: Export Destination Storage Container for COSI convert + condition: and(succeeded(), eq(variables.ENVIRONMENT, 'tme'), eq(variables.GENERATE_PUBLISHING_INFO, 'True'), eq(variables['OS_SKU'], 'AzureContainerLinux')) + - task: AzureCLI@2 inputs: azureSubscription: $(VHD_ARM_SERVICE_CONNECTION) @@ -621,6 +691,7 @@ steps: RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME) STORAGE_ACCT_BLOB_URL: $(CLASSIC_BLOB) OFFER_NAME: $(OS_SKU) + IMAGE_VERSION: $(IMAGE_VERSION) - task: PublishPipelineArtifact@1 condition: and(succeeded(), eq(variables.GENERATE_PUBLISHING_INFO, 'True')) diff --git a/.pipelines/templates/acl-image-versions.yaml b/.pipelines/templates/acl-image-versions.yaml new file mode 100644 index 00000000000..e30de2d2d3f --- /dev/null +++ b/.pipelines/templates/acl-image-versions.yaml @@ -0,0 +1,3 @@ +variables: + ACL_AMD64_IMAGE_VERSION: '20261027.1238771.0' + ACL_ARM64_IMAGE_VERSION: '20261027.1238770.0' diff --git a/.pipelines/templates/e2e-gc-template.yaml b/.pipelines/templates/e2e-gc-template.yaml new file mode 100644 index 00000000000..2e4327eb87d --- /dev/null +++ b/.pipelines/templates/e2e-gc-template.yaml @@ -0,0 +1,34 @@ +parameters: + - name: ARM_SERVICE_CONNECTION + type: string + - name: SUBSCRIPTION_ID + type: string + - name: DRY_RUN + type: boolean + default: false + - name: NO_WAIT + type: boolean + default: false + +jobs: + - job: e2e_gc + displayName: Garbage Collection + timeoutInMinutes: 0 + steps: + - checkout: self + fetchTags: false + fetchDepth: 1 + + - task: AzureCLI@2 + inputs: + azureSubscription: ${{ parameters.ARM_SERVICE_CONNECTION }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + chmod +x .pipelines/scripts/e2e_gc.sh + .pipelines/scripts/e2e_gc.sh + env: + SUBSCRIPTION_ID: ${{ parameters.SUBSCRIPTION_ID }} + DRY_RUN: ${{ parameters.DRY_RUN }} + NO_WAIT: ${{ parameters.NO_WAIT }} + displayName: Garbage collect E2E resource groups diff --git a/.pipelines/templates/e2e-template.yaml b/.pipelines/templates/e2e-template.yaml index 3a343978393..f5974634cfb 100644 --- a/.pipelines/templates/e2e-template.yaml +++ b/.pipelines/templates/e2e-template.yaml @@ -35,12 +35,24 @@ parameters: because they cannot verify the not-opted-in behavior on such subscriptions. Msft tenant E2E subs auto-inject; the RCV1P subscription does not. default: true + - name: COSI_ARTIFACT_NAMES + type: object + displayName: List of COSI publishing info pipeline artifact names to download. Leave empty to skip COSI validation. + default: [] + - name: ACL_BASE_IMAGE_SOURCE + type: string + default: acg + values: + - acg + - acldevel jobs: - job: e2e condition: and(succeeded(), ne(variables.SKIP_E2E_TESTS, 'true')) variables: - group: ${{parameters.variableGroup}} # all variables prefixed with E2E_* come from this variable group + - name: COSI_ARTIFACTS_DIR + value: "" pool: name: $(E2E_POOL_NAME) timeoutInMinutes: ${{ parameters.jobTimeoutInMinutes }} @@ -53,6 +65,21 @@ jobs: fetchTags: false fetchDepth: 1 + - ${{ if gt(length(parameters.COSI_ARTIFACT_NAMES), 0) }}: + - bash: | + echo "##vso[task.setvariable variable=COSI_ARTIFACTS_DIR]$(Pipeline.Workspace)/cosi-info/$(Build.BuildId)-$(System.JobAttempt)" + echo "##vso[task.setvariable variable=TEST_TIMEOUT_VMSS]${TEST_TIMEOUT_VMSS:-30m}" + displayName: Set COSI artifact directory + condition: and(succeeded(), eq(variables.ENVIRONMENT, 'tme'), eq(variables.GENERATE_PUBLISHING_INFO, 'True')) + + - ${{ each ARTIFACT_NAME in parameters.COSI_ARTIFACT_NAMES }}: + - task: DownloadPipelineArtifact@2 + displayName: "Download COSI info: ${{ ARTIFACT_NAME }}" + condition: and(succeeded(), eq(variables.ENVIRONMENT, 'tme'), eq(variables.GENERATE_PUBLISHING_INFO, 'True')) + inputs: + artifact: "${{ ARTIFACT_NAME }}" + path: "$(COSI_ARTIFACTS_DIR)/${{ ARTIFACT_NAME }}" + - ${{ if eq(parameters.useVhdMetadataArtifacts, true) }}: - task: DownloadPipelineArtifact@2 displayName: Download E2E VHD metadata @@ -78,6 +105,10 @@ jobs: echo "##vso[task.setvariable variable=E2E_VHD_METADATA_FILE]$metadata_file" displayName: Merge E2E VHD metadata + - template: ./.acl-base-image-signed-vars.yaml + parameters: + ACL_BASE_IMAGE_SOURCE: ${{ parameters.ACL_BASE_IMAGE_SOURCE }} + - task: AzureCLI@2 inputs: azureSubscription: $(E2E_ARM_SERVICE_CONNECTION) @@ -98,6 +129,8 @@ jobs: RCV1P_TAGS_AUTO_INJECTED: ${{parameters.rcv1pTagsAutoInjected}} ${{ if eq(parameters.useVhdMetadataArtifacts, true) }}: E2E_VHD_METADATA_FILE: $(E2E_VHD_METADATA_FILE) + COSI_ARTIFACTS_DIR: $(COSI_ARTIFACTS_DIR) + ACL_BASE_IMAGE_SIGNED: $(ACL_BASE_IMAGE_SIGNED) - task: PublishTestResults@2 displayName: Upload test results diff --git a/Makefile b/Makefile index deb1212a203..113182872a3 100644 --- a/Makefile +++ b/Makefile @@ -204,7 +204,7 @@ endif ginkgoBuild: generate make -C ./test/e2e ginkgo-build -test: test-aks-node-controller test-aks-live-patching +test: test-aks-node-controller test-aks-live-patching test-cosi-uploader go build -mod=readonly ./... && go test ./... test-aks-node-controller: @@ -213,10 +213,14 @@ test-aks-node-controller: test-aks-live-patching: pushd aks-live-patching && go build -mod=readonly ./... && go test ./... && popd +test-cosi-uploader: + pushd vhdbuilder/cosi-uploader && go build -mod=readonly ./... && go test ./... && popd + lint: $(TOOLSBIN)/golangci-lint run ./... pushd aks-node-controller && $(TOOLSBIN)/golangci-lint run ./... && popd pushd aks-live-patching && $(TOOLSBIN)/golangci-lint run ./... && popd + pushd vhdbuilder/cosi-uploader && $(TOOLSBIN)/golangci-lint run ./... && popd .PHONY: test-style test-style: validate-go validate-shell validate-copyright-headers diff --git a/aks-node-controller/README.md b/aks-node-controller/README.md index 0539665d43b..9e82770c6b6 100644 --- a/aks-node-controller/README.md +++ b/aks-node-controller/README.md @@ -54,6 +54,50 @@ the provisioning config in CustomData; the automatic size fallback uses the same config-only payload. Leaving the new flag false preserves legacy, Phase 2, and Phase 2.5 behavior. +### Azure cloud trust-store capabilities + +The shared cloud runtime provides platform detection, bounded HTTP and command +execution, public-file writes, and guest-agent events. The trust-store helpers +write downloaded contents without PEM/X.509 filtering, update the platform's +existing CA tools, and preserve Ubuntu's OpenSSL bundle/symlink behavior. + +Cloud commands run in their own process groups with a two-minute timeout, or +the caller's earlier cancellation. Cancellation kills the entire group. A +five-second `WaitDelay` bounds output draining after cancellation or command +exit, including when a descendant leaves the group but retains an output pipe. + +Cache cleanup removes flat staging files, including hidden files, without +recursively deleting directories or removing installed system-trust certificates. +`AKS_CLOUD_LOG_CERTIFICATES=true` enables public-certificate-only diagnostics; +parsing for logging does not gate installation. + +These are internal capabilities only. Existing provisioning still uses +`init-aks-cloud.sh`. + +### Azure cloud certificate retrieval capabilities + +WireServer helpers retrieve legacy certificate packages and RCV1P root/intermediate +operation groups. Requests preserve the shell's `comp=acmspackage&type=...&ext=...` +parameter order and raw values without additional URL escaping. WireServer's +special-character decoding contract is not verified. Request/retry bounds, flat +download caching, and per-certificate download skipping are preserved. + +RCV1P manifests are decoded as JSON from `OperationsInfo[].ResouceFileName`; +the field's misspelling is part of the protocol. JSON whitespace does not affect +parsing, and same-named fields at other levels are ignored. Invalid JSON or an +incompatible `OperationsInfo` structure returns an error rather than falling back +to text matching. + +HTTP 2xx responses must also be read without errors. Detectable transfer +truncation, such as a body shorter than `Content-Length`, is retried rather than +returning partial certificate bytes, unlike the shell's status-only success +check. This is not certificate-content validation: a close-delimited response +without a length or chunk-framing error can still contain incomplete content. + +Retrieval does not install certificates or make opt-in decisions. Refresh +orchestration, the standalone refresh command, scheduling, and native provisioning +activation are supplied by later layers; they are not enabled here. + ### Validating and normalizing AKSNodeConfig Before serializing a typed config, producers such as RP should call: diff --git a/aks-node-controller/app.go b/aks-node-controller/app.go index 71942951b3e..a5720a652fe 100644 --- a/aks-node-controller/app.go +++ b/aks-node-controller/app.go @@ -97,11 +97,16 @@ type App struct { // Authorization header for the check-hotfix LPS fetch. When nil, the real IMDS endpoint // is queried. fetchAttestedToken func(ctx context.Context) (string, error) + // fetchVMResourceID overrides retrieval of the VM's ARM resource ID from IMDS compute + // metadata, sent to the LPS as a lookup hint. When nil, the real IMDS endpoint is queried. + fetchVMResourceID func(ctx context.Context) (string, error) // applyEmbeddedHotfix overrides embedded script application for tests. applyEmbeddedHotfix func(string) error // grpcDialContext overrides how the gRPC LPS client dials, letting tests point the client at // an in-process (bufconn) server. When nil, the real TLS dial to the apiserver front is used. grpcDialContext func(ctx context.Context, target string) (net.Conn, error) + // refreshCloudCertificates overrides standalone cloud certificate refresh for testing. + refreshCloudCertificates func(context.Context, string) error } // provision.json values are emitted as strings by the shell jq invocation. @@ -181,6 +186,19 @@ func (a *App) Run(ctx context.Context, args []string) int { return nil }, }, + { + Name: "refresh-cloud-certificates", + Usage: "Refresh Azure cloud CA certificates without configuring repositories", + Flags: []cli.Flag{ + &cli.StringFlag{Name: "location", Required: true, Usage: "Azure region used to select the certificate endpoint"}, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + if cmd.Args().Len() != 0 { + return fmt.Errorf("unexpected refresh-cloud-certificates arguments: %s", strings.Join(cmd.Args().Slice(), " ")) + } + return a.runCloudCertificateRefresh(ctx, cmd.String("location")) + }, + }, { Name: "download-hotfix", Usage: "Download the requested hotfix binary", diff --git a/aks-node-controller/checkhotfix.go b/aks-node-controller/checkhotfix.go index ec6d529a43f..5f377c4fa74 100644 --- a/aks-node-controller/checkhotfix.go +++ b/aks-node-controller/checkhotfix.go @@ -54,6 +54,11 @@ const ( // the LPS Authorization token. IMDS is reachable pre-kubelet (the same primitive Secure // TLS Bootstrap uses), so this works before any kube credential exists. imdsAttestedDocURL = "http://169.254.169.254/metadata/attested/document?api-version=2025-04-07" + + // imdsInstanceComputeURL returns the IMDS compute metadata, whose resourceId is sent to the + // LPS as a lookup hint. It is unsigned: the LPS trusts it only after confirming the VM it + // names carries the vmId from the signed attested document. + imdsInstanceComputeURL = "http://169.254.169.254/metadata/instance/compute?api-version=2025-04-07&format=json" ) // Timeout tuning for the IMDS and LPS fetches. The generic transport/retry mechanics live in @@ -369,12 +374,58 @@ func fetchIMDSAttestedToken(ctx context.Context) (string, error) { // fetchIMDSAttestedTokenOnce performs a single IMDS attested-document GET and returns the // document signature. func fetchIMDSAttestedTokenOnce(ctx context.Context) (string, error) { + var doc struct { + Signature string `json:"signature"` + } + if err := getIMDSJSON(ctx, imdsAttestedDocURL, &doc); err != nil { + return "", err + } + if doc.Signature == "" { + return "", fmt.Errorf("imds attested document had empty signature") + } + return doc.Signature, nil +} + +// vmResourceID returns this VM's ARM resource ID, sent to the LPS so it can look the VM up +// directly instead of enumerating the node resource group. The fetch is overridable for +// testing via App.fetchVMResourceID. +func (a *App) vmResourceID(ctx context.Context) (string, error) { + if a.fetchVMResourceID != nil { + return a.fetchVMResourceID(ctx) + } + return fetchIMDSVMResourceID(ctx) +} + +// fetchIMDSVMResourceID queries IMDS compute metadata for the VM's resource ID, with the same +// retry budget as the attested-token fetch. +func fetchIMDSVMResourceID(ctx context.Context) (string, error) { + return common.RetryStringFetch(ctx, imdsMaxAttempts, fetchIMDSVMResourceIDOnce) +} + +// fetchIMDSVMResourceIDOnce performs a single IMDS compute-metadata GET and returns the +// resourceId. +func fetchIMDSVMResourceIDOnce(ctx context.Context) (string, error) { + var compute struct { + ResourceID string `json:"resourceId"` + } + if err := getIMDSJSON(ctx, imdsInstanceComputeURL, &compute); err != nil { + return "", err + } + if compute.ResourceID == "" { + return "", fmt.Errorf("imds compute metadata had empty resourceId") + } + return compute.ResourceID, nil +} + +// getIMDSJSON performs a single IMDS GET, bounded by imdsFetchTimeout, and decodes the JSON +// response into out. +func getIMDSJSON(ctx context.Context, url string, out any) error { ctx, cancel := context.WithTimeout(ctx, imdsFetchTimeout) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, imdsAttestedDocURL, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - return "", err + return err } req.Header.Set("Metadata", "true") @@ -391,26 +442,17 @@ func fetchIMDSAttestedTokenOnce(ctx context.Context) (string, error) { } resp, err := imdsClient.Do(req) if err != nil { - return "", err + return err } defer resp.Body.Close() body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) if err != nil { - return "", err + return err } if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("imds returned status %d", resp.StatusCode) + return fmt.Errorf("imds returned status %d", resp.StatusCode) } - var doc struct { - Signature string `json:"signature"` - } - if err := json.Unmarshal(body, &doc); err != nil { - return "", err - } - if doc.Signature == "" { - return "", fmt.Errorf("imds attested document had empty signature") - } - return doc.Signature, nil + return json.Unmarshal(body, out) } // parseHotfixConfig extracts the hotfix pointer from an LPS response body. The body is the diff --git a/aks-node-controller/checkhotfix_grpc.go b/aks-node-controller/checkhotfix_grpc.go index 4b3dcf750d1..ebd78d49241 100644 --- a/aks-node-controller/checkhotfix_grpc.go +++ b/aks-node-controller/checkhotfix_grpc.go @@ -36,6 +36,13 @@ const ( // the "authorization" metadata header (per the live-patching client example / contract), // mirroring the embargoed HTTP service's header-based auth. lpsAttestedMetadataKey = "authorization" + + // lpsVMResourceIDMetadataKey is the gRPC request-metadata key carrying the VM's ARM resource ID + // from IMDS compute metadata. It is an unsigned lookup hint: it lets the service fetch this one + // VM instead of enumerating the node resource group, and the service accepts it only if that + // VM's vmId matches the one in the signed attested document. It is optional; when absent the + // service falls back to the resource group enumeration. + lpsVMResourceIDMetadataKey = "x-aks-vm-resource-id" ) // lpsGRPCStatusError is a non-benign gRPC failure from the live-patching service. fallbackAllowed @@ -53,8 +60,8 @@ func (e *lpsGRPCStatusError) Error() string { // fetchHotfixOverGRPC performs the GetComponentConfig call against the live-patching service: it // resolves the apiserver FQDN + cluster CA from the available node bootstrap input, carries the -// IMDS attested-data document in gRPC metadata, and returns the opaque response bytes for the -// shared parse/stage path. +// IMDS attested-data document (and, when available, the VM resource ID) in gRPC metadata, and +// returns the opaque response bytes for the shared parse/stage path. // The gRPC status is mapped onto the benign-vs-fatal taxonomy so handleFetchError is unchanged. func (a *App) fetchHotfixOverGRPC(ctx context.Context) ([]byte, error) { fqdn, caPEM, err := a.lpsTargetFromNodeConfig() @@ -67,6 +74,14 @@ func (a *App) fetchHotfixOverGRPC(ctx context.Context) ([]byte, error) { return nil, fmt.Errorf("imds attested token: %w", err) } + // The resource ID only speeds up the service's VM lookup, so a failure here is not fatal: + // send the request without it and let the service fall back to its slower path. + resourceID, err := a.vmResourceID(ctx) + if err != nil { + slog.Info("check-hotfix could not read VM resource ID from IMDS; sending request without it", "error", err) + resourceID = "" + } + conn, err := a.dialLPSGRPC(fqdn, caPEM) if err != nil { return nil, fmt.Errorf("dialing LPS gRPC: %w", err) @@ -79,8 +94,12 @@ func (a *App) fetchHotfixOverGRPC(ctx context.Context) ([]byte, error) { ctx, cancel := context.WithTimeout(ctx, lpsFetchTimeout) defer cancel() - // Carry the attested-data document that authenticates the node in request metadata. + // Carry the attested-data document that authenticates the node in request metadata, plus + // the VM resource ID when known. ctx = metadata.AppendToOutgoingContext(ctx, lpsAttestedMetadataKey, token) + if resourceID != "" { + ctx = metadata.AppendToOutgoingContext(ctx, lpsVMResourceIDMetadataKey, resourceID) + } client := lpsv1.NewLivePatchingServiceClient(conn) resp, err := client.GetComponentConfig(ctx, &lpsv1.GetComponentConfigRequest{ComponentName: ancComponentName}) diff --git a/aks-node-controller/checkhotfix_grpc_test.go b/aks-node-controller/checkhotfix_grpc_test.go index 05456cdd570..9c664d8ae9a 100644 --- a/aks-node-controller/checkhotfix_grpc_test.go +++ b/aks-node-controller/checkhotfix_grpc_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/base64" + "errors" "net" "os" "path/filepath" @@ -28,14 +29,16 @@ type mockLPSServer struct { err error // captured request state for assertions. - gotComponent string - gotToken []string + gotComponent string + gotToken []string + gotResourceID []string } func (m *mockLPSServer) GetComponentConfig(ctx context.Context, req *lpsv1.GetComponentConfigRequest) (*lpsv1.GetComponentConfigResponse, error) { m.gotComponent = req.GetComponentName() if md, ok := metadata.FromIncomingContext(ctx); ok { m.gotToken = md.Get(lpsAttestedMetadataKey) + m.gotResourceID = md.Get(lpsVMResourceIDMetadataKey) } if m.err != nil { return nil, m.err @@ -60,8 +63,12 @@ func startMockLPS(t *testing.T, srv *mockLPSServer) func(context.Context, string } } -// newGRPCTestApp wires an App for the gRPC transport: a node config carrying the apiserver FQDN, a -// stub attested token (so no real IMDS call), and the bufconn dialer for the given mock server. +// testVMResourceID is the stub IMDS compute resourceId newGRPCTestApp injects. +const testVMResourceID = "/subscriptions/sub/resourceGroups/MC_rg/providers/Microsoft.Compute/virtualMachines/aks-nodepool1-12345678-vms1" + +// newGRPCTestApp wires an App for the gRPC transport: a node config carrying the apiserver FQDN, +// stub attested token and VM resource ID (so no real IMDS call), and the bufconn dialer for the +// given mock server. func newGRPCTestApp(t *testing.T, srv *mockLPSServer) *TestApp { t.Helper() tt := NewTestApp(t, TestAppConfig{}) @@ -76,6 +83,9 @@ func newGRPCTestApp(t *testing.T, srv *mockLPSServer) *TestApp { tt.App.fetchAttestedToken = func(context.Context) (string, error) { return "attested-doc-token", nil } + tt.App.fetchVMResourceID = func(context.Context) (string, error) { + return testVMResourceID, nil + } return tt } @@ -91,9 +101,31 @@ func TestFetchHotfixOverGRPC_Success(t *testing.T) { require.NoError(t, err) assert.Equal(t, body, got) - // The request must carry the ANC component name and the attested-data token in metadata. + // The request must carry the ANC component name, the attested-data token, and the VM + // resource ID in metadata. assert.Equal(t, ancComponentName, srv.gotComponent) assert.Equal(t, []string{"attested-doc-token"}, srv.gotToken) + assert.Equal(t, []string{testVMResourceID}, srv.gotResourceID) +} + +func TestFetchHotfixOverGRPC_ResourceIDUnavailableStillSends(t *testing.T) { + // The resource ID is only a lookup hint: an IMDS compute failure must not fail the fetch, + // and the request goes out with the token but without the resource ID key. + body := lpsPointerBody(t, map[string]string{"202604.01": "202604.01.1"}) + srv := &mockLPSServer{resp: &lpsv1.GetComponentConfigResponse{ + ComponentName: ancComponentName, + Config: string(body), + }} + tt := newGRPCTestApp(t, srv) + tt.App.fetchVMResourceID = func(context.Context) (string, error) { + return "", errors.New("imds compute down") + } + + got, err := tt.App.fetchHotfixOverGRPC(context.Background()) + require.NoError(t, err) + assert.Equal(t, body, got) + assert.Equal(t, []string{"attested-doc-token"}, srv.gotToken) + assert.Empty(t, srv.gotResourceID) } func TestCheckHotfix_GRPCSuccessReadAndWrite(t *testing.T) { diff --git a/aks-node-controller/checkhotfix_test.go b/aks-node-controller/checkhotfix_test.go index 64a60b4e2b8..72fdf0a71e3 100644 --- a/aks-node-controller/checkhotfix_test.go +++ b/aks-node-controller/checkhotfix_test.go @@ -7,6 +7,8 @@ import ( "errors" "fmt" "log/slog" + "net/http" + "net/http/httptest" "os" "path/filepath" "runtime" @@ -568,6 +570,68 @@ func TestAttestedToken_InjectionOverridesIMDS(t *testing.T) { }) } +func TestVMResourceID_InjectionOverridesIMDS(t *testing.T) { + t.Run("injected resource ID is returned without networking", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + tt.App.fetchVMResourceID = func(context.Context) (string, error) { + return "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm", nil + } + rid, err := tt.App.vmResourceID(context.Background()) + require.NoError(t, err) + assert.Equal(t, "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm", rid) + }) + + t.Run("injected error propagates", func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + tt.App.fetchVMResourceID = func(context.Context) (string, error) { + return "", errors.New("imds down") + } + _, err := tt.App.vmResourceID(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "imds down") + }) +} + +func TestGetIMDSJSON(t *testing.T) { + t.Run("sends Metadata header and decodes the body", func(t *testing.T) { + var gotHeader string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotHeader = r.Header.Get("Metadata") + _, _ = w.Write([]byte(`{"resourceId":"/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm","name":"vm"}`)) + })) + defer server.Close() + + var compute struct { + ResourceID string `json:"resourceId"` + } + require.NoError(t, getIMDSJSON(context.Background(), server.URL, &compute)) + assert.Equal(t, "true", gotHeader) + assert.Equal(t, "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm", compute.ResourceID) + }) + + t.Run("non-200 status is an error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + var out map[string]any + err := getIMDSJSON(context.Background(), server.URL, &out) + require.Error(t, err) + assert.Contains(t, err.Error(), "500") + }) + + t.Run("malformed body is an error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`not json`)) + })) + defer server.Close() + + var out map[string]any + require.Error(t, getIMDSJSON(context.Background(), server.URL, &out)) + }) +} + func TestLPSTargetFromNodeConfig(t *testing.T) { // A minimal AKSNodeConfig in the on-disk shape: MarshalConfigurationV1 sets // UseProtoNames=true, so production JSON uses proto (snake_case) field names. diff --git a/aks-node-controller/cloud_certificate_store.go b/aks-node-controller/cloud_certificate_store.go new file mode 100644 index 00000000000..9c559bb1e53 --- /dev/null +++ b/aks-node-controller/cloud_certificate_store.go @@ -0,0 +1,126 @@ +package main + +import ( + "bytes" + "context" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "sort" + "strings" +) + +func (c *cloudInitializer) clearCertificateCache() { + directory := c.path("/root/AzureCACertificates") + if err := os.MkdirAll(directory, 0o755); err != nil { + slog.Warn("could not create Azure cloud certificate cache", "error", err) + return + } + entries, err := os.ReadDir(directory) + if err != nil { + slog.Warn("could not read Azure cloud certificate cache", "error", err) + return + } + // Certificate downloads are written directly here as files, without subdirectories. + for _, entry := range entries { + if err := os.Remove(filepath.Join(directory, entry.Name())); err != nil { + slog.Warn("could not remove cached Azure cloud certificate", "name", entry.Name(), "error", err) + } + } +} + +func publicCloudCertificates(data []byte) ([]byte, error) { + var result bytes.Buffer + for { + block, rest := pem.Decode(data) + if block == nil { + break + } + data = rest + if block.Type != "CERTIFICATE" { + continue + } + if _, err := x509.ParseCertificate(block.Bytes); err != nil { + return nil, fmt.Errorf("invalid CA certificate: %w", err) + } + if err := pem.Encode(&result, &pem.Block{Type: "CERTIFICATE", Bytes: block.Bytes}); err != nil { + return nil, err + } + } + if result.Len() == 0 { + return nil, errors.New("response contains no PEM certificates") + } + return result.Bytes(), nil +} + +func (c *cloudInitializer) installCertificates(ctx context.Context, platform cloudPlatform, certificates map[string][]byte) error { + destination, update := "/usr/local/share/ca-certificates", "update-ca-certificates" + switch platform.ID { + case osReleaseIDAzureLinux, osReleaseIDMariner, osReleaseIDAzureContainerLinux: + destination, update = "/etc/pki/ca-trust/source/anchors", "update-ca-trust" + case osReleaseIDFlatcar: + destination = "/etc/ssl/certs" + } + names := make([]string, 0, len(certificates)) + for name := range certificates { + names = append(names, name) + } + sort.Strings(names) + installed := 0 + for _, name := range names { + data := certificates[name] + if !strings.HasSuffix(name, ".crt") { + continue + } + targetName := name + if platform.ID == osReleaseIDFlatcar { + targetName = strings.TrimSuffix(name, ".crt") + ".pem" + } + if err := c.writeFile(filepath.Join(destination, targetName), data); err != nil { + return err + } + installed++ + if os.Getenv("AKS_CLOUD_LOG_CERTIFICATES") == "true" { + public, err := publicCloudCertificates(data) + if err != nil { + slog.Warn("could not decode public Azure cloud CA certificates for logging", "name", name, "error", err) + continue + } + slog.Info("Azure cloud public CA certificate", "name", name, "certificate", string(public)) + } + } + if installed == 0 { + return errors.New("no *.crt files in Azure cloud certificate cache to install") + } + if _, err := c.command(ctx, nil, update); err != nil { + return err + } + if platform.ID == osReleaseIDUbuntu { + return c.copyOpenSSLBundle() + } + return nil +} + +func (c *cloudInitializer) copyOpenSSLBundle() error { + source, target := c.path("/etc/ssl/certs/ca-certificates.crt"), c.path("/usr/lib/ssl/cert.pem") + sourceInfo, err := os.Stat(source) + if err != nil { + return err + } + targetInfo, err := os.Stat(target) + if err == nil && os.SameFile(sourceInfo, targetInfo) { + return nil + } + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + data, err := os.ReadFile(source) + if err != nil { + return err + } + return c.writeFile("/usr/lib/ssl/cert.pem", data) +} diff --git a/aks-node-controller/cloud_certificate_store_behavior_test.go b/aks-node-controller/cloud_certificate_store_behavior_test.go new file mode 100644 index 00000000000..01e807de391 --- /dev/null +++ b/aks-node-controller/cloud_certificate_store_behavior_test.go @@ -0,0 +1,103 @@ +package main + +import ( + "encoding/pem" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCloudCertificateStorePlatformInstallation(t *testing.T) { + for _, tc := range []struct{ id, target, command string }{ + {osReleaseIDUbuntu, "/usr/local/share/ca-certificates/root.crt", "update-ca-certificates"}, + {osReleaseIDMariner, "/etc/pki/ca-trust/source/anchors/root.crt", "update-ca-trust"}, + {osReleaseIDAzureLinux, "/etc/pki/ca-trust/source/anchors/root.crt", "update-ca-trust"}, + {osReleaseIDAzureContainerLinux, "/etc/pki/ca-trust/source/anchors/root.crt", "update-ca-trust"}, + {osReleaseIDFlatcar, "/etc/ssl/certs/root.pem", "update-ca-certificates"}, + } { + t.Run(tc.id, func(t *testing.T) { + c := newTestCloudInitializer(t, "ID="+tc.id+"\n") + platform, err := c.platform() + require.NoError(t, err) + var commands []string + c.run = func(cmd *exec.Cmd) error { + commands = append(commands, cmd.Args[0]) + return nil + } + payload := []byte("unparsed certificate payload\n") + require.NoError(t, c.installCertificates(t.Context(), platform, map[string][]byte{ + "root.crt": payload, + "skip.cer": []byte("ignored"), + })) + got, err := os.ReadFile(c.path(tc.target)) + require.NoError(t, err) + require.Equal(t, payload, got) + require.Equal(t, []string{tc.command}, commands) + _, err = os.Stat(c.path(filepath.Join(filepath.Dir(tc.target), "skip.cer"))) + require.ErrorIs(t, err, os.ErrNotExist) + if tc.id == osReleaseIDUbuntu { + bundle, err := os.ReadFile(c.path("/usr/lib/ssl/cert.pem")) + require.NoError(t, err) + require.Equal(t, "system-ca-bundle", string(bundle)) + } + }) + } +} + +func TestCloudCertificateStoreRejectsMissingCRTFiles(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + c.run = func(*exec.Cmd) error { + t.Fatal("empty certificate sets must not invoke trust-store tools") + return nil + } + err := c.installCertificates(t.Context(), cloudPlatform{ID: osReleaseIDUbuntu}, map[string][]byte{ + "root.cer": []byte("ignored"), + }) + require.ErrorContains(t, err, "no *.crt files") +} + +func TestCloudCertificateStoreClearsOnlyFlatStagingFiles(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + for _, name := range []string{"root.crt", ".hidden.crt", "nested/keep.crt"} { + require.NoError(t, c.writeFile("/root/AzureCACertificates/"+name, []byte(name))) + } + installed := "/usr/local/share/ca-certificates/installed.crt" + require.NoError(t, c.writeFile(installed, []byte("installed CA"))) + c.clearCertificateCache() + for _, name := range []string{"root.crt", ".hidden.crt"} { + _, err := os.Stat(c.path("/root/AzureCACertificates/" + name)) + require.ErrorIs(t, err, os.ErrNotExist) + } + for _, name := range []string{"/root/AzureCACertificates/nested/keep.crt", installed} { + _, err := os.Stat(c.path(name)) + require.NoError(t, err) + } +} + +func TestCloudCertificateStoreLoggingPreservesRawPayload(t *testing.T) { + t.Setenv("AKS_CLOUD_LOG_CERTIFICATES", "true") + logs := installLogCapturer(t) + c := newTestCloudInitializer(t, "ID=azurelinux\n") + payload := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte("not-a-real-private-key")}) + payload = append(payload, cloudTestCertificate(t)...) + require.NoError(t, c.installCertificates(t.Context(), cloudPlatform{ID: osReleaseIDAzureLinux}, map[string][]byte{ + "root.crt": payload, + })) + got, err := os.ReadFile(c.path("/etc/pki/ca-trust/source/anchors/root.crt")) + require.NoError(t, err) + require.Equal(t, payload, got) + found := false + for _, record := range logs.getRecords() { + for _, value := range record.Attrs { + require.NotContains(t, value, "PRIVATE KEY") + if strings.Contains(value, "BEGIN CERTIFICATE") { + found = true + } + } + } + require.True(t, found) +} diff --git a/aks-node-controller/cloud_certificate_store_test.go b/aks-node-controller/cloud_certificate_store_test.go new file mode 100644 index 00000000000..f8f37c6198d --- /dev/null +++ b/aks-node-controller/cloud_certificate_store_test.go @@ -0,0 +1,18 @@ +package main + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCloudCertificateUbuntuBundleSymlink(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + require.NoError(t, os.MkdirAll(c.path("/usr/lib/ssl"), 0o755)) + require.NoError(t, os.Symlink(c.path("/etc/ssl/certs/ca-certificates.crt"), c.path("/usr/lib/ssl/cert.pem"))) + require.NoError(t, c.copyOpenSSLBundle()) + info, err := os.Lstat(c.path("/usr/lib/ssl/cert.pem")) + require.NoError(t, err) + require.NotZero(t, info.Mode()&os.ModeSymlink) +} diff --git a/aks-node-controller/cloud_certificate_transport.go b/aks-node-controller/cloud_certificate_transport.go new file mode 100644 index 00000000000..04246ab34e6 --- /dev/null +++ b/aks-node-controller/cloud_certificate_transport.go @@ -0,0 +1,179 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "maps" + "net/http" + "path/filepath" + "strings" + "time" + + "github.com/Azure/agentbaker/aks-node-controller/common" +) + +const cloudRequestAttempts = 10 + +func (c *cloudInitializer) requestCertificates(ctx context.Context, path string) ([]byte, error) { + attempt := 0 + delay, nextDelay := 3*time.Second, time.Duration(0) + return common.RetryFetch(ctx, cloudRequestAttempts, nil, func(ctx context.Context) ([]byte, error) { + if attempt > 0 { + if err := c.wait(ctx, nextDelay); err != nil { + return nil, err + } + } + attempt++ + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.wireEndpoint+path, nil) + if err != nil { + return nil, err + } + nextDelay = delay + resp, err := c.wireClient.Do(req) + if err != nil { + return nil, err + } + body, err := readCloudResponse(resp) + if err != nil { + return nil, err + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return body, nil + } + if resp.StatusCode == http.StatusForbidden && bytes.Contains(body, []byte("RequestRateLimitExceeded")) { + delay *= 2 + } + return nil, fmt.Errorf("WireServer certificate request %s returned HTTP %d", path, resp.StatusCode) + }) +} + +func certificatePackagePath(kind, extension string) string { + // Match the shell's raw query values without assuming WireServer's decoding contract. + return "/machine?comp=acmspackage&type=" + kind + "&ext=" + extension +} + +func validCloudCertificateName(name string) bool { + return name != "" && name != "." && name != ".." && filepath.Base(name) == name && + !strings.ContainsRune(name, 0) +} + +func (c *cloudInitializer) legacyCertificates(ctx context.Context) (map[string][]byte, error) { + body, err := c.requestCertificates(ctx, certificatePackagePath("cacertificates", "json")) + if err != nil { + return nil, fmt.Errorf("retrieve legacy Azure cloud certificates: %w", err) + } + var response struct { + Certificates []struct { + Name json.RawMessage + Body json.RawMessage `json:"CertBody"` + } + } + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("decode legacy Azure cloud certificates: %w", err) + } + if response.Certificates == nil { + return nil, errors.New("legacy Azure cloud response is missing a Certificates array") + } + certificates := make(map[string][]byte) + for _, cert := range response.Certificates { + var name string + if err := json.Unmarshal(cert.Name, &name); err != nil || !validCloudCertificateName(name) { + slog.Warn("rejecting invalid Azure cloud certificate filename") + continue + } + var certBody *string + if err := json.Unmarshal(cert.Body, &certBody); err != nil { + return nil, fmt.Errorf("decode legacy Azure cloud certificate body %s: %w", name, err) + } + if certBody == nil { + return nil, fmt.Errorf("legacy Azure cloud certificate %s is missing CertBody", name) + } + // Shell command substitution trims trailing LFs before CRLF normalization. + data := strings.ReplaceAll(strings.TrimRight(*certBody, "\n"), "\r\n", "\n") + "\n" + name = strings.ReplaceAll(name, ".cer", ".crt") + if err := c.writeFile("/root/AzureCACertificates/"+name, []byte(data)); err != nil { + return nil, fmt.Errorf("cache legacy Azure cloud certificate %s: %w", name, err) + } + certificates[name] = []byte(data) + } + return certificates, nil +} + +func (c *cloudInitializer) operationCertificates(ctx context.Context) (map[string][]byte, error) { + certificates := make(map[string][]byte) + for _, kind := range []string{"operationrequestsroot", "operationrequestsintermediate"} { + group, err := c.operationCertificateGroup(ctx, kind) + if err != nil { + return nil, err + } + maps.Copy(certificates, group) + } + return certificates, nil +} + +func (c *cloudInitializer) operationCertificateGroup(ctx context.Context, kind string) (map[string][]byte, error) { + body, err := c.requestCertificates(ctx, certificatePackagePath(kind, "json")) + if err != nil { + return nil, fmt.Errorf("retrieve Azure cloud %s: %w", kind, err) + } + var response struct { + Operations []struct { + // ResouceFileName is the spelling used by the WireServer protocol. + Name json.RawMessage `json:"ResouceFileName"` + } `json:"OperationsInfo"` + } + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("decode Azure cloud %s: %w", kind, err) + } + certificates := make(map[string][]byte) + found := false + for _, operation := range response.Operations { + var name string + if err := json.Unmarshal(operation.Name, &name); err != nil || name == "" { + slog.Warn("ignoring missing or non-string Azure cloud certificate filename") + continue + } + found = true + if !validCloudCertificateName(name) { + slog.Warn("rejecting invalid Azure cloud certificate filename") + continue + } + data, err := c.operationCertificate(ctx, name) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + slog.Warn("could not retrieve Azure cloud certificate, continuing with remaining certificates", "name", name, "error", err) + continue + } + if err := c.writeFile("/root/AzureCACertificates/"+name, data); err != nil { + return nil, fmt.Errorf("cache Azure cloud certificate %s: %w", name, err) + } + certificates[name] = data + } + if !found { + return nil, fmt.Errorf("cloud %s contains no certificate filenames", kind) + } + return certificates, nil +} + +func (c *cloudInitializer) operationCertificate(ctx context.Context, filename string) ([]byte, error) { + name, extension := filename, filename + if suffix := filepath.Ext(filename); suffix != "" { + name = strings.TrimSuffix(filename, suffix) + extension = strings.TrimPrefix(suffix, ".") + } + body, err := c.requestCertificates(ctx, certificatePackagePath(name, extension)) + if err != nil { + return nil, fmt.Errorf("retrieve Azure cloud certificate %s: %w", filename, err) + } + body = bytes.TrimRight(body, "\n") + if len(body) == 0 { + return nil, fmt.Errorf("cloud certificate %s response is empty", filename) + } + return append(body, '\n'), nil +} diff --git a/aks-node-controller/cloud_certificate_transport_behavior_test.go b/aks-node-controller/cloud_certificate_transport_behavior_test.go new file mode 100644 index 00000000000..7145af50073 --- /dev/null +++ b/aks-node-controller/cloud_certificate_transport_behavior_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCloudLegacyCertificateTransport(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + c.run = func(*exec.Cmd) error { + t.Fatal("certificate retrieval must not invoke trust-store tools") + return nil + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query := r.URL.Query() + if r.URL.Path != "/machine" || query.Get("comp") != "acmspackage" || + query.Get("type") != "cacertificates" || query.Get("ext") != "json" { + t.Errorf("unexpected legacy certificate request: %s", r.URL) + w.WriteHeader(http.StatusBadRequest) + return + } + _, _ = io.WriteString(w, `{"Certificates":[{"Name":"root.cer","CertBody":"first\r\nsecond\r\n"},{"Name":"../escape.cer","CertBody":"ignored"}]}`) + })) + defer server.Close() + c.wireEndpoint = server.URL + certificates, err := c.legacyCertificates(t.Context()) + require.NoError(t, err) + payload := []byte("first\nsecond\r\n") + require.Equal(t, map[string][]byte{"root.crt": payload}, certificates) + cached, err := os.ReadFile(c.path("/root/AzureCACertificates/root.crt")) + require.NoError(t, err) + require.Equal(t, payload, cached) + _, err = os.Stat(c.path("/root/escape.crt")) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestCloudOperationCertificateTransport(t *testing.T) { + c := newTestCloudInitializer(t, "ID=azurelinux\n") + c.run = func(*exec.Cmd) error { + t.Fatal("certificate retrieval must not invoke trust-store tools") + return nil + } + serveCloudOperationTestResponses(t, c, + []string{"missing.crt", "empty.crt", "root+ca.crt", "unfiltered.cer"}, + []string{"intermediate.crt"}) + certificates, err := c.operationCertificates(t.Context()) + require.NoError(t, err) + payload := []byte("certificate payload\n") + require.Equal(t, map[string][]byte{ + "root+ca.crt": payload, + "unfiltered.cer": payload, + "intermediate.crt": payload, + }, certificates) + for name, want := range certificates { + cached, readErr := os.ReadFile(c.path("/root/AzureCACertificates/" + name)) + require.NoError(t, readErr) + require.Equal(t, want, cached) + } + _, err = os.Stat(c.path("/etc/pki/ca-trust/source/anchors")) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func TestCloudCertificateTransportPreservesShellQuery(t *testing.T) { + for _, tc := range []struct { + name string + filename string + wantRequestURI string + }{ + { + name: "ordinary resource", + filename: "root.crt", + wantRequestURI: "/machine?comp=acmspackage&type=root&ext=crt", + }, + { + name: "literal plus", + filename: "root+ca.crt", + wantRequestURI: "/machine?comp=acmspackage&type=root+ca&ext=crt", + }, + { + name: "percent-escaped plus", + filename: "root%2Bca.crt", + wantRequestURI: "/machine?comp=acmspackage&type=root%2Bca&ext=crt", + }, + { + name: "ampersand", + filename: "root&ca.crt", + wantRequestURI: "/machine?comp=acmspackage&type=root&ca&ext=crt", + }, + { + name: "plus in extension", + filename: "root.crt+ca", + wantRequestURI: "/machine?comp=acmspackage&type=root&ext=crt+ca", + }, + { + name: "percent escape in extension", + filename: "root.c%2Brt", + wantRequestURI: "/machine?comp=acmspackage&type=root&ext=c%2Brt", + }, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.RequestURI != tc.wantRequestURI { + t.Errorf("certificate request differs from init-aks-cloud.sh: got %q, want %q", r.RequestURI, tc.wantRequestURI) + } + _, _ = io.WriteString(w, "raw certificate payload\n\n") + })) + defer server.Close() + c.wireEndpoint = server.URL + payload, err := c.operationCertificate(t.Context(), tc.filename) + require.NoError(t, err) + require.Equal(t, "raw certificate payload\n", string(payload)) + }) + } +} diff --git a/aks-node-controller/cloud_certificate_transport_json_test.go b/aks-node-controller/cloud_certificate_transport_json_test.go new file mode 100644 index 00000000000..4396708e926 --- /dev/null +++ b/aks-node-controller/cloud_certificate_transport_json_test.go @@ -0,0 +1,174 @@ +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCloudOperationCertificateGroupJSON(t *testing.T) { + payloads := map[string][]byte{ + "root.crt": cloudTestCertificate(t), + "second.crt": cloudTestCertificate(t), + } + testCases := []struct { + name string + response string + wantNames []string + wantErr string + }{ + { + name: "one space after colon", + response: `{"OperationsInfo":[{"ResouceFileName": "root.crt"}]}`, + wantNames: []string{"root.crt"}, + }, + { + name: "compact JSON", + response: `{"OperationsInfo":[{"ResouceFileName":"root.crt"}]}`, + wantNames: []string{"root.crt"}, + }, + { + name: "indented JSON", + response: `{ + "OperationsInfo": [ + { + "ResouceFileName": "root.crt" + } + ] +}`, + wantNames: []string{"root.crt"}, + }, + { + name: "tabs CRLF and whitespace around colon", + response: "\r\n{\r\n\t\"OperationsInfo\"\t:\t[\r\n\t{\"ResouceFileName\"\r\n:\t \"root.crt\"}\r\n]\r\n}\t\r\n", + wantNames: []string{"root.crt"}, + }, + { + name: "JSON escaped filename", + response: `{"OperationsInfo":[{"ResouceFileName":"\u0072oot.crt"}]}`, + wantNames: []string{"root.crt"}, + }, + { + name: "ignore filenames at other levels", + response: `{ + "ResouceFileName": "wrong.crt", + "Metadata": {"ResouceFileName": "nested.crt"}, + "OperationsInfo": [ + {"ResouceFileName": "root.crt", "Metadata": {"ResouceFileName": "nested-entry.crt"}} + ] +}`, + wantNames: []string{"root.crt"}, + }, + { + name: "skip non-string and missing names", + response: `{"OperationsInfo":[ + {"ResouceFileName":42}, + {"ResouceFileName":null}, + {"OtherField":"ignored"}, + {"ResouceFileName":"root.crt"} +]}`, + wantNames: []string{"root.crt"}, + }, + { + name: "multiple resources", + response: `{"OperationsInfo":[{"ResouceFileName":"root.crt"},{"ResouceFileName":"second.crt"}]}`, + wantNames: []string{"root.crt", "second.crt"}, + }, + { + name: "truncated JSON containing a filename", + response: `{"OperationsInfo":[{"ResouceFileName": "root.crt"}]`, + wantErr: "decode Azure cloud", + }, + { + name: "filename only at the wrong level", + response: `{"ResouceFileName": "root.crt"}`, + wantErr: "contains no certificate filenames", + }, + { + name: "top-level array", + response: `[{"ResouceFileName": "root.crt"}]`, + wantErr: "decode Azure cloud", + }, + { + name: "OperationsInfo is not an array", + response: `{"OperationsInfo":{"ResouceFileName": "root.crt"}}`, + wantErr: "decode Azure cloud", + }, + { + name: "corrected spelling is not the protocol field", + response: `{"OperationsInfo":[{"ResourceFileName": "root.crt"}]}`, + wantErr: "contains no certificate filenames", + }, + { + name: "null OperationsInfo", + response: `{"OperationsInfo":null}`, + wantErr: "contains no certificate filenames", + }, + { + name: "empty OperationsInfo", + response: `{"OperationsInfo":[]}`, + wantErr: "contains no certificate filenames", + }, + { + name: "no string filenames", + response: `{"OperationsInfo":[{"ResouceFileName":42},{"ResouceFileName":null}]}`, + wantErr: "contains no certificate filenames", + }, + } + for _, kind := range []string{"operationrequestsroot", "operationrequestsintermediate"} { + for _, tc := range testCases { + t.Run(kind+"/"+tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + manifestURI := "/machine?comp=acmspackage&type=" + kind + "&ext=json" + var manifests, downloads atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method + " " + r.RequestURI { + case http.MethodGet + " " + manifestURI: + manifests.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, tc.response) + case http.MethodGet + " /machine?comp=acmspackage&type=root&ext=crt": + downloads.Add(1) + _, _ = w.Write(payloads["root.crt"]) + case http.MethodGet + " /machine?comp=acmspackage&type=second&ext=crt": + downloads.Add(1) + _, _ = w.Write(payloads["second.crt"]) + default: + t.Errorf("unexpected certificate request: %s %s", r.Method, r.RequestURI) + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + c.wireEndpoint = server.URL + + certificates, err := c.operationCertificateGroup(t.Context(), kind) + require.EqualValues(t, 1, manifests.Load()) + require.EqualValues(t, len(tc.wantNames), downloads.Load()) + if tc.wantErr != "" { + require.ErrorContains(t, err, tc.wantErr) + require.Nil(t, certificates) + _, statErr := os.Stat(c.path("/root/AzureCACertificates")) + require.ErrorIs(t, statErr, os.ErrNotExist) + return + } + require.NoError(t, err) + want := make(map[string][]byte, len(tc.wantNames)) + for _, name := range tc.wantNames { + want[name] = payloads[name] + cached, readErr := os.ReadFile(c.path("/root/AzureCACertificates/" + name)) + require.NoError(t, readErr) + require.Equal(t, payloads[name], cached) + } + require.Equal(t, want, certificates) + cached, readErr := os.ReadDir(c.path("/root/AzureCACertificates")) + require.NoError(t, readErr) + require.Len(t, cached, len(tc.wantNames)) + }) + } + } +} diff --git a/aks-node-controller/cloud_certificate_transport_test.go b/aks-node-controller/cloud_certificate_transport_test.go new file mode 100644 index 00000000000..5ca92e7db52 --- /dev/null +++ b/aks-node-controller/cloud_certificate_transport_test.go @@ -0,0 +1,246 @@ +package main + +import ( + "context" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "os" + "strconv" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCloudCertificateRequestRetryAndCancellation(t *testing.T) { + for _, tc := range []struct { + name string + status int + body string + failures int32 + wantWaits []time.Duration + }{ + {"transient", http.StatusServiceUnavailable, "do-not-log-body", 2, []time.Duration{3 * time.Second, 3 * time.Second}}, + {"rate limited", http.StatusForbidden, "RequestRateLimitExceeded", 2, []time.Duration{3 * time.Second, 6 * time.Second}}, + {"exhausted", http.StatusServiceUnavailable, "do-not-log-body", cloudRequestAttempts, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if attempts.Add(1) <= tc.failures { + w.WriteHeader(tc.status) + _, _ = io.WriteString(w, tc.body) + return + } + _, _ = io.WriteString(w, "success") + })) + defer server.Close() + c.wireEndpoint = server.URL + var waits []time.Duration + c.wait = func(_ context.Context, delay time.Duration) error { + waits = append(waits, delay) + return nil + } + body, err := c.requestCertificates(t.Context(), "/test") + if tc.failures == cloudRequestAttempts { + require.Error(t, err) + require.NotContains(t, err.Error(), tc.body) + require.EqualValues(t, cloudRequestAttempts, attempts.Load()) + require.Len(t, waits, cloudRequestAttempts-1) + } else { + require.NoError(t, err) + require.Equal(t, "success", string(body)) + require.Equal(t, tc.wantWaits, waits) + } + }) + } + t.Run("cancellation during backoff", func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + c.wireEndpoint = server.URL + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + c.wait = func(ctx context.Context, _ time.Duration) error { + cancel() + return ctx.Err() + } + _, err := c.requestCertificates(ctx, "/test") + require.ErrorIs(t, err, context.Canceled) + require.EqualValues(t, 1, attempts.Load()) + }) +} + +func TestCloudCertificateTruncatedHTTP200(t *testing.T) { + complete := cloudTestCertificate(t) + for _, tc := range []struct { + name string + failures int32 + }{ + {"recover on retry", 1}, + {"exhaust retries", cloudRequestAttempts}, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempt := attempts.Add(1) + w.Header().Set("Content-Length", strconv.Itoa(len(complete))) + w.WriteHeader(http.StatusOK) + if attempt <= tc.failures { + // Returning fewer bytes than Content-Length closes the response early. + _, _ = w.Write(complete[:len(complete)/2]) + return + } + _, _ = w.Write(complete) + })) + defer server.Close() + c.wireEndpoint = server.URL + waits := 0 + c.wait = func(ctx context.Context, _ time.Duration) error { + waits++ + return ctx.Err() + } + + body, err := c.requestCertificates(t.Context(), certificatePackagePath("root", "crt")) + wantAttempts := tc.failures + 1 + if tc.failures == cloudRequestAttempts { + wantAttempts = cloudRequestAttempts + require.ErrorIs(t, err, io.ErrUnexpectedEOF) + require.Nil(t, body) + } else { + require.NoError(t, err) + require.Equal(t, complete, body) + } + require.Equal(t, wantAttempts, attempts.Load()) + require.EqualValues(t, wantAttempts-1, waits) + }) + } +} + +func TestCloudOperationCertificateTruncatedDownload(t *testing.T) { + complete := cloudTestCertificate(t) + for _, failures := range []int32{1, cloudRequestAttempts} { + t.Run(strconv.Itoa(int(failures)), func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.RequestURI { + case "/machine?comp=acmspackage&type=operationrequestsroot&ext=json": + _, _ = io.WriteString(w, `{"OperationsInfo":[{"ResouceFileName":"truncated.crt"},{"ResouceFileName":"complete.crt"}]}`) + case "/machine?comp=acmspackage&type=truncated&ext=crt": + attempt := attempts.Add(1) + w.Header().Set("Content-Length", strconv.Itoa(len(complete))) + w.WriteHeader(http.StatusOK) + if attempt <= failures { + _, _ = w.Write(complete[:len(complete)/2]) + return + } + _, _ = w.Write(complete) + case "/machine?comp=acmspackage&type=complete&ext=crt": + _, _ = w.Write(complete) + default: + t.Errorf("unexpected certificate request: %s", r.RequestURI) + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + c.wireEndpoint = server.URL + + certificates, err := c.operationCertificateGroup(t.Context(), "operationrequestsroot") + require.NoError(t, err) + want := map[string][]byte{"complete.crt": complete} + wantAttempts := failures + 1 + if failures == cloudRequestAttempts { + wantAttempts = cloudRequestAttempts + _, statErr := os.Stat(c.path("/root/AzureCACertificates/truncated.crt")) + require.ErrorIs(t, statErr, os.ErrNotExist) + } else { + want["truncated.crt"] = complete + } + require.Equal(t, wantAttempts, attempts.Load()) + require.Equal(t, want, certificates) + for name, payload := range want { + cached, readErr := os.ReadFile(c.path("/root/AzureCACertificates/" + name)) + require.NoError(t, readErr) + require.Equal(t, payload, cached) + } + }) + } +} + +func TestCloudCertificateCloseDelimitedResponse(t *testing.T) { + complete := cloudTestCertificate(t) + partial := complete[:len(complete)/2] + c := newTestCloudInitializer(t, cloudTestUbuntu) + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + hijacker, ok := w.(http.Hijacker) + if !ok { + t.Error("test server does not support connection hijacking") + return + } + conn, writer, err := hijacker.Hijack() + if err != nil { + t.Errorf("hijack test connection: %v", err) + return + } + defer conn.Close() + if _, err := writer.WriteString("HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n" + string(partial)); err != nil { + t.Errorf("write close-delimited response: %v", err) + return + } + if err := writer.Flush(); err != nil { + t.Errorf("flush close-delimited response: %v", err) + } + })) + defer server.Close() + c.wireEndpoint = server.URL + + body, err := c.requestCertificates(t.Context(), certificatePackagePath("root", "crt")) + require.NoError(t, err) + require.Equal(t, partial, body) + require.EqualValues(t, 1, attempts.Load()) +} + +func TestCloudCertificateRequestsDoNotFollowRedirects(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var redirected atomic.Int32 + target := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { redirected.Add(1) })) + defer target.Close() + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer source.Close() + c.wireEndpoint = source.URL + _, err := c.requestCertificates(t.Context(), "/test") + require.ErrorContains(t, err, "HTTP 302") + require.Zero(t, redirected.Load()) +} + +func TestCloudCertificateNamesAndPublicContent(t *testing.T) { + for _, name := range []string{"", ".", "..", "../root.crt", "/root.crt", "root\x00.crt"} { + require.False(t, validCloudCertificateName(name), name) + } + for _, name := range []string{"root+ca.crt", `..\root.crt`, "root\n.crt"} { + require.True(t, validCloudCertificateName(name), name) + } + cert := cloudTestCertificate(t) + private := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte("not-a-real-private-key")}) + got, err := publicCloudCertificates(append(private, cert...)) + require.NoError(t, err) + require.Equal(t, cert, got) + for _, bad := range [][]byte{nil, private, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte("invalid")})} { + _, err = publicCloudCertificates(bad) + require.Error(t, err) + } +} diff --git a/aks-node-controller/cloud_certificates.go b/aks-node-controller/cloud_certificates.go new file mode 100644 index 00000000000..6dffa5563c3 --- /dev/null +++ b/aks-node-controller/cloud_certificates.go @@ -0,0 +1,342 @@ +package main + +import ( + "bytes" + "context" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "log/slog" + "maps" + "net/http" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/Azure/agentbaker/aks-node-controller/common" +) + +func (c *cloudInitializer) requestCertificates(ctx context.Context, path string) ([]byte, error) { + attempt := 0 + delay, nextDelay := 3*time.Second, time.Duration(0) + return common.RetryFetch(ctx, cloudRequestAttempts, nil, func(ctx context.Context) ([]byte, error) { + if attempt > 0 { + if err := c.wait(ctx, nextDelay); err != nil { + return nil, err + } + } + attempt++ + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.wireEndpoint+path, nil) + if err != nil { + return nil, err + } + nextDelay = delay + resp, err := c.wireClient.Do(req) + if err != nil { + return nil, err + } + body, err := readCloudResponse(resp) + if err != nil { + return nil, err + } + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return body, nil + } + if resp.StatusCode == http.StatusForbidden && bytes.Contains(body, []byte("RequestRateLimitExceeded")) { + delay *= 2 + } + return nil, fmt.Errorf("WireServer certificate request %s returned HTTP %d", path, resp.StatusCode) + }) +} + +func (c *cloudInitializer) refresh(ctx context.Context, platform cloudPlatform, location string) (bool, error) { + c.clearCertificateCache() + legacy := strings.HasPrefix(location, "ussec") || strings.HasPrefix(location, "usnat") + slog.Info("selecting Azure cloud certificate endpoint", "legacy", legacy, "location", location) + if !legacy { + body, err := c.requestCertificates(ctx, "/acms/isOptedInForRootCerts") + if err != nil { + return false, fmt.Errorf("check Azure cloud root certificate opt-in: %w", err) + } + if len(bytes.TrimRight(body, "\n")) == 0 { + return false, errors.New("cloud root certificate opt-in response is empty") + } + var response map[string]json.RawMessage + if err := json.Unmarshal(body, &response); err != nil { + slog.Warn("could not decode Azure cloud root certificate opt-in, skipping certificate installation", "error", err) + return false, nil + } + if !bytes.Equal(bytes.TrimSpace(response["IsOptedInForRootCerts"]), []byte("true")) { + slog.Info("Azure cloud root certificates are not opted in, skipping certificate installation") + return false, nil + } + } + var certificates map[string][]byte + var err error + if legacy { + certificates, err = c.legacyCertificates(ctx) + } else { + certificates, err = c.operationCertificates(ctx) + } + if err != nil { + return false, err + } + if err := c.installCertificates(ctx, platform, certificates); err != nil { + if ctx.Err() != nil { + return false, ctx.Err() + } + if legacy { + slog.Warn("legacy Azure cloud CA certificate installation failed, continuing cloud initialization", "error", err) + return true, nil + } + return false, fmt.Errorf("install Azure cloud CA certificates: %w", err) + } + slog.Info("installed Azure cloud CA certificates") + return true, nil +} + +func (c *cloudInitializer) clearCertificateCache() { + directory := c.path("/root/AzureCACertificates") + if err := os.MkdirAll(directory, 0o755); err != nil { + slog.Warn("could not create Azure cloud certificate cache", "error", err) + return + } + entries, err := os.ReadDir(directory) + if err != nil { + slog.Warn("could not read Azure cloud certificate cache", "error", err) + return + } + // Certificate downloads are written directly here as files, without subdirectories. + for _, entry := range entries { + if err := os.Remove(filepath.Join(directory, entry.Name())); err != nil { + slog.Warn("could not remove cached Azure cloud certificate", "name", entry.Name(), "error", err) + } + } +} + +func certificatePackagePath(kind, extension string) string { + return "/machine?" + url.Values{ + "comp": {"acmspackage"}, + "type": {kind}, + "ext": {extension}, + }.Encode() +} + +func validCloudCertificateName(name string) bool { + return name != "" && name != "." && name != ".." && filepath.Base(name) == name && + !strings.ContainsRune(name, 0) +} + +func (c *cloudInitializer) legacyCertificates(ctx context.Context) (map[string][]byte, error) { + body, err := c.requestCertificates(ctx, certificatePackagePath("cacertificates", "json")) + if err != nil { + return nil, fmt.Errorf("retrieve legacy Azure cloud certificates: %w", err) + } + var response struct { + Certificates []struct { + Name json.RawMessage + Body json.RawMessage `json:"CertBody"` + } + } + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("decode legacy Azure cloud certificates: %w", err) + } + if response.Certificates == nil { + return nil, errors.New("legacy Azure cloud response is missing a Certificates array") + } + certificates := make(map[string][]byte) + for _, cert := range response.Certificates { + var name string + if err := json.Unmarshal(cert.Name, &name); err != nil || !validCloudCertificateName(name) { + slog.Warn("rejecting invalid Azure cloud certificate filename") + continue + } + var certBody *string + if err := json.Unmarshal(cert.Body, &certBody); err != nil { + return nil, fmt.Errorf("decode legacy Azure cloud certificate body %s: %w", name, err) + } + if certBody == nil { + return nil, fmt.Errorf("legacy Azure cloud certificate %s is missing CertBody", name) + } + // Shell command substitution trims trailing LFs before CRLF normalization. + data := strings.ReplaceAll(strings.TrimRight(*certBody, "\n"), "\r\n", "\n") + "\n" + name = strings.ReplaceAll(name, ".cer", ".crt") + if err := c.writeFile("/root/AzureCACertificates/"+name, []byte(data)); err != nil { + return nil, fmt.Errorf("cache legacy Azure cloud certificate %s: %w", name, err) + } + certificates[name] = []byte(data) + } + return certificates, nil +} + +func (c *cloudInitializer) operationCertificates(ctx context.Context) (map[string][]byte, error) { + certificates := make(map[string][]byte) + for _, kind := range []string{"operationrequestsroot", "operationrequestsintermediate"} { + group, err := c.operationCertificateGroup(ctx, kind) + if err != nil { + return nil, err + } + maps.Copy(certificates, group) + } + return certificates, nil +} + +func (c *cloudInitializer) operationCertificateGroup(ctx context.Context, kind string) (map[string][]byte, error) { + body, err := c.requestCertificates(ctx, certificatePackagePath(kind, "json")) + if err != nil { + return nil, fmt.Errorf("retrieve Azure cloud %s: %w", kind, err) + } + var response struct { + Operations []struct { + // ResouceFileName is the spelling used by the WireServer protocol. + Name json.RawMessage `json:"ResouceFileName"` + } `json:"OperationsInfo"` + } + if err := json.Unmarshal(body, &response); err != nil { + return nil, fmt.Errorf("decode Azure cloud %s: %w", kind, err) + } + certificates := make(map[string][]byte) + found := false + for _, operation := range response.Operations { + var name string + if err := json.Unmarshal(operation.Name, &name); err != nil || name == "" { + slog.Warn("ignoring missing or non-string Azure cloud certificate filename") + continue + } + found = true + if !validCloudCertificateName(name) { + slog.Warn("rejecting invalid Azure cloud certificate filename") + continue + } + data, err := c.operationCertificate(ctx, name) + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + slog.Warn("could not retrieve Azure cloud certificate, continuing with remaining certificates", "name", name, "error", err) + continue + } + if err := c.writeFile("/root/AzureCACertificates/"+name, data); err != nil { + return nil, fmt.Errorf("cache Azure cloud certificate %s: %w", name, err) + } + certificates[name] = data + } + if !found { + return nil, fmt.Errorf("cloud %s contains no certificate filenames", kind) + } + return certificates, nil +} + +func (c *cloudInitializer) operationCertificate(ctx context.Context, filename string) ([]byte, error) { + name, extension := filename, filename + if suffix := filepath.Ext(filename); suffix != "" { + name = strings.TrimSuffix(filename, suffix) + extension = strings.TrimPrefix(suffix, ".") + } + body, err := c.requestCertificates(ctx, certificatePackagePath(name, extension)) + if err != nil { + return nil, fmt.Errorf("retrieve Azure cloud certificate %s: %w", filename, err) + } + body = bytes.TrimRight(body, "\n") + if len(body) == 0 { + return nil, fmt.Errorf("cloud certificate %s response is empty", filename) + } + return append(body, '\n'), nil +} + +func publicCloudCertificates(data []byte) ([]byte, error) { + var result bytes.Buffer + for { + block, rest := pem.Decode(data) + if block == nil { + break + } + data = rest + if block.Type != "CERTIFICATE" { + continue + } + if _, err := x509.ParseCertificate(block.Bytes); err != nil { + return nil, fmt.Errorf("invalid CA certificate: %w", err) + } + if err := pem.Encode(&result, &pem.Block{Type: "CERTIFICATE", Bytes: block.Bytes}); err != nil { + return nil, err + } + } + if result.Len() == 0 { + return nil, errors.New("response contains no PEM certificates") + } + return result.Bytes(), nil +} + +func (c *cloudInitializer) installCertificates(ctx context.Context, platform cloudPlatform, certificates map[string][]byte) error { + destination, update := "/usr/local/share/ca-certificates", "update-ca-certificates" + switch platform.ID { + case osReleaseIDAzureLinux, osReleaseIDMariner, osReleaseIDAzureContainerLinux: + destination, update = "/etc/pki/ca-trust/source/anchors", "update-ca-trust" + case osReleaseIDFlatcar: + destination = "/etc/ssl/certs" + } + names := make([]string, 0, len(certificates)) + for name := range certificates { + names = append(names, name) + } + sort.Strings(names) + installed := 0 + for _, name := range names { + data := certificates[name] + if !strings.HasSuffix(name, ".crt") { + continue + } + targetName := name + if platform.ID == osReleaseIDFlatcar { + targetName = strings.TrimSuffix(name, ".crt") + ".pem" + } + if err := c.writeFile(filepath.Join(destination, targetName), data); err != nil { + return err + } + installed++ + if os.Getenv("AKS_CLOUD_LOG_CERTIFICATES") == "true" { + public, err := publicCloudCertificates(data) + if err != nil { + slog.Warn("could not decode public Azure cloud CA certificates for logging", "name", name, "error", err) + continue + } + slog.Info("Azure cloud public CA certificate", "name", name, "certificate", string(public)) + } + } + if installed == 0 { + return errors.New("no *.crt files in Azure cloud certificate cache to install") + } + if _, err := c.command(ctx, nil, update); err != nil { + return err + } + if platform.ID == osReleaseIDUbuntu { + return c.copyOpenSSLBundle() + } + return nil +} + +func (c *cloudInitializer) copyOpenSSLBundle() error { + source, target := c.path("/etc/ssl/certs/ca-certificates.crt"), c.path("/usr/lib/ssl/cert.pem") + sourceInfo, err := os.Stat(source) + if err != nil { + return err + } + targetInfo, err := os.Stat(target) + if err == nil && os.SameFile(sourceInfo, targetInfo) { + return nil + } + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + data, err := os.ReadFile(source) + if err != nil { + return err + } + return c.writeFile("/usr/lib/ssl/cert.pem", data) +} diff --git a/aks-node-controller/cloud_certificates_test.go b/aks-node-controller/cloud_certificates_test.go new file mode 100644 index 00000000000..eeb6c4eeca4 --- /dev/null +++ b/aks-node-controller/cloud_certificates_test.go @@ -0,0 +1,559 @@ +package main + +import ( + "context" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCloudCertificateRefreshModes(t *testing.T) { + cert := cloudTestCertificate(t) + for _, platform := range []struct{ name, release, directory, filename, command string }{ + {"Ubuntu", cloudTestUbuntu, "/usr/local/share/ca-certificates", "root.crt", "update-ca-certificates"}, + {"Mariner", "ID=mariner\n", "/etc/pki/ca-trust/source/anchors", "root.crt", "update-ca-trust"}, + {"AzureLinux", "ID=azurelinux\n", "/etc/pki/ca-trust/source/anchors", "root.crt", "update-ca-trust"}, + {"Flatcar", "ID=flatcar\n", "/etc/ssl/certs", "root.pem", "update-ca-certificates"}, + {"ACL", "ID=azurecontainerlinux\n", "/etc/pki/ca-trust/source/anchors", "root.crt", "update-ca-trust"}, + } { + for _, mode := range []struct{ name, location, optIn string }{ + {"opt-out", "westus2", `{"IsOptedInForRootCerts":false}`}, + {"rcv1p", "westus2", `{"IsOptedInForRootCerts":true}`}, + {"legacy", " US NAT East ", "not used"}, + } { + t.Run(platform.name+"/"+mode.name, func(t *testing.T) { + c := newTestCloudInitializer(t, platform.release) + serveCloudTestResponses(t, c, cert, mode.optIn) + var commands []string + c.run = func(cmd *exec.Cmd) error { + commands = append(commands, cmd.Args[0]) + return nil + } + require.NoError(t, c.refreshCertificates(t.Context(), mode.location)) + if mode.name == "opt-out" { + require.Empty(t, commands) + return + } + require.Equal(t, []string{platform.command}, commands) + installed, err := os.ReadFile(c.path(filepath.Join(platform.directory, platform.filename))) + require.NoError(t, err) + expected := cert + if mode.name == "legacy" { + expected = []byte(strings.TrimSuffix(string(cert), "\n") + "\r\n") + } + require.Equal(t, expected, installed) + }) + } + } +} + +func TestCloudCertificateRefreshNormalizesLocation(t *testing.T) { + cert := cloudTestCertificate(t) + for _, tc := range []struct { + name, location string + legacy bool + }{ + {"ussec lowercase", "ussecgov", true}, + {"ussec mixed case", "usSecGov", true}, + {"ussec whitespace", " \tUS SEC GOV\r\n", true}, + {"usnat lowercase", "usnateast", true}, + {"usnat mixed case", "usNatEast", true}, + {"usnat whitespace", " US NAT East ", true}, + {"rcv1p lowercase", "westus2", false}, + {"rcv1p mixed case and whitespace", " \tWest US2\n", false}, + {"rcv1p empty", "", false}, + {"rcv1p whitespace only", " \t\n", false}, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + requests := serveCloudTestResponses(t, c, cert, `{"IsOptedInForRootCerts":true}`) + var commands []string + c.run = func(cmd *exec.Cmd) error { + commands = append(commands, cmd.Args[0]) + return nil + } + platform, err := c.platform() + require.NoError(t, err) + refreshed, err := c.refresh(t.Context(), platform, tc.location) + require.NoError(t, err) + require.True(t, refreshed) + require.Equal(t, []string{"update-ca-certificates"}, commands) + wantRequests := 5 + wantFirst := "/acms/isOptedInForRootCerts?" + if tc.legacy { + wantRequests = 1 + wantFirst = "/machine?comp=acmspackage&type=cacertificates&ext=json" + } + require.Len(t, requests, wantRequests) + require.Equal(t, wantFirst, <-requests) + }) + } +} + +func TestCloudCertificateRefreshRejectsInvalidLocationBeforeSideEffects(t *testing.T) { + for _, location := range []string{`westus";echo bad`, "westus%bad", "ussec/gov"} { + t.Run(location, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + requests := serveCloudTestResponses(t, c, nil, `{"IsOptedInForRootCerts":false}`) + c.run = func(*exec.Cmd) error { + t.Fatal("invalid locations must not update the trust store") + return nil + } + require.NoError(t, c.writeFile("/root/AzureCACertificates/stale.crt", []byte("keep"))) + platform, err := c.platform() + require.NoError(t, err) + refreshed, err := c.refresh(t.Context(), platform, location) + require.ErrorContains(t, err, "cloud location contains unsupported characters") + require.False(t, refreshed) + require.Empty(t, requests) + cached, err := os.ReadFile(c.path("/root/AzureCACertificates/stale.crt")) + require.NoError(t, err) + require.Equal(t, "keep", string(cached)) + }) + } +} + +func TestCloudRootCertificateOptIn(t *testing.T) { + for _, tc := range []struct { + name, response string + wantOptedIn bool + wantError string + }{ + {"opted in", `{"IsOptedInForRootCerts":true}`, true, ""}, + {"opted out", `{"IsOptedInForRootCerts":false}`, false, ""}, + {"malformed JSON", "not json", false, ""}, + {"missing field", `{}`, false, ""}, + {"wrong field case", `{"isOptedInForRootCerts":true}`, false, ""}, + {"string true", `{"IsOptedInForRootCerts":"true"}`, false, ""}, + {"whitespace only", " ", false, ""}, + {"empty response", "", false, "response is empty"}, + {"newlines only", "\n\n", false, "response is empty"}, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + requests := serveCloudTestResponses(t, c, nil, tc.response) + c.run = func(*exec.Cmd) error { + t.Fatal("opt-in checks must not update the trust store") + return nil + } + require.NoError(t, c.writeFile("/root/AzureCACertificates/stale.crt", []byte("keep"))) + optedIn, err := c.isOptedInForRootCerts(t.Context()) + if tc.wantError != "" { + require.ErrorContains(t, err, tc.wantError) + } else { + require.NoError(t, err) + } + require.Equal(t, tc.wantOptedIn, optedIn) + require.Len(t, requests, 1) + require.Equal(t, "/acms/isOptedInForRootCerts?", <-requests) + cached, err := os.ReadFile(c.path("/root/AzureCACertificates/stale.crt")) + require.NoError(t, err) + require.Equal(t, "keep", string(cached)) + }) + } +} + +func TestCloudRootCertificateOptInRequestErrors(t *testing.T) { + for _, tc := range []struct { + name string + canceled bool + }{ + {"HTTP failure", false}, + {"cancellation", true}, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + c.wireEndpoint = server.URL + c.wireClient = server.Client() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if tc.canceled { + cancel() + } + optedIn, err := c.isOptedInForRootCerts(ctx) + require.False(t, optedIn) + require.ErrorContains(t, err, "check Azure cloud root certificate opt-in") + if tc.canceled { + require.ErrorIs(t, err, context.Canceled) + require.Zero(t, requests.Load()) + } else { + require.Positive(t, requests.Load()) + } + }) + } +} + +func TestCloudCertificateRefreshNonTrueOptInResponsesSkip(t *testing.T) { + for _, response := range []string{ + "not json", " ", `{}`, `null`, `[]`, `{"IsOptedInForRootCerts":false}`, + `{"IsOptedInForRootCerts":null}`, `{"IsOptedInForRootCerts":"false"}`, + `{"IsOptedInForRootCerts":"true"}`, `{"IsOptedInForRootCerts":1}`, + `{"isOptedInForRootCerts":true}`, + } { + t.Run(response, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + requests := serveCloudTestResponses(t, c, nil, response) + c.run = func(*exec.Cmd) error { + t.Fatal("non-true opt-in must not update the trust store") + return nil + } + require.NoError(t, c.refreshCertificates(t.Context(), "westus2")) + require.Len(t, requests, 1) + }) + } +} + +func TestCloudLegacyCertificateRefreshTrustFailureIsBestEffort(t *testing.T) { + c := newTestCloudInitializer(t, "ID=azurelinux\n") + serveCloudTestResponses(t, c, cloudTestCertificate(t), "not used") + c.run = func(cmd *exec.Cmd) error { + require.Equal(t, "update-ca-trust", cmd.Args[0]) + return errors.New("trust store update failed") + } + require.NoError(t, c.refreshCertificates(t.Context(), "usnateast")) +} + +func TestCloudCertificateOptInEmptyResponsesFail(t *testing.T) { + for _, response := range []string{"", "\n", "\n\n"} { + t.Run(fmt.Sprintf("%q", response), func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + serveCloudTestResponses(t, c, nil, response) + require.ErrorContains(t, c.refreshCertificates(t.Context(), "westus2"), "response is empty") + }) + } +} + +func TestCloudCertificateRefreshClearsCachedFiles(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + serveCloudTestResponses(t, c, nil, `{"IsOptedInForRootCerts":false}`) + for _, name := range []string{"stale.crt", ".stale.crt", "certificate-without-extension"} { + require.NoError(t, c.writeFile("/root/AzureCACertificates/"+name, []byte("stale"))) + } + require.NoError(t, c.writeFile("/usr/local/share/ca-certificates/existing.crt", []byte("keep"))) + require.NoError(t, c.refreshCertificates(t.Context(), "westus2")) + entries, err := os.ReadDir(c.path("/root/AzureCACertificates")) + require.NoError(t, err) + require.Empty(t, entries) + data, err := os.ReadFile(c.path("/usr/local/share/ca-certificates/existing.crt")) + require.NoError(t, err) + require.Equal(t, "keep", string(data)) +} + +func TestCloudCertificatePayloadsArePassedToTrustStoreTools(t *testing.T) { + for _, tc := range []struct{ name, location, want string }{ + {"legacy", "usnateast", "not a PEM certificate\r\n"}, + {"rcv1p", "westus2", "not a PEM certificate\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("AKS_CLOUD_LOG_CERTIFICATES", "true") + logs := installLogCapturer(t) + c := newTestCloudInitializer(t, cloudTestUbuntu) + serveCloudTestResponses(t, c, []byte("not a PEM certificate\n"), `{"IsOptedInForRootCerts":true}`) + var commands []string + c.run = func(cmd *exec.Cmd) error { + commands = append(commands, cmd.Args[0]) + return nil + } + require.NoError(t, c.refreshCertificates(t.Context(), tc.location)) + got, err := os.ReadFile(c.path("/usr/local/share/ca-certificates/root.crt")) + require.NoError(t, err) + require.Equal(t, tc.want, string(got)) + require.Equal(t, []string{"update-ca-certificates"}, commands) + for _, record := range logs.getRecords() { + for _, value := range record.Attrs { + require.NotContains(t, value, "not a PEM certificate") + } + } + }) + } +} + +func TestCloudCertificateOperationDownloadFailures(t *testing.T) { + for _, tc := range []struct { + name string + roots []string + intermediates []string + wantInstalled []string + wantError bool + }{ + {"partial group", []string{"missing.crt", "root.crt"}, []string{"intermediate.crt"}, []string{"root.crt", "intermediate.crt"}, false}, + {"empty root group after failures", []string{"missing.crt"}, []string{"intermediate.crt"}, []string{"intermediate.crt"}, false}, + {"empty certificate response", []string{"empty.crt"}, []string{"intermediate.crt"}, []string{"intermediate.crt"}, false}, + {"non-crt root", []string{"root.cer"}, []string{"intermediate.crt"}, []string{"intermediate.crt"}, false}, + {"all downloads fail", []string{"missing.crt"}, []string{"missing.crt"}, nil, true}, + {"no crt files", []string{"root.cer"}, []string{"intermediate.cer"}, nil, true}, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, "ID=azurelinux\n") + serveCloudOperationTestResponses(t, c, tc.roots, tc.intermediates) + var commands []string + c.run = func(cmd *exec.Cmd) error { + commands = append(commands, cmd.Args[0]) + return nil + } + err := c.refreshCertificates(t.Context(), "westus2") + if tc.wantError { + require.ErrorContains(t, err, "no *.crt files") + require.Empty(t, commands) + return + } + require.NoError(t, err) + require.Equal(t, []string{"update-ca-trust"}, commands) + for _, name := range tc.wantInstalled { + data, readErr := os.ReadFile(c.path("/etc/pki/ca-trust/source/anchors/" + name)) + require.NoError(t, readErr) + require.Equal(t, "certificate payload\n", string(data)) + } + for _, name := range []string{"missing.crt", "empty.crt", "root.cer"} { + _, err = os.Stat(c.path("/etc/pki/ca-trust/source/anchors/" + name)) + require.ErrorIs(t, err, os.ErrNotExist) + } + }) + } +} + +func TestCloudLegacyCertificateInstallationDoesNotIgnoreCancellation(t *testing.T) { + c := newTestCloudInitializer(t, "ID=azurelinux\n") + serveCloudTestResponses(t, c, cloudTestCertificate(t), "not used") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + c.run = func(*exec.Cmd) error { + cancel() + return ctx.Err() + } + require.ErrorIs(t, c.refreshCertificates(ctx, "usnateast"), context.Canceled) +} + +func TestCloudCertificateRequestRetryAndCancellation(t *testing.T) { + for _, tc := range []struct { + name string + status int + body string + failures int32 + wantWaits []time.Duration + }{ + {"transient", http.StatusServiceUnavailable, "do-not-log-body", 2, []time.Duration{3 * time.Second, 3 * time.Second}}, + {"rate limited", http.StatusForbidden, "RequestRateLimitExceeded", 2, []time.Duration{3 * time.Second, 6 * time.Second}}, + {"exhausted", http.StatusServiceUnavailable, "do-not-log-body", cloudRequestAttempts, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if attempts.Add(1) <= tc.failures { + w.WriteHeader(tc.status) + _, _ = io.WriteString(w, tc.body) + return + } + _, _ = io.WriteString(w, "success") + })) + defer server.Close() + c.wireEndpoint = server.URL + var waits []time.Duration + c.wait = func(_ context.Context, delay time.Duration) error { + waits = append(waits, delay) + return nil + } + body, err := c.requestCertificates(t.Context(), "/test") + if tc.failures == cloudRequestAttempts { + require.Error(t, err) + require.NotContains(t, err.Error(), tc.body) + require.EqualValues(t, cloudRequestAttempts, attempts.Load()) + require.Len(t, waits, cloudRequestAttempts-1) + } else { + require.NoError(t, err) + require.Equal(t, "success", string(body)) + require.Equal(t, tc.wantWaits, waits) + } + }) + } + t.Run("cancellation during backoff", func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + c.wireEndpoint = server.URL + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + c.wait = func(ctx context.Context, _ time.Duration) error { + cancel() + return ctx.Err() + } + _, err := c.requestCertificates(ctx, "/test") + require.ErrorIs(t, err, context.Canceled) + require.EqualValues(t, 1, attempts.Load()) + }) +} + +func TestCloudCertificateRequestsDoNotFollowRedirects(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + var redirected atomic.Int32 + target := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { redirected.Add(1) })) + defer target.Close() + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer source.Close() + c.wireEndpoint = source.URL + _, err := c.requestCertificates(t.Context(), "/test") + require.ErrorContains(t, err, "HTTP 302") + require.Zero(t, redirected.Load()) +} + +func TestCloudCertificateResourceNamesAreURLEncoded(t *testing.T) { + c := newTestCloudInitializer(t, "ID=azurelinux\n") + cert := cloudTestCertificate(t) + var fetched atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch kind := r.URL.Query().Get("type"); { + case r.URL.Path == "/acms/isOptedInForRootCerts": + _, _ = io.WriteString(w, `{"IsOptedInForRootCerts":true}`) + case kind == "operationrequestsroot": + _, _ = io.WriteString(w, `{"OperationsInfo":[{"ResouceFileName":"root+ca.crt"}]}`) + case kind == "operationrequestsintermediate": + _, _ = io.WriteString(w, `{"OperationsInfo":[{"ResouceFileName":"intermediate.crt"}]}`) + case kind == "root+ca", kind == "intermediate": + fetched.Store(true) + _, _ = w.Write(cert) + default: + t.Errorf("incorrectly encoded certificate resource: %s", r.URL) + w.WriteHeader(http.StatusBadRequest) + } + })) + defer server.Close() + c.wireEndpoint = server.URL + require.NoError(t, c.refreshCertificates(t.Context(), "westus2")) + require.True(t, fetched.Load()) + installed, err := os.ReadFile(c.path("/etc/pki/ca-trust/source/anchors/root+ca.crt")) + require.NoError(t, err) + require.Equal(t, cert, installed) +} + +func TestCloudCertificateResponseLimits(t *testing.T) { + _, err := readCloudResponse(&http.Response{ + Body: io.NopCloser(strings.NewReader(strings.Repeat("x", cloudResponseLimit+1))), + }) + require.ErrorContains(t, err, "size limit") +} + +func TestCloudCertificateNamesAndPublicContent(t *testing.T) { + for _, name := range []string{"", ".", "..", "../root.crt", "/root.crt", "root\x00.crt"} { + require.False(t, validCloudCertificateName(name), name) + } + for _, name := range []string{"root+ca.crt", `..\root.crt`, "root\n.crt"} { + require.True(t, validCloudCertificateName(name), name) + } + cert := cloudTestCertificate(t) + private := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte("not-a-real-private-key")}) + got, err := publicCloudCertificates(append(private, cert...)) + require.NoError(t, err) + require.Equal(t, cert, got) + for _, bad := range [][]byte{nil, private, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte("invalid")})} { + _, err = publicCloudCertificates(bad) + require.Error(t, err) + } +} + +func TestCloudCertificateLoggingIsOptInAndPublicOnly(t *testing.T) { + for _, enabled := range []string{"false", "true"} { + t.Run(enabled, func(t *testing.T) { + t.Setenv("AKS_CLOUD_LOG_CERTIFICATES", enabled) + logs := installLogCapturer(t) + c := newTestCloudInitializer(t, "ID=azurelinux\n") + cert := cloudTestCertificate(t) + payload := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte("not-a-real-private-key")}) + payload = append(payload, cert...) + serveCloudTestResponses(t, c, payload, `{"IsOptedInForRootCerts":true}`) + require.NoError(t, c.refreshCertificates(t.Context(), "westus2")) + found := false + for _, record := range logs.getRecords() { + for _, value := range record.Attrs { + require.NotContains(t, value, "PRIVATE KEY") + if strings.Contains(value, "BEGIN CERTIFICATE") { + found = true + } + } + } + require.Equal(t, enabled == "true", found) + got, err := os.ReadFile(c.path("/etc/pki/ca-trust/source/anchors/root.crt")) + require.NoError(t, err) + require.Equal(t, payload, got) + }) + } +} + +func TestCloudCertificateInvalidOperationsDoNotInstall(t *testing.T) { + for _, response := range []string{ + `{}`, `{"OperationsInfo":[]}`, `{"OperationsInfo":[{"ResouceFileName":"../escape.crt"}]}`, + `{"OperationsInfo":[{"ResouceFileName":"/escape.crt"}]}`, `{"OperationsInfo":[{"ResouceFileName":"missing-extension"}]}`, + } { + t.Run(response, func(t *testing.T) { + c := newTestCloudInitializer(t, "ID=azurelinux\n") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/acms/isOptedInForRootCerts" { + _, _ = io.WriteString(w, `{"IsOptedInForRootCerts":true}`) + } else { + _, _ = io.WriteString(w, response) + } + })) + defer server.Close() + c.wireEndpoint = server.URL + c.run = func(*exec.Cmd) error { + t.Fatal("invalid operations must not install certificates") + return nil + } + require.Error(t, c.refreshCertificates(t.Context(), "westus2")) + _, err := os.Stat(c.path("/etc/pki/ca-trust/source/anchors")) + require.ErrorIs(t, err, os.ErrNotExist) + }) + } +} + +func TestCloudCertificateUbuntuBundleSymlink(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + require.NoError(t, os.MkdirAll(c.path("/usr/lib/ssl"), 0o755)) + require.NoError(t, os.Symlink(c.path("/etc/ssl/certs/ca-certificates.crt"), c.path("/usr/lib/ssl/cert.pem"))) + require.NoError(t, c.copyOpenSSLBundle()) + info, err := os.Lstat(c.path("/usr/lib/ssl/cert.pem")) + require.NoError(t, err) + require.NotZero(t, info.Mode()&os.ModeSymlink) +} + +func TestCloudPeriodicRefreshDoesNotInitializeRepositoriesOrSchedules(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + requests := serveCloudTestResponses(t, c, cloudTestCertificate(t), `{"IsOptedInForRootCerts":true}`) + var commands []string + c.run = func(cmd *exec.Cmd) error { + commands = append(commands, strings.Join(cmd.Args, " ")) + return nil + } + require.NoError(t, c.refreshCertificates(t.Context(), "westus2")) + require.Equal(t, []string{"update-ca-certificates"}, commands) + require.Len(t, requests, 5) + _, err := os.Stat(c.path(defaultAptSourcesDir)) + require.ErrorIs(t, err, os.ErrNotExist) + events := c.eventLogger.Events() + require.Len(t, events, 1) + require.Equal(t, "AKS.AKSNodeController.CloudCertificateRefresh", events[0].TaskName) +} diff --git a/aks-node-controller/cloud_refresh.go b/aks-node-controller/cloud_refresh.go new file mode 100644 index 00000000000..4a183a62b19 --- /dev/null +++ b/aks-node-controller/cloud_refresh.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "errors" + "log/slog" + "strings" + "time" + "unicode" +) + +const cloudRefreshTimeout = 15 * time.Minute + +func (a *App) runCloudCertificateRefresh(ctx context.Context, location string) error { + if strings.TrimSpace(location) == "" { + return errors.New("cloud certificate refresh requires a location") + } + var err error + if a.refreshCloudCertificates != nil { + err = a.refreshCloudCertificates(ctx, location) + } else { + initializer := newCloudInitializer(a) + defer initializer.close() + err = initializer.refreshCertificates(ctx, location) + } + if err != nil { + slog.Error("Azure cloud certificate refresh failed", "error", err) + } + return err +} + +func (c *cloudInitializer) refreshCertificates(ctx context.Context, location string) error { + refreshCtx, cancel := context.WithTimeout(ctx, cloudRefreshTimeout) + defer cancel() + return c.withEvent("CloudCertificateRefresh", func() error { + platform, err := c.platform() + if err != nil { + return err + } + _, err = c.refresh(refreshCtx, platform, location) + return err + }) +} + +func normalizeCloudLocation(location string) (string, error) { + location = strings.ToLower(strings.Map(func(r rune) rune { + if unicode.IsSpace(r) { + return -1 + } + return r + }, location)) + if strings.IndexFunc(location, func(r rune) bool { + return !strings.ContainsRune("abcdefghijklmnopqrstuvwxyz0123456789-", r) + }) != -1 { + return "", errors.New("cloud location contains unsupported characters") + } + if location == "" { + slog.Warn("Azure cloud location is empty, using rcv1p certificate endpoints") + } + return location, nil +} diff --git a/aks-node-controller/cloud_refresh_test.go b/aks-node-controller/cloud_refresh_test.go new file mode 100644 index 00000000000..06c98447237 --- /dev/null +++ b/aks-node-controller/cloud_refresh_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNormalizeCloudLocation(t *testing.T) { + location, err := normalizeCloudLocation(" US Sec East \t\n") + require.NoError(t, err) + require.Equal(t, "usseceast", location) + for _, value := range []string{`westus";echo bad`, "westus%bad", "westus\\bad"} { + _, err = normalizeCloudLocation(value) + require.Error(t, err) + } +} + +func TestCloudCertificateRefreshCommand(t *testing.T) { + for _, tc := range []struct { + name string + args []string + err error + wantCall bool + wantError bool + }{ + {"valid", []string{"--location", "usnateast"}, nil, true, false}, + {"refresh failure", []string{"--location", "usnateast"}, errors.New("refresh failed"), true, true}, + {"missing location", nil, nil, false, true}, + {"empty location", []string{"--location", " "}, nil, false, true}, + {"unexpected argument", []string{"--location", "usnateast", "extra"}, nil, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + tt := NewTestApp(t, TestAppConfig{}) + called := false + tt.App.refreshCloudCertificates = func(_ context.Context, location string) error { + called = true + require.Equal(t, "usnateast", location) + return tc.err + } + code := tt.App.Run(t.Context(), append([]string{"aks-node-controller", "refresh-cloud-certificates"}, tc.args...)) + require.Equal(t, tc.wantCall, called) + require.Equal(t, tc.wantError, code != 0) + }) + } +} diff --git a/aks-node-controller/cloud_runtime.go b/aks-node-controller/cloud_runtime.go new file mode 100644 index 00000000000..4781d41b8e4 --- /dev/null +++ b/aks-node-controller/cloud_runtime.go @@ -0,0 +1,197 @@ +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/Azure/agentbaker/aks-node-controller/common" + "github.com/Azure/agentbaker/aks-node-controller/helpers" +) + +const cloudResponseLimit = 8 << 20 + +const ( + cloudCommandTimeout = 2 * time.Minute + cloudCommandWaitDelay = 5 * time.Second +) + +type cloudPlatform struct { + ID, Version, Codename string +} + +type cloudInitializer struct { + run func(*exec.Cmd) error + wait func(context.Context, time.Duration) error + wireClient *http.Client + repoClient *http.Client + wireEndpoint string + eventLogger *helpers.EventLogger + rootDir string + osReleasePath string + executable string +} + +func newCloudInitializer(a *App) *cloudInitializer { + transport := common.NewBaseTransport(common.HTTPTransportOptions{ + DialTimeout: 10 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: 30 * time.Second, + }) + repoTransport := transport.Clone() + repoTransport.Proxy = http.ProxyFromEnvironment + return &cloudInitializer{ + run: a.cmdRun, + wait: waitCloudRetry, + wireEndpoint: wireServerEndpoint, + wireClient: &http.Client{ + Transport: transport, + Timeout: 30 * time.Second, + // WireServer responses must not redirect to another endpoint. + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + }, + repoClient: &http.Client{Transport: repoTransport, Timeout: 30 * time.Second}, + eventLogger: a.eventLogger, + osReleasePath: a.osReleasePath, + } +} + +func (c *cloudInitializer) close() { + c.wireClient.CloseIdleConnections() + c.repoClient.CloseIdleConnections() +} + +func (c *cloudInitializer) platform() (cloudPlatform, error) { + path := c.osReleasePath + if path == "" { + path = c.path(defaultOSReleasePath) + } + data, err := os.ReadFile(path) + if err != nil { + return cloudPlatform{}, fmt.Errorf("read cloud initialization OS release: %w", err) + } + values := parseNodeCustomDataOSRelease(data) + id := strings.ToLower(values["ID"]) + switch { + case strings.Contains(values["NAME"], "Ubuntu"), id == osReleaseIDUbuntu: + id = osReleaseIDUbuntu + case strings.Contains(id, osReleaseIDFlatcar): + id = osReleaseIDFlatcar + case id == osReleaseIDAzureContainerLinux, + id == osReleaseIDAzureLinux && strings.ToLower(values["VARIANT_ID"]) == osReleaseIDAzureContainerLinux: + id = osReleaseIDAzureContainerLinux + case strings.Contains(values["NAME"], "Mariner"), id == osReleaseIDMariner: + id = osReleaseIDMariner + case strings.Contains(values["NAME"], "Microsoft Azure Linux"), id == osReleaseIDAzureLinux: + id = osReleaseIDAzureLinux + default: + return cloudPlatform{}, fmt.Errorf("unsupported cloud initialization OS %q", id) + } + return cloudPlatform{ID: id, Version: values["VERSION_ID"], Codename: values["VERSION_CODENAME"]}, nil +} + +func (c *cloudInitializer) path(path string) string { + if c.rootDir == "" { + return path + } + return filepath.Join(c.rootDir, strings.TrimPrefix(path, "/")) +} + +func (c *cloudInitializer) writeFile(path string, data []byte) error { + return writeCloudFile(c.path(path), data) +} + +func writeCloudFile(path string, data []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + root, err := os.OpenRoot(filepath.Dir(path)) + if err != nil { + return err + } + defer root.Close() + // #nosec G306 -- Public CAs, package keys, and repository metadata must be readable by unprivileged clients. + return root.WriteFile(filepath.Base(path), data, 0o644) +} + +func (c *cloudInitializer) withEvent(task string, action func() error) error { + start := time.Now() + err := action() + level, message := helpers.EventLevelInformational, "Completed" + if err != nil { + level, message = helpers.EventLevelError, err.Error() + slog.Error("Azure cloud operation failed", "task", task, "error", err) + } + if c.eventLogger != nil { + c.eventLogger.LogEvent(task, message, level, start, time.Now()) + } + return err +} + +func (c *cloudInitializer) command(ctx context.Context, input []byte, name string, args ...string) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, cloudCommandTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, name, args...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if errors.Is(err, syscall.ESRCH) { + return os.ErrProcessDone + } + return err + } + // A descendant can leave the process group but still hold an output pipe open. + cmd.WaitDelay = cloudCommandWaitDelay + cmd.Env = append(os.Environ(), "LC_ALL=C") + cmd.Stdin = bytes.NewReader(input) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + run := c.run + if run == nil { + run = cmdRunner + } + if err := run(cmd); err != nil { + return stdout.Bytes(), fmt.Errorf("%s: %w: %s", name, err, strings.TrimSpace(stderr.String())) + } + if name == "dnf" { + // DNF can report unavailable repositories on stderr despite exiting zero. + _, _ = stdout.Write(stderr.Bytes()) + } + return stdout.Bytes(), nil +} + +func waitCloudRetry(ctx context.Context, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func readCloudResponse(resp *http.Response) ([]byte, error) { + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, cloudResponseLimit+1)) + if err != nil { + return nil, err + } + if len(body) > cloudResponseLimit { + return nil, errors.New("cloud response exceeds size limit") + } + return body, nil +} diff --git a/aks-node-controller/cloud_runtime_events_test.go b/aks-node-controller/cloud_runtime_events_test.go new file mode 100644 index 00000000000..a85583a5010 --- /dev/null +++ b/aks-node-controller/cloud_runtime_events_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCloudRuntimeOperationEvents(t *testing.T) { + for _, tc := range []struct { + name string + err error + }{ + {"completed", nil}, + {"failed", errors.New("operation failed")}, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, cloudTestUbuntu) + err := c.withEvent("CloudRuntimeProbe", func() error { return tc.err }) + require.ErrorIs(t, err, tc.err) + events := c.eventLogger.Events() + require.Len(t, events, 1) + require.Equal(t, "AKS.AKSNodeController.CloudRuntimeProbe", events[0].TaskName) + }) + } +} diff --git a/aks-node-controller/cloud_runtime_test.go b/aks-node-controller/cloud_runtime_test.go new file mode 100644 index 00000000000..6cff4f13d66 --- /dev/null +++ b/aks-node-controller/cloud_runtime_test.go @@ -0,0 +1,235 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCloudFileWrites(t *testing.T) { + t.Run("create and replace", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "certificates", "root.crt") + require.NoError(t, writeCloudFile(path, []byte("original"))) + require.NoError(t, writeCloudFile(path, []byte("replacement"))) + data, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, "replacement", string(data)) + }) + t.Run("reject symlinks outside the destination directory", func(t *testing.T) { + directory := t.TempDir() + outside := filepath.Join(t.TempDir(), "outside.crt") + require.NoError(t, os.WriteFile(outside, []byte("original"), 0o600)) + path := filepath.Join(directory, "root.crt") + require.NoError(t, os.Symlink(outside, path)) + require.Error(t, writeCloudFile(path, []byte("replacement"))) + data, err := os.ReadFile(outside) + require.NoError(t, err) + require.Equal(t, "original", string(data)) + }) +} + +func TestCloudPlatform(t *testing.T) { + for _, tc := range []struct { + name, release, want string + }{ + {"Ubuntu", cloudTestUbuntu, osReleaseIDUbuntu}, + {"Mariner", "ID=mariner\nNAME=CBL-Mariner\nVERSION_ID=2.0\n", "mariner"}, + {"AzureLinux", "ID=azurelinux\nNAME=\"Microsoft Azure Linux\"\nVERSION_ID=3.0\n", osReleaseIDAzureLinux}, + {"OSGuard", "ID=azurelinux\nVARIANT_ID=osguard\nVERSION_ID=3.0\n", osReleaseIDAzureLinux}, + {"Flatcar", "ID=flatcar\n", osReleaseIDFlatcar}, + {"ACL", "ID=azurecontainerlinux\n", osReleaseIDAzureContainerLinux}, + {"ACL variant", "ID=azurelinux\nVARIANT_ID=azurecontainerlinux\n", osReleaseIDAzureContainerLinux}, + {"Mixed-case ACL variant", "ID=azurelinux\nVARIANT_ID=AzureContainerLinux\n", osReleaseIDAzureContainerLinux}, + {"unsupported", "ID=debian\n", ""}, + {"missing ID", "", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + c := newTestCloudInitializer(t, tc.release) + platform, err := c.platform() + if tc.want == "" { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tc.want, platform.ID) + } + }) + } +} + +func TestCloudCertificateResponseLimits(t *testing.T) { + _, err := readCloudResponse(&http.Response{ + Body: io.NopCloser(strings.NewReader(strings.Repeat("x", cloudResponseLimit+1))), + }) + require.ErrorContains(t, err, "size limit") +} + +func TestCloudCommandSuccess(t *testing.T) { + executable, err := os.Executable() + require.NoError(t, err) + c := &cloudInitializer{} + output, err := c.command(t.Context(), []byte("input"), executable, + "-test.run=^TestCloudCommandHelperProcess$", "--", "echo", "") + require.NoError(t, err) + require.Equal(t, "C:input", string(output)) +} + +func TestCloudCommandAlreadyCanceled(t *testing.T) { + executable, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + called := false + c := &cloudInitializer{run: func(cmd *exec.Cmd) error { + called = true + return cmd.Run() + }} + output, err := c.command(ctx, nil, executable, + "-test.run=^TestCloudCommandHelperProcess$", "--", "echo", "") + require.ErrorIs(t, err, context.Canceled) + require.False(t, called, "an already-canceled context must not invoke the command runner") + require.Nil(t, output) +} + +func TestCloudCommandCancellation(t *testing.T) { + executable, err := os.Executable() + require.NoError(t, err) + for _, tc := range []struct { + name string + mode string + returnWithin time.Duration + }{ + {"kills descendants with inherited output", "group", 2 * time.Second}, + {"bounds output draining after a descendant leaves the group", "detached", 7 * time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + readyPath := filepath.Join(t.TempDir(), "child.pid") + ctx, cancel := context.WithCancel(t.Context()) + started := make(chan *os.Process, 1) + finished := make(chan struct{}) + c := &cloudInitializer{run: func(cmd *exec.Cmd) error { + if err := cmd.Start(); err != nil { + return err + } + started <- cmd.Process + return cmd.Wait() + }} + var parent, child *os.Process + t.Cleanup(func() { + cleanupCloudCommand(t, cancel, finished, parent, child) + }) + var output []byte + var commandErr error + go func() { + defer close(finished) + output, commandErr = c.command(ctx, nil, executable, + "-test.run=^TestCloudCommandHelperProcess$", "--", tc.mode, readyPath) + }() + select { + case parent = <-started: + case <-finished: + t.Fatalf("command exited before starting its child: %v", commandErr) + case <-time.After(5 * time.Second): + t.Fatal("command did not start") + } + var childPID int + require.Eventually(t, func() bool { + data, err := os.ReadFile(readyPath) + if err != nil { + return false + } + childPID, err = strconv.Atoi(strings.TrimSpace(string(data))) + return err == nil && childPID > 0 + }, 5*time.Second, 10*time.Millisecond, "command did not start its output-holding child") + var err error + child, err = os.FindProcess(childPID) + require.NoError(t, err) + + cancel() + select { + case <-finished: + require.Error(t, commandErr) + require.Contains(t, string(output), "stdout-marker") + require.ErrorContains(t, commandErr, "stderr-marker") + case <-time.After(tc.returnWithin): + t.Fatalf("command did not return within %s of cancellation", tc.returnWithin) + } + }) + } +} + +func cleanupCloudCommand(t *testing.T, cancel context.CancelFunc, finished <-chan struct{}, parent, child *os.Process) { + t.Helper() + cancel() + killCloudCommandTestProcess(t, child) + killCloudCommandTestProcess(t, parent) + select { + case <-finished: + case <-time.After(5 * time.Second): + t.Error("command did not finish after subprocess cleanup") + } + if child != nil { + if err := child.Release(); err != nil { + t.Errorf("release child process: %v", err) + } + } +} + +func killCloudCommandTestProcess(t *testing.T, process *os.Process) { + t.Helper() + if process == nil { + return + } + if err := process.Kill(); err != nil && + !errors.Is(err, os.ErrProcessDone) && !errors.Is(err, syscall.ESRCH) { + t.Errorf("clean up process %d: %v", process.Pid, err) + } +} + +func TestCloudCommandHelperProcess(t *testing.T) { + if len(os.Args) != 5 || os.Args[2] != "--" { + return + } + runCloudCommandHelperProcess(t) + os.Exit(0) +} + +func runCloudCommandHelperProcess(t *testing.T) { + t.Helper() + switch os.Args[3] { + case "echo": + input, err := io.ReadAll(os.Stdin) + require.NoError(t, err) + _, err = fmt.Fprintf(os.Stdout, "%s:%s", os.Getenv("LC_ALL"), input) + require.NoError(t, err) + case "hold": + time.Sleep(time.Minute) + case "group", "detached": + executable, err := os.Executable() + require.NoError(t, err) + child := exec.Command(executable, os.Args[1], "--", "hold", os.Args[4]) + child.Stdout, child.Stderr = os.Stdout, os.Stderr + if os.Args[3] == "detached" { + child.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + } + require.NoError(t, child.Start()) + defer killCloudCommandTestProcess(t, child.Process) + fmt.Fprintln(os.Stdout, "stdout-marker") + fmt.Fprintln(os.Stderr, "stderr-marker") + require.NoError(t, os.WriteFile(os.Args[4], []byte(strconv.Itoa(child.Process.Pid)), 0o600)) + require.NoError(t, child.Wait()) + default: + t.Fatalf("unknown command helper mode %q", os.Args[3]) + } +} diff --git a/aks-node-controller/cloud_test_helpers_test.go b/aks-node-controller/cloud_test_helpers_test.go new file mode 100644 index 00000000000..0efb19cc093 --- /dev/null +++ b/aks-node-controller/cloud_test_helpers_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "io" + "math/big" + "net/http" + "net/http/httptest" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/Azure/agentbaker/aks-node-controller/helpers" + "github.com/stretchr/testify/require" +) + +const cloudTestUbuntu = "ID=ubuntu\nNAME=Ubuntu\nVERSION_ID=22.04\nVERSION_CODENAME=jammy\n" + +func newTestCloudInitializer(t *testing.T, release string) *cloudInitializer { + t.Helper() + c := newCloudInitializer(&App{ + cmdRun: func(cmd *exec.Cmd) error { + if filepath.Base(cmd.Path) == "gpg" { + _, _ = io.WriteString(cmd.Stdout, "dearmored-key") + } + return nil + }, + eventLogger: helpers.NewEventLogger(t.TempDir()), + }) + c.rootDir = t.TempDir() + c.executable = hotfixBinaryPath + c.wait = func(ctx context.Context, _ time.Duration) error { return ctx.Err() } + require.NoError(t, c.writeFile(defaultOSReleasePath, []byte(release))) + require.NoError(t, c.writeFile("/etc/ssl/certs/ca-certificates.crt", []byte("system-ca-bundle"))) + t.Cleanup(c.close) + return c +} + +func cloudTestCertificate(t *testing.T) []byte { + t.Helper() + public, private, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "ANC cloud initialization test CA"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, public, private) + require.NoError(t, err) + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +func serveCloudOperationTestResponses(t *testing.T, c *cloudInitializer, roots, intermediates []string) { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch kind := r.URL.Query().Get("type"); { + case r.URL.Path == "/acms/isOptedInForRootCerts": + _, _ = io.WriteString(w, `{"IsOptedInForRootCerts":true}`) + case kind == "operationrequestsroot", kind == "operationrequestsintermediate": + names := roots + if kind == "operationrequestsintermediate" { + names = intermediates + } + operations := make([]map[string]string, 0, len(names)) + for _, name := range names { + operations = append(operations, map[string]string{"ResouceFileName": name}) + } + _ = json.NewEncoder(w).Encode(map[string]any{"OperationsInfo": operations}) + case kind == "missing": + w.WriteHeader(http.StatusServiceUnavailable) + case kind == "empty": + _, _ = io.WriteString(w, "\n\n") + default: + _, _ = io.WriteString(w, "certificate payload\n\n") + } + })) + t.Cleanup(server.Close) + c.wireEndpoint = server.URL +} diff --git a/aks-node-controller/embeddednodecustomdata.go b/aks-node-controller/embeddednodecustomdata.go index 67615d54fcb..1f1ee70c0e5 100644 --- a/aks-node-controller/embeddednodecustomdata.go +++ b/aks-node-controller/embeddednodecustomdata.go @@ -20,6 +20,7 @@ const ( osReleaseIDAzureLinux = "azurelinux" osReleaseIDFlatcar = "flatcar" osReleaseIDUbuntu = "ubuntu" + osReleaseIDMariner = "mariner" osVariantIDOSGuard = "osguard" ) @@ -27,7 +28,7 @@ type nodeCustomDataPlatform string const ( nodeCustomDataPlatformUbuntu nodeCustomDataPlatform = "ubuntu" - nodeCustomDataPlatformMariner nodeCustomDataPlatform = "mariner" + nodeCustomDataPlatformMariner nodeCustomDataPlatform = osReleaseIDMariner nodeCustomDataPlatformUnsupported nodeCustomDataPlatform = "unsupported" ) diff --git a/aks-node-controller/parser/consts.go b/aks-node-controller/parser/consts.go index c52803b7189..f36166ad3c2 100644 --- a/aks-node-controller/parser/consts.go +++ b/aks-node-controller/parser/consts.go @@ -27,7 +27,6 @@ const ( cseConfigLocalDNSScriptFilepath = "/opt/azure/containers/provision_configs_localdns.sh" cseConfigKubeletScriptFilepath = "/opt/azure/containers/provision_configs_kubelet.sh" cseConfigNetworkScriptFilepath = "/opt/azure/containers/provision_configs_network.sh" - cseConfigAddonsScriptFilepath = "/opt/azure/containers/provision_configs_addons.sh" cseConfigChronyScriptFilepath = "/opt/azure/containers/provision_configs_chrony.sh" customSearchDomainsCSEScriptFilepath = "/opt/azure/containers/setup-custom-search-domains.sh" dhcpV6ServiceCSEScriptFilepath = "/etc/systemd/system/dhcpv6.service" diff --git a/aks-node-controller/parser/helper.go b/aks-node-controller/parser/helper.go index 1008e668509..b69eae671af 100644 --- a/aks-node-controller/parser/helper.go +++ b/aks-node-controller/parser/helper.go @@ -312,10 +312,6 @@ func getCSEConfigNetworkFilepath() string { return cseConfigNetworkScriptFilepath } -func getCSEConfigAddonsFilepath() string { - return cseConfigAddonsScriptFilepath -} - func getCSEConfigChronyFilepath() string { return cseConfigChronyScriptFilepath } diff --git a/aks-node-controller/parser/helper_test.go b/aks-node-controller/parser/helper_test.go index d01a86134a5..97ab2f8e957 100644 --- a/aks-node-controller/parser/helper_test.go +++ b/aks-node-controller/parser/helper_test.go @@ -21,6 +21,7 @@ import ( _ "embed" "encoding/base64" "encoding/json" + "fmt" "math" "reflect" "regexp" @@ -546,6 +547,68 @@ func Test_createSortedKeyValueInt32Pairs(t *testing.T) { } } +func TestContainerdImagePullTemplates(t *testing.T) { + for _, version := range []string{"1.7.22", "2.2.4"} { + for _, noGPU := range []bool{false, true} { + for _, tc := range []struct { + name string + settings *aksnodeconfigv1.ContainerdConfig + }{ + {"absent", nil}, + {"empty", &aksnodeconfigv1.ContainerdConfig{}}, + {"false", &aksnodeconfigv1.ContainerdConfig{ImagePullWithSyncFs: proto.Bool(false)}}, + {"true", &aksnodeconfigv1.ContainerdConfig{ImagePullWithSyncFs: proto.Bool(true)}}, + {"zero timeout", &aksnodeconfigv1.ContainerdConfig{ImagePullProgressTimeout: proto.String("0s")}}, + {"both", &aksnodeconfigv1.ContainerdConfig{ImagePullWithSyncFs: proto.Bool(true), ImagePullProgressTimeout: proto.String("1m30s")}}, + } { + t.Run(fmt.Sprintf("%s/noGPU=%t/%s", version, noGPU, tc.name), func(t *testing.T) { + assertContainerdImagePullTemplate(t, tc.settings, noGPU, version) + }) + } + } + } +} + +func assertContainerdImagePullTemplate(t *testing.T, settings *aksnodeconfigv1.ContainerdConfig, noGPU bool, version string) { + t.Helper() + cfg := &aksnodeconfigv1.Configuration{ + KubernetesVersion: "1.34.0", + GpuConfig: &aksnodeconfigv1.GpuConfig{EnableNvidia: proto.Bool(true)}, + ContainerdConfig: settings, + } + rendered, err := containerdConfigFromAKSNodeConfig(cfg, noGPU, version) + require.NoError(t, err) + plugin := "io.containerd.grpc.v1.cri" + if isContainerdV2(version) { + plugin = "io.containerd.cri.v1.images" + } + header := `[plugins."` + plugin + `"]` + "\n" + start := strings.Index(rendered, header) + require.NotEqual(t, -1, start) + section := rendered[start+len(header):] + if next := regexp.MustCompile(`(?m)^[ \t]*\[`).FindStringIndex(section); next != nil { + section = section[:next[0]] + } + if settings != nil && settings.ImagePullWithSyncFs != nil { + require.Contains(t, section, fmt.Sprintf(" image_pull_with_sync_fs = %t\n", settings.GetImagePullWithSyncFs())) + require.Equal(t, 1, strings.Count(rendered, "image_pull_with_sync_fs")) + } else { + require.NotContains(t, rendered, "image_pull_with_sync_fs") + } + if settings != nil && settings.ImagePullProgressTimeout != nil { + require.Contains(t, section, fmt.Sprintf(" image_pull_progress_timeout = %q\n", settings.GetImagePullProgressTimeout())) + require.Equal(t, 1, strings.Count(rendered, "image_pull_progress_timeout")) + } else { + require.NotContains(t, rendered, "image_pull_progress_timeout") + } + if settings == nil || (settings.ImagePullWithSyncFs == nil && settings.ImagePullProgressTimeout == nil) { + cfg.ContainerdConfig = nil + baseline, err := containerdConfigFromAKSNodeConfig(cfg, noGPU, version) + require.NoError(t, err) + require.Equal(t, baseline, rendered) + } +} + func Test_getContainerdConfig(t *testing.T) { type args struct { aksnodeconfig *aksnodeconfigv1.Configuration diff --git a/aks-node-controller/parser/parser.go b/aks-node-controller/parser/parser.go index 7f83919d1da..4015dbde80e 100644 --- a/aks-node-controller/parser/parser.go +++ b/aks-node-controller/parser/parser.go @@ -142,7 +142,6 @@ func getCSEEnv(ctx context.Context, config *aksnodeconfigv1.Configuration, gpuCo "CSE_CONFIG_LOCALDNS_FILEPATH": getCSEConfigLocalDNSFilepath(), "CSE_CONFIG_KUBELET_FILEPATH": getCSEConfigKubeletFilepath(), "CSE_CONFIG_NETWORK_FILEPATH": getCSEConfigNetworkFilepath(), - "CSE_CONFIG_ADDONS_FILEPATH": getCSEConfigAddonsFilepath(), "CSE_CONFIG_CHRONY_FILEPATH": getCSEConfigChronyFilepath(), "AZURE_PRIVATE_REGISTRY_SERVER": config.GetAzurePrivateRegistryServer(), "HAS_CUSTOM_SEARCH_DOMAIN": fmt.Sprintf("%v", getHasSearchDomain(config.GetCustomSearchDomainConfig())), diff --git a/aks-node-controller/parser/parser_test.go b/aks-node-controller/parser/parser_test.go index fc6109fc6ab..da20f558e6d 100644 --- a/aks-node-controller/parser/parser_test.go +++ b/aks-node-controller/parser/parser_test.go @@ -7,6 +7,7 @@ import ( "log" "os" "os/exec" + "path/filepath" "strings" "testing" @@ -17,8 +18,38 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" ) +func TestBuildCSECmdContainerdImagePullSettings(t *testing.T) { + for _, version := range []string{"1.7.22", "2.2.4"} { + t.Run(version, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "containerd"), []byte(fmt.Sprintf(`#!/bin/sh +[ "$1" = "--version" ] || exit 1 +echo "containerd containerd.io %s test" +`, version)), 0755)) + t.Setenv("PATH", dir) + cfg := &aksnodeconfigv1.Configuration{ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ + ImagePullWithSyncFs: proto.Bool(false), + ImagePullProgressTimeout: proto.String("0s"), + }} + cmd, err := BuildCSECmd(context.Background(), cfg, nil) + require.NoError(t, err) + env := environToMap(cmd.Env) + for _, key := range []string{"CONTAINERD_CONFIG_CONTENT", "CONTAINERD_CONFIG_NO_GPU_CONTENT"} { + content, err := base64.StdEncoding.DecodeString(env[key]) + require.NoError(t, err) + require.Contains(t, string(content), "image_pull_with_sync_fs = false") + require.Contains(t, string(content), `image_pull_progress_timeout = "0s"`) + } + require.NotContains(t, env, "ANC_CONFIGURE_CONTAINERD_BINARY") + require.NotContains(t, env, "CONTAINERD_IMAGE_PULL_WITH_SYNC_FS") + require.NotContains(t, env, "CONTAINERD_IMAGE_PULL_PROGRESS_TIMEOUT") + }) + } +} + func TestBuildCSECmd(t *testing.T) { tests := []struct { name string diff --git a/aks-node-controller/parser/templates/containerd.toml.gtpl b/aks-node-controller/parser/templates/containerd.toml.gtpl index a2e4b1a26a9..475aeea83c5 100644 --- a/aks-node-controller/parser/templates/containerd.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd.toml.gtpl @@ -15,6 +15,14 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} enable_tar_index = false {{- end}} [plugins."io.containerd.grpc.v1.cri"] +{{- with .GetContainerdConfig }} + {{- if ne .ImagePullWithSyncFs nil }} + image_pull_with_sync_fs = {{ .GetImagePullWithSyncFs }} + {{- end }} + {{- if ne .ImagePullProgressTimeout nil }} + image_pull_progress_timeout = "{{ .GetImagePullProgressTimeout }}" + {{- end }} +{{- end }} sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}" enable_cdi = true [plugins."io.containerd.grpc.v1.cri".containerd] diff --git a/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl b/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl index 1addca22cdb..d4bea918fec 100644 --- a/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_no_GPU.toml.gtpl @@ -15,6 +15,14 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} enable_tar_index = false {{- end}} [plugins."io.containerd.grpc.v1.cri"] +{{- with .GetContainerdConfig }} + {{- if ne .ImagePullWithSyncFs nil }} + image_pull_with_sync_fs = {{ .GetImagePullWithSyncFs }} + {{- end }} + {{- if ne .ImagePullProgressTimeout nil }} + image_pull_progress_timeout = "{{ .GetImagePullProgressTimeout }}" + {{- end }} +{{- end }} sandbox_image = "{{ .KubeBinaryConfig.GetPodInfraContainerImageUrl }}" [plugins."io.containerd.grpc.v1.cri".containerd] {{- if .GetIsKata }} diff --git a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl index d99504ac7e5..52f2ca39521 100644 --- a/aks-node-controller/parser/templates/containerd_v2.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2.toml.gtpl @@ -15,6 +15,14 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} enable_tar_index = false {{- end}} [plugins."io.containerd.cri.v1.images"] +{{- with .GetContainerdConfig }} + {{- if ne .ImagePullWithSyncFs nil }} + image_pull_with_sync_fs = {{ .GetImagePullWithSyncFs }} + {{- end }} + {{- if ne .ImagePullProgressTimeout nil }} + image_pull_progress_timeout = "{{ .GetImagePullProgressTimeout }}" + {{- end }} +{{- end }} {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" disable_snapshot_annotations = false diff --git a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl index 64438ad1e41..3213d6e8e27 100644 --- a/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl +++ b/aks-node-controller/parser/templates/containerd_v2_no_GPU.toml.gtpl @@ -15,6 +15,14 @@ root = "{{.KubeletConfig.GetContainerDataDir}}"{{- end}} enable_tar_index = false {{- end}} [plugins."io.containerd.cri.v1.images"] +{{- with .GetContainerdConfig }} + {{- if ne .ImagePullWithSyncFs nil }} + image_pull_with_sync_fs = {{ .GetImagePullWithSyncFs }} + {{- end }} + {{- if ne .ImagePullProgressTimeout nil }} + image_pull_progress_timeout = "{{ .GetImagePullProgressTimeout }}" + {{- end }} +{{- end }} {{- if .GetEnableArtifactStreaming }} snapshotter = "overlaybd" disable_snapshot_annotations = false diff --git a/aks-node-controller/pkg/gen/aksnodeconfig/v1/containerd_config.pb.go b/aks-node-controller/pkg/gen/aksnodeconfig/v1/containerd_config.pb.go index 54d3a35de59..d81e9f245d6 100644 --- a/aks-node-controller/pkg/gen/aksnodeconfig/v1/containerd_config.pb.go +++ b/aks-node-controller/pkg/gen/aksnodeconfig/v1/containerd_config.pb.go @@ -31,6 +31,12 @@ type ContainerdConfig struct { ContainerdVersion string `protobuf:"bytes,2,opt,name=containerd_version,json=containerdVersion,proto3" json:"containerd_version,omitempty"` // The URL for downloading the containerd package. ContainerdPackageUrl string `protobuf:"bytes,3,opt,name=containerd_package_url,json=containerdPackageUrl,proto3" json:"containerd_package_url,omitempty"` + // Force filesystem synchronization during image unpacking. Absence preserves + // the existing setting. Enabling this may select local pulls on containerd 2.x. + ImagePullWithSyncFs *bool `protobuf:"varint,4,opt,name=image_pull_with_sync_fs,json=imagePullWithSyncFs,proto3,oneof" json:"image_pull_with_sync_fs,omitempty"` + // Maximum time without image download progress, in Go duration format. + // Zero disables the timeout. Absence preserves the existing setting. + ImagePullProgressTimeout *string `protobuf:"bytes,5,opt,name=image_pull_progress_timeout,json=imagePullProgressTimeout,proto3,oneof" json:"image_pull_progress_timeout,omitempty"` } func (x *ContainerdConfig) Reset() { @@ -84,13 +90,27 @@ func (x *ContainerdConfig) GetContainerdPackageUrl() string { return "" } +func (x *ContainerdConfig) GetImagePullWithSyncFs() bool { + if x != nil && x.ImagePullWithSyncFs != nil { + return *x.ImagePullWithSyncFs + } + return false +} + +func (x *ContainerdConfig) GetImagePullProgressTimeout() string { + if x != nil && x.ImagePullProgressTimeout != nil { + return *x.ImagePullProgressTimeout + } + return "" +} + var File_aksnodeconfig_v1_containerd_config_proto protoreflect.FileDescriptor var file_aksnodeconfig_v1_containerd_config_proto_rawDesc = []byte{ 0x0a, 0x28, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x10, 0x61, 0x6b, 0x73, 0x6e, - 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x76, 0x31, 0x22, 0xb8, 0x01, 0x0a, + 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x76, 0x31, 0x22, 0xf3, 0x02, 0x0a, 0x10, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x3f, 0x0a, 0x1c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x5f, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x75, 0x72, 0x6c, 0x5f, 0x62, 0x61, 0x73, @@ -102,13 +122,25 @@ var file_aksnodeconfig_v1_containerd_config_proto_rawDesc = []byte{ 0x6e, 0x12, 0x34, 0x0a, 0x16, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x50, 0x61, 0x63, - 0x6b, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x42, 0x5a, 0x5a, 0x58, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x61, 0x67, 0x65, 0x6e, - 0x74, 0x62, 0x61, 0x6b, 0x65, 0x72, 0x2f, 0x61, 0x6b, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2d, - 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, - 0x65, 0x6e, 0x2f, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x2f, 0x76, 0x31, 0x3b, 0x61, 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6b, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x39, 0x0a, 0x17, 0x69, 0x6d, 0x61, 0x67, 0x65, + 0x5f, 0x70, 0x75, 0x6c, 0x6c, 0x5f, 0x77, 0x69, 0x74, 0x68, 0x5f, 0x73, 0x79, 0x6e, 0x63, 0x5f, + 0x66, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x13, 0x69, 0x6d, 0x61, 0x67, + 0x65, 0x50, 0x75, 0x6c, 0x6c, 0x57, 0x69, 0x74, 0x68, 0x53, 0x79, 0x6e, 0x63, 0x46, 0x73, 0x88, + 0x01, 0x01, 0x12, 0x42, 0x0a, 0x1b, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x75, 0x6c, 0x6c, + 0x5f, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, + 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x18, 0x69, 0x6d, 0x61, 0x67, 0x65, + 0x50, 0x75, 0x6c, 0x6c, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x54, 0x69, 0x6d, 0x65, + 0x6f, 0x75, 0x74, 0x88, 0x01, 0x01, 0x42, 0x1a, 0x0a, 0x18, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, + 0x5f, 0x70, 0x75, 0x6c, 0x6c, 0x5f, 0x77, 0x69, 0x74, 0x68, 0x5f, 0x73, 0x79, 0x6e, 0x63, 0x5f, + 0x66, 0x73, 0x42, 0x1e, 0x0a, 0x1c, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x75, 0x6c, + 0x6c, 0x5f, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, + 0x75, 0x74, 0x42, 0x5a, 0x5a, 0x58, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x61, 0x6b, 0x65, + 0x72, 0x2f, 0x61, 0x6b, 0x73, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2d, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x6b, + 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2f, 0x76, 0x31, 0x3b, 0x61, + 0x6b, 0x73, 0x6e, 0x6f, 0x64, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x76, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -140,6 +172,7 @@ func file_aksnodeconfig_v1_containerd_config_proto_init() { if File_aksnodeconfig_v1_containerd_config_proto != nil { return } + file_aksnodeconfig_v1_containerd_config_proto_msgTypes[0].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ diff --git a/aks-node-controller/pkg/nodeconfigutils/utils_test.go b/aks-node-controller/pkg/nodeconfigutils/utils_test.go index a296a82df8c..5bea3f89bdc 100644 --- a/aks-node-controller/pkg/nodeconfigutils/utils_test.go +++ b/aks-node-controller/pkg/nodeconfigutils/utils_test.go @@ -16,6 +16,34 @@ import ( "google.golang.org/protobuf/proto" ) +func TestContainerdImagePullPresenceRoundTrip(t *testing.T) { + for _, raw := range []string{ + `{"containerdConfig":{"imagePullWithSyncFs":false,"imagePullProgressTimeout":"0s"}}`, + `{"containerd_config":{"image_pull_with_sync_fs":true,"image_pull_progress_timeout":"30s"}}`, + `{"containerd_config":{}}`, + } { + cfg, err := UnmarshalConfigurationV1([]byte(raw)) + require.NoError(t, err) + encoded, err := MarshalConfigurationV1(cfg) + require.NoError(t, err) + restored, err := UnmarshalConfigurationV1(encoded) + require.NoError(t, err) + require.True(t, proto.Equal(cfg, restored)) + wire, err := proto.Marshal(cfg) + require.NoError(t, err) + decoded := &aksnodeconfigv1.Configuration{} + require.NoError(t, proto.Unmarshal(wire, decoded)) + require.True(t, proto.Equal(cfg, decoded)) + if cfg.ContainerdConfig.ImagePullWithSyncFs != nil { + require.Contains(t, string(encoded), `"image_pull_with_sync_fs"`) + require.NotNil(t, decoded.ContainerdConfig.ImagePullWithSyncFs) + require.NotNil(t, decoded.ContainerdConfig.ImagePullProgressTimeout) + } else { + require.NotContains(t, string(encoded), "image_pull_") + } + } +} + func TestUnmarshalConfigurationV1(t *testing.T) { tests := []struct { name string diff --git a/aks-node-controller/pkg/nodeconfigutils/validation.go b/aks-node-controller/pkg/nodeconfigutils/validation.go index 81ae9941ce3..bda1208ec10 100644 --- a/aks-node-controller/pkg/nodeconfigutils/validation.go +++ b/aks-node-controller/pkg/nodeconfigutils/validation.go @@ -6,6 +6,7 @@ import ( "sort" "strconv" "strings" + "time" aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" "github.com/Masterminds/semver/v3" @@ -22,6 +23,9 @@ func validateAndNormalize(cfg *aksnodeconfigv1.Configuration) error { if err := validateCustomLinuxOSConfig(cfg.GetCustomLinuxOsConfig()); err != nil { return err } + if err := validateContainerdConfig(cfg.GetContainerdConfig()); err != nil { + return err + } kubelet := cfg.GetKubeletConfig() if kubelet == nil { @@ -57,6 +61,19 @@ func validateAndNormalize(cfg *aksnodeconfigv1.Configuration) error { return nil } +func validateContainerdConfig(cfg *aksnodeconfigv1.ContainerdConfig) error { + if cfg != nil && cfg.ImagePullProgressTimeout != nil { + duration, err := time.ParseDuration(cfg.GetImagePullProgressTimeout()) + if err != nil { + return fmt.Errorf("containerd_config.image_pull_progress_timeout: %w", err) + } + if duration < 0 { + return fmt.Errorf("containerd_config.image_pull_progress_timeout must not be negative") + } + } + return nil +} + func validateCustomLinuxOSConfig(osConfig *aksnodeconfigv1.CustomLinuxOsConfig) error { for _, field := range []struct { name, value string diff --git a/aks-node-controller/pkg/nodeconfigutils/validation_test.go b/aks-node-controller/pkg/nodeconfigutils/validation_test.go index 30a02be0f11..0b46de02af3 100644 --- a/aks-node-controller/pkg/nodeconfigutils/validation_test.go +++ b/aks-node-controller/pkg/nodeconfigutils/validation_test.go @@ -8,6 +8,29 @@ import ( "google.golang.org/protobuf/proto" ) +func TestContainerdImagePullValidationWithoutKubelet(t *testing.T) { + for _, timeout := range []string{"0", "0s", "30s", "5m", "1m30s"} { + cfg := &aksnodeconfigv1.Configuration{ + Version: "v1", + ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ + ImagePullWithSyncFs: proto.Bool(false), + ImagePullProgressTimeout: proto.String(timeout), + }, + } + require.NoError(t, validateAndNormalize(cfg)) + require.NoError(t, validateAndNormalize(cfg)) + require.Equal(t, timeout, cfg.ContainerdConfig.GetImagePullProgressTimeout()) + } + for _, timeout := range []string{"", "-1s", "invalid", "5", "999999999999999999999h", "5m\n"} { + cfg := &aksnodeconfigv1.Configuration{ + Version: "v1", + ContainerdConfig: &aksnodeconfigv1.ContainerdConfig{ImagePullProgressTimeout: proto.String(timeout)}, + } + require.ErrorContains(t, validateAndNormalize(cfg), "containerd_config.image_pull_progress_timeout") + } + require.NoError(t, validateAndNormalize(&aksnodeconfigv1.Configuration{Version: "v1"})) +} + func TestValidateAndNormalize(t *testing.T) { for _, version := range []string{"1.10.0", "1.11.0", "1.20.0", "1.24.0", "1.25.0", "1.34.0", "1.34.0-rc.1"} { t.Run(version, func(t *testing.T) { diff --git a/aks-node-controller/proto/aksnodeconfig/v1/containerd_config.proto b/aks-node-controller/proto/aksnodeconfig/v1/containerd_config.proto index 7a528c8f726..5eeccb015ca 100644 --- a/aks-node-controller/proto/aksnodeconfig/v1/containerd_config.proto +++ b/aks-node-controller/proto/aksnodeconfig/v1/containerd_config.proto @@ -12,4 +12,12 @@ message ContainerdConfig { // The URL for downloading the containerd package. string containerd_package_url = 3; + + // Force filesystem synchronization during image unpacking. Absence preserves + // the existing setting. Enabling this may select local pulls on containerd 2.x. + optional bool image_pull_with_sync_fs = 4; + + // Maximum time without image download progress, in Go duration format. + // Zero disables the timeout. Absence preserves the existing setting. + optional string image_pull_progress_timeout = 5; } diff --git a/apiserver/apiserver.go b/apiserver/apiserver.go index fae2dded218..b0eb75d3e6b 100644 --- a/apiserver/apiserver.go +++ b/apiserver/apiserver.go @@ -97,6 +97,11 @@ func (api *APIServer) ListenAndServe(ctx context.Context) error { errors := make(chan error) go func() { + // Plaintext HTTP (not ListenAndServeTLS) is intentional here, risk-accepted + // By-Design - see the authn/authz rationale in NewRouter (routers.go). + // This server is only reachable from the AKS RP control plane over the + // cluster-internal network; TLS termination/authentication is handled at + // the network boundary, not in this process. errors <- svr.ListenAndServe() }() diff --git a/apiserver/routers.go b/apiserver/routers.go index e5d199f4f9f..fe68cda2605 100644 --- a/apiserver/routers.go +++ b/apiserver/routers.go @@ -43,6 +43,27 @@ func (api *APIServer) NewRouter() *mux.Router { router.Methods("GET").Path("/healthz").Name("healthz").HandlerFunc(healthz) // global timeout and panic handlers. + // + // No authentication middleware (JWT/mTLS/API-key) is intentionally added here; + // this is a risk-accepted By-Design decision, not an oversight. + // + // Rationale: + // - This APIServer is an internal sidecar invoked only by the AKS RP control + // plane over the cluster-internal network; it is not exposed to the public + // internet. + // - The enforced security boundary is network isolation (pod/cluster network + // policy), configured outside this repo in the RP deployment, not app-layer + // auth. + // - The handlers registered above are stateless template generators: they + // render node bootstrap data / image config from request input and do not + // read tenant secrets from a backend store. + // - Adding real authentication (mTLS or token-based) would require a + // cert/token issuance and rotation mechanism plus a coordinated change on + // the RP caller side (a different repo/system) - out of scope for a + // point fix here. + // + // If this router is ever exposed beyond the trusted RP-only network boundary, + // authentication middleware must be added here. router.Use(timeoutHandler(), recoveryHandler()) return router diff --git a/docs/npd-onboarding.md b/docs/npd-onboarding.md index 60d470e9078..afd7302c925 100644 --- a/docs/npd-onboarding.md +++ b/docs/npd-onboarding.md @@ -25,11 +25,16 @@ rpm/tdnf on Azure Linux. - CSE starts the baked service in `nodePrep`, after kubelet configuration and GPU setup. This also runs on PIS-cached images. No public-settings file is generated; monitor defaults and hardware selection come from the config package. -- Pins exist for Ubuntu 20.04, 22.04, 24.04 and 26.04 and Azure Linux 3.0, amd64 - and arm64. The config package content is the same on every release, so Ubuntu - 26.04, which has no config build of its own, uses the 24.04 package (see below). - Azure Linux OSGuard, Kata, Azure Container Linux, Mariner 2 and other OSes remain +- Pins exist for Ubuntu 20.04, 22.04, 24.04 and 26.04 and Azure Linux 3.0 (including + Kata), amd64 and arm64. The config package content is the same on every release, so + Ubuntu 26.04, which has no config build of its own, uses the 24.04 package (see below). + Azure Linux OSGuard, Azure Container Linux, Mariner 2 and other OSes remain extension-managed. +- Azure Linux 3 Kata uses the same packages, tdnf lock and handler as Azure Linux 3. + NPD runs on the Kata host, not inside Kata guest VMs. The Kata host kernel lacks PSI, + so pressure monitors fall back to load and iowait. LPC config updates change only + the held NPD config package; Kata's OS, kernel and Kata packages still update only + through images (unattended upgrades and securityPatch stay excluded for Kata). ## VHD package pins (`components.json`) diff --git a/e2e/README.md b/e2e/README.md index 2d7c56c02ee..a552388edf9 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -117,6 +117,30 @@ alongside cluster preparation. Setup, validators, and cleanup batches share `scenario/concurrent.go`, which waits for every task without canceling peers on failure. Each caller decides whether an already-canceled context prevents its tasks from starting. +## Shared Windows Gen2 images + +The ordinary WS2022 and WS2025 Gen2 images are published as `TrustedLaunchSupported`. +Each has a separate scenario that enables Secure Boot and vTPM on the VMSS: + +| Image | Standard scenario | Trusted Launch scenario | +| --- | --- | --- | +| `windows-2022-containerd-gen2` | `Windows2022Gen2AzureNetwork` | `Windows2022Gen2_TrustedLaunch_SharedImage` | +| `windows-2025-gen2` | `Windows2025Gen2` | `Windows2025Gen2_TrustedLaunch_SharedImage` | + +Separate nodes are needed because their VM security profiles differ. These direct +image tests do not change AKS RP's OS eligibility checks. + +The builder reuses already-capable staging definitions without `startsAtVersion` +without modifying them. Explicit boundaries must be valid and at or below the +build version; new capability migrations still establish and verify a boundary. +The VHD-build gate validates the actual build's image, not historical versions. +Production publication and RP image-version admission remain separate checks. + +`Windows2025Gen2TrustedLaunch` continues to validate the dedicated +`windows-2025-gen2-tl` image during the shared-image rollout. +The shared-image scenarios run in both the Windows VHD-build gate, using that +build's image, and the general Windows E2E job, using the main-tagged test image. + ## VM size configuration Resource SKU catalogues are loaded once per location using the runner's single @@ -136,6 +160,60 @@ which defaults to `Standard_D2ds_v6`. Set it with `--mana-vm-sku` or `MANA_VM_SK Choose a size that supports MANA and NVMe. This setting controls both the bootstrap configuration and the VMSS SKU. Command-line arguments take precedence over environment variables. +## ACL COSI Validation + +The four ACL Trusted Launch variants accept `ACL_BASE_IMAGE_SOURCE` (`acg` or +`acldevel`) and optional `ACL_BASE_IMAGE_VERSION_AMD64` and +`ACL_BASE_IMAGE_VERSION_ARM64` gallery-version overrides. Each takes precedence +over the existing shared `ACL_BASE_IMAGE_VERSION` fallback. Use separate values +when the selected architectures have different gallery versions. The overrides +default to a single space, because the Run pipeline form treats an empty default +as required; leave that space in all three to keep the pinned versions. Values +are trimmed. Both the PR and release pipelines expose these parameters. +The production ACG source is the default. Secure Boot stays enabled for signed sources; +selecting `acldevel` disables Secure Boot only for ACL test VMs. Older base images +without `trident-acl-agent` still build; images containing the agent enable its +kubeconfig-triggered startup. + +The dev source uses `acldevel` for AMD64 and `acldevel-arm64` for ARM64, with +architecture-specific version pins in the +[base-image template](../.pipelines/templates/.acl-base-image-vars.yaml). +The pipeline resolves `IMAGE_VERSION` once; the ACL image-stamping step fails +if it does not receive that value. + +The stamp records the original provisioning image and survives A/B updates of +`/usr`. Until Trident has an updatable current-version source, the agent reads +`/dev/null` with a `0.0.0` fallback rather than treating the stamp as the active +version. This deliberately disables current-version-based `AlreadyAtTarget` +shortcuts. Nebraska affinity cookie handling is enabled for annotation-driven +updates, including cookie persistence across reboot. + +ACL CVM builds keep the production ACG source and are not part of the +four-variant COSI validation. + +TME publishing runs convert the selected ACL variants in architecture-matched +jobs and produce `cosi-publishing-info-*` artifacts with a base64 +`metadata_sha384` value. The E2E job downloads only those variants' artifacts. +Non-publishing and non-TME runs do not convert, download, or validate COSI +artifacts. Cold Azure Linux/Mariner convert agents install the native signed +Microsoft OSS repository package before installing Azure CLI. + +`ACL_COSIUpdate_AMD64`, `ACL_COSIUpdate_ARM64`, and their `_FIPS` variants use +separate nodes because they stage, finalize, reboot, and commit an A/B update. +They verify node identity, readiness, and a post-update workload. The COSI is +staged in a unique test-owned blob on the already-allowed E2E storage endpoint; +cleanup deletes that blob. These scenarios do not change shared firewall rules. +They exercise the Trident update path, not annotation-driven agent orchestration. + +When COSI artifacts are enabled, the TME publishing E2E job defaults +`TEST_TIMEOUT_VMSS` to `30m` for staging, provisioning, validation, and reboot. +Explicit timeout overrides are preserved. Ordinary runs retain the `17m` +VM lifecycle default. The separate `30m` COSI download limit cannot extend +the enclosing VM lifecycle deadline. + +Run offline unit tests with `go test -short ./...` from this directory. Live +four-variant builds and update scenarios require the pipeline service connections. + ## Gallery replication When selecting a gallery image by version or tag, the runner adds the test region diff --git a/e2e/config/azure.go b/e2e/config/azure.go index b39f2284763..c6b1b7a1200 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -47,7 +47,7 @@ type AzureClient struct { StorageContainers *armstorage.BlobContainersClient CacheRulesClient *armcontainerregistry.CacheRulesClient Core *azcore.Client - Credential *azidentity.AzureCLICredential + Credential azcore.TokenCredential Maintenance *armcontainerservice.MaintenanceConfigurationsClient NetworkInterfaces *armnetwork.InterfacesClient PrivateDNSZoneGroup *armnetwork.PrivateDNSZoneGroupsClient @@ -489,6 +489,25 @@ func (a *AzureClient) assignRolesToVMIdentity(ctx context.Context, principalID * return nil } +// EnsureResourceGroupRoles restores CI runner access lost when a shared RG is garbage-collected. +func (a *AzureClient) EnsureResourceGroupRoles(ctx context.Context, scope string) error { + principalID, err := getCurrentPrincipalID(ctx, a.Credential) + if err != nil { + return fmt.Errorf("resolve current principal: %w", err) + } + for _, roleDefID := range []string{ + // Compute Gallery Artifacts Publisher. + "/providers/Microsoft.Authorization/roleDefinitions/85a2d0d9-2eba-4c9c-b355-11c2cc0788ab", + // Container Registry Data Importer and Data Reader. + "/providers/Microsoft.Authorization/roleDefinitions/577a9874-89fd-4f24-9dbd-b5034d0ad23a", + } { + if err := a.assignRole(ctx, scope, principalID, roleDefID); err != nil { + return err + } + } + return nil +} + // assignBlobContributorToCurrentPrincipal grants "Storage Blob Data Contributor" on the // e2e blob container to the principal currently authenticated against ARM (the test // runner: ADO service-connection SP in pipelines, or the developer's user identity @@ -504,7 +523,7 @@ func (a *AzureClient) assignRolesToVMIdentity(ctx context.Context, principalID * // Idempotent: uses a deterministic role-assignment name derived from // (scope, principalID, roleDefinitionID) so re-runs recreate the same assignment ID // instead of accumulating duplicate assignments (Azure caps at ~2000/sub); a pre-existing -// role assignment returns 409 Conflict which is swallowed. +// role assignment is accepted. func (a *AzureClient) assignBlobContributorToCurrentPrincipal(ctx context.Context) error { principalID, err := getCurrentPrincipalID(ctx, a.Credential) if err != nil { @@ -519,10 +538,12 @@ func (a *AzureClient) assignBlobContributorToCurrentPrincipal(ctx context.Contex ) // Storage Blob Data Contributor built-in role. roleDefID := "/providers/Microsoft.Authorization/roleDefinitions/ba92f5b4-2d11-453d-a403-e96b0029c9fe" - // Deterministic assignment name so re-runs produce the same GUID and hit the 409 - // swallow-path below instead of creating a new assignment each run. + return a.assignRole(ctx, scope, principalID, roleDefID) +} + +func (a *AzureClient) assignRole(ctx context.Context, scope, principalID, roleDefID string) error { uid := uuid.NewSHA1(uuid.NameSpaceOID, []byte(scope+"|"+principalID+"|"+roleDefID)).String() - _, err = a.RoleAssignments.Create(ctx, scope, uid, armauthorization.RoleAssignmentCreateParameters{ + _, err := a.RoleAssignments.Create(ctx, scope, uid, armauthorization.RoleAssignmentCreateParameters{ Properties: &armauthorization.RoleAssignmentProperties{ PrincipalID: to.Ptr(principalID), RoleDefinitionID: to.Ptr(roleDefID), @@ -534,10 +555,10 @@ func (a *AzureClient) assignBlobContributorToCurrentPrincipal(ctx context.Contex }, nil) var respError *azcore.ResponseError if err != nil { - if errors.As(err, &respError) && respError.StatusCode == http.StatusConflict { + if errors.As(err, &respError) && respError.StatusCode == http.StatusConflict && respError.ErrorCode == "RoleAssignmentExists" { return nil } - return fmt.Errorf("assign Storage Blob Data Contributor role to current principal %s: %w", principalID, err) + return fmt.Errorf("assign role %s to principal %s at %s: %w", roleDefID, principalID, scope, err) } return nil } diff --git a/e2e/config/config.go b/e2e/config/config.go index a6dc015a80d..f8e80986365 100644 --- a/e2e/config/config.go +++ b/e2e/config/config.go @@ -42,6 +42,7 @@ func PrivateACRName(location string) string { } type Configuration struct { + ACLBaseImageSigned bool ACRSecretName string AzureContainerRegistrytargetRepository string BlobContainer string @@ -91,6 +92,7 @@ type Configuration struct { func DefaultConfiguration() *Configuration { return &Configuration{ + ACLBaseImageSigned: true, ACRSecretName: "acr-secret-code2", AzureContainerRegistrytargetRepository: "aks-managed-repository/*", BlobContainer: "abe2e", diff --git a/e2e/config/flags.go b/e2e/config/flags.go index e0042cbe376..14af5f0b4b3 100644 --- a/e2e/config/flags.go +++ b/e2e/config/flags.go @@ -5,6 +5,7 @@ import "github.com/urfave/cli/v3" func Flags() []cli.Flag { defaults := DefaultConfiguration() return []cli.Flag{ + &cli.BoolFlag{Name: "acl-base-image-signed", Value: defaults.ACLBaseImageSigned, Destination: &Config.ACLBaseImageSigned, Sources: cli.EnvVars("ACL_BASE_IMAGE_SIGNED"), Usage: "Whether the ACL base image is signed (default true). Set to false for non-prod (e.g. acldevel-sourced) images whose kernel/shim is not enrolled in the platform Secure Boot db, which disables Secure Boot for ACL scenarios"}, &cli.StringFlag{Name: "acr-secret-name", Value: defaults.ACRSecretName, Destination: &Config.ACRSecretName, Sources: cli.EnvVars("ACR_SECRET_NAME")}, &cli.StringFlag{Name: "acr-target-repository", Value: defaults.AzureContainerRegistrytargetRepository, Destination: &Config.AzureContainerRegistrytargetRepository, Sources: cli.EnvVars("ACR_TARGET_REPOSITORY")}, &cli.StringFlag{Name: "blob-container", Value: defaults.BlobContainer, Destination: &Config.BlobContainer, Sources: cli.EnvVars("BLOB_CONTAINER")}, diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 7d6f7eb8b24..549c59d5062 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -334,6 +334,7 @@ type Image struct { Distro datamodel.Distro Name string OS OS + SharedGalleryImageID VHDResourceID Version string Gallery *Gallery UnsupportedKubeletNodeIP bool diff --git a/e2e/go.mod b/e2e/go.mod index d7b31262598..14e4219efa4 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -95,6 +95,7 @@ require ( k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect k8s.io/streaming v0.36.1 // indirect k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + mvdan.cc/sh/v3 v3.14.1 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect diff --git a/e2e/go.sum b/e2e/go.sum index e55e35627c3..c9da6d4eb8d 100644 --- a/e2e/go.sum +++ b/e2e/go.sum @@ -97,6 +97,8 @@ github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474= +github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= @@ -164,8 +166,8 @@ github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTU github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw= github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= @@ -252,6 +254,8 @@ k8s.io/streaming v0.36.1 h1:L+K68n4Gg940BGNNYtUBvL1WTLL0YnKT3s+P1MNAmR4= k8s.io/streaming v0.36.1/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +mvdan.cc/sh/v3 v3.14.1 h1:bXkhQWNHCs0KZEChF8hYS6FC+T2N9mUZLbQv9blditI= +mvdan.cc/sh/v3 v3.14.1/go.mod h1:syYCoFET8w9tvevxiXUtY8/ICrU+l26jHmhJDra3Vwo= sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4= sigs.k8s.io/controller-runtime v0.24.1/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= diff --git a/e2e/scenario/cosi_download.go b/e2e/scenario/cosi_download.go new file mode 100644 index 00000000000..ca3b8028717 --- /dev/null +++ b/e2e/scenario/cosi_download.go @@ -0,0 +1,114 @@ +package scenario + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" + + "github.com/Azure/agentbaker/e2e/logging" +) + +// cosiDownloadTimeout is the maximum time allowed for downloading and +// streaming through the entire COSI file. ACL COSIs can be multi-GB. +const cosiDownloadTimeout = 30 * time.Minute + +// cosiDownloadRetries is the number of attempts made to download a COSI file +// before giving up. Multi-GB COSI downloads are occasionally interrupted +// mid-stream by transient network errors, so the whole download is retried +// from scratch on failure. +const cosiDownloadRetries = 3 + +// cosiDownloadRetryBackoff is the delay between COSI download retry attempts. +const cosiDownloadRetryBackoff = 15 * time.Second + +// downloadCOSIFileOnce performs a single attempt at downloading the COSI file +// at cosiURL to destPath on local disk. +func downloadCOSIFileOnce(ctx context.Context, cosiURL, destPath string) (err error) { + ctx, cancel := context.WithTimeout(ctx, cosiDownloadTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, cosiURL, nil) + if err != nil { + return fmt.Errorf("creating HTTP request for COSI download: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("downloading COSI file: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("COSI download returned non-200 status: %d", resp.StatusCode) + } + + out, err := os.Create(destPath) + if err != nil { + return fmt.Errorf("creating local COSI file %s: %w", destPath, err) + } + defer func() { + if closeErr := out.Close(); err == nil { + err = closeErr + } + }() + + if _, err := io.Copy(out, resp.Body); err != nil { + return fmt.Errorf("writing COSI file to %s: %w", destPath, err) + } + return nil +} + +// downloadCOSIFileWithRetry downloads the COSI file at cosiURL to destPath on +// local disk, retrying the whole download on failure. Multi-GB COSI +// downloads are occasionally interrupted mid-stream, and since a partially +// read tar stream can't be resumed, the simplest reliable fix is to retry +// the entire download from scratch. +func downloadCOSIFileWithRetry(ctx context.Context, cosiURL, destPath string) error { + var lastErr error + for attempt := 1; attempt <= cosiDownloadRetries; attempt++ { + if attempt > 1 { + logging.Logf(ctx, "retrying COSI download (attempt %d/%d) after error: %v", attempt, cosiDownloadRetries, lastErr) + select { + case <-time.After(cosiDownloadRetryBackoff): + case <-ctx.Done(): + return ctx.Err() + } + } + if err := downloadCOSIFileOnce(ctx, cosiURL, destPath); err != nil { + lastErr = err + continue + } + return nil + } + return fmt.Errorf("download COSI file after %d attempts: %w", cosiDownloadRetries, lastErr) +} + +// cosiDownloadDir returns a scratch directory for downloading COSI files, +// along with a cleanup function the caller must defer. COSIs can be +// multi-GB, and the OS temp dir is too small on some ADO runners. When the +// ADO agent's own temp directory is available (AGENT_TEMPDIRECTORY, set via +// Agent.TempDirectory), which lives on the same, larger volume as the +// checked-out source (Agent.WorkFolder/_work), prefer that instead. +func cosiDownloadDir() (string, func(), error) { + base := os.Getenv("AGENT_TEMPDIRECTORY") + downloadDir, err := os.MkdirTemp(base, "acl-cosi-*") + if err != nil { + if base != "" { + return "", nil, fmt.Errorf("creating COSI download directory under AGENT_TEMPDIRECTORY: %w", err) + } + return "", nil, fmt.Errorf("creating COSI download directory: %w", err) + } + return downloadDir, func() { os.RemoveAll(downloadDir) }, nil +} + +// sanitizeURL removes query parameters (which may contain SAS tokens) from a +// URL for safe logging. +func sanitizeURL(u string) string { + if idx := strings.IndexByte(u, '?'); idx >= 0 { + return u[:idx] + "?" + } + return u +} diff --git a/e2e/scenario/cosi_test.go b/e2e/scenario/cosi_test.go new file mode 100644 index 00000000000..ee11ab50110 --- /dev/null +++ b/e2e/scenario/cosi_test.go @@ -0,0 +1,119 @@ +package scenario + +import ( + "bytes" + "context" + "crypto/sha512" + "encoding/base64" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Azure/agentbaker/e2e/config" + "github.com/Azure/agentbaker/e2e/logging" + "github.com/stretchr/testify/require" +) + +const aclCOSIAMD64ImageID = "/SharedGalleries/035db282-f1c8-4ce7-b78f-2a7265d5398c-ACLDEVEL/Images/acldevel/Versions/0.20260827.1192019" + +func TestValidateCOSIUpdateInput(t *testing.T) { + validHash := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0xab}, sha512.Size384)) + require.NoError(t, validateCOSIUpdateInput("https://download.example.com/acl.cosi", validHash)) + require.ErrorContains(t, validateCOSIUpdateInput("http://download.example.com/acl.cosi", validHash), "HTTPS") + require.ErrorContains(t, validateCOSIUpdateInput("https://download.example.com/acl.cosi", "abcd"), "48 bytes") +} + +func TestResolveCOSISharedGalleryImageReference(t *testing.T) { + imageReference, err := resolveImageReference(context.Background(), &config.Image{ + SharedGalleryImageID: aclCOSIAMD64ImageID, + Version: "would-trigger-gallery-lookup-without-shared-id", + }, "westus2") + + require.NoError(t, err) + require.Nil(t, imageReference.ID) + require.Equal(t, aclCOSIAMD64ImageID, *imageReference.SharedGalleryImageID) +} + +func TestStageCOSIUpdateImage(t *testing.T) { + t.Setenv("AGENT_TEMPDIRECTORY", t.TempDir()) + payload := []byte("test COSI bytes") + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + _, _ = response.Write(payload) + })) + defer server.Close() + names := map[string]bool{} + for _, uploadErr := range []error{nil, errors.New("upload failed")} { + var localPath string + blobName, signedURL, err := stageCOSIUpdateImage(t.Context(), server.URL, func(ctx context.Context, name string, file *os.File) (string, error) { + require.NoError(t, ctx.Err()) + require.True(t, strings.HasPrefix(name, "cosi-updates/acl-cosi-")) + require.False(t, names[name]) + names[name] = true + contents, readErr := io.ReadAll(file) + require.NoError(t, readErr) + require.Equal(t, payload, contents) + localPath = file.Name() + return "https://storage.invalid/test.cosi?sig=private", uploadErr + }) + require.ErrorIs(t, err, uploadErr) + require.True(t, names[blobName]) + require.Equal(t, "https://storage.invalid/test.cosi?sig=private", signedURL) + require.NoFileExists(t, localPath) + } +} + +func TestCheckTridentUpdateResult(t *testing.T) { + for _, test := range []struct { + name string + result *podExecResult + err error + wantError string + }{ + {name: "success", result: &podExecResult{exitCode: "0", stdout: "https://storage.invalid/image?sig=private"}}, + {name: "exit failure", result: &podExecResult{exitCode: "1", stderr: "https://storage.invalid/image?sig=private"}, wantError: "exited with code 1"}, + {name: "transport failure", err: errors.New("https://storage.invalid/image?sig=private"), wantError: "SSH command error"}, + {name: "missing result", wantError: "returned no result"}, + } { + t.Run(test.name, func(t *testing.T) { + logger := &executionLogger{} + err := checkTridentUpdateResult(logging.WithLogger(t.Context(), logger), "stage", test.result, test.err) + if test.wantError == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, test.wantError) + require.NotContains(t, err.Error(), "sig=private") + } + require.NotContains(t, strings.Join(logger.logs, "\n"), "sig=private") + }) + } +} + +func TestLoadCOSIPublishingInfo(t *testing.T) { + dir := t.TempDir() + t.Setenv("COSI_ARTIFACTS_DIR", dir) + _, exists := loadCOSIPublishingInfo("not-built") + require.False(t, exists) + require.NoError(t, os.Mkdir(filepath.Join(dir, "built"), 0700)) + info := cosiPublishingInfo{ + CosiURL: "https://download.invalid/image.cosi", + MetadataSHA384: base64.StdEncoding.EncodeToString(make([]byte, sha512.Size384)), + } + writeInfo := func() { + data, err := json.Marshal(info) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "built", "cosi-publishing-info.json"), data, 0600)) + } + writeInfo() + actual, exists := loadCOSIPublishingInfo("built") + require.True(t, exists) + require.Equal(t, info, actual) + info.MetadataSHA384 = "bad-hash" + writeInfo() + require.Panics(t, func() { loadCOSIPublishingInfo("built") }) +} diff --git a/e2e/scenario/gc_deadline.go b/e2e/scenario/gc_deadline.go index bfd00c6d224..27be06251f0 100644 --- a/e2e/scenario/gc_deadline.go +++ b/e2e/scenario/gc_deadline.go @@ -36,6 +36,12 @@ func ensureResourceGroup(ctx context.Context, azure *config.AzureClient, cfg *co } if cfg.InADO { renewResourceGroupDeadline(ctx, azure, cfg, rg) + if rg.ID == nil || *rg.ID == "" { + return armresources.ResourceGroup{}, fmt.Errorf("RG %q has no resource ID", name) + } + if err := azure.EnsureResourceGroupRoles(ctx, *rg.ID); err != nil { + return armresources.ResourceGroup{}, fmt.Errorf("restore roles for RG %q: %w", name, err) + } } return rg, nil } diff --git a/e2e/scenario/gc_deadline_test.go b/e2e/scenario/gc_deadline_test.go index a2962fae14a..b56dc03a19f 100644 --- a/e2e/scenario/gc_deadline_test.go +++ b/e2e/scenario/gc_deadline_test.go @@ -2,6 +2,7 @@ package scenario import ( "context" + "encoding/base64" "encoding/json" "io" "net/http" @@ -13,23 +14,38 @@ import ( "github.com/Azure/agentbaker/e2e/config" "github.com/Azure/agentbaker/e2e/logging" "github.com/Azure/agentbaker/e2e/toolkit" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v3" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3" "github.com/stretchr/testify/require" ) +type gcTestCredential struct{} + +func (gcTestCredential) GetToken(context.Context, policy.TokenRequestOptions) (azcore.AccessToken, error) { + payload := base64.RawURLEncoding.EncodeToString([]byte(`{"oid":"11111111-1111-4111-8111-111111111111"}`)) + return azcore.AccessToken{Token: "e30." + payload + ".signature"}, nil +} + func gcTestAzure(t *testing.T, respond vmssCreationTestPolicy) *config.AzureClient { t.Helper() opts := &arm.ClientOptions{ClientOptions: policy.ClientOptions{PerCallPolicies: []policy.Policy{respond}}} - client := &config.AzureClient{} + client := &config.AzureClient{Credential: gcTestCredential{}} var err error client.ResourceGroup, err = armresources.NewResourceGroupsClient("test", nil, opts) require.NoError(t, err) client.Tags, err = armresources.NewTagsClient("test", nil, opts) require.NoError(t, err) + rolePolicy := vmssCreationTestPolicy(func(req *http.Request) *http.Response { + return gcResponse(req, http.StatusOK, `{}`) + }) + client.RoleAssignments, err = armauthorization.NewRoleAssignmentsClient("test", nil, + &arm.ClientOptions{ClientOptions: policy.ClientOptions{PerCallPolicies: []policy.Policy{rolePolicy}}}) + require.NoError(t, err) return client } @@ -38,6 +54,31 @@ func gcResponse(req *http.Request, status int, body string) *http.Response { Body: io.NopCloser(strings.NewReader(body)), Request: req} } +func gcRoleAssignmentPolicy(t *testing.T, scope string, assignments, names map[string]string) vmssCreationTestPolicy { + t.Helper() + return func(req *http.Request) *http.Response { + require.Equal(t, http.MethodPut, req.Method) + prefix := scope + "/providers/Microsoft.Authorization/roleAssignments/" + require.True(t, strings.HasPrefix(req.URL.Path, prefix), "role must be scoped to the shared RG") + var assignment armauthorization.RoleAssignmentCreateParameters + require.NoError(t, json.NewDecoder(req.Body).Decode(&assignment)) + require.NotNil(t, assignment.Properties) + require.Equal(t, to.Ptr("11111111-1111-4111-8111-111111111111"), assignment.Properties.PrincipalID) + require.NotNil(t, assignment.Properties.RoleDefinitionID) + role := *assignment.Properties.RoleDefinitionID + name := strings.TrimPrefix(req.URL.Path, prefix) + if previous, ok := names[role]; ok { + require.Equal(t, previous, name, "assignment name must remain stable across runs and recreation") + } + names[role] = name + if _, exists := assignments[role]; exists { + return gcResponse(req, http.StatusConflict, `{"error":{"code":"RoleAssignmentExists","message":"already assigned"}}`) + } + assignments[role] = name + return gcResponse(req, http.StatusCreated, `{}`) + } +} + func TestLocalResourceGroupDeadlineIsUntouched(t *testing.T) { for _, buildID := range []string{"local", "12345"} { for _, node := range []bool{false, true} { @@ -250,6 +291,9 @@ func TestEnsureResourceGroup(t *testing.T) { return gcResponse(req, status, string(body)) }) logger := &executionLogger{} + // Local setup must not require role-assignment credentials. + azure.Credential = nil + azure.RoleAssignments = nil rg, err := ensureResourceGroup(logging.WithLogger(t.Context(), logger), azure, &config.Configuration{}, "westus3") if tt.failure != "" { require.ErrorContains(t, err, tt.failure) @@ -263,6 +307,86 @@ func TestEnsureResourceGroup(t *testing.T) { } } +func TestEnsureResourceGroupRestoresScopedRoles(t *testing.T) { + for _, tt := range []struct { + name string + exists bool + }{ + {name: "new resource group"}, + {name: "existing resource group without roles", exists: true}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + const scope = "/subscriptions/test/resourceGroups/abe2e-westus3" + const galleryRole = "/providers/Microsoft.Authorization/roleDefinitions/85a2d0d9-2eba-4c9c-b355-11c2cc0788ab" + const importRole = "/providers/Microsoft.Authorization/roleDefinitions/577a9874-89fd-4f24-9dbd-b5034d0ad23a" + groupExists := tt.exists + assignments := map[string]string{} + names := map[string]string{} + respond := vmssCreationTestPolicy(func(req *http.Request) *http.Response { + require.True(t, strings.EqualFold(scope, req.URL.Path)) + if req.Method == http.MethodGet && !groupExists { + return gcResponse(req, http.StatusNotFound, `{}`) + } + if req.Method == http.MethodPut { + groupExists = true + } else { + require.Equal(t, http.MethodGet, req.Method) + } + return gcResponse(req, http.StatusOK, + `{"name":"abe2e-westus3","id":"`+scope+`","tags":{"deletion_due_time":"2000-01-01T08:00:00Z"}}`) + }) + azure := gcTestAzure(t, respond) + var err error + rolePolicy := gcRoleAssignmentPolicy(t, scope, assignments, names) + azure.RoleAssignments, err = armauthorization.NewRoleAssignmentsClient("test", nil, + &arm.ClientOptions{ClientOptions: policy.ClientOptions{PerCallPolicies: []policy.Policy{rolePolicy}}}) + require.NoError(t, err) + cfg := &config.Configuration{InADO: true} + for attempt := range 3 { + if attempt == 2 { + groupExists = false + clear(assignments) + } + _, err = ensureResourceGroup(t.Context(), azure, cfg, "westus3") + require.NoError(t, err) + require.Len(t, assignments, 2) + require.Contains(t, assignments, galleryRole) + require.Contains(t, assignments, importRole) + } + }) + } +} + +func TestEnsureResourceGroupSurfacesRoleAssignmentErrors(t *testing.T) { + for _, tt := range []struct { + name string + status int + code string + }{ + {name: "permission denied", status: http.StatusForbidden, code: "AuthorizationFailed"}, + {name: "unexpected conflict", status: http.StatusConflict, code: "RoleAssignmentUpdateNotPermitted"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + azure := gcTestAzure(t, func(req *http.Request) *http.Response { + require.Equal(t, http.MethodGet, req.Method) + return gcResponse(req, http.StatusOK, + `{"name":"abe2e-westus3","id":"/subscriptions/test/resourceGroups/abe2e-westus3","tags":{"deletion_due_time":"2000-01-01T08:00:00Z"}}`) + }) + rolePolicy := vmssCreationTestPolicy(func(req *http.Request) *http.Response { + return gcResponse(req, tt.status, `{"error":{"code":"`+tt.code+`","message":"cannot assign role"}}`) + }) + var err error + azure.RoleAssignments, err = armauthorization.NewRoleAssignmentsClient("test", nil, + &arm.ClientOptions{ClientOptions: policy.ClientOptions{PerCallPolicies: []policy.Policy{rolePolicy}}}) + require.NoError(t, err) + _, err = ensureResourceGroup(t.Context(), azure, &config.Configuration{InADO: true}, "westus3") + require.ErrorContains(t, err, tt.code) + }) + } +} + func TestSaturdayDeadline(t *testing.T) { for _, tt := range []struct { name, earliest, want string diff --git a/e2e/scenario/network_validation_test.go b/e2e/scenario/network_validation_test.go index 26de498b33f..fd7c0086922 100644 --- a/e2e/scenario/network_validation_test.go +++ b/e2e/scenario/network_validation_test.go @@ -12,6 +12,7 @@ import ( func TestValidateRxBufferDefaultDoesNotSkipBySKU(t *testing.T) { for _, distro := range []datamodel.Distro{ + datamodel.AKSAzureLinuxV2Gen2, datamodel.AKSAzureLinuxV3Gen2, datamodel.AKSACLGen2TL, datamodel.AKSUbuntuContainerd2404Gen2, @@ -33,6 +34,16 @@ func TestValidateRxBufferDefaultDoesNotSkipBySKU(t *testing.T) { } } +func TestValidateRxBufferDefaultSkipsOldVHDs(t *testing.T) { + s := &Scenario{ + Config: Config{VHD: &config.Image{ + Distro: datamodel.AKSAzureLinuxV2Gen2, + SkipOldVHDValidations: true, + }}, + } + require.NoError(t, ValidateRxBufferDefault(logging.WithLogger(t.Context(), t), s)) +} + func TestParseCurrentRxBuffer(t *testing.T) { for _, tc := range []struct { name, output, want string diff --git a/e2e/scenario/provision.go b/e2e/scenario/provision.go index c0044fb1a2c..c5e6ad53435 100644 --- a/e2e/scenario/provision.go +++ b/e2e/scenario/provision.go @@ -41,7 +41,7 @@ func runScenarioFlow(ctx context.Context, name string, s *Scenario) error { } func scriptlessUnsupported(s *Scenario) bool { - return s.IsWindows() || len(s.Config.CustomDataWriteFiles) > 0 || s.VHDCaching || config.Config.TestPreProvision || s.VHD.Distro == datamodel.AKSAzureLinuxV2Gen2 + return s.Config.SkipScriptlessNBCCSECmd || s.IsWindows() || len(s.Config.CustomDataWriteFiles) > 0 || s.VHDCaching || config.Config.TestPreProvision || s.VHD.Distro == datamodel.AKSAzureLinuxV2Gen2 } func runVHDCachingScenario(ctx context.Context, name string, original *Scenario) error { @@ -429,10 +429,7 @@ func annotateVMSSCreateError(s *Scenario, err error) error { func maybeSkipScenario(ctx context.Context, name string, s *Scenario) error { s.Tags = s.EffectiveTags() - _, err := CachedPrepareVHD(ctx, GetVHDRequest{ - Image: *s.VHD, - Location: s.Location, - }) + _, err := resolveImageReference(ctx, s.VHD, s.Location) if err != nil { if config.Config.IgnoreScenariosWithMissingVHD && errors.Is(err, config.ErrNotFound) { return &skipError{message: fmt.Sprintf("scenario %q image for VHD %s was not found: %s", name, s.VHD.Distro, err)} @@ -729,6 +726,22 @@ func addTrustedLaunchToVMSS(properties *armcompute.VirtualMachineScaleSetPropert return properties } +// aclVMSSSecurityProfile deploys ACL VMs as TrustedLaunch. ACL builds set +// ENABLE_TRUSTED_LAUNCH, so their SIG image definitions are created with +// SecurityType=TrustedLaunch and reject Standard deployments; TrustedLaunch also +// works once those definitions move to TrustedLaunchSupported. Secure Boot is +// disabled only when baseImageSigned is false, which unsigned/dev ACL builds +// (e.g. acldevel-sourced images) need: their kernel/shim is not enrolled in the +// platform's Secure Boot db, so with Secure Boot on, UEFI firmware returns +// "Access denied" loading the kernel EFI stub and boot never completes. +func aclVMSSSecurityProfile(properties *armcompute.VirtualMachineScaleSetProperties, baseImageSigned bool) *armcompute.VirtualMachineScaleSetProperties { + properties = addTrustedLaunchToVMSS(properties) + if !baseImageSigned { + properties.VirtualMachineProfile.SecurityProfile.UefiSettings.SecureBootEnabled = to.Ptr(false) + } + return properties +} + func createVMExtensionLinuxAKSNode(ctx context.Context, location *string) (*armcompute.VirtualMachineScaleSetExtension, error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() diff --git a/e2e/scenario/scenario.go b/e2e/scenario/scenario.go index 2d5121a16fd..124fc3b705f 100755 --- a/e2e/scenario/scenario.go +++ b/e2e/scenario/scenario.go @@ -67,7 +67,7 @@ var _ = Register(&Scenario{ AKSNodeConfigMutator: func(_ *Cluster, config *aksnodeconfigv1.Configuration) { }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) }, Validator: func(ctx context.Context, s *Scenario) error { return errors.Join( @@ -92,7 +92,7 @@ var _ = Register(&Scenario{ } }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) }, Validator: func(ctx context.Context, s *Scenario) error { return errors.Join( @@ -120,7 +120,7 @@ var _ = Register(&Scenario{ nbc.IsARM64 = true }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) vmss.SKU.Name = to.Ptr("Standard_D2pds_v6") }, Validator: func(ctx context.Context, s *Scenario) error { @@ -144,7 +144,7 @@ var _ = Register(&Scenario{ nbc.AgentPoolProfile.LocalDNSProfile = nil }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) }, Validator: func(ctx context.Context, s *Scenario) error { return errors.Join( @@ -185,7 +185,7 @@ var _ = Register(&Scenario{ Cluster: ClusterAzureNetwork, VHD: config.VHDACLGen2TL, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) }, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.NetworkPlugin = string(armcontainerservice.NetworkPluginAzure) @@ -216,7 +216,7 @@ var _ = Register(&Scenario{ Cluster: ClusterKubenet, VHD: config.VHDACLGen2TL, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) }, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.SecureTLSBootstrappingConfig = &datamodel.SecureTLSBootstrappingConfig{ @@ -235,7 +235,7 @@ var _ = Register(&Scenario{ Cluster: ClusterKubenet, VHD: config.VHDACLGen2TL, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) }, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.SSHStatus = datamodel.SSHOff @@ -263,6 +263,20 @@ func init() { } } +// skipIfACLBaseImageUnsigned skips ACL GPU scenarios when running against an +// unsigned/dev ACL base image (ACL_BASE_IMAGE_SIGNED=false, e.g. acldevel-sourced). +// Unsigned dev builds are pinned to a specific VERSION_ID that is not guaranteed to +// have a matching NVIDIA GPU sysext (nvidia-container-toolkit, nvidia-driver-*, etc.) +// published to MCR yet, since GPU sysext publishing lags the dev image build. That +// causes GPU provisioning to fail with an unrelated-looking VMExtensionProvisioningError +// rather than a real product regression, so skip GPU scenarios for unsigned builds. +func skipIfACLBaseImageUnsigned(context.Context) string { + if !config.Config.ACLBaseImageSigned { + return "ACL_BASE_IMAGE_SIGNED=false (unsigned/dev ACL base image); matching NVIDIA GPU sysexts are not guaranteed to be published for this build" + } + return "" +} + func aclGPUScenario(name, vmSize, location string) *Scenario { return &Scenario{ Name: name, @@ -271,6 +285,7 @@ func aclGPUScenario(name, vmSize, location string) *Scenario { Tags: Tags{ GPU: true, }, + SkipIf: skipIfACLBaseImageUnsigned, Config: Config{ Cluster: ClusterKubenet, VHD: config.VHDACLGen2TL, @@ -282,7 +297,7 @@ func aclGPUScenario(name, vmSize, location string) *Scenario { }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.SKU.Name = to.Ptr(vmSize) - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) }, Validator: func(ctx context.Context, s *Scenario) error { return errors.Join( @@ -302,6 +317,7 @@ func aclGRIDScenario(name, vmSize string) *Scenario { Tags: Tags{ GPU: true, }, + SkipIf: skipIfACLBaseImageUnsigned, Config: Config{ Cluster: ClusterKubenet, VHD: config.VHDACLGen2TL, @@ -313,7 +329,7 @@ func aclGRIDScenario(name, vmSize string) *Scenario { }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.SKU.Name = to.Ptr(vmSize) - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) }, Validator: func(ctx context.Context, s *Scenario) error { return errors.Join( @@ -734,7 +750,7 @@ func newUbuntu2204EmbeddedScriptHotfixScenario() *Scenario { Payload: payload, }} // Older VHDs do not have the modules sourced by the current provision config. - for _, suffix := range []string{"gpu", "localdns", "kubelet", "network", "addons", "chrony"} { + for _, suffix := range []string{"gpu", "localdns", "kubelet", "network", "chrony"} { name := "cse_config_" + suffix + ".sh" module, err := os.ReadFile(repoPath("parts/linux/cloud-init/artifacts/" + name)) if err != nil { @@ -971,7 +987,7 @@ var _ = Register(&Scenario{ Cluster: ClusterAzureNetworkIsolated, VHD: config.VHDACLGen2TL, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) }, BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { nbc.OutboundType = datamodel.OutboundTypeBlock @@ -3492,7 +3508,7 @@ var _ = Register(&Scenario{ nbc.StandardSecondaryNICCount = 1 }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) addSecondaryNIC(vmss) }, Validator: func(ctx context.Context, s *Scenario) error { @@ -3660,7 +3676,7 @@ var _ = Register(&Scenario{ nbc.AgentPoolProfile.CustomNodeLabels["kubernetes.azure.com/azure-cni-overlay"] = "true" }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) DualStackVMConfigMutator(vmss) addDualStackSecondaryNIC(vmss) }, @@ -3806,3 +3822,36 @@ var _ = Register(&Scenario{ }, }, }) + +// The 1.33 inputs need a separate node to exercise the natural Dalec gate rather +// than the enforced-install path or the suite's default Kubernetes version. +var _ = Register(&Scenario{ + Name: "Ubuntu2204_Dalec_CredentialProvider_Kubernetes133", + Description: "Tests Ubuntu 22.04 installs the Dalec credential provider at the natural Kubernetes 1.33 gate", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2204Gen2Containerd, + VMConfigMutator: EmptyVMConfigMutator, + BootstrapConfigMutatorWithError: func(_ context.Context, _ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) error { + return configureUbuntuDalecCredentialProvider(nbc, + components.GetExpectedPackageVersions("kubernetes-binaries", "default", "current")) + }, + Validator: func(ctx context.Context, s *Scenario) error { + return errors.Join( + ValidateFileExists(ctx, s, "/var/lib/kubelet/credential-provider/acr-credential-provider"), + ValidateFileHasContent(ctx, s, "/var/log/azure/cluster-provision.log", "installCredentialProviderFromPkg"), + ) + }, + }, +}) + +func configureUbuntuDalecCredentialProvider(nbc *datamodel.NodeBootstrappingConfiguration, kubeletVersions []string) error { + version, err := credentialProviderVersion(kubeletVersions, "kubernetes-binaries/default/current", 33) + if err != nil { + return err + } + nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion = version + nbc.KubeletConfig["--image-credential-provider-config"] = "/var/lib/kubelet/credential-provider-config.yaml" + nbc.KubeletConfig["--image-credential-provider-bin-dir"] = "/var/lib/kubelet/credential-provider" + return nil +} diff --git a/e2e/scenario/scenario_cosi.go b/e2e/scenario/scenario_cosi.go new file mode 100644 index 00000000000..39cd9c8cd11 --- /dev/null +++ b/e2e/scenario/scenario_cosi.go @@ -0,0 +1,544 @@ +package scenario + +import ( + "context" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/Azure/agentbaker/e2e/config" + "github.com/Azure/agentbaker/e2e/logging" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v8" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + remoteCOSIConfigPath = "/home/azureuser/update-config.yaml" + cosiAMD64PublishingArtifact = "cosi-publishing-info-acl-tl-gen2" + cosiARM64PublishingArtifact = "cosi-publishing-info-acl-arm64-tl-gen2" + cosiAMD64FIPSPublishingArtifact = "cosi-publishing-info-acl-fips-tl-gen2" + cosiARM64FIPSPublishingArtifact = "cosi-publishing-info-acl-arm64-fips-tl-gen2" +) + +// cosiPublishingInfo mirrors the JSON written by convert-vhd-to-cosi.sh. +type cosiPublishingInfo struct { + CosiURL string `json:"cosi_url"` + MetadataSHA384 string `json:"metadata_sha384"` +} + +// loadCOSIPublishingInfo reads cosi-publishing-info.json from a downloaded +// pipeline artifact directory. artifactName is the pipeline artifact name +// (e.g., "cosi-publishing-info-acl-tl-gen2"). Returns ok=false if +// COSI_ARTIFACTS_DIR is not set or the artifact was not downloaded (variant +// not built for this run). This runs at package init time (via Register), +// so a malformed artifact -- as opposed to a missing one -- is a hard +// failure and panics, matching the invariant-violation convention used +// elsewhere in this package (see registry.go). +func loadCOSIPublishingInfo(artifactName string) (cosiPublishingInfo, bool) { + dir := os.Getenv("COSI_ARTIFACTS_DIR") + if dir == "" { + return cosiPublishingInfo{}, false + } + infoPath := filepath.Join(dir, artifactName, "cosi-publishing-info.json") + data, err := os.ReadFile(infoPath) + if err != nil { + if os.IsNotExist(err) { + return cosiPublishingInfo{}, false + } + panic(fmt.Sprintf("reading %s: %v", infoPath, err)) + } + var info cosiPublishingInfo + if err := json.Unmarshal(data, &info); err != nil { + panic(fmt.Sprintf("parsing %s: %v", infoPath, err)) + } + if info.CosiURL == "" { + panic(fmt.Sprintf("cosi_url is empty in %s", infoPath)) + } + if info.MetadataSHA384 == "" { + panic(fmt.Sprintf("metadata_sha384 is empty in %s", infoPath)) + } + if err := validateCOSIUpdateInput(info.CosiURL, info.MetadataSHA384); err != nil { + panic(fmt.Sprintf("invalid COSI publishing artifact %s: %v", infoPath, err)) + } + return info, true +} + +var _ = Register(cosiUpdateAMD64Scenario()) +var _ = Register(cosiUpdateARM64Scenario()) +var _ = Register(cosiUpdateAMD64FIPSScenario()) +var _ = Register(cosiUpdateARM64FIPSScenario()) + +// cosiUpdateAMD64Scenario builds the ACL_COSIUpdate_AMD64 scenario. If the +// COSI publishing artifact isn't available (or its update inputs are +// invalid), the scenario is still registered but with a SkipReason, so +// registry listings and validation still see it -- it just doesn't run. +func cosiUpdateAMD64Scenario() *Scenario { + const name = "ACL_COSIUpdate_AMD64" + const description = "Tests that an AMD64 ACL node remains Ready after a COSI A/B update" + + info, ok := loadCOSIPublishingInfo(cosiAMD64PublishingArtifact) + if !ok { + return &Scenario{ + Name: name, + Description: description, + SkipReason: "COSI artifact not available for acl-tl-gen2, skipping COSI update test", + } + } + if err := validateCOSIUpdateInput(info.CosiURL, info.MetadataSHA384); err != nil { + return &Scenario{ + Name: name, + Description: description, + SkipReason: fmt.Sprintf("invalid COSI update input: %v", err), + } + } + + return &Scenario{ + Name: name, + Description: description, + SkipReason: "COSI update tests are temporarily disabled", + Location: "westus3", + Tags: Tags{ + COSIUpdate: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDACLGen2TL, + SkipScriptlessNBCCSECmd: true, + WaitForSSHAfterReboot: 5 * time.Minute, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) + }, + Validator: func(ctx context.Context, s *Scenario) error { + return validateACLCOSIUpdate(ctx, s, info.CosiURL, info.MetadataSHA384) + }, + }, + } +} + +// cosiUpdateARM64Scenario builds the ACL_COSIUpdate_ARM64 scenario. Same +// shape as cosiUpdateAMD64Scenario, but on ARM64 (Cobalt 100 / Standard_D2pds_v6, +// the only currently-supported ARM64 SKU for ACL -- see ACL_ARM64 in scenario.go). +func cosiUpdateARM64Scenario() *Scenario { + const name = "ACL_COSIUpdate_ARM64" + const description = "Tests that an ARM64 ACL node remains Ready after a COSI A/B update" + + info, ok := loadCOSIPublishingInfo(cosiARM64PublishingArtifact) + if !ok { + return &Scenario{ + Name: name, + Description: description, + SkipReason: "COSI artifact not available for acl-arm64-tl-gen2, skipping COSI update test", + } + } + if err := validateCOSIUpdateInput(info.CosiURL, info.MetadataSHA384); err != nil { + return &Scenario{ + Name: name, + Description: description, + SkipReason: fmt.Sprintf("invalid COSI update input: %v", err), + } + } + + return &Scenario{ + Name: name, + Description: description, + SkipReason: "COSI update tests are temporarily disabled", + Location: "westus3", + Tags: Tags{ + COSIUpdate: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDACLArm64Gen2TL, + // v6 (Cobalt 100) only supports NVMe disk controllers, not ResourceDisk + UseNVMe: true, + SkipScriptlessNBCCSECmd: true, + WaitForSSHAfterReboot: 5 * time.Minute, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + // Ampere Altra (v5) doesn't support TrustedLaunch; Cobalt 100 (v6) does + nbc.AgentPoolProfile.VMSize = "Standard_D2pds_v6" + nbc.IsARM64 = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) + vmss.SKU.Name = to.Ptr("Standard_D2pds_v6") + }, + Validator: func(ctx context.Context, s *Scenario) error { + return validateACLCOSIUpdate(ctx, s, info.CosiURL, info.MetadataSHA384) + }, + }, + } +} + +// cosiUpdateAMD64FIPSScenario builds the ACL_COSIUpdate_AMD64_FIPS scenario. +// Same shape as cosiUpdateAMD64Scenario, but on the FIPS VHD (LocalDNS isn't +// currently supported on FIPS-enabled VHDs; mirrors ACLGen2FIPSTL in scenario.go). +func cosiUpdateAMD64FIPSScenario() *Scenario { + const name = "ACL_COSIUpdate_AMD64_FIPS" + const description = "Tests that an AMD64 FIPS ACL node remains Ready after a COSI A/B update" + + info, ok := loadCOSIPublishingInfo(cosiAMD64FIPSPublishingArtifact) + if !ok { + return &Scenario{ + Name: name, + Description: description, + SkipReason: "COSI artifact not available for acl-fips-tl-gen2, skipping COSI update test", + } + } + if err := validateCOSIUpdateInput(info.CosiURL, info.MetadataSHA384); err != nil { + return &Scenario{ + Name: name, + Description: description, + SkipReason: fmt.Sprintf("invalid COSI update input: %v", err), + } + } + + return &Scenario{ + Name: name, + Description: description, + SkipReason: "COSI update tests are temporarily disabled", + Location: "westus3", + Tags: Tags{ + COSIUpdate: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDACLGen2FIPSTL, + SkipScriptlessNBCCSECmd: true, + WaitForSSHAfterReboot: 5 * time.Minute, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + // LocalDNS isn't currently supported on FIPS-enabled VHDs; mirror ACLGen2FIPSTL. + nbc.AgentPoolProfile.LocalDNSProfile = nil + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) + }, + Validator: func(ctx context.Context, s *Scenario) error { + return validateACLCOSIUpdate(ctx, s, info.CosiURL, info.MetadataSHA384) + }, + }, + } +} + +// cosiUpdateARM64FIPSScenario builds the ACL_COSIUpdate_ARM64_FIPS scenario, +// combining the ARM64 (Cobalt 100 / Standard_D2pds_v6) and FIPS (no LocalDNS) +// requirements of cosiUpdateARM64Scenario and cosiUpdateAMD64FIPSScenario. +func cosiUpdateARM64FIPSScenario() *Scenario { + const name = "ACL_COSIUpdate_ARM64_FIPS" + const description = "Tests that an ARM64 FIPS ACL node remains Ready after a COSI A/B update" + + info, ok := loadCOSIPublishingInfo(cosiARM64FIPSPublishingArtifact) + if !ok { + return &Scenario{ + Name: name, + Description: description, + SkipReason: "COSI artifact not available for acl-arm64-fips-tl-gen2, skipping COSI update test", + } + } + if err := validateCOSIUpdateInput(info.CosiURL, info.MetadataSHA384); err != nil { + return &Scenario{ + Name: name, + Description: description, + SkipReason: fmt.Sprintf("invalid COSI update input: %v", err), + } + } + + return &Scenario{ + Name: name, + Description: description, + SkipReason: "COSI update tests are temporarily disabled", + Location: "westus3", + Tags: Tags{ + COSIUpdate: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDACLArm64Gen2FIPSTL, + // v6 (Cobalt 100) only supports NVMe disk controllers, not ResourceDisk + UseNVMe: true, + SkipScriptlessNBCCSECmd: true, + WaitForSSHAfterReboot: 5 * time.Minute, + BootstrapConfigMutator: func(_ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) { + // Ampere Altra (v5) doesn't support TrustedLaunch; Cobalt 100 (v6) does + nbc.AgentPoolProfile.VMSize = "Standard_D2pds_v6" + nbc.IsARM64 = true + // LocalDNS isn't currently supported on FIPS-enabled VHDs; mirror ACLGen2FIPSTL. + nbc.AgentPoolProfile.LocalDNSProfile = nil + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) + vmss.SKU.Name = to.Ptr("Standard_D2pds_v6") + }, + Validator: func(ctx context.Context, s *Scenario) error { + return validateACLCOSIUpdate(ctx, s, info.CosiURL, info.MetadataSHA384) + }, + }, + } +} + +func validateACLCOSIUpdate(ctx context.Context, s *Scenario, rawCosiURL, rawMetadataHash string) error { + cosiURL := strings.TrimSpace(rawCosiURL) + metadataHash := strings.TrimSpace(rawMetadataHash) + if err := validateCOSIUpdateInput(cosiURL, metadataHash); err != nil { + return err + } + blobName, stagedURL, err := stageCOSIUpdateImage(ctx, cosiURL, config.Azure.UploadAndGetSignedLink) + if blobName != "" { + s.Cleanup(func(cleanupCtx context.Context) error { + _, deleteErr := config.Azure.Blob.DeleteBlob(cleanupCtx, config.Config.BlobContainer, blobName, nil) + if isNotFoundErr(deleteErr) { + return nil + } + return deleteErr + }) + } + if err != nil { + return fmt.Errorf("stage COSI in test storage: %w", err) + } + cosiURL = stagedURL + + // Diagnostic-only: capture trident-acl-agent's systemd status and + // relevant files before and after the update, to help root-cause why it + // enters a failed state after a COSI update. The test never disables or + // masks the agent -- it drives Trident directly via CLI/gRPC (stage, + // finalize, commit below) alongside whatever the agent does on its own. + logACLAgentDiagnostics(ctx, s, "before COSI update") + defer logACLAgentDiagnostics(ctx, s, "after COSI update") + + beforeBootID, err := runCOSICommand(ctx, s, "cat /proc/sys/kernel/random/boot_id") + if err != nil { + return fmt.Errorf("get boot ID before COSI update: %w", err) + } + beforeBootID = strings.TrimSpace(beforeBootID) + if beforeBootID == "" { + return fmt.Errorf("boot ID before COSI update is empty") + } + + beforeNode, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(ctx, s.Runtime.VM.KubeName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get node before COSI update: %w", err) + } + if !strings.EqualFold(beforeBootID, beforeNode.Status.NodeInfo.BootID) { + return fmt.Errorf("host and Kubernetes boot IDs must match before the update: host=%s, k8s=%s", beforeBootID, beforeNode.Status.NodeInfo.BootID) + } + + // Trident's Host Configuration expects the image SHA384 as a 96-character + // lowercase hex string, but the COSI publishing artifact stores it base64 + // encoded (see convert-vhd-to-cosi.sh). Convert before embedding it below. + metadataHashBytes, err := base64.StdEncoding.DecodeString(metadataHash) + if err != nil { + return fmt.Errorf("decode COSI metadata SHA-384: %w", err) + } + metadataHashHex := hex.EncodeToString(metadataHashBytes) + + updateConfig := fmt.Sprintf("image:\n url: %s\n sha384: %s\ninternalParams:\n forceAbUpdate: true\n noTransition: true\n", strconv.Quote(cosiURL), metadataHashHex) + encodedConfig := base64.StdEncoding.EncodeToString([]byte(updateConfig)) + writeConfigCommand := fmt.Sprintf("printf '%%s' %s | base64 --decode > %s && chmod 0600 %s", shellQuote(encodedConfig), remoteCOSIConfigPath, remoteCOSIConfigPath) + if _, err := runCOSICommand(ctx, s, writeConfigCommand); err != nil { + return fmt.Errorf("write Trident update host configuration: %w", err) + } + + version, err := runCOSICommand(ctx, s, "sudo trident --version") + if err != nil { + return fmt.Errorf("get trident version: %w", err) + } + logging.Logf(ctx, "Trident version: %s", strings.TrimSpace(version)) + + stageCommand := fmt.Sprintf("sudo trident update -v trace %s --allowed-operations=stage", remoteCOSIConfigPath) + logging.Logf(ctx, "Trident update (stage) SSH command: %s", stageCommand) + stageResult, stageErr := runSSHCommand(ctx, s.Runtime.VM.SSHClient, stageCommand, false) + if err := checkTridentUpdateResult(ctx, "stage", stageResult, stageErr); err != nil { + return err + } + + finalizeCommand := fmt.Sprintf("sudo trident update -v trace %s --allowed-operations=finalize", remoteCOSIConfigPath) + logging.Logf(ctx, "Trident update (finalize) SSH command: %s", finalizeCommand) + finalizeResult, finalizeErr := runSSHCommand(ctx, s.Runtime.VM.SSHClient, finalizeCommand, false) + if err := checkTridentUpdateResult(ctx, "finalize", finalizeResult, finalizeErr); err != nil { + return err + } + + if err := RebootVMAndWaitForSSH(ctx, s); err != nil { + return fmt.Errorf("reboot VM after COSI update: %w", err) + } + + afterBootIDRaw, err := runCOSICommand(ctx, s, "cat /proc/sys/kernel/random/boot_id") + if err != nil { + return fmt.Errorf("get boot ID after COSI update: %w", err) + } + afterBootID := strings.TrimSpace(afterBootIDRaw) + if afterBootID == "" { + return fmt.Errorf("boot ID after COSI update is empty") + } + if beforeBootID == afterBootID { + return fmt.Errorf("boot ID must change after the COSI update reboot") + } + + if err := requireTridentStatus(ctx, s, "ab-update-finalized", ""); err != nil { + return err + } + + if _, err := execScriptOnVMForScenarioValidateExitCode(ctx, s, "sudo trident grpc-client commit -v trace", 0, "failed to commit the COSI update"); err != nil { + return err + } + if err := requireTridentStatus(ctx, s, "provisioned", "volume-b"); err != nil { + return err + } + if err := waitForSameNodeReadyAfterCOSIUpdate(ctx, s, beforeNode, afterBootID); err != nil { + return err + } + postUpdatePod := podHTTPServerLinux(s) + postUpdatePod.Name += "-cosi-post-update" + if err := ValidatePodRunning(ctx, s, postUpdatePod); err != nil { + return err + } + return nil +} + +func stageCOSIUpdateImage(ctx context.Context, sourceURL string, upload func(context.Context, string, *os.File) (string, error)) (string, string, error) { + dir, cleanup, err := cosiDownloadDir() + if err != nil { + return "", "", err + } + defer cleanup() + localPath := filepath.Join(dir, "update.cosi") + if err := downloadCOSIFileWithRetry(ctx, sourceURL, localPath); err != nil { + return "", "", err + } + file, err := os.Open(localPath) + if err != nil { + return "", "", err + } + defer file.Close() + // The test storage endpoint is already allowed by the shared firewall. + blobName := "cosi-updates/" + filepath.Base(dir) + ".cosi" + signedURL, err := upload(ctx, blobName, file) + return blobName, signedURL, err +} + +func validateCOSIUpdateInput(rawURL, metadataSHA384 string) error { + parsedURL, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil { + return fmt.Errorf("parse COSI update URL: %w", err) + } + if !strings.EqualFold(parsedURL.Scheme, "https") || parsedURL.Host == "" { + return fmt.Errorf("COSI update URL must be an absolute HTTPS URL") + } + + metadataHash, err := base64.StdEncoding.DecodeString(strings.TrimSpace(metadataSHA384)) + if err != nil { + return fmt.Errorf("decode COSI metadata SHA-384: %w", err) + } + if len(metadataHash) != sha512.Size384 { + return fmt.Errorf("COSI metadata SHA-384 must decode to %d bytes, got %d", sha512.Size384, len(metadataHash)) + } + return nil +} + +func runCOSICommand(ctx context.Context, s *Scenario, command string) (string, error) { + result, err := runSSHCommand(ctx, s.Runtime.VM.SSHClient, command, false) + if err != nil { + return "", err + } + if result.exitCode != "0" { + return "", fmt.Errorf("command failed with exit code %s: %s", result.exitCode, sanitizeURL(result.stderr)) + } + return result.stdout, nil +} + +// logACLAgentDiagnostics runs a fixed set of read-only diagnostic commands -- +// systemd status, the unit's full journal history (not just the short tail +// systemctl status shows), and relevant config files -- and logs their +// output, ignoring any errors or non-zero exit codes. It never fails the +// test; it exists purely to help root-cause why trident-acl-agent enters a +// failed state after a COSI update. +func logACLAgentDiagnostics(ctx context.Context, s *Scenario, label string) { + commands := []struct { + desc string + cmd string + }{ + {"systemctl status trident-acl-agent.path/.service", "sudo systemctl status trident-acl-agent.path trident-acl-agent.service"}, + {"journalctl -u trident-acl-agent.path/.service (all available logs)", "sudo journalctl -u trident-acl-agent.path -u trident-acl-agent.service --no-pager"}, + {"kubelet kubeconfig present", "sudo test -s /var/lib/kubelet/kubeconfig"}, + {"/opt/azure/containers/image-version", "sudo cat /opt/azure/containers/image-version"}, + } + for _, c := range commands { + result, err := runSSHCommand(ctx, s.Runtime.VM.SSHClient, c.cmd, false) + if err != nil { + logging.Logf(ctx, "[acl-agent-diagnostics %s] %s: SSH error: %v", label, c.desc, err) + continue + } + logging.Logf(ctx, "[acl-agent-diagnostics %s] %s: exit=%s\nstdout:\n%s\nstderr:\n%s", label, c.desc, result.exitCode, sanitizeURL(result.stdout), sanitizeURL(result.stderr)) + } +} + +func checkTridentUpdateResult(ctx context.Context, step string, result *podExecResult, err error) error { + if err != nil { + return fmt.Errorf("Trident update (%s) SSH command error: %s", step, sanitizeURL(err.Error())) + } + if result == nil { + return fmt.Errorf("Trident update (%s) returned no result", step) + } + logging.Logf(ctx, "Trident update (%s) exited with code %s", step, result.exitCode) + logging.Logf(ctx, "Trident update (%s) stdout: %s", step, sanitizeURL(result.stdout)) + logging.Logf(ctx, "Trident update (%s) stderr: %s", step, sanitizeURL(result.stderr)) + if result.exitCode != "0" { + return fmt.Errorf("Trident update (%s) exited with code %s", step, result.exitCode) + } + return nil +} + +func requireTridentStatus(ctx context.Context, s *Scenario, servicingState, activeVolume string) error { + status, err := runCOSICommand(ctx, s, "sudo trident get status") + if err != nil { + return fmt.Errorf("get trident status: %w", err) + } + if !strings.Contains(status, "servicingState: "+servicingState) { + return fmt.Errorf("expected trident status to contain servicingState: %s, got: %s", servicingState, sanitizeURL(status)) + } + if activeVolume != "" && !strings.Contains(status, "abActiveVolume: "+activeVolume) { + return fmt.Errorf("expected trident status to contain abActiveVolume: %s, got: %s", activeVolume, sanitizeURL(status)) + } + return nil +} + +func waitForSameNodeReadyAfterCOSIUpdate(ctx context.Context, s *Scenario, beforeNode *corev1.Node, hostBootID string) error { + err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 10*time.Minute, true, func(pollCtx context.Context) (bool, error) { + node, err := s.Runtime.Kube.Typed.CoreV1().Nodes().Get(pollCtx, beforeNode.Name, metav1.GetOptions{}) + if err != nil { + logging.Logf(pollCtx, "waiting for node %s after COSI update: %v", beforeNode.Name, err) + return false, nil + } + if node.UID != beforeNode.UID { + return false, fmt.Errorf("node %s was replaced during the COSI update", beforeNode.Name) + } + if !strings.EqualFold(node.Status.NodeInfo.BootID, hostBootID) { + return false, nil + } + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { + return true, nil + } + } + return false, nil + }) + if err != nil { + return fmt.Errorf("same Kubernetes node did not return Ready after the COSI update: %w", err) + } + return nil +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" +} diff --git a/e2e/scenario/scenario_credential_provider_test.go b/e2e/scenario/scenario_credential_provider_test.go new file mode 100644 index 00000000000..7626c332a47 --- /dev/null +++ b/e2e/scenario/scenario_credential_provider_test.go @@ -0,0 +1,342 @@ +package scenario + +import ( + "context" + "errors" + "fmt" + "path" + "reflect" + "strings" + "testing" + + "github.com/Azure/agentbaker/e2e/components" + "github.com/Azure/agentbaker/e2e/config" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v8" + "github.com/stretchr/testify/require" +) + +func credentialProviderTestNBC() *datamodel.NodeBootstrappingConfiguration { + return &datamodel.NodeBootstrappingConfiguration{ + ContainerService: &datamodel.ContainerService{ + Properties: &datamodel.Properties{ + OrchestratorProfile: &datamodel.OrchestratorProfile{}, + WindowsProfile: &datamodel.WindowsProfile{CseScriptsPackageURL: "published-cse"}, + }, + }, + K8sComponents: &datamodel.K8sComponents{}, + KubeletConfig: map[string]string{}, + } +} + +func TestCredentialProviderScenariosRegisteredOnce(t *testing.T) { + for _, tc := range []struct { + name string + vhd *config.Image + cluster func(context.Context, ClusterRequest) (*Cluster, error) + minor uint64 + }{ + {"Ubuntu2204_Dalec_CredentialProvider_Kubernetes133", config.VHDUbuntu2204Gen2Containerd, ClusterKubenet, 33}, + {"Windows2022_Dalec_CredentialProvider", config.VHDWindows2022ContainerdGen2, ClusterAzureNetwork, 33}, + {"Windows2022Gen1_Dalec_CredentialProvider", config.VHDWindows2022Containerd, ClusterAzureNetwork, 33}, + {"Windows2022_Legacy_CredentialProvider_Kubernetes132", config.VHDWindows2022ContainerdGen2, ClusterAzureNetwork, 32}, + {"Windows2022Gen1_Legacy_CredentialProvider_Kubernetes132", config.VHDWindows2022Containerd, ClusterAzureNetwork, 32}, + } { + t.Run(tc.name, func(t *testing.T) { + var matches []*Scenario + for _, s := range List() { + if s.Name == tc.name { + matches = append(matches, s) + } + } + require.Len(t, matches, 1) + s := matches[0] + require.Equal(t, tc.vhd, s.VHD) + require.Equal(t, reflect.ValueOf(tc.cluster).Pointer(), reflect.ValueOf(s.Cluster).Pointer()) + require.Nil(t, s.Runtime, "listing must not prepare a scenario") + require.Empty(t, s.SkipReason) + require.Nil(t, s.BootstrapConfigMutator) + require.NotNil(t, s.BootstrapConfigMutatorWithError) + require.NotNil(t, s.Validator) + require.False(t, s.SkipDefaultValidation, "node readiness and workload validation must run") + require.NotNil(t, s.VMConfigMutator) + vmss := &armcompute.VirtualMachineScaleSet{Tags: map[string]*string{}} + s.VMConfigMutator(vmss) + require.Empty(t, vmss.Tags, "the natural gate must not set ShouldEnforceKubePMCInstall") + + if tc.name == "Ubuntu2204_Dalec_CredentialProvider_Kubernetes133" { + nbc := credentialProviderTestNBC() + require.NoError(t, s.BootstrapConfigMutatorWithError(t.Context(), nil, nbc)) + version := nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion + require.True(t, strings.HasPrefix(version, "1.33.")) + require.Contains(t, components.GetExpectedPackageVersions("kubernetes-binaries", "default", "current"), "v"+version) + require.Equal(t, "/var/lib/kubelet/credential-provider-config.yaml", nbc.KubeletConfig["--image-credential-provider-config"]) + require.Equal(t, "/var/lib/kubelet/credential-provider", nbc.KubeletConfig["--image-credential-provider-bin-dir"]) + require.Equal(t, "published-cse", nbc.ContainerService.Properties.WindowsProfile.CseScriptsPackageURL) + } else { + require.NotNil(t, s.SkipIf) + nbc := credentialProviderTestNBC() + require.NoError(t, s.BootstrapConfigMutatorWithError(t.Context(), nil, nbc)) + expected := credentialProviderTestNBC() + require.NoError(t, configureWindowsCredentialProvider(expected, + components.GetExpectedPackageVersions("kubernetes-binaries", "windows", "default"), + components.GetExpectedPackageVersions("azure-acr-credential-provider", "windows", "default"), tc.minor)) + require.Equal(t, expected, nbc) + require.True(t, strings.HasPrefix(nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion, fmt.Sprintf("1.%d.", tc.minor))) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.ErrorIs(t, s.BootstrapConfigMutatorWithError(ctx, nil, credentialProviderTestNBC()), context.Canceled) + } + }) + } + for _, s := range List() { + require.NotEqual(t, "Ubuntu2204_PMC_CredentialProvider_Kubernetes133", s.Name, "the old dedicated PMC name must not remain registered") + } +} + +func TestCredentialProviderWindowsEligibility(t *testing.T) { + savedConfig := *config.Config + t.Cleanup(func() { *config.Config = savedConfig }) + + for _, minor := range []uint64{32, 33} { + for _, vhd := range []*config.Image{config.VHDWindows2022ContainerdGen2, config.VHDWindows2022Containerd} { + t.Run(fmt.Sprintf("1.%d/%s", minor, vhd.Name), func(t *testing.T) { + // Construct before configuring the lane: registration must not latch eligibility. + image := *vhd + s := newWindowsCredentialProviderScenario("CredentialProvider", &image, minor) + require.Empty(t, s.SkipReason) + require.Nil(t, s.Runtime) + for _, tc := range []struct { + name string + currentBuild string + tagName string + tagValue string + version string + metadataFile string + wantSkip string + }{ + {name: "standalone main", currentBuild: "123", tagName: "branch", tagValue: "refs/heads/main", wantSkip: "require buildId=BUILD_BUILDID"}, + {name: "older build", currentBuild: "123", tagName: "buildId", tagValue: "122", wantSkip: "require buildId=BUILD_BUILDID"}, + {name: "local explicit build", tagName: "buildId", tagValue: "123", wantSkip: "require buildId=BUILD_BUILDID"}, + {name: "empty build IDs", tagName: "buildId", wantSkip: "require buildId=BUILD_BUILDID"}, + {name: "empty selector", currentBuild: "123", tagName: "buildId", wantSkip: "require buildId=BUILD_BUILDID"}, + {name: "wrong tag", currentBuild: "123", tagName: "branch", tagValue: "123", wantSkip: "require buildId=BUILD_BUILDID"}, + {name: "pinned image overrides current build", currentBuild: "123", tagName: "buildId", tagValue: "123", version: "202601.01.0", wantSkip: "pinned versions and VHD metadata overrides"}, + {name: "metadata overrides current build", currentBuild: "123", tagName: "buildId", tagValue: "123", metadataFile: "images.json", wantSkip: "pinned versions and VHD metadata overrides"}, + {name: "current build", currentBuild: "123", tagName: "buildId", tagValue: "123"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("BUILD_BUILDID", tc.currentBuild) + // Neither raw VHD_BUILD_ID nor the runner's BUILD_ID proves eligibility. + t.Setenv("VHD_BUILD_ID", "123") + config.Config.BuildID = "123" + config.Config.SIGVersionTagName = tc.tagName + config.Config.SIGVersionTagValue = tc.tagValue + config.Config.VHDMetadataFile = tc.metadataFile + image.Version = tc.version + + reason := s.SkipIf(t.Context()) + if tc.wantSkip != "" { + require.Contains(t, reason, tc.wantSkip) + outcome := Run(t.Context(), s.Name, s.Name, s) + require.NoError(t, outcome.Error) + require.Equal(t, reason, outcome.SkipReason) + } else { + require.Empty(t, reason) + for _, runErr := range []error{nil, errors.New("credential-provider validation failed")} { + ran := false + outcome := runExecution(t.Context(), s.Name, s.Name, s, + func(ctx context.Context, _ string, prepared *Scenario) error { + ran = true + require.NotSame(t, s, prepared) + require.Equal(t, &image, prepared.VHD) + nbc := credentialProviderTestNBC() + require.NoError(t, prepared.BootstrapConfigMutatorWithError(ctx, nil, nbc)) + require.Equal(t, "published-cse", nbc.ContainerService.Properties.WindowsProfile.CseScriptsPackageURL) + return runErr + }) + require.True(t, ran) + require.Empty(t, outcome.SkipReason) + require.ErrorIs(t, outcome.Error, runErr) + } + } + require.Empty(t, s.SkipReason) + require.Nil(t, s.Runtime) + }) + } + }) + } + } +} + +func TestCredentialProviderVersion(t *testing.T) { + for _, minor := range []uint64{32, 33} { + prefix := fmt.Sprintf("1.%d", minor) + otherMinor := uint64(32) + if minor == 32 { + otherMinor = 33 + } + for _, tc := range []struct { + name string + versions []string + want string + }{ + {"numeric patch order", []string{"v" + prefix + ".9", "v" + prefix + ".13", "v" + prefix + ".12"}, prefix + ".13"}, + {"legacy versions without v", []string{"1.31.8", prefix + ".3", "1.34.1"}, prefix + ".3"}, + {"absent", nil, ""}, + {"other minors cannot replace gate", []string{"v1.31.11", "v1.34.11", "v" + prefix + "0.1", fmt.Sprintf("v2.%d.1", minor)}, ""}, + {"opposite side of gate cannot replace minor", []string{fmt.Sprintf("v1.%d.12", otherMinor)}, ""}, + {"invalid incomplete and prerelease", []string{"invalid", "v" + prefix, "v" + prefix + ".0-rc.1", "v" + prefix + ".01", "vv" + prefix + ".2"}, ""}, + {"higher prerelease excluded", []string{prefix + ".9", "v" + prefix + ".14-rc.1", prefix + ".13-beta.1"}, prefix + ".9"}, + } { + t.Run(prefix+"/"+tc.name, func(t *testing.T) { + got, err := credentialProviderVersion(tc.versions, "test/windows/default", minor) + if tc.want == "" { + require.ErrorContains(t, err, "cached "+prefix+" patch") + require.ErrorContains(t, err, "components.json test/windows/default") + return + } + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } + } +} + +func TestCredentialProviderScenarioMetadataSetup(t *testing.T) { + t.Run("Ubuntu Dalec selects cached patch", func(t *testing.T) { + nbc := credentialProviderTestNBC() + require.NoError(t, configureUbuntuDalecCredentialProvider(nbc, []string{"v1.33.9", "v1.33.12"})) + require.Equal(t, "1.33.12", nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion) + require.Empty(t, nbc.K8sComponents.WindowsPackageURL) + require.Empty(t, nbc.K8sComponents.WindowsCredentialProviderURL) + }) + t.Run("Ubuntu Dalec missing minor fails", func(t *testing.T) { + require.ErrorContains(t, configureUbuntuDalecCredentialProvider(credentialProviderTestNBC(), []string{"v1.34.1"}), "kubernetes-binaries/default/current") + }) + for _, minor := range []uint64{32, 33} { + t.Run(fmt.Sprintf("Windows 1.%d", minor), func(t *testing.T) { + t.Run("Windows independently selects kubelet and legacy provider", func(t *testing.T) { + nbc := credentialProviderTestNBC() + kubeletVersion := fmt.Sprintf("1.%d.12", minor) + providerVersion := fmt.Sprintf("1.%d.3", minor) + require.NoError(t, configureWindowsCredentialProvider(nbc, []string{"v" + kubeletVersion}, []string{providerVersion}, minor)) + require.Equal(t, kubeletVersion, nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion) + require.Equal(t, fmt.Sprintf("https://packages.aks.azure.com/kubernetes/v%s/windowszip/v%s-1int.zip", kubeletVersion, kubeletVersion), nbc.K8sComponents.WindowsPackageURL) + legacyURL, err := windowsLegacyCredentialProviderURL(providerVersion) + require.NoError(t, err) + require.Equal(t, legacyURL, nbc.K8sComponents.WindowsCredentialProviderURL) + require.Contains(t, nbc.K8sComponents.WindowsCredentialProviderURL, providerVersion) + require.NotContains(t, nbc.K8sComponents.WindowsCredentialProviderURL, kubeletVersion) + require.Equal(t, `C:\k\credential-provider-config.yaml`, nbc.KubeletConfig["--image-credential-provider-config"]) + require.Equal(t, `C:\var\lib\kubelet\credential-provider`, nbc.KubeletConfig["--image-credential-provider-bin-dir"]) + require.Equal(t, "published-cse", nbc.ContainerService.Properties.WindowsProfile.CseScriptsPackageURL) + }) + t.Run("Windows missing kubelet never falls back to Linux metadata", func(t *testing.T) { + nbc := credentialProviderTestNBC() + require.ErrorContains(t, configureWindowsCredentialProvider(nbc, []string{"v1.34.1"}, []string{fmt.Sprintf("1.%d.3", minor)}, minor), "kubernetes-binaries/windows/default") + require.Equal(t, credentialProviderTestNBC(), nbc) + }) + t.Run("Windows missing legacy provider never infers kubelet release", func(t *testing.T) { + nbc := credentialProviderTestNBC() + require.ErrorContains(t, configureWindowsCredentialProvider(nbc, []string{fmt.Sprintf("v1.%d.12", minor)}, []string{"1.31.8"}, minor), "azure-acr-credential-provider/windows/default") + require.Equal(t, credentialProviderTestNBC(), nbc) + }) + }) + } +} + +func TestCredentialProviderWindowsPreservesCSEURL(t *testing.T) { + t.Chdir(t.TempDir()) + for _, minor := range []uint64{32, 33} { + t.Run(fmt.Sprintf("1.%d", minor), func(t *testing.T) { + for _, url := range []string{"", "https://packages.aks.azure.com/aks/windows/cse/", "https://example.invalid/custom-cse.zip"} { + t.Run(url, func(t *testing.T) { + nbc := credentialProviderTestNBC() + nbc.ContainerService.Properties.WindowsProfile.CseScriptsPackageURL = url + require.NoError(t, windowsCredentialProviderMutator(t.Context(), nil, nbc, minor)) + require.Equal(t, url, nbc.ContainerService.Properties.WindowsProfile.CseScriptsPackageURL) + }) + } + + nbc := credentialProviderTestNBC() + require.NoError(t, windowsCredentialProviderMutator(t.Context(), nil, nbc, minor)) + + version := nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion + require.Contains(t, components.GetExpectedPackageVersions("kubernetes-binaries", "windows", "default"), "v"+version) + require.Equal(t, fmt.Sprintf("https://packages.aks.azure.com/kubernetes/v%s/windowszip/v%s-1int.zip", version, version), nbc.K8sComponents.WindowsPackageURL) + legacyVersion, err := credentialProviderVersion(components.GetExpectedPackageVersions("azure-acr-credential-provider", "windows", "default"), "azure-acr-credential-provider/windows/default", minor) + require.NoError(t, err) + legacyURL, err := windowsLegacyCredentialProviderURL(legacyVersion) + require.NoError(t, err) + require.Equal(t, legacyURL, nbc.K8sComponents.WindowsCredentialProviderURL) + }) + } +} + +func TestCredentialProviderWindowsCancellation(t *testing.T) { + for _, minor := range []uint64{32, 33} { + nbc := credentialProviderTestNBC() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.ErrorIs(t, windowsCredentialProviderMutator(ctx, nil, nbc, minor), context.Canceled) + require.Equal(t, credentialProviderTestNBC(), nbc) + } +} + +func TestCredentialProviderWindowsLegacyURLMetadata(t *testing.T) { + metadata := []byte(`{"Packages":[{"name":"azure-acr-credential-provider","downloadURIs":{"windows":{"default":{"downloadURL":"https://metadata.example.invalid/releases/${version}/provider-${version}.tar.gz"}}}}]}`) + got, err := resolveWindowsLegacyCredentialProviderURL(metadata, "1.32.8") + require.NoError(t, err) + require.Equal(t, "https://metadata.example.invalid/releases/1.32.8/provider-1.32.8.tar.gz", got) + + for _, tc := range []struct { + name string + data string + }{ + {"invalid JSON", `{`}, + {"missing package", `{"Packages":[]}`}, + {"missing windows", `{"Packages":[{"name":"azure-acr-credential-provider","downloadURIs":{"default":{}}}]}`}, + {"missing URL", `{"Packages":[{"name":"azure-acr-credential-provider","downloadURIs":{"windows":{"default":{}}}}]}`}, + {"unversioned URL", strings.ReplaceAll(string(metadata), "${version}", "fixed")}, + {"unresolved variable", strings.ReplaceAll(string(metadata), "provider-", "${unknown}-")}, + {"invalid URL", strings.ReplaceAll(string(metadata), "https://metadata.example.invalid", "://broken")}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := resolveWindowsLegacyCredentialProviderURL([]byte(tc.data), "1.32.8") + require.ErrorContains(t, err, "components.json azure-acr-credential-provider/windows/default/downloadURL") + }) + } +} + +func TestCredentialProviderWindowsLegacyLog(t *testing.T) { + version, err := credentialProviderVersion(components.GetExpectedPackageVersions("azure-acr-credential-provider", "windows", "default"), "azure-acr-credential-provider/windows/default", 32) + require.NoError(t, err) + legacyURL, err := windowsLegacyCredentialProviderURL(version) + require.NoError(t, err) + filename := path.Base(legacyURL) + for _, tc := range []struct { + name string + log string + wantError bool + }{ + {"cache copy", fmt.Sprintf(`Using cached version of %s - Copying file from C:\akse-cache\credential-provider\%s to C:\tmp\credentialprovider.tar.gz`, filename, filename), false}, + {"completed download", fmt.Sprintf(`Downloaded file %s to C:\tmp\credentialprovider.tar.gz in 12 ms`, legacyURL), false}, + {"remapped download", fmt.Sprintf(`Downloaded file https://packages.aks.azure.us/binaries/%s to C:\tmp\credentialprovider.tar.gz in 12 ms`, filename), false}, + {"generic pre-selection", fmt.Sprintf(`Download credential provider binary from %s to C:\var\lib\kubelet\credential-provider`, legacyURL), true}, + {"attempted download", fmt.Sprintf(`Updated URL %s -> %s to download %s to C:\tmp\credentialprovider.tar.gz`, legacyURL, legacyURL, filename), true}, + {"different provider version", `Using cached version of other-provider.tar.gz - Copying file from C:\cache\other-provider.tar.gz to C:\tmp\credentialprovider.tar.gz`, true}, + {"different destination", fmt.Sprintf(`Downloaded file %s to C:\tmp\unrelated.tar.gz in 12 ms`, legacyURL), true}, + {"unrelated lines", fmt.Sprintf("Using cached version of %s - Copying file from C:\\cache\\%s\nunrelated to C:\\tmp\\credentialprovider.tar.gz", filename, filename), true}, + {"missing log", "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + err := validateWindowsLegacyCredentialProviderLog(tc.log, legacyURL) + if tc.wantError { + require.ErrorContains(t, err, "expected legacy credential-provider") + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/e2e/scenario/scenario_localdns_hosts.go b/e2e/scenario/scenario_localdns_hosts.go index e83b5df8336..75776b78d73 100644 --- a/e2e/scenario/scenario_localdns_hosts.go +++ b/e2e/scenario/scenario_localdns_hosts.go @@ -22,7 +22,7 @@ func init() { {name: "Ubuntu2604Minimal", vhd: config.VHDUbuntu2604MinimalGen2Containerd}, {name: "AzureLinuxV3", vhd: config.VHDAzureLinuxV3Gen2}, {name: "ACL", vhd: config.VHDACLGen2TL, vmConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) }}, } diff --git a/e2e/scenario/scenario_rcv1p.go b/e2e/scenario/scenario_rcv1p.go index 7b68fefabb8..6b08668cf08 100644 --- a/e2e/scenario/scenario_rcv1p.go +++ b/e2e/scenario/scenario_rcv1p.go @@ -348,7 +348,7 @@ var _ = Register(&Scenario{ Cluster: ClusterKubenet, VHD: config.VHDACLGen2TL, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + vmss.Properties = aclVMSSSecurityProfile(vmss.Properties, config.Config.ACLBaseImageSigned) rcv1pOptInVMConfigMutator(vmss) }, Validator: func(ctx context.Context, s *Scenario) error { diff --git a/e2e/scenario/scenario_win.go b/e2e/scenario/scenario_win.go index 31fc797d187..4dbfc1e5410 100644 --- a/e2e/scenario/scenario_win.go +++ b/e2e/scenario/scenario_win.go @@ -4,6 +4,12 @@ import ( "context" "errors" "fmt" + "net/url" + "os" + "path" + "path/filepath" + "runtime" + "strings" "time" @@ -11,6 +17,7 @@ import ( "github.com/Azure/agentbaker/e2e/components" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Masterminds/semver/v3" + "github.com/tidwall/gjson" "github.com/Azure/agentbaker/e2e/config" "github.com/Azure/agentbaker/pkg/agent/datamodel" @@ -140,6 +147,33 @@ var _ = Register(&Scenario{ }, }) +var _ = Register(&Scenario{ + Name: "Windows2022Gen2_TrustedLaunch_SharedImage", + Description: "Windows Server 2022 Gen2 Trusted Launch (Secure Boot + vTPM)", + Config: Config{ + Cluster: ClusterAzureNetwork, + VHD: config.VHDWindows2022ContainerdGen2, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + }, + BootstrapConfigMutator: EmptyBootstrapConfigMutator, + Validator: func(ctx context.Context, s *Scenario) error { + return errors.Join( + ValidateWindowsVersionFromWindowsSettings(ctx, s, "2022-containerd-gen2"), + ValidateWindowsProductName(ctx, s, "Windows Server 2022 Datacenter"), + ValidateWindowsDisplayVersion(ctx, s, "21H2"), + ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote"), + ValidateKubeletArgs(ctx, s), + ValidateContainerdWindowsPriorityClass(ctx, s), + ValidateCiliumIsNotRunningWindows(ctx, s), + ValidateDotnetNotInstalledWindows(ctx, s), + ValidateWindowsSystemServicesRestartConfiguration(ctx, s), + ValidateCollectWindowsLogsScript(ctx, s), + ) + }, + }, +}) + var _ = Register(&Scenario{ Name: "Windows2022Gen2AzureOverlayNetworkDualStack", SkipReason: "Dual stack tests are not working yet", @@ -220,34 +254,46 @@ var _ = Register(&Scenario{ }, }) -var _ = Register(&Scenario{ - Name: "Windows2025Gen2TrustedLaunch", - Description: "Windows Server 2025 Gen2 Trusted Launch (Secure Boot + vTPM)", - Config: Config{ - Cluster: ClusterAzureNetwork, - VHD: config.VHDWindows2025Gen2TL, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) - }, - BootstrapConfigMutatorWithError: func(_ context.Context, _ *Cluster, configuration *datamodel.NodeBootstrappingConfiguration) error { - return Windows2025BootstrapConfigMutator(configuration) - }, - Validator: func(ctx context.Context, s *Scenario) error { - return errors.Join( - ValidateWindowsVersionFromWindowsSettings(ctx, s, "2025-gen2-tl"), - ValidateWindowsProductName(ctx, s, "Windows Server 2025 Datacenter"), - ValidateWindowsDisplayVersion(ctx, s, "24H2"), - ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote"), - ValidateKubeletArgs(ctx, s), - ValidateContainerdWindowsPriorityClass(ctx, s), - ValidateCiliumIsNotRunningWindows(ctx, s), - ValidateDotnetNotInstalledWindows(ctx, s), - ValidateWindowsSystemServicesRestartConfiguration(ctx, s), - ValidateCollectWindowsLogsScript(ctx, s), - ) - }, - }, -}) +//nolint:gochecknoinits // Register both image variants before the runner selects scenarios. +func init() { + for _, tc := range []struct { + name string + vhd *config.Image + windowsSKU string + }{ + {"Windows2025Gen2TrustedLaunch", config.VHDWindows2025Gen2TL, "2025-gen2-tl"}, + {"Windows2025Gen2_TrustedLaunch_SharedImage", config.VHDWindows2025Gen2, "2025-gen2"}, + } { + Register(&Scenario{ + Name: tc.name, + Description: "Windows Server 2025 Gen2 Trusted Launch (Secure Boot + vTPM)", + Config: Config{ + Cluster: ClusterAzureNetwork, + VHD: tc.vhd, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + }, + BootstrapConfigMutatorWithError: func(_ context.Context, _ *Cluster, configuration *datamodel.NodeBootstrappingConfiguration) error { + return Windows2025BootstrapConfigMutator(configuration) + }, + Validator: func(ctx context.Context, s *Scenario) error { + return errors.Join( + ValidateWindowsVersionFromWindowsSettings(ctx, s, tc.windowsSKU), + ValidateWindowsProductName(ctx, s, "Windows Server 2025 Datacenter"), + ValidateWindowsDisplayVersion(ctx, s, "24H2"), + ValidateFileHasContent(ctx, s, "/k/kubeletstart.ps1", "--container-runtime=remote"), + ValidateKubeletArgs(ctx, s), + ValidateContainerdWindowsPriorityClass(ctx, s), + ValidateCiliumIsNotRunningWindows(ctx, s), + ValidateDotnetNotInstalledWindows(ctx, s), + ValidateWindowsSystemServicesRestartConfiguration(ctx, s), + ValidateCollectWindowsLogsScript(ctx, s), + ) + }, + }, + }) + } +} var _ = Register(&Scenario{ Name: "Windows2025Gen2_WindowsCiliumNetworking", @@ -615,3 +661,176 @@ var _ = Register(&Scenario{ }, }, }) + +// The Kubernetes minors on either side of the gate require separate nodes for each generation. +// Only this pipeline run's VHDs establish that branch CSE and components were baked in. +var _ = Register(newWindowsCredentialProviderScenario("Windows2022_Dalec_CredentialProvider", config.VHDWindows2022ContainerdGen2, 33)) +var _ = Register(newWindowsCredentialProviderScenario("Windows2022Gen1_Dalec_CredentialProvider", config.VHDWindows2022Containerd, 33)) +var _ = Register(newWindowsCredentialProviderScenario("Windows2022_Legacy_CredentialProvider_Kubernetes132", config.VHDWindows2022ContainerdGen2, 32)) +var _ = Register(newWindowsCredentialProviderScenario("Windows2022Gen1_Legacy_CredentialProvider_Kubernetes132", config.VHDWindows2022Containerd, 32)) + +func newWindowsCredentialProviderScenario(name string, vhd *config.Image, minor uint64) *Scenario { + description := "Tests Windows 2022 VHD-embedded CSE selects the Dalec credential provider at Kubernetes 1.33 with a stock RP legacy URL, without falling back" + if minor == 32 { + description = "Tests Windows 2022 VHD-embedded CSE installs the legacy credential provider at Kubernetes 1.32 below the Dalec gate" + } + return &Scenario{ + Name: name, + Description: description, + SkipIf: func(context.Context) string { + return skipWindowsCredentialProvider(vhd) + }, + Config: Config{ + Cluster: ClusterAzureNetwork, + VHD: vhd, + VMConfigMutator: EmptyVMConfigMutator, + BootstrapConfigMutatorWithError: func(ctx context.Context, cluster *Cluster, nbc *datamodel.NodeBootstrappingConfiguration) error { + return windowsCredentialProviderMutator(ctx, cluster, nbc, minor) + }, + Validator: func(ctx context.Context, s *Scenario) error { + if minor == 32 { + return errors.Join( + ValidateFileExists(ctx, s, `C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe`), + ValidateFileExists(ctx, s, `C:\k\credential-provider-config.yaml`), + ValidateFileExcludesContent(ctx, s, `C:\AzureData\CustomDataSetupScript.log`, "Using dalec credential provider"), + ValidateFileExcludesContent(ctx, s, `C:\AzureData\CustomDataSetupScript.log`, "Failed to install dalec credential provider, falling back to RP URL."), + validateWindowsLegacyCredentialProvider(ctx, s), + ) + } + return errors.Join( + ValidateFileExists(ctx, s, `C:\var\lib\kubelet\credential-provider\acr-credential-provider.exe`), + ValidateFileExists(ctx, s, `C:\k\credential-provider-config.yaml`), + ValidateFileHasContent(ctx, s, `C:\AzureData\CustomDataSetupScript.log`, "Using dalec credential provider"), + ValidateFileExcludesContent(ctx, s, `C:\AzureData\CustomDataSetupScript.log`, "Failed to install dalec credential provider, falling back to RP URL."), + ) + }, + }, + } +} + +func skipWindowsCredentialProvider(vhd *config.Image) string { + const reason = "Windows credential-provider scenarios require a VHD built with branch CSE in the current pipeline run" + // Explicit versions and metadata take precedence over the gallery tag selector. + if vhd.Version != "" || config.Config.VHDMetadataFile != "" { + return reason + "; pinned versions and VHD metadata overrides do not establish that provenance" + } + // e2e_run.sh maps VHD_BUILD_ID to the selector, but it may name an older build. + // BUILD_BUILDID is Azure DevOps' current run ID, shared by CSE packaging and VHD build. + buildID := os.Getenv("BUILD_BUILDID") + if buildID == "" || config.Config.SIGVersionTagName != "buildId" || config.Config.SIGVersionTagValue != buildID { + return fmt.Sprintf("%s; require buildId=BUILD_BUILDID (current run %q), got %s=%q", + reason, buildID, config.Config.SIGVersionTagName, config.Config.SIGVersionTagValue) + } + return "" +} + +func windowsCredentialProviderMutator(ctx context.Context, _ *Cluster, nbc *datamodel.NodeBootstrappingConfiguration, minor uint64) error { + if err := ctx.Err(); err != nil { + return err + } + return configureWindowsCredentialProvider(nbc, + components.GetExpectedPackageVersions("kubernetes-binaries", "windows", "default"), + components.GetExpectedPackageVersions("azure-acr-credential-provider", "windows", "default"), minor) +} + +func configureWindowsCredentialProvider(nbc *datamodel.NodeBootstrappingConfiguration, kubeletVersions, legacyProviderVersions []string, minor uint64) error { + version, err := credentialProviderVersion(kubeletVersions, "kubernetes-binaries/windows/default", minor) + if err != nil { + return err + } + // Legacy provider releases differ from Kubernetes patches; use their own + // cached metadata to keep the populated RP fallback URL valid. + legacyVersion, err := credentialProviderVersion(legacyProviderVersions, "azure-acr-credential-provider/windows/default", minor) + if err != nil { + return err + } + legacyURL, err := windowsLegacyCredentialProviderURL(legacyVersion) + if err != nil { + return err + } + nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion = version + nbc.K8sComponents.WindowsPackageURL = fmt.Sprintf("https://packages.aks.azure.com/kubernetes/v%s/windowszip/v%s-1int.zip", version, version) + nbc.K8sComponents.WindowsCredentialProviderURL = legacyURL + nbc.KubeletConfig["--image-credential-provider-config"] = `C:\k\credential-provider-config.yaml` + nbc.KubeletConfig["--image-credential-provider-bin-dir"] = `C:\var\lib\kubelet\credential-provider` + return nil +} + +func credentialProviderVersion(versions []string, metadataPath string, minor uint64) (string, error) { + var latest *semver.Version + for _, raw := range versions { + version, err := semver.StrictNewVersion(components.RemoveLeadingV(raw)) + if err != nil || version.Major() != 1 || version.Minor() != minor || version.Prerelease() != "" { + continue + } + if latest == nil || version.GreaterThan(latest) { + latest = version + } + } + if latest == nil { + return "", fmt.Errorf("credential-provider scenario requires a cached 1.%d patch in components.json %s", minor, metadataPath) + } + return latest.String(), nil +} + +func windowsLegacyCredentialProviderURL(version string) (string, error) { + // Match the version helper's source-relative lookup, independent of the runner's CWD. + _, filename, _, _ := runtime.Caller(0) + metadataPath := filepath.Join(filepath.Dir(filename), "..", "..", "parts", "common", "components.json") + data, err := os.ReadFile(metadataPath) + if err != nil { + return "", fmt.Errorf("read credential-provider metadata: %w", err) + } + return resolveWindowsLegacyCredentialProviderURL(data, version) +} + +func resolveWindowsLegacyCredentialProviderURL(data []byte, version string) (string, error) { + const metadataPath = "components.json azure-acr-credential-provider/windows/default/downloadURL" + if !gjson.ValidBytes(data) { + return "", fmt.Errorf("%s: invalid JSON", metadataPath) + } + template := gjson.GetBytes(data, `Packages.#(name="azure-acr-credential-provider").downloadURIs.windows.default.downloadURL`) + if template.Type != gjson.String || !strings.Contains(template.String(), "${version}") { + return "", fmt.Errorf("%s: requires a downloadURL template containing ${version}", metadataPath) + } + resolved := strings.ReplaceAll(template.String(), "${version}", version) + parsed, err := url.Parse(resolved) + if err != nil || parsed.Scheme != "https" || parsed.Host == "" || strings.ContainsAny(resolved, "$ \t\r\n") { + return "", fmt.Errorf("%s: invalid resolved URL %q", metadataPath, resolved) + } + return resolved, nil +} + +func validateWindowsLegacyCredentialProvider(ctx context.Context, s *Scenario) error { + version, err := credentialProviderVersion(components.GetExpectedPackageVersions("azure-acr-credential-provider", "windows", "default"), "azure-acr-credential-provider/windows/default", 32) + if err != nil { + return err + } + legacyURL, err := windowsLegacyCredentialProviderURL(version) + if err != nil { + return err + } + log, err := getFileContent(ctx, s, `C:\AzureData\CustomDataSetupScript.log`) + if err != nil { + return err + } + return validateWindowsLegacyCredentialProviderLog(log, legacyURL) +} + +func validateWindowsLegacyCredentialProviderLog(log, legacyURL string) error { + parsed, err := url.Parse(legacyURL) + if err != nil { + return fmt.Errorf("parse legacy credential-provider URL: %w", err) + } + filename := path.Base(parsed.Path) + for _, line := range strings.Split(log, "\n") { + // DownloadFileOverHttp emits these only after selecting the legacy path. + // Match the archive name because Update-BaseUrl may remap the download host. + cached := strings.Contains(line, "Using cached version of "+filename+" - Copying file from ") + downloaded := strings.Contains(line, "Downloaded file ") && strings.Contains(line, "/"+filename+" to ") + if (cached || downloaded) && strings.Contains(line, `\credentialprovider.tar.gz`) { + return nil + } + } + return fmt.Errorf("expected legacy credential-provider cache-copy or completed-download log for %s to credentialprovider.tar.gz", filename) +} diff --git a/e2e/scenario/types.go b/e2e/scenario/types.go index 27a2c4456dd..40850c8aed3 100644 --- a/e2e/scenario/types.go +++ b/e2e/scenario/types.go @@ -11,6 +11,7 @@ import ( aksnodeconfigv1 "github.com/Azure/agentbaker/aks-node-controller/pkg/gen/aksnodeconfig/v1" "github.com/Azure/agentbaker/e2e/config" + "github.com/Azure/agentbaker/e2e/logging" "github.com/Azure/agentbaker/pkg/agent/datamodel" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v8" @@ -28,6 +29,7 @@ type Tags struct { WASM bool Kata bool BootstrapTokenFallback bool + COSIUpdate bool KubeletCustomConfig bool Scriptless bool VHDCaching bool @@ -189,6 +191,9 @@ type Config struct { // It shouldn't be used for majority of scenarios, currently only used for scenarios where the node is not expected to be reachable via ssh SkipSSHConnectivityValidation bool + // SkipScriptlessNBCCSECmd prevents RunScenario from automatically repeating the scenario using scriptless NBC provisioning. + SkipScriptlessNBCCSECmd bool + // WaitForSSHAfterReboot if set to non-zero duration, SSH connectivity validation will retry with exponential backoff // for up to this duration when encountering reboot-related errors. This is useful for scenarios where the node // reboots during provisioning (e.g., MIG-enabled GPU nodes). Default (zero value) means no retry. @@ -224,16 +229,10 @@ func (s *Scenario) PrepareVMSSModel(ctx context.Context, vmss *armcompute.Virtua if s.VHD == nil { return fmt.Errorf("scenario VHD is nil") } - resourceID, err := CachedPrepareVHD(ctx, GetVHDRequest{ - Image: *s.VHD, - Location: s.Location, - }) + imageReference, err := resolveImageReference(ctx, s.VHD, s.Location) if err != nil { return fmt.Errorf("prepare VHD: %w", err) } - if resourceID == "" { - return fmt.Errorf("VHD selector returned an empty resource ID") - } if vmss == nil { return fmt.Errorf("input virtual machine scale set is nil") } @@ -256,9 +255,7 @@ func (s *Scenario) PrepareVMSSModel(ctx context.Context, vmss *armcompute.Virtua if vmss.Properties.VirtualMachineProfile.StorageProfile == nil { vmss.Properties.VirtualMachineProfile.StorageProfile = &armcompute.VirtualMachineScaleSetStorageProfile{} } - vmss.Properties.VirtualMachineProfile.StorageProfile.ImageReference = &armcompute.ImageReference{ - ID: to.Ptr(string(resourceID)), - } + vmss.Properties.VirtualMachineProfile.StorageProfile.ImageReference = imageReference // Override OS disk size if the VHD requires a non-default size. if s.VHD.OSDiskSizeGB > 0 { @@ -272,6 +269,22 @@ func (s *Scenario) PrepareVMSSModel(ctx context.Context, vmss *armcompute.Virtua return nil } +func resolveImageReference(ctx context.Context, image *config.Image, location string) (*armcompute.ImageReference, error) { + if image.SharedGalleryImageID != "" { + logging.Logf(ctx, "Using shared gallery image ID: %s", image.SharedGalleryImageID) + return &armcompute.ImageReference{SharedGalleryImageID: to.Ptr(string(image.SharedGalleryImageID))}, nil + } + + resourceID, err := CachedPrepareVHD(ctx, GetVHDRequest{Image: *image, Location: location}) + if err != nil { + return nil, err + } + if resourceID == "" { + return nil, fmt.Errorf("VHD selector returned an empty resource ID") + } + return &armcompute.ImageReference{ID: to.Ptr(string(resourceID))}, nil +} + func (s *Scenario) RemoveBoostrapKubeconfig() bool { if s.Runtime == nil { return false diff --git a/e2e/scenario/validators.go b/e2e/scenario/validators.go index bd318d124a6..c1497e4cb50 100644 --- a/e2e/scenario/validators.go +++ b/e2e/scenario/validators.go @@ -3932,6 +3932,11 @@ func ValidateStaleCachedKubeBinariesRemoved(ctx context.Context, s *Scenario) er // ValidateRxBufferDefault validates rx buffer config using default values based on VM's CPU count func ValidateRxBufferDefault(ctx context.Context, s *Scenario) error { + if s.VHD != nil && s.VHD.SkipOldVHDValidations { + logging.Logf(ctx, "Skipping RX-buffer validation: legacy image %s may not contain the NIC-tuning artifacts", s.VHD.Distro) + return nil + } + // Query the VM's actual CPU count using nproc cpuCountCmd := "nproc" result, err := execScriptOnVMForScenarioValidateExitCode(ctx, s, cpuCountCmd, 0, "could not get CPU count from VM") diff --git a/go.mod b/go.mod index ee334a598b2..a1b1e690f66 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/stretchr/testify v1.12.1 github.com/vincent-petithory/dataurl v1.0.0 gopkg.in/yaml.v3 v3.0.1 + mvdan.cc/sh/v3 v3.14.1 ) require ( @@ -31,12 +32,11 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/kr/text v0.2.0 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/spf13/pflag v1.0.9 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/net v0.55.0 // indirect - golang.org/x/sys v0.45.0 // indirect + golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.37.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect diff --git a/go.sum b/go.sum index f82287fd07f..dbc5b03402c 100644 --- a/go.sum +++ b/go.sum @@ -21,7 +21,6 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687 h1:uSmlDgJGbUB0bwQBcZomBTottKwEDF5fF8UjSwKSzWM= github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687/go.mod h1:Salmysdw7DAVuobBW/LwsKKgpyCPHUhjyJoMJD+ZJiI= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -32,6 +31,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474= +github.com/go-quicktest/qt v1.102.0/go.mod h1:p4lGIVX+8Wa6ZPNDvqcxq36XpUDLh42FLetFU7odllI= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -74,8 +75,8 @@ github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRb github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= @@ -115,8 +116,8 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= @@ -147,3 +148,5 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +mvdan.cc/sh/v3 v3.14.1 h1:bXkhQWNHCs0KZEChF8hYS6FC+T2N9mUZLbQv9blditI= +mvdan.cc/sh/v3 v3.14.1/go.mod h1:syYCoFET8w9tvevxiXUtY8/ICrU+l26jHmhJDra3Vwo= diff --git a/hotfix/hotfix_generate.py b/hotfix/hotfix_generate.py index 432c96eb0ae..c50a9a04776 100644 --- a/hotfix/hotfix_generate.py +++ b/hotfix/hotfix_generate.py @@ -75,7 +75,6 @@ "cse_config_localdns.sh": "provisionConfigsLocalDNS", "cse_config_kubelet.sh": "provisionConfigsKubelet", "cse_config_network.sh": "provisionConfigsNetwork", - "cse_config_addons.sh": "provisionConfigsAddons", "cse_config_chrony.sh": "provisionConfigsChrony", # CSE main "cse_main.sh": "provisionScript", diff --git a/hotfix/hotfix_generate_test.py b/hotfix/hotfix_generate_test.py index 7626a0863c7..b321195994b 100644 --- a/hotfix/hotfix_generate_test.py +++ b/hotfix/hotfix_generate_test.py @@ -161,7 +161,8 @@ def test_config_modules_are_independently_hotfixable(self): self.assertEqual({variable}, blocks[0][0]) self.assertIn(f"- path: {{{{{path_function}}}}}", template) self.assertIn('permissions: "0744"', template) - self.assertIn("encoding: gzip", template) + self.assertIn("{{GetCloudInitFileEncodingLine}}", template) + self.assertIn("| GetCloudInitFileContent}}", template) self.assertNotIn("{{if", template) def test_mariner_package_update_is_not_embedded_in_custom_data(self): diff --git a/hotfix/render-nodecustomdata/main_test.go b/hotfix/render-nodecustomdata/main_test.go index d490001e6c4..55102a76c80 100644 --- a/hotfix/render-nodecustomdata/main_test.go +++ b/hotfix/render-nodecustomdata/main_test.go @@ -18,18 +18,16 @@ func TestRenderSelectedHotfixScripts(t *testing.T) { const initCloudBlock = `{{if IsAKSCustomCloud}} - path: {{GetInitAKSCloudFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "initAKSCloud"}} + content: {{GetVariableProperty "cloudInitData" "initAKSCloud" | GetCloudInitFileContent}} {{end}} ` const helperBlock = `- path: {{GetCSEHelpersScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSource"}} + content: {{GetVariableProperty "cloudInitData" "provisionSource" | GetCloudInitFileContent}} ` for _, distro := range []datamodel.Distro{ datamodel.AKSUbuntuContainerd2204Gen2, @@ -47,8 +45,9 @@ func TestRenderSelectedHotfixScripts(t *testing.T) { require.NoError(t, err) var payload struct { WriteFiles []struct { - Path string `yaml:"path"` - Content string `yaml:"content"` + Path string `yaml:"path"` + Encoding string `yaml:"encoding"` + Content string `yaml:"content"` } `yaml:"write_files"` } require.NoError(t, yaml.Unmarshal([]byte(rendered), &payload)) @@ -62,6 +61,7 @@ func TestRenderSelectedHotfixScripts(t *testing.T) { require.Len(t, payload.WriteFiles, 1+wantInitCount) for _, file := range payload.WriteFiles { require.NotEmpty(t, file.Content) + require.Empty(t, file.Encoding) } }) } diff --git a/image-fetcher/go.mod b/image-fetcher/go.mod index 4da4bb83d1f..7dec2d7d070 100644 --- a/image-fetcher/go.mod +++ b/image-fetcher/go.mod @@ -32,7 +32,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.18.7 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/sys/sequential v0.6.0 // indirect diff --git a/image-fetcher/go.sum b/image-fetcher/go.sum index b3a49c20ad0..ace668a48ad 100644 --- a/image-fetcher/go.sum +++ b/image-fetcher/go.sum @@ -84,8 +84,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= diff --git a/packer.mk b/packer.mk index 059016fd9fa..c36258e57e6 100755 --- a/packer.mk +++ b/packer.mk @@ -117,6 +117,25 @@ generate-publishing-info: az-login convert-sig-to-classic-storage-account-blob: az-login @./vhdbuilder/packer/convert-sig-to-classic-storage-account-blob.sh +convert-vhd-to-cosi: az-login + @./vhdbuilder/packer/imagecustomizer/scripts/convert-vhd-to-cosi.sh + +# cosi-uploader is its own Go module (vhdbuilder/cosi-uploader/go.mod). +# Go 1.27 enables cgo-less OpenSSL by default and rejects the old experiment. +build-cosi-uploader: + @echo "Building cosi-uploader binary" + @mkdir -p bin + @go_experiment=""; \ + if GOEXPERIMENT=ms_nocgo_opensslcrypto go env GOEXPERIMENT >/dev/null 2>&1; then \ + go_experiment=ms_nocgo_opensslcrypto; \ + fi; \ + cd vhdbuilder/cosi-uploader && GOEXPERIMENT="$$go_experiment" CGO_ENABLED=0 go build -o ../../bin/cosi-uploader . + @go version -m bin/cosi-uploader | grep -q 'microsoft_systemcrypto=1$$' || \ + { echo "cosi-uploader must be built with Microsoft Go system crypto" >&2; exit 1; } + +upload-cosi-to-pmc: build-cosi-uploader + @./vhdbuilder/packer/imagecustomizer/scripts/upload-cosi-to-pmc.sh + scanning-vhd: az-login @./vhdbuilder/packer/vhd-scanning.sh diff --git a/parts/common/components.json b/parts/common/components.json index 10df81e0980..fea4a772e7b 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -8,8 +8,8 @@ { "comment": "Windows 2022 servercore. Should have latest version and latest version -1", "renovateTag": "registry=https://mcr.microsoft.com, name=windows/servercore", - "latestVersion": "10.0.20348.5761", - "previousLatestVersion": "10.0.20348.5622", + "latestVersion": "10.0.20348.5773", + "previousLatestVersion": "10.0.20348.5761", "windowsSkuMatch": "2022-containerd*" }, { @@ -21,8 +21,8 @@ { "comment": "Windows 2022 servercore for Windows 2025. Should have latest version and latest version -1", "renovateTag": "registry=https://mcr.microsoft.com, name=windows/servercore", - "latestVersion": "10.0.20348.5761", - "previousLatestVersion": "10.0.20348.5622", + "latestVersion": "10.0.20348.5773", + "previousLatestVersion": "10.0.20348.5761", "windowsSkuMatch": "2025*" }, { @@ -34,8 +34,8 @@ { "comment": "Windows 2025 servercore for windows 2025. Should have latest version and latest version -1", "renovateTag": "registry=https://mcr.microsoft.com, name=windows/servercore", - "latestVersion": "10.0.26100.33650", - "previousLatestVersion": "10.0.26100.33438", + "latestVersion": "10.0.26100.33674", + "previousLatestVersion": "10.0.26100.33650", "windowsSkuMatch": "2025*" }, { @@ -350,8 +350,8 @@ "multiArchVersionsV2": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=containernetworking/cilium/cilium-distroless", - "latestVersion": "v1.18.14-260923", - "previousLatestVersion": "v1.18.12-260811", + "latestVersion": "v1.19.8-261006", + "previousLatestVersion": "v1.19.6-260811", "containerImagePrefetch": { "latestVersion": { "binaries": [ @@ -362,8 +362,7 @@ }, { "renovateTag": "registry=https://mcr.microsoft.com, name=containernetworking/cilium/cilium-distroless", - "latestVersion": "v1.19.8-260923", - "previousLatestVersion": "v1.19.6-260811", + "latestVersion": "v1.20.2-261006", "containerImagePrefetch": { "latestVersion": { "binaries": [ @@ -381,8 +380,8 @@ "multiArchVersionsV2": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=containernetworking/cilium/cilium-distroless-init", - "latestVersion": "v1.18.14-260923", - "previousLatestVersion": "v1.18.12-260811", + "latestVersion": "v1.19.8-261006", + "previousLatestVersion": "v1.19.6-260811", "containerImagePrefetch": { "latestVersion": { "binaries": [ @@ -393,8 +392,7 @@ }, { "renovateTag": "registry=https://mcr.microsoft.com, name=containernetworking/cilium/cilium-distroless-init", - "latestVersion": "v1.19.8-260923", - "previousLatestVersion": "v1.19.6-260811", + "latestVersion": "v1.20.2-261006", "containerImagePrefetch": { "latestVersion": { "binaries": [ @@ -423,7 +421,7 @@ "multiArchVersionsV2": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/secrets-store/driver", - "latestVersion": "v1.6.1-1" + "latestVersion": "v1.5.7-6" } ] }, @@ -433,7 +431,7 @@ "multiArchVersionsV2": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/azure/secrets-store/provider-azure", - "latestVersion": "v1.8.2-11" + "latestVersion": "v1.8.2-16" } ] }, @@ -644,21 +642,13 @@ "multiArchVersionsV2": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/livenessprobe", - "latestVersion": "v2.19.0" - }, - { - "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/livenessprobe", - "latestVersion": "v2.18.0" + "latestVersion": "v2.20.0-8" } ], "windowsVersions": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/livenessprobe", - "latestVersion": "v2.19.0" - }, - { - "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/livenessprobe", - "latestVersion": "v2.18.0" + "latestVersion": "v2.20.0-8" } ] }, @@ -668,21 +658,13 @@ "multiArchVersionsV2": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/csi-node-driver-registrar", - "latestVersion": "v2.17.0" - }, - { - "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/csi-node-driver-registrar", - "latestVersion": "v2.16.0" + "latestVersion": "v2.18.0-8" } ], "windowsVersions": [ { "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/csi-node-driver-registrar", - "latestVersion": "v2.17.0" - }, - { - "renovateTag": "registry=https://mcr.microsoft.com, name=oss/v2/kubernetes-csi/csi-node-driver-registrar", - "latestVersion": "v2.16.0" + "latestVersion": "v2.18.0-8" } ] }, @@ -832,52 +814,43 @@ } }, { - "name": "windows credential provider", - "windowsDownloadLocation": "c:\\akse-cache\\credential-provider\\", + "name": "windows credential provider dalec", + "windowsDownloadLocation": "c:\\akse-cache\\azure-acr-credential-provider\\", "downloadURIs": { "windows": { "default": { "versionsV2": [ { + "k8sVersion": "1.32", "renovateTag": "", - "latestVersion": "1.30.15" + "latestVersion": "1.32.11-1" }, { + "k8sVersion": "1.33", "renovateTag": "", - "latestVersion": "1.31.9" + "latestVersion": "1.33.20-1", + "previousLatestVersion": "1.33.6-1" }, { + "k8sVersion": "1.34", "renovateTag": "", - "latestVersion": "1.32.8" + "latestVersion": "1.34.16-1", + "previousLatestVersion": "1.34.3-1" }, { + "k8sVersion": "1.35", "renovateTag": "", - "latestVersion": "1.33.3" - } - ], - "downloadURL": "https://packages.aks.azure.com/cloud-provider-azure/v${version}/binaries/azure-acr-credential-provider-windows-amd64-v${version}.tar.gz" - } - } - } - }, - { - "name": "windows credential provider dalec", - "windowsDownloadLocation": "c:\\akse-cache\\azure-acr-credential-provider\\", - "downloadURIs": { - "windows": { - "default": { - "versionsV2": [ - { - "renovateTag": "", - "latestVersion": "1.32.11-1" + "latestVersion": "1.35.11-1" }, { + "k8sVersion": "1.36", "renovateTag": "", - "latestVersion": "1.33.6-1" + "latestVersion": "1.36.8-1" }, { + "k8sVersion": "1.37", "renovateTag": "", - "latestVersion": "1.34.3-1" + "latestVersion": "1.37.1-1" } ], "downloadURL": "https://packages.aks.azure.com/dalec-packages/azure-acr-credential-provider/$($version.Split('-')[0])/windows/amd64/azure-acr-credential-provider_${version}_amd64.zip" @@ -933,7 +906,7 @@ "versionsV2": [ { "renovateTag": "github-releases=oras-project/oras", - "latestVersion": "1.3.4" + "latestVersion": "1.3.6" } ], "downloadURL": "https://github.com/oras-project/oras/releases/download/v${version}/oras_${version}_linux_${CPU_ARCH}.tar.gz" @@ -954,7 +927,7 @@ "versionsV2": [ { "renovateTag": "github-releases=oras-project/oras", - "latestVersion": "1.3.4" + "latestVersion": "1.3.6" } ], "downloadURL": "https://github.com/oras-project/oras/releases/download/v${version}/oras_${version}_windows_amd64.zip" @@ -972,7 +945,7 @@ "versionsV2": [ { "renovateTag": "name=aks-secure-tls-bootstrap-client, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.1.4" + "latestVersion": "1.1.6" } ] }, @@ -980,7 +953,7 @@ "versionsV2": [ { "renovateTag": "name=aks-secure-tls-bootstrap-client, repository=production, os=ubuntu, release=24.04", - "latestVersion": "1.1.4" + "latestVersion": "1.1.6" } ] }, @@ -988,7 +961,7 @@ "versionsV2": [ { "renovateTag": "name=aks-secure-tls-bootstrap-client, repository=production, os=ubuntu, release=22.04", - "latestVersion": "1.1.4" + "latestVersion": "1.1.6" } ] }, @@ -996,7 +969,7 @@ "versionsV2": [ { "renovateTag": "name=aks-secure-tls-bootstrap-client, repository=production, os=ubuntu, release=20.04", - "latestVersion": "1.1.4" + "latestVersion": "1.1.6" } ] } @@ -1006,7 +979,7 @@ "versionsV2": [ { "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/ms-oss/x86_64/repodata, name=aks-secure-tls-bootstrap-client, os=azurelinux, release=3.0", - "latestVersion": "1.1.4" + "latestVersion": "1.1.6" } ] } @@ -1016,7 +989,7 @@ "versionsV2": [ { "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/ms-oss/x86_64/repodata, name=aks-secure-tls-bootstrap-client, os=azurelinux, release=3.0", - "latestVersion": "1.1.4" + "latestVersion": "1.1.6" } ] } @@ -1026,7 +999,7 @@ "versionsV2": [ { "renovateTag": "OCI_registry=https://mcr.microsoft.com, name=aks-secure-tls-bootstrap/v2/aks-secure-tls-bootstrap-client-sysext", - "latestVersion": "v1.1.4-2-azlinux3" + "latestVersion": "v1.1.6-1-azlinux3" } ], "downloadURL": "mcr.microsoft.com/aks-secure-tls-bootstrap/v2/aks-secure-tls-bootstrap-client-sysext:${version}-${SYSTEMD_ARCH}" @@ -1037,7 +1010,7 @@ "versionsV2": [ { "renovateTag": "", - "latestVersion": "1.1.4" + "latestVersion": "1.1.6" } ], "downloadURL": "https://github.com/Azure/aks-secure-tls-bootstrap/releases/download/client/v${version}/windows-amd64.zip" @@ -1471,8 +1444,8 @@ { "k8sVersion": "1.36", "renovateTag": "name=kubelet, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.36.4", - "previousLatestVersion": "1.36.3" + "latestVersion": "1.36.5", + "previousLatestVersion": "1.36.4" } ] }, @@ -1481,20 +1454,20 @@ { "k8sVersion": "1.34", "renovateTag": "name=kubelet, repository=production, os=ubuntu, release=24.04", - "latestVersion": "1.34.11", - "previousLatestVersion": "1.34.10" + "latestVersion": "1.34.12", + "previousLatestVersion": "1.34.11" }, { "k8sVersion": "1.35", "renovateTag": "name=kubelet, repository=production, os=ubuntu, release=24.04", - "latestVersion": "1.35.8", - "previousLatestVersion": "1.35.7" + "latestVersion": "1.35.9", + "previousLatestVersion": "1.35.8" }, { "k8sVersion": "1.36", "renovateTag": "name=kubelet, repository=production, os=ubuntu, release=24.04", - "latestVersion": "1.36.4", - "previousLatestVersion": "1.36.3" + "latestVersion": "1.36.5", + "previousLatestVersion": "1.36.4" } ] }, @@ -1503,20 +1476,20 @@ { "k8sVersion": "1.34", "renovateTag": "name=kubelet, repository=production, os=ubuntu, release=22.04", - "latestVersion": "1.34.11", - "previousLatestVersion": "1.34.10" + "latestVersion": "1.34.12", + "previousLatestVersion": "1.34.11" }, { "k8sVersion": "1.35", "renovateTag": "name=kubelet, repository=production, os=ubuntu, release=22.04", - "latestVersion": "1.35.8", - "previousLatestVersion": "1.35.7" + "latestVersion": "1.35.9", + "previousLatestVersion": "1.35.8" }, { "k8sVersion": "1.36", "renovateTag": "name=kubelet, repository=production, os=ubuntu, release=22.04", - "latestVersion": "1.36.4", - "previousLatestVersion": "1.36.3" + "latestVersion": "1.36.5", + "previousLatestVersion": "1.36.4" } ] }, @@ -1525,8 +1498,8 @@ { "k8sVersion": "1.34", "renovateTag": "name=kubelet, repository=production, os=ubuntu, release=20.04", - "latestVersion": "1.34.11", - "previousLatestVersion": "1.34.10" + "latestVersion": "1.34.12", + "previousLatestVersion": "1.34.11" } ] } @@ -1537,20 +1510,20 @@ { "k8sVersion": "1.34", "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=kubelet, os=azurelinux, release=3.0", - "latestVersion": "1.34.11", - "previousLatestVersion": "1.34.10" + "latestVersion": "1.34.12", + "previousLatestVersion": "1.34.11" }, { "k8sVersion": "1.35", "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=kubelet, os=azurelinux, release=3.0", - "latestVersion": "1.35.8", - "previousLatestVersion": "1.35.7" + "latestVersion": "1.35.9", + "previousLatestVersion": "1.35.8" }, { "k8sVersion": "1.36", "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=kubelet, os=azurelinux, release=3.0", - "latestVersion": "1.36.4", - "previousLatestVersion": "1.36.3" + "latestVersion": "1.36.5", + "previousLatestVersion": "1.36.4" } ] } @@ -1561,20 +1534,20 @@ { "k8sVersion": "1.34", "renovateTag": "OCI_registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/kubelet-sysext", - "latestVersion": "v1.34.11-4-azlinux3", - "previousLatestVersion": "v1.34.10-9-azlinux3" + "latestVersion": "v1.34.12-10-azlinux3", + "previousLatestVersion": "v1.34.11-16-azlinux3" }, { "k8sVersion": "1.35", "renovateTag": "OCI_registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/kubelet-sysext", - "latestVersion": "v1.35.8-6-azlinux3", - "previousLatestVersion": "v1.35.7-11-azlinux3" + "latestVersion": "v1.35.9-4-azlinux3", + "previousLatestVersion": "v1.35.8-12-azlinux3" }, { "k8sVersion": "1.36", "renovateTag": "OCI_registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/kubelet-sysext", - "latestVersion": "v1.36.4-6-azlinux3", - "previousLatestVersion": "v1.36.3-7-azlinux3" + "latestVersion": "v1.36.5-5-azlinux3", + "previousLatestVersion": "v1.36.4-10-azlinux3" } ], "downloadURL": "mcr.microsoft.com/oss/v2/kubernetes/kubelet-sysext:${version}-${SYSTEMD_ARCH}" @@ -1593,8 +1566,8 @@ { "k8sVersion": "1.36", "renovateTag": "name=kubectl, repository=production, os=ubuntu, release=26.04", - "latestVersion": "1.36.4", - "previousLatestVersion": "1.36.3" + "latestVersion": "1.36.5", + "previousLatestVersion": "1.36.4" } ] }, @@ -1603,20 +1576,20 @@ { "k8sVersion": "1.34", "renovateTag": "name=kubectl, repository=production, os=ubuntu, release=24.04", - "latestVersion": "1.34.11", - "previousLatestVersion": "1.34.10" + "latestVersion": "1.34.12", + "previousLatestVersion": "1.34.11" }, { "k8sVersion": "1.35", "renovateTag": "name=kubectl, repository=production, os=ubuntu, release=24.04", - "latestVersion": "1.35.8", - "previousLatestVersion": "1.35.7" + "latestVersion": "1.35.9", + "previousLatestVersion": "1.35.8" }, { "k8sVersion": "1.36", "renovateTag": "name=kubectl, repository=production, os=ubuntu, release=24.04", - "latestVersion": "1.36.4", - "previousLatestVersion": "1.36.3" + "latestVersion": "1.36.5", + "previousLatestVersion": "1.36.4" } ] }, @@ -1625,20 +1598,20 @@ { "k8sVersion": "1.34", "renovateTag": "name=kubectl, repository=production, os=ubuntu, release=22.04", - "latestVersion": "1.34.11", - "previousLatestVersion": "1.34.10" + "latestVersion": "1.34.12", + "previousLatestVersion": "1.34.11" }, { "k8sVersion": "1.35", "renovateTag": "name=kubectl, repository=production, os=ubuntu, release=22.04", - "latestVersion": "1.35.8", - "previousLatestVersion": "1.35.7" + "latestVersion": "1.35.9", + "previousLatestVersion": "1.35.8" }, { "k8sVersion": "1.36", "renovateTag": "name=kubectl, repository=production, os=ubuntu, release=22.04", - "latestVersion": "1.36.4", - "previousLatestVersion": "1.36.3" + "latestVersion": "1.36.5", + "previousLatestVersion": "1.36.4" } ] }, @@ -1647,8 +1620,8 @@ { "k8sVersion": "1.34", "renovateTag": "name=kubectl, repository=production, os=ubuntu, release=20.04", - "latestVersion": "1.34.11", - "previousLatestVersion": "1.34.10" + "latestVersion": "1.34.12", + "previousLatestVersion": "1.34.11" } ] } @@ -1659,20 +1632,20 @@ { "k8sVersion": "1.34", "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=kubectl, os=azurelinux, release=3.0", - "latestVersion": "1.34.11", - "previousLatestVersion": "1.34.10" + "latestVersion": "1.34.12", + "previousLatestVersion": "1.34.11" }, { "k8sVersion": "1.35", "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=kubectl, os=azurelinux, release=3.0", - "latestVersion": "1.35.8", - "previousLatestVersion": "1.35.7" + "latestVersion": "1.35.9", + "previousLatestVersion": "1.35.8" }, { "k8sVersion": "1.36", "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=kubectl, os=azurelinux, release=3.0", - "latestVersion": "1.36.4", - "previousLatestVersion": "1.36.3" + "latestVersion": "1.36.5", + "previousLatestVersion": "1.36.4" } ] } @@ -1683,20 +1656,20 @@ { "k8sVersion": "1.34", "renovateTag": "OCI_registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/kubectl-sysext", - "latestVersion": "v1.34.11-1-azlinux3", - "previousLatestVersion": "v1.34.10-7-azlinux3" + "latestVersion": "v1.34.12-10-azlinux3", + "previousLatestVersion": "v1.34.11-16-azlinux3" }, { "k8sVersion": "1.35", "renovateTag": "OCI_registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/kubectl-sysext", - "latestVersion": "v1.35.8-1-azlinux3", - "previousLatestVersion": "v1.35.7-5-azlinux3" + "latestVersion": "v1.35.9-4-azlinux3", + "previousLatestVersion": "v1.35.8-1-azlinux3" }, { "k8sVersion": "1.36", "renovateTag": "OCI_registry=https://mcr.microsoft.com, name=oss/v2/kubernetes/kubectl-sysext", - "latestVersion": "v1.36.4-1-azlinux3", - "previousLatestVersion": "v1.36.3-5-azlinux3" + "latestVersion": "v1.36.5-5-azlinux3", + "previousLatestVersion": "v1.36.4-10-azlinux3" } ], "downloadURL": "mcr.microsoft.com/oss/v2/kubernetes/kubectl-sysext:${version}-${SYSTEMD_ARCH}" @@ -1774,10 +1747,25 @@ { "name": "azure-acr-credential-provider", "downloadLocation": "/opt/credentialprovider/downloads", + "windowsDownloadLocation": "c:\\akse-cache\\credential-provider\\", "downloadURIs": { "windows": { "default": { - "versionsV2": [] + "versionsV2": [ + { + "renovateTag": "", + "latestVersion": "1.31.9" + }, + { + "renovateTag": "", + "latestVersion": "1.32.8" + }, + { + "renovateTag": "", + "latestVersion": "1.33.3" + } + ], + "downloadURL": "https://packages.aks.azure.com/cloud-provider-azure/v${version}/binaries/azure-acr-credential-provider-windows-amd64-v${version}.tar.gz" } }, "default": { @@ -1882,7 +1870,6 @@ { "name": "azure-acr-credential-provider-pmc", "downloadLocation": "/opt/azure-acr-credential-provider/downloads", - "windowsDownloadLocation": "c:\\akse-cache\\azure-acr-credential-provider\\", "downloadURIs": { "ubuntu": { "r2604": { @@ -2300,6 +2287,14 @@ "OSGUARD/v3.0": { "versionsV2": [] } + }, + "azurelinuxkata": { + "v3.0": { + "versionsV2": [{ + "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/cloud-native/x86_64/repodata, name=node-problem-detector-kubernetes, os=azurelinux, release=3.0", + "latestVersion": "0.8.25" + }] + } } } }, @@ -2348,6 +2343,14 @@ "OSGUARD/v3.0": { "versionsV2": [] } + }, + "azurelinuxkata": { + "v3.0": { + "versionsV2": [{ + "renovateTag": "RPM_registry=https://packages.microsoft.com/azurelinux/3.0/prod/ms-oss/x86_64/repodata, name=node-problem-detector-aks-config, os=azurelinux, release=3.0", + "latestVersion": "1.0.0" + }] + } } } }, @@ -2433,7 +2436,7 @@ "versionsV2": [ { "renovateTag": "github-releases=Azure/WALinuxAgent", - "latestVersion": "2.15.0.1" + "latestVersion": "2.16.0.2" } ] } diff --git a/parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh b/parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh index b549696ca10..0bf141f7cb2 100644 --- a/parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh +++ b/parts/linux/cloud-init/artifacts/acl/cse_helpers_acl.sh @@ -21,5 +21,3 @@ apt_get_update() { apt_get_dist_upgrade() { stub } - -#EOF diff --git a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh index cdbb11d85e9..f3db5805620 100644 --- a/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh +++ b/parts/linux/cloud-init/artifacts/acl/cse_install_acl.sh @@ -169,16 +169,12 @@ installACLGPUSysext() { installGPUDriverSysext() { # ACL NVIDIA GPU driver sysext registry paths: # Registry path uses major.minor (e.g. 3.0), tag uses full VERSION_ID (e.g. 3.0.20260304). - # # 1. NVIDIA proprietary driver: # mcr.microsoft.com/azurelinux/3.0/azure-container-linux/nvidia-driver-cuda:${VERSION_ID}... - # # 2. NVIDIA OpenRM driver: # mcr.microsoft.com/azurelinux/3.0/azure-container-linux/nvidia-driver-cuda-open:${VERSION_ID}... - # # 3. NVIDIA GRID (vGPU guest) driver for converged GPU sizes: # mcr.microsoft.com/azurelinux/3.0/azure-container-linux/nvidia-driver-vgpu:${VERSION_ID}... - # # NVIDIA_GPU_DRIVER_TYPE is set by AgentBaker based on the GPU SKU maps in # gpu_components.go. Converged sizes get "grid"; RTX PRO 6000 BSE v6 gets # "grid-v20" (Ubuntu-only, rejected below); modern CUDA SKUs get "cuda-lts" and legacy @@ -251,5 +247,3 @@ installToolFromLocalRepo() { stub return 1 } - -#EOF diff --git a/parts/linux/cloud-init/artifacts/acl/trident-acl-agent-override.conf b/parts/linux/cloud-init/artifacts/acl/trident-acl-agent-override.conf new file mode 100644 index 00000000000..0f5daca5659 --- /dev/null +++ b/parts/linux/cloud-init/artifacts/acl/trident-acl-agent-override.conf @@ -0,0 +1,44 @@ +[Service] +# Every setting the agent reads is pinned, including those matching its current +# defaults, so an upstream default change cannot silently alter AKS behaviour. +# Review these when the Trident version in the base image moves. + +# Agent default is acl.microsoft.com; the AKS contract uses acl.azure.com. A +# mismatch is silent: the RP writes a key the agent never watches. +Environment="TRIDENT_ACL_AGENT_KUBERNETES_ANNOTATION_PREFIX=acl.azure.com" +Environment="TRIDENT_ACL_AGENT_KUBERNETES_KUBECONFIG=/var/lib/kubelet/kubeconfig" + +# tridentd's gRPC socket. Without it the agent falls back to the Trident CLI, +# whose finalize reboots before a terminal status can be written. +Environment="TRIDENT_ACL_AGENT_TRIDENT_SOCKET=unix:///run/trident/trident.sock" + +Environment="TRIDENT_ACL_AGENT_ORCHESTRATION_MODE=annotations" +Environment="TRIDENT_ACL_AGENT_ORCHESTRATION_STATE_PATH=/var/lib/trident-acl-agent/state.json" +Environment="TRIDENT_ACL_AGENT_ORCHESTRATION_STAGE_TIMEOUT=20m" +Environment="TRIDENT_ACL_AGENT_ORCHESTRATION_FINALIZE_TIMEOUT=10m" +Environment="TRIDENT_ACL_AGENT_ORCHESTRATION_HEARTBEAT_INTERVAL=60s" +Environment="TRIDENT_ACL_AGENT_ORCHESTRATION_COOKIE_ENABLED=true" + +# The root-partition stamp survives A/B updates of /usr and is not the active +# version. Use a sentinel until Trident has an updatable version source; this +# deliberately disables current-version-based AlreadyAtTarget shortcuts. +Environment="TRIDENT_ACL_AGENT_CURRENT_VERSION_PATH=/dev/null" +Environment="TRIDENT_ACL_AGENT_CURRENT_VERSION_KEY=IMAGE_VERSION" +Environment="TRIDENT_ACL_AGENT_CURRENT_VERSION_FALLBACK=0.0.0" + +# Not set: NODE_NAME and API_SERVER are per-node and per-cluster and default +# correctly; NEBRASKA_* is unused in annotation mode. + +StateDirectory=trident-acl-agent + +# Trident before 0.29 exits cleanly until kubelet registers the Node. Restart it +# after a pause so path unit re-triggers can't hit systemd's start limit. +Restart=always +RestartSec=5 + +# Diagnostics only. Each of these absences makes the agent behave differently, +# and this records which one in the journal that is collected on failure. +ExecStartPre=-/usr/bin/sh -c '\ +for p in /var/lib/kubelet/kubeconfig /run/trident/trident.sock; do \ + [ -e "$p" ] && echo "trident-acl-agent preflight: present $p" || echo "trident-acl-agent preflight: MISSING $p"; \ +done' diff --git a/parts/linux/cloud-init/artifacts/acl/trident-acl-agent.path b/parts/linux/cloud-init/artifacts/acl/trident-acl-agent.path new file mode 100644 index 00000000000..c5346ffd009 --- /dev/null +++ b/parts/linux/cloud-init/artifacts/acl/trident-acl-agent.path @@ -0,0 +1,19 @@ +[Unit] +Description=Start the Trident ACL Agent once kubelet's kubeconfig exists +Documentation=https://microsoft.github.io/trident/ + +# The service unit lives on /usr, which an A/B update replaces; this path unit +# lives on the root partition, which it does not. Without this condition, an +# image whose /usr lacks the agent leaves both units failed with the reason +# buried in the path unit's journal. Skipped units are not failed units, and +# systemd logs the missing path by name. +ConditionPathExists=/usr/lib/systemd/system/trident-acl-agent.service + +[Path] +# The agent exits if kubelet's kubeconfig is absent, which it is until kubelet +# finishes TLS bootstrapping. Wait for the file rather than enabling the service. +PathExists=/var/lib/kubelet/kubeconfig +Unit=trident-acl-agent.service + +[Install] +WantedBy=multi-user.target diff --git a/parts/linux/cloud-init/artifacts/aks-check-network.sh b/parts/linux/cloud-init/artifacts/aks-check-network.sh index 52986666eff..e724690336b 100644 --- a/parts/linux/cloud-init/artifacts/aks-check-network.sh +++ b/parts/linux/cloud-init/artifacts/aks-check-network.sh @@ -41,7 +41,10 @@ function logs_to_events { # local vars here allow for nested function tracking # installContainerRuntime for example local task=$1; shift - local eventsFileName=$(date +%s%3N) + # The name must stay digits-only for WALinuxAgent to collect it. A nanosecond + # timestamp plus the emitting process's PID is unique by construction: two live + # processes cannot share a PID, and two forks of date cannot share a nanosecond. + local eventsFileName="$(date +%s%N)${BASHPID}" local startTime=$(date +"%F %T.%3N") ${@} diff --git a/parts/linux/cloud-init/artifacts/azlosguard/cse_helpers_osguard.sh b/parts/linux/cloud-init/artifacts/azlosguard/cse_helpers_osguard.sh index 29f838102a1..ca2570d056d 100644 --- a/parts/linux/cloud-init/artifacts/azlosguard/cse_helpers_osguard.sh +++ b/parts/linux/cloud-init/artifacts/azlosguard/cse_helpers_osguard.sh @@ -56,5 +56,3 @@ tdnf_download() { done echo Executed tdnf install -y \"$@\" $i times; } - -#EOF diff --git a/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh b/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh index 120ac952ec4..c98f21bfbbd 100644 --- a/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh +++ b/parts/linux/cloud-init/artifacts/azlosguard/cse_install_osguard.sh @@ -118,5 +118,3 @@ installToolFromLocalRepo() { stub return 1 } - -#EOF diff --git a/parts/linux/cloud-init/artifacts/block_wireserver.sh b/parts/linux/cloud-init/artifacts/block_wireserver.sh index 0c61458d74f..7cccd95541c 100644 --- a/parts/linux/cloud-init/artifacts/block_wireserver.sh +++ b/parts/linux/cloud-init/artifacts/block_wireserver.sh @@ -1,13 +1,10 @@ #!/bin/bash # Disallow container from reaching out to the special IP address 168.63.129.16 # for TCP protocol (which http uses) -# # 168.63.129.16 contains protected settings that have priviledged info. # HostGAPlugin (Host-GuestAgent-Plugin) is a web server process that runs on the physical host that serves the operational and diagnostic needs of the in-VM Guest Agent. # IT listens on both port 80 and 32526 hence access is only needed for agent but not the containers. -# # The host can still reach 168.63.129.16 because it goes through the OUTPUT chain, not FORWARD. -# # Note: we should not block all traffic to 168.63.129.16. For example UDP traffic is still needed # for DNS. diff --git a/parts/linux/cloud-init/artifacts/cgroup-cpu-telemetry.service b/parts/linux/cloud-init/artifacts/cgroup-cpu-telemetry.service index ac49fa7eb9b..7497abe927a 100644 --- a/parts/linux/cloud-init/artifacts/cgroup-cpu-telemetry.service +++ b/parts/linux/cloud-init/artifacts/cgroup-cpu-telemetry.service @@ -1,5 +1,7 @@ [Unit] Description=Emit per-service cgroup CPU usage and throttling telemetry +After=kubelet.service containerd.service +ConditionPathExists=/sys/fs/cgroup/cgroup.controllers [Service] Type=oneshot diff --git a/parts/linux/cloud-init/artifacts/cgroup-cpu-telemetry.sh b/parts/linux/cloud-init/artifacts/cgroup-cpu-telemetry.sh index f4feeee7b7d..8fc4af3fcbd 100644 --- a/parts/linux/cloud-init/artifacts/cgroup-cpu-telemetry.sh +++ b/parts/linux/cloud-init/artifacts/cgroup-cpu-telemetry.sh @@ -64,9 +64,10 @@ getServiceCPUUsage() { } EVENTS_LOGGING_DIR=/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/ -# WALinuxAgent only collects event files matching ^[0-9]+\.json$ and silently drops -# anything else, so the name must stay digits-only. -EVENTS_FILE_NAME=$(date +%s%3N) +# The name must stay digits-only for WALinuxAgent to collect it. A nanosecond +# timestamp plus the emitting process's PID is unique by construction: two live +# processes cannot share a PID, and two forks of date cannot share a nanosecond. +EVENTS_FILE_NAME="$(date +%s%N)${BASHPID}" STARTTIME_FORMATTED=$(date +"%F %T.%3N") ENDTIME_FORMATTED=$(date +"%F %T.%3N") CGROUP_VERSION=$(stat -fc %T /sys/fs/cgroup) diff --git a/parts/linux/cloud-init/artifacts/cgroup-memory-telemetry.service b/parts/linux/cloud-init/artifacts/cgroup-memory-telemetry.service index 45c3f2bdc1a..d40e9783772 100644 --- a/parts/linux/cloud-init/artifacts/cgroup-memory-telemetry.service +++ b/parts/linux/cloud-init/artifacts/cgroup-memory-telemetry.service @@ -1,5 +1,6 @@ [Unit] Description=Emit system cgroup memory telemetry +After=kubelet.service containerd.service OnSuccess=service-execution-telemetry@cgroup\x2dmemory\x2dtelemetry.service.service OnFailure=service-execution-telemetry@cgroup\x2dmemory\x2dtelemetry.service.service diff --git a/parts/linux/cloud-init/artifacts/cgroup-memory-telemetry.sh b/parts/linux/cloud-init/artifacts/cgroup-memory-telemetry.sh index 0e8c40df61b..f4c6aaed81c 100644 --- a/parts/linux/cloud-init/artifacts/cgroup-memory-telemetry.sh +++ b/parts/linux/cloud-init/artifacts/cgroup-memory-telemetry.sh @@ -36,7 +36,10 @@ getServiceMemory() { } EVENTS_LOGGING_DIR=/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/ -EVENTS_FILE_NAME=$(date +%s%3N) +# The name must stay digits-only for WALinuxAgent to collect it. A nanosecond +# timestamp plus the emitting process's PID is unique by construction: two live +# processes cannot share a PID, and two forks of date cannot share a nanosecond. +EVENTS_FILE_NAME="$(date +%s%N)${BASHPID}" STARTTIME_FORMATTED=$(date +"%F %T.%3N") ENDTIME_FORMATTED=$(date +"%F %T.%3N") CGROUP_VERSION=$(stat -fc %T /sys/fs/cgroup) diff --git a/parts/linux/cloud-init/artifacts/cgroup-pressure-telemetry.service b/parts/linux/cloud-init/artifacts/cgroup-pressure-telemetry.service index 4675824ae37..281a5465792 100644 --- a/parts/linux/cloud-init/artifacts/cgroup-pressure-telemetry.service +++ b/parts/linux/cloud-init/artifacts/cgroup-pressure-telemetry.service @@ -1,5 +1,7 @@ [Unit] Description=Emit system cgroup pressure telemetry +After=kubelet.service containerd.service +ConditionPathExists=/sys/fs/cgroup/cgroup.controllers OnSuccess=service-execution-telemetry@cgroup\x2dpressure\x2dtelemetry.service.service OnFailure=service-execution-telemetry@cgroup\x2dpressure\x2dtelemetry.service.service diff --git a/parts/linux/cloud-init/artifacts/cgroup-pressure-telemetry.sh b/parts/linux/cloud-init/artifacts/cgroup-pressure-telemetry.sh index 6b705b14c1d..6bee2b8f321 100644 --- a/parts/linux/cloud-init/artifacts/cgroup-pressure-telemetry.sh +++ b/parts/linux/cloud-init/artifacts/cgroup-pressure-telemetry.sh @@ -1,6 +1,9 @@ #!/bin/bash set -uo pipefail +# shellcheck source=./cgroup-telemetry-events.sh +source /opt/scripts/cgroup-telemetry-events.sh || exit 1 + getServicePressure() { local service_cgroup="$1" local cpu_pressure @@ -41,6 +44,9 @@ getServicePressure() { } EVENTS_LOGGING_DIR=/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/ +# The name must stay digits-only for WALinuxAgent to collect it. A nanosecond +# timestamp plus the emitting process's PID is unique by construction: two live +# processes cannot share a PID, and two forks of date cannot share a nanosecond. EVENTS_FILE_NAME="$(date +%s%N)${BASHPID}" STARTTIME_FORMATTED=$(date +"%F %T.%3N") ENDTIME_FORMATTED=$(date +"%F %T.%3N") @@ -304,55 +310,6 @@ fi message_string=$(echo $message_string | sed 's/\\//g' | sed 's/^.\(.*\).$/\1/') -if ! EVENT_JSON=$( jq -cn \ - --arg Timestamp "${STARTTIME_FORMATTED}" \ - --arg OperationId "${ENDTIME_FORMATTED}" \ - --arg Version "1.23" \ - --arg TaskName "${TASK_NAME}" \ - --arg EventLevel "${eventlevel}" \ - --arg Message "${message_string}" \ - --arg EventPid "0" \ - --arg EventTid "0" \ - --arg id "${EVENTS_FILE_NAME}" ' - # These byte limits are multiples of 3, so their Base64 length bounds are exact on jq 1.5. - def fits: (.Message | @base64 | length) <= 4096 and (tojson | @base64 | length) <= 8192; - {Timestamp: $Timestamp, OperationId: $OperationId, Version: $Version, TaskName: $TaskName, - EventLevel: $EventLevel, Message: $Message, EventPid: $EventPid, EventTid: $EventTid} as $event | - if ($event | fits) then [$event] else - ($Message | fromjson) as $payload | - ($payload.Pressure | to_entries) as $entries | - ((($entries | length) + 3) / 4 | floor) as $count | - [range(0; $count) as $index | - $payload | - .Pressure = ($entries[$index * 4:($index + 1) * 4] | from_entries) | - .TelemetryChunk = {id: $id, index: $index, count: $count, encoding: "json"} | - tojson as $message | $event | .Message = $message] - end | - if length > 0 and all(.[]; fits) then .[] - else error("pressure event exceeds guest agent transport limits") end -'); then - echo "pressure telemetry: failed to bound event payload" >&2 - exit 1 -fi - -if [ -z "${EVENT_JSON}" ]; then - echo "pressure telemetry: no events generated" >&2 - exit 1 -fi - -mkdir -p "${EVENTS_LOGGING_DIR}" || exit 1 -event_index=0 -while IFS= read -r event; do - printf -v target '%s/%s%06d.json' "${EVENTS_LOGGING_DIR%/}" "${EVENTS_FILE_NAME}" "${event_index}" - # WALinuxAgent deletes all entries in events/, including hidden staging files. - temporary=$(mktemp "${EVENTS_LOGGING_DIR%/}/../.pressure-event.XXXXXXXX") || exit 1 - # Publish complete files only, without overwriting another sample or fragment. - if ! printf '%s\n' "${event}" > "${temporary}" || - ! chmod 0644 "${temporary}" || ! ln "${temporary}" "${target}"; then - echo "pressure telemetry: failed to publish ${target}" >&2 - rm -f "${temporary}" - exit 1 - fi - rm -f "${temporary}" || exit 1 - event_index=$((event_index + 1)) -done <<< "${EVENT_JSON}" +publishCgroupTelemetry "${EVENTS_LOGGING_DIR}" "${EVENTS_FILE_NAME}" \ + "${STARTTIME_FORMATTED}" "${ENDTIME_FORMATTED}" "${TASK_NAME}" "${eventlevel}" \ + "${message_string}" Pressure pressure || exit 1 diff --git a/parts/linux/cloud-init/artifacts/cgroup-telemetry-events.sh b/parts/linux/cloud-init/artifacts/cgroup-telemetry-events.sh new file mode 100644 index 00000000000..9d34a0a82d4 --- /dev/null +++ b/parts/linux/cloud-init/artifacts/cgroup-telemetry-events.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Publish a cgroup observation, splitting the named object only when necessary. +# Each fragment is checked before any file is published. +publishCgroupTelemetry() { + local events_dir="$1" event_id="$2" start_time="$3" end_time="$4" + local task_name="$5" event_level="$6" message="$7" field="$8" label="$9" + local events event_index event target temporary + + if ! events=$(jq -cn \ + --arg Timestamp "${start_time}" \ + --arg OperationId "${end_time}" \ + --arg Version "1.23" \ + --arg TaskName "${task_name}" \ + --arg EventLevel "${event_level}" \ + --arg Message "${message}" \ + --arg EventPid "0" \ + --arg EventTid "0" \ + --arg id "${event_id}" \ + --arg field "${field}" ' + # These byte limits are multiples of 3, so their Base64 length bounds are exact on jq 1.5. + def fits: (.Message | @base64 | length) <= 4096 and (tojson | @base64 | length) <= 8192; + {Timestamp: $Timestamp, OperationId: $OperationId, Version: $Version, TaskName: $TaskName, + EventLevel: $EventLevel, Message: $Message, EventPid: $EventPid, EventTid: $EventTid} as $event | + if ($event | fits) then [$event] else + ($Message | fromjson) as $payload | + ($payload[$field] | to_entries) as $entries | + ((($entries | length) + 3) / 4 | floor) as $count | + [range(0; $count) as $index | + $payload | + .[$field] = ($entries[$index * 4:($index + 1) * 4] | from_entries) | + .TelemetryChunk = {id: $id, index: $index, count: $count, encoding: "json"} | + tojson as $chunk | $event | .Message = $chunk] + end | + if length > 0 and all(.[]; fits) then .[] + else error("cgroup event exceeds guest agent transport limits") end +'); then + echo "${label} telemetry: failed to bound event payload" >&2 + return 1 + fi + + if [ -z "${events}" ]; then + echo "${label} telemetry: no events generated" >&2 + return 1 + fi + + mkdir -p "${events_dir}" || return 1 + event_index=0 + while IFS= read -r event; do + printf -v target '%s/%s%06d.json' "${events_dir%/}" "${event_id}" "${event_index}" + # WALinuxAgent deletes all entries in events/, including hidden staging files. + temporary=$(mktemp "${events_dir%/}/../.${label}-event.XXXXXXXX") || return 1 + if ! printf '%s\n' "${event}" > "${temporary}" || + ! chmod 0644 "${temporary}" || ! ln "${temporary}" "${target}"; then + echo "${label} telemetry: failed to publish ${target}" >&2 + rm -f "${temporary}" + return 1 + fi + rm -f "${temporary}" || return 1 + event_index=$((event_index + 1)) + done <<< "${events}" +} diff --git a/parts/linux/cloud-init/artifacts/cgroup-telemetry.md b/parts/linux/cloud-init/artifacts/cgroup-telemetry.md index 5a37f12da07..0cb8d817c2b 100644 --- a/parts/linux/cloud-init/artifacts/cgroup-telemetry.md +++ b/parts/linux/cloud-init/artifacts/cgroup-telemetry.md @@ -60,7 +60,26 @@ outside `events/`, so WALinuxAgent's queue cleanup cannot delete them before publication. Splitting increases event count without changing WALinuxAgent's collection quotas; it prevents size truncation, not other ingestion failures. This -pressure-only implementation has no dependency on a new VHD or shared helper. +shared `cgroup-telemetry-events.sh` helper is installed in the VHD alongside +the collectors and can also publish other cgroup observation types. + +# Collector unit activation + +All three cgroup collector units order themselves with +`After=kubelet.service containerd.service`. Ordering only applies when the +units share a start transaction, so this affects the `OnBootSec=0min` +invocation alone: it stops the first observation from reporting `"Not Found"` +for services whose cgroup does not exist yet. The five-minute invocations are +unaffected, and no `Wants=` or `Requires=` is declared, so a collector still +runs when kubelet or containerd never start. + +The CPU and pressure collectors additionally declare +`ConditionPathExists=/sys/fs/cgroup/cgroup.controllers`. Both require cgroup +v2 and exit non-zero otherwise, which would surface as a failed unit on a node +that boots cgroup v1 even though the VHD was built on v2. The condition makes +systemd skip the unit instead. The memory collector carries no such condition +because it supports cgroup v1 and emits +`AKS.Runtime.memory_telemetry_cgroupv1` there. # Recurring service execution telemetry diff --git a/parts/linux/cloud-init/artifacts/cloud-init-status-check.sh b/parts/linux/cloud-init/artifacts/cloud-init-status-check.sh index a0227e4622c..026bea74ee4 100644 --- a/parts/linux/cloud-init/artifacts/cloud-init-status-check.sh +++ b/parts/linux/cloud-init/artifacts/cloud-init-status-check.sh @@ -48,7 +48,10 @@ handleCloudInitStatus() { local startTime=$(date +"%F %T.%3N") local endTime=$(date +"%F %T.%3N") local task="AKS.CSE.CloudInitStatusCheck" - local eventsFileName=$(date +%s%3N) + # The name must stay digits-only for WALinuxAgent to collect it. A nanosecond + # timestamp plus the emitting process's PID is unique by construction: two live + # processes cannot share a PID, and two forks of date cannot share a nanosecond. + local eventsFileName="$(date +%s%N)${BASHPID}" jsonString=$( jq -n \ --arg Timestamp "${startTime}" \ --arg OperationId "${endTime}" \ diff --git a/parts/linux/cloud-init/artifacts/cse_cmd.sh b/parts/linux/cloud-init/artifacts/cse_cmd.sh index 6e84a0f20d4..a08d3b35d6d 100644 --- a/parts/linux/cloud-init/artifacts/cse_cmd.sh +++ b/parts/linux/cloud-init/artifacts/cse_cmd.sh @@ -132,7 +132,6 @@ CSE_CONFIG_GPU_FILEPATH="{{GetCSEConfigGPUScriptFilepath}}" CSE_CONFIG_LOCALDNS_FILEPATH="{{GetCSEConfigLocalDNSScriptFilepath}}" CSE_CONFIG_KUBELET_FILEPATH="{{GetCSEConfigKubeletScriptFilepath}}" CSE_CONFIG_NETWORK_FILEPATH="{{GetCSEConfigNetworkScriptFilepath}}" -CSE_CONFIG_ADDONS_FILEPATH="{{GetCSEConfigAddonsScriptFilepath}}" CSE_CONFIG_CHRONY_FILEPATH="{{GetCSEConfigChronyScriptFilepath}}" AZURE_PRIVATE_REGISTRY_SERVER="{{GetPrivateAzureRegistryServer}}" HAS_CUSTOM_SEARCH_DOMAIN="{{HasCustomSearchDomain}}" diff --git a/parts/linux/cloud-init/artifacts/cse_config.sh b/parts/linux/cloud-init/artifacts/cse_config.sh index 8930eae9c2d..bf39be1bd7a 100755 --- a/parts/linux/cloud-init/artifacts/cse_config.sh +++ b/parts/linux/cloud-init/artifacts/cse_config.sh @@ -7,7 +7,6 @@ source "${CSE_CONFIG_GPU_FILEPATH:-${BASH_SOURCE[0]%.sh}_gpu.sh}" source "${CSE_CONFIG_LOCALDNS_FILEPATH:-${BASH_SOURCE[0]%.sh}_localdns.sh}" source "${CSE_CONFIG_KUBELET_FILEPATH:-${BASH_SOURCE[0]%.sh}_kubelet.sh}" source "${CSE_CONFIG_NETWORK_FILEPATH:-${BASH_SOURCE[0]%.sh}_network.sh}" -source "${CSE_CONFIG_ADDONS_FILEPATH:-${BASH_SOURCE[0]%.sh}_addons.sh}" source "${CSE_CONFIG_CHRONY_FILEPATH:-${BASH_SOURCE[0]%.sh}_chrony.sh}" configureAdminUser(){ @@ -506,7 +505,17 @@ configureK8s() { if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" = "false" ] && [ -z "${TLS_BOOTSTRAP_TOKEN:-}" ]; then # only create the client cert and key if we're not using vanilla/secure TLS bootstrapping if [ -n "${KUBELET_CLIENT_CONTENT}" ]; then - echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.key + # client.key is the kubelet client private key; path is overridable for testing. + # Create under a restrictive umask so the file is never briefly world-readable + # between creation and chmod (avoids a TOCTOU window), then chmod again to cover + # the case where the file already existed with looser permissions. + local client_key_path="${CLIENT_KEY_PATH:-/etc/kubernetes/certs/client.key}" + # Guard both operations: if either fails, stop before writing the private key + # rather than silently falling through and writing it with whatever (possibly + # world-readable) permissions the file ended up with. + (umask 077 && touch "${client_key_path}") || return "$ERR_KUBELET_CLIENT_KEY_PERM_FAIL" + chmod 0600 "${client_key_path}" || return "$ERR_KUBELET_CLIENT_KEY_PERM_FAIL" + echo "${KUBELET_CLIENT_CONTENT}" | base64 -d > "${client_key_path}" fi if [ -n "${KUBELET_CLIENT_CERT_CONTENT}" ]; then echo "${KUBELET_CLIENT_CERT_CONTENT}" | base64 -d > /etc/kubernetes/certs/client.crt @@ -636,16 +645,17 @@ EOF ensureArtifactStreaming() { waitForContainerdReady || exit $ERR_ARTIFACT_STREAMING_INSTALL - retrycmd_if_failure 120 5 25 systemctl --quiet enable --now acr-mirror overlaybd-tcmu overlaybd-snapshotter || exit $ERR_ARTIFACT_STREAMING_INSTALL local acr_mirror_setup="${ACR_MIRROR_SETUP_SCRIPT:-/opt/acr/tools/mirror/setup.sh}" if [ -x "$acr_mirror_setup" ]; then - "$acr_mirror_setup" aks + "$acr_mirror_setup" aks || exit $ERR_ARTIFACT_STREAMING_INSTALL else echo "Older acr-mirror package is detected, using old acr-config enablement" # setup.sh is only available in acr-mirror 1.0.0 and above - "${ACR_CONFIG_BIN:-/opt/acr/bin/acr-config}" --enable-containerd 'azurecr.io' + "${ACR_CONFIG_BIN:-/opt/acr/bin/acr-config}" --enable-containerd 'azurecr.io' || exit $ERR_ARTIFACT_STREAMING_INSTALL fi + + retrycmd_if_failure 120 5 25 systemctl --quiet enable --now acr-mirror overlaybd-tcmu overlaybd-snapshotter || exit $ERR_ARTIFACT_STREAMING_INSTALL } ensureSnapshotUpdate() { @@ -701,52 +711,11 @@ ensureSysctl() { retrycmd_if_failure 24 5 25 sysctl --system } -ensureK8sControlPlane() { - if $REBOOTREQUIRED || [ "$NO_OUTBOUND" = "true" ]; then - return - fi - retrycmd_if_failure 120 5 25 $KUBECTL 2>/dev/null cluster-info || exit $ERR_K8S_RUNNING_TIMEOUT -} - createKubeManifestDir() { KUBEMANIFESTDIR=/etc/kubernetes/manifests mkdir -p $KUBEMANIFESTDIR } -writeKubeConfig() { - KUBECONFIGDIR=/home/$ADMINUSER/.kube - KUBECONFIGFILE=$KUBECONFIGDIR/config - mkdir -p $KUBECONFIGDIR - touch $KUBECONFIGFILE - chown $ADMINUSER:$ADMINUSER $KUBECONFIGDIR - chown $ADMINUSER:$ADMINUSER $KUBECONFIGFILE - chmod 700 $KUBECONFIGDIR - chmod 600 $KUBECONFIGFILE - set +x - echo " ---- -apiVersion: v1 -clusters: -- cluster: - certificate-authority-data: \"$CA_CERTIFICATE\" - server: $KUBECONFIG_SERVER - name: \"$MASTER_FQDN\" -contexts: -- context: - cluster: \"$MASTER_FQDN\" - user: \"$MASTER_FQDN-admin\" - name: \"$MASTER_FQDN\" -current-context: \"$MASTER_FQDN\" -kind: Config -users: -- name: \"$MASTER_FQDN-admin\" - user: - client-certificate-data: \"$KUBECONFIG_CERTIFICATE\" - client-key-data: \"$KUBECONFIG_KEY\" -" > $KUBECONFIGFILE - set -x -} - disableSSH() { local start_time attempt_id unit result=0 start_time=$(date -u +"%F %T.%3N") @@ -822,5 +791,3 @@ get_compute_sku() { fi echo "$vm_sku" } - -#EOF diff --git a/parts/linux/cloud-init/artifacts/cse_config_addons.sh b/parts/linux/cloud-init/artifacts/cse_config_addons.sh deleted file mode 100644 index 5148a1b0df5..00000000000 --- a/parts/linux/cloud-init/artifacts/cse_config_addons.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -configClusterAutoscalerAddon() { - CLUSTER_AUTOSCALER_ADDON_FILE=/etc/kubernetes/addons/cluster-autoscaler-deployment.yaml - sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_ID | base64)|g" $CLUSTER_AUTOSCALER_ADDON_FILE - sed -i "s||$(echo $SERVICE_PRINCIPAL_CLIENT_SECRET | base64)|g" $CLUSTER_AUTOSCALER_ADDON_FILE - sed -i "s||$(echo $SUBSCRIPTION_ID | base64)|g" $CLUSTER_AUTOSCALER_ADDON_FILE - sed -i "s||$(echo $TENANT_ID | base64)|g" $CLUSTER_AUTOSCALER_ADDON_FILE - sed -i "s||$(echo $RESOURCE_GROUP | base64)|g" $CLUSTER_AUTOSCALER_ADDON_FILE -} - -configACIConnectorAddon() { - ACI_CONNECTOR_CREDENTIALS=$(printf "{\"clientId\": \"%s\", \"clientSecret\": \"%s\", \"tenantId\": \"%s\", \"subscriptionId\": \"%s\", \"activeDirectoryEndpointUrl\": \"https://login.microsoftonline.com\",\"resourceManagerEndpointUrl\": \"https://management.azure.com/\", \"activeDirectoryGraphResourceId\": \"https://graph.windows.net/\", \"sqlManagementEndpointUrl\": \"https://management.core.windows.net:8443/\", \"galleryEndpointUrl\": \"https://gallery.azure.com/\", \"managementEndpointUrl\": \"https://management.core.windows.net/\"}" "$SERVICE_PRINCIPAL_CLIENT_ID" "$SERVICE_PRINCIPAL_CLIENT_SECRET" "$TENANT_ID" "$SUBSCRIPTION_ID" | base64 -w 0) - - openssl req -newkey rsa:4096 -new -nodes -x509 -days 3650 -keyout /etc/kubernetes/certs/aci-connector-key.pem -out /etc/kubernetes/certs/aci-connector-cert.pem -subj "/C=US/ST=CA/L=virtualkubelet/O=virtualkubelet/OU=virtualkubelet/CN=virtualkubelet" - ACI_CONNECTOR_KEY=$(base64 /etc/kubernetes/certs/aci-connector-key.pem -w0) - ACI_CONNECTOR_CERT=$(base64 /etc/kubernetes/certs/aci-connector-cert.pem -w0) - - ACI_CONNECTOR_ADDON_FILE=/etc/kubernetes/addons/aci-connector-deployment.yaml - sed -i "s||$ACI_CONNECTOR_CREDENTIALS|g" $ACI_CONNECTOR_ADDON_FILE - sed -i "s||$RESOURCE_GROUP|g" $ACI_CONNECTOR_ADDON_FILE - sed -i "s||$ACI_CONNECTOR_CERT|g" $ACI_CONNECTOR_ADDON_FILE - sed -i "s||$ACI_CONNECTOR_KEY|g" $ACI_CONNECTOR_ADDON_FILE -} - -configAzurePolicyAddon() { - AZURE_POLICY_ADDON_FILE=/etc/kubernetes/addons/azure-policy-deployment.yaml - sed -i "s||/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP|g" $AZURE_POLICY_ADDON_FILE -} - -#EOF diff --git a/parts/linux/cloud-init/artifacts/cse_config_chrony.sh b/parts/linux/cloud-init/artifacts/cse_config_chrony.sh index 36989a20c06..7a8a2d7b0dd 100644 --- a/parts/linux/cloud-init/artifacts/cse_config_chrony.sh +++ b/parts/linux/cloud-init/artifacts/cse_config_chrony.sh @@ -109,7 +109,6 @@ apply_chrony_configuration() { # At the same time it retains some protection against one of the entries being # down (compare to just using one of the lines). See (LP: #1754358) for the # discussion. -# # About using servers from the NTP Pool Project in general see (LP: #104525). # Approved by Ubuntu Technical Board on 2011-02-08. # See https://www.pool.ntp.org/join.html for more information. diff --git a/parts/linux/cloud-init/artifacts/cse_config_gpu.sh b/parts/linux/cloud-init/artifacts/cse_config_gpu.sh index 3ad27398952..77e3b52b2e7 100755 --- a/parts/linux/cloud-init/artifacts/cse_config_gpu.sh +++ b/parts/linux/cloud-init/artifacts/cse_config_gpu.sh @@ -40,13 +40,36 @@ installGPUDriverImage() { # three known codes: 1 (MIG mode on with no MIG instances, which never succeeds), 2 (panic before the # driver userspace is ready), 127 (nvidia-smi missing on pre-reorder aks-gpu images). Anything else # still fails the unit. GPU pods reach the device via containerd's nvidia-container-runtime, not CDI. +# +# Grace-Blackwell also needs a wider START timeout. nvidia-container-toolkit 1.19.1 left this +# Type=oneshot unit at systemd's oneshot default of TimeoutStartSec=infinity; 1.20.1 (aks-gpu #177) +# ships /usr/lib/systemd/system/nvidia-cdi-refresh.service.d/10-container-engines.conf which orders +# the container engines after the unit AND pins TimeoutStartSec=90s. On GB200/GB300 the unit's +# `nvidia-ctk cdi generate` is slow -- ~80s to enumerate the GPUs + driver libs/firmware over the +# arm64 Grace NVLink/nvml path, plus a cold ~30s `nvidia-smi -L` precheck (>90s total) -- so systemd +# SIGTERMs it. A timeout is result=timeout (SIGTERM), NOT one of the tolerated SuccessExitStatus +# codes, so the unit still fails and install.sh's `systemctl restart` returns non-zero (CSE 84). +# x86 generates in <1s (measured: 8x H200 cdi generate <1s), so it keeps the 90s default. +# +# The timeout override lives in a SEPARATE, higher-sorted drop-in. systemd merges drop-ins in lexical +# filename order (last wins) across all dirs, so to beat the toolkit's 10-container-engines.conf our +# file must sort AFTER it -- hence the 90- prefix. (A 10-aks-* name sorts before 10-container-engines +# and loses -- verified: it leaves TimeoutStartUSec at 90s.) Verified live on GB300 that the 90- file +# raises TimeoutStartUSec 90s -> 5min. NVIDIA_CDI_REFRESH_DROP_IN="/etc/systemd/system/nvidia-cdi-refresh.service.d/10-aks-tolerate-generate-failure.conf" +NVIDIA_CDI_REFRESH_TIMEOUT_DROP_IN="/etc/systemd/system/nvidia-cdi-refresh.service.d/90-aks-cdi-refresh-timeout.conf" +NVIDIA_CDI_REFRESH_TIMEOUT_START_SEC="${NVIDIA_CDI_REFRESH_TIMEOUT_START_SEC:-300}" configureNvidiaCDIRefresh() { mkdir -p "$(dirname "$NVIDIA_CDI_REFRESH_DROP_IN")" || return 1 printf '[Service]\nSuccessExitStatus=1 2 127\n' > "$NVIDIA_CDI_REFRESH_DROP_IN" || return 1 - # Drop-ins are read when systemd loads a unit, so writing this before the toolkit is installed - # means the unit picks it up on its very first start. + # arm64 Grace-Blackwell (GB200/GB300) only: widen the start timeout so the slow cdi generate can + # finish. x86 generates in <1s, so leave it at the systemd default (no file written there). + if [ "$(getCPUArch)" = "arm64" ]; then + printf '[Service]\nTimeoutStartSec=%s\n' "$NVIDIA_CDI_REFRESH_TIMEOUT_START_SEC" > "$NVIDIA_CDI_REFRESH_TIMEOUT_DROP_IN" || return 1 + fi + # Drop-ins are read when systemd loads a unit, so writing these before the toolkit is installed + # means the unit picks them up on its very first start. systemctl daemon-reload || true } @@ -406,7 +429,6 @@ startNvidiaManagedExpServices() { # MIG strategy controls how nvidia-device-plugin exposes MIG instances to Kubernetes: # - "single": All MIG devices exposed as generic nvidia.com/gpu resources # - "mixed": MIG devices exposed with specific types like nvidia.com/mig-1g.5gb - # # We only use "mixed" when explicitly specified via NVIDIA_MIG_STRATEGY. # Otherwise, we default to "single" which is the safer/simpler option. # Note: NVIDIA_MIG_STRATEGY values from RP are "None", "Single", "Mixed". @@ -548,5 +570,3 @@ EOF # path and treat a slow/failed start as non-fatal. logs_to_events "AKS.CSE.start.nvidia-dcgm-exporter" "systemctlEnableAndStartNoBlock nvidia-dcgm-exporter 30" || echo "warning: nvidia-dcgm-exporter could not be enqueued; GPU metrics will start asynchronously" } - -#EOF diff --git a/parts/linux/cloud-init/artifacts/cse_config_kubelet.sh b/parts/linux/cloud-init/artifacts/cse_config_kubelet.sh index eac8c7afb37..7e35c9119ac 100644 --- a/parts/linux/cloud-init/artifacts/cse_config_kubelet.sh +++ b/parts/linux/cloud-init/artifacts/cse_config_kubelet.sh @@ -206,44 +206,6 @@ ensurePodInfraContainerImage() { rm -f ${POD_INFRA_CONTAINER_IMAGE_TAR} } -validateKubeletNodeLabels() { - local labels="$1" - local validated_labels="" - local delimiter="" - - # Return empty if no labels provided - if [ -z "$labels" ]; then - echo "No labels found in KUBELET_NODE_LABELS" - return 0 - fi - - # Split labels by comma and process each - IFS=',' read -ra LABEL_ARRAY <<< "$labels" - for label in "${LABEL_ARRAY[@]}"; do - # Split each label into key and value - # shellcheck disable=SC3010 - if [[ "$label" == *"="* ]]; then - key="${label%%=*}" - value="${label#*=}" - - # Check if key length exceeds 63 characters - if [ ${#key} -gt 63 ]; then - echo "Warning: Label key '$key' exceeds 63 characters, truncating to 63 characters" >&2 - key="${key:0:63}" - fi - - # Rebuild the label with potentially truncated key - validated_labels="${validated_labels}${delimiter}${key}=${value}" - fi - - # Set delimiter for subsequent labels - delimiter="," - done - - # Update the global variable with validated labels - KUBELET_NODE_LABELS="$validated_labels" -} - ensureKubelet() { KUBELET_DEFAULT_FILE=/etc/default/kubelet mkdir -p /etc/default @@ -373,13 +335,10 @@ EOF #!/bin/bash # Disallow container from reaching out to the special IP address 168.63.129.16 # for TCP protocol (which http uses) -# # 168.63.129.16 contains protected settings that have priviledged info. # HostGAPlugin (Host-GuestAgent-Plugin) is a web server process that runs on the physical host that serves the operational and diagnostic needs of the in-VM Guest Agent. # IT listens on both port 80 and 32526 hence access is only needed for agent but not the containers. -# # The host can still reach 168.63.129.16 because it goes through the OUTPUT chain, not FORWARD. -# # Note: we should not block all traffic to 168.63.129.16. For example UDP traffic is still needed # for DNS. @@ -596,5 +555,3 @@ setKubeletNodeIPFlag() { KUBELET_FLAGS="$KUBELET_FLAGS --node-ip=$nodeIPArg" fi } - -#EOF diff --git a/parts/linux/cloud-init/artifacts/cse_config_localdns.sh b/parts/linux/cloud-init/artifacts/cse_config_localdns.sh index 7c159ac5a09..447fc50f845 100644 --- a/parts/linux/cloud-init/artifacts/cse_config_localdns.sh +++ b/parts/linux/cloud-init/artifacts/cse_config_localdns.sh @@ -319,5 +319,3 @@ disableAKSLocalDNSHostsSetup() { echo "disableAKSLocalDNSHostsSetup complete" } - -#EOF diff --git a/parts/linux/cloud-init/artifacts/cse_config_network.sh b/parts/linux/cloud-init/artifacts/cse_config_network.sh index 6d3d37c30b8..44e8157b2ed 100644 --- a/parts/linux/cloud-init/artifacts/cse_config_network.sh +++ b/parts/linux/cloud-init/artifacts/cse_config_network.sh @@ -1,6 +1,11 @@ #!/bin/bash configPrivateClusterHosts() { + if [ ! -f /etc/systemd/system/reconcile-private-hosts.service ]; then + echo "Skipping private hosts reconciliation: service is absent" + return 0 + fi + mkdir -p /etc/systemd/system/reconcile-private-hosts.service.d/ touch /etc/systemd/system/reconcile-private-hosts.service.d/10-fqdn.conf tee /etc/systemd/system/reconcile-private-hosts.service.d/10-fqdn.conf > /dev/null < /dev/null; then - return 0 - fi - if isMarinerOrAzureLinux "$OS"; then - tdnf install -y jq && echo "jq was installed: $(jq --version)" - else - apt_get_install 5 1 60 jq && echo "jq was installed: $(jq --version)" - fi -} - # Returns the JSON for the given package on the given OS and OS version. getPackageJSON() { local package=$1 @@ -1224,19 +1215,6 @@ updatePackageVersions() { return } -# sets MULTI_ARCH_VERSIONS to multiArchVersionsV2 if it exists, otherwise multiArchVersions -updateMultiArchVersions() { - MULTI_ARCH_VERSIONS=() - readarray -t MULTI_ARCH_VERSIONS < <(jq -r "if .multiArchVersionsV2 == null then .multiArchVersions // [] else .multiArchVersionsV2 // [] | map(.latestVersion, .previousLatestVersion) end | .[] | values" <<< "$1") - return -} - -# sets PACKAGE_DOWNLOAD_URL to the URL of the package based on the os and osVersion -updatePackageDownloadURL() { - PACKAGE_DOWNLOAD_URL=$(jq -r ".downloadURL | values" <<< "$(getPackageJSON "$@")") - return -} - # Get latestVersion for a given k8sVersion from components.json based on the os and osVersion getLatestPkgVersionFromK8sVersion() { local k8sVersion="$1" @@ -1360,28 +1338,6 @@ removeKubeletFlag() { fi } -verify_DNS_health(){ - local domain_name=$1 - if [ -z "$domain_name" ]; then - echo "DNS domain is empty" - return $ERR_DNS_HEALTH_FAIL - fi - - dig_check_no_domain=$(dig +norec +short +tries=5 +timeout=5 .) - if [ "$?" -ne 0 ]; then - echo "Failed to resolve root domain '.'" - return $ERR_DNS_HEALTH_FAIL - fi - - dig_check_domain=$(dig +tries=5 +timeout=5 +short $domain_name) - ret_code=$? - if [ "$ret_code" -ne 0 ] || [ -z "$dig_check_domain" ]; then - echo "Failed to resolve domain $domain_name return code: $ret_code" - return $ERR_DNS_HEALTH_FAIL - fi - echo "DNS health check passed" -} - resolve_packages_source_url() { local retries=5 local wait_sleep=1 @@ -1554,51 +1510,6 @@ oras_login_with_kubelet_identity() { echo "successfully logged in to acr '$acr_url' with identity token" } -configureSSHService() { - local os_param="${1:-$OS}" - local os_version_param="${2:-$OS_VERSION}" - - # If not Ubuntu, no changes needed - if [ "$os_param" != "$UBUNTU_OS_NAME" ]; then - return 0 - fi - - # Only for Ubuntu 22.10+ or newer socket activation is used, for earlier versions no changes needed - if semverCompare "22.10" "$os_version_param" ; then - return 0 - fi - - # For Ubuntu 22.10+ with socket-based SSH activation - # disable socket if is active - if systemctl is-active --quiet ssh.socket; then - # Socket is active, disable it and switch to service-based activation - systemctl disable --now ssh.socket || echo "Warning: Could not disable ssh.socket" - fi - - # Remove the socket-based configuration files if present - if [ -f /etc/systemd/system/ssh.service.d/00-socket.conf ]; then - rm /etc/systemd/system/ssh.service.d/00-socket.conf || echo "Warning: Could not remove 00-socket.conf" - fi - - if [ -f /etc/systemd/system/ssh.socket.d/addresses.conf ]; then - rm /etc/systemd/system/ssh.socket.d/addresses.conf || echo "Warning: Could not remove addresses.conf" - fi - - # For all Ubuntu versions, just make sure ssh service is enabled and running - if ! systemctl is-enabled --quiet ssh.service; then - echo "Enabling SSH service..." - systemctlEnableAndStart ssh 30 || return $ERR_SYSTEMCTL_START_FAIL - fi - # Verify SSH service is now running - if ! systemctl is-active --quiet ssh; then - echo "Error: Failed to start SSH service after configuration changes" - return $ERR_SYSTEMCTL_START_FAIL - fi - - echo "SSH service successfully reconfigured and started" - return 0 -} - # Helper to extract tarballs with --no-same-owner for portability and security # Usage: extract_tarball [tar options] # Always uses --no-same-owner @@ -1680,14 +1591,12 @@ function get_sandbox_image_from_containerd_config() { # kube-reserved and system-reserved enforcement tiers (Node Memory Hardening F2/F5). # It MUST be called before kubelet starts so that /kubereserved.slice and /system.slice # exist and are managed by systemd before the first kubelet enforcement pass. -# # The function: # - Asserts cgroupv2 unified hierarchy (cgroupv1 is not supported by this feature # because mixed/legacy hierarchies cannot reliably enforce per-slice MemoryMax). # - Drops a /etc/systemd/system/kubereserved.slice unit (system.slice ships with systemd). # - Triggers `systemctl daemon-reload` and `systemctl start kubereserved.slice` so the # cgroup is materialised at /sys/fs/cgroup/kubereserved.slice prior to kubelet boot. -# # Inputs (env, all optional — the function is a no-op if the RP did not opt in): # KUBE_RESERVED_CGROUP — absolute cgroup name, e.g. "/kubereserved.slice" # SYSTEM_RESERVED_CGROUP — absolute cgroup name, e.g. "/system.slice" @@ -1697,7 +1606,6 @@ function get_sandbox_image_from_containerd_config() { # from KUBELET_FLAGS as a fallback. Both vars are unset (empty string) when the RP did # not opt the pool into Node Memory Hardening, which keeps ensureKubeletCgroupHierarchy # a no-op for non-hardened pools. -# # When kubelet config-file mode is enabled, --kube-reserved-cgroup / # --system-reserved-cgroup are filtered out of KUBELET_FLAGS by the RP # (TranslatedKubeletConfigFlags) and rendered into kubeletconfig.json instead, so diff --git a/parts/linux/cloud-init/artifacts/cse_install.sh b/parts/linux/cloud-init/artifacts/cse_install.sh index 4ea0d1fb47f..eb1666493b9 100755 --- a/parts/linux/cloud-init/artifacts/cse_install.sh +++ b/parts/linux/cloud-init/artifacts/cse_install.sh @@ -42,10 +42,6 @@ createManDbAutoUpdateFlagFile() { touch $MAN_DB_AUTO_UPDATE_FLAG_FILEPATH } -cleanupContainerdDlFiles() { - rm -rf $CONTAINERD_DOWNLOADS_DIR -} - # After the centralized packages changes, the containerd versions are only available in the components.json. installContainerdWithComponentsJson() { os=${UBUNTU_OS_NAME} @@ -178,7 +174,8 @@ downloadCredentialProvider() { # if BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER is set to non-empty string if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then local credential_provider_download_url_for_oras="${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}/${K8S_REGISTRY_REPO}/azure-acr-credential-provider:v${cred_version_for_oras}-linux-${CPU_ARCH}" - CREDENTIAL_PROVIDER_TGZ_TMP="${CREDENTIAL_PROVIDER_DOWNLOAD_URL##*/}" # Use bash builtin ## to remove all chars ("*") up to the final "/" + # Use bash builtin ## to remove all chars ("*") up to the final "/" + CREDENTIAL_PROVIDER_TGZ_TMP="${CREDENTIAL_PROVIDER_DOWNLOAD_URL##*/}" retrycmd_get_tarball_from_registry_with_oras 120 5 "$CREDENTIAL_PROVIDER_DOWNLOAD_DIR/$CREDENTIAL_PROVIDER_TGZ_TMP" "${credential_provider_download_url_for_oras}" || exit $ERR_ORAS_PULL_CREDENTIAL_PROVIDER return elif isRegistryUrl "${CREDENTIAL_PROVIDER_DOWNLOAD_URL}"; then @@ -190,7 +187,8 @@ downloadCredentialProvider() { return fi - CREDENTIAL_PROVIDER_TGZ_TMP="${CREDENTIAL_PROVIDER_DOWNLOAD_URL##*/}" # Use bash builtin ## to remove all chars ("*") up to the final "/" + # Use bash builtin ## to remove all chars ("*") up to the final "/" + CREDENTIAL_PROVIDER_TGZ_TMP="${CREDENTIAL_PROVIDER_DOWNLOAD_URL##*/}" echo "$CREDENTIAL_PROVIDER_DOWNLOAD_DIR/$CREDENTIAL_PROVIDER_TGZ_TMP ... $CREDENTIAL_PROVIDER_DOWNLOAD_URL" # Only apply per-operation budget during real CSE runs (CSE_STARTTIME_SECONDS set). # During VHD build, use 0 (unlimited) to avoid flakiness from transient network issues. @@ -212,31 +210,6 @@ installCredentialProviderFromUrl() { rm -rf ${CREDENTIAL_PROVIDER_DOWNLOAD_DIR} } -# TODO (alburgess) have oras version managed by dependant or Renovate -installOras() { - ORAS_DOWNLOAD_DIR="/opt/oras/downloads" - ORAS_EXTRACTED_DIR=${1} # Use components.json var for /opt/bin for linux-vhd-content-test.sh binary file checks. - ORAS_DOWNLOAD_URL=${2} - ORAS_VERSION=${3} - - mkdir -p $ORAS_DOWNLOAD_DIR - - echo "Installing Oras version $ORAS_VERSION..." - ORAS_TMP=${ORAS_DOWNLOAD_URL##*/} # Use bash builtin ## to remove all chars ("*") up to the final "/" - retrycmd_get_tarball 120 5 60 "$ORAS_DOWNLOAD_DIR/${ORAS_TMP}" ${ORAS_DOWNLOAD_URL} 300 || exit $ERR_ORAS_DOWNLOAD_ERROR - - if [ ! -f "$ORAS_DOWNLOAD_DIR/${ORAS_TMP}" ]; then - echo "File $ORAS_DOWNLOAD_DIR/${ORAS_TMP} does not exist." - exit $ERR_ORAS_DOWNLOAD_ERROR - fi - - echo "File $ORAS_DOWNLOAD_DIR/${ORAS_TMP} exists." - # no-same-owner because the files in the tarball are owned by 1001:admin - extract_tarball "$ORAS_DOWNLOAD_DIR/${ORAS_TMP}" "$ORAS_EXTRACTED_DIR/" - rm -r "$ORAS_DOWNLOAD_DIR" - echo "Oras version $ORAS_VERSION installed successfully." -} - # this is called called during node provisioning - # if secure TLS bootstrapping is disabled, this will simply remove the client binary from disk. # otherwise, if a custom URL is provided, it will use the custom URL to overwrite the existing installation @@ -314,7 +287,8 @@ downloadAzureCNI() { logs_to_events "AKS.CSE.logDownloadURL" "echo $VNET_CNI_PLUGINS_URL" VNET_CNI_PLUGINS_URL=$(update_base_url $VNET_CNI_PLUGINS_URL) - CNI_TGZ_TMP=${VNET_CNI_PLUGINS_URL##*/} # Use bash builtin ## to remove all chars ("*") up to the final "/" + # Use bash builtin ## to remove all chars ("*") up to the final "/" + CNI_TGZ_TMP=${VNET_CNI_PLUGINS_URL##*/} retrycmd_get_tarball 120 5 60 "$CNI_DOWNLOADS_DIR/${CNI_TGZ_TMP}" ${VNET_CNI_PLUGINS_URL} 300 || exit $ERR_CNI_DOWNLOAD_TIMEOUT } @@ -378,17 +352,6 @@ installCNILegacy() { chown -R root:root $CNI_BIN_DIR } -downloadCrictl() { - # if $1 is empty, take ${CRICTL_DOWNLOAD_DIR} as default value. Otherwise take $1 as the value - downloadDir=${1:-${CRICTL_DOWNLOAD_DIR}} - mkdir -p $downloadDir - url=${2} - logs_to_events "AKS.CSE.logDownloadURL" "echo $url" - url=$(update_base_url $url) - crictlTgzTmp=${url##*/} - retrycmd_curl_file 10 5 60 "$downloadDir/${crictlTgzTmp}" ${url} 300 || exit $ERR_CRICTL_DOWNLOAD_TIMEOUT -} - installCrictl() { CPU_ARCH=$(getCPUArch) currentVersion=$(crictl --version 2>/dev/null | sed 's/crictl version //g') @@ -421,7 +384,8 @@ setupCNIDirs() { } installAzureCNI() { - CNI_TGZ_TMP=${VNET_CNI_PLUGINS_URL##*/} # Use bash builtin ## to remove all chars ("*") up to the final "/" + # Use bash builtin ## to remove all chars ("*") up to the final "/" + CNI_TGZ_TMP=${VNET_CNI_PLUGINS_URL##*/} CNI_DIR_TMP=${CNI_TGZ_TMP%.tgz} # Use bash builtin % to remove the .tgz to look for a folder rather than tgz if [ -d "$CNI_DOWNLOADS_DIR/${CNI_DIR_TMP}" ]; then @@ -637,42 +601,6 @@ installKubeletKubectlFromURL() { rm -rf /opt/bin/kubelet-* /opt/bin/kubectl-* /home/hyperkube-downloads & } -pullContainerImage() { - CLI_TOOL=$1 - CONTAINER_IMAGE_URL=$2 - PULL_RETRIES=10 - PULL_WAIT_SLEEP_SECONDS=1 - PULL_TIMEOUT_SECONDS=600 # 10 minutes - - echo "pulling the image ${CONTAINER_IMAGE_URL} using ${CLI_TOOL} with a timeout of ${PULL_TIMEOUT_SECONDS}s" - - code=0 - if [ "${CLI_TOOL,,}" = "ctr" ]; then - retrycmd_if_failure $PULL_RETRIES $PULL_WAIT_SLEEP_SECONDS $PULL_TIMEOUT_SECONDS /opt/azure/containers/image-fetcher cache-image --image "$CONTAINER_IMAGE_URL" || code=$? - elif [ "${CLI_TOOL,,}" = "crictl" ]; then - retrycmd_if_failure $PULL_RETRIES $PULL_WAIT_SLEEP_SECONDS $PULL_TIMEOUT_SECONDS crictl pull $CONTAINER_IMAGE_URL || code=$? - else - retrycmd_if_failure $PULL_RETRIES $PULL_WAIT_SLEEP_SECONDS $PULL_TIMEOUT_SECONDS docker pull $CONTAINER_IMAGE_URL || code=$? - fi - - if [ "$code" -ne 0 ]; then - if [ "$code" -ne 124 ]; then - echo "failed to pull image ${CONTAINER_IMAGE_URL} using ${CLI_TOOL}, exit code: $code" - return $code - fi - echo "timed out pulling image ${CONTAINER_IMAGE_URL} via ${CLI_TOOL}" - if [ "${CLI_TOOL,,}" = "ctr" ]; then - return $ERR_CONTAINERD_CTR_IMG_PULL_TIMEOUT - elif [ "${CLI_TOOL,,}" = "crictl" ]; then - return $ERR_CONTAINERD_CRICTL_IMG_PULL_TIMEOUT - else - return $ERR_CONTAINERD_DOCKER_IMG_PULL_TIMEOUT - fi - fi - - echo "successfully pulled image ${CONTAINER_IMAGE_URL} using ${CLI_TOOL}" -} - retagContainerImage() { CLI_TOOL=$1 CONTAINER_IMAGE_URL=$2 @@ -753,23 +681,6 @@ cleanUpKubeProxyImages() { echo $(date),$(hostname), endCleanUpKubeProxyImages } -cleanupRetaggedImages() { - if [ "${TARGET_CLOUD}" != "AzureChinaCloud" ]; then - if [ "${CLI_TOOL}" = "crictl" ]; then - images_to_delete=$(crictl images | awk '{print $1":"$2}' | grep -E '^mcr\.(azk8s|azure)\.cn/' | tr ' ' '\n') # TODO(fseldow): remove azk8s when mcr.azk8s.cn is fully deprecated - else - images_to_delete=$(ctr --namespace k8s.io images list | awk '{print $1}' | grep -E '^mcr\.(azk8s|azure)\.cn/' | tr ' ' '\n') - fi - if [ -n "${images_to_delete}" ]; then - echo "${images_to_delete}" | while read -r image; do - removeContainerImage ${CLI_TOOL} ${image} - done - fi - else - echo "skipping container cleanup for AzureChinaCloud" - fi -} - cleanUpContainerImages() { export KUBERNETES_VERSION export CLI_TOOL @@ -780,10 +691,6 @@ cleanUpContainerImages() { bash -c cleanUpKubeProxyImages & } -cleanUpContainerd() { - rm -Rf $CONTAINERD_DOWNLOADS_DIR -} - getInstallModeAndCleanupContainerImages() { local SKIP_BINARY_CLEANUP=$1 local IS_VHD=$2 @@ -812,17 +719,6 @@ getInstallModeAndCleanupContainerImages() { echo "${FULL_INSTALL_REQUIRED,,}" } -overrideNetworkConfig() { - CONFIG_FILEPATH="/etc/cloud/cloud.cfg.d/80_azure_net_config.cfg" - mkdir -p "${CONFIG_FILEPATH%/*}" - touch ${CONFIG_FILEPATH} - cat <>${CONFIG_FILEPATH} -datasource: - Azure: - apply_network_config: false -EOF -} - # ==== GPU driver functions ==== # Shared between Azure Linux (Mariner) and ACL distro install scripts. # These functions are only invoked on GPU-enabled VM SKUs during provisioning; @@ -883,5 +779,3 @@ EOF systemctl enable nvidia-persistenced.service || exit 1 systemctl restart nvidia-persistenced.service || exit 1 } - -#EOF diff --git a/parts/linux/cloud-init/artifacts/cse_main.sh b/parts/linux/cloud-init/artifacts/cse_main.sh index 0c3ffe92253..19a3c1dd765 100755 --- a/parts/linux/cloud-init/artifacts/cse_main.sh +++ b/parts/linux/cloud-init/artifacts/cse_main.sh @@ -65,7 +65,6 @@ fi # Writes a modprobe blacklist rule and unloads the module if loaded. # Safe to run repeatedly during VHD build or provisioning; idempotent (overwrites with same content if already present). # Defined in cse_main.sh (not sourced) to support scriptless provisioning. -# # Usage: disableVulnerableKernelModule disableVulnerableKernelModule() { local mod="$1" @@ -122,7 +121,6 @@ reconcileVulnerableKernelModuleMitigation() { # future VHDs, while nodePrep covers PIS-cached or already-released VHDs that may skip # basePrep or carry stale modprobe files. # To add a new CVE mitigation, add a disableVulnerableKernelModule call below. - # # Ubuntu 20.04 remains in scope except 5.4.0-1164-azure-fips and newer ABIs in the # 5.4 Azure FIPS stream: linux-azure-fips 5.4.0-1164.170+fips1 fixes Copy Fail and # DirtyFrag; Focal 5.4 is not affected by Fragnesia. Other 20.04 streams stay blocked. @@ -133,7 +131,6 @@ reconcileVulnerableKernelModuleMitigation() { # 6.8.0-1058-azure (generic fallback 6.8.0-124-generic). Keep the apply for older # or unknown 22.04 / 24.04 kernel flavors. Fixed 22.04 / 24.04 kernels remove stale # deny rules so in-support VHDs no longer block legitimate module use after the fix. - # # AzureLinux 3.0 (regular and Kata) is excluded: kernel 6.6.139.1-1.azl3 and later fix Copy # Fail / DirtyFrag / Fragnesia upstream, so the modprobe blacklist is no longer # required. Newly-built AzL3 VHDs also no longer ship the four entries in modprobe-CIS.conf; @@ -144,12 +141,10 @@ reconcileVulnerableKernelModuleMitigation() { # VHD upgrade. AzureLinux OSGuard (hardened secure-boot variant) is intentionally kept in # scope as defense-in-depth: OSGuard workloads are security-sensitive and do not require # the affected kernel modules. - # # Mariner / AzureLinux 2.0 (AzL2) images are frozen (see FrozenCBLMarinerV2AndAzureLinuxV2SIGImageVersion=202512.06.0), # so they cannot pick up new modprobe-CIS.conf entries for these 2026 CVEs via VHD refresh. # Keep the runtime apply enabled for AzL2/Mariner while those images remain supported. # See https://github.com/Azure/AKS/issues/5753. - # if isUbuntu "$OS"; then if ubuntuKernelNeedsVulnerableModuleMitigation; then disableVulnerableKernelModule "algif_aead" "CVE-2026-31431 (Copy Fail)" @@ -449,7 +444,7 @@ function nodePrep { # This function first creates the systemd drop-in directory for kubelet.service. # Pay attention to ordering relative to other functions that create kubelet drop-ins. - logs_to_events "AKS.CSE.configureK8s" configureK8s + logs_to_events "AKS.CSE.configureK8s" configureK8s || exit "$ERR_KUBELET_CLIENT_KEY_PERM_FAIL" if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" = "true" ]; then # Depends on configureK8s, ensureKubeCACert, and installSecureTLSBootstrapClient @@ -457,6 +452,12 @@ function nodePrep { fi if [ -n "${OUTBOUND_COMMAND}" ]; then + # OUTBOUND_COMMAND is intentionally expanded unquoted here (not a scanner oversight): + # it is a full shell command (e.g. `curl -v --insecure --proxy-insecure `) built by + # the RP from config.OutboundType/CloudSpecConfig (see GetOutboundCommand in baker.go), + # not an arbitrary string from an external caller. Quoting it would break word-splitting + # into argv and make the outbound connectivity check fail to execute. Do not add quotes + # here without also changing it to an array (e.g. OUTBOUND_COMMAND=(...)) end to end. retrycmd_if_failure 20 1 15 $OUTBOUND_COMMAND >> /var/log/azure/cluster-provision-cse-output.log 2>&1 || exit $ERR_OUTBOUND_CONN_FAIL; fi if [ -n "${BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER}" ]; then @@ -731,21 +732,18 @@ function nodePrep { } # The provisioning is split into two stages to support VHD image creation workflows: -# # basePrep: Base image preparation # - Installs and configures all required components (kubelet, containerd, etc.) # - Sets up system configurations that are common across all nodes # - DOES NOT join the node to any cluster # - After this stage, users can add customizations (e.g., pre-pull additional container images) # - The VM can then be captured as a VHD image for use as a node pool base image -# # nodePrep: Cluster integration and hardware setup # - Performs cluster-specific configurations # - Configures hardware-specific components (GPU drivers, MIG partitions, etc.) # - Establishes connection to the API server # - Joins the node to the cluster # - Only runs when actually provisioning a node, not when creating VHD images -# # In typical deployments, both stages run sequentially during node provisioning. # For VHD image creation workflows, only basePrep runs initially, and nodePrep runs later # when nodes are created from that VHD image. diff --git a/parts/linux/cloud-init/artifacts/cse_start.sh b/parts/linux/cloud-init/artifacts/cse_start.sh index 1b28fd5abfc..3f61049bcdc 100644 --- a/parts/linux/cloud-init/artifacts/cse_start.sh +++ b/parts/linux/cloud-init/artifacts/cse_start.sh @@ -28,7 +28,10 @@ KUBELET_START_TIME_FORMATTED=$(date -d "${KUBELET_START_TIME}" +"%F %T.%3N" ) KUBELET_READY_TIME_FORMATTED="$(date -d "$(journalctl -u kubelet | grep NodeReady | cut -d' ' -f1-3)" +"%F %T.%3N")" SYSTEMD_SUMMARY=$(systemd-analyze || true) CSE_ENDTIME_FORMATTED=$(date +"%F %T.%3N") -EVENTS_FILE_NAME=$(date +%s%3N) +# The name must stay digits-only for WALinuxAgent to collect it. A nanosecond +# timestamp plus the emitting process's PID is unique by construction: two live +# processes cannot share a PID, and two forks of date cannot share a nanosecond. +EVENTS_FILE_NAME="$(date +%s%N)${BASHPID}" EXECUTION_DURATION=$(($(date +%s) - $(date -d "$CSE_STARTTIME" +%s))) SCRIPTLESS_MODE="none" diff --git a/parts/linux/cloud-init/artifacts/emit-kubelet-active-flags.sh b/parts/linux/cloud-init/artifacts/emit-kubelet-active-flags.sh index 1c5be2fe00f..e8aa190d824 100644 --- a/parts/linux/cloud-init/artifacts/emit-kubelet-active-flags.sh +++ b/parts/linux/cloud-init/artifacts/emit-kubelet-active-flags.sh @@ -100,7 +100,10 @@ emitKubeletActiveFlagsEvent() { fi message="$(getKubeletActiveFlagsJSON)" now="$(date +"%F %T.%3N")" - eventsFileName="$(date +%s%3N)" + # The name must stay digits-only for WALinuxAgent to collect it. A nanosecond + # timestamp plus the emitting process's PID is unique by construction: two live + # processes cannot share a PID, and two forks of date cannot share a nanosecond. + eventsFileName="$(date +%s%N)${BASHPID}" mkdir -p "${EVENTS_LOGGING_DIR}" jq -n \ --arg Timestamp "${now}" \ diff --git a/parts/linux/cloud-init/artifacts/flatcar/cse_helpers_flatcar.sh b/parts/linux/cloud-init/artifacts/flatcar/cse_helpers_flatcar.sh index 97c788e6483..2b3557cbe36 100755 --- a/parts/linux/cloud-init/artifacts/flatcar/cse_helpers_flatcar.sh +++ b/parts/linux/cloud-init/artifacts/flatcar/cse_helpers_flatcar.sh @@ -21,5 +21,3 @@ apt_get_update() { apt_get_dist_upgrade() { stub } - -#EOF diff --git a/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh b/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh index fbdea58bf58..1c73384bcee 100755 --- a/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh +++ b/parts/linux/cloud-init/artifacts/flatcar/cse_install_flatcar.sh @@ -149,5 +149,3 @@ installToolFromLocalRepo() { stub return 1 } - -#EOF diff --git a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh index 54b2a928150..b2abd419d60 100644 --- a/parts/linux/cloud-init/artifacts/init-aks-cloud.sh +++ b/parts/linux/cloud-init/artifacts/init-aks-cloud.sh @@ -40,7 +40,10 @@ EVENTS_LOGGING_DIR="/var/log/azure/Microsoft.Azure.Extensions.CustomScript/event logs_to_events() { local task=$1; shift local eventsFileName - eventsFileName=$(date +%s%3N) + # The name must stay digits-only for WALinuxAgent to collect it. A nanosecond + # timestamp plus the emitting process's PID is unique by construction: two live + # processes cannot share a PID, and two forks of date cannot share a nanosecond. + eventsFileName="$(date +%s%N)${BASHPID}" local startTime startTime=$(date +"%F %T.%3N") @@ -76,7 +79,10 @@ emit_event() { local message=$2 local level=${3:-Informational} local eventsFileName - eventsFileName=$(date +%s%3N) + # The name must stay digits-only for WALinuxAgent to collect it. A nanosecond + # timestamp plus the emitting process's PID is unique by construction: two live + # processes cannot share a PID, and two forks of date cannot share a nanosecond. + eventsFileName="$(date +%s%N)${BASHPID}" local timestamp timestamp=$(date +"%F %T.%3N") @@ -146,7 +152,6 @@ function make_request_with_retry { # 0 - opted in (wireserver confirmed IsOptedInForRootCerts=true) # 1 - not opted in (wireserver responded with false; valid, skip certs) # 2 - wireserver unreachable after retries (caller must treat as fatal) -# # Wireserver unreachable must be fatal (return 2) rather than silently skipping certs. # If the subscription is opted in for hardened root certs but we silently fall back to # the distro's default trust store, we leave a security hole — the node would trust CAs @@ -266,7 +271,7 @@ function retrieve_rcv1p_certs { } function install_certs_to_trust_store { - mkdir -p /root/AzureCACertificates + mkdir -p /root/AzureCACertificates || return 1 # Guard against empty glob: if no *.crt files exist, bash leaves the literal # '*.crt', which would fail cp. @@ -275,7 +280,7 @@ function install_certs_to_trust_store { return 1 fi - local cert + local cert destcert for cert in /root/AzureCACertificates/*.crt; do log_certificates "$cert" done @@ -748,5 +753,3 @@ EOF systemctl daemon-reload systemctl enable --now azure-ca-refresh.timer fi - -#EOF diff --git a/parts/linux/cloud-init/artifacts/mariner/aks-networkd-recovery.service b/parts/linux/cloud-init/artifacts/mariner/aks-networkd-recovery.service new file mode 100644 index 00000000000..7cbe694c6b4 --- /dev/null +++ b/parts/linux/cloud-init/artifacts/mariner/aks-networkd-recovery.service @@ -0,0 +1,11 @@ +[Unit] +Description=Recover failed Azure Linux eth0 networkd configuration +After=systemd-networkd.service +ConditionPathExists=/opt/azure/containers/provision.complete + +[Service] +Type=oneshot +ExecStart=/opt/azure/containers/aks-networkd-recovery.sh +TimeoutStartSec=45s +StandardOutput=journal +StandardError=journal diff --git a/parts/linux/cloud-init/artifacts/mariner/aks-networkd-recovery.sh b/parts/linux/cloud-init/artifacts/mariner/aks-networkd-recovery.sh new file mode 100644 index 00000000000..32804abb7a5 --- /dev/null +++ b/parts/linux/cloud-init/artifacts/mariner/aks-networkd-recovery.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +recover_networkd_link() { + local ifindex state + + if ! ifindex=$(cat /sys/class/net/eth0/ifindex); then + echo "ERROR: Cannot find eth0; not attempting networkd recovery." >&2 + return 0 + fi + if ! state=$(busctl --system --auto-start=no --timeout=5s get-property \ + org.freedesktop.network1 "/org/freedesktop/network1/link/_${ifindex}" \ + org.freedesktop.network1.Link AdministrativeState); then + echo "ERROR: Cannot query eth0 administrative state; not attempting recovery." >&2 + return 0 + fi + case "$state" in + 's "failed"') ;; + 's "pending"'|'s "initialized"'|'s "configuring"'|'s "configured"'|'s "unmanaged"'|'s "linger"') + return 0 + ;; + *) + echo "ERROR: Unexpected eth0 administrative state: $state" >&2 + return 0 + ;; + esac + + echo "eth0 is failed; requesting networkd reconfiguration (index $ifindex)." + if ! timeout 20s networkctl reconfigure "$ifindex"; then + echo "ERROR: eth0 reconfiguration request failed or timed out." >&2 + return 0 + fi + echo "eth0 reconfiguration requested; connectivity recovery is not yet confirmed." +} + +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + set -euo pipefail + recover_networkd_link +fi diff --git a/parts/linux/cloud-init/artifacts/mariner/aks-networkd-recovery.timer b/parts/linux/cloud-init/artifacts/mariner/aks-networkd-recovery.timer new file mode 100644 index 00000000000..7783ce1b9f1 --- /dev/null +++ b/parts/linux/cloud-init/artifacts/mariner/aks-networkd-recovery.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Check Azure Linux eth0 networkd state every five minutes + +[Timer] +OnCalendar=*:0/5 +AccuracySec=1s + +[Install] +WantedBy=timers.target diff --git a/parts/linux/cloud-init/artifacts/mariner/cse_helpers_mariner.sh b/parts/linux/cloud-init/artifacts/mariner/cse_helpers_mariner.sh index 494bf32f444..48795611149 100755 --- a/parts/linux/cloud-init/artifacts/mariner/cse_helpers_mariner.sh +++ b/parts/linux/cloud-init/artifacts/mariner/cse_helpers_mariner.sh @@ -78,4 +78,3 @@ dnf_download() { done echo Executed dnf download --resolve --downloaddir="\"${downloadDir}\"" "$@" $i times; } -#EOF diff --git a/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh b/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh index 8d19ce74b2c..15e6deb56b2 100755 --- a/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh +++ b/parts/linux/cloud-init/artifacts/mariner/cse_install_mariner.sh @@ -164,16 +164,12 @@ downloadGPUDrivers() { fi # Mariner CUDA rpm name comes in the following format: - # # 1. NVIDIA proprietary driver: # cuda-%{nvidia gpu driver version}_%{kernel source version}.%{kernel release version}.{mariner rpm postfix} - # # 2. NVIDIA OpenRM driver: # cuda-open[-hwe]-%{nvidia gpu driver version}_%{kernel source version}.%{kernel release version}.{mariner rpm postfix} - # # 3. NVIDIA GRID (vGPU guest) driver for converged GPU sizes: # nvidia-vgpu-guest-driver-%{version}_%{kernel version}.{mariner rpm postfix} - # # NVIDIA_GPU_DRIVER_TYPE is set by AgentBaker based on the GPU SKU maps in # gpu_components.go. Converged sizes get "grid"; RTX PRO 6000 BSE v6 gets # "grid-v20" (Ubuntu-only, rejected below); modern CUDA SKUs get "cuda-lts" and legacy @@ -294,7 +290,8 @@ installNvidiaContainerToolkit() { # Check if the version is empty and set the default if needed if [ -z "$MARINER_NVIDIA_CONTAINER_TOOLKIT_VERSION" ]; then - echo "nvidia-container-toolkit not found in components.json" # Expected for older VHD with new CSE + # Expected for older VHD with new CSE + echo "nvidia-container-toolkit not found in components.json" MARINER_NVIDIA_CONTAINER_TOOLKIT_VERSION="1.16.2" fi @@ -844,5 +841,3 @@ downloadContainerdFromVersion() { downloadContainerdFromURL() { echo "downloadContainerdFromURL not implemented for mariner" } - -#EOF diff --git a/parts/linux/cloud-init/artifacts/mariner/mariner-package-update.sh b/parts/linux/cloud-init/artifacts/mariner/mariner-package-update.sh index df11d663c5d..7084648a924 100755 --- a/parts/linux/cloud-init/artifacts/mariner/mariner-package-update.sh +++ b/parts/linux/cloud-init/artifacts/mariner/mariner-package-update.sh @@ -25,7 +25,7 @@ knead_emit_event() { local message="$2" local level="${3:-Informational}" local events_file_name - events_file_name="$(date +%s%3N)" + events_file_name="$(date +%s%N)${BASHPID}" local timestamp timestamp="$(date +"%F %T.%3N")" local event_json diff --git a/parts/linux/cloud-init/artifacts/measure-tls-bootstrapping-latency.sh b/parts/linux/cloud-init/artifacts/measure-tls-bootstrapping-latency.sh index c1c51cd1030..cdecea6c2d8 100644 --- a/parts/linux/cloud-init/artifacts/measure-tls-bootstrapping-latency.sh +++ b/parts/linux/cloud-init/artifacts/measure-tls-bootstrapping-latency.sh @@ -18,7 +18,10 @@ WATCH_TIMEOUT_SECONDS=${WATCH_TIMEOUT_SECONDS:-300} # default to 5 minutes createGuestAgentEvent() { local task=$1; startTime=$2; endTime=$3; local eventsFileName - eventsFileName=$(date +%s%3N) + # The name must stay digits-only for WALinuxAgent to collect it. A nanosecond + # timestamp plus the emitting process's PID is unique by construction: two live + # processes cannot share a PID, and two forks of date cannot share a nanosecond. + eventsFileName="$(date +%s%N)${BASHPID}" json_string=$( jq -n \ --arg Timestamp "${startTime}" \ diff --git a/parts/linux/cloud-init/artifacts/remove-bootstrap-kubeconfig.sh b/parts/linux/cloud-init/artifacts/remove-bootstrap-kubeconfig.sh index 9dce85039c5..8af38a4e5b9 100644 --- a/parts/linux/cloud-init/artifacts/remove-bootstrap-kubeconfig.sh +++ b/parts/linux/cloud-init/artifacts/remove-bootstrap-kubeconfig.sh @@ -8,7 +8,7 @@ KUBELET_CLIENT_CURRENT_PEM_PATH="${KUBELET_CLIENT_CURRENT_PEM_PATH:-/var/lib/kub logs_to_events() { local task=$1; shift - local eventsFileName=$(date +%s%3N) + local eventsFileName="$(date +%s%N)${BASHPID}" local startTime=$(date +"%F %T.%3N") ${@} diff --git a/parts/linux/cloud-init/artifacts/service-execution-telemetry.sh b/parts/linux/cloud-init/artifacts/service-execution-telemetry.sh index 01f4dd2f334..4396347a1ab 100644 --- a/parts/linux/cloud-init/artifacts/service-execution-telemetry.sh +++ b/parts/linux/cloud-init/artifacts/service-execution-telemetry.sh @@ -175,9 +175,10 @@ main() { } + if $MEMORY_PEAK_AVAILABLE then {MemoryPeakBytes: $MEMORY_PEAK_BYTES} else {} end') event_timestamp=$(date +"%F %T.%3N") - # WALinuxAgent only collects extension event files matching ^[0-9]+\.json$ and - # silently drops anything else, so the name must stay digits-only. - event_file_name="$(date +%s%3N).json" + # The name must stay digits-only for WALinuxAgent to collect it. A nanosecond + # timestamp plus the emitting process's PID is unique by construction: two live + # processes cannot share a PID, and two forks of date cannot share a nanosecond. + event_file_name="$(date +%s%N)${BASHPID}.json" event_json=$(jq -n \ --arg Timestamp "${event_timestamp}" \ --arg OperationId "${event_timestamp}" \ diff --git a/parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh b/parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh index e8472d62ad4..0899fb4e1d2 100755 --- a/parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh +++ b/parts/linux/cloud-init/artifacts/setup-custom-search-domains.sh @@ -9,4 +9,9 @@ wait_for_apt_locks # this script is called from `cse_main.sh` where we return ERR_CUSTOM_SEARCH_DOMAINS_FAIL in case this script ends with non-zero exit code retrycmd_if_failure 10 5 120 apt-get -y install realmd sssd sssd-tools samba-common samba samba-common python2.7 samba-libs packagekit || exit 1 wait_for_apt_locks +# disable command tracing so the realm password isn't written to provisioning logs +set +x echo "${CUSTOM_SEARCH_REALM_PASSWORD}" | realm join -U ${CUSTOM_SEARCH_REALM_USER}@$(echo "${CUSTOM_SEARCH_DOMAIN_NAME}" | tr /a-z/ /A-Z/) $(echo "${CUSTOM_SEARCH_DOMAIN_NAME}" | tr /a-z/ /A-Z/) +realm_join_status=$? +set -x +exit "${realm_join_status}" diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_helpers_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_helpers_ubuntu.sh index 92d4014b26b..1bf10219006 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_helpers_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_helpers_ubuntu.sh @@ -321,4 +321,3 @@ apt_get_download() { popd || return 1 return "$ret" } -#EOF diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index 0d86742de72..438c4a9daac 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -191,57 +191,6 @@ updatePMCRepository() { fi } -updateAptWithNvidiaPkg() { - readonly nvidia_gpg_keyring_path="/etc/apt/keyrings/nvidia.gpg" - mkdir -p "$(dirname "${nvidia_gpg_keyring_path}")" - - readonly nvidia_sources_list_path="/etc/apt/sources.list.d/nvidia.list" - local cpu_arch=$(getCPUArch) # Returns amd64 or arm64 - local repo_arch="" - local nvidia_ubuntu_release="" - - if [ "$cpu_arch" = "amd64" ]; then - repo_arch="x86_64" - elif [ "$cpu_arch" = "arm64" ]; then - repo_arch="sbsa" - else - echo "Unknown CPU architecture: ${cpu_arch}" - return - fi - - if [ "${UBUNTU_RELEASE}" = "22.04" ]; then - nvidia_ubuntu_release="ubuntu2204" - elif [ "${UBUNTU_RELEASE}" = "24.04" ]; then - nvidia_ubuntu_release="ubuntu2404" - elif [ "${UBUNTU_RELEASE}" = "26.04" ]; then - nvidia_ubuntu_release="ubuntu2604" - else - echo "NVIDIA repo setup is not supported on Ubuntu ${UBUNTU_RELEASE}" - return - fi - - # Construct URLs based on detected architecture and Ubuntu version - echo "deb [arch=${cpu_arch} signed-by=${nvidia_gpg_keyring_path}] https://developer.download.nvidia.com/compute/cuda/repos/${nvidia_ubuntu_release}/${repo_arch} /" > ${nvidia_sources_list_path} - - # Add NVIDIA repository - local nvidia_gpg_key_name="3bf863cc.pub" - if [ "${nvidia_ubuntu_release}" = "ubuntu2604" ]; then - nvidia_gpg_key_name="60DF8A40.pub" - fi - local nvidia_gpg_key_url="https://developer.download.nvidia.com/compute/cuda/repos/${nvidia_ubuntu_release}/${repo_arch}/${nvidia_gpg_key_name}" - - # Download the armored NVIDIA repo key and dearmor it into a binary keyring. - # apt only accepts a signed-by keyring with a .gpg (binary) or .asc (armored) extension; - # NVIDIA publishes an ASCII-armored *.pub, so a raw .pub file is rejected as an "unsupported - # filetype" and the key is ignored (the repo then fails to verify with NO_PUBKEY). Newer apt - # (e.g. 3.x on Ubuntu 26.04) enforces this strictly, so dearmor to nvidia.gpg. - local nvidia_gpg_key_tmp="/tmp/${nvidia_gpg_key_name}" - retrycmd_curl_file 120 5 25 "${nvidia_gpg_key_tmp}" "${nvidia_gpg_key_url}" 300 || exit $ERR_NVIDIA_GPG_KEY_DOWNLOAD_TIMEOUT - gpg --dearmor < "${nvidia_gpg_key_tmp}" > "${nvidia_gpg_keyring_path}" || exit $ERR_NVIDIA_GPG_KEY_DOWNLOAD_TIMEOUT - rm -f "${nvidia_gpg_key_tmp}" - apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT -} - isPackageInstalled() { local packageName="${1}" if dpkg -l "${packageName}" 2>/dev/null | grep -q "^ii"; then @@ -313,7 +262,6 @@ removeNvidiaRepos() { # skip_nvidia_driver_install=true): non-GPU VMs, and GPU VMs opted out via --gpu-driver None or the # skip toggle/tag. There the driver is dead weight (wasted disk; nvidia.ko rebuilt on every kernel # patch) and, on an opted-out GPU node, unused attack surface. -# # The prebaked module MAY already be loaded when we run: cuda(-lts) prebakes auto-load nvidia.ko at # boot (~5s, well before CSE), so on a cuda/cuda-lts SKU opted out via --gpu-driver None the module # is resident even though ensureGPUDrivers never ran. (grid prebakes do not auto-load, so grid nodes @@ -668,19 +616,6 @@ downloadPkgFromVersion() { echo "Succeeded to download ${packageName} version ${fullPackageVersion} for requested version ${packageVersion}" } -installContainerd() { - local packageVersion="${3:-}" - CONTAINERD_DOWNLOADS_DIR="${1:-$CONTAINERD_DOWNLOADS_DIR}" - eval containerdOverrideDownloadURL="${2:-}" - - # the user-defined package URL is always picked first, and the other options won't be tried when this one fails - if [ ! -z "${containerdOverrideDownloadURL}" ]; then - installContainerdFromOverride ${containerdOverrideDownloadURL} || exit $ERR_CONTAINERD_INSTALL_TIMEOUT - return 0 - fi - installContainerdWithAptGet "${packageVersion}" "${CONTAINERD_DOWNLOADS_DIR}" || exit $ERR_CONTAINERD_INSTALL_TIMEOUT -} - installContainerdFromOverride() { containerdOverrideDownloadURL=$1 echo "Installing containerd from user input: ${containerdOverrideDownloadURL}" @@ -842,7 +777,8 @@ ensureRunc() { fi fi - CPU_ARCH=$(getCPUArch) #amd64 or arm64 + # amd64 or arm64 + CPU_ARCH=$(getCPUArch) CURRENT_VERSION="" if command -v runc &> /dev/null; then CURRENT_VERSION=$(runc --version | head -n1 | sed 's/runc version //') @@ -907,5 +843,3 @@ ensureRunc() { logResolvedPackageVersion "moby-runc" "${TARGET_VERSION}" "${fullPackageVersion}" apt_get_install 20 30 120 "moby-runc=${fullPackageVersion}" --allow-downgrades || exit $ERR_RUNC_INSTALL_TIMEOUT } - -#EOF diff --git a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh index 3a2fd5d2af2..c2af5594fd9 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/ubuntu-snapshot-update.sh @@ -28,7 +28,7 @@ knead_emit_event() { local message="$2" local level="${3:-Informational}" local events_file_name - events_file_name="$(date +%s%3N)" + events_file_name="$(date +%s%N)${BASHPID}" local timestamp timestamp="$(date +"%F %T.%3N")" local event_json diff --git a/parts/linux/cloud-init/artifacts/validate-kubelet-credentials.sh b/parts/linux/cloud-init/artifacts/validate-kubelet-credentials.sh index cb54b66b408..f3b1dc1d8f4 100644 --- a/parts/linux/cloud-init/artifacts/validate-kubelet-credentials.sh +++ b/parts/linux/cloud-init/artifacts/validate-kubelet-credentials.sh @@ -15,7 +15,10 @@ RETRY_TIMEOUT_SECONDS=${CREDENTIAL_VALIDATION_RETRY_TIMEOUT_SECONDS:-5} # TODO: move logs_to_events out of provision_source.sh to an independent script so other provisioning/runtime scripts can cleanly source it as needed. logs_to_events() { local task=$1; shift - local eventsFileName=$(date +%s%3N) + # The name must stay digits-only for WALinuxAgent to collect it. A nanosecond + # timestamp plus the emitting process's PID is unique by construction: two live + # processes cannot share a PID, and two forks of date cannot share a nanosecond. + local eventsFileName="$(date +%s%N)${BASHPID}" local startTime=$(date +"%F %T.%3N") ${@} diff --git a/parts/linux/cloud-init/nodecustomdata.yml b/parts/linux/cloud-init/nodecustomdata.yml index 200eeb0ce5f..6c59344a200 100644 --- a/parts/linux/cloud-init/nodecustomdata.yml +++ b/parts/linux/cloud-init/nodecustomdata.yml @@ -28,201 +28,151 @@ write_files: {{if IsAKSCustomCloud}} - path: {{GetInitAKSCloudFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "initAKSCloud"}} + content: {{GetVariableProperty "cloudInitData" "initAKSCloud" | GetCloudInitFileContent}} {{end}} {{- else }} - path: {{GetCSEHelpersScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSource"}} + content: {{GetVariableProperty "cloudInitData" "provisionSource" | GetCloudInitFileContent}} {{if IsACL }} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceACL"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceACL" | GetCloudInitFileContent}} {{- else if IsAzlOSGuard}} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceAzlOSGuard"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceAzlOSGuard" | GetCloudInitFileContent}} {{- else if IsMariner}} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceMariner"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceMariner" | GetCloudInitFileContent}} {{- else if IsFlatcar }} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceFlatcar"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceFlatcar" | GetCloudInitFileContent}} {{- else }} - path: {{GetCSEHelpersScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionSourceUbuntu"}} + content: {{GetVariableProperty "cloudInitData" "provisionSourceUbuntu" | GetCloudInitFileContent}} {{end}} {{ if not IsCustomImage -}} - path: /opt/azure/containers/provision_start.sh permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionStartScript"}} + content: {{GetVariableProperty "cloudInitData" "provisionStartScript" | GetCloudInitFileContent}} {{- end }} - path: /opt/azure/containers/provision.sh permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionScript"}} + content: {{GetVariableProperty "cloudInitData" "provisionScript" | GetCloudInitFileContent}} - path: {{GetCSEInstallScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstalls"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstalls" | GetCloudInitFileContent}} {{if IsACL }} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsACL"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsACL" | GetCloudInitFileContent}} {{- else if IsAzlOSGuard}} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsAzlOSGuard"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsAzlOSGuard" | GetCloudInitFileContent}} {{- else if IsMariner}} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsMariner"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsMariner" | GetCloudInitFileContent}} {{- else if IsFlatcar }} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsFlatcar"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsFlatcar" | GetCloudInitFileContent}} {{- else }} - path: {{GetCSEInstallScriptDistroFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionInstallsUbuntu"}} + content: {{GetVariableProperty "cloudInitData" "provisionInstallsUbuntu" | GetCloudInitFileContent}} {{end}} - path: {{GetCSEConfigScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigs"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigs" | GetCloudInitFileContent}} - path: {{GetCSEConfigGPUScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsGPU"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsGPU" | GetCloudInitFileContent}} - path: {{GetCSEConfigLocalDNSScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsLocalDNS"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsLocalDNS" | GetCloudInitFileContent}} - path: {{GetCSEConfigKubeletScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsKubelet"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsKubelet" | GetCloudInitFileContent}} - path: {{GetCSEConfigNetworkScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsNetwork"}} - -- path: {{GetCSEConfigAddonsScriptFilepath}} - permissions: "0744" - encoding: gzip - owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsAddons"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsNetwork" | GetCloudInitFileContent}} - path: {{GetCSEConfigChronyScriptFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "provisionConfigsChrony"}} + content: {{GetVariableProperty "cloudInitData" "provisionConfigsChrony" | GetCloudInitFileContent}} {{if IsAKSCustomCloud}} - path: {{GetInitAKSCloudFilepath}} permissions: "0744" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "initAKSCloud"}} + content: {{GetVariableProperty "cloudInitData" "initAKSCloud" | GetCloudInitFileContent}} {{end}} -- path: /etc/systemd/system/reconcile-private-hosts.service - permissions: "0644" - encoding: gzip - owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "reconcilePrivateHostsService"}} - - path: /etc/systemd/system/kubelet.service permissions: "0600" - encoding: gzip - owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "kubeletSystemdService"}} - -- path: /opt/azure-network/configure-azure-network.sh - permissions: "0755" - encoding: gzip + {{GetCloudInitFileEncodingLine}} owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "configureAzureNetworkScript"}} + content: {{GetVariableProperty "cloudInitData" "kubeletSystemdService" | GetCloudInitFileContent}} -- path: /etc/udev/rules.d/99-azure-network.rules - permissions: "0644" - encoding: gzip - owner: root - content: !!binary | - {{GetVariableProperty "cloudInitData" "azureNetworkUdevRule"}} {{- end }} diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 91eacc7f9fb..b01d4553f7b 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "text/template" + "unicode/utf8" "github.com/Azure/agentbaker/parts" "github.com/Azure/agentbaker/pkg/agent/datamodel" @@ -164,14 +165,12 @@ func (t *TemplateGenerator) getLinuxNodeBootstrappingPayload(config *datamodel.N return t.getScriptlessBoothook(config) } - // this might seem strange that we're encoding the custom data to a JSON string and then extracting it, but without that serialisation and deserialisation - // lots of tests fail. var encoded string if config.IsFlatcar() || config.IsACL() { customData := getCustomDataFromJSON(t.getFlatcarLinuxNodeCustomDataJSONObject(config)) encoded = base64.StdEncoding.EncodeToString([]byte(customData)) } else { - customData := getCustomDataFromJSON(t.getLinuxNodeCustomDataJSONObject(config)) + customData := t.getLinuxNodeCustomData(config) encoded = getBase64EncodedGzippedCustomScriptFromStr(customData) } return encoded @@ -188,7 +187,7 @@ type encodedFile struct { func (t *TemplateGenerator) getScriptlessBoothook(config *datamodel.NodeBootstrappingConfiguration) string { config.DisableCustomData = true config.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(t.getLinuxNodeCustomDataJSONObject(config)) + nodeCustomData := t.getLinuxNodeCustomData(config) encodedNodeCustomData := getBase64EncodedGzippedCustomScriptFromStr(nodeCustomData) // hotfixJSONFile is optional: only VHDs that bake a static default hotfix @@ -343,20 +342,17 @@ func buildScriptlessCustomData(cloudInitTemplate, fileListTemplate, separator st return fmt.Sprintf(cloudInitTemplate, strings.Join(fileList, separator)) } -// GetLinuxNodeCustomDataJSONObject returns Linux customData JSON object in the form. -// { "customData": "" }. -func (t *TemplateGenerator) getLinuxNodeCustomDataJSONObject(config *datamodel.NodeBootstrappingConfiguration) string { - // get parameters - parameters := getParameters(config) - // get variable cloudInit - variables := getCustomDataVariables(config) - str, e := t.getSingleLineForTemplate(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, getBakerFuncMap(config, parameters, variables), true) - - if e != nil { - panic(e) +func (t *TemplateGenerator) getLinuxNodeCustomData(config *datamodel.NodeBootstrappingConfiguration) string { + b, err := parts.Templates.ReadFile(kubernetesNodeCustomDataYaml) + if err != nil { + panic(err) + } + str, err := RenderLinuxNodeCustomDataTemplate(b, config) + if err != nil { + panic(err) } - return fmt.Sprintf("{\"customData\": \"%s\"}", str) + return str } const ( @@ -573,9 +569,6 @@ func (t *TemplateGenerator) getNodeBootstrappingCmd(config *datamodel.NodeBootst return t.getWindowsNodeCSECommand(config) } if supportsScriptlessProvisioning(config) { - if config.IsCustomDataOnlyProvisioningEnabled() { - return "" - } if config.ScriptlessCSEProvisionMode { cseCmd := getBase64EncodedGzippedCustomScriptFromStr(t.getScriptlessNBCCmd(config)) return fmt.Sprintf(cseScriptlessPhase2Template, cseCmd) @@ -677,6 +670,8 @@ func (t *TemplateGenerator) getSingleLine(textFilename string, profile interface // same variables and functions used by the production AgentBaker path. // Callers must supply a complete configuration, as required by the production // rendering helpers. +// File entries must use GetCloudInitFileEncodingLine and GetCloudInitFileContent +// to select the encoding for the delivery mode. func RenderLinuxNodeCustomDataTemplate(templateContent []byte, config *datamodel.NodeBootstrappingConfiguration) (string, error) { parameters := getParameters(config) variables := getCustomDataVariables(config) @@ -694,10 +689,46 @@ func RenderLinuxNodeCustomDataTemplate(templateContent []byte, config *datamodel return buffer.String(), nil } +// Traditional custom data is gzipped as a whole, so its files need no inner encoding. +func usePlainCloudInit(config *datamodel.NodeBootstrappingConfiguration) bool { + return !config.EnableScriptlessCSECmd && !config.IsFlatcar() && !config.IsACL() +} + +func getCloudInitFileEncodingLine(config *datamodel.NodeBootstrappingConfiguration) string { + if usePlainCloudInit(config) { + return "" + } + return "encoding: " + encodingGZIP +} + +func getCloudInitFileContent(config *datamodel.NodeBootstrappingConfiguration, content string) (string, error) { + if usePlainCloudInit(config) { + if !utf8.ValidString(content) { + return "", fmt.Errorf("cloud-init file content contains invalid UTF-8") + } + // YAML quoting preserves Unicode line breaks that JSON leaves unescaped. + encoded, err := yaml.Marshal(&yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Style: yaml.DoubleQuotedStyle, + Value: content, + }) + return strings.TrimSuffix(string(encoded), "\n"), err + } + return "!!binary |\n " + getBase64EncodedGzippedCustomScriptFromStr(content), nil +} + // getTemplateFuncMap returns the general purpose template func map from getContainerServiceFuncMap. func getBakerFuncMap(config *datamodel.NodeBootstrappingConfiguration, params paramsMap, variables paramsMap) template.FuncMap { funcMap := getContainerServiceFuncMap(config) + funcMap["GetCloudInitFileEncodingLine"] = func() string { + return getCloudInitFileEncodingLine(config) + } + funcMap["GetCloudInitFileContent"] = func(content string) (string, error) { + return getCloudInitFileContent(config, content) + } + funcMap["GetParameter"] = func(s string) interface{} { if v, ok := params[s].(paramsMap); ok && v != nil { if v["value"] == nil { @@ -1482,9 +1513,6 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration "GetCSEConfigNetworkScriptFilepath": func() string { return cseConfigNetworkScriptFilepath }, - "GetCSEConfigAddonsScriptFilepath": func() string { - return cseConfigAddonsScriptFilepath - }, "GetCSEConfigChronyScriptFilepath": func() string { return cseConfigChronyScriptFilepath }, diff --git a/pkg/agent/baker_test.go b/pkg/agent/baker_test.go index d0d34b9e0cd..9b7acc68439 100644 --- a/pkg/agent/baker_test.go +++ b/pkg/agent/baker_test.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/gomega" "github.com/stretchr/testify/require" "github.com/vincent-petithory/dataurl" + "gopkg.in/yaml.v3" ) // this regex looks for groups of the following forms, returning KEY and VALUE as submatches. @@ -132,6 +133,61 @@ func newNodeCustomDataRenderConfig(distro datamodel.Distro) *datamodel.NodeBoots } } +func TestLinuxNodeBootstrappingPayloadSizeBudget(t *testing.T) { + for _, distro := range []datamodel.Distro{ + datamodel.AKSUbuntuContainerd2204Gen2, + datamodel.AKSUbuntuContainerd2404Gen2, + datamodel.AKSUbuntuMinimalContainerd2604Gen2, + datamodel.AKSCBLMarinerV2Gen2, + datamodel.AKSAzureLinuxV2Gen2, + datamodel.AKSAzureLinuxV3Gen2, + } { + for _, cloud := range []struct { + name string + custom bool + }{ + {name: "public"}, + {name: "custom", custom: true}, + } { + t.Run(fmt.Sprintf("%s/%s", distro, cloud.name), func(t *testing.T) { + config := newNodeCustomDataRenderConfig(distro) + // Exercise legacy NPS-style delivery with provisioning scripts in customData. + config.EnableKubeletConfigFile = true + config.EnableScriptlessCSECmd = false + config.EnableScriptlessNBCCSECmd = false + config.EnableScriptlessAKSNodeConfig = false + config.PreProvisionOnly = false + config.AgentPoolProfile.LocalDNSProfile = &datamodel.LocalDNSProfile{EnableLocalDNS: true} + if cloud.custom { + config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{ + Name: "akscustom", + } + } + + payload := InitializeTemplateGenerator().getLinuxNodeBootstrappingPayload(config) + compressed, err := base64.StdEncoding.Strict().DecodeString(payload) + require.NoError(t, err) + decoded, err := getGzipDecodedValue(compressed) + require.NoError(t, err) + require.True(t, strings.HasPrefix(string(decoded), "#cloud-config\n")) + require.False(t, config.ScriptlessCSEProvisionMode) + require.Contains(t, string(decoded), cseConfigLocalDNSScriptFilepath) + + // Ensure the custom-cloud measurement includes its required initializer. + if cloud.custom { + require.Contains(t, string(decoded), initAKSCloudFilepath) + } else { + require.NotContains(t, string(decoded), initAKSCloudFilepath) + } + + // Check the final base64-encoded payload against Compute's customData limit. + t.Logf("customData: %d encoded characters, %d characters remaining", len(payload), MaxCustomDataLength-len(payload)) + require.LessOrEqual(t, len(payload), MaxCustomDataLength, "legacy customData exceeds Compute's encoded size limit") + }) + } + } +} + func TestWindowsPreProvisionCustomDataOmitsTLSBootstrapToken(t *testing.T) { const bootstrapToken = "bake00.0123456789abcdef" @@ -1264,7 +1320,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) encodedNodeCustomData := getBase64EncodedGzippedCustomScriptFromStr(nodeCustomData) Expect(string(decodedPayload)).To(ContainSubstring(aksNodeCustomDataFilepath)) @@ -1478,7 +1534,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) Expect(nodeCustomData).To(ContainSubstring(initAKSCloudFilepath)) Expect(nodeCustomData).To(ContainSubstring("permissions: \"0744\"")) @@ -1495,7 +1551,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) Expect(nodeCustomData).To(ContainSubstring(initAKSCloudFilepath)) Expect(nodeCustomData).To(ContainSubstring("permissions: \"0744\"")) @@ -1512,7 +1568,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) Expect(nodeCustomData).To(ContainSubstring(initAKSCloudFilepath)) Expect(nodeCustomData).To(ContainSubstring("permissions: \"0744\"")) @@ -1530,7 +1586,7 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { renderConfig := *config renderConfig.EnableScriptlessCSECmd = true - nodeCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(&renderConfig)) + nodeCustomData := templateGenerator.getLinuxNodeCustomData(&renderConfig) Expect(nodeCustomData).To(ContainSubstring(initAKSCloudFilepath)) Expect(nodeCustomData).To(ContainSubstring("permissions: \"0744\"")) @@ -1548,11 +1604,17 @@ var _ = Describe("getLinuxNodeBootstrappingPayload", func() { decompressedPayload, err := getGzipDecodedValue(decodedPayload) Expect(err).NotTo(HaveOccurred()) - expectedCustomData := getCustomDataFromJSON(templateGenerator.getLinuxNodeCustomDataJSONObject(config)) + expectedCustomData := templateGenerator.getLinuxNodeCustomData(config) Expect(string(decompressedPayload)).To(Equal(expectedCustomData)) - Expect(string(decompressedPayload)).NotTo(ContainSubstring(aksNodeCustomDataFilepath)) - Expect(string(decompressedPayload)).NotTo(ContainSubstring(aksNbcCmdFilepath)) + // Script contents can reference these paths; only delivery as write_files + // entries would indicate that the scriptless path was used. + var cloudConfig cloudInit + Expect(yaml.Unmarshal(decompressedPayload, &cloudConfig)).To(Succeed()) + for _, file := range cloudConfig.WriteFiles { + Expect(file.Path).NotTo(Equal(aksNodeCustomDataFilepath)) + Expect(file.Path).NotTo(Equal(aksNbcCmdFilepath)) + } }) }) @@ -1680,7 +1742,7 @@ var _ = Describe("getNodeBootstrappingCmd", func() { Expect(string(payload)).NotTo(ContainSubstring("# azure-experimental-node-ready")) }) - It("should put provisioning in CustomData and omit CSE when CustomData-only provisioning is enabled", func() { + It("should put provisioning in CustomData and retain the CSE wait command when CustomData-only provisioning is enabled", func() { templateGenerator := InitializeTemplateGenerator() config := newScriptlessCmdTestConfig() config.CustomDataOnlyProvisioning = true @@ -1696,7 +1758,7 @@ var _ = Describe("getNodeBootstrappingCmd", func() { Expect(string(payload)).To(ContainSubstring("/var/lib/waagent/experimental_skip_ready_report")) Expect(string(payload)).To(ContainSubstring("if [ -s /sys/class/dmi/id/product_uuid ]; then")) Expect(string(payload)).To(ContainSubstring("cat /sys/class/dmi/id/product_uuid > /var/lib/waagent/provisioned")) - Expect(templateGenerator.getNodeBootstrappingCmd(config)).To(BeEmpty()) + Expect(templateGenerator.getNodeBootstrappingCmd(config)).To(Equal("/opt/azure/containers/aks-node-controller provision-wait")) }) It("should omit the Ready reporting handoff when CSE delivery is already selected", func() { diff --git a/pkg/agent/cloud_init_encoding_test.go b/pkg/agent/cloud_init_encoding_test.go new file mode 100644 index 00000000000..310213cb683 --- /dev/null +++ b/pkg/agent/cloud_init_encoding_test.go @@ -0,0 +1,192 @@ +package agent + +import ( + "encoding/base64" + "fmt" + "strings" + "testing" + "text/template" + + "github.com/Azure/agentbaker/parts" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestTraditionalCloudInitUsesSingleCompression(t *testing.T) { + template, err := parts.Templates.ReadFile(kubernetesNodeCustomDataYaml) + require.NoError(t, err) + for _, distro := range []datamodel.Distro{datamodel.AKSUbuntuContainerd2204Gen2, datamodel.AKSAzureLinuxV2Gen2, datamodel.AKSAzureLinuxV3Gen2} { + for _, custom := range []bool{false, true} { + for _, dns := range []bool{false, true} { + for _, preProvision := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/custom=%t/localdns=%t/preProvision=%t", distro, custom, dns, preProvision), func(t *testing.T) { + config := newNodeCustomDataRenderConfig(distro) + config.PreProvisionOnly = preProvision + config.AgentPoolProfile.LocalDNSProfile = &datamodel.LocalDNSProfile{EnableLocalDNS: dns} + if custom { + config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{Name: "akscustom"} + } + assertTraditionalCloudInitPayload(t, template, config) + }) + } + } + } + } +} + +func assertTraditionalCloudInitPayload(t *testing.T, template []byte, config *datamodel.NodeBootstrappingConfiguration) { + t.Helper() + generator := InitializeTemplateGenerator() + original, err := generator.getSingleLine(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, legacyCloudInitFuncMap(config), true) + require.NoError(t, err) + payload := generator.getLinuxNodeBootstrappingPayload(config) + compressed, err := base64.StdEncoding.Strict().DecodeString(payload) + require.NoError(t, err) + decoded, err := getGzipDecodedValue(compressed) + require.NoError(t, err) + require.True(t, strings.HasPrefix(string(decoded), "#cloud-config\n")) + rendered, err := RenderLinuxNodeCustomDataTemplate(template, config) + require.NoError(t, err) + require.Equal(t, rendered, string(decoded)) + var before, after cloudInit + require.NoError(t, yaml.Unmarshal([]byte(original), &before)) + require.NoError(t, yaml.Unmarshal(decoded, &after)) + var document struct { + WriteFiles []map[string]any `yaml:"write_files"` + } + require.NoError(t, yaml.Unmarshal(decoded, &document)) + for _, file := range document.WriteFiles { + _, hasEncoding := file["encoding"] + require.False(t, hasEncoding, "plain files must use cloud-init's default, not an invalid empty encoding") + } + beforeFiles, err := buildIgnitionTarEntries(before) + require.NoError(t, err) + afterFiles, err := buildIgnitionTarEntries(after) + require.NoError(t, err) + require.Equal(t, beforeFiles, afterFiles) + for _, file := range after.WriteFiles { + require.Empty(t, file.Encoding) + } + require.Less(t, len(payload), len(getBase64EncodedGzippedCustomScriptFromStr(original))) + t.Logf("customData: %d encoded characters", len(payload)) + require.LessOrEqual(t, len(payload), MaxCustomDataLength) +} + +func TestCloudInitEncodingLeavesOtherModesUnchanged(t *testing.T) { + template, err := parts.Templates.ReadFile(kubernetesNodeCustomDataYaml) + require.NoError(t, err) + generator := InitializeTemplateGenerator() + for _, distro := range []datamodel.Distro{datamodel.AKSUbuntuContainerd2204Gen2, datamodel.AKSAzureLinuxV3Gen2, datamodel.AKSFlatcarGen2, datamodel.AKSACLGen2TL} { + for _, scriptless := range []bool{false, true} { + if !scriptless && distro != datamodel.AKSFlatcarGen2 && distro != datamodel.AKSACLGen2TL { + continue + } + t.Run(fmt.Sprintf("%s/scriptless=%t", distro, scriptless), func(t *testing.T) { + config := newNodeCustomDataRenderConfig(distro) + config.EnableScriptlessCSECmd = scriptless + config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{Name: "akscustom"} + original, err := generator.getSingleLine(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, legacyCloudInitFuncMap(config), true) + require.NoError(t, err) + rendered, err := RenderLinuxNodeCustomDataTemplate(template, config) + require.NoError(t, err) + require.Equal(t, original, rendered) + require.Equal(t, original, generator.getLinuxNodeCustomData(config)) + }) + } + } +} + +func legacyCloudInitFuncMap(config *datamodel.NodeBootstrappingConfiguration) template.FuncMap { + funcs := getBakerFuncMap(config, getParameters(config), getCustomDataVariables(config)) + funcs["GetCloudInitFileEncodingLine"] = func() string { return "encoding: " + encodingGZIP } + funcs["GetCloudInitFileContent"] = func(content string) string { + return "!!binary |\n " + getBase64EncodedGzippedCustomScriptFromStr(content) + } + return funcs +} + +func TestCloudInitRenderingPreservesAdditionalFields(t *testing.T) { + input := []byte("#cloud-config\nruncmd:\n- echo hello\nwrite_files:\n- path: /opt/example\n append: true\n defer: true\n content: \"a\\tb\\n\\n\"\n") + output, err := RenderLinuxNodeCustomDataTemplate(input, newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2)) + require.NoError(t, err) + require.Equal(t, string(input), output) +} + +func TestPlainCloudInitFileContentPreservesWhitespace(t *testing.T) { + input := []byte(`#cloud-config +write_files: +- path: /opt/example + {{GetCloudInitFileEncodingLine}} + content: {{.Name | GetCloudInitFileContent}} +`) + for _, content := range []string{ + "", + "tab\tquote\"backslash\\", + "trailing newline\n\n", + "\u00e9\r\n", + "next\u0085line\u2028paragraph\u2029end", + "nul\x00escape\x1b", + "echo \"$HOME\"; printf '%s\\n' 'a: # & < >'\n", + strings.Repeat("long line: # \"\\ ", 1024), + } { + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) + config.AgentPoolProfile.Name = content + output, err := RenderLinuxNodeCustomDataTemplate(input, config) + require.NoError(t, err) + var parsed cloudInit + require.NoError(t, yaml.Unmarshal([]byte(output), &parsed)) + require.Len(t, parsed.WriteFiles, 1) + require.Empty(t, parsed.WriteFiles[0].Encoding) + require.Equal(t, content, parsed.WriteFiles[0].Content) + } +} + +func TestPlainCloudInitFileContentRejectsInvalidUTF8(t *testing.T) { + input := []byte("#cloud-config\nwrite_files:\n- path: /opt/example\n content: {{.Name | GetCloudInitFileContent}}\n") + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) + config.AgentPoolProfile.Name = string([]byte{'a', 0xff, 'b'}) + output, err := RenderLinuxNodeCustomDataTemplate(input, config) + require.ErrorContains(t, err, "invalid UTF-8") + require.Empty(t, output) +} + +func TestEncodedCloudInitFileContentUsesRawBytes(t *testing.T) { + for _, distro := range []datamodel.Distro{datamodel.AKSUbuntuContainerd2204Gen2, datamodel.AKSFlatcarGen2, datamodel.AKSACLGen2TL} { + for i, content := range []string{"", "#!/bin/bash\nprintf 'hello\\n'\n", "\x00\xff\r\n\u0085\u2028\u2029"} { + t.Run(fmt.Sprintf("%s/%d", distro, i), func(t *testing.T) { + config := newNodeCustomDataRenderConfig(distro) + config.EnableScriptlessCSECmd = true + encoded, err := getCloudInitFileContent(config, content) + require.NoError(t, err) + var file cloudInitWriteFile + require.NoError(t, yaml.Unmarshal([]byte("content: "+encoded+"\n"), &file)) + decoded, err := getGzipDecodedValue([]byte(file.Content)) + require.NoError(t, err) + require.Equal(t, []byte(content), decoded) + }) + } + } +} + +func BenchmarkTraditionalCloudInitEncoding(b *testing.B) { + for _, old := range []bool{true, false} { + b.Run(fmt.Sprintf("legacy=%t", old), func(b *testing.B) { + config := newNodeCustomDataRenderConfig(datamodel.AKSUbuntuContainerd2204Gen2) + generator := InitializeTemplateGenerator() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if old { + rendered, err := generator.getSingleLineForTemplate(kubernetesNodeCustomDataYaml, config.AgentPoolProfile, legacyCloudInitFuncMap(config), true) + if err != nil { + b.Fatal(err) + } + customData := getCustomDataFromJSON(fmt.Sprintf("{\"customData\": \"%s\"}", rendered)) + _ = getBase64EncodedGzippedCustomScriptFromStr(customData) + } else { + _ = generator.getLinuxNodeBootstrappingPayload(config) + } + } + }) + } +} diff --git a/pkg/agent/const.go b/pkg/agent/const.go index a9277323fc0..064c30fdb20 100644 --- a/pkg/agent/const.go +++ b/pkg/agent/const.go @@ -64,7 +64,6 @@ const ( kubernetesCSEConfigLocalDNS = "linux/cloud-init/artifacts/cse_config_localdns.sh" kubernetesCSEConfigKubelet = "linux/cloud-init/artifacts/cse_config_kubelet.sh" kubernetesCSEConfigNetwork = "linux/cloud-init/artifacts/cse_config_network.sh" - kubernetesCSEConfigAddons = "linux/cloud-init/artifacts/cse_config_addons.sh" kubernetesCSEConfigChrony = "linux/cloud-init/artifacts/cse_config_chrony.sh" kubernetesCSESendLogs = "linux/cloud-init/artifacts/cse_send_logs.py" kubernetesCSERedactCloudConfig = "linux/cloud-init/artifacts/cse_redact_cloud_config.py" @@ -122,7 +121,6 @@ const ( cseConfigLocalDNSScriptFilepath = "/opt/azure/containers/provision_configs_localdns.sh" cseConfigKubeletScriptFilepath = "/opt/azure/containers/provision_configs_kubelet.sh" cseConfigNetworkScriptFilepath = "/opt/azure/containers/provision_configs_network.sh" - cseConfigAddonsScriptFilepath = "/opt/azure/containers/provision_configs_addons.sh" cseConfigChronyScriptFilepath = "/opt/azure/containers/provision_configs_chrony.sh" customSearchDomainsCSEScriptFilepath = "/opt/azure/containers/setup-custom-search-domains.sh" dhcpV6ServiceCSEScriptFilepath = "/etc/systemd/system/dhcpv6.service" diff --git a/pkg/agent/datamodel/sig_config.go b/pkg/agent/datamodel/sig_config.go index 51058441fa1..6df93dd51a8 100644 --- a/pkg/agent/datamodel/sig_config.go +++ b/pkg/agent/datamodel/sig_config.go @@ -984,7 +984,7 @@ func GetMaintainedLinuxSIGImageConfigMap() map[Distro]SigImageConfig { // to ensure that missing image versions under a new image definition does not block AgentBaker rollout. // Definitions only need to be added to this list that have not yet been built and released at least once by // the time it's corresponding SIGImageConfigTemplate is added to AgentBaker. - definitionsToSkip := []string{"aclkatagen2TL", "aclgen2fipsCVM"} + definitionsToSkip := []string{"aclkatagen2TL"} maintained := map[Distro]SigImageConfig{} for _, m := range imageConfigMaps { diff --git a/pkg/agent/datamodel/sig_config_test.go b/pkg/agent/datamodel/sig_config_test.go index f580dff5a46..ce107971653 100644 --- a/pkg/agent/datamodel/sig_config_test.go +++ b/pkg/agent/datamodel/sig_config_test.go @@ -41,10 +41,9 @@ var _ = Describe("GetMaintainedLinuxSIGImageConfigMap", func() { AKSACLGen2FIPSTL: SIGACLGen2FIPSTLImageConfigTemplate.WithOptions(), AKSACLArm64Gen2FIPSTL: SIGACLArm64Gen2FIPSTLImageConfigTemplate.WithOptions(), AKSACLCVMGen2: SIGACLCVMGen2ImageConfigTemplate.WithOptions(), + AKSACLCVMGen2FIPS: SIGACLCVMGen2FIPSImageConfigTemplate.WithOptions(), // TODO: add back once "aclkatagen2TL" has at least one valid image version. // AKSACLKataGen2TL: SIGACLKataGen2TLImageConfigTemplate.WithOptions(), - // TODO: add once "aclgen2fipsCVM" has at least one valid image version. - // AKSACLCVMGen2FIPS: SIGACLCVMGen2FIPSImageConfigTemplate.WithOptions(), } actual := GetMaintainedLinuxSIGImageConfigMap() for distro, config := range expected { diff --git a/pkg/agent/live_patching_delivery_test.go b/pkg/agent/live_patching_delivery_test.go index b616b2bcf0e..04ca8b846c2 100644 --- a/pkg/agent/live_patching_delivery_test.go +++ b/pkg/agent/live_patching_delivery_test.go @@ -18,7 +18,9 @@ func TestLivePatchingUsesVHDInsteadOfCustomData(t *testing.T) { require.NotContains(t, rendered, "/opt/azure/containers/ubuntu-snapshot-update.sh") require.NotContains(t, rendered, "/opt/azure/containers/security-update.sh") require.NotContains(t, rendered, "/opt/azure/containers/npd-update.sh") - size := len(getBase64EncodedGzippedCustomScriptFromStr(rendered)) + payload := InitializeTemplateGenerator().getLinuxNodeBootstrappingPayload(config) + require.Equal(t, getBase64EncodedGzippedCustomScriptFromStr(rendered), payload) + size := len(payload) t.Logf("traditional custom data: %d encoded characters", size) require.Less(t, size, MaxCustomDataLength) } diff --git a/pkg/agent/mariner_live_patching_delivery_test.go b/pkg/agent/mariner_live_patching_delivery_test.go index e7cf41fc494..7b9cb9293f3 100644 --- a/pkg/agent/mariner_live_patching_delivery_test.go +++ b/pkg/agent/mariner_live_patching_delivery_test.go @@ -20,7 +20,9 @@ func TestMarinerLivePatchingUsesVHDInsteadOfCustomData(t *testing.T) { require.NotContains(t, rendered, "/opt/azure/containers/mariner-package-update.sh") require.NotContains(t, rendered, "/opt/azure/containers/security-update.sh") require.NotContains(t, rendered, "/opt/azure/containers/npd-update.sh") - size := len(getBase64EncodedGzippedCustomScriptFromStr(rendered)) + payload := InitializeTemplateGenerator().getLinuxNodeBootstrappingPayload(config) + require.Equal(t, getBase64EncodedGzippedCustomScriptFromStr(rendered), payload) + size := len(payload) t.Logf("traditional custom data: %d encoded characters", size) require.Less(t, size, MaxCustomDataLength) }) diff --git a/pkg/agent/script_minification_artifacts_test.go b/pkg/agent/script_minification_artifacts_test.go new file mode 100644 index 00000000000..02a5336d99a --- /dev/null +++ b/pkg/agent/script_minification_artifacts_test.go @@ -0,0 +1,253 @@ +package agent + +import ( + "bytes" + "encoding/base64" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "text/template" + + "github.com/Azure/agentbaker/parts" + "github.com/Azure/agentbaker/pkg/agent/datamodel" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/stretchr/testify/require" + "mvdan.cc/sh/v3/syntax" +) + +func TestCSEScriptMinificationArtifacts(t *testing.T) { + outputDir := os.Getenv("CSE_SCRIPT_DIFF_DIR") + if outputDir != "" { + require.True(t, filepath.IsAbs(outputDir), "CSE_SCRIPT_DIFF_DIR must be an absolute path") + } + scripts := discoverCSEScripts(t) + require.NotEmpty(t, scripts) + + for _, distro := range []datamodel.Distro{ + datamodel.AKSUbuntuContainerd2204Gen2, + datamodel.AKSUbuntuContainerd2404Gen2, + datamodel.AKSUbuntuMinimalContainerd2604Gen2, + datamodel.AKSCBLMarinerV2Gen2, + datamodel.AKSAzureLinuxV2Gen2, + datamodel.AKSAzureLinuxV3Gen2, + } { + for _, cloud := range []string{"public", "custom"} { + t.Run(string(distro)+"/"+cloud, func(t *testing.T) { + config := newNodeCustomDataRenderConfig(distro) + config.EnableKubeletConfigFile = true + config.AgentPoolProfile.LocalDNSProfile = &datamodel.LocalDNSProfile{EnableLocalDNS: true} + if cloud == "custom" { + config.ContainerService.Properties.CustomCloudEnv = &datamodel.CustomCloudEnv{Name: "akscustom"} + } + dir := filepath.Join(outputDir, string(distro), cloud) + for _, script := range scripts { + t.Run(script, func(t *testing.T) { + // Existing comment removal is part of the accepted baseline. Render + // without the production minifier or encoder so their changes remain visible. + raw, err := parts.Templates.ReadFile(script) + require.NoError(t, err) + templ, err := template.New("ContainerService template"). + Option("missingkey=error"). + Funcs(getContainerServiceFuncMap(config)). + Parse(string(removeComments(raw))) + require.NoError(t, err) + var rendered bytes.Buffer + require.NoError(t, templ.Execute(&rendered, config.ContainerService)) + before := []byte(strings.ReplaceAll(rendered.String(), "\r\n", "\n")) + encoded := getBase64EncodedGzippedCustomScript(script, config) + compressed, err := base64.StdEncoding.Strict().DecodeString(encoded) + require.NoError(t, err) + after, err := getGzipDecodedValue(compressed) + require.NoError(t, err) + if outputDir != "" { + writeMinificationDiff(t, dir, script, before, after) + } + require.Empty(t, shellCodeDiff(t, script, before, after), + "production rendering changed shell code (-source +production)") + cseValidateBashSyntax(t, script, before) + cseValidateBashSyntax(t, script, after) + }) + } + }) + } + } + if outputDir != "" { + t.Logf("script diffs written to %s", outputDir) + } +} + +func shellCodeDiff(t *testing.T, name string, before, after []byte) string { + t.Helper() + type shellCode struct { + Shebang string + File *syntax.File + } + parse := func(script []byte) shellCode { + t.Helper() + // Discard comments, but retain literal content (including heredocs). + file, err := syntax.NewParser(syntax.Variant(syntax.LangBash)).Parse(bytes.NewReader(script), name) + require.NoError(t, err) + code := shellCode{File: file} + if bytes.HasPrefix(script, []byte("#!")) { + code.Shebang, _, _ = strings.Cut(string(script), "\n") + } + return code + } + return cmp.Diff(parse(before), parse(after), + cmpopts.IgnoreTypes(syntax.Pos{}), + // Minification may spell ${name} as $name; the parsed parameter and + // all expansion operators must still match. + cmpopts.IgnoreFields(syntax.ParamExp{}, "Short"), + ) +} + +func TestShellCodeDiff(t *testing.T) { + tests := []struct { + name string + before string + after string + equal bool + }{ + { + name: "comments whitespace and statement separators", + before: "#!/bin/bash\n# comment\n\nif true; then\n echo ok # trailing comment\nfi\n", + after: "#!/bin/bash\nif true;then echo ok;fi\n", + equal: true, + }, + { + name: "short parameter expansion", + before: `echo "${value}"` + "\n", + after: `echo "$value"` + "\n", + equal: true, + }, + { + name: "preserved heredoc and multiline literal", + before: "# comment\ncat <<'EOF'\n# literal\n content\n\nEOF\n\n echo 'first\n\nlast'\n", + after: "cat <<'EOF'\n# literal\n content\n\nEOF\necho 'first\n\nlast'\n", + equal: true, + }, + { + name: "changed parameter boundary", + before: "echo ${value}suffix\n", + after: "echo $valuesuffix\n", + }, + { + name: "changed positional parameter", + before: "echo ${10}\n", + after: "echo $10\n", + }, + { + name: "added command", + before: "echo ok\n", + after: "echo ok; unexpected_command\n", + }, + { + name: "removed command", + before: "check_configuration\necho ok\n", + after: "echo ok\n", + }, + { + name: "changed argument", + before: "echo ok\n", + after: "echo unexpected\n", + }, + { + name: "injected command substitution", + before: `echo "ok"` + "\n", + after: `echo "$(unexpected_command)"` + "\n", + }, + { + name: "injected process substitution", + before: "cat input\n", + after: "cat <(unexpected_command)\n", + }, + { + name: "changed control flow", + before: "check_configuration && start_service\n", + after: "check_configuration; start_service\n", + }, + { + name: "background command", + before: "check_configuration; start_service\n", + after: "check_configuration & start_service\n", + }, + { + name: "changed redirect", + before: "echo ok > output\n", + after: "echo ok >> output\n", + }, + { + name: "changed function body", + before: "setup() { echo ok; }\n", + after: "setup() { echo ok; unexpected_command; }\n", + }, + { + name: "removed quoting", + before: `echo "$value"` + "\n", + after: "echo $value\n", + }, + { + name: "changed whitespace inside literal", + before: "echo 'a b'\n", + after: "echo 'a b'\n", + }, + { + name: "removed comment-like literal content", + before: "echo 'first\n# literal\nlast'\n", + after: "echo 'first\nlast'\n", + }, + { + name: "changed heredoc payload", + before: "cat <<'EOF'\n# literal\n content\n\nEOF\n", + after: "cat <<'EOF'\ncontent\nEOF\n", + }, + { + name: "enabled heredoc expansion", + before: "cat <<'EOF'\n$variable\nEOF\n", + after: "cat < <-count=1> <./...>' + The contents of file "${COMMAND_LOG}" should include 'go <-count=1> <-timeout> <75m> <./...>' The contents of file "${COMMAND_LOG}" should include 'go <.> <--parallel> <60> <--suite-timeout> <75m> <--retries> <2> <--log-dir> <--junit-file>' The contents of file "${COMMAND_LOG}" should include '<--output> ' End @@ -123,7 +123,7 @@ EOF bash "${ROOT_DIR}/.pipelines/scripts/e2e_run.sh" The status should be failure The output should be present - The contents of file "${COMMAND_LOG}" should include 'go <-count=1> <./...>' + The contents of file "${COMMAND_LOG}" should include 'go <-count=1> <-timeout> <80m> <./...>' The contents of file "${COMMAND_LOG}" should not include 'go ' End End diff --git a/spec/packer_spec.sh b/spec/packer_spec.sh new file mode 100644 index 00000000000..f3f25ce9022 --- /dev/null +++ b/spec/packer_spec.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +Describe 'build-cosi-uploader' + setup() { + export MOCK_GO_VERSION=1.27 + export MOCK_SYSTEM_CRYPTO=1 + export MOCK_BUILD_STATUS=0 + + go() { + case "$1 $2" in + 'env GOEXPERIMENT') + [ "$MOCK_GO_VERSION" = 1.26 ] + ;; + 'build -o') + printf 'GOEXPERIMENT=%s CGO_ENABLED=%s\n' "${GOEXPERIMENT-}" "${CGO_ENABLED-}" + return "$MOCK_BUILD_STATUS" + ;; + 'version -m') + printf '\tbuild\tmicrosoft_systemcrypto=%s\n' "$MOCK_SYSTEM_CRYPTO" + ;; + *) return 1 ;; + esac + } + export -f go + } + BeforeEach 'setup' + + It 'uses the OpenSSL experiment on Go 1.26' + MOCK_GO_VERSION=1.26 + When run make --no-print-directory -f packer.mk SHELL=/bin/bash build-cosi-uploader + The status should be success + The output should include 'GOEXPERIMENT=ms_nocgo_opensslcrypto CGO_ENABLED=0' + End + + It 'uses default system crypto on Go 1.27' + When run make --no-print-directory -f packer.mk SHELL=/bin/bash build-cosi-uploader + The status should be success + The output should include 'GOEXPERIMENT= CGO_ENABLED=0' + End + + It 'rejects a binary without system crypto' + MOCK_SYSTEM_CRYPTO=0 + When run make --no-print-directory -f packer.mk SHELL=/bin/bash build-cosi-uploader + The status should be failure + The output should include 'Building cosi-uploader binary' + The error should include 'cosi-uploader must be built with Microsoft Go system crypto' + End + + It 'propagates a compiler failure' + MOCK_BUILD_STATUS=17 + When run make --no-print-directory -f packer.mk SHELL=/bin/bash build-cosi-uploader + The status should be failure + The output should include 'Building cosi-uploader binary' + The error should include 'Error 17' + End +End \ No newline at end of file diff --git a/spec/parts/linux/cloud-init/artifacts/aks-networkd-recovery_spec.sh b/spec/parts/linux/cloud-init/artifacts/aks-networkd-recovery_spec.sh new file mode 100644 index 00000000000..e0bae20ac2f --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/aks-networkd-recovery_spec.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +Describe 'Azure Linux networkd recovery' + setup() { + Include ./parts/linux/cloud-init/artifacts/mariner/aks-networkd-recovery.sh + TEST_DIR="${SHELLSPEC_TMPBASE}/networkd-recovery" + mkdir -p "$TEST_DIR" + TEST_STATE='s "failed"' + TEST_IFINDEX=2 + TEST_QUERY_STATUS=0 + TEST_RECOVERY_STATUS=0 + } + cleanup() { rm -rf "${TEST_DIR}"; } + BeforeEach 'setup' + AfterEach 'cleanup' + + cat() { + if [ "$1" = /sys/class/net/eth0/ifindex ]; then + [ -n "$TEST_IFINDEX" ] || return 1 + printf '%s\n' "$TEST_IFINDEX" + else + command cat "$@" + fi + } + busctl() { + printf '%s\n' "$*" > "$TEST_DIR/query" + [ "$TEST_QUERY_STATUS" -eq 0 ] || return "$TEST_QUERY_STATUS" + printf '%s\n' "$TEST_STATE" + } + timeout() { + printf '%s\n' "$*" > "$TEST_DIR/reconfigure" + return "$TEST_RECOVERY_STATUS" + } + + Describe 'non-failed states' + Parameters + pending + initialized + configuring + configured + unmanaged + linger + End + It "leaves a $1 interface alone" + TEST_STATE="s \"$1\"" + When call recover_networkd_link + The status should be success + The output should be blank + The path "$TEST_DIR/reconfigure" should not be exist + End + End + + It 'immediately reconfigures only the failed interface' + When call recover_networkd_link + The status should be success + The output should include 'eth0 is failed' + The output should include 'connectivity recovery is not yet confirmed' + The contents of file "$TEST_DIR/reconfigure" should equal '20s networkctl reconfigure 2' + The contents of file "$TEST_DIR/query" should equal '--system --auto-start=no --timeout=5s get-property org.freedesktop.network1 /org/freedesktop/network1/link/_2 org.freedesktop.network1.Link AdministrativeState' + End + + It 'reports a missing eth0 without attempting recovery' + TEST_IFINDEX='' + When call recover_networkd_link + The status should be success + The stderr should include 'Cannot find eth0' + The path "$TEST_DIR/reconfigure" should not be exist + End + + It 'does not treat a D-Bus error as a failed interface' + TEST_QUERY_STATUS=1 + When call recover_networkd_link + The status should be success + The stderr should include 'Cannot query eth0 administrative state' + The path "$TEST_DIR/reconfigure" should not be exist + End + + It 'rejects unexpected administrative state output' + TEST_STATE='unexpected' + When call recover_networkd_link + The status should be success + The stderr should include 'Unexpected eth0 administrative state' + The path "$TEST_DIR/reconfigure" should not be exist + End + + Describe 'unsuccessful recovery requests' + Parameters + 1 + 124 + End + It "reports a failed or timed-out request (exit $1)" + TEST_RECOVERY_STATUS="$1" + When call recover_networkd_link + The status should be success + The output should include 'requesting networkd reconfiguration' + The stderr should include 'failed or timed out' + The output should not include 'eth0 reconfiguration requested' + End + End +End diff --git a/spec/parts/linux/cloud-init/artifacts/cgroup_telemetry_spec.sh b/spec/parts/linux/cloud-init/artifacts/cgroup_telemetry_spec.sh index 3a71845d50a..90506d7ee6f 100644 --- a/spec/parts/linux/cloud-init/artifacts/cgroup_telemetry_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cgroup_telemetry_spec.sh @@ -62,6 +62,7 @@ Describe 'cgroup telemetry' prepare_pressure_script() { sed \ + -e "s|/opt/scripts/cgroup-telemetry-events.sh|${PWD}/parts/linux/cloud-init/artifacts/cgroup-telemetry-events.sh|" \ -e "s|EVENTS_LOGGING_DIR=/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events/|EVENTS_LOGGING_DIR=${EVENTS_ROOT}/|" \ -e 's|CGROUP_VERSION=$(stat -fc %T /sys/fs/cgroup)|CGROUP_VERSION=cgroup2fs|' \ -e "s|CGROUP=\"/sys/fs/cgroup\"|CGROUP=\"${CGROUP_ROOT}\"|" \ @@ -113,6 +114,34 @@ Describe 'cgroup telemetry' ' "${EVENTS_ROOT}"/*.json } + freeze_event_timestamp() { + mkdir -p "${TEST_ROOT}/bin" + cat > "${TEST_ROOT}/bin/date" <<'STUB' +#!/bin/bash +if [ "$1" = "+%s%N" ]; then + echo 1700000000000000000 + exit 0 +fi +exec /bin/date "$@" +STUB + chmod +x "${TEST_ROOT}/bin/date" + } + + emitted_event_task_count() { + jq -s 'map(.TaskName) | unique | length' "${EVENTS_ROOT}"/*.json + } + + emitted_event_file_names_class() { + local path + for path in "${EVENTS_ROOT}"/*; do + if ! printf '%s' "$(basename "${path}")" | grep -Eq '^[0-9]+\.json$'; then + echo "not-collectable:$(basename "${path}")" + return + fi + done + echo collectable + } + BeforeEach 'setup_cgroup_telemetry_test' AfterEach 'cleanup_cgroup_telemetry_test' @@ -250,7 +279,7 @@ Describe 'cgroup telemetry' It 'enforces exact message and serialized event byte limits with ASCII and Unicode' check_pressure_size_boundaries() { local fits_filter - fits_filter=$(sed -n '/^ def fits:/p' ./parts/linux/cloud-init/artifacts/cgroup-pressure-telemetry.sh) + fits_filter=$(sed -n '/^ def fits:/p' ./parts/linux/cloud-init/artifacts/cgroup-telemetry-events.sh) [ -n "${fits_filter}" ] || return 1 jq -ne "${fits_filter}"' def repeat_text($text; $count): [range(0; $count) | $text] | join(""); @@ -277,6 +306,71 @@ Describe 'cgroup telemetry' The output should equal true End + Describe 'shared cgroup event publication' + Parameters + Pressure + CPUUsage + Memory + End + + It 'splits and reconstructs the requested object without losing its entries' + publish_and_validate() { + local field="$1" message + # shellcheck source=./parts/linux/cloud-init/artifacts/cgroup-telemetry-events.sh + source ./parts/linux/cloud-init/artifacts/cgroup-telemetry-events.sh + message=$(jq -cn --arg field "${field}" ' + {CgroupVersion: "cgroupv2", ($field): + ([range(0; 20) | {key: ("metric" + tostring), value: ([range(0; 200) | "x"] | join(""))}] | from_entries)}') + publishCgroupTelemetry "${EVENTS_ROOT}" 123 start end test 4 "${message}" "${field}" test || return 1 + jq -se --arg field "${field}" ' + length as $count | + length > 1 and + ([.[].Message | fromjson | .[$field]] | add | length == 20) and + ([.[].Message | fromjson | .TelemetryChunk.index] | sort == [range(0; length)]) and + all(.[]; (.Message | fromjson | .TelemetryChunk.count) == $count) + ' "${EVENTS_ROOT}"/*.json + } + + When call publish_and_validate "$1" + The status should be success + The output should equal true + End + + It 'keeps small messages unchanged without chunk metadata' + publish_small() { + local field="$1" message + # shellcheck source=./parts/linux/cloud-init/artifacts/cgroup-telemetry-events.sh + source ./parts/linux/cloud-init/artifacts/cgroup-telemetry-events.sh + message=$(jq -cn --arg field "${field}" '{CgroupVersion: "cgroupv2", ($field): {metric: "value"}}') + publishCgroupTelemetry "${EVENTS_ROOT}" 123 start end test 4 "${message}" "${field}" test || return 1 + jq -se --arg message "${message}" 'length == 1 and .[0].Message == $message and + (.[0].Message | fromjson | has("TelemetryChunk") | not)' "${EVENTS_ROOT}"/*.json + } + + When call publish_small "$1" + The status should be success + The output should equal true + End + + It 'rejects oversized individual entries before publishing any chunks' + publish_oversized() { + local field="$1" message + # shellcheck source=./parts/linux/cloud-init/artifacts/cgroup-telemetry-events.sh + source ./parts/linux/cloud-init/artifacts/cgroup-telemetry-events.sh + message=$(jq -cn --arg field "${field}" '{($field): {metric: ([range(0; 4000) | "x"] | join(""))}}') + publishCgroupTelemetry "${EVENTS_ROOT}" 123 start end test 4 "${message}" "${field}" test + } + remaining_event_files() { + find "${EVENTS_ROOT}" -mindepth 1 -print + } + + When call publish_oversized "$1" + The status should be failure + The stderr should include 'test telemetry: failed to bound event payload' + The result of function remaining_event_files should equal "" + End + End + Context 'when jq succeeds without emitting events' Mock jq exit 0 @@ -476,4 +570,29 @@ Describe 'cgroup telemetry' The contents of file "${EVENTS_ROOT}"/* should include '\"walinuxagent_service_memory\":\"Not Found\"' End End + + It 'gives collectors distinct event file names within the same millisecond' + for cgroup_path in \ + . \ + system.slice \ + azure.slice \ + kubepods.slice \ + user.slice \ + system.slice/containerd.service \ + system.slice/kubelet.service; do + create_memory_stat "${cgroup_path}" 1 2 + create_pressure_files "${cgroup_path}" + done + create_cpu_stat_v2 system.slice/containerd.service + create_cpu_stat_v2 system.slice/kubelet.service + prepare_memory_script + prepare_pressure_script + prepare_cpu_script + freeze_event_timestamp + + When run env PATH="${TEST_ROOT}/bin:${PATH}" bash -c 'bash "$1" && bash "$2" && bash "$3"' _ "${TEST_ROOT}/cgroup-memory-telemetry.sh" "${TEST_ROOT}/cgroup-pressure-telemetry.sh" "${TEST_ROOT}/cgroup-cpu-telemetry.sh" + The status should be success + The result of function emitted_event_task_count should equal 3 + The result of function emitted_event_file_names_class should equal collectable + End End diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh index 2f8b394df05..808ede5508c 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_gpu_spec.sh @@ -5,7 +5,6 @@ Describe 'cse_config_gpu.sh' CSE_CONFIG_LOCALDNS_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_localdns.sh" CSE_CONFIG_KUBELET_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_kubelet.sh" CSE_CONFIG_NETWORK_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_network.sh" - CSE_CONFIG_ADDONS_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_addons.sh" Include "./parts/linux/cloud-init/artifacts/cse_config.sh" Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" Describe 'GPU driver dispatch' @@ -130,8 +129,8 @@ Describe 'cse_config_gpu.sh' End End Describe 'ensureArtifactStreaming' - # ensureArtifactStreaming enables the acr-mirror/overlaybd services and then - # runs the version-appropriate enablement path: + # ensureArtifactStreaming runs the version-appropriate enablement path + # before enabling the acr-mirror/overlaybd services: # - acr-mirror 1.0.0+ -> setup.sh aks # - older packages -> acr-config --enable-containerd # The enablement binary paths are overridable (ACR_MIRROR_SETUP_SCRIPT / @@ -148,7 +147,7 @@ Describe 'cse_config_gpu.sh' AfterEach 'cleanup_streaming' waitForContainerdReady() { - return 0 + return "${CONTAINERD_READY_RC:-0}" } systemctl() { echo "systemctl $@" @@ -159,18 +158,19 @@ Describe 'cse_config_gpu.sh' } install_setup_sh() { - printf '#!/bin/sh\necho "setup.sh $@"\n' > "${ACR_MIRROR_SETUP_SCRIPT}" + printf '#!/bin/sh\necho "setup.sh $@"\nexit %s\n' "${SETUP_RC:-0}" > "${ACR_MIRROR_SETUP_SCRIPT}" chmod +x "${ACR_MIRROR_SETUP_SCRIPT}" } install_acr_config() { - printf '#!/bin/sh\necho "acr-config $@"\n' > "${ACR_CONFIG_BIN}" + printf '#!/bin/sh\necho "acr-config $@"\nexit %s\n' "${SETUP_RC:-0}" > "${ACR_CONFIG_BIN}" chmod +x "${ACR_CONFIG_BIN}" } It 'uses setup.sh aks when acr-mirror 1.0.0+ is installed' install_setup_sh When run ensureArtifactStreaming - The output should include "setup.sh aks" + The line 1 of output should equal "setup.sh aks" + The line 2 of output should equal "retrycmd_if_failure 120 5 25 systemctl --quiet enable --now acr-mirror overlaybd-tcmu overlaybd-snapshotter" The output should not include "Older acr-mirror package" The status should be success End @@ -179,7 +179,8 @@ Describe 'cse_config_gpu.sh' install_acr_config When run ensureArtifactStreaming The output should include "Older acr-mirror package is detected" - The output should include "acr-config --enable-containerd azurecr.io" + The line 2 of output should equal "acr-config --enable-containerd azurecr.io" + The line 3 of output should equal "retrycmd_if_failure 120 5 25 systemctl --quiet enable --now acr-mirror overlaybd-tcmu overlaybd-snapshotter" The status should be success End @@ -188,7 +189,32 @@ Describe 'cse_config_gpu.sh' install_setup_sh When run ensureArtifactStreaming The output should include "retrycmd_if_failure" - The output should not include "setup.sh aks" + The line 1 of output should equal "setup.sh aks" + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + End + + It 'does not configure or start streaming when containerd is not ready' + CONTAINERD_READY_RC=1 + install_setup_sh + When run ensureArtifactStreaming + The output should be blank + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + End + + It 'does not start streaming services when setup.sh fails' + SETUP_RC=1 + install_setup_sh + When run ensureArtifactStreaming + The output should equal "setup.sh aks" + The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" + End + + It 'does not start streaming services when legacy acr-config fails' + SETUP_RC=1 + install_acr_config + When run ensureArtifactStreaming + The output should include "acr-config --enable-containerd azurecr.io" + The output should not include "retrycmd_if_failure" The status should equal "$ERR_ARTIFACT_STREAMING_INSTALL" End End @@ -714,6 +740,9 @@ Describe 'cse_config_gpu.sh' setup_cdi_dropin() { CDI_TEST_DIR=$(mktemp -d) NVIDIA_CDI_REFRESH_DROP_IN="${CDI_TEST_DIR}/nvidia-cdi-refresh.service.d/10-aks-tolerate-generate-failure.conf" + # The timeout override is a separate, higher-sorted drop-in so it beats the toolkit's own + # 10-container-engines.conf (TimeoutStartSec=90s) in systemd's lexical drop-in merge. + NVIDIA_CDI_REFRESH_TIMEOUT_DROP_IN="${CDI_TEST_DIR}/nvidia-cdi-refresh.service.d/90-aks-cdi-refresh-timeout.conf" } cleanup_cdi_dropin() { rm -rf "${CDI_TEST_DIR}" @@ -722,6 +751,10 @@ Describe 'cse_config_gpu.sh' AfterEach 'cleanup_cdi_dropin' Describe 'configureNvidiaCDIRefresh' + # The TimeoutStartSec widening is arm64 (Grace-Blackwell) only; default the mock to arm64 + # so the shared drop-in assertions below exercise the GB path, and override per-test. + getCPUArch() { echo "${MOCK_CPU_ARCH:-arm64}"; } + It 'treats the known nvidia-ctk failures as success and reloads systemd' systemctl() { echo "systemctl $*"; return 0; } @@ -735,6 +768,33 @@ Describe 'cse_config_gpu.sh' The contents of file "${NVIDIA_CDI_REFRESH_DROP_IN}" should include "SuccessExitStatus=1 2 127" End + It 'writes the timeout override to a 90- drop-in on arm64 (must out-sort toolkit 10-container-engines.conf)' + MOCK_CPU_ARCH="arm64" + systemctl() { return 0; } + + When call configureNvidiaCDIRefresh + + The status should be success + # The 90- prefix is load-bearing: systemd merges drop-ins in lexical order (last wins), + # and the toolkit's own 10-container-engines.conf pins TimeoutStartSec=90s. A 10-aks-* + # name sorts before it and loses, so the override must sort after it. + The path "${NVIDIA_CDI_REFRESH_TIMEOUT_DROP_IN}" should be file + The contents of file "${NVIDIA_CDI_REFRESH_TIMEOUT_DROP_IN}" should include "TimeoutStartSec=300" + # the tolerate-failure drop-in stays free of the timeout (separation keeps ordering clear) + The contents of file "${NVIDIA_CDI_REFRESH_DROP_IN}" should not include "TimeoutStartSec" + End + + It 'does not write the timeout drop-in on x86 (generates in <1s, default timeout is fine)' + MOCK_CPU_ARCH="amd64" + systemctl() { return 0; } + + When call configureNvidiaCDIRefresh + + The status should be success + The contents of file "${NVIDIA_CDI_REFRESH_DROP_IN}" should include "SuccessExitStatus=1 2 127" + The path "${NVIDIA_CDI_REFRESH_TIMEOUT_DROP_IN}" should not be exist + End + It 'keeps the units restartable so the .path trigger can still refresh the spec' systemctl() { return 0; } diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_kubelet_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_kubelet_spec.sh index ec08df4e898..1c981bf4a7d 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_kubelet_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_kubelet_spec.sh @@ -5,7 +5,6 @@ Describe 'cse_config_kubelet.sh' CSE_CONFIG_LOCALDNS_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_localdns.sh" CSE_CONFIG_KUBELET_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_kubelet.sh" CSE_CONFIG_NETWORK_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_network.sh" - CSE_CONFIG_ADDONS_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_addons.sh" Include "./parts/linux/cloud-init/artifacts/cse_config.sh" Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" Describe 'configureKubeletServing' diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_localdns_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_localdns_spec.sh index c57a5fe7b70..16926ab44dc 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_localdns_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_localdns_spec.sh @@ -18,7 +18,6 @@ Describe 'cse_config_localdns.sh' CSE_CONFIG_LOCALDNS_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_localdns.sh" CSE_CONFIG_KUBELET_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_kubelet.sh" CSE_CONFIG_NETWORK_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_network.sh" - CSE_CONFIG_ADDONS_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_addons.sh" Include "./parts/linux/cloud-init/artifacts/cse_config.sh" Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" Describe 'enableLocalDNS' diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_network_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_network_spec.sh index 28e8259b36f..da71a5914b2 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_network_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_network_spec.sh @@ -5,9 +5,80 @@ Describe 'cse_config_network.sh' CSE_CONFIG_LOCALDNS_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_localdns.sh" CSE_CONFIG_KUBELET_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_kubelet.sh" CSE_CONFIG_NETWORK_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_network.sh" - CSE_CONFIG_ADDONS_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_addons.sh" Include "./parts/linux/cloud-init/artifacts/cse_config.sh" Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" + Describe 'configPrivateClusterHosts' + function [ { + case "$*" in + "! -f /etc/systemd/system/reconcile-private-hosts.service ]") + test "$private_hosts_service_available" != true ;; + *) command [ "$@" ;; + esac + } + + mkdir() { echo "mkdir $*"; } + touch() { echo "touch $*"; } + tee() { cat; } + systemctlEnableAndStart() { echo "systemctlEnableAndStart $*"; } + + It 'configures and starts the baked service when available' + private_hosts_service_available=true + API_SERVER_NAME="api.example.com" + When call configPrivateClusterHosts + The status should be success + The output should include 'mkdir -p /etc/systemd/system/reconcile-private-hosts.service.d/' + The output should include 'touch /etc/systemd/system/reconcile-private-hosts.service.d/10-fqdn.conf' + The output should include 'systemctlEnableAndStart reconcile-private-hosts 30' + The output should not include 'Skipping' + End + + It 'skips reconciliation when the baked service is absent' + private_hosts_service_available=false + When call configPrivateClusterHosts + The status should be success + The output should equal 'Skipping private hosts reconciliation: service is absent' + End + End + Describe 'ensureAzureNetworkConfig' + function [ { + case "$*" in + "! -x /opt/azure-network/configure-azure-network.sh ]") + test "$network_script_available" != true ;; + "! -f /etc/udev/rules.d/99-azure-network.rules ]") + test "$network_rule_available" != true ;; + *) command [ "$@" ;; + esac + } + + udevadm() { + echo "udevadm $*" + } + + It 'reloads and triggers udev when both baked artifacts are available' + network_script_available=true + network_rule_available=true + When call ensureAzureNetworkConfig + The status should be success + The output should include 'udevadm control --reload-rules' + The output should include 'udevadm trigger --subsystem-match=net --action=add' + The output should include 'udevadm settle --timeout=10' + The output should not include 'Skipping' + End + + Parameters + false true + true false + false false + End + + It 'skips tuning when either baked artifact is unavailable' + network_script_available="$1" + network_rule_available="$2" + When call ensureAzureNetworkConfig + The status should be success + The output should equal 'Skipping Azure NIC tuning: image does not contain required files' + End + End Describe 'getPrimaryNicIP' It 'should return the correct IP when a single network interface is attached to the VM' IMDS_INSTANCE_METADATA_CACHE_FILE="spec/parts/linux/cloud-init/artifacts/imds_mocks/network/single_nic.json" diff --git a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh index a1a11a97a09..0749177aa03 100755 --- a/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_config_spec.sh @@ -9,7 +9,6 @@ Describe 'cse_config.sh' CSE_CONFIG_LOCALDNS_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_localdns.sh" CSE_CONFIG_KUBELET_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_kubelet.sh" CSE_CONFIG_NETWORK_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_network.sh" - CSE_CONFIG_ADDONS_FILEPATH="./parts/linux/cloud-init/artifacts/cse_config_addons.sh" Include "./parts/linux/cloud-init/artifacts/cse_config.sh" Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" Describe 'configureTransparentHugePageSystemdService' @@ -1071,4 +1070,76 @@ PubkeyAuthentication no" The output should include "ensureSwapFileFstabEntry /swapfile" End End + Describe 'configureK8s client key permissions' + # Use a throwaway temp dir rather than the real /etc/kubernetes/certs path: when + # ShellSpec is installed natively (spec/README.md's "Option 2") instead of via the + # recommended container, tests run directly against the host filesystem, and this + # test would otherwise delete/overwrite a real node's kubelet credential. + setup_client_key() { + CLIENT_KEY_TEST_DIR="$(mktemp -d "${SHELLSPEC_TMPBASE:-${TMPDIR:-/tmp}}/client-key.XXXXXX")" + CLIENT_KEY_PATH="${CLIENT_KEY_TEST_DIR}/client.key" + APISERVER_PUBLIC_KEY="" + ENABLE_SECURE_TLS_BOOTSTRAPPING="false" + unset TLS_BOOTSTRAP_TOKEN + KUBELET_CLIENT_CONTENT="Y2xpZW50LWtleQ==" + KUBELET_CLIENT_CERT_CONTENT="" + KUBELET_CONFIG_FILE_ENABLED="false" + } + + cleanup_client_key() { + set +x + command rm -rf "${CLIENT_KEY_TEST_DIR}" + } + + invoke_configureK8s() { + { + configureK8s + local status=$? + set +x + return "${status}" + } 2>/dev/null + } + + BeforeEach 'setup_client_key' + AfterEach 'cleanup_client_key' + + It 'writes the kubelet client private key with mode 0600' + When call invoke_configureK8s + + The status should be success + The contents of file "${CLIENT_KEY_PATH}" should equal 'client-key' + The value "$(stat -c '%a' "${CLIENT_KEY_PATH}")" should equal '600' + End + + It 'creates the client key as 0600 at creation time, not only after a later chmod' + # Stub chmod as a no-op so the only thing that can have set a restrictive mode + # is the umask-scoped touch itself; this proves there is no world-readable window. + chmod() { :; } + + When call invoke_configureK8s + + The status should be success + The value "$(stat -c '%a' "${CLIENT_KEY_PATH}")" should equal '600' + End + + It 'does not write the private key if chmod fails' + chmod() { return 1; } + + When call invoke_configureK8s + + The status should equal "$ERR_KUBELET_CLIENT_KEY_PERM_FAIL" + # touch still ran before the guarded chmod, so an empty file may exist; + # what matters is the secret content was never written into it. + The contents of file "${CLIENT_KEY_PATH}" should not equal 'client-key' + End + + It 'does not write the private key if touch fails' + touch() { return 1; } + + When call invoke_configureK8s + + The status should equal "$ERR_KUBELET_CLIENT_KEY_PERM_FAIL" + The path "${CLIENT_KEY_PATH}" should not be exist + End + End End diff --git a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh index 905f47c78cf..0b77dd845d2 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_helpers_spec.sh @@ -15,8 +15,9 @@ readContainerImage() { echo "$containerImage" } -Describe 'cse_helpers.sh' +Describe 'cse_helpers.sh and build helpers' Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" + Include "./vhdbuilder/scripts/linux/tool_installs.sh" Describe 'isAzureLinuxArm64BaseImage' Parameters diff --git a/spec/parts/linux/cloud-init/artifacts/cse_install_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_install_spec.sh index 3095fd7f1c9..139157295d4 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_install_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_install_spec.sh @@ -13,45 +13,6 @@ Describe 'cse_install.sh' Include "./parts/linux/cloud-init/artifacts/cse_install.sh" Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" - Describe 'pullContainerImage' - retrycmd_if_failure() { - printf 'retry argument: %s\n' "$@" - return "${MOCK_PULL_STATUS:-0}" - } - - It 'uses cache-image with the existing retry policy for containerd' - When call pullContainerImage ctr 'registry.example/image:v1' - The status should be success - The output should include 'retry argument: 10' - The output should include 'retry argument: 1' - The output should include 'retry argument: 600' - The output should include 'retry argument: /opt/azure/containers/image-fetcher' - The output should include 'retry argument: cache-image' - The output should include 'retry argument: --image' - The output should include 'retry argument: registry.example/image:v1' - End - - It 'propagates image-fetcher failures' - MOCK_PULL_STATUS=7 - When call pullContainerImage ctr 'registry.example/image:v1' - The status should equal 7 - The output should include 'failed to pull image registry.example/image:v1 using ctr, exit code: 7' - End - - Parameters - crictl - docker - End - - It "preserves the $1 pull command" - When call pullContainerImage "$1" 'registry.example/image:v1' - The status should be success - The output should include "retry argument: $1" - The output should include 'retry argument: pull' - The output should not include 'retry argument: cache-image' - End - End - Describe 'installContainerRuntime' logs_to_events() { echo "mock logs to events calling with $1" diff --git a/spec/parts/linux/cloud-init/artifacts/cse_main_disable_modules_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_main_disable_modules_spec.sh index 6eaeb94d114..47e94677869 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_main_disable_modules_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_main_disable_modules_spec.sh @@ -290,7 +290,7 @@ Describe 'Ubuntu 20.04 Azure FIPS mitigation policy consistency' fi for source_file in \ - vhdbuilder/packer/packer_source.sh \ + vhdbuilder/packer/install-dependencies.sh \ vhdbuilder/packer/cleanup-vhd.sh \ vhdbuilder/packer/test/linux-vhd-content-test.sh; do eval "$(sed -n '/^kernelVersionGe()/,/^}/p' "$source_file")" diff --git a/spec/parts/linux/cloud-init/artifacts/cse_retry_helpers_spec.sh b/spec/parts/linux/cloud-init/artifacts/cse_retry_helpers_spec.sh index 882b394ed4f..41e1eac22ec 100644 --- a/spec/parts/linux/cloud-init/artifacts/cse_retry_helpers_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/cse_retry_helpers_spec.sh @@ -29,7 +29,6 @@ Describe 'long running cse helper functions' "systemctl_stop" 1 1 1 1 "nonexistent.service" "systemctl_disable" 1 1 1 1 "nonexistent.service" "_systemctl_retry_svc_operation" 1 1 1 1 "nonexistent.service" "restart" - "sysctl_reload" 1 1 1 1 "retrycmd_get_aad_access_token" $ERR_ORAS_IMDS_TIMEOUT 1 1 "http://nonexistent.local/token" "retrycmd_get_refresh_token_for_oras" $ERR_ORAS_PULL_NETWORK_TIMEOUT 1 1 "dummy.registry" "tenant-id" "fake-token" "retrycmd_curl_file" 1 1 1 1 "/tmp/nonexistent" "https://dummy.url/file" diff --git a/spec/parts/linux/cloud-init/artifacts/event_file_naming_spec.sh b/spec/parts/linux/cloud-init/artifacts/event_file_naming_spec.sh new file mode 100644 index 00000000000..a43ee91be27 --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/event_file_naming_spec.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +Describe 'WALinuxAgent event file naming' + ARTIFACTS_DIR="./parts/linux/cloud-init/artifacts" + COMPLIANT_NAME='="\$\(date \+%s%N\)\$\{BASHPID\}(\.json)?"$' + + emitters() { + grep -rlE --include='*.sh' 'EVENTS_LOGGING_(DIR|PATH)|events_logging_dir' "${ARTIFACTS_DIR}" | sort + } + + emitters_without_compliant_name() { + local file + for file in $(emitters); do + if ! grep -qE "${COMPLIANT_NAME}" "${file}"; then + echo "${file##*/}" + fi + done + echo done + } + + noncompliant_name_expressions() { + grep -rhE --include='*.sh' '(EVENTS_FILE_NAME|eventsFileName|event_file_name|events_file_name)=.*\$\(date \+%s%[0-9]*N\)' "${ARTIFACTS_DIR}" \ + | grep -vE "${COMPLIANT_NAME}" \ + | sed 's/^[[:space:]]*//' + echo done + } + + compliant_name_count() { + grep -rhE --include='*.sh' "${COMPLIANT_NAME}" "${ARTIFACTS_DIR}" | wc -l | tr -d ' ' + } + + It 'builds every emitted event file name from a nanosecond timestamp and the process id' + When call emitters_without_compliant_name + The output should equal done + End + + It 'leaves no event file name built from a coarser or non-unique expression' + When call noncompliant_name_expressions + The output should equal done + End + + It 'covers all known emission sites' + When call compliant_name_count + The output should equal 16 + End + + Describe 'repeated emission from a single process' + setup_events_dir() { + EVENTS_TEST_ROOT=$(mktemp -d) + EVENTS_LOGGING_DIR="${EVENTS_TEST_ROOT}/" + } + + cleanup_events_dir() { + rm -rf "${EVENTS_TEST_ROOT}" + } + + BeforeEach 'setup_events_dir' + AfterEach 'cleanup_events_dir' + + Include "./parts/linux/cloud-init/artifacts/cse_helpers.sh" + + emit_twice_then_count() { + logs_to_events "AKS.Spec.first" true + logs_to_events "AKS.Spec.second" true + find "${EVENTS_TEST_ROOT}" -maxdepth 1 -type f -name '*.json' | wc -l | tr -d ' ' + } + + # Both calls run in the same shell, so they share a BASHPID and only the + # timestamp separates them. This guards that contract end to end; it is + # not a measurement of the resolution actually required. + It 'does not overwrite the previous event when logs_to_events is called back to back' + When call emit_twice_then_count + The output should equal 2 + End + End +End diff --git a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_logging_spec.sh b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_logging_spec.sh index 9c9993034f1..1845878f892 100644 --- a/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_logging_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/init_aks_cloud_logging_spec.sh @@ -167,6 +167,14 @@ Describe 'init-aks-cloud.sh certificate logging' install_certs_to_trust_store } + It 'stops immediately when creating the certificate directory fails' + mkdir() { echo 'mkdir failed' >&2; return 1; } + When call install_with_diagnostics + The status should eq 1 + The stdout should be blank + The stderr should eq 'mkdir failed' + End + It 'logs public PEMs through the actual installation path when opted in' make_certificate mkdir "${TEST_DIR}/certs" diff --git a/spec/parts/linux/cloud-init/artifacts/kata_payload_spec.sh b/spec/parts/linux/cloud-init/artifacts/kata_payload_spec.sh index bfed1242fef..1abe79feeab 100644 --- a/spec/parts/linux/cloud-init/artifacts/kata_payload_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/kata_payload_spec.sh @@ -39,9 +39,6 @@ Describe 'Azure Linux 3 Kata payload copies' esac } - bakeModprobeCISWithoutVulnerableModules() { :; } - configureServiceExecutionTelemetryForSystemdVersion() { :; } - systemctl() { printf 'systemd 255\n'; } signed_payload_is_intact() { local file diff --git a/spec/parts/linux/cloud-init/artifacts/service_execution_telemetry_spec.sh b/spec/parts/linux/cloud-init/artifacts/service_execution_telemetry_spec.sh index 8d30a4e4737..c4f625c2cb1 100644 --- a/spec/parts/linux/cloud-init/artifacts/service_execution_telemetry_spec.sh +++ b/spec/parts/linux/cloud-init/artifacts/service_execution_telemetry_spec.sh @@ -1,7 +1,7 @@ #!/bin/bash Describe 'service execution telemetry' - Include './vhdbuilder/packer/packer_source.sh' + eval "$(sed -n '/^configureServiceExecutionTelemetryForSystemdVersion()/,/^}/p' ./vhdbuilder/packer/install-dependencies.sh)" setup_service_execution_telemetry_test() { TEST_ROOT=$(mktemp -d) @@ -277,7 +277,7 @@ EOF cp ./parts/linux/cloud-init/artifacts/cgroup-pressure-telemetry.service "${CGROUP_PRESSURE_TELEMETRY_SERVICE_DEST}" cp ./parts/linux/cloud-init/artifacts/aks-log-collector.service "${AKS_LOG_COLLECTOR_SERVICE_DEST}" - When call configureServiceExecutionTelemetryForSystemdVersion 245 + When call configureServiceExecutionTelemetryForSystemdVersion 245 "${TEST_ROOT}" The status should be success The contents of file "${CGROUP_MEMORY_TELEMETRY_SERVICE_DEST}" should not include "OnSuccess=" The contents of file "${CGROUP_MEMORY_TELEMETRY_SERVICE_DEST}" should not include "OnFailure=" @@ -294,7 +294,7 @@ EOF cp ./parts/linux/cloud-init/artifacts/cgroup-pressure-telemetry.service "${CGROUP_PRESSURE_TELEMETRY_SERVICE_DEST}" cp ./parts/linux/cloud-init/artifacts/aks-log-collector.service "${AKS_LOG_COLLECTOR_SERVICE_DEST}" - When call configureServiceExecutionTelemetryForSystemdVersion 249 + When call configureServiceExecutionTelemetryForSystemdVersion 249 "${TEST_ROOT}" The status should be success The contents of file "${CGROUP_MEMORY_TELEMETRY_SERVICE_DEST}" should include 'OnSuccess=service-execution-telemetry@cgroup\x2dmemory\x2dtelemetry.service.service' The contents of file "${CGROUP_MEMORY_TELEMETRY_SERVICE_DEST}" should not include "ExecStopPost=-/bin/systemctl" @@ -309,7 +309,7 @@ EOF cp ./parts/linux/cloud-init/artifacts/cgroup-pressure-telemetry.service "${CGROUP_PRESSURE_TELEMETRY_SERVICE_DEST}" cp ./parts/linux/cloud-init/artifacts/aks-log-collector.service "${AKS_LOG_COLLECTOR_SERVICE_DEST}" - When call configureServiceExecutionTelemetryForSystemdVersion 258 + When call configureServiceExecutionTelemetryForSystemdVersion 258 "${TEST_ROOT}" The status should be success The contents of file "${CGROUP_MEMORY_TELEMETRY_SERVICE_DEST}" should not include "CPUAccounting=" The contents of file "${CGROUP_MEMORY_TELEMETRY_SERVICE_DEST}" should not include "MemoryAccounting=" @@ -325,7 +325,7 @@ EOF cp ./parts/linux/cloud-init/artifacts/cgroup-pressure-telemetry.service "${CGROUP_PRESSURE_TELEMETRY_SERVICE_DEST}" cp ./parts/linux/cloud-init/artifacts/aks-log-collector.service "${AKS_LOG_COLLECTOR_SERVICE_DEST}" - When call configureServiceExecutionTelemetryForSystemdVersion 256 + When call configureServiceExecutionTelemetryForSystemdVersion 256 "${TEST_ROOT}" The status should be success The contents of file "${CGROUP_MEMORY_TELEMETRY_SERVICE_DEST}" should not include "--snapshot-memory-peak" The contents of file "${CGROUP_PRESSURE_TELEMETRY_SERVICE_DEST}" should not include "--snapshot-memory-peak" diff --git a/spec/parts/linux/cloud-init/artifacts/setup_custom_search_domains_spec.sh b/spec/parts/linux/cloud-init/artifacts/setup_custom_search_domains_spec.sh new file mode 100644 index 00000000000..33331649900 --- /dev/null +++ b/spec/parts/linux/cloud-init/artifacts/setup_custom_search_domains_spec.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +Describe 'setup-custom-search-domains.sh' + setup() { + TEST_DIR=$(mktemp -d) + mkdir -p "${TEST_DIR}/bin" + cat > "${TEST_DIR}/helpers.sh" <<'EOF' +wait_for_apt_locks() { :; } +systemctl_restart() { :; } +retrycmd_if_failure() { return 0; } +EOF + : > "${TEST_DIR}/distro_helpers.sh" + cat > "${TEST_DIR}/bin/realm" <<'EOF' +#!/bin/sh +cat >/dev/null +exit "${REALM_EXIT_CODE:-0}" +EOF + cat > "${TEST_DIR}/bin/tee" <<'EOF' +#!/bin/sh +cat >/dev/null +EOF + cat > "${TEST_DIR}/bin/run-script" <"${TEST_DIR}/stderr" +EOF + chmod +x "${TEST_DIR}/bin/realm" "${TEST_DIR}/bin/tee" "${TEST_DIR}/bin/run-script" + } + + cleanup() { + rm -rf "${TEST_DIR}" + } + + BeforeEach 'setup' + AfterEach 'cleanup' + + It 'preserves realm join failures without tracing the password' + When run env \ + PATH="${TEST_DIR}/bin:${PATH}" \ + CSE_HELPERS_FILEPATH="${TEST_DIR}/helpers.sh" \ + CSE_DISTRO_HELPERS_FILEPATH="${TEST_DIR}/distro_helpers.sh" \ + CUSTOM_SEARCH_REALM_PASSWORD='realm-password-sentinel' \ + CUSTOM_SEARCH_REALM_USER='test-user' \ + CUSTOM_SEARCH_DOMAIN_NAME='example.com' \ + REALM_EXIT_CODE=37 \ + "${TEST_DIR}/bin/run-script" + + The status should equal 37 + The value "$(grep -F 'realm-password-sentinel' "${TEST_DIR}/stderr")" should equal '' + End +End diff --git a/spec/shellspec.Dockerfile b/spec/shellspec.Dockerfile index a7d9313712e..f478f9cba98 100644 --- a/spec/shellspec.Dockerfile +++ b/spec/shellspec.Dockerfile @@ -30,7 +30,7 @@ FROM aksdataplanedev.azurecr.io/shellspec/shellspec-debian:0.28.1 RUN sed -i -e 's/\(deb\|security\).debian.org/archive.debian.org/g' /etc/apt/sources.list && \ apt-get update && \ apt-get install -y --no-install-recommends \ - gawk jq curl dnsutils openssl \ + gawk jq curl dnsutils make openssl \ libcurl4 libdw1 libelf1 libssl1.1 zlib1g && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/spec/vhdbuilder/packer/acl_unsigned_secure_boot_spec.sh b/spec/vhdbuilder/packer/acl_unsigned_secure_boot_spec.sh new file mode 100644 index 00000000000..70026af5237 --- /dev/null +++ b/spec/vhdbuilder/packer/acl_unsigned_secure_boot_spec.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# shellcheck disable=SC2016,SC2034 + +Describe 'Trusted Launch Secure Boot for unsigned ACL base images' + run_test_options() ( + set -eu + ENABLE_TRUSTED_LAUNCH=True TARGET_COMMAND_STRING="--size Standard_D2pds_v5" + eval "$(sed -n '/^if \[ "${ENABLE_TRUSTED_LAUNCH,,}" = "true" \]; then$/,/^fi$/p' vhdbuilder/packer/test/run-test.sh)" + printf '%s\n' "$TARGET_COMMAND_STRING" + ) + vhd_scanning_options() ( + set -eu + OS_TYPE=Linux ENABLE_TRUSTED_LAUNCH=True VM_OPTIONS="--size Standard_D8pds_v6" + eval "$(sed -n '/^if \[ "${OS_TYPE}" = "Linux" \] && \[ "${ENABLE_TRUSTED_LAUNCH,,}" = "true" \]; then$/,/^fi$/p' vhdbuilder/packer/vhd-scanning.sh)" + printf '%s\n' "$VM_OPTIONS" + ) + + Parameters + run_test_options unset 'enable-secure-boot true' + run_test_options true 'enable-secure-boot true' + run_test_options false 'enable-secure-boot false' + vhd_scanning_options unset 'enable-secure-boot true' + vhd_scanning_options true 'enable-secure-boot true' + vhd_scanning_options false 'enable-secure-boot false' + End + It "$1 with ACL_BASE_IMAGE_SIGNED=$2 uses --$3" + if [ "$2" = unset ]; then + unset ACL_BASE_IMAGE_SIGNED + else + export ACL_BASE_IMAGE_SIGNED="$2" + fi + When run "$1" + The status should be success + The output should include "--security-type TrustedLaunch --$3 --enable-vtpm true" + End +End diff --git a/spec/vhdbuilder/packer/ensure_sig_trusted_launch_supported_spec.sh b/spec/vhdbuilder/packer/ensure_sig_trusted_launch_supported_spec.sh new file mode 100644 index 00000000000..403f77f76da --- /dev/null +++ b/spec/vhdbuilder/packer/ensure_sig_trusted_launch_supported_spec.sh @@ -0,0 +1,291 @@ +#!/bin/bash +# shellcheck disable=SC2329 + +Describe 'ensure_sig_trusted_launch_supported' + Include './vhdbuilder/packer/produce-packer-settings-functions.sh' + + setup() { + TEST_DIR="${SHELLSPEC_WORKDIR}/trusted-launch.${SHELLSPEC_EXAMPLE_NO}" + mkdir -p "${TEST_DIR}" + AZ_CALLS="${TEST_DIR}/az-calls" + UPDATE_FEATURES="${TEST_DIR}/update-features" + : > "${AZ_CALLS}" + AZURE_RESOURCE_GROUP_NAME="test-rg" + SIG_GALLERY_NAME="test-gallery" + SIG_IMAGE_NAME="windows-2025-gen2" + SIG_IMAGE_VERSION="26100.1.261006" + DEFINITION_ID="/subscriptions/test-sub/resourceGroups/test-rg/providers/Microsoft.Compute/galleries/test-gallery/images/windows-2025-gen2" + INITIAL_DEFINITION='{"id":"definition-id","name":"windows-2025-gen2","type":"Microsoft.Compute/galleries/images","location":"eastus","tags":{"preserve":"true"},"properties":{"hyperVGeneration":"V2","provisioningState":"Succeeded","features":[{"name":"DiskControllerTypes","value":"SCSI,NVMe"}]}}' + UPDATED_DEFINITION='{"properties":{"provisioningState":"Succeeded","features":[{"name":"SecurityType","value":"TrustedLaunchSupported","startsAtVersion":"26100.1.261006"}]}}' + FAIL_OPERATION="" + } + BeforeEach 'setup' + + az() { + printf '%s\n' "$*" >> "${AZ_CALLS}" + case "$1 $2 $3" in + 'resource show --ids') + if [ "${FAIL_OPERATION}" = "show" ]; then return 1; fi + if [ -f "${TEST_DIR}/created-definition" ]; then + cat "${TEST_DIR}/created-definition" + else + printf '%s\n' "${INITIAL_DEFINITION}" + fi + ;; + 'resource update --ids') + if [ "${FAIL_OPERATION}" = "update" ]; then return 1; fi + while [ "$#" -gt 0 ]; do + case "$1" in + properties.features=*) + printf '%s\n' "${1#properties.features=}" > "${UPDATE_FEATURES}" + break + ;; + esac + shift + done + printf '%s\n' "${UPDATED_DEFINITION}" + ;; + 'sig show --resource-group') + printf '%s\n' '{"provisioningState":"Succeeded"}' + ;; + 'sig image-definition show') + if [ ! -f "${TEST_DIR}/created-definition" ]; then return 1; fi + printf '{"id":"%s"}\n' "${DEFINITION_ID}" + ;; + 'sig image-definition create') + if [ "${FAIL_OPERATION}" = "create" ]; then return 23; fi + local security_type="Standard" + case "$*" in *SecurityType=TrustedLaunchSupported*) security_type="TrustedLaunchSupported" ;; esac + jq --arg security_type "${security_type}" ' + .properties.features += [{name: "SecurityType", value: $security_type}] + ' <<<"${INITIAL_DEFINITION}" > "${TEST_DIR}/created-definition" + ;; + *) return 1 ;; + esac + } + + It 'adds versioned capability while preserving the existing definition' + When call ensure_sig_trusted_launch_supported "${DEFINITION_ID}" + The status should be success + The output should include 'from version 26100.1.261006' + The contents of file "${AZ_CALLS}" should include "resource show --ids ${DEFINITION_ID} --api-version 2024-03-03" + The contents of file "${AZ_CALLS}" should include "resource update --ids ${DEFINITION_ID} --api-version 2024-03-03 --set properties.allowUpdateImage=true properties.features=" + The contents of file "${AZ_CALLS}" should include '--remove properties.provisioningState' + The contents of file "${AZ_CALLS}" should not include '--no-wait' + The contents of file "${AZ_CALLS}" should not include 'image-definition wait' + The value "$(grep -c '^resource show ' "${AZ_CALLS}")" should equal 1 + The value "$(jq -c '.' "${UPDATE_FEATURES}")" should equal '[{"name":"DiskControllerTypes","value":"SCSI,NVMe"},{"name":"SecurityType","value":"TrustedLaunchSupported","startsAtVersion":"26100.1.261006"}]' + End + + It 'creates a new shared Windows definition capable directly without a migration' + MODE="windowsVhdMode" + ARCHITECTURE="x64" + FEATURE_FLAGS="" + HYPERV_GENERATION="V2" + OS_TYPE="Windows" + AZURE_LOCATION="eastus" + ENABLE_TRUSTED_LAUNCH="False" + TRUSTED_LAUNCH_SUPPORTED="True" + + When call ensure_sig_vhd_exists + The status should be success + The output should include 'Creating image definition windows-2025-gen2' + The output should not include 'Updating image definition' + The stderr should equal '' + The contents of file "${AZ_CALLS}" should include 'DiskControllerTypes=SCSI,NVMe SecurityType=TrustedLaunchSupported' + The contents of file "${AZ_CALLS}" should not include 'resource update' + The contents of file "${AZ_CALLS}" should not include 'image-definition wait' + The value "$(grep -c '^sig image-definition create ' "${AZ_CALLS}")" should equal 1 + The value "$(grep -c '^sig image-definition show ' "${AZ_CALLS}")" should equal 1 + The file "${UPDATE_FEATURES}" should not be exist + The value "$(jq -c '.properties.features' "${TEST_DIR}/created-definition")" should equal '[{"name":"DiskControllerTypes","value":"SCSI,NVMe"},{"name":"SecurityType","value":"TrustedLaunchSupported"}]' + End + + It 'propagates a new shared Windows definition creation failure without migration' + MODE="windowsVhdMode" + ARCHITECTURE="x64" + FEATURE_FLAGS="" + HYPERV_GENERATION="V2" + OS_TYPE="Windows" + AZURE_LOCATION="eastus" + ENABLE_TRUSTED_LAUNCH="False" + TRUSTED_LAUNCH_SUPPORTED="True" + FAIL_OPERATION="create" + + When call ensure_sig_vhd_exists + The status should equal 23 + The output should include 'Creating image definition windows-2025-gen2' + The contents of file "${AZ_CALLS}" should not include 'resource update' + The contents of file "${AZ_CALLS}" should not include 'image-definition wait' + The file "${UPDATE_FEATURES}" should not be exist + End + + It 'migrates an existing Standard staging definition during build setup' + MODE="windowsVhdMode" + ENABLE_TRUSTED_LAUNCH="False" + TRUSTED_LAUNCH_SUPPORTED="True" + jq '.properties.features += [{name:"SecurityType",value:"Standard"}]' \ + <<<"${INITIAL_DEFINITION}" > "${TEST_DIR}/created-definition" + + When call ensure_sig_vhd_exists + The status should be success + The output should include 'Updating image definition' + The stderr should equal '' + The contents of file "${AZ_CALLS}" should not include 'image-definition create' + The contents of file "${AZ_CALLS}" should include '--set properties.allowUpdateImage=true properties.features=' + The value "$(grep -c '^resource show ' "${AZ_CALLS}")" should equal 1 + The value "$(jq -c '.' "${UPDATE_FEATURES}")" should equal '[{"name":"DiskControllerTypes","value":"SCSI,NVMe"},{"name":"SecurityType","value":"TrustedLaunchSupported","startsAtVersion":"26100.1.261006"}]' + End + + It 'does not move the boundary of an already-supported definition' + INITIAL_DEFINITION='{"properties":{"hyperVGeneration":"V2","features":[{"name":"SecurityType","value":"TrustedLaunchSupported","startsAtVersion":"26100.1.260901"}]}}' + + When call ensure_sig_trusted_launch_supported "${DEFINITION_ID}" + The status should be success + The output should equal '' + The contents of file "${AZ_CALLS}" should not include 'resource update' + The file "${UPDATE_FEATURES}" should not be exist + End + + Describe 'existing capability without a boundary' + Parameters + absent '{"name":"SecurityType","value":"TrustedLaunchSupported"}' + null '{"name":"SecurityType","value":"TrustedLaunchSupported","startsAtVersion":null}' + End + + It "accepts an $1 boundary without modifying the staging definition" + INITIAL_DEFINITION=$(jq --argjson feature "$2" ' + .properties.features += [$feature] + ' <<<"${INITIAL_DEFINITION}") + + When call ensure_sig_trusted_launch_supported "${DEFINITION_ID}" + The status should be success + The output should equal '' + The stderr should equal '' + The contents of file "${AZ_CALLS}" should not include 'resource update' + The contents of file "${AZ_CALLS}" should not include 'image-definition wait' + The file "${UPDATE_FEATURES}" should not be exist + End + End + + It 'reuses an existing supported staging definition without a boundary during build setup' + MODE="windowsVhdMode" + ENABLE_TRUSTED_LAUNCH="False" + TRUSTED_LAUNCH_SUPPORTED="True" + jq '.properties.features += [{name:"SecurityType",value:"TrustedLaunchSupported"}]' \ + <<<"${INITIAL_DEFINITION}" > "${TEST_DIR}/created-definition" + + When call ensure_sig_vhd_exists + The status should be success + The output should include 'Image definition windows-2025-gen2 existing' + The stderr should equal '' + The contents of file "${AZ_CALLS}" should not include 'resource update' + The contents of file "${AZ_CALLS}" should not include 'image-definition wait' + The contents of file "${AZ_CALLS}" should not include 'image-definition create' + The file "${UPDATE_FEATURES}" should not be exist + End + + Describe 'valid existing capability boundaries' + Parameters + equal '26100.1.261006' + numeric '26100.1.9' + minor '26100.0.999999' + End + + It "preserves an existing $1 boundary at or below the build version" + INITIAL_DEFINITION=$(jq --arg version "$2" '.properties.hyperVGeneration = "V2" | .properties.features[0].startsAtVersion = $version' <<<"${UPDATED_DEFINITION}") + + When call ensure_sig_trusted_launch_supported "${DEFINITION_ID}" + The status should be success + The output should equal '' + The contents of file "${AZ_CALLS}" should not include 'resource update' + The file "${UPDATE_FEATURES}" should not be exist + End + End + + Describe 'invalid existing capability boundaries' + Parameters + empty '' + malformed 'invalid' + short '26100.1' + nonnumeric '26100.1.bad' + future '26100.1.261007' + futureminor '26100.10.1' + outofrange '26100.1.2147483648' + End + + It "rejects an existing $1 boundary without updating it" + INITIAL_DEFINITION=$(jq --arg version "$2" ' + .properties.hyperVGeneration = "V2" | + .properties.features[0].startsAtVersion = $version + ' <<<"${UPDATED_DEFINITION}") + + When call ensure_sig_trusted_launch_supported "${DEFINITION_ID}" + The status should be failure + The stderr should include 'cannot support build version' + The output should equal '' + The contents of file "${AZ_CALLS}" should not include 'resource update' + The file "${UPDATE_FEATURES}" should not be exist + End + End + + It 'refuses to mark a Gen1 definition as capable' + INITIAL_DEFINITION='{"properties":{"hyperVGeneration":"V1","features":[]}}' + + When call ensure_sig_trusted_launch_supported "${DEFINITION_ID}" + The status should be failure + The stderr should include 'must be Gen2' + The contents of file "${AZ_CALLS}" should not include 'resource update' + End + + Describe 'Azure failures' + Parameters + show 'Failed to read image definition' + update 'Failed to update Trusted Launch support' + End + + It "stops when $1 fails" + FAIL_OPERATION="$1" + + When call ensure_sig_trusted_launch_supported "${DEFINITION_ID}" + The status should be failure + The stderr should include "$2" + The output should not include 'did not converge' + End + End + + It 'rejects a completed update with incorrect capability' + UPDATED_DEFINITION='{"properties":{"provisioningState":"Succeeded","features":[{"name":"SecurityType","value":"TrustedLaunch"}]}}' + + When call ensure_sig_trusted_launch_supported "${DEFINITION_ID}" + The status should be failure + The output should include 'Updating image definition' + The stderr should include 'did not converge to TrustedLaunchSupported' + End + + It 'rejects an update that has not succeeded' + UPDATED_DEFINITION=$(jq '.properties.provisioningState = "Updating"' <<<"${UPDATED_DEFINITION}") + + When call ensure_sig_trusted_launch_supported "${DEFINITION_ID}" + The status should be failure + The output should include 'Updating image definition' + The stderr should include 'did not converge to TrustedLaunchSupported' + End + + Describe 'returned capability boundary' + Parameters + missing '{"properties":{"provisioningState":"Succeeded","features":[{"name":"SecurityType","value":"TrustedLaunchSupported"}]}}' + older '{"properties":{"provisioningState":"Succeeded","features":[{"name":"SecurityType","value":"TrustedLaunchSupported","startsAtVersion":"26100.1.260901"}]}}' + newer '{"properties":{"provisioningState":"Succeeded","features":[{"name":"SecurityType","value":"TrustedLaunchSupported","startsAtVersion":"26100.1.261007"}]}}' + End + + It "rejects a completed update with a $1 boundary" + UPDATED_DEFINITION="$2" + + When call ensure_sig_trusted_launch_supported "${DEFINITION_ID}" + The status should be failure + The output should include 'Updating image definition' + The stderr should include 'did not converge to TrustedLaunchSupported' + End + End +End diff --git a/spec/vhdbuilder/packer/ensure_sig_vhd_exists_spec.sh b/spec/vhdbuilder/packer/ensure_sig_vhd_exists_spec.sh index ea07aa4c7aa..0ae454c107b 100644 --- a/spec/vhdbuilder/packer/ensure_sig_vhd_exists_spec.sh +++ b/spec/vhdbuilder/packer/ensure_sig_vhd_exists_spec.sh @@ -18,6 +18,7 @@ Describe 'ensure_sig_vhd_exists function' HYPERV_GENERATION="" OS_TYPE="" ENABLE_TRUSTED_LAUNCH="" + TRUSTED_LAUNCH_SUPPORTED="False" # Mock variables to control az command behavior MOCK_AZ_SIG_SHOW_STATE="" @@ -61,6 +62,7 @@ Describe 'ensure_sig_vhd_exists function' return 0 ;; "create") + MOCK_AZ_SIG_IMAGE_DEFINITION_EXISTS="true" echo "az $*" return 0 ;; @@ -125,7 +127,84 @@ Describe 'ensure_sig_vhd_exists function' BeforeEach 'setup_environment' + Describe 'shared Windows Gen2 image creation' + Parameters + windows-2022-containerd-gen2 False True TrustedLaunchSupported + windows-2025-gen2 False True TrustedLaunchSupported + windows-2025-gen2-tl True True TrustedLaunch + windows-2025-gen2 False False '' + End + + It "creates $1 with TL enabled=$2 supported=$3" + MODE="windowsVhdMode" + AZURE_RESOURCE_GROUP_NAME="test-rg" + SIG_GALLERY_NAME="test-gallery" + SIG_IMAGE_NAME="$1" + AZURE_LOCATION="eastus" + OS_TYPE="Windows" + HYPERV_GENERATION="V2" + ARCHITECTURE="x64" + ENABLE_TRUSTED_LAUNCH="$2" + TRUSTED_LAUNCH_SUPPORTED="$3" + MOCK_AZ_SIG_SHOW_EXISTS="true" + MOCK_AZ_SIG_SHOW_STATE="Succeeded" + MOCK_AZ_SIG_IMAGE_DEFINITION_EXISTS="false" + + # shellcheck disable=SC2329 + ensure_sig_trusted_launch_supported() { + echo "Reconciled $1" + } + + When call ensure_sig_vhd_exists + The status should be success + The output should include "--gallery-image-definition $1" + The output should include 'DiskControllerTypes=SCSI,NVMe' + if [ -n "$4" ]; then + The output should include "SecurityType=$4" + else + The output should not include 'SecurityType=' + fi + The output should not include 'Reconciled' + End + End + Describe 'Basic function execution' + It 'reconciles capability for an existing shared Windows Gen2 definition' + MODE="windowsVhdMode" + ENABLE_TRUSTED_LAUNCH="False" + TRUSTED_LAUNCH_SUPPORTED="True" + MOCK_AZ_SIG_SHOW_EXISTS="true" + MOCK_AZ_SIG_SHOW_STATE="Succeeded" + MOCK_AZ_SIG_IMAGE_DEFINITION_EXISTS="true" + + # shellcheck disable=SC2329 + ensure_sig_trusted_launch_supported() { + echo "Reconciled $1" + } + + When call ensure_sig_vhd_exists + The status should be success + The output should include 'Reconciled mock-value' + End + + It 'propagates an existing shared Windows capability update failure' + MODE="windowsVhdMode" + ENABLE_TRUSTED_LAUNCH="False" + TRUSTED_LAUNCH_SUPPORTED="True" + MOCK_AZ_SIG_SHOW_EXISTS="true" + MOCK_AZ_SIG_SHOW_STATE="Succeeded" + MOCK_AZ_SIG_IMAGE_DEFINITION_EXISTS="true" + + # shellcheck disable=SC2329 + ensure_sig_trusted_launch_supported() { + return 1 + } + + When call ensure_sig_vhd_exists + The status should be failure + The output should include 'Image definition' + End + It 'should complete successfully with minimal setup when gallery does not exist' MODE="linuxVhdMode" AZURE_RESOURCE_GROUP_NAME="test-rg" diff --git a/spec/vhdbuilder/packer/generate_vhd_publishing_info_spec.sh b/spec/vhdbuilder/packer/generate_vhd_publishing_info_spec.sh index 8edecbcf47b..e5cca863c26 100644 --- a/spec/vhdbuilder/packer/generate_vhd_publishing_info_spec.sh +++ b/spec/vhdbuilder/packer/generate_vhd_publishing_info_spec.sh @@ -4,7 +4,8 @@ Describe 'generate-vhd-publishing-info.sh' setup() { SCRIPT_PATH="${PWD}/vhdbuilder/packer/generate-vhd-publishing-info.sh" - TEST_DIR=$(mktemp -d "${SHELLSPEC_WORKDIR}/publishing-info.XXXXXX") + TEST_DIR="${SHELLSPEC_WORKDIR}/publishing-info.${SHELLSPEC_EXAMPLE_NO}" + mkdir -p "${TEST_DIR}" OUTPUT="${TEST_DIR}/vhd-publishing-info.json" export AZ_CALLS="${TEST_DIR}/az-calls" : > "$AZ_CALLS" @@ -221,6 +222,21 @@ Describe 'generate-vhd-publishing-info.sh' End Describe 'Windows output' + Parameters + windows-2022-containerd-gen2 + windows-2025-gen2 + End + + It "publishes $1 capability without relying on a TL image name" + OS_NAME=windows OFFER_NAME=Windows SKU_NAME="$1" SECURITY_TYPE_FEATURE=TrustedLaunchSupported + AZ_VERSIONS='' + + When call generate_publishing_info + The status should be success + The output should include 'COPY ME ---> https://example.invalid/images/test.vhd' + The value "$(read_metadata '[.os_name, .sku_name, .hyperv_generation, .security_type_feature]')" should equal "[\"windows\",\"$1\",\"V2\",\"TrustedLaunchSupported\"]" + End + Parameters '' stale-gallery diff --git a/spec/vhdbuilder/packer/imagecustomizer/bootstrap_cosi_convert_tools_spec.sh b/spec/vhdbuilder/packer/imagecustomizer/bootstrap_cosi_convert_tools_spec.sh new file mode 100644 index 00000000000..eabf59dd3c4 --- /dev/null +++ b/spec/vhdbuilder/packer/imagecustomizer/bootstrap_cosi_convert_tools_spec.sh @@ -0,0 +1,261 @@ +#!/bin/bash +# shellcheck disable=SC2034,SC2329 + +# Tests for vhdbuilder/packer/imagecustomizer/scripts/bootstrap-cosi-convert-tools.sh. +# Only the functions are sourced (guarded by ${__SOURCED__:+return}); every +# system command is stubbed, so nothing is installed. + +Describe 'bootstrap-cosi-convert-tools.sh' + setup() { + # shellcheck disable=SC1090 + __SOURCED__=1 . "./vhdbuilder/packer/imagecustomizer/scripts/bootstrap-cosi-convert-tools.sh" + stub_dir="$(mktemp -d)" + } + cleanup() { rm -rf "$stub_dir"; } + BeforeEach 'setup' + AfterEach 'cleanup' + + # Creates executable stubs named $@ in stub_dir. + make_stubs() { + local name + for name in "$@"; do + printf '#!/bin/sh\nexit 0\n' > "$stub_dir/$name" + chmod +x "$stub_dir/$name" + done + } + + # Links the real tools that install_azcopy needs into stub_dir. + link_tools() { + local name + for name in "$@"; do + ln -s "$(command -v "$name")" "$stub_dir/$name" + done + } + + Describe 'detect_package_manager' + detect_with_path() { + PATH="$stub_dir" detect_package_manager + local rc=$? + echo "PKG=$PKG" + return $rc + } + + It 'prefers apt when both apt-get and tdnf exist' + make_stubs apt-get tdnf + When call detect_with_path + The status should be success + The line 1 of output should equal 'Package manager: apt' + The line 2 of output should equal 'PKG=apt' + End + + It 'uses tdnf on Azure Linux' + make_stubs tdnf + When call detect_with_path + The status should be success + The line 2 of output should equal 'PKG=tdnf' + End + + It 'falls back to dnf' + make_stubs dnf + When call detect_with_path + The status should be success + The line 2 of output should equal 'PKG=dnf' + End + + It 'fails without a supported package manager' + When call detect_with_path + The status should be failure + The line 1 of output should include 'No supported package manager' + End + End + + Describe 'require_passwordless_sudo' + It 'fails when sudo prompts for a password' + sudo() { return 1; } + When call require_passwordless_sudo + The status should be failure + The output should include 'does not have passwordless sudo' + End + + It 'passes when sudo -n succeeds' + sudo() { return 0; } + When call require_passwordless_sudo + The status should be success + End + End + + Describe 'pkg_install' + sudo() { echo "sudo $*"; } + + It 'uses tdnf on Azure Linux' + PKG=tdnf + When call pkg_install make jq + The output should equal 'sudo tdnf install -y make jq' + End + + It 'uses noninteractive apt-get without recommends' + PKG=apt + When call pkg_install make + The output should equal 'sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends make' + End + + It 'fails before a package manager is detected' + PKG="" + When call pkg_install make + The status should be failure + The output should include 'pkg_install called before detect_package_manager' + End + End + + Describe 'install_azure_cli' + az() { echo 'azure-cli 2.77.0'; } + + It 'keeps an existing Azure CLI without changing repositories' + PKG=tdnf + sudo() { return 1; } + source() { return 1; } + When call install_azure_cli + The status should be success + The output should equal 'azure-cli 2.77.0' + End + + Describe 'cold RPM installation' + Parameters + tdnf azurelinux azurelinux-repos-ms-oss + dnf azurelinux azurelinux-repos-ms-oss + tdnf mariner mariner-repos-microsoft + dnf mariner mariner-repos-microsoft + End + + It "enables the native $2 repository before installing with $1" + PKG="$1" + TEST_DISTRO="$2" + TEST_REPOSITORY_PACKAGE="$3" + TEST_REPOSITORY_READY=false + unset -f az + source() { ID="$TEST_DISTRO"; } + sudo() { + echo "sudo $*" + case "$*" in + "$PKG install -y $TEST_REPOSITORY_PACKAGE") + TEST_REPOSITORY_READY=true + ;; + "$PKG install -y azure-cli") + if [ "$TEST_REPOSITORY_READY" != true ]; then + echo 'Native Azure CLI repository is not configured' >&2 + return 1 + fi + az() { echo 'azure-cli 2.77.0'; } + ;; + *) return 1 ;; + esac + } + install_without_az() { + set -e + PATH="$stub_dir" install_azure_cli + } + When run install_without_az + The status should be success + The line 1 of output should equal "sudo $1 install -y $3" + The line 2 of output should equal "sudo $1 install -y azure-cli" + The line 3 of output should equal 'azure-cli 2.77.0' + The stderr should equal '' + End + End + + It 'stops when native repository installation fails' + PKG=tdnf + unset -f az + source() { ID=azurelinux; } + sudo() { echo 'Repository installation failed' >&2; return 17; } + install_without_az() { + set -e + PATH="$stub_dir" install_azure_cli + } + When run install_without_az + The status should equal 17 + The output should equal '' + The stderr should equal 'Repository installation failed' + End + + It 'preserves configured repositories on other RPM distributions' + PKG=dnf + unset -f az + source() { ID=rhel; } + sudo() { + echo "sudo $*" + az() { echo 'azure-cli 2.77.0'; } + } + install_without_az() { PATH="$stub_dir" install_azure_cli; } + When call install_without_az + The status should be success + The line 1 of output should equal 'sudo dnf install -y azure-cli' + The line 2 of output should equal 'azure-cli 2.77.0' + The stderr should equal '' + End + End + + Describe 'azcopy_release_for_machine' + Parameters + aarch64 arm64 4db9a4b48abc7775f1a5d6d928afc42361dcc57bbfcde23ac82e4c419a0dc8fc + arm64 arm64 4db9a4b48abc7775f1a5d6d928afc42361dcc57bbfcde23ac82e4c419a0dc8fc + x86_64 amd64 7549424d56ab2d8b4033c84c2a9bb167dc2dcbb23998acd7fffb37bc1a71a267 + End + + It "pins the $2 release for $1" + When call azcopy_release_for_machine "$1" + The status should be success + The output should equal "https://github.com/Azure/azure-storage-azcopy/releases/download/v${AZCOPY_VERSION}/azcopy_linux_$2_${AZCOPY_VERSION}.tar.gz $3" + End + End + + It 'rejects an unsupported machine architecture' + When call azcopy_release_for_machine riscv64 + The status should be failure + The output should include 'Unsupported machine architecture for azcopy: riscv64' + End + + Describe 'install_azcopy' + It 'refuses an archive whose checksum does not match the pin' + link_tools mktemp sha256sum rm + uname() { echo x86_64; } + curl() { printf 'tampered' > "${@: -1}"; } + install_with_path() { PATH="$stub_dir" install_azcopy; } + When call install_with_path + The status should be failure + The output should include 'azcopy archive checksum mismatch' + End + End + + Describe 'install_go' + go() { echo 'go version go1.27.1 linux/amd64'; } + + It 'always installs the pinned Microsoft Go on Ubuntu, even if Go exists' + PKG=apt + mkdir -p "$stub_dir/hack" + printf '#!/bin/sh\necho setup_golang ran\n' > "$stub_dir/hack/setup_golang.sh" + chmod +x "$stub_dir/hack/setup_golang.sh" + install_in_dir() { cd "$stub_dir" && install_go; } + When call install_in_dir + The line 1 of output should equal 'setup_golang ran' + The line 2 of output should equal 'go version go1.27.1 linux/amd64' + End + + It 'keeps an existing Go on Azure Linux' + PKG=tdnf + sudo() { echo "sudo $*"; } + When call install_go + The output should equal 'go version go1.27.1 linux/amd64' + End + + It 'installs golang on Azure Linux when Go is missing' + PKG=tdnf + unset -f go + sudo() { echo "sudo $*"; go() { echo 'go version go1.27.1 linux/arm64'; }; } + install_without_go() { PATH="$stub_dir" install_go; } + When call install_without_go + The line 1 of output should equal 'sudo tdnf install -y golang' + The line 2 of output should equal 'go version go1.27.1 linux/arm64' + End + End +End diff --git a/spec/vhdbuilder/packer/imagecustomizer/convert_vhd_to_cosi_spec.sh b/spec/vhdbuilder/packer/imagecustomizer/convert_vhd_to_cosi_spec.sh new file mode 100644 index 00000000000..733ffa44e8f --- /dev/null +++ b/spec/vhdbuilder/packer/imagecustomizer/convert_vhd_to_cosi_spec.sh @@ -0,0 +1,427 @@ +#!/bin/bash + +# Tests for generate_cosi_package_version in +# vhdbuilder/packer/imagecustomizer/scripts/convert-vhd-to-cosi.sh. +# +# Nebraska validates COSI package versions as strict SemVer, which rejects +# leading zeros in numeric components (e.g. 202608.06.0). These tests pin the +# normalization: leading zeros stripped, -fips appended only for FIPS builds, +# and never duplicated. + +Describe 'generate_cosi_package_version' + setup() { + # Source only the functions (guarded by ${__SOURCED__:+return}), not the + # main conversion flow. + # shellcheck disable=SC1090 + __SOURCED__=1 . "./vhdbuilder/packer/imagecustomizer/scripts/convert-vhd-to-cosi.sh" + } + BeforeEach 'setup' + + Describe 'strips per-component leading zeros' + It 'normalizes day 01 (non-FIPS)' + When call generate_cosi_package_version "202608.01.0" "false" + The status should be success + The output should equal "202608.1.0" + End + + It 'normalizes day 06 (non-FIPS)' + When call generate_cosi_package_version "202608.06.0" "false" + The status should be success + The output should equal "202608.6.0" + End + + It 'leaves day 10 unchanged (non-FIPS)' + When call generate_cosi_package_version "202608.10.0" "false" + The status should be success + The output should equal "202608.10.0" + End + End + + Describe 'appends -fips only for FIPS builds' + It 'emits -fips when ENABLE_FIPS is true' + When call generate_cosi_package_version "202608.06.0" "true" + The status should be success + The output should equal "202608.6.0-fips" + End + + It 'omits -fips when ENABLE_FIPS is false' + When call generate_cosi_package_version "202608.06.0" "false" + The status should be success + The output should equal "202608.6.0" + End + + It 'treats ENABLE_FIPS case-insensitively (True)' + When call generate_cosi_package_version "202608.10.2" "True" + The status should be success + The output should equal "202608.10.2-fips" + End + + It 'does not duplicate -fips when the version already contains it' + When call generate_cosi_package_version "202608.06.0-fips" "true" + The status should be success + The output should equal "202608.6.0-fips" + End + End + + Describe 'rejects a core that is not exactly three numeric components' + It 'fails with too few components' + When call generate_cosi_package_version "202608.06" "false" + The status should be failure + The error should be present + End + + It 'fails with too many components' + When call generate_cosi_package_version "202608.06.0.1" "false" + The status should be failure + The error should be present + End + + It 'fails with a non-numeric component' + When call generate_cosi_package_version "202608.aug.0" "false" + The status should be failure + The error should be present + End + End +End + +Describe 'immutable VHD handoff' + setup_handoff() { + export DESTINATION_STORAGE_CONTAINER=https://storage.invalid/staging + export CAPTURED_SIG_VERSION=test + export STORAGE_ACCOUNT_NAME=mock + export VHD_CONTAINER_NAME=immutable + export MOCK_DESTINATION_EXISTS=false + export MOCK_SOURCE_EXISTS=true + export MOCK_EXISTS_STATUS=0 + export MOCK_COPY_STATUS=success + export MOCK_COMPLETE_AFTER_WAIT=0 + export MOCK_START_STATUS=0 + export MOCK_SHOW_STATUS=0 + + az() { + case "$*" in + 'storage blob exists --account-name '*) + printf '%s\n' "$MOCK_DESTINATION_EXISTS" + return "$MOCK_EXISTS_STATUS" + ;; + 'storage blob exists --blob-url '*) + printf '%s\n' "$MOCK_SOURCE_EXISTS" + ;; + 'storage blob copy start '*) + echo COPY_STARTED + return "$MOCK_START_STATUS" + ;; + 'storage blob show '*) + printf '%s\n' "$MOCK_COPY_STATUS" + return "$MOCK_SHOW_STATUS" + ;; + *) return 1 ;; + esac + } + azcopy() { echo SOURCE_REMOVED; } + sleep() { + [ "$1" = 15 ] || return 1 + echo COPY_WAITED + if [ "$MOCK_COMPLETE_AFTER_WAIT" = 1 ]; then + MOCK_COPY_STATUS=success + fi + } + export -f az azcopy sleep + } + BeforeEach 'setup_handoff' + + It 'waits for a successful copy before deleting the source' + MOCK_COPY_STATUS=pending + MOCK_COMPLETE_AFTER_WAIT=1 + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/copy-vhd-to-immutable.sh + The status should be success + The line 3 of output should equal COPY_WAITED + The line 5 of output should equal SOURCE_REMOVED + The output should not include 'task.setvariable' + End + + It 'reuses a successful immutable copy on retry' + MOCK_DESTINATION_EXISTS=true + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/copy-vhd-to-immutable.sh + The status should be success + The output should include SOURCE_REMOVED + The output should not include COPY_STARTED + The output should not include COPY_WAITED + End + + It 'waits for an existing pending copy instead of overwriting it' + MOCK_DESTINATION_EXISTS=true + MOCK_COPY_STATUS=pending + MOCK_COMPLETE_AFTER_WAIT=1 + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/copy-vhd-to-immutable.sh + The status should be success + The line 2 of output should equal COPY_WAITED + The output should include SOURCE_REMOVED + The output should not include COPY_STARTED + End + + It 'succeeds when the staging source is already removed' + MOCK_DESTINATION_EXISTS=true + MOCK_SOURCE_EXISTS=false + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/copy-vhd-to-immutable.sh + The status should be success + The output should include 'is already removed' + The output should not include COPY_STARTED + The output should not include SOURCE_REMOVED + End + + It 'stops when checking the destination fails' + MOCK_EXISTS_STATUS=1 + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/copy-vhd-to-immutable.sh + The status should be failure + The output should be blank + End + + It 'rejects an unknown destination existence result' + MOCK_DESTINATION_EXISTS=unknown + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/copy-vhd-to-immutable.sh + The status should be failure + The output should include 'Unable to determine whether the immutable VHD exists' + The output should not include COPY_STARTED + The output should not include SOURCE_REMOVED + End + + It 'does not delete a source with an unknown existence result' + MOCK_SOURCE_EXISTS=unknown + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/copy-vhd-to-immutable.sh + The status should be failure + The output should include 'Unable to determine whether the staging VHD exists' + The output should not include SOURCE_REMOVED + End + + Describe 'unsuccessful copy states' + Parameters + failed + aborted + # shellcheck disable=SC2286 + '' + End + It 'preserves the source when the copy is not successful' + MOCK_COPY_STATUS="$1" + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/copy-vhd-to-immutable.sh + The status should be failure + The output should include 'VHD copy to immutable container finished with status' + The output should not include SOURCE_REMOVED + End + End + + It 'preserves the source when the copy stays pending until timeout' + MOCK_COPY_STATUS=pending + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/copy-vhd-to-immutable.sh + The status should be failure + The output should include 'Timed out waiting for the immutable VHD copy' + The output should not include SOURCE_REMOVED + End + + It 'stops when starting the copy fails' + MOCK_START_STATUS=1 + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/copy-vhd-to-immutable.sh + The status should be failure + The output should include COPY_STARTED + The output should not include COPY_WAITED + The output should not include SOURCE_REMOVED + End + + It 'preserves the source when reading the copy status fails' + MOCK_SHOW_STATUS=1 + When run bash ./vhdbuilder/packer/imagecustomizer/scripts/copy-vhd-to-immutable.sh + The status should be failure + The output should include COPY_STARTED + The output should not include SOURCE_REMOVED + End +End + +Describe 'COSI artifact names' + setup_artifacts() { + convert_script="$(pwd)/vhdbuilder/packer/imagecustomizer/scripts/convert-vhd-to-cosi.sh" + upload_script="$(pwd)/vhdbuilder/packer/imagecustomizer/scripts/upload-cosi-to-pmc.sh" + test_dir="$(mktemp -d)" + mkdir -p "$test_dir/bin" + ln -s /bin/echo "$test_dir/bin/cosi-uploader" + export CAPTURED_SIG_VERSION=202609.21.0 + export IMAGE_VERSION=202609.21.0 + export DESTINATION_STORAGE_CONTAINER=https://storage.invalid/vhds + export IMG_CUSTOMIZER_CONTAINER=registry.invalid/imagecustomizer@sha256:0000000000000000000000000000000000000000000000000000000000000000 + export EXPECTED_IMG_CUSTOMIZER_REF="$IMG_CUSTOMIZER_CONTAINER" + export MOCK_PRIMARY_PULL_FAIL=0 + export MOCK_COSI_METADATA=true + export AFD_DOWNLOAD_HOSTNAME=download.invalid + export AFD_UPLOAD_ENDPOINT=https://upload.invalid + export COSI_CONTAINER=cosi + export ARCHITECTURE=X86_64 + export MOCK_AZ_TENANT_STATUS=0 + unset AZCOPY_TENANT_ID + + az() { + [ "$*" = 'account show --query tenantId --output tsv' ] || return 1 + [ "$MOCK_AZ_TENANT_STATUS" = 0 ] || return 1 + echo mock-tenant + } + # Read the login settings from a child process, as the real azcopy would. + azcopy() { + bash -c 'printf "%s %s %s\n" "${AZCOPY_AUTO_LOGIN_TYPE:-}" "${AZCOPY_TENANT_ID:-}" "$1"' _ "$2" > "$PWD/azcopy-call" + } + docker() { + if [ "$1" = pull ] && [ "$4" = "$IMG_CUSTOMIZER_CONTAINER" ] && [ "$MOCK_PRIMARY_PULL_FAIL" = 1 ]; then + return 1 + elif [ "$1" = run ]; then + [ "$9" = "$EXPECTED_IMG_CUSTOMIZER_REF" ] || return 1 + if [ "$MOCK_COSI_METADATA" = true ]; then + printf '%s\n' '{"version":"1.2"}' > "$PWD/cosi-convert/metadata.json" + tar -cf "$PWD/cosi-convert/out/${CAPTURED_SIG_VERSION}.cosi" -C "$PWD/cosi-convert" metadata.json + else + tar -cf "$PWD/cosi-convert/out/${CAPTURED_SIG_VERSION}.cosi" --files-from=/dev/null + fi + fi + } + export -f az azcopy docker + } + cleanup_artifacts() { rm -rf "$test_dir"; } + BeforeEach 'setup_artifacts' + AfterEach 'cleanup_artifacts' + + run_convert() { + cd "$test_dir" || return 1 + bash "$convert_script" + } + + check_vhd_download() { + cd "$test_dir" || return 1 + bash "$convert_script" >/dev/null || return $? + cat azcopy-call + } + + check_artifact_name() { + cd "$test_dir" || return 1 + bash "$convert_script" >/dev/null || return $? + jq -r .cosi_url cosi-publishing-info.json + bash "$upload_script" + } + + check_build_version() { + cd "$test_dir" || return 1 + date() { printf '%s\n' 202610.01.0; } + export -f date + bash "$convert_script" >/dev/null || return $? + jq -r .image_version cosi-publishing-info.json + } + + check_metadata_hash() { + cd "$test_dir" || return 1 + bash "$convert_script" >/dev/null || return $? + local expected_hash + expected_hash=$(printf '%s\n' '{"version":"1.2"}' | openssl dgst -sha384 -binary | base64) + [ "$(jq -r .metadata_sha384 cosi-publishing-info.json)" = "$expected_hash" ] + } + + It 'publishes the base64 SHA384 of the embedded metadata' + export SKU_NAME=aclgen2TL + When call check_metadata_hash + The status should be success + The output should be blank + End + + It 'rejects COSI output without embedded metadata' + export SKU_NAME=aclgen2TL MOCK_COSI_METADATA=false + When call check_artifact_name + The status should be failure + The output should be blank + The error should include 'metadata.json: Not found in archive' + End + + It 'keeps the build version when conversion starts on a later UTC date' + export SKU_NAME=aclgen2arm64fipsTL ARCHITECTURE=ARM64 ENABLE_FIPS=true + When call check_build_version + The status should be success + The output should equal '202609.21.0-fips' + End + + It 'rejects conversion without the build-resolved version' + export SKU_NAME=aclgen2arm64TL + unset IMAGE_VERSION + When run bash "$convert_script" + The status should be failure + The output should equal 'IMAGE_VERSION was not set!' + End + + It 'rejects conversion without ARCHITECTURE' + export SKU_NAME=aclgen2TL + unset ARCHITECTURE + When run bash "$convert_script" + The status should be failure + The output should equal 'ARCHITECTURE was not set!' + End + + It 'downloads the immutable VHD with the Azure CLI login and its tenant' + export SKU_NAME=aclgen2arm64TL ARCHITECTURE=ARM64 + When call check_vhd_download + The status should be success + The output should equal "AZCLI mock-tenant ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" + End + + It 'keeps an explicitly configured azcopy tenant' + export SKU_NAME=aclgen2TL AZCOPY_TENANT_ID=explicit-tenant MOCK_AZ_TENANT_STATUS=1 + When call check_vhd_download + The status should be success + The output should equal "AZCLI explicit-tenant ${DESTINATION_STORAGE_CONTAINER}/${CAPTURED_SIG_VERSION}.vhd" + End + + It 'stops before downloading when the Azure CLI tenant is unavailable' + export SKU_NAME=aclgen2TL MOCK_AZ_TENANT_STATUS=1 + When call run_convert + The status should be failure + The output should include 'Unable to read the Azure CLI tenant for the azcopy login' + The output should not include 'Downloading VHD' + End + + It 'rejects a mutable primary image before downloading or converting' + export SKU_NAME=aclgen2arm64TL IMG_CUSTOMIZER_CONTAINER=mcr.microsoft.com/azurelinux/imagecustomizer:1.5 + When run bash "$convert_script" + The status should be failure + The output should be blank + The error should include 'ImageCustomizer reference must be pinned with a sha256 digest' + End + + It 'stops when the ImageCustomizer pull fails' + export SKU_NAME=aclgen2arm64TL + MOCK_PRIMARY_PULL_FAIL=1 + When call check_artifact_name + The status should be failure + The output should be blank + End + + Describe 'variants with a shared capture version' + Parameters + aclgen2TL X86_64 false + aclgen2fipsTL X86_64 true + aclgen2arm64TL ARM64 false + aclgen2arm64fipsTL ARM64 true + End + It 'uses the SKU in both the metadata URL and upload destination' + export SKU_NAME="$1" ARCHITECTURE="$2" ENABLE_FIPS="$3" + When call check_artifact_name + The status should be success + The line 1 of output should equal "https://download.invalid/cosi/${SKU_NAME}-${CAPTURED_SIG_VERSION}.cosi" + The line 2 of output should include "--blob ${SKU_NAME}-${CAPTURED_SIG_VERSION}.cosi --file ${test_dir}/${SKU_NAME}-${CAPTURED_SIG_VERSION}.cosi" + End + End + + It 'rejects conversion without a SKU' + unset SKU_NAME + When run bash "$convert_script" + The status should be failure + The output should equal 'SKU_NAME was not set!' + End + + It 'rejects upload without a SKU' + unset SKU_NAME + When run bash "$upload_script" + The status should be failure + The output should equal 'SKU_NAME was not set!' + End +End diff --git a/spec/vhdbuilder/packer/post_install_dependencies_spec.sh b/spec/vhdbuilder/packer/post_install_dependencies_spec.sh new file mode 100644 index 00000000000..7a7ca1e2cfd --- /dev/null +++ b/spec/vhdbuilder/packer/post_install_dependencies_spec.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# shellcheck disable=SC2016,SC2034,SC2329 + +Describe 'ACL image version stamp' + setup_image_version() { + TEST_ROOT="$(mktemp -d)" + TEST_SCRIPT="${TEST_ROOT}/image-version.sh" + export TEST_IMAGE_VERSION_FILE="${TEST_ROOT}/containers/image-version" + export VHD_LOGS_FILEPATH="${TEST_ROOT}/vhd-install.complete" + export OS=AZURELINUX OS_VARIANT=AZURECONTAINERLINUX + unset IMAGE_VERSION + + # Run only the stamp block, redirecting its output path away from the host. + sed -n '/^if isACL "$OS" "$OS_VARIANT"; then$/,/^fi$/p' \ + ./vhdbuilder/packer/post-install-dependencies.sh | + sed 's|^ IMAGE_VERSION_FILEPATH=/opt/azure/containers/image-version$| IMAGE_VERSION_FILEPATH="${TEST_IMAGE_VERSION_FILE}"|' \ + > "$TEST_SCRIPT" + + isACL() { + [ "$1" = AZURELINUX ] && [ "$2" = AZURECONTAINERLINUX ] + } + export -f isACL + } + + cleanup_image_version() { + rm -rf "$TEST_ROOT" + } + + BeforeEach 'setup_image_version' + AfterEach 'cleanup_image_version' + + Describe 'required input' + Parameters + unset + empty + End + + It "rejects an $1 IMAGE_VERSION without creating a stamp" + if [ "$1" = empty ]; then + export IMAGE_VERSION="" + fi + + When run bash -eu "$TEST_SCRIPT" + The status should be failure + The error should include 'ERROR: IMAGE_VERSION must be set for ACL image builds' + The output should be blank + The path "$TEST_IMAGE_VERSION_FILE" should not be exist + The path "$VHD_LOGS_FILEPATH" should not be exist + End + End + + Describe 'supplied input' + Parameters + 202610.26.0 + 202608.06.0 + End + + It "records the supplied version $1 unchanged" + export IMAGE_VERSION="$1" + + When run bash -eu "$TEST_SCRIPT" + The status should be success + The output should be blank + The contents of file "$TEST_IMAGE_VERSION_FILE" should equal "IMAGE_VERSION=$1" + The contents of file "$VHD_LOGS_FILEPATH" should include "IMAGE_VERSION=$1" + End + End + + It 'preserves an existing stamp when IMAGE_VERSION is missing' + mkdir -p "$(dirname "$TEST_IMAGE_VERSION_FILE")" + printf '%s\n' 'IMAGE_VERSION=existing' > "$TEST_IMAGE_VERSION_FILE" + + When run bash -eu "$TEST_SCRIPT" + The status should be failure + The error should include 'ERROR: IMAGE_VERSION must be set for ACL image builds' + The output should be blank + The contents of file "$TEST_IMAGE_VERSION_FILE" should equal 'IMAGE_VERSION=existing' + The path "$VHD_LOGS_FILEPATH" should not be exist + End + + It 'does not require IMAGE_VERSION for a non-ACL image' + export OS=UBUNTU OS_VARIANT="" + + When run bash -eu "$TEST_SCRIPT" + The status should be success + The output should be blank + The path "$TEST_IMAGE_VERSION_FILE" should not be exist + The path "$VHD_LOGS_FILEPATH" should not be exist + End +End diff --git a/spec/vhdbuilder/packer/pre_install_dependencies_spec.sh b/spec/vhdbuilder/packer/pre_install_dependencies_spec.sh index a88e5a1f07b..f254439c4e4 100644 --- a/spec/vhdbuilder/packer/pre_install_dependencies_spec.sh +++ b/spec/vhdbuilder/packer/pre_install_dependencies_spec.sh @@ -82,3 +82,42 @@ Describe 'installAzureLinuxArm64DualKernel' The path "${BOOT_DIR}/grub2/arm64-efi" should not be exist End End + +Describe 'enableTridentACLAgent' + BeforeAll "eval \"\$(sed -n '/^enableTridentACLAgent()/,/^}/p' './vhdbuilder/packer/pre-install-dependencies.sh')\"" + + setup_agent() { + MOCK_SERVICE_STATUS=0 + MOCK_ENABLE_STATUS=0 + systemctl() { + [ "$*" = 'cat trident-acl-agent.service' ] || return 1 + return "$MOCK_SERVICE_STATUS" + } + systemctlEnableAndStart() { + printf '%s\n' "$*" + return "$MOCK_ENABLE_STATUS" + } + } + BeforeEach 'setup_agent' + + It 'enables the kubeconfig trigger when the base contains the agent' + When call enableTridentACLAgent + The status should be success + The output should equal 'trident-acl-agent.path 30' + End + + It 'preserves compatibility with base images without the agent' + MOCK_SERVICE_STATUS=1 + When call enableTridentACLAgent + The status should be success + The output should include 'skipping startup trigger' + The output should not include 'trident-acl-agent.path 30' + End + + It 'propagates failure to enable an installed agent' + MOCK_ENABLE_STATUS=1 + When call enableTridentACLAgent + The status should be failure + The output should equal 'trident-acl-agent.path 30' + End +End diff --git a/spec/vhdbuilder/packer/resolve_security_type_feature_spec.sh b/spec/vhdbuilder/packer/resolve_security_type_feature_spec.sh new file mode 100644 index 00000000000..d35beee4ebb --- /dev/null +++ b/spec/vhdbuilder/packer/resolve_security_type_feature_spec.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +Describe 'resolve_security_type_feature' + Include './vhdbuilder/packer/produce-packer-settings-functions.sh' + + Parameters + False True None TrustedLaunchSupported + True True None TrustedLaunch + True False None TrustedLaunch + False False cvm ConfidentialVMSupported + False False None Standard + true true None TrustedLaunch + false true None TrustedLaunchSupported + End + + It "resolves enabled=$1 supported=$2 flags=$3 to $4" + ENABLE_TRUSTED_LAUNCH="$1" + TRUSTED_LAUNCH_SUPPORTED="$2" + FEATURE_FLAGS="$3" + + When call resolve_security_type_feature + The status should be success + The variable SECURITY_TYPE_FEATURE should equal "$4" + End +End diff --git a/spec/vhdbuilder/packer/test/linux_vhd_content_test_helpers_spec.sh b/spec/vhdbuilder/packer/test/linux_vhd_content_test_helpers_spec.sh index 8d0d025b5cf..ea5263575ea 100644 --- a/spec/vhdbuilder/packer/test/linux_vhd_content_test_helpers_spec.sh +++ b/spec/vhdbuilder/packer/test/linux_vhd_content_test_helpers_spec.sh @@ -3,6 +3,34 @@ # ShellSpec tests for content-test helpers and call sites +Describe 'eMudhra root removal' + BeforeAll "eval \"\$(sed -n '/^testEmudhraRootRemoval()/,/^}/p' vhdbuilder/packer/test/linux-vhd-content-test.sh)\"" + err() { echo "$1:Error: $2" >&2; } + openssl() { + if [ "$1" = crl2pkcs7 ]; then + printf 'subject=CN = %s\n' "$CONTROL" "$ROOT" + return "$READ_STATUS" + fi + cat + } + Parameters + 'DigiCert Global Root G2' '' 0 /dev/null 0 + 'DigiCert Global Root G2' 'emSign ECC Root CA - G3' 0 /dev/null 1 + 'DigiCert Global Root G2' 'AAA Certificate Services' 0 /dev/null 1 + '' '' 0 /dev/null 1 + 'DigiCert Global Root G2' '' 1 /dev/null 1 + 'DigiCert Global Root G2' '' 0 "${SHELLSPEC_WORKDIR}/absent-ca-bundle" 1 + End + It "checks root '$2', read status $3 and bundle $4" + CONTROL="$1" ROOT="$2" READ_STATUS="$3" + When call testEmudhraRootRemoval "$4" + The status should equal "$5" + if [ "$5" -ne 0 ]; then + The stderr should include 'testEmudhraRootRemoval:Error:' + fi + End +End + Describe 'Linux content-test legacy parameters' read_legacy_parameters() { eval "$(sed -n '/^OS_VERSION=/,/^AGENTBAKER_REPOSITORY_URL=/p' vhdbuilder/packer/test/linux-vhd-content-test.sh)" diff --git a/staging/cse/windows/configfunc.ps1 b/staging/cse/windows/configfunc.ps1 index 298d15d7373..a7b0d804666 100644 --- a/staging/cse/windows/configfunc.ps1 +++ b/staging/cse/windows/configfunc.ps1 @@ -564,6 +564,266 @@ function Validate-CredentialProviderConfigFlags { } } +# Returns $null for malformed or incompatible package versions so resolvers can skip them. +function ConvertTo-DalecCredentialProviderVersion { + Param( + [Parameter(Mandatory=$true)][AllowNull()][AllowEmptyString()][string]$PackageVersion, + [Parameter(Mandatory=$true)][version]$KubeVersion + ) + $match = [regex]::Match($PackageVersion, '\A(?(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*))-(?0|[1-9][0-9]*)\z') + # A four-part System.Version compares patch and package revision numerically. + $candidateVersion = $null + if (-not $match.Success -or + -not [version]::TryParse("$($match.Groups['version'].Value).$($match.Groups['revision'].Value)", [ref]$candidateVersion) -or + $candidateVersion.Major -ne $KubeVersion.Major -or $candidateVersion.Minor -ne $KubeVersion.Minor) { + return $null + } + return $candidateVersion +} + +function Resolve-DalecCredentialProviderPackage { + <# + .SYNOPSIS + Selects a cached Dalec credential provider ZIP for k8s >= 1.33, consulting + baked metadata for a download only when no matching archive is cached. + Matches the Kubernetes major/minor and selects the highest patch, then package revision. + Returns $null when neither source applies; cache access errors throw. + #> + Param( + [Parameter(Mandatory=$true)][string]$KubeBinariesVersion + ) + + $kubeVersion = $null + if (-not [version]::TryParse($KubeBinariesVersion.TrimStart('v'), [ref]$kubeVersion)) { + Write-Log "Invalid Kubernetes version '$KubeBinariesVersion'; cannot select a cached Dalec credential provider, falling back to RP URL" + return $null + } + if ($kubeVersion -lt [version]"1.33.0") { + Write-Log "Kubernetes $KubeBinariesVersion is below the Dalec credential provider gate; using RP URL" + return $null + } + + # PROCESSOR_ARCHITEW6432 identifies the native architecture in a 32-bit process. + $architecture = if ($env:PROCESSOR_ARCHITEW6432) { $env:PROCESSOR_ARCHITEW6432 } else { $env:PROCESSOR_ARCHITECTURE } + if ($architecture -ne 'AMD64') { + Write-Log "No supported cached Dalec credential provider architecture for '$architecture'; falling back to RP URL" + return $null + } + $files = @() + if ([string]::IsNullOrEmpty($global:CacheDir)) { + Write-Log "Dalec credential provider cache root is not configured; checking baked metadata" + } else { + $cacheDir = Join-Path $global:CacheDir 'azure-acr-credential-provider' + try { + if (Test-Path -LiteralPath $cacheDir -PathType Container -ErrorAction Stop) { + $files = Get-ChildItem -LiteralPath $cacheDir -File -ErrorAction Stop + } else { + Write-Log "Dalec credential provider cache directory '$cacheDir' is missing; checking baked metadata" + } + } catch { + throw "Failed to inspect Dalec credential provider cache '$cacheDir': $_" + } + } + + $selectedFile = $null + $selectedVersion = $null + $selectedPackageVersion = $null + foreach ($file in $files) { + $match = [regex]::Match($file.Name, '\Aazure-acr-credential-provider_(?.+)_amd64\.zip\z') + if ($file.PSIsContainer -or -not $match.Success) { + continue + } + $candidateVersion = ConvertTo-DalecCredentialProviderVersion -PackageVersion $match.Groups['packageVersion'].Value -KubeVersion $kubeVersion + if ($null -eq $candidateVersion) { + continue + } + if ($null -eq $selectedVersion -or $candidateVersion -gt $selectedVersion) { + $selectedVersion = $candidateVersion + $selectedFile = $file.FullName + $selectedPackageVersion = $match.Groups['packageVersion'].Value + } + } + if ($null -eq $selectedFile) { + Write-Log "No cached Dalec credential provider for Kubernetes $($kubeVersion.Major).$($kubeVersion.Minor) on $architecture; checking baked metadata" + return Resolve-DalecCredentialProviderPackageFromMetadata -KubeVersion $kubeVersion + } + Write-Log "Selected cached Dalec credential provider v$selectedPackageVersion from '$selectedFile'" + return @{ + Version = $selectedPackageVersion + CachedFile = $selectedFile + } +} + +# Called only after cache discovery misses, so cached packages do not depend on metadata. +function Resolve-DalecCredentialProviderPackageFromMetadata { + Param( + [Parameter(Mandatory=$true)][version]$KubeVersion + ) + $metadataPath = 'c:\k\components.json' + if (-not (Test-Path -LiteralPath $metadataPath -PathType Leaf -ErrorAction Stop)) { + Write-Log "Dalec credential provider metadata not found at '$metadataPath'; falling back to RP URL" + return $null + } + $content = Get-Content -LiteralPath $metadataPath -ErrorAction Stop | Out-String + try { + $metadata = $content | ConvertFrom-Json -ErrorAction Stop + } catch [System.ArgumentException] { + Write-Log "Invalid Dalec credential provider metadata in '$metadataPath'; falling back to RP URL. Error: $_" + return $null + } + # Older VHDs placed newer Windows pins under PMC; retain that precedence for their metadata. + # Consolidated metadata keeps Windows pins only in the dedicated Dalec component. + $package = $metadata.Packages | Where-Object { $_.name -eq 'azure-acr-credential-provider-pmc' } | Select-Object -First 1 + if ($null -eq $package.downloadURIs.windows.default) { + $package = $metadata.Packages | Where-Object { $_.name -eq 'windows credential provider dalec' } | Select-Object -First 1 + } + $windowsPackage = $package.downloadURIs.windows.default + $selectedVersion = $null + $selectedPackageVersion = $null + foreach ($entry in $windowsPackage.versionsV2) { + $candidateVersion = ConvertTo-DalecCredentialProviderVersion -PackageVersion ([string]$entry.latestVersion) -KubeVersion $KubeVersion + if ($null -eq $candidateVersion) { + continue + } + if ($null -eq $selectedVersion -or $candidateVersion -gt $selectedVersion) { + $selectedVersion = $candidateVersion + $selectedPackageVersion = $entry.latestVersion + } + } + if ($null -eq $selectedVersion -or $windowsPackage.downloadURL -isnot [string] -or [string]::IsNullOrEmpty($windowsPackage.downloadURL)) { + Write-Log "No Dalec credential provider download in baked metadata for Kubernetes $($KubeVersion.Major).$($KubeVersion.Minor); falling back to RP URL" + return $null + } + # Substitute the supported template tokens literally; never evaluate metadata as PowerShell. + $resolvedUrl = $windowsPackage.downloadURL.Replace('${version}', $selectedPackageVersion) + $resolvedUrl = $resolvedUrl.Replace('$($version.Split(''-'')[0])', $selectedPackageVersion.Split('-')[0]) + $uri = $null + if ($resolvedUrl.Contains('$') -or -not [uri]::TryCreate($resolvedUrl, [UriKind]::Absolute, [ref]$uri) -or + $uri.Scheme -ne 'https' -or + [IO.Path]::GetFileName($uri.AbsolutePath) -cne "azure-acr-credential-provider_${selectedPackageVersion}_amd64.zip") { + Write-Log "Invalid Dalec credential provider download URL in baked metadata; falling back to RP URL" + return $null + } + Write-Log "Selected Dalec credential provider v$selectedPackageVersion from baked metadata for download" + return @{ + Version = $selectedPackageVersion + CachedFile = $null + Url = $uri.AbsoluteUri + } +} + +# Invoke-RestMethod -TimeoutSec on Windows PowerShell only bounds receiving response headers; a +# stalled body can still block for ReadWriteTimeout (300s) per read. TimeoutSec > 0 bounds the whole +# attempt (connect, headers and body) by waiting on async operations against one deadline and aborting +# the request when it passes. TimeoutSec = 0 keeps the synchronous .NET defaults used by Invoke-RestMethod. +function Save-DalecCredentialProviderHttpResponse { + Param( + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$Url, + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$DestinationPath, + [Parameter(Mandatory=$true)][ValidateRange(0, 2147483)][int]$TimeoutSec + ) + $path = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($DestinationPath) + $timeoutMs = [long]$TimeoutSec * 1000 + $timer = [System.Diagnostics.Stopwatch]::StartNew() + $request = [System.Net.WebRequest]::Create($Url) + $request.Method = 'GET' + $waitForTask = { + param($Task) + $remainingMs = $timeoutMs - $timer.ElapsedMilliseconds + # WaitAny does not throw for faulted tasks; GetResult then surfaces the original exception. + if ($remainingMs -le 0 -or [System.Threading.Tasks.Task]::WaitAny(@($Task), [int]$remainingMs) -lt 0) { + $request.Abort() + throw [System.TimeoutException]::new("Download of '$Url' exceeded $TimeoutSec seconds.") + } + return $Task.GetAwaiter().GetResult() + } + $response = $null + $source = $null + $target = $null + try { + $response = if ($timeoutMs -gt 0) { & $waitForTask $request.GetResponseAsync() } else { $request.GetResponse() } + $source = $response.GetResponseStream() + $target = [System.IO.File]::Create($path) + $buffer = New-Object byte[] 81920 + $totalBytes = [long]0 + while ($true) { + $read = if ($timeoutMs -gt 0) { + & $waitForTask $source.ReadAsync($buffer, 0, $buffer.Length) + } else { + $source.Read($buffer, 0, $buffer.Length) + } + if ($read -le 0) { + break + } + $target.Write($buffer, 0, $read) + $totalBytes += $read + } + if ($response.ContentLength -ge 0 -and $totalBytes -ne $response.ContentLength) { + throw "Download of '$Url' was truncated: received $totalBytes of $($response.ContentLength) bytes." + } + } catch { + if ($null -ne $target) { + $target.Dispose() + $target = $null + Remove-Item -LiteralPath $path -Force -ErrorAction SilentlyContinue + } + throw + } finally { + if ($null -ne $target) { $target.Dispose() } + if ($null -ne $source) { $source.Dispose() } + if ($null -ne $response) { $response.Dispose() } + } +} + +# Throws rather than exiting so the caller can fall back to the RP cache/URL. +# Defaults match DownloadFileOverHttp; pass a smaller budget only when a fallback exists. +function Get-DalecCredentialProviderPackage { + Param( + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$Url, + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$DestinationPath, + [Parameter(Mandatory=$false)][ValidateRange(1, 100)][int]$Retries = 5, + [Parameter(Mandatory=$false)][ValidateRange(0, 3600)][int]$RetryDelaySeconds = 10, + [Parameter(Mandatory=$false)][ValidateRange(0, 3600)][int]$TimeoutSec = 0 + ) + # Match the shared downloader for Windows PowerShell's .NET Framework HTTP stack. + $secureProtocols = @() + $insecureProtocols = @([System.Net.SecurityProtocolType]::SystemDefault, [System.Net.SecurityProtocolType]::Ssl3) + foreach ($protocol in [System.Enum]::GetValues([System.Net.SecurityProtocolType])) { + if ($insecureProtocols -notcontains $protocol) { + $secureProtocols += $protocol + } + } + [System.Net.ServicePointManager]::SecurityProtocol = $secureProtocols + + $mappedUrl = Update-BaseUrl -InitialUrl $Url + Write-Log "Downloading dalec credential provider $Url -> $mappedUrl to $DestinationPath (attempts: $Retries, per-attempt timeout: $(if ($TimeoutSec -gt 0) { "$TimeoutSec s" } else { 'default' }), retry delay: $RetryDelaySeconds s)" + $downloadTimer = [System.Diagnostics.Stopwatch]::StartNew() + try { + $arglist = @{ Url = $mappedUrl; DestinationPath = $DestinationPath; TimeoutSec = $TimeoutSec } + Retry-Command -Command "Save-DalecCredentialProviderHttpResponse" -Args $arglist -Retries $Retries -RetryDelaySeconds $RetryDelaySeconds + } finally { + $downloadTimer.Stop() + Write-Log "Dalec credential provider download elapsed time: $($downloadTimer.ElapsedMilliseconds) ms ($mappedUrl)" + } + + if ($null -ne $global:AppInsightsClient) { + $evt = New-Object "Microsoft.ApplicationInsights.DataContracts.EventTelemetry" + $evt.Name = "FileDownload" + $evt.Properties["FileName"] = [IO.Path]::GetFileName(([uri]$mappedUrl).AbsolutePath) + $evt.Metrics["DurationMs"] = $downloadTimer.ElapsedMilliseconds + $global:AppInsightsClient.TrackEvent($evt) + } +} + +# Throws on extraction failure so installation can fall back to the RP URL. +function Expand-DalecCredentialProviderPackage { + Param( + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$Path, + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$DestinationPath + ) + Expand-Archive -Path $Path -DestinationPath $DestinationPath -Force -ErrorAction Stop +} + function Install-CredentialProvider { Param( [Parameter(Mandatory=$true)][string] @@ -587,14 +847,78 @@ function Install-CredentialProvider { Write-Log "Create credential provider configuration file" Config-CredentialProvider -KubeDir $KubeDir -CredentialProviderConfPath $global:credentialProviderConfigPath -CustomCloudContainerRegistryDNSSuffix $CustomCloudContainerRegistryDNSSuffix - Write-Log "Download credential provider binary from $global:CredentialProviderURL to $global:credentialProviderBinDir" + Write-Log "Preparing credential provider installation in $global:credentialProviderBinDir" $tempDir = New-TemporaryDirectory if ([string]::IsNullOrEmpty($global:BootstrapProfileContainerRegistryServer)) { - $credentialproviderbinaryPackage = "$tempDir\credentialprovider.tar.gz" - DownloadFileOverHttp -Url $global:CredentialProviderURL -DestinationPath $credentialproviderbinaryPackage -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CREDEDNTIAL_PROVIDER - tar -xzf $credentialproviderbinaryPackage -C $tempDir - if ($LASTEXITCODE -ne 0) { - throw "Failed to extract the '$credentialproviderbinaryPackage' archive." + # Only select Dalec when the RP URL is empty or matches a stock legacy URL. + # If the caller provided a custom/hotfix URL, honor it as-is. + $credentialProviderUri = $null + $isCredentialProviderUri = $false + if (-not [string]::IsNullOrEmpty($global:CredentialProviderURL)) { + $isCredentialProviderUri = [Uri]::TryCreate( + [string]$global:CredentialProviderURL, + [UriKind]::Absolute, + [ref]$credentialProviderUri + ) + } + $stockLegacyHosts = @( + 'packages.aks.azure.com', + 'packages.aks.azure.us', + 'acs-mirror.azureedge.net' + ) + $allowDalecSelection = [string]::IsNullOrEmpty($global:CredentialProviderURL) -or + ($isCredentialProviderUri -and + $credentialProviderUri.Host -in $stockLegacyHosts -and + $credentialProviderUri.AbsolutePath -match '/cloud-provider-azure/v\d+\.\d+\.\d+/binaries/azure-acr-credential-provider-windows-amd64-v\d+\.\d+\.\d+\.tar\.gz$') + $dalecPackage = if ($allowDalecSelection) { + Resolve-DalecCredentialProviderPackage -KubeBinariesVersion $global:KubeBinariesVersion + } else { $null } + + $useLegacyPackage = $null -eq $dalecPackage + if (-not $useLegacyPackage) { + Write-Log "Using dalec credential provider v$($dalecPackage.Version)" + try { + if ($dalecPackage.CachedFile) { + Write-Log "Using cached dalec package: $($dalecPackage.CachedFile)" + $credentialproviderbinaryPackage = $dalecPackage.CachedFile + } else { + $credentialproviderbinaryPackage = "$tempDir\credentialprovider.zip" + $dalecUri = [UriBuilder]::new($dalecPackage.Url) + if ($isCredentialProviderUri) { + $dalecUri.Host = $credentialProviderUri.Host + } + $downloadBudget = if ([string]::IsNullOrEmpty($global:CredentialProviderURL)) { + # No RP fallback: a failed download fails provisioning, so keep the shared downloader's budget. + @{ Retries = 5; RetryDelaySeconds = 10; TimeoutSec = 0 } + } else { + # RP fallback exists: cap the Dalec attempt at two 15-second tries before falling back. + @{ Retries = 2; RetryDelaySeconds = 1; TimeoutSec = 15 } + } + Get-DalecCredentialProviderPackage -Url $dalecUri.Uri.AbsoluteUri -DestinationPath $credentialproviderbinaryPackage @downloadBudget + } + Expand-DalecCredentialProviderPackage -Path $credentialproviderbinaryPackage -DestinationPath $tempDir + if (-not (Test-Path -LiteralPath "$tempDir\azure-acr-credential-provider.exe" -PathType Leaf -ErrorAction Stop)) { + throw "Dalec credential provider archive '$credentialproviderbinaryPackage' does not contain azure-acr-credential-provider.exe." + } + } catch { + if ([string]::IsNullOrEmpty($global:CredentialProviderURL)) { + throw + } + Write-Log "Failed to install dalec credential provider, falling back to RP URL. Error: $_" + $useLegacyPackage = $true + } + } + if ($useLegacyPackage) { + # If neither Dalec source works, the shared helper checks the RP cache before downloading. + if ([string]::IsNullOrEmpty($global:CredentialProviderURL)) { + throw "No credential provider source: CredentialProviderURL is empty and neither a cached Dalec package nor a matching download in baked metadata is available for Kubernetes $global:KubeBinariesVersion on this node." + } + $credentialproviderbinaryPackage = "$tempDir\credentialprovider.tar.gz" + DownloadFileOverHttp -Url $global:CredentialProviderURL -DestinationPath $credentialproviderbinaryPackage -ExitCode $global:WINDOWS_CSE_ERROR_DOWNLOAD_CREDEDNTIAL_PROVIDER + tar -xzf $credentialproviderbinaryPackage -C $tempDir + if ($LASTEXITCODE -ne 0) { + throw "Failed to extract the '$credentialproviderbinaryPackage' archive." + } } } else { # network isolated cluster @@ -629,10 +953,10 @@ function Install-CredentialProvider { } Create-Directory -FullPath $global:credentialProviderBinDir - cp "$tempDir\azure-acr-credential-provider.exe" "$global:credentialProviderBinDir\acr-credential-provider.exe" + Copy-Item "$tempDir\azure-acr-credential-provider.exe" "$global:credentialProviderBinDir\acr-credential-provider.exe" -ErrorAction Stop # acr-credential-provider.exe cannot be found by kubelet through provider name before the fix https://github.com/kubernetes/kubernetes/pull/120291 # so we copy the exe file to acr-credential-provider to make all 1.29 release work. - cp "$global:credentialProviderBinDir\acr-credential-provider.exe" "$global:credentialProviderBinDir\acr-credential-provider" + Copy-Item "$global:credentialProviderBinDir\acr-credential-provider.exe" "$global:credentialProviderBinDir\acr-credential-provider" -ErrorAction Stop del $tempDir -Recurse } catch { Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_INSTALL_CREDENTIAL_PROVIDER -ErrorMessage "Error installing credential provider. Error: $_" diff --git a/staging/cse/windows/configfunc.tests.ps1 b/staging/cse/windows/configfunc.tests.ps1 index 4e94c326d84..bed15a5f1a5 100644 --- a/staging/cse/windows/configfunc.tests.ps1 +++ b/staging/cse/windows/configfunc.tests.ps1 @@ -569,7 +569,34 @@ Describe 'Install-OpenSSH' { } Describe 'Install-CredentialProvider' { + BeforeAll { + $script:savedCacheDir = $global:CacheDir + $script:savedArchitecture = $env:PROCESSOR_ARCHITECTURE + $script:savedNativeArchitecture = $env:PROCESSOR_ARCHITEW6432 + $script:savedPackageDownloadFqdn = $global:PackageDownloadFqdn + $script:downloadCredentialProvider = ${function:DownloadFileOverHttp} + function Write-Log { + param([string]$Message) + $script:credentialProviderLogs.Add($Message) + } + } + AfterAll { + $global:CacheDir = $script:savedCacheDir + $env:PROCESSOR_ARCHITECTURE = $script:savedArchitecture + $env:PROCESSOR_ARCHITEW6432 = $script:savedNativeArchitecture + $global:PackageDownloadFqdn = $script:savedPackageDownloadFqdn + } + BeforeEach { + $script:credentialProviderLogs = [System.Collections.Generic.List[string]]::new() + $script:expectedMetadataReads = 0 + $script:expectedDalecDownloads = 0 + $env:PROCESSOR_ARCHITECTURE = 'AMD64' + $env:PROCESSOR_ARCHITEW6432 = $null + $global:CacheDir = Join-Path $TestDrive ([guid]::NewGuid().ToString()) + $script:dalecCacheDir = Join-Path $global:CacheDir 'azure-acr-credential-provider' + New-Item -ItemType Directory -Path $script:dalecCacheDir -Force | Out-Null + $script:cachedDalecFile = (New-Item -ItemType File -Path (Join-Path $script:dalecCacheDir 'azure-acr-credential-provider_1.33.6-1_amd64.zip')).FullName $global:credentialProviderConfigPath = "" $global:credentialProviderBinDir = "" $global:KubeletConfigArgs = @( @@ -593,8 +620,15 @@ Describe 'Install-CredentialProvider' { $script:lastDownloadReference = $Reference } Mock AKS-Expand-Archive + Mock Save-DalecCredentialProviderHttpResponse -MockWith { throw 'Unexpected Dalec download' } + Mock Test-Path -MockWith { $false } -ParameterFilter { $LiteralPath -eq 'c:\k\components.json' } + Mock Get-Content -MockWith { throw 'Unexpected metadata read' } + Mock Expand-DalecCredentialProviderPackage + Mock Test-Path -MockWith { $true } -ParameterFilter { + $LiteralPath -eq 'C:\temp\credprovider\azure-acr-credential-provider.exe' -and $PathType -eq 'Leaf' + } Mock Create-Directory - Mock cp + Mock Copy-Item Mock del Mock tar -MockWith { $global:LASTEXITCODE = 0 } Mock Get-Command -MockWith { @@ -607,9 +641,217 @@ Describe 'Install-CredentialProvider' { } } AfterEach { + Assert-MockCalled Save-DalecCredentialProviderHttpResponse -Times $script:expectedDalecDownloads -Exactly + Assert-MockCalled Get-Content -Times $script:expectedMetadataReads -Exactly + Assert-MockCalled del -Times 0 -ParameterFilter { $Path -like "$global:CacheDir*" } + Assert-MockCalled Copy-Item -Times 0 -ParameterFilter { $Destination -like "$global:CacheDir*" } $global:BootstrapProfileContainerRegistryServer = $null } + Context 'when a Dalec download is present in metadata' { + BeforeEach { + $global:BootstrapProfileContainerRegistryServer = '' + $global:KubeBinariesVersion = '1.34.0' + $global:CredentialProviderURL = 'https://packages.aks.azure.com/cloud-provider-azure/v1.34.0/binaries/azure-acr-credential-provider-windows-amd64-v1.34.0.tar.gz' + $global:PackageDownloadFqdn = 'packages.aks.azure.com' + $script:expectedMetadataReads = 1 + $script:expectedDalecDownloads = 1 + Mock Test-Path -MockWith { $true } -ParameterFilter { $LiteralPath -eq 'c:\k\components.json' } + Mock Get-Content -MockWith { + @' +{"Packages":[{"name":"azure-acr-credential-provider-pmc","downloadURIs":{"windows":{"default":{ + "versionsV2":[{"latestVersion":"1.34.19-1"}], + "downloadURL":"https://packages.aks.azure.com/dalec-packages/azure-acr-credential-provider/$($version.Split('-')[0])/windows/amd64/azure-acr-credential-provider_${version}_amd64.zip" +}}}}]} +'@ + } + Mock Save-DalecCredentialProviderHttpResponse + Mock Resolve-PackagesDownloadFqdn -MockWith { throw 'Unexpected public endpoint probe' } + } + + It 'uses an older matching cached ZIP without reading or downloading newer metadata' { + $file = New-Item -ItemType File -Path (Join-Path $script:dalecCacheDir 'azure-acr-credential-provider_1.34.3-1_amd64.zip') + $script:expectedMetadataReads = 0 + $script:expectedDalecDownloads = 0 + Mock Get-Content -MockWith { throw 'A cache hit must not initialize metadata' } + + Install-CredentialProvider -KubeDir 'c:\k' + Assert-MockCalled Expand-DalecCredentialProviderPackage -Times 1 -ParameterFilter { $Path -eq $file.FullName } + Assert-MockCalled DownloadFileOverHttp -Times 0 + Assert-MockCalled Test-Path -Times 0 -ParameterFilter { $LiteralPath -eq 'c:\k\components.json' } + } + + It 'downloads the pinned Dalec ZIP into temporary storage after a cache miss' { + Install-CredentialProvider -KubeDir 'c:\k' + Assert-MockCalled Save-DalecCredentialProviderHttpResponse -Times 1 -Exactly -ParameterFilter { + $Url -eq 'https://packages.aks.azure.com/dalec-packages/azure-acr-credential-provider/1.34.19/windows/amd64/azure-acr-credential-provider_1.34.19-1_amd64.zip' -and + $DestinationPath -eq 'C:\temp\credprovider\credentialprovider.zip' + } + Assert-MockCalled Expand-DalecCredentialProviderPackage -Times 1 -ParameterFilter { + $Path -eq 'C:\temp\credprovider\credentialprovider.zip' -and $DestinationPath -eq 'C:\temp\credprovider' + } + Assert-MockCalled DownloadFileOverHttp -Times 0 + $script:credentialProviderLogs | Should -Contain 'Using dalec credential provider v1.34.19-1' + ($script:credentialProviderLogs -join "`n") | Should -Not -Match 'Failed to install dalec credential provider' + } + + It 'still checks metadata when the entire cache directory is missing' { + $global:CacheDir = Join-Path $TestDrive 'missing-cache' + Install-CredentialProvider -KubeDir 'c:\k' + Assert-MockCalled Expand-DalecCredentialProviderPackage -Times 1 + Assert-MockCalled DownloadFileOverHttp -Times 0 + } + + It 'downloads from metadata when the RP URL is empty' { + $global:CredentialProviderURL = '' + Install-CredentialProvider -KubeDir 'c:\k' + Assert-MockCalled Save-DalecCredentialProviderHttpResponse -Times 1 -Exactly -ParameterFilter { $TimeoutSec -eq 0 } + Assert-MockCalled Expand-DalecCredentialProviderPackage -Times 1 + Assert-MockCalled DownloadFileOverHttp -Times 0 + } + + It 'keeps the full retry budget for transient failures when no RP URL is available' { + $global:CredentialProviderURL = '' + $script:expectedDalecDownloads = 5 + $script:dalecAttempts = 0 + Mock Save-DalecCredentialProviderHttpResponse -MockWith { + $script:dalecAttempts++ + if ($script:dalecAttempts -lt 5) { + throw 'Transient Dalec download failure' + } + } + Mock Start-Sleep + + Install-CredentialProvider -KubeDir 'c:\k' + Assert-MockCalled Save-DalecCredentialProviderHttpResponse -Times 5 -Exactly -ParameterFilter { $TimeoutSec -eq 0 } + Assert-MockCalled Start-Sleep -Times 4 -Exactly -ParameterFilter { $Seconds -eq 10 } + Assert-MockCalled Expand-DalecCredentialProviderPackage -Times 1 -Exactly + Assert-MockCalled DownloadFileOverHttp -Times 0 + Assert-MockCalled Set-ExitCode -Times 0 + } + + It 'fails provisioning only after the full retry budget when no RP URL is available' { + $global:CredentialProviderURL = '' + $script:expectedDalecDownloads = 5 + Mock Save-DalecCredentialProviderHttpResponse -MockWith { throw 'Dalec download failed' } + Mock Start-Sleep + + { Install-CredentialProvider -KubeDir 'c:\k' } | Should -Throw '*Dalec download failed*' + Assert-MockCalled Save-DalecCredentialProviderHttpResponse -Times 5 -Exactly + Assert-MockCalled Start-Sleep -Times 4 -Exactly -ParameterFilter { $Seconds -eq 10 } + Assert-MockCalled DownloadFileOverHttp -Times 0 + Assert-MockCalled Set-ExitCode -Times 1 -Exactly -ParameterFilter { + $ExitCode -eq $global:WINDOWS_CSE_ERROR_INSTALL_CREDENTIAL_PROVIDER + } + } + + It 'preserves sovereign and public mirror mapping for ' -TestCases @( + @{ rpHost = 'packages.aks.azure.us'; preferredHost = 'packages.aks.azure.com'; downloadHost = 'packages.aks.azure.us' }, + @{ rpHost = 'packages.aks.azure.com'; preferredHost = 'acs-mirror.azureedge.net'; downloadHost = 'acs-mirror.azureedge.net' } + ) { + param($rpHost, $preferredHost, $downloadHost) + $global:CredentialProviderURL = $global:CredentialProviderURL.Replace('packages.aks.azure.com', $rpHost) + $global:PackageDownloadFqdn = $preferredHost + Install-CredentialProvider -KubeDir 'c:\k' + Assert-MockCalled Save-DalecCredentialProviderHttpResponse -Times 1 -Exactly -ParameterFilter { + $Url -eq "https://$downloadHost/dalec-packages/azure-acr-credential-provider/1.34.19/windows/amd64/azure-acr-credential-provider_1.34.19-1_amd64.zip" + } + Assert-MockCalled Resolve-PackagesDownloadFqdn -Times 0 + Assert-MockCalled DownloadFileOverHttp -Times 0 + } + + It 'bounds failed Dalec downloads before falling back to the RP path' { + $script:expectedDalecDownloads = 2 + Mock Save-DalecCredentialProviderHttpResponse -MockWith { throw 'Dalec download failed' } + Mock Start-Sleep + + Install-CredentialProvider -KubeDir 'c:\k' + Assert-MockCalled Save-DalecCredentialProviderHttpResponse -Times 2 -Exactly -ParameterFilter { $TimeoutSec -eq 15 } + Assert-MockCalled Start-Sleep -Times 1 -Exactly + Assert-MockCalled Start-Sleep -Times 1 -Exactly -ParameterFilter { $Seconds -eq 1 } + Assert-MockCalled DownloadFileOverHttp -Times 1 -Exactly -ParameterFilter { $Url -eq $global:CredentialProviderURL } + Assert-MockCalled Expand-DalecCredentialProviderPackage -Times 0 + Assert-MockCalled tar -Times 1 + ($script:credentialProviderLogs -join "`n") | Should -Match 'Failed to install dalec credential provider, falling back to RP URL.' + } + + It 'succeeds on the second bounded Dalec attempt without falling back' { + $script:expectedDalecDownloads = 2 + $script:dalecAttempts = 0 + Mock Save-DalecCredentialProviderHttpResponse -MockWith { + $script:dalecAttempts++ + if ($script:dalecAttempts -eq 1) { + throw 'Transient Dalec download failure' + } + } + Mock Start-Sleep + + Install-CredentialProvider -KubeDir 'c:\k' + Assert-MockCalled Save-DalecCredentialProviderHttpResponse -Times 2 -Exactly -ParameterFilter { $TimeoutSec -eq 15 } + Assert-MockCalled Start-Sleep -Times 1 -Exactly -ParameterFilter { $Seconds -eq 1 } + Assert-MockCalled Expand-DalecCredentialProviderPackage -Times 1 -Exactly + Assert-MockCalled DownloadFileOverHttp -Times 0 + } + + It 'reuses an available RP archive after exhausting the bounded Dalec download budget' { + $script:expectedDalecDownloads = 2 + $script:dalecAttempts = 0 + $legacyFile = New-Item -ItemType File -Path (Join-Path $global:CacheDir 'azure-acr-credential-provider-windows-amd64-v1.34.0.tar.gz') + Mock Save-DalecCredentialProviderHttpResponse -MockWith { + $script:dalecAttempts++ + throw 'Dalec request timed out' + } + Mock Start-Sleep + Mock DownloadFileOverHttp -MockWith { + param($Url, $DestinationPath, $ExitCode) + $script:dalecAttempts | Should -Be 2 + & $script:downloadCredentialProvider -Url $Url -DestinationPath $DestinationPath -ExitCode $ExitCode + } + Mock Copy-Item + + Install-CredentialProvider -KubeDir 'c:\k' + Assert-MockCalled Save-DalecCredentialProviderHttpResponse -Times 2 -Exactly -ParameterFilter { $TimeoutSec -eq 15 } + Assert-MockCalled Start-Sleep -Times 1 -Exactly + Assert-MockCalled Start-Sleep -Times 1 -Exactly -ParameterFilter { $Seconds -eq 1 } + Assert-MockCalled DownloadFileOverHttp -Times 1 -Exactly + Assert-MockCalled Expand-DalecCredentialProviderPackage -Times 0 + Assert-MockCalled tar -Times 1 -Exactly + Assert-MockCalled Copy-Item -Times 1 -Exactly -ParameterFilter { + $Path -eq $legacyFile.FullName -and $Destination -eq 'C:\temp\credprovider\credentialprovider.tar.gz' + } + } + + It 'falls back to the RP path if the downloaded Dalec ZIP cannot be extracted' { + Mock Expand-DalecCredentialProviderPackage -MockWith { throw 'Invalid downloaded ZIP' } + Install-CredentialProvider -KubeDir 'c:\k' + Assert-MockCalled DownloadFileOverHttp -Times 1 -Exactly -ParameterFilter { $Url -eq $global:CredentialProviderURL } + Assert-MockCalled tar -Times 1 + } + + It 'fails with the download error when no RP URL is available' { + $global:CredentialProviderURL = '' + $script:expectedDalecDownloads = 0 + Mock Get-DalecCredentialProviderPackage -MockWith { throw 'Dalec download failed' } + { Install-CredentialProvider -KubeDir 'c:\k' } | Should -Throw '*Dalec download failed*' + Assert-MockCalled DownloadFileOverHttp -Times 0 + } + + It 'uses the RP path when metadata does not list the requested minor' { + $global:KubeBinariesVersion = '1.35.0' + $script:expectedDalecDownloads = 0 + Install-CredentialProvider -KubeDir 'c:\k' + Assert-MockCalled DownloadFileOverHttp -Times 1 -Exactly -ParameterFilter { $Url -eq $global:CredentialProviderURL } + } + + It 'uses the RP path when metadata is malformed' { + $script:expectedDalecDownloads = 0 + Mock Get-Content -MockWith { '{malformed' } + Install-CredentialProvider -KubeDir 'c:\k' + Assert-MockCalled DownloadFileOverHttp -Times 1 -Exactly -ParameterFilter { $Url -eq $global:CredentialProviderURL } + ($script:credentialProviderLogs -join "`n") | Should -Match 'Invalid Dalec credential provider metadata.*falling back to RP URL' + } + } + It 'returns early when out-of-tree credential provider flags are not configured' { $global:KubeletConfigArgs = @("--address=0.0.0.0") @@ -642,6 +884,239 @@ Describe 'Install-CredentialProvider' { Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' $script:lastDownloadReference | Should -Be 'myregistry.azurecr.io/aks/packages/kubernetes/azure-acr-credential-provider:v1.31.9' } + + It 'extracts the cached Dalec ZIP directly into a temporary directory with a stock legacy RP URL' { + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.33.3" + $global:CredentialProviderURL = 'https://packages.aks.azure.com/cloud-provider-azure/v1.33.3/binaries/azure-acr-credential-provider-windows-amd64-v1.33.3.tar.gz' + + Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' + Assert-MockCalled -CommandName 'Expand-DalecCredentialProviderPackage' -Times 1 -Exactly -ParameterFilter { + $Path -eq $script:cachedDalecFile -and $DestinationPath -eq 'C:\temp\credprovider' + } + Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 0 + Assert-MockCalled -CommandName 'tar' -Times 0 + Assert-MockCalled -CommandName 'del' -Times 1 -Exactly -ParameterFilter { $Path -eq 'C:\temp\credprovider' } + $script:credentialProviderLogs | Should -Contain 'Using dalec credential provider v1.33.6-1' + ($script:credentialProviderLogs -join "`n") | Should -Not -Match 'Failed to install dalec credential provider, falling back to RP URL.' + } + + It 'uses dalec path for k8s >= 1.33 when the RP URL is empty' { + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.33.3" + $global:CredentialProviderURL = "" + + Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' + Assert-MockCalled -CommandName 'Expand-DalecCredentialProviderPackage' -Times 1 -ParameterFilter { $Path -eq $script:cachedDalecFile } + Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 0 + Assert-MockCalled -CommandName 'tar' -Times 0 + } + + It 'uses cached Dalec or the sovereign RP URL with cache hit ' -TestCases @( + @{ cacheHit = $true }, + @{ cacheHit = $false } + ) { + param($cacheHit) + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.33.3" + $global:CredentialProviderURL = 'https://packages.aks.azure.us/cloud-provider-azure/v1.33.3/binaries/azure-acr-credential-provider-windows-amd64-v1.33.3.tar.gz' + + if (-not $cacheHit) { + $global:KubeBinariesVersion = "1.34.0" + } + Mock Resolve-PackagesDownloadFqdn -MockWith { throw 'Sovereign downloads must not probe public endpoints' } + + Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' + if ($cacheHit) { + Assert-MockCalled -CommandName 'Expand-DalecCredentialProviderPackage' -Times 1 -ParameterFilter { + $Path -eq $script:cachedDalecFile + } + Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 0 + Assert-MockCalled -CommandName 'tar' -Times 0 + } else { + Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 1 -Exactly -ParameterFilter { + $Url -eq $global:CredentialProviderURL + } + Assert-MockCalled -CommandName 'Expand-DalecCredentialProviderPackage' -Times 0 + Assert-MockCalled -CommandName 'tar' -Times 1 + } + Assert-MockCalled -CommandName 'Resolve-PackagesDownloadFqdn' -Times 0 + } + + It 'uses dalec path for k8s >= 1.33 with the stock legacy mirror URL' { + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.33.3" + $global:CredentialProviderURL = 'https://acs-mirror.azureedge.net/cloud-provider-azure/v1.33.3/binaries/azure-acr-credential-provider-windows-amd64-v1.33.3.tar.gz' + + Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' + Assert-MockCalled -CommandName 'Expand-DalecCredentialProviderPackage' -Times 1 + Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 0 + Assert-MockCalled -CommandName 'tar' -Times 0 + } + + It 'falls back to RP legacy URL when the cache directory is missing' { + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.33.3" + $global:CredentialProviderURL = 'https://packages.aks.azure.us/cloud-provider-azure/v1.33.3/binaries/azure-acr-credential-provider-windows-amd64-v1.33.3.tar.gz' + + $global:CacheDir = Join-Path $TestDrive 'missing-cache' + + Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' + Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 1 -ParameterFilter { + $Url -eq $global:CredentialProviderURL + } + Assert-MockCalled -CommandName 'Expand-DalecCredentialProviderPackage' -Times 0 + Assert-MockCalled -CommandName 'tar' -Times 1 + ($script:credentialProviderLogs -join "`n") | Should -Match 'cache directory .* is missing; checking baked metadata' + ($script:credentialProviderLogs -join "`n") | Should -Match 'metadata not found.*falling back to RP URL' + } + + It 'reuses a legacy archive through the unchanged shared helper on a Dalec cache miss' { + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.34.0" + $global:CredentialProviderURL = 'https://packages.aks.azure.com/cloud-provider-azure/v1.34.0/binaries/azure-acr-credential-provider-windows-amd64-v1.34.0.tar.gz' + $legacyFile = New-Item -ItemType File -Path (Join-Path $global:CacheDir 'azure-acr-credential-provider-windows-amd64-v1.34.0.tar.gz') + Mock DownloadFileOverHttp -MockWith { + param($Url, $DestinationPath, $ExitCode) + & $script:downloadCredentialProvider -Url $Url -DestinationPath $DestinationPath -ExitCode $ExitCode + } + Mock Copy-Item + + Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' + Assert-MockCalled Copy-Item -Times 1 -Exactly -ParameterFilter { + $Path -eq $legacyFile.FullName -and $Destination -eq 'C:\temp\credprovider\credentialprovider.tar.gz' + } + Assert-MockCalled Expand-DalecCredentialProviderPackage -Times 0 + Assert-MockCalled tar -Times 1 + } + + It 'falls back to RP legacy URL when dalec extraction fails' { + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.33.3" + $global:CredentialProviderURL = 'https://packages.aks.azure.us/cloud-provider-azure/v1.33.3/binaries/azure-acr-credential-provider-windows-amd64-v1.33.3.tar.gz' + + Mock Expand-DalecCredentialProviderPackage -MockWith { + throw 'dalec extraction failed' + } + + Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' + Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 1 -ParameterFilter { + $Url -eq $global:CredentialProviderURL + } + Assert-MockCalled -CommandName 'Expand-DalecCredentialProviderPackage' -Times 1 + Assert-MockCalled -CommandName 'tar' -Times 1 + ($script:credentialProviderLogs -join "`n") | Should -Match 'Failed to install dalec credential provider, falling back to RP URL. Error: dalec extraction failed' + } + + It 'fails with the extraction error when a cached archive is unusable and the RP URL is empty' { + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.33.3" + $global:CredentialProviderURL = "" + Mock Expand-DalecCredentialProviderPackage -MockWith { throw 'dalec extraction failed' } + + { Install-CredentialProvider -KubeDir 'c:\k' } | Should -Throw '*dalec extraction failed*' + Assert-MockCalled DownloadFileOverHttp -Times 0 + Assert-MockCalled tar -Times 0 + Assert-MockCalled Set-ExitCode -Times 1 -ParameterFilter { $ExitCode -eq $global:WINDOWS_CSE_ERROR_INSTALL_CREDENTIAL_PROVIDER } + } + + It 'surfaces a cache access error instead of silently falling back to the RP URL' { + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.33.3" + $global:CredentialProviderURL = 'https://packages.aks.azure.com/cloud-provider-azure/v1.33.3/binaries/azure-acr-credential-provider-windows-amd64-v1.33.3.tar.gz' + Mock Get-ChildItem -MockWith { Write-Error 'Cache access denied' } + + { Install-CredentialProvider -KubeDir 'c:\k' } | Should -Throw '*Failed to inspect Dalec credential provider cache*Cache access denied*' + Assert-MockCalled DownloadFileOverHttp -Times 0 + Assert-MockCalled Expand-DalecCredentialProviderPackage -Times 0 + Assert-MockCalled Set-ExitCode -Times 1 -ParameterFilter { $ExitCode -eq $global:WINDOWS_CSE_ERROR_INSTALL_CREDENTIAL_PROVIDER } + } + + It 'fails clearly when the RP URL, matching cached Dalec ZIP and metadata download are unavailable' { + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.35.7" + $global:CredentialProviderURL = "" + + { Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' } | Should -Throw '*No credential provider source*neither a cached Dalec package nor a matching download in baked metadata*' + Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 0 + Assert-MockCalled -CommandName 'tar' -Times 0 + Assert-MockCalled -CommandName 'Set-ExitCode' -Times 1 -ParameterFilter { $ExitCode -eq $global:WINDOWS_CSE_ERROR_INSTALL_CREDENTIAL_PROVIDER } + } + + It 'falls back to the RP URL when no cached Dalec ZIP matches the Kubernetes minor' { + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.35.0" + $global:CredentialProviderURL = 'https://packages.aks.azure.com/cloud-provider-azure/v1.35.0/binaries/azure-acr-credential-provider-windows-amd64-v1.35.0.tar.gz' + + Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' + Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 1 -ParameterFilter { + $Url -eq $global:CredentialProviderURL + } + Assert-MockCalled -CommandName 'tar' -Times 1 + Assert-MockCalled -CommandName 'Expand-DalecCredentialProviderPackage' -Times 0 + ($script:credentialProviderLogs -join "`n") | Should -Match 'No cached Dalec credential provider for Kubernetes 1.35.*checking baked metadata' + ($script:credentialProviderLogs -join "`n") | Should -Match 'metadata not found.*falling back to RP URL' + } + + It 'uses legacy tar path for k8s 1.32 even with a matching cached Dalec ZIP' { + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.32.8" + $global:CredentialProviderURL = 'https://packages.aks.azure.com/cloud-provider-azure/v1.32.8/binaries/azure-acr-credential-provider-windows-amd64-v1.32.8.tar.gz' + + New-Item -ItemType File -Path (Join-Path $script:dalecCacheDir 'azure-acr-credential-provider_1.32.19-1_amd64.zip') | Out-Null + + Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' + Assert-MockCalled -CommandName 'tar' -Times 1 + Assert-MockCalled -CommandName 'Expand-DalecCredentialProviderPackage' -Times 0 + Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 1 -ParameterFilter { + $Url -eq 'https://packages.aks.azure.com/cloud-provider-azure/v1.32.8/binaries/azure-acr-credential-provider-windows-amd64-v1.32.8.tar.gz' + } + ($script:credentialProviderLogs -join "`n") | Should -Not -Match 'Using dalec credential provider|Failed to install dalec credential provider' + } + + It 'honors custom URL and skips dalec for k8s >= 1.33 with non-stock URL' { + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.33.3" + $global:CredentialProviderURL = 'https://custom-mirror.example.com/hotfix/azure-acr-credential-provider-1.33.3-hotfix.tar.gz' + + Mock Resolve-DalecCredentialProviderPackage + + Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' + # Custom URL does not match stock pattern, so dalec resolver should NOT be called + Assert-MockCalled -CommandName 'Resolve-DalecCredentialProviderPackage' -Times 0 + Assert-MockCalled -CommandName 'tar' -Times 1 + Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 1 -ParameterFilter { + $Url -eq 'https://custom-mirror.example.com/hotfix/azure-acr-credential-provider-1.33.3-hotfix.tar.gz' + } + } + + It 'honors a custom mirror URL that uses the stock legacy path' { + $global:BootstrapProfileContainerRegistryServer = "" + $global:KubeBinariesVersion = "1.33.3" + $global:CredentialProviderURL = 'https://custom-mirror.example.com/cloud-provider-azure/v1.33.3/binaries/azure-acr-credential-provider-windows-amd64-v1.33.3.tar.gz' + + Mock Resolve-DalecCredentialProviderPackage + + Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' + Assert-MockCalled -CommandName 'Resolve-DalecCredentialProviderPackage' -Times 0 + Assert-MockCalled -CommandName 'tar' -Times 1 + Assert-MockCalled -CommandName 'DownloadFileOverHttp' -Times 1 -ParameterFilter { + $Url -eq $global:CredentialProviderURL + } + } + + It 'uses NI ORAS path unchanged when BootstrapProfileContainerRegistryServer is set' { + $global:BootstrapProfileContainerRegistryServer = "myregistry.azurecr.io" + $global:KubeBinariesVersion = "1.33.3" + $global:CredentialProviderURL = 'https://packages.aks.azure.com/cloud-provider-azure/v1.33.3/binaries/azure-acr-credential-provider-windows-amd64-v1.33.3.tar.gz' + + Mock Resolve-DalecCredentialProviderPackage + + Install-CredentialProvider -KubeDir 'c:\k' -CustomCloudContainerRegistryDNSSuffix '' + # NI path should use ORAS, not dalec resolve + Assert-MockCalled -CommandName 'Resolve-DalecCredentialProviderPackage' -Times 0 + $script:lastDownloadReference | Should -Be 'myregistry.azurecr.io/aks/packages/kubernetes/azure-acr-credential-provider:v1.33.3' + } } Describe 'New-CsiProxyService' { @@ -776,3 +1251,618 @@ Describe 'New-HostsConfigService' { } } } + +Describe 'ConvertTo-DalecCredentialProviderVersion' { + It 'converts compatible package into a four-part numeric version' -TestCases @( + @{ packageVersion = '1.33.19-10'; expected = '1.33.19.10' }, + @{ packageVersion = '1.33.0-0'; expected = '1.33.0.0' }, + @{ packageVersion = '1.33.2147483647-2147483647'; expected = '1.33.2147483647.2147483647' } + ) { + param($packageVersion, $expected) + + $result = ConvertTo-DalecCredentialProviderVersion -PackageVersion $packageVersion -KubeVersion '1.33.3' + + $result | Should -BeOfType ([version]) + $result | Should -Be ([version]$expected) + } + + It 'rejects malformed or incompatible package ' -TestCases @( + @{ packageVersion = $null }, + @{ packageVersion = '' }, + @{ packageVersion = '1.34.19-1' }, + @{ packageVersion = '2.33.19-1' }, + @{ packageVersion = 'v1.33.19-1' }, + @{ packageVersion = '1.33.19-rc.1-1' }, + @{ packageVersion = '1.33.19' }, + @{ packageVersion = '1.33.19.0-1' }, + @{ packageVersion = '01.33.19-1' }, + @{ packageVersion = '1.033.19-1' }, + @{ packageVersion = '1.33.019-1' }, + @{ packageVersion = '1.33.19-01' }, + @{ packageVersion = '1.33.2147483648-1' }, + @{ packageVersion = '1.33.19-2147483648' }, + @{ packageVersion = "1.33.19-1`n" } + ) { + param($packageVersion) + + ConvertTo-DalecCredentialProviderVersion -PackageVersion $packageVersion -KubeVersion '1.33.3' | Should -Be $null + } +} + +Describe 'Resolve-DalecCredentialProviderPackageFromMetadata' { + BeforeEach { + $script:metadata = @{ + Packages = @(@{ + name = 'azure-acr-credential-provider-pmc' + downloadURIs = @{ + windows = @{ + default = @{ + versionsV2 = @(@{ latestVersion = '1.33.19-1' }) + downloadURL = 'https://packages.aks.azure.com/dalec-packages/azure-acr-credential-provider/$($version.Split(''-'')[0])/windows/amd64/azure-acr-credential-provider_${version}_amd64.zip' + } + } + } + }) + } + Mock Test-Path -MockWith { $true } -ParameterFilter { $LiteralPath -eq 'c:\k\components.json' } + Mock Get-Content -MockWith { $script:metadata | ConvertTo-Json -Depth 10 -Compress } + } + + It 'resolves the pinned version and URL from older PMC-hosted Windows metadata' { + $result = Resolve-DalecCredentialProviderPackageFromMetadata -KubeVersion '1.33.3' + $result.Version | Should -Be '1.33.19-1' + $result.Url | Should -Be 'https://packages.aks.azure.com/dalec-packages/azure-acr-credential-provider/1.33.19/windows/amd64/azure-acr-credential-provider_1.33.19-1_amd64.zip' + $result.CachedFile | Should -Be $null + Assert-MockCalled Get-Content -Times 1 -Exactly -ParameterFilter { $LiteralPath -eq 'c:\k\components.json' } + } + + It 'orders numeric patches and revisions and excludes other minors, prereleases and malformed versions' { + $script:metadata.Packages[0].downloadURIs.windows.default.versionsV2 = @( + foreach ($version in @('1.33.9-99', '1.33.19-10', '1.33.19-2', '1.34.99-1', '2.33.99-1', '1.33.99-rc.1-1', '1.33.99-01', '1.33.99', 'invalid')) { + @{ latestVersion = $version } + } + ) + $result = Resolve-DalecCredentialProviderPackageFromMetadata -KubeVersion '1.33.0' + $result.Version | Should -Be '1.33.19-10' + $result.Url | Should -Be 'https://packages.aks.azure.com/dalec-packages/azure-acr-credential-provider/1.33.19/windows/amd64/azure-acr-credential-provider_1.33.19-10_amd64.zip' + } + + It 'supports consolidated Windows Dalec metadata alongside Linux-only PMC metadata' { + $script:metadata.Packages[0].name = 'windows credential provider dalec' + $script:metadata.Packages[0].downloadURIs.windows.default.versionsV2[0].previousLatestVersion = '1.33.6-1' + $script:metadata.Packages += @{ + name = 'azure-acr-credential-provider-pmc' + downloadURIs = @{ + ubuntu = @{ r2204 = @{ versionsV2 = @(@{ latestVersion = '1.33.99' }) } } + } + } + $result = Resolve-DalecCredentialProviderPackageFromMetadata -KubeVersion '1.33.3' + $result.Version | Should -Be '1.33.19-1' + $result.Url | Should -Be 'https://packages.aks.azure.com/dalec-packages/azure-acr-credential-provider/1.33.19/windows/amd64/azure-acr-credential-provider_1.33.19-1_amd64.zip' + } + + It 'retains PMC Windows precedence when older metadata contains both Dalec blocks' { + $script:metadata.Packages += @{ + name = 'windows credential provider dalec' + downloadURIs = @{ + windows = @{ + default = @{ + versionsV2 = @(@{ latestVersion = '1.33.6-1' }) + downloadURL = 'https://example.invalid/azure-acr-credential-provider_${version}_amd64.zip' + } + } + } + } + $result = Resolve-DalecCredentialProviderPackageFromMetadata -KubeVersion '1.33.3' + $result.Version | Should -Be '1.33.19-1' + $result.Url | Should -Be 'https://packages.aks.azure.com/dalec-packages/azure-acr-credential-provider/1.33.19/windows/amd64/azure-acr-credential-provider_1.33.19-1_amd64.zip' + } + + It 'does not use matching versions from the legacy component ' -TestCases @( + @{ name = 'windows credential provider' }, + @{ name = 'azure-acr-credential-provider' } + ) { + param($name) + $script:metadata.Packages[0].name = $name + Resolve-DalecCredentialProviderPackageFromMetadata -KubeVersion '1.33.3' | Should -Be $null + } + + It 'returns no download for a missing minor, package or URL' { + Resolve-DalecCredentialProviderPackageFromMetadata -KubeVersion '1.34.0' | Should -Be $null + $script:metadata.Packages[0].downloadURIs.windows.default.downloadURL = '' + Resolve-DalecCredentialProviderPackageFromMetadata -KubeVersion '1.33.3' | Should -Be $null + $script:metadata.Packages = @() + Resolve-DalecCredentialProviderPackageFromMetadata -KubeVersion '1.33.3' | Should -Be $null + } + + It 'rejects an unsupported or unresolved download template