From 5d266909be7d34f1e1e96a2834339f71e9d8b13c Mon Sep 17 00:00:00 2001 From: IEvangelist <7679720+IEvangelist@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:33:55 -0600 Subject: [PATCH 1/4] Bound primary apt mirror attempts in shared CLI E2E image builds Move shared image orchestration into a testable shell entry point, reserve fallback time inside the existing job budget, and fail closed on unsafe cancellation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build-cli-e2e-image.yml | 98 +-------- .../workflows/warm-cli-e2e-image-cache.yml | 1 + docs/ci/cli-e2e-images.md | 21 ++ eng/github-ci/test-trigger-map.yml | 4 + eng/scripts/build-cli-e2e-images.sh | 78 +++++++ .../Pipelines/CliE2EImageBuildTests.cs | 32 +++ .../Pipelines/cli-e2e-image-build.harness.ts | 199 ++++++++++++++++++ .../TestTriggerMap/TestTriggerMapTests.cs | 4 + 8 files changed, 340 insertions(+), 97 deletions(-) create mode 100644 eng/scripts/build-cli-e2e-images.sh create mode 100644 tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs create mode 100644 tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts diff --git a/.github/workflows/build-cli-e2e-image.yml b/.github/workflows/build-cli-e2e-image.yml index 71a806f8c16..735525c3525 100644 --- a/.github/workflows/build-cli-e2e-image.yml +++ b/.github/workflows/build-cli-e2e-image.yml @@ -18,7 +18,6 @@ on: env: CLI_E2E_DOTNET_IMAGE_ARTIFACT: cli-e2e-dotnet-image CLI_E2E_DOTNET_IMAGE_TAG: aspire-cli-e2e-dotnet:prebuilt - CLI_E2E_DOTNET_IMAGE_CACHE_SCOPE: cli-e2e-dotnet CLI_E2E_POLYGLOT_IMAGE_ARTIFACT: cli-e2e-polyglot-image CLI_E2E_POLYGLOT_IMAGE_TAG: aspire-cli-e2e-polyglot:prebuilt CLI_E2E_POLYGLOT_JAVA_IMAGE_ARTIFACT: cli-e2e-polyglot-java-image @@ -26,7 +25,6 @@ env: CLI_E2E_POLYGLOT_GO_IMAGE_ARTIFACT: cli-e2e-polyglot-go-image CLI_E2E_POLYGLOT_GO_IMAGE_TAG: aspire-cli-e2e-polyglot-go:prebuilt CLI_E2E_POLYGLOT_BASE_IMAGE_TAG: aspire-e2e-polyglot-base:latest - CLI_E2E_POLYGLOT_BASE_IMAGE_CACHE_SCOPE: cli-e2e-polyglot-base CLI_E2E_INCLUDE_POLYGLOT_IMAGES: ${{ inputs.includePolyglotImages }} jobs: @@ -57,101 +55,7 @@ jobs: - name: Build CLI E2E Docker image shell: bash - run: | - set -euo pipefail - - docker buildx create --name cli-e2e-builder --use - docker buildx inspect --bootstrap - - build_image() { - local display_name="$1" - local dockerfile="$2" - local tag="$3" - local cache_scope="$4" - local mirror="$5" - local -a build_args=(--build-arg SKIP_SOURCE_BUILD=true) - - if [[ -n "$mirror" ]]; then - echo "Building $display_name with Ubuntu apt mirror: $mirror" - build_args+=(--build-arg "UBUNTU_APT_MIRROR=$mirror") - else - echo "Building $display_name with the default Ubuntu apt sources" - fi - - docker buildx build \ - --load \ - --cache-from "type=gha,scope=$cache_scope" \ - --cache-to "type=gha,scope=$cache_scope,mode=max,ignore-error=true" \ - "${build_args[@]}" \ - -f "$dockerfile" \ - -t "$tag" \ - . - } - - build_java_image() { - local mirror="$1" - local -a build_args=() - - if [[ -n "$mirror" ]]; then - echo "Building Java polyglot image with Ubuntu apt mirror: $mirror" - build_args+=(--build-arg "UBUNTU_APT_MIRROR=$mirror") - else - echo "Building Java polyglot image with the default Ubuntu apt sources" - fi - - DOCKER_BUILDKIT=1 docker build \ - "${build_args[@]}" \ - -f tests/Shared/Docker/Dockerfile.e2e-polyglot-java \ - -t "$CLI_E2E_POLYGLOT_JAVA_IMAGE_TAG" \ - . - } - - build_go_image() { - local mirror="$1" - local -a build_args=() - - if [[ -n "$mirror" ]]; then - echo "Building Go polyglot image with Ubuntu apt mirror: $mirror" - build_args+=(--build-arg "UBUNTU_APT_MIRROR=$mirror") - else - echo "Building Go polyglot image with the default Ubuntu apt sources" - fi - - DOCKER_BUILDKIT=1 docker build \ - "${build_args[@]}" \ - -f tests/Shared/Docker/Dockerfile.e2e-polyglot-go \ - -t "$CLI_E2E_POLYGLOT_GO_IMAGE_TAG" \ - . - } - - build_with_mirror_retry() { - local display_name="$1" - local dockerfile="$2" - local tag="$3" - local cache_scope="$4" - - if ! build_image "$display_name" "$dockerfile" "$tag" "$cache_scope" "http://azure.archive.ubuntu.com/ubuntu/"; then - echo "$display_name build failed with the Azure Ubuntu apt mirror; retrying with default Ubuntu apt sources" - build_image "$display_name" "$dockerfile" "$tag" "$cache_scope" "" - fi - } - - build_with_mirror_retry ".NET image" "tests/Shared/Docker/Dockerfile.e2e" "$CLI_E2E_DOTNET_IMAGE_TAG" "$CLI_E2E_DOTNET_IMAGE_CACHE_SCOPE" - - if [[ "$CLI_E2E_INCLUDE_POLYGLOT_IMAGES" == "true" ]]; then - build_with_mirror_retry "polyglot base image" "tests/Shared/Docker/Dockerfile.e2e-polyglot-base" "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG" "$CLI_E2E_POLYGLOT_BASE_IMAGE_CACHE_SCOPE" - docker tag "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG" "$CLI_E2E_POLYGLOT_IMAGE_TAG" - - if ! build_java_image "http://azure.archive.ubuntu.com/ubuntu/"; then - echo "Java polyglot image build failed with the Azure Ubuntu apt mirror; retrying with default Ubuntu apt sources" - build_java_image "" - fi - - if ! build_go_image "http://azure.archive.ubuntu.com/ubuntu/"; then - echo "Go polyglot image build failed with the Azure Ubuntu apt mirror; retrying with default Ubuntu apt sources" - build_go_image "" - fi - fi + run: bash eng/scripts/build-cli-e2e-images.sh - name: Save CLI E2E Docker image tarballs if: ${{ inputs.uploadImageArtifacts }} diff --git a/.github/workflows/warm-cli-e2e-image-cache.yml b/.github/workflows/warm-cli-e2e-image-cache.yml index 0f4d88e9162..482738941b1 100644 --- a/.github/workflows/warm-cli-e2e-image-cache.yml +++ b/.github/workflows/warm-cli-e2e-image-cache.yml @@ -16,6 +16,7 @@ on: - '.dockerignore' - 'eng/scripts/get-aspire-cli.sh' - 'eng/scripts/get-aspire-cli-pr.sh' + - 'eng/scripts/build-cli-e2e-images.sh' - 'tests/Shared/Docker/Dockerfile.e2e' - 'tests/Shared/Docker/Dockerfile.e2e-polyglot-base' - 'tests/Shared/Docker/Dockerfile.e2e-polyglot-go' diff --git a/docs/ci/cli-e2e-images.md b/docs/ci/cli-e2e-images.md index 8a315f11684..e4363407743 100644 --- a/docs/ci/cli-e2e-images.md +++ b/docs/ci/cli-e2e-images.md @@ -29,6 +29,27 @@ Consumer workflows download image artifacts into `${{ github.workspace }}/cli-e2 The reusable image build workflow also has an `uploadImageArtifacts` input. It defaults to `true` for test workflows because each isolated test job needs the image tarballs. Cache-warming workflows set it to `false` so they only build the images and export BuildKit cache layers. +### Bounded apt mirror fallback + +Both test builds and the cache warmer run `eng/scripts/build-cli-e2e-images.sh`. +Each image (.NET, polyglot base, Java and Go) first uses the Azure Ubuntu apt +mirror, with a five-minute limit. A failed or timed-out primary attempt retries +once with the Dockerfile's default Ubuntu apt sources. The fallback's failure +exit code and both attempts' output are preserved; there is no third attempt. + +The job still has a 45-minute cap. Four primary attempts can consume at most +20 minutes plus two minutes of cancellation grace, leaving 23 minutes for +fallback builds, setup, saving and uploading images. This prevents a slow but +successful apt transfer from consuming the entire job before fallback runs. +Observed throttled transfers took 38-41 minutes for 82-108 MB. + +GNU `timeout` signals the Docker client's entire process group with `TERM` so +BuildKit can cancel the build session before fallback starts. If the client +cannot stop within another 30 seconds, `timeout` kills the process group and the +script fails without fallback: a forced kill cannot establish that the remote +build acknowledged cancellation. Interrupts, termination and timeout execution +errors also fail without retrying. + ## Cross-PR BuildKit cache The image build workflow exports BuildKit layers to the GitHub Actions cache with stable scopes: diff --git a/eng/github-ci/test-trigger-map.yml b/eng/github-ci/test-trigger-map.yml index f0854d1365b..13c86688c68 100644 --- a/eng/github-ci/test-trigger-map.yml +++ b/eng/github-ci/test-trigger-map.yml @@ -432,6 +432,10 @@ path_rules: - eng/scripts/load-cli-e2e-images.sh targets: [test:Aspire.Cli.EndToEnd.Tests] reason: builds or loads the CLI E2E images consumed only by Aspire.Cli.EndToEnd.Tests + - paths: + - eng/scripts/build-cli-e2e-images.sh + targets: [test:Aspire.Cli.EndToEnd.Tests, test:Infrastructure.Tests] + reason: shared image build entry point and its shell behavior tests - paths: - eng/scripts/pack-cli-npm-package.ps1 targets: [test:Aspire.Cli.Tests, test:Infrastructure.Tests] diff --git a/eng/scripts/build-cli-e2e-images.sh b/eng/scripts/build-cli-e2e-images.sh new file mode 100644 index 00000000000..8640ddd0138 --- /dev/null +++ b/eng/scripts/build-cli-e2e-images.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -euo pipefail + +build_with_mirror_retry() { + local primary_timeout="$1" + local display_name="$2" + shift 2 + local status + + echo "Building $display_name with Azure Ubuntu apt mirror (limit: $primary_timeout)" + # Do not use --foreground: timeout must signal the entire client process group. + # TERM lets Docker cancel its BuildKit session before default-source fallback. + if timeout --verbose --signal=TERM --kill-after=30s "$primary_timeout" \ + "$@" --build-arg "UBUNTU_APT_MIRROR=http://azure.archive.ubuntu.com/ubuntu/" .; then + return 0 + else + status=$? + fi + + case "$status" in + 124) + echo "$display_name primary mirror build exceeded $primary_timeout; retrying with default Ubuntu apt sources" >&2 + ;; + 125|126|127|130|137|143) + # A forced kill cannot confirm BuildKit acknowledged cancellation. Do + # not overlap another build or turn workflow cancellation into a retry. + echo "$display_name primary build stopped with exit code $status; not retrying after cancellation or timeout execution failure" >&2 + return "$status" + ;; + *) + echo "$display_name build failed with the Azure Ubuntu apt mirror (exit code $status); retrying with default Ubuntu apt sources" >&2 + ;; + esac + + "$@" . +} + +build_cli_e2e_images() { + # Four primary attempts can spend at most 20m plus 2m cancellation grace, + # leaving 23m of the 45m job for default sources, setup and artifact handling. + # Throttled apt downloads have otherwise consumed 38-41m in a single image. + local primary_timeout=5m + local dotnet_cache_scope=cli-e2e-dotnet + local polyglot_cache_scope=cli-e2e-polyglot-base + + docker buildx create --name cli-e2e-builder --use + docker buildx inspect --bootstrap + + build_with_mirror_retry "$primary_timeout" ".NET image" docker buildx build \ + --load \ + --cache-from "type=gha,scope=$dotnet_cache_scope" \ + --cache-to "type=gha,scope=$dotnet_cache_scope,mode=max,ignore-error=true" \ + --build-arg SKIP_SOURCE_BUILD=true \ + -f tests/Shared/Docker/Dockerfile.e2e \ + -t "$CLI_E2E_DOTNET_IMAGE_TAG" + + if [[ "$CLI_E2E_INCLUDE_POLYGLOT_IMAGES" == "true" ]]; then + build_with_mirror_retry "$primary_timeout" "polyglot base image" docker buildx build \ + --load \ + --cache-from "type=gha,scope=$polyglot_cache_scope" \ + --cache-to "type=gha,scope=$polyglot_cache_scope,mode=max,ignore-error=true" \ + --build-arg SKIP_SOURCE_BUILD=true \ + -f tests/Shared/Docker/Dockerfile.e2e-polyglot-base \ + -t "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG" + docker tag "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG" "$CLI_E2E_POLYGLOT_IMAGE_TAG" + + DOCKER_BUILDKIT=1 build_with_mirror_retry "$primary_timeout" "Java polyglot image" docker build \ + -f tests/Shared/Docker/Dockerfile.e2e-polyglot-java \ + -t "$CLI_E2E_POLYGLOT_JAVA_IMAGE_TAG" + DOCKER_BUILDKIT=1 build_with_mirror_retry "$primary_timeout" "Go polyglot image" docker build \ + -f tests/Shared/Docker/Dockerfile.e2e-polyglot-go \ + -t "$CLI_E2E_POLYGLOT_GO_IMAGE_TAG" + fi +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + build_cli_e2e_images +fi diff --git a/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs b/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs new file mode 100644 index 00000000000..a73baab507e --- /dev/null +++ b/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.TestUtilities; +using Xunit; + +namespace Infrastructure.Tests; + +public sealed class CliE2EImageBuildTests(ITestOutputHelper output) +{ + [Theory] + [InlineData("success")] + [InlineData("failure")] + [InlineData("fallback-failure")] + [InlineData("slow-success")] + [InlineData("stall")] + [InlineData("forced-kill")] + [InlineData("cancelled")] + [InlineData("all-images")] + [InlineData("dotnet-only")] + [RequiresTools(["node", "bash", "timeout"])] + public async Task ImageBuildPreservesFallbackAndCancellation(string scenario) + { + Assert.SkipWhen(!OperatingSystem.IsLinux(), "GNU timeout process-group behavior requires Linux."); + using var command = new NodeCommand(output).WithTimeout(TimeSpan.FromMinutes(2)); + var result = await command.ExecuteScriptAsync( + Path.Combine(RepoRoot.Path, "tests", "Infrastructure.Tests", "Pipelines", "cli-e2e-image-build.harness.ts"), + RepoRoot.Path, scenario); + + result.EnsureSuccessful(); + } +} diff --git a/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts b/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts new file mode 100644 index 00000000000..bc510aebe67 --- /dev/null +++ b/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts @@ -0,0 +1,199 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn, spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const mirrorArg = 'UBUNTU_APT_MIRROR=http://azure.archive.ubuntu.com/ubuntu/'; + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + // /proc/PID/stat starts "123 (sleep) S ...". A killed orphan can briefly + // remain as a zombie ('Z') until init reaps it. + return fs.readFileSync(`/proc/${pid}/stat`, 'utf8').split(') ')[1][0] !== 'Z'; + } catch (error) { + if (error instanceof Error && 'code' in error && (error.code === 'ESRCH' || error.code === 'ENOENT')) { + return false; + } + throw error; + } +} + +function stubDocker(): void { + const root = process.env.IMAGE_BUILD_TEST_ROOT!; + const scenario = process.env.IMAGE_BUILD_TEST_SCENARIO!; + const args = process.argv.slice(3); + fs.appendFileSync(path.join(root, 'trace'), JSON.stringify({ args, buildkit: process.env.DOCKER_BUILDKIT }) + '\n'); + if (!args.includes('build')) { + return; + } + if (!args.includes(mirrorArg)) { + if (scenario === 'stall' || scenario === 'slow-success') { + const childPid = Number(fs.readFileSync(path.join(root, 'child-pid'), 'utf8')); + assert.equal(isAlive(childPid), false, 'Fallback started before primary child stopped'); + assert.equal(fs.readFileSync(path.join(root, 'cancelled'), 'utf8'), 'acknowledged'); + } + console.log('default sources output'); + console.error('default sources diagnostic'); + process.exit(scenario === 'fallback-failure' ? 42 : 0); + } + + console.log('primary mirror output'); + console.error('primary mirror diagnostic'); + fs.writeFileSync(path.join(root, 'primary-pid'), String(process.pid)); + switch (scenario) { + case 'success': + return; + case 'cancelled': + process.kill(process.pid, 'SIGTERM'); + return; + case 'stall': + case 'slow-success': + case 'forced-kill': { + // A real descendant, not a mocked timeout, must stop before retry. + const child = spawn('bash', ['-c', ` + ${scenario === 'forced-kill' ? "trap '' TERM" : ''} + echo "$$" > "$IMAGE_BUILD_TEST_ROOT/child-pid" + exec sleep 300 + `], { stdio: 'inherit' }); + if (scenario === 'forced-kill') { + process.on('SIGTERM', () => {}); + } else { + process.on('SIGTERM', () => { + child.once('exit', () => { + fs.writeFileSync(path.join(root, 'cancelled'), 'acknowledged'); + process.exit(143); + }); + if (child.exitCode !== null || child.signalCode !== null) { + fs.writeFileSync(path.join(root, 'cancelled'), 'acknowledged'); + process.exit(143); + } + }); + } + if (scenario === 'slow-success') { + setTimeout(() => process.exit(0), 30000); + } else { + setInterval(() => {}, 1000); + } + return; + } + default: + process.exit(17); + } +} + +function testBuild(repo: string, scenario: string): void { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'aspire-image-build-')); + try { + const bin = path.join(workspace, 'commands with spaces'); + fs.mkdirSync(bin); + fs.writeFileSync(path.join(bin, 'docker'), `#!/usr/bin/env bash +exec "$IMAGE_BUILD_TEST_NODE" "$IMAGE_BUILD_TEST_HARNESS" --stub "$@" +`, { mode: 0o755 }); + const script = path.join(repo, 'eng', 'scripts', 'build-cli-e2e-images.sh'); + const orchestration = scenario === 'all-images' || scenario === 'dotnet-only'; + const started = Date.now(); + const result = spawnSync('bash', orchestration + ? [script] + : ['-c', 'source "$1"; build_with_mirror_retry 2s "test image" docker build -f "Dockerfile with spaces" -t test:primary', 'bash', script], { + env: { + ...process.env, + PATH: `${bin}${path.delimiter}${process.env.PATH}`, + IMAGE_BUILD_TEST_NODE: process.execPath, + IMAGE_BUILD_TEST_HARNESS: fileURLToPath(import.meta.url), + IMAGE_BUILD_TEST_ROOT: workspace, + IMAGE_BUILD_TEST_SCENARIO: scenario, + CLI_E2E_INCLUDE_POLYGLOT_IMAGES: scenario === 'all-images' ? 'true' : 'false', + CLI_E2E_DOTNET_IMAGE_TAG: 'test:dotnet', + CLI_E2E_POLYGLOT_BASE_IMAGE_TAG: 'test:base', + CLI_E2E_POLYGLOT_IMAGE_TAG: 'test:polyglot', + CLI_E2E_POLYGLOT_JAVA_IMAGE_TAG: 'test:java', + CLI_E2E_POLYGLOT_GO_IMAGE_TAG: 'test:go' + }, + encoding: 'utf8', + timeout: 60000, + killSignal: 'SIGKILL' + }); + assert.ifError(result.error); + const output = result.stdout + result.stderr; + const expectedStatus = scenario === 'fallback-failure' ? 42 + : scenario === 'forced-kill' ? 137 + : scenario === 'cancelled' ? 143 : 0; + assert.equal(result.status, expectedStatus, output); + const trace = fs.readFileSync(path.join(workspace, 'trace'), 'utf8').trim().split('\n').map(line => JSON.parse(line)); + if (orchestration) { + const builds = trace.filter(entry => entry.args.includes('build')); + const variants = scenario === 'all-images' ? ['dotnet', 'base', 'java', 'go'] : ['dotnet']; + assert.equal(builds.length, variants.length * 2, output); + for (const [index, variant] of variants.entries()) { + const primary = builds[index * 2]; + const fallback = builds[index * 2 + 1]; + assert.equal(primary.args[primary.args.indexOf('-t') + 1], `test:${variant}`); + assert.deepEqual(primary.args.slice(-3), ['--build-arg', mirrorArg, '.']); + assert.deepEqual(fallback.args, [...primary.args.slice(0, -3), '.']); + if (variant === 'java' || variant === 'go') { + assert.equal(primary.buildkit, '1'); + assert.equal(fallback.buildkit, '1'); + } else { + assert.ok(primary.args.includes('--load')); + assert.ok(primary.args.includes('SKIP_SOURCE_BUILD=true')); + assert.ok(primary.args.includes(`type=gha,scope=cli-e2e-${variant === 'dotnet' ? 'dotnet' : 'polyglot-base'}`)); + } + } + assert.deepEqual(trace.filter(entry => !entry.args.includes('build')).map(entry => entry.args), [ + ['buildx', 'create', '--name', 'cli-e2e-builder', '--use'], + ['buildx', 'inspect', '--bootstrap'], + ...(scenario === 'all-images' ? [['tag', 'test:base', 'test:polyglot']] : []) + ]); + } else { + const expectedArgs = ['build', '-f', 'Dockerfile with spaces', '-t', 'test:primary']; + const noFallback = ['success', 'forced-kill', 'cancelled'].includes(scenario); + assert.deepEqual(trace.map(entry => entry.args), [ + [...expectedArgs, '--build-arg', mirrorArg, '.'], + ...(noFallback ? [] : [[...expectedArgs, '.']]) + ]); + assert.ok(output.includes('primary mirror output'), output); + assert.ok(output.includes('primary mirror diagnostic'), output); + if (!noFallback) { + assert.ok(output.includes('default sources output'), output); + assert.ok(output.includes('default sources diagnostic'), output); + } + if (scenario === 'stall' || scenario === 'slow-success') { + assert.match(output, /exceeded 2s; retrying with default Ubuntu apt sources/); + assert.ok(Date.now() - started < 15000, output); + } + if (scenario === 'forced-kill' || scenario === 'cancelled') { + assert.match(output, /not retrying after cancellation or timeout execution failure/); + } + } + const childPath = path.join(workspace, 'child-pid'); + assert.equal(isAlive(Number(fs.readFileSync(path.join(workspace, 'primary-pid'), 'utf8'))), false, output); + if (fs.existsSync(childPath)) { + assert.equal(isAlive(Number(fs.readFileSync(childPath, 'utf8'))), false, output); + } + console.log(output); + console.log(`Passed: ${scenario}`); + } finally { + for (const name of ['primary-pid', 'child-pid']) { + const pidPath = path.join(workspace, name); + if (fs.existsSync(pidPath)) { + const pid = Number(fs.readFileSync(pidPath, 'utf8')); + if (isAlive(pid)) { + process.kill(pid, 'SIGKILL'); + } + } + } + fs.rmSync(workspace, { recursive: true, force: true }); + } +} + +if (process.argv[2] === '--stub') { + stubDocker(); +} else { + testBuild(process.argv[2], process.argv[3]); +} diff --git a/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs b/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs index 5ebab541c9b..cef09f1a3e1 100644 --- a/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs +++ b/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs @@ -278,6 +278,10 @@ private static IReadOnlySet LoadSolutionProjectPaths() "eng/scripts/load-cli-e2e-images.sh", ["test:Aspire.Cli.EndToEnd.Tests"] }, + { + "eng/scripts/build-cli-e2e-images.sh", + ["test:Aspire.Cli.EndToEnd.Tests", "test:Infrastructure.Tests"] + }, { "eng/clipack/Common.projitems", [ From 4bb054c23f15155d1cb8501e88607a18f157e2f5 Mon Sep 17 00:00:00 2001 From: IEvangelist <7679720+IEvangelist@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:04:38 -0600 Subject: [PATCH 2/4] Restore inherited apt sources and preserve external cancellation Snapshot original Ubuntu sources before configuring mirrors so derived-image fallbacks restore defaults. Normalize cancellation in a supervising shell even when Docker exits with code 1 after TERM. Add external-signal and real Ubuntu restoration regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/ci/cli-e2e-images.md | 6 ++ eng/scripts/build-cli-e2e-images.sh | 18 +++++- .../Pipelines/CliE2EImageBuildTests.cs | 13 ++++ .../Pipelines/cli-e2e-image-build.harness.ts | 64 +++++++++++++++++-- .../Docker/configure-ubuntu-apt-mirror.sh | 25 +++++++- 5 files changed, 119 insertions(+), 7 deletions(-) diff --git a/docs/ci/cli-e2e-images.md b/docs/ci/cli-e2e-images.md index e4363407743..dde0ceacaf1 100644 --- a/docs/ci/cli-e2e-images.md +++ b/docs/ci/cli-e2e-images.md @@ -50,6 +50,12 @@ script fails without fallback: a forced kill cannot establish that the remote build acknowledged cancellation. Interrupts, termination and timeout execution errors also fail without retrying. +The mirror configuration script preserves the base image's original Ubuntu +sources before replacing them. An empty mirror restores those files, including +in derived Java/Go images, without changing third-party repositories. A +supervising shell preserves external cancellation as a signal exit even when +the Docker client handles the signal and exits with an ordinary failure code. + ## Cross-PR BuildKit cache The image build workflow exports BuildKit layers to the GitHub Actions cache with stable scopes: diff --git a/eng/scripts/build-cli-e2e-images.sh b/eng/scripts/build-cli-e2e-images.sh index 8640ddd0138..9fab0374603 100644 --- a/eng/scripts/build-cli-e2e-images.sh +++ b/eng/scripts/build-cli-e2e-images.sh @@ -10,8 +10,24 @@ build_with_mirror_retry() { echo "Building $display_name with Azure Ubuntu apt mirror (limit: $primary_timeout)" # Do not use --foreground: timeout must signal the entire client process group. # TERM lets Docker cancel its BuildKit session before default-source fallback. + # Normalize external signals in a supervising shell: Docker may handle TERM + # and exit 1, which otherwise looks like an ordinary build failure to timeout. + # On a deadline timeout still returns 124; external cancellation returns 143/130. if timeout --verbose --signal=TERM --kill-after=30s "$primary_timeout" \ - "$@" --build-arg "UBUNTU_APT_MIRROR=http://azure.archive.ubuntu.com/ubuntu/" .; then + bash -c ' + cancel() { + trap "" INT TERM + if [[ -n "${client_pid:-}" ]]; then + wait "$client_pid" + fi + exit "$1" + } + trap "cancel 143" TERM + trap "cancel 130" INT + "$@" & + client_pid=$! + wait "$client_pid" + ' bash "$@" --build-arg "UBUNTU_APT_MIRROR=http://azure.archive.ubuntu.com/ubuntu/" .; then return 0 else status=$? diff --git a/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs b/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs index a73baab507e..ab61eb8de57 100644 --- a/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs +++ b/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs @@ -16,6 +16,7 @@ public sealed class CliE2EImageBuildTests(ITestOutputHelper output) [InlineData("stall")] [InlineData("forced-kill")] [InlineData("cancelled")] + [InlineData("external-cancel")] [InlineData("all-images")] [InlineData("dotnet-only")] [RequiresTools(["node", "bash", "timeout"])] @@ -29,4 +30,16 @@ public async Task ImageBuildPreservesFallbackAndCancellation(string scenario) result.EnsureSuccessful(); } + + [Fact] + [RequiresTools(["node", "docker"])] + public async Task DerivedImageFallbackRestoresOriginalAptSources() + { + using var command = new NodeCommand(output).WithTimeout(TimeSpan.FromMinutes(2)); + var result = await command.ExecuteScriptAsync( + Path.Combine(RepoRoot.Path, "tests", "Infrastructure.Tests", "Pipelines", "cli-e2e-image-build.harness.ts"), + RepoRoot.Path, "inherited-sources"); + + result.EnsureSuccessful(); + } } diff --git a/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts b/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts index bc510aebe67..efa42d412a6 100644 --- a/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts +++ b/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts @@ -52,6 +52,18 @@ function stubDocker(): void { case 'cancelled': process.kill(process.pid, 'SIGTERM'); return; + case 'external-cancel': { + process.on('SIGTERM', () => { + console.error('Client handled external TERM and exited 1'); + process.exit(1); + }); + // Signal the real timeout supervisor, not the client or its wrapper. + // /proc/PID/stat starts "123 (bash) S 456 ...", where 456 is PPID. + const timeoutPid = Number(fs.readFileSync(`/proc/${process.ppid}/stat`, 'utf8').split(') ')[1].split(' ')[1]); + process.kill(timeoutPid, 'SIGTERM'); + setInterval(() => {}, 1000); + return; + } case 'stall': case 'slow-success': case 'forced-kill': { @@ -100,7 +112,7 @@ exec "$IMAGE_BUILD_TEST_NODE" "$IMAGE_BUILD_TEST_HARNESS" --stub "$@" const started = Date.now(); const result = spawnSync('bash', orchestration ? [script] - : ['-c', 'source "$1"; build_with_mirror_retry 2s "test image" docker build -f "Dockerfile with spaces" -t test:primary', 'bash', script], { + : ['-c', 'source "$1"; build_with_mirror_retry 5s "test image" docker build -f "Dockerfile with spaces" -t test:primary', 'bash', script], { env: { ...process.env, PATH: `${bin}${path.delimiter}${process.env.PATH}`, @@ -123,7 +135,7 @@ exec "$IMAGE_BUILD_TEST_NODE" "$IMAGE_BUILD_TEST_HARNESS" --stub "$@" const output = result.stdout + result.stderr; const expectedStatus = scenario === 'fallback-failure' ? 42 : scenario === 'forced-kill' ? 137 - : scenario === 'cancelled' ? 143 : 0; + : scenario === 'cancelled' || scenario === 'external-cancel' ? 143 : 0; assert.equal(result.status, expectedStatus, output); const trace = fs.readFileSync(path.join(workspace, 'trace'), 'utf8').trim().split('\n').map(line => JSON.parse(line)); if (orchestration) { @@ -152,7 +164,7 @@ exec "$IMAGE_BUILD_TEST_NODE" "$IMAGE_BUILD_TEST_HARNESS" --stub "$@" ]); } else { const expectedArgs = ['build', '-f', 'Dockerfile with spaces', '-t', 'test:primary']; - const noFallback = ['success', 'forced-kill', 'cancelled'].includes(scenario); + const noFallback = ['success', 'forced-kill', 'cancelled', 'external-cancel'].includes(scenario); assert.deepEqual(trace.map(entry => entry.args), [ [...expectedArgs, '--build-arg', mirrorArg, '.'], ...(noFallback ? [] : [[...expectedArgs, '.']]) @@ -164,12 +176,15 @@ exec "$IMAGE_BUILD_TEST_NODE" "$IMAGE_BUILD_TEST_HARNESS" --stub "$@" assert.ok(output.includes('default sources diagnostic'), output); } if (scenario === 'stall' || scenario === 'slow-success') { - assert.match(output, /exceeded 2s; retrying with default Ubuntu apt sources/); + assert.match(output, /exceeded 5s; retrying with default Ubuntu apt sources/); assert.ok(Date.now() - started < 15000, output); } - if (scenario === 'forced-kill' || scenario === 'cancelled') { + if (scenario === 'forced-kill' || scenario === 'cancelled' || scenario === 'external-cancel') { assert.match(output, /not retrying after cancellation or timeout execution failure/); } + if (scenario === 'external-cancel') { + assert.match(output, /Client handled external TERM and exited 1/); + } } const childPath = path.join(workspace, 'child-pid'); assert.equal(isAlive(Number(fs.readFileSync(path.join(workspace, 'primary-pid'), 'utf8'))), false, output); @@ -192,8 +207,47 @@ exec "$IMAGE_BUILD_TEST_NODE" "$IMAGE_BUILD_TEST_HARNESS" --stub "$@" } } +function testInheritedSources(repo: string): void { + const script = path.join(repo, 'tests', 'Shared', 'Docker', 'configure-ubuntu-apt-mirror.sh'); + const result = spawnSync('docker', [ + 'run', '--rm', '--interactive', + 'mcr.microsoft.com/mirror/docker/library/ubuntu:24.04', 'sh', '-ec', ` + cat > /configure-mirror.sh + cp /etc/apt/sources.list.d/ubuntu.sources /tmp/original.sources + echo 'deb https://packages.example.invalid stable main' > /etc/apt/sources.list.d/third-party.list + for legacy in absent present; do + rm -rf /etc/apt/aspire-default-ubuntu-sources + if [ "$legacy" = present ]; then + echo '# original legacy sources' > /etc/apt/sources.list + else + rm -f /etc/apt/sources.list + fi + sh /configure-mirror.sh http://azure.archive.ubuntu.com/ubuntu/ + grep -q 'http://azure.archive.ubuntu.com/ubuntu' /etc/apt/sources.list.d/ubuntu.sources + # The derived layer must not overwrite the base layer's snapshot. + sh /configure-mirror.sh https://different.example.invalid/ubuntu + sh /configure-mirror.sh "" + cmp /tmp/original.sources /etc/apt/sources.list.d/ubuntu.sources + if [ "$legacy" = present ]; then + test "$(cat /etc/apt/sources.list)" = '# original legacy sources' + else + test ! -f /etc/apt/sources.list + fi + test "$(cat /etc/apt/sources.list.d/third-party.list)" = 'deb https://packages.example.invalid stable main' + sh /configure-mirror.sh "" + cmp /tmp/original.sources /etc/apt/sources.list.d/ubuntu.sources + done + ` + ], { encoding: 'utf8', timeout: 90000, input: fs.readFileSync(script) }); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stdout + result.stderr); + console.log('Passed: inherited-sources (base/derived configuration, original legacy absence/presence and third-party sources)'); +} + if (process.argv[2] === '--stub') { stubDocker(); +} else if (process.argv[3] === 'inherited-sources') { + testInheritedSources(process.argv[2]); } else { testBuild(process.argv[2], process.argv[3]); } diff --git a/tests/Shared/Docker/configure-ubuntu-apt-mirror.sh b/tests/Shared/Docker/configure-ubuntu-apt-mirror.sh index 72f3943bc51..249c2a2bf10 100644 --- a/tests/Shared/Docker/configure-ubuntu-apt-mirror.sh +++ b/tests/Shared/Docker/configure-ubuntu-apt-mirror.sh @@ -2,8 +2,21 @@ set -eu ubuntu_apt_mirror="${1:-}" +ubuntu_sources_file="/etc/apt/sources.list.d/ubuntu.sources" +default_sources_directory="/etc/apt/aspire-default-ubuntu-sources" if [ -z "$ubuntu_apt_mirror" ]; then + # Derived Java/Go images inherit the base image's mirror configuration. + # Restore the original files, not just an empty ARG, for default-source retries. + if [ -d "$default_sources_directory" ]; then + for source in sources.list sources.list.d/ubuntu.sources; do + if [ -f "$default_sources_directory/$source" ]; then + cp "$default_sources_directory/$source" "/etc/apt/$source" + else + rm -f "/etc/apt/$source" + fi + done + fi exit 0 fi @@ -34,7 +47,17 @@ fi ubuntu_apt_mirror="${ubuntu_apt_mirror%/}" ubuntu_apt_components="${UBUNTU_APT_COMPONENTS:-main restricted universe multiverse}" -ubuntu_sources_file="/etc/apt/sources.list.d/ubuntu.sources" + +# Keep one pristine snapshot across base and derived layers, including absence +# of a legacy sources.list. Other repositories (NodeSource, GitHub) are untouched. +if [ ! -d "$default_sources_directory" ]; then + mkdir -p "$default_sources_directory/sources.list.d" + for source in sources.list sources.list.d/ubuntu.sources; do + if [ -f "/etc/apt/$source" ]; then + cp "/etc/apt/$source" "$default_sources_directory/$source" + fi + done +fi mkdir -p "$(dirname "$ubuntu_sources_file")" From 296e3e9ccd574ea70403d0493da945b284ddd3b1 Mon Sep 17 00:00:00 2001 From: IEvangelist <7679720+IEvangelist@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:37:02 -0600 Subject: [PATCH 3/4] Preserve external cancellation during mirror deadline cleanup Track primary deadlines and external signals independently; retain process-group TERM and fail-closed kill grace. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../workflows/warm-cli-e2e-image-cache.yml | 1 + docs/ci/cli-e2e-images.md | 10 +-- eng/github-ci/test-trigger-map.yml | 1 + eng/scripts/build-cli-e2e-images.sh | 22 +----- eng/scripts/run-bounded-image-build.ts | 79 +++++++++++++++++++ .../Pipelines/CliE2EImageBuildTests.cs | 5 +- .../Pipelines/cli-e2e-image-build.harness.ts | 37 ++++++--- .../TestTriggerMap/TestTriggerMapTests.cs | 4 + 8 files changed, 121 insertions(+), 38 deletions(-) create mode 100644 eng/scripts/run-bounded-image-build.ts diff --git a/.github/workflows/warm-cli-e2e-image-cache.yml b/.github/workflows/warm-cli-e2e-image-cache.yml index 482738941b1..9e4b0aba2ee 100644 --- a/.github/workflows/warm-cli-e2e-image-cache.yml +++ b/.github/workflows/warm-cli-e2e-image-cache.yml @@ -17,6 +17,7 @@ on: - 'eng/scripts/get-aspire-cli.sh' - 'eng/scripts/get-aspire-cli-pr.sh' - 'eng/scripts/build-cli-e2e-images.sh' + - 'eng/scripts/run-bounded-image-build.ts' - 'tests/Shared/Docker/Dockerfile.e2e' - 'tests/Shared/Docker/Dockerfile.e2e-polyglot-base' - 'tests/Shared/Docker/Dockerfile.e2e-polyglot-go' diff --git a/docs/ci/cli-e2e-images.md b/docs/ci/cli-e2e-images.md index dde0ceacaf1..c9d7ea08aa5 100644 --- a/docs/ci/cli-e2e-images.md +++ b/docs/ci/cli-e2e-images.md @@ -43,18 +43,18 @@ fallback builds, setup, saving and uploading images. This prevents a slow but successful apt transfer from consuming the entire job before fallback runs. Observed throttled transfers took 38-41 minutes for 82-108 MB. -GNU `timeout` signals the Docker client's entire process group with `TERM` so +The Node.js build supervisor signals the Docker client's process group with `TERM` so BuildKit can cancel the build session before fallback starts. If the client -cannot stop within another 30 seconds, `timeout` kills the process group and the +cannot stop within another 30 seconds, the supervisor kills the process group and the script fails without fallback: a forced kill cannot establish that the remote -build acknowledged cancellation. Interrupts, termination and timeout execution +build acknowledged cancellation. Interrupts, termination and supervisor execution errors also fail without retrying. The mirror configuration script preserves the base image's original Ubuntu sources before replacing them. An empty mirror restores those files, including in derived Java/Go images, without changing third-party repositories. A -supervising shell preserves external cancellation as a signal exit even when -the Docker client handles the signal and exits with an ordinary failure code. +supervisor tracks external cancellation independently of its deadline, including +signals during cleanup and clients that exit with an ordinary failure code. ## Cross-PR BuildKit cache diff --git a/eng/github-ci/test-trigger-map.yml b/eng/github-ci/test-trigger-map.yml index 13c86688c68..cb132f968fe 100644 --- a/eng/github-ci/test-trigger-map.yml +++ b/eng/github-ci/test-trigger-map.yml @@ -434,6 +434,7 @@ path_rules: reason: builds or loads the CLI E2E images consumed only by Aspire.Cli.EndToEnd.Tests - paths: - eng/scripts/build-cli-e2e-images.sh + - eng/scripts/run-bounded-image-build.ts targets: [test:Aspire.Cli.EndToEnd.Tests, test:Infrastructure.Tests] reason: shared image build entry point and its shell behavior tests - paths: diff --git a/eng/scripts/build-cli-e2e-images.sh b/eng/scripts/build-cli-e2e-images.sh index 9fab0374603..22bcecb394f 100644 --- a/eng/scripts/build-cli-e2e-images.sh +++ b/eng/scripts/build-cli-e2e-images.sh @@ -8,26 +8,8 @@ build_with_mirror_retry() { local status echo "Building $display_name with Azure Ubuntu apt mirror (limit: $primary_timeout)" - # Do not use --foreground: timeout must signal the entire client process group. - # TERM lets Docker cancel its BuildKit session before default-source fallback. - # Normalize external signals in a supervising shell: Docker may handle TERM - # and exit 1, which otherwise looks like an ordinary build failure to timeout. - # On a deadline timeout still returns 124; external cancellation returns 143/130. - if timeout --verbose --signal=TERM --kill-after=30s "$primary_timeout" \ - bash -c ' - cancel() { - trap "" INT TERM - if [[ -n "${client_pid:-}" ]]; then - wait "$client_pid" - fi - exit "$1" - } - trap "cancel 143" TERM - trap "cancel 130" INT - "$@" & - client_pid=$! - wait "$client_pid" - ' bash "$@" --build-arg "UBUNTU_APT_MIRROR=http://azure.archive.ubuntu.com/ubuntu/" .; then + if node "$(dirname "${BASH_SOURCE[0]}")/run-bounded-image-build.ts" "$primary_timeout" \ + "$@" --build-arg "UBUNTU_APT_MIRROR=http://azure.archive.ubuntu.com/ubuntu/" .; then return 0 else status=$? diff --git a/eng/scripts/run-bounded-image-build.ts b/eng/scripts/run-bounded-image-build.ts new file mode 100644 index 00000000000..c081a12f500 --- /dev/null +++ b/eng/scripts/run-bounded-image-build.ts @@ -0,0 +1,79 @@ +#!/usr/bin/env node +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +import { spawn } from 'node:child_process'; +import { constants } from 'node:os'; + +const [duration, executable, ...args] = process.argv.slice(2); +const match = /^([1-9]\d*)(s|m)$/.exec(duration ?? ''); +if (!match || !executable || process.platform !== 'linux') { + console.error('Usage (Linux): node run-bounded-image-build.ts [arguments...]'); + process.exit(125); +} + +const milliseconds = Number(match[1]) * (match[2] === 'm' ? 60000 : 1000); +if (!Number.isSafeInteger(milliseconds) || milliseconds > 2147483647) { + console.error(`Invalid image build duration: ${duration}`); + process.exit(125); +} + +// A separate session lets us signal the client and all of its descendants, +// without signaling the workflow shell or other builds on the runner. +const child = spawn(executable, args, { detached: true, stdio: 'inherit' }); +let deadlineReached = false; +let cancellationExitCode = 0; +let forcedKill = false; +let deadline: NodeJS.Timeout | undefined; +let grace: NodeJS.Timeout | undefined; + +function signalGroup(signal: NodeJS.Signals): void { + if (child.pid === undefined) { + return; + } + try { + process.kill(-child.pid, signal); + } catch (error) { + // The client may finish between the signal event and the group kill. + if (!(error instanceof Error && 'code' in error && error.code === 'ESRCH')) { + throw error; + } + } +} + +function stopClient(): void { + signalGroup('SIGTERM'); + grace ??= setTimeout(() => { + forcedKill = true; + console.error('Image build did not acknowledge cancellation within 30s; killing its process group'); + signalGroup('SIGKILL'); + }, 30000); +} + +function cancel(exitCode: number): void { + // Keep cancellation independent of the deadline, including signals arriving + // during the deadline's cleanup grace. GNU timeout returns 124 in that race. + cancellationExitCode = exitCode; + console.error('Image build externally cancelled; default-source retry is disabled'); + stopClient(); +} + +process.on('SIGTERM', () => cancel(143)); +process.on('SIGINT', () => cancel(130)); +child.once('spawn', () => { + deadline = setTimeout(() => { + deadlineReached = true; + console.error(`Image build exceeded ${duration}; cancelling its process group`); + stopClient(); + }, milliseconds); +}); +child.once('error', error => { + console.error(`Could not start image build: ${error.message}`); + process.exit('code' in error && error.code === 'ENOENT' ? 127 : 125); +}); +child.once('exit', (code, signal) => { + clearTimeout(deadline); + clearTimeout(grace); + const clientExitCode = code ?? (signal ? 128 + constants.signals[signal] : 1); + process.exit(forcedKill ? 137 : cancellationExitCode || (deadlineReached ? 124 : clientExitCode)); +}); diff --git a/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs b/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs index ab61eb8de57..39811ad0122 100644 --- a/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs +++ b/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs @@ -17,12 +17,13 @@ public sealed class CliE2EImageBuildTests(ITestOutputHelper output) [InlineData("forced-kill")] [InlineData("cancelled")] [InlineData("external-cancel")] + [InlineData("cleanup-cancel")] [InlineData("all-images")] [InlineData("dotnet-only")] - [RequiresTools(["node", "bash", "timeout"])] + [RequiresTools(["node", "bash"])] public async Task ImageBuildPreservesFallbackAndCancellation(string scenario) { - Assert.SkipWhen(!OperatingSystem.IsLinux(), "GNU timeout process-group behavior requires Linux."); + Assert.SkipWhen(!OperatingSystem.IsLinux(), "Image build process-group behavior requires Linux."); using var command = new NodeCommand(output).WithTimeout(TimeSpan.FromMinutes(2)); var result = await command.ExecuteScriptAsync( Path.Combine(RepoRoot.Path, "tests", "Infrastructure.Tests", "Pipelines", "cli-e2e-image-build.harness.ts"), diff --git a/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts b/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts index efa42d412a6..4988e2b1f8c 100644 --- a/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts +++ b/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts @@ -52,15 +52,26 @@ function stubDocker(): void { case 'cancelled': process.kill(process.pid, 'SIGTERM'); return; - case 'external-cancel': { + case 'external-cancel': + case 'cleanup-cancel': { + let stopping = false; process.on('SIGTERM', () => { - console.error('Client handled external TERM and exited 1'); - process.exit(1); + if (stopping) { + return; + } + stopping = true; + if (scenario === 'cleanup-cancel') { + console.error('Deadline cleanup started; sending external TERM to supervisor'); + process.kill(process.ppid, 'SIGTERM'); + } + setTimeout(() => { + console.error('Client handled external TERM and exited 1'); + process.exit(1); + }, 100); }); - // Signal the real timeout supervisor, not the client or its wrapper. - // /proc/PID/stat starts "123 (bash) S 456 ...", where 456 is PPID. - const timeoutPid = Number(fs.readFileSync(`/proc/${process.ppid}/stat`, 'utf8').split(') ')[1].split(' ')[1]); - process.kill(timeoutPid, 'SIGTERM'); + if (scenario === 'external-cancel') { + process.kill(process.ppid, 'SIGTERM'); + } setInterval(() => {}, 1000); return; } @@ -135,7 +146,7 @@ exec "$IMAGE_BUILD_TEST_NODE" "$IMAGE_BUILD_TEST_HARNESS" --stub "$@" const output = result.stdout + result.stderr; const expectedStatus = scenario === 'fallback-failure' ? 42 : scenario === 'forced-kill' ? 137 - : scenario === 'cancelled' || scenario === 'external-cancel' ? 143 : 0; + : ['cancelled', 'external-cancel', 'cleanup-cancel'].includes(scenario) ? 143 : 0; assert.equal(result.status, expectedStatus, output); const trace = fs.readFileSync(path.join(workspace, 'trace'), 'utf8').trim().split('\n').map(line => JSON.parse(line)); if (orchestration) { @@ -164,7 +175,7 @@ exec "$IMAGE_BUILD_TEST_NODE" "$IMAGE_BUILD_TEST_HARNESS" --stub "$@" ]); } else { const expectedArgs = ['build', '-f', 'Dockerfile with spaces', '-t', 'test:primary']; - const noFallback = ['success', 'forced-kill', 'cancelled', 'external-cancel'].includes(scenario); + const noFallback = ['success', 'forced-kill', 'cancelled', 'external-cancel', 'cleanup-cancel'].includes(scenario); assert.deepEqual(trace.map(entry => entry.args), [ [...expectedArgs, '--build-arg', mirrorArg, '.'], ...(noFallback ? [] : [[...expectedArgs, '.']]) @@ -179,12 +190,16 @@ exec "$IMAGE_BUILD_TEST_NODE" "$IMAGE_BUILD_TEST_HARNESS" --stub "$@" assert.match(output, /exceeded 5s; retrying with default Ubuntu apt sources/); assert.ok(Date.now() - started < 15000, output); } - if (scenario === 'forced-kill' || scenario === 'cancelled' || scenario === 'external-cancel') { + if (['forced-kill', 'cancelled', 'external-cancel', 'cleanup-cancel'].includes(scenario)) { assert.match(output, /not retrying after cancellation or timeout execution failure/); } - if (scenario === 'external-cancel') { + if (scenario === 'external-cancel' || scenario === 'cleanup-cancel') { assert.match(output, /Client handled external TERM and exited 1/); } + if (scenario === 'cleanup-cancel') { + assert.match(output, /Deadline cleanup started; sending external TERM to supervisor/); + assert.match(output, /Image build exceeded 5s/); + } } const childPath = path.join(workspace, 'child-pid'); assert.equal(isAlive(Number(fs.readFileSync(path.join(workspace, 'primary-pid'), 'utf8'))), false, output); diff --git a/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs b/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs index cef09f1a3e1..f85c9e92d88 100644 --- a/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs +++ b/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs @@ -282,6 +282,10 @@ private static IReadOnlySet LoadSolutionProjectPaths() "eng/scripts/build-cli-e2e-images.sh", ["test:Aspire.Cli.EndToEnd.Tests", "test:Infrastructure.Tests"] }, + { + "eng/scripts/run-bounded-image-build.ts", + ["test:Aspire.Cli.EndToEnd.Tests", "test:Infrastructure.Tests"] + }, { "eng/clipack/Common.projitems", [ From b8274fc8e0377c70ada8aa9eecfbdbc4069c6ff7 Mon Sep 17 00:00:00 2001 From: IEvangelist <7679720+IEvangelist@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:50:31 -0600 Subject: [PATCH 4/4] Fail closed when the image client is killed during cleanup Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- eng/scripts/run-bounded-image-build.ts | 3 ++- .../Pipelines/CliE2EImageBuildTests.cs | 1 + .../Pipelines/cli-e2e-image-build.harness.ts | 10 +++++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/eng/scripts/run-bounded-image-build.ts b/eng/scripts/run-bounded-image-build.ts index c081a12f500..271a16c877d 100644 --- a/eng/scripts/run-bounded-image-build.ts +++ b/eng/scripts/run-bounded-image-build.ts @@ -75,5 +75,6 @@ child.once('exit', (code, signal) => { clearTimeout(deadline); clearTimeout(grace); const clientExitCode = code ?? (signal ? 128 + constants.signals[signal] : 1); - process.exit(forcedKill ? 137 : cancellationExitCode || (deadlineReached ? 124 : clientExitCode)); + // An external kill/OOM during cleanup is not acknowledged cancellation. + process.exit(forcedKill || clientExitCode === 137 ? 137 : cancellationExitCode || (deadlineReached ? 124 : clientExitCode)); }); diff --git a/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs b/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs index 39811ad0122..f89a12d6d86 100644 --- a/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs +++ b/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs @@ -18,6 +18,7 @@ public sealed class CliE2EImageBuildTests(ITestOutputHelper output) [InlineData("cancelled")] [InlineData("external-cancel")] [InlineData("cleanup-cancel")] + [InlineData("cleanup-kill")] [InlineData("all-images")] [InlineData("dotnet-only")] [RequiresTools(["node", "bash"])] diff --git a/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts b/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts index 4988e2b1f8c..1d4cb25aec5 100644 --- a/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts +++ b/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts @@ -52,6 +52,10 @@ function stubDocker(): void { case 'cancelled': process.kill(process.pid, 'SIGTERM'); return; + case 'cleanup-kill': + process.on('SIGTERM', () => process.kill(process.pid, 'SIGKILL')); + setInterval(() => {}, 1000); + return; case 'external-cancel': case 'cleanup-cancel': { let stopping = false; @@ -145,7 +149,7 @@ exec "$IMAGE_BUILD_TEST_NODE" "$IMAGE_BUILD_TEST_HARNESS" --stub "$@" assert.ifError(result.error); const output = result.stdout + result.stderr; const expectedStatus = scenario === 'fallback-failure' ? 42 - : scenario === 'forced-kill' ? 137 + : scenario === 'forced-kill' || scenario === 'cleanup-kill' ? 137 : ['cancelled', 'external-cancel', 'cleanup-cancel'].includes(scenario) ? 143 : 0; assert.equal(result.status, expectedStatus, output); const trace = fs.readFileSync(path.join(workspace, 'trace'), 'utf8').trim().split('\n').map(line => JSON.parse(line)); @@ -175,7 +179,7 @@ exec "$IMAGE_BUILD_TEST_NODE" "$IMAGE_BUILD_TEST_HARNESS" --stub "$@" ]); } else { const expectedArgs = ['build', '-f', 'Dockerfile with spaces', '-t', 'test:primary']; - const noFallback = ['success', 'forced-kill', 'cancelled', 'external-cancel', 'cleanup-cancel'].includes(scenario); + const noFallback = ['success', 'forced-kill', 'cleanup-kill', 'cancelled', 'external-cancel', 'cleanup-cancel'].includes(scenario); assert.deepEqual(trace.map(entry => entry.args), [ [...expectedArgs, '--build-arg', mirrorArg, '.'], ...(noFallback ? [] : [[...expectedArgs, '.']]) @@ -190,7 +194,7 @@ exec "$IMAGE_BUILD_TEST_NODE" "$IMAGE_BUILD_TEST_HARNESS" --stub "$@" assert.match(output, /exceeded 5s; retrying with default Ubuntu apt sources/); assert.ok(Date.now() - started < 15000, output); } - if (['forced-kill', 'cancelled', 'external-cancel', 'cleanup-cancel'].includes(scenario)) { + if (['forced-kill', 'cleanup-kill', 'cancelled', 'external-cancel', 'cleanup-cancel'].includes(scenario)) { assert.match(output, /not retrying after cancellation or timeout execution failure/); } if (scenario === 'external-cancel' || scenario === 'cleanup-cancel') {