diff --git a/.github/workflows/build-cli-e2e-image.yml b/.github/workflows/build-cli-e2e-image.yml index 71a806f8c16..735525c3525 100644 --- a/.github/workflows/build-cli-e2e-image.yml +++ b/.github/workflows/build-cli-e2e-image.yml @@ -18,7 +18,6 @@ on: env: CLI_E2E_DOTNET_IMAGE_ARTIFACT: cli-e2e-dotnet-image CLI_E2E_DOTNET_IMAGE_TAG: aspire-cli-e2e-dotnet:prebuilt - CLI_E2E_DOTNET_IMAGE_CACHE_SCOPE: cli-e2e-dotnet CLI_E2E_POLYGLOT_IMAGE_ARTIFACT: cli-e2e-polyglot-image CLI_E2E_POLYGLOT_IMAGE_TAG: aspire-cli-e2e-polyglot:prebuilt CLI_E2E_POLYGLOT_JAVA_IMAGE_ARTIFACT: cli-e2e-polyglot-java-image @@ -26,7 +25,6 @@ env: CLI_E2E_POLYGLOT_GO_IMAGE_ARTIFACT: cli-e2e-polyglot-go-image CLI_E2E_POLYGLOT_GO_IMAGE_TAG: aspire-cli-e2e-polyglot-go:prebuilt CLI_E2E_POLYGLOT_BASE_IMAGE_TAG: aspire-e2e-polyglot-base:latest - CLI_E2E_POLYGLOT_BASE_IMAGE_CACHE_SCOPE: cli-e2e-polyglot-base CLI_E2E_INCLUDE_POLYGLOT_IMAGES: ${{ inputs.includePolyglotImages }} jobs: @@ -57,101 +55,7 @@ jobs: - name: Build CLI E2E Docker image shell: bash - run: | - set -euo pipefail - - docker buildx create --name cli-e2e-builder --use - docker buildx inspect --bootstrap - - build_image() { - local display_name="$1" - local dockerfile="$2" - local tag="$3" - local cache_scope="$4" - local mirror="$5" - local -a build_args=(--build-arg SKIP_SOURCE_BUILD=true) - - if [[ -n "$mirror" ]]; then - echo "Building $display_name with Ubuntu apt mirror: $mirror" - build_args+=(--build-arg "UBUNTU_APT_MIRROR=$mirror") - else - echo "Building $display_name with the default Ubuntu apt sources" - fi - - docker buildx build \ - --load \ - --cache-from "type=gha,scope=$cache_scope" \ - --cache-to "type=gha,scope=$cache_scope,mode=max,ignore-error=true" \ - "${build_args[@]}" \ - -f "$dockerfile" \ - -t "$tag" \ - . - } - - build_java_image() { - local mirror="$1" - local -a build_args=() - - if [[ -n "$mirror" ]]; then - echo "Building Java polyglot image with Ubuntu apt mirror: $mirror" - build_args+=(--build-arg "UBUNTU_APT_MIRROR=$mirror") - else - echo "Building Java polyglot image with the default Ubuntu apt sources" - fi - - DOCKER_BUILDKIT=1 docker build \ - "${build_args[@]}" \ - -f tests/Shared/Docker/Dockerfile.e2e-polyglot-java \ - -t "$CLI_E2E_POLYGLOT_JAVA_IMAGE_TAG" \ - . - } - - build_go_image() { - local mirror="$1" - local -a build_args=() - - if [[ -n "$mirror" ]]; then - echo "Building Go polyglot image with Ubuntu apt mirror: $mirror" - build_args+=(--build-arg "UBUNTU_APT_MIRROR=$mirror") - else - echo "Building Go polyglot image with the default Ubuntu apt sources" - fi - - DOCKER_BUILDKIT=1 docker build \ - "${build_args[@]}" \ - -f tests/Shared/Docker/Dockerfile.e2e-polyglot-go \ - -t "$CLI_E2E_POLYGLOT_GO_IMAGE_TAG" \ - . - } - - build_with_mirror_retry() { - local display_name="$1" - local dockerfile="$2" - local tag="$3" - local cache_scope="$4" - - if ! build_image "$display_name" "$dockerfile" "$tag" "$cache_scope" "http://azure.archive.ubuntu.com/ubuntu/"; then - echo "$display_name build failed with the Azure Ubuntu apt mirror; retrying with default Ubuntu apt sources" - build_image "$display_name" "$dockerfile" "$tag" "$cache_scope" "" - fi - } - - build_with_mirror_retry ".NET image" "tests/Shared/Docker/Dockerfile.e2e" "$CLI_E2E_DOTNET_IMAGE_TAG" "$CLI_E2E_DOTNET_IMAGE_CACHE_SCOPE" - - if [[ "$CLI_E2E_INCLUDE_POLYGLOT_IMAGES" == "true" ]]; then - build_with_mirror_retry "polyglot base image" "tests/Shared/Docker/Dockerfile.e2e-polyglot-base" "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG" "$CLI_E2E_POLYGLOT_BASE_IMAGE_CACHE_SCOPE" - docker tag "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG" "$CLI_E2E_POLYGLOT_IMAGE_TAG" - - if ! build_java_image "http://azure.archive.ubuntu.com/ubuntu/"; then - echo "Java polyglot image build failed with the Azure Ubuntu apt mirror; retrying with default Ubuntu apt sources" - build_java_image "" - fi - - if ! build_go_image "http://azure.archive.ubuntu.com/ubuntu/"; then - echo "Go polyglot image build failed with the Azure Ubuntu apt mirror; retrying with default Ubuntu apt sources" - build_go_image "" - fi - fi + run: bash eng/scripts/build-cli-e2e-images.sh - name: Save CLI E2E Docker image tarballs if: ${{ inputs.uploadImageArtifacts }} diff --git a/.github/workflows/warm-cli-e2e-image-cache.yml b/.github/workflows/warm-cli-e2e-image-cache.yml index 0f4d88e9162..9e4b0aba2ee 100644 --- a/.github/workflows/warm-cli-e2e-image-cache.yml +++ b/.github/workflows/warm-cli-e2e-image-cache.yml @@ -16,6 +16,8 @@ on: - '.dockerignore' - 'eng/scripts/get-aspire-cli.sh' - 'eng/scripts/get-aspire-cli-pr.sh' + - 'eng/scripts/build-cli-e2e-images.sh' + - 'eng/scripts/run-bounded-image-build.ts' - 'tests/Shared/Docker/Dockerfile.e2e' - 'tests/Shared/Docker/Dockerfile.e2e-polyglot-base' - 'tests/Shared/Docker/Dockerfile.e2e-polyglot-go' diff --git a/docs/ci/cli-e2e-images.md b/docs/ci/cli-e2e-images.md index 8a315f11684..c9d7ea08aa5 100644 --- a/docs/ci/cli-e2e-images.md +++ b/docs/ci/cli-e2e-images.md @@ -29,6 +29,33 @@ Consumer workflows download image artifacts into `${{ github.workspace }}/cli-e2 The reusable image build workflow also has an `uploadImageArtifacts` input. It defaults to `true` for test workflows because each isolated test job needs the image tarballs. Cache-warming workflows set it to `false` so they only build the images and export BuildKit cache layers. +### Bounded apt mirror fallback + +Both test builds and the cache warmer run `eng/scripts/build-cli-e2e-images.sh`. +Each image (.NET, polyglot base, Java and Go) first uses the Azure Ubuntu apt +mirror, with a five-minute limit. A failed or timed-out primary attempt retries +once with the Dockerfile's default Ubuntu apt sources. The fallback's failure +exit code and both attempts' output are preserved; there is no third attempt. + +The job still has a 45-minute cap. Four primary attempts can consume at most +20 minutes plus two minutes of cancellation grace, leaving 23 minutes for +fallback builds, setup, saving and uploading images. This prevents a slow but +successful apt transfer from consuming the entire job before fallback runs. +Observed throttled transfers took 38-41 minutes for 82-108 MB. + +The Node.js build supervisor signals the Docker client's process group with `TERM` so +BuildKit can cancel the build session before fallback starts. If the client +cannot stop within another 30 seconds, the supervisor kills the process group and the +script fails without fallback: a forced kill cannot establish that the remote +build acknowledged cancellation. Interrupts, termination and supervisor execution +errors also fail without retrying. + +The mirror configuration script preserves the base image's original Ubuntu +sources before replacing them. An empty mirror restores those files, including +in derived Java/Go images, without changing third-party repositories. A +supervisor tracks external cancellation independently of its deadline, including +signals during cleanup and clients that exit with an ordinary failure code. + ## Cross-PR BuildKit cache The image build workflow exports BuildKit layers to the GitHub Actions cache with stable scopes: diff --git a/eng/github-ci/test-trigger-map.yml b/eng/github-ci/test-trigger-map.yml index f0854d1365b..cb132f968fe 100644 --- a/eng/github-ci/test-trigger-map.yml +++ b/eng/github-ci/test-trigger-map.yml @@ -432,6 +432,11 @@ path_rules: - eng/scripts/load-cli-e2e-images.sh targets: [test:Aspire.Cli.EndToEnd.Tests] reason: builds or loads the CLI E2E images consumed only by Aspire.Cli.EndToEnd.Tests + - paths: + - eng/scripts/build-cli-e2e-images.sh + - eng/scripts/run-bounded-image-build.ts + targets: [test:Aspire.Cli.EndToEnd.Tests, test:Infrastructure.Tests] + reason: shared image build entry point and its shell behavior tests - paths: - eng/scripts/pack-cli-npm-package.ps1 targets: [test:Aspire.Cli.Tests, test:Infrastructure.Tests] diff --git a/eng/scripts/build-cli-e2e-images.sh b/eng/scripts/build-cli-e2e-images.sh new file mode 100644 index 00000000000..22bcecb394f --- /dev/null +++ b/eng/scripts/build-cli-e2e-images.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail + +build_with_mirror_retry() { + local primary_timeout="$1" + local display_name="$2" + shift 2 + local status + + echo "Building $display_name with Azure Ubuntu apt mirror (limit: $primary_timeout)" + if node "$(dirname "${BASH_SOURCE[0]}")/run-bounded-image-build.ts" "$primary_timeout" \ + "$@" --build-arg "UBUNTU_APT_MIRROR=http://azure.archive.ubuntu.com/ubuntu/" .; then + return 0 + else + status=$? + fi + + case "$status" in + 124) + echo "$display_name primary mirror build exceeded $primary_timeout; retrying with default Ubuntu apt sources" >&2 + ;; + 125|126|127|130|137|143) + # A forced kill cannot confirm BuildKit acknowledged cancellation. Do + # not overlap another build or turn workflow cancellation into a retry. + echo "$display_name primary build stopped with exit code $status; not retrying after cancellation or timeout execution failure" >&2 + return "$status" + ;; + *) + echo "$display_name build failed with the Azure Ubuntu apt mirror (exit code $status); retrying with default Ubuntu apt sources" >&2 + ;; + esac + + "$@" . +} + +build_cli_e2e_images() { + # Four primary attempts can spend at most 20m plus 2m cancellation grace, + # leaving 23m of the 45m job for default sources, setup and artifact handling. + # Throttled apt downloads have otherwise consumed 38-41m in a single image. + local primary_timeout=5m + local dotnet_cache_scope=cli-e2e-dotnet + local polyglot_cache_scope=cli-e2e-polyglot-base + + docker buildx create --name cli-e2e-builder --use + docker buildx inspect --bootstrap + + build_with_mirror_retry "$primary_timeout" ".NET image" docker buildx build \ + --load \ + --cache-from "type=gha,scope=$dotnet_cache_scope" \ + --cache-to "type=gha,scope=$dotnet_cache_scope,mode=max,ignore-error=true" \ + --build-arg SKIP_SOURCE_BUILD=true \ + -f tests/Shared/Docker/Dockerfile.e2e \ + -t "$CLI_E2E_DOTNET_IMAGE_TAG" + + if [[ "$CLI_E2E_INCLUDE_POLYGLOT_IMAGES" == "true" ]]; then + build_with_mirror_retry "$primary_timeout" "polyglot base image" docker buildx build \ + --load \ + --cache-from "type=gha,scope=$polyglot_cache_scope" \ + --cache-to "type=gha,scope=$polyglot_cache_scope,mode=max,ignore-error=true" \ + --build-arg SKIP_SOURCE_BUILD=true \ + -f tests/Shared/Docker/Dockerfile.e2e-polyglot-base \ + -t "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG" + docker tag "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG" "$CLI_E2E_POLYGLOT_IMAGE_TAG" + + DOCKER_BUILDKIT=1 build_with_mirror_retry "$primary_timeout" "Java polyglot image" docker build \ + -f tests/Shared/Docker/Dockerfile.e2e-polyglot-java \ + -t "$CLI_E2E_POLYGLOT_JAVA_IMAGE_TAG" + DOCKER_BUILDKIT=1 build_with_mirror_retry "$primary_timeout" "Go polyglot image" docker build \ + -f tests/Shared/Docker/Dockerfile.e2e-polyglot-go \ + -t "$CLI_E2E_POLYGLOT_GO_IMAGE_TAG" + fi +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + build_cli_e2e_images +fi diff --git a/eng/scripts/run-bounded-image-build.ts b/eng/scripts/run-bounded-image-build.ts new file mode 100644 index 00000000000..271a16c877d --- /dev/null +++ b/eng/scripts/run-bounded-image-build.ts @@ -0,0 +1,80 @@ +#!/usr/bin/env node +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +import { spawn } from 'node:child_process'; +import { constants } from 'node:os'; + +const [duration, executable, ...args] = process.argv.slice(2); +const match = /^([1-9]\d*)(s|m)$/.exec(duration ?? ''); +if (!match || !executable || process.platform !== 'linux') { + console.error('Usage (Linux): node run-bounded-image-build.ts [arguments...]'); + process.exit(125); +} + +const milliseconds = Number(match[1]) * (match[2] === 'm' ? 60000 : 1000); +if (!Number.isSafeInteger(milliseconds) || milliseconds > 2147483647) { + console.error(`Invalid image build duration: ${duration}`); + process.exit(125); +} + +// A separate session lets us signal the client and all of its descendants, +// without signaling the workflow shell or other builds on the runner. +const child = spawn(executable, args, { detached: true, stdio: 'inherit' }); +let deadlineReached = false; +let cancellationExitCode = 0; +let forcedKill = false; +let deadline: NodeJS.Timeout | undefined; +let grace: NodeJS.Timeout | undefined; + +function signalGroup(signal: NodeJS.Signals): void { + if (child.pid === undefined) { + return; + } + try { + process.kill(-child.pid, signal); + } catch (error) { + // The client may finish between the signal event and the group kill. + if (!(error instanceof Error && 'code' in error && error.code === 'ESRCH')) { + throw error; + } + } +} + +function stopClient(): void { + signalGroup('SIGTERM'); + grace ??= setTimeout(() => { + forcedKill = true; + console.error('Image build did not acknowledge cancellation within 30s; killing its process group'); + signalGroup('SIGKILL'); + }, 30000); +} + +function cancel(exitCode: number): void { + // Keep cancellation independent of the deadline, including signals arriving + // during the deadline's cleanup grace. GNU timeout returns 124 in that race. + cancellationExitCode = exitCode; + console.error('Image build externally cancelled; default-source retry is disabled'); + stopClient(); +} + +process.on('SIGTERM', () => cancel(143)); +process.on('SIGINT', () => cancel(130)); +child.once('spawn', () => { + deadline = setTimeout(() => { + deadlineReached = true; + console.error(`Image build exceeded ${duration}; cancelling its process group`); + stopClient(); + }, milliseconds); +}); +child.once('error', error => { + console.error(`Could not start image build: ${error.message}`); + process.exit('code' in error && error.code === 'ENOENT' ? 127 : 125); +}); +child.once('exit', (code, signal) => { + clearTimeout(deadline); + clearTimeout(grace); + const clientExitCode = code ?? (signal ? 128 + constants.signals[signal] : 1); + // An external kill/OOM during cleanup is not acknowledged cancellation. + process.exit(forcedKill || clientExitCode === 137 ? 137 : cancellationExitCode || (deadlineReached ? 124 : clientExitCode)); +}); diff --git a/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs b/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs new file mode 100644 index 00000000000..f89a12d6d86 --- /dev/null +++ b/tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Aspire.TestUtilities; +using Xunit; + +namespace Infrastructure.Tests; + +public sealed class CliE2EImageBuildTests(ITestOutputHelper output) +{ + [Theory] + [InlineData("success")] + [InlineData("failure")] + [InlineData("fallback-failure")] + [InlineData("slow-success")] + [InlineData("stall")] + [InlineData("forced-kill")] + [InlineData("cancelled")] + [InlineData("external-cancel")] + [InlineData("cleanup-cancel")] + [InlineData("cleanup-kill")] + [InlineData("all-images")] + [InlineData("dotnet-only")] + [RequiresTools(["node", "bash"])] + public async Task ImageBuildPreservesFallbackAndCancellation(string scenario) + { + Assert.SkipWhen(!OperatingSystem.IsLinux(), "Image build process-group behavior requires Linux."); + using var command = new NodeCommand(output).WithTimeout(TimeSpan.FromMinutes(2)); + var result = await command.ExecuteScriptAsync( + Path.Combine(RepoRoot.Path, "tests", "Infrastructure.Tests", "Pipelines", "cli-e2e-image-build.harness.ts"), + RepoRoot.Path, scenario); + + result.EnsureSuccessful(); + } + + [Fact] + [RequiresTools(["node", "docker"])] + public async Task DerivedImageFallbackRestoresOriginalAptSources() + { + using var command = new NodeCommand(output).WithTimeout(TimeSpan.FromMinutes(2)); + var result = await command.ExecuteScriptAsync( + Path.Combine(RepoRoot.Path, "tests", "Infrastructure.Tests", "Pipelines", "cli-e2e-image-build.harness.ts"), + RepoRoot.Path, "inherited-sources"); + + result.EnsureSuccessful(); + } +} diff --git a/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts b/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts new file mode 100644 index 00000000000..1d4cb25aec5 --- /dev/null +++ b/tests/Infrastructure.Tests/Pipelines/cli-e2e-image-build.harness.ts @@ -0,0 +1,272 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn, spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const mirrorArg = 'UBUNTU_APT_MIRROR=http://azure.archive.ubuntu.com/ubuntu/'; + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + // /proc/PID/stat starts "123 (sleep) S ...". A killed orphan can briefly + // remain as a zombie ('Z') until init reaps it. + return fs.readFileSync(`/proc/${pid}/stat`, 'utf8').split(') ')[1][0] !== 'Z'; + } catch (error) { + if (error instanceof Error && 'code' in error && (error.code === 'ESRCH' || error.code === 'ENOENT')) { + return false; + } + throw error; + } +} + +function stubDocker(): void { + const root = process.env.IMAGE_BUILD_TEST_ROOT!; + const scenario = process.env.IMAGE_BUILD_TEST_SCENARIO!; + const args = process.argv.slice(3); + fs.appendFileSync(path.join(root, 'trace'), JSON.stringify({ args, buildkit: process.env.DOCKER_BUILDKIT }) + '\n'); + if (!args.includes('build')) { + return; + } + if (!args.includes(mirrorArg)) { + if (scenario === 'stall' || scenario === 'slow-success') { + const childPid = Number(fs.readFileSync(path.join(root, 'child-pid'), 'utf8')); + assert.equal(isAlive(childPid), false, 'Fallback started before primary child stopped'); + assert.equal(fs.readFileSync(path.join(root, 'cancelled'), 'utf8'), 'acknowledged'); + } + console.log('default sources output'); + console.error('default sources diagnostic'); + process.exit(scenario === 'fallback-failure' ? 42 : 0); + } + + console.log('primary mirror output'); + console.error('primary mirror diagnostic'); + fs.writeFileSync(path.join(root, 'primary-pid'), String(process.pid)); + switch (scenario) { + case 'success': + return; + case 'cancelled': + process.kill(process.pid, 'SIGTERM'); + return; + case 'cleanup-kill': + process.on('SIGTERM', () => process.kill(process.pid, 'SIGKILL')); + setInterval(() => {}, 1000); + return; + case 'external-cancel': + case 'cleanup-cancel': { + let stopping = false; + process.on('SIGTERM', () => { + if (stopping) { + return; + } + stopping = true; + if (scenario === 'cleanup-cancel') { + console.error('Deadline cleanup started; sending external TERM to supervisor'); + process.kill(process.ppid, 'SIGTERM'); + } + setTimeout(() => { + console.error('Client handled external TERM and exited 1'); + process.exit(1); + }, 100); + }); + if (scenario === 'external-cancel') { + process.kill(process.ppid, 'SIGTERM'); + } + setInterval(() => {}, 1000); + return; + } + case 'stall': + case 'slow-success': + case 'forced-kill': { + // A real descendant, not a mocked timeout, must stop before retry. + const child = spawn('bash', ['-c', ` + ${scenario === 'forced-kill' ? "trap '' TERM" : ''} + echo "$$" > "$IMAGE_BUILD_TEST_ROOT/child-pid" + exec sleep 300 + `], { stdio: 'inherit' }); + if (scenario === 'forced-kill') { + process.on('SIGTERM', () => {}); + } else { + process.on('SIGTERM', () => { + child.once('exit', () => { + fs.writeFileSync(path.join(root, 'cancelled'), 'acknowledged'); + process.exit(143); + }); + if (child.exitCode !== null || child.signalCode !== null) { + fs.writeFileSync(path.join(root, 'cancelled'), 'acknowledged'); + process.exit(143); + } + }); + } + if (scenario === 'slow-success') { + setTimeout(() => process.exit(0), 30000); + } else { + setInterval(() => {}, 1000); + } + return; + } + default: + process.exit(17); + } +} + +function testBuild(repo: string, scenario: string): void { + const workspace = fs.mkdtempSync(path.join(os.tmpdir(), 'aspire-image-build-')); + try { + const bin = path.join(workspace, 'commands with spaces'); + fs.mkdirSync(bin); + fs.writeFileSync(path.join(bin, 'docker'), `#!/usr/bin/env bash +exec "$IMAGE_BUILD_TEST_NODE" "$IMAGE_BUILD_TEST_HARNESS" --stub "$@" +`, { mode: 0o755 }); + const script = path.join(repo, 'eng', 'scripts', 'build-cli-e2e-images.sh'); + const orchestration = scenario === 'all-images' || scenario === 'dotnet-only'; + const started = Date.now(); + const result = spawnSync('bash', orchestration + ? [script] + : ['-c', 'source "$1"; build_with_mirror_retry 5s "test image" docker build -f "Dockerfile with spaces" -t test:primary', 'bash', script], { + env: { + ...process.env, + PATH: `${bin}${path.delimiter}${process.env.PATH}`, + IMAGE_BUILD_TEST_NODE: process.execPath, + IMAGE_BUILD_TEST_HARNESS: fileURLToPath(import.meta.url), + IMAGE_BUILD_TEST_ROOT: workspace, + IMAGE_BUILD_TEST_SCENARIO: scenario, + CLI_E2E_INCLUDE_POLYGLOT_IMAGES: scenario === 'all-images' ? 'true' : 'false', + CLI_E2E_DOTNET_IMAGE_TAG: 'test:dotnet', + CLI_E2E_POLYGLOT_BASE_IMAGE_TAG: 'test:base', + CLI_E2E_POLYGLOT_IMAGE_TAG: 'test:polyglot', + CLI_E2E_POLYGLOT_JAVA_IMAGE_TAG: 'test:java', + CLI_E2E_POLYGLOT_GO_IMAGE_TAG: 'test:go' + }, + encoding: 'utf8', + timeout: 60000, + killSignal: 'SIGKILL' + }); + assert.ifError(result.error); + const output = result.stdout + result.stderr; + const expectedStatus = scenario === 'fallback-failure' ? 42 + : scenario === 'forced-kill' || scenario === 'cleanup-kill' ? 137 + : ['cancelled', 'external-cancel', 'cleanup-cancel'].includes(scenario) ? 143 : 0; + assert.equal(result.status, expectedStatus, output); + const trace = fs.readFileSync(path.join(workspace, 'trace'), 'utf8').trim().split('\n').map(line => JSON.parse(line)); + if (orchestration) { + const builds = trace.filter(entry => entry.args.includes('build')); + const variants = scenario === 'all-images' ? ['dotnet', 'base', 'java', 'go'] : ['dotnet']; + assert.equal(builds.length, variants.length * 2, output); + for (const [index, variant] of variants.entries()) { + const primary = builds[index * 2]; + const fallback = builds[index * 2 + 1]; + assert.equal(primary.args[primary.args.indexOf('-t') + 1], `test:${variant}`); + assert.deepEqual(primary.args.slice(-3), ['--build-arg', mirrorArg, '.']); + assert.deepEqual(fallback.args, [...primary.args.slice(0, -3), '.']); + if (variant === 'java' || variant === 'go') { + assert.equal(primary.buildkit, '1'); + assert.equal(fallback.buildkit, '1'); + } else { + assert.ok(primary.args.includes('--load')); + assert.ok(primary.args.includes('SKIP_SOURCE_BUILD=true')); + assert.ok(primary.args.includes(`type=gha,scope=cli-e2e-${variant === 'dotnet' ? 'dotnet' : 'polyglot-base'}`)); + } + } + assert.deepEqual(trace.filter(entry => !entry.args.includes('build')).map(entry => entry.args), [ + ['buildx', 'create', '--name', 'cli-e2e-builder', '--use'], + ['buildx', 'inspect', '--bootstrap'], + ...(scenario === 'all-images' ? [['tag', 'test:base', 'test:polyglot']] : []) + ]); + } else { + const expectedArgs = ['build', '-f', 'Dockerfile with spaces', '-t', 'test:primary']; + const noFallback = ['success', 'forced-kill', 'cleanup-kill', 'cancelled', 'external-cancel', 'cleanup-cancel'].includes(scenario); + assert.deepEqual(trace.map(entry => entry.args), [ + [...expectedArgs, '--build-arg', mirrorArg, '.'], + ...(noFallback ? [] : [[...expectedArgs, '.']]) + ]); + assert.ok(output.includes('primary mirror output'), output); + assert.ok(output.includes('primary mirror diagnostic'), output); + if (!noFallback) { + assert.ok(output.includes('default sources output'), output); + assert.ok(output.includes('default sources diagnostic'), output); + } + if (scenario === 'stall' || scenario === 'slow-success') { + assert.match(output, /exceeded 5s; retrying with default Ubuntu apt sources/); + assert.ok(Date.now() - started < 15000, output); + } + if (['forced-kill', 'cleanup-kill', 'cancelled', 'external-cancel', 'cleanup-cancel'].includes(scenario)) { + assert.match(output, /not retrying after cancellation or timeout execution failure/); + } + if (scenario === 'external-cancel' || scenario === 'cleanup-cancel') { + assert.match(output, /Client handled external TERM and exited 1/); + } + if (scenario === 'cleanup-cancel') { + assert.match(output, /Deadline cleanup started; sending external TERM to supervisor/); + assert.match(output, /Image build exceeded 5s/); + } + } + const childPath = path.join(workspace, 'child-pid'); + assert.equal(isAlive(Number(fs.readFileSync(path.join(workspace, 'primary-pid'), 'utf8'))), false, output); + if (fs.existsSync(childPath)) { + assert.equal(isAlive(Number(fs.readFileSync(childPath, 'utf8'))), false, output); + } + console.log(output); + console.log(`Passed: ${scenario}`); + } finally { + for (const name of ['primary-pid', 'child-pid']) { + const pidPath = path.join(workspace, name); + if (fs.existsSync(pidPath)) { + const pid = Number(fs.readFileSync(pidPath, 'utf8')); + if (isAlive(pid)) { + process.kill(pid, 'SIGKILL'); + } + } + } + fs.rmSync(workspace, { recursive: true, force: true }); + } +} + +function testInheritedSources(repo: string): void { + const script = path.join(repo, 'tests', 'Shared', 'Docker', 'configure-ubuntu-apt-mirror.sh'); + const result = spawnSync('docker', [ + 'run', '--rm', '--interactive', + 'mcr.microsoft.com/mirror/docker/library/ubuntu:24.04', 'sh', '-ec', ` + cat > /configure-mirror.sh + cp /etc/apt/sources.list.d/ubuntu.sources /tmp/original.sources + echo 'deb https://packages.example.invalid stable main' > /etc/apt/sources.list.d/third-party.list + for legacy in absent present; do + rm -rf /etc/apt/aspire-default-ubuntu-sources + if [ "$legacy" = present ]; then + echo '# original legacy sources' > /etc/apt/sources.list + else + rm -f /etc/apt/sources.list + fi + sh /configure-mirror.sh http://azure.archive.ubuntu.com/ubuntu/ + grep -q 'http://azure.archive.ubuntu.com/ubuntu' /etc/apt/sources.list.d/ubuntu.sources + # The derived layer must not overwrite the base layer's snapshot. + sh /configure-mirror.sh https://different.example.invalid/ubuntu + sh /configure-mirror.sh "" + cmp /tmp/original.sources /etc/apt/sources.list.d/ubuntu.sources + if [ "$legacy" = present ]; then + test "$(cat /etc/apt/sources.list)" = '# original legacy sources' + else + test ! -f /etc/apt/sources.list + fi + test "$(cat /etc/apt/sources.list.d/third-party.list)" = 'deb https://packages.example.invalid stable main' + sh /configure-mirror.sh "" + cmp /tmp/original.sources /etc/apt/sources.list.d/ubuntu.sources + done + ` + ], { encoding: 'utf8', timeout: 90000, input: fs.readFileSync(script) }); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stdout + result.stderr); + console.log('Passed: inherited-sources (base/derived configuration, original legacy absence/presence and third-party sources)'); +} + +if (process.argv[2] === '--stub') { + stubDocker(); +} else if (process.argv[3] === 'inherited-sources') { + testInheritedSources(process.argv[2]); +} else { + testBuild(process.argv[2], process.argv[3]); +} diff --git a/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs b/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs index 5ebab541c9b..f85c9e92d88 100644 --- a/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs +++ b/tests/Infrastructure.Tests/TestTriggerMap/TestTriggerMapTests.cs @@ -278,6 +278,14 @@ private static IReadOnlySet LoadSolutionProjectPaths() "eng/scripts/load-cli-e2e-images.sh", ["test:Aspire.Cli.EndToEnd.Tests"] }, + { + "eng/scripts/build-cli-e2e-images.sh", + ["test:Aspire.Cli.EndToEnd.Tests", "test:Infrastructure.Tests"] + }, + { + "eng/scripts/run-bounded-image-build.ts", + ["test:Aspire.Cli.EndToEnd.Tests", "test:Infrastructure.Tests"] + }, { "eng/clipack/Common.projitems", [ diff --git a/tests/Shared/Docker/configure-ubuntu-apt-mirror.sh b/tests/Shared/Docker/configure-ubuntu-apt-mirror.sh index 72f3943bc51..249c2a2bf10 100644 --- a/tests/Shared/Docker/configure-ubuntu-apt-mirror.sh +++ b/tests/Shared/Docker/configure-ubuntu-apt-mirror.sh @@ -2,8 +2,21 @@ set -eu ubuntu_apt_mirror="${1:-}" +ubuntu_sources_file="/etc/apt/sources.list.d/ubuntu.sources" +default_sources_directory="/etc/apt/aspire-default-ubuntu-sources" if [ -z "$ubuntu_apt_mirror" ]; then + # Derived Java/Go images inherit the base image's mirror configuration. + # Restore the original files, not just an empty ARG, for default-source retries. + if [ -d "$default_sources_directory" ]; then + for source in sources.list sources.list.d/ubuntu.sources; do + if [ -f "$default_sources_directory/$source" ]; then + cp "$default_sources_directory/$source" "/etc/apt/$source" + else + rm -f "/etc/apt/$source" + fi + done + fi exit 0 fi @@ -34,7 +47,17 @@ fi ubuntu_apt_mirror="${ubuntu_apt_mirror%/}" ubuntu_apt_components="${UBUNTU_APT_COMPONENTS:-main restricted universe multiverse}" -ubuntu_sources_file="/etc/apt/sources.list.d/ubuntu.sources" + +# Keep one pristine snapshot across base and derived layers, including absence +# of a legacy sources.list. Other repositories (NodeSource, GitHub) are untouched. +if [ ! -d "$default_sources_directory" ]; then + mkdir -p "$default_sources_directory/sources.list.d" + for source in sources.list sources.list.d/ubuntu.sources; do + if [ -f "/etc/apt/$source" ]; then + cp "/etc/apt/$source" "$default_sources_directory/$source" + fi + done +fi mkdir -p "$(dirname "$ubuntu_sources_file")"