Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 1 addition & 97 deletions .github/workflows/build-cli-e2e-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,13 @@ on:
env:
CLI_E2E_DOTNET_IMAGE_ARTIFACT: cli-e2e-dotnet-image
CLI_E2E_DOTNET_IMAGE_TAG: aspire-cli-e2e-dotnet:prebuilt
CLI_E2E_DOTNET_IMAGE_CACHE_SCOPE: cli-e2e-dotnet
CLI_E2E_POLYGLOT_IMAGE_ARTIFACT: cli-e2e-polyglot-image
CLI_E2E_POLYGLOT_IMAGE_TAG: aspire-cli-e2e-polyglot:prebuilt
CLI_E2E_POLYGLOT_JAVA_IMAGE_ARTIFACT: cli-e2e-polyglot-java-image
CLI_E2E_POLYGLOT_JAVA_IMAGE_TAG: aspire-cli-e2e-polyglot-java:prebuilt
CLI_E2E_POLYGLOT_GO_IMAGE_ARTIFACT: cli-e2e-polyglot-go-image
CLI_E2E_POLYGLOT_GO_IMAGE_TAG: aspire-cli-e2e-polyglot-go:prebuilt
CLI_E2E_POLYGLOT_BASE_IMAGE_TAG: aspire-e2e-polyglot-base:latest
CLI_E2E_POLYGLOT_BASE_IMAGE_CACHE_SCOPE: cli-e2e-polyglot-base
CLI_E2E_INCLUDE_POLYGLOT_IMAGES: ${{ inputs.includePolyglotImages }}

jobs:
Expand Down Expand Up @@ -57,101 +55,7 @@ jobs:

- name: Build CLI E2E Docker image
shell: bash
run: |
set -euo pipefail

docker buildx create --name cli-e2e-builder --use
docker buildx inspect --bootstrap

build_image() {
local display_name="$1"
local dockerfile="$2"
local tag="$3"
local cache_scope="$4"
local mirror="$5"
local -a build_args=(--build-arg SKIP_SOURCE_BUILD=true)

if [[ -n "$mirror" ]]; then
echo "Building $display_name with Ubuntu apt mirror: $mirror"
build_args+=(--build-arg "UBUNTU_APT_MIRROR=$mirror")
else
echo "Building $display_name with the default Ubuntu apt sources"
fi

docker buildx build \
--load \
--cache-from "type=gha,scope=$cache_scope" \
--cache-to "type=gha,scope=$cache_scope,mode=max,ignore-error=true" \
"${build_args[@]}" \
-f "$dockerfile" \
-t "$tag" \
.
}

build_java_image() {
local mirror="$1"
local -a build_args=()

if [[ -n "$mirror" ]]; then
echo "Building Java polyglot image with Ubuntu apt mirror: $mirror"
build_args+=(--build-arg "UBUNTU_APT_MIRROR=$mirror")
else
echo "Building Java polyglot image with the default Ubuntu apt sources"
fi

DOCKER_BUILDKIT=1 docker build \
"${build_args[@]}" \
-f tests/Shared/Docker/Dockerfile.e2e-polyglot-java \
-t "$CLI_E2E_POLYGLOT_JAVA_IMAGE_TAG" \
.
}

build_go_image() {
local mirror="$1"
local -a build_args=()

if [[ -n "$mirror" ]]; then
echo "Building Go polyglot image with Ubuntu apt mirror: $mirror"
build_args+=(--build-arg "UBUNTU_APT_MIRROR=$mirror")
else
echo "Building Go polyglot image with the default Ubuntu apt sources"
fi

DOCKER_BUILDKIT=1 docker build \
"${build_args[@]}" \
-f tests/Shared/Docker/Dockerfile.e2e-polyglot-go \
-t "$CLI_E2E_POLYGLOT_GO_IMAGE_TAG" \
.
}

build_with_mirror_retry() {
local display_name="$1"
local dockerfile="$2"
local tag="$3"
local cache_scope="$4"

if ! build_image "$display_name" "$dockerfile" "$tag" "$cache_scope" "http://azure.archive.ubuntu.com/ubuntu/"; then
echo "$display_name build failed with the Azure Ubuntu apt mirror; retrying with default Ubuntu apt sources"
build_image "$display_name" "$dockerfile" "$tag" "$cache_scope" ""
fi
}

build_with_mirror_retry ".NET image" "tests/Shared/Docker/Dockerfile.e2e" "$CLI_E2E_DOTNET_IMAGE_TAG" "$CLI_E2E_DOTNET_IMAGE_CACHE_SCOPE"

if [[ "$CLI_E2E_INCLUDE_POLYGLOT_IMAGES" == "true" ]]; then
build_with_mirror_retry "polyglot base image" "tests/Shared/Docker/Dockerfile.e2e-polyglot-base" "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG" "$CLI_E2E_POLYGLOT_BASE_IMAGE_CACHE_SCOPE"
docker tag "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG" "$CLI_E2E_POLYGLOT_IMAGE_TAG"

if ! build_java_image "http://azure.archive.ubuntu.com/ubuntu/"; then
echo "Java polyglot image build failed with the Azure Ubuntu apt mirror; retrying with default Ubuntu apt sources"
build_java_image ""
fi

if ! build_go_image "http://azure.archive.ubuntu.com/ubuntu/"; then
echo "Go polyglot image build failed with the Azure Ubuntu apt mirror; retrying with default Ubuntu apt sources"
build_go_image ""
fi
fi
run: bash eng/scripts/build-cli-e2e-images.sh

- name: Save CLI E2E Docker image tarballs
if: ${{ inputs.uploadImageArtifacts }}
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/warm-cli-e2e-image-cache.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ on:
- '.dockerignore'
- 'eng/scripts/get-aspire-cli.sh'
- 'eng/scripts/get-aspire-cli-pr.sh'
- 'eng/scripts/build-cli-e2e-images.sh'
- 'eng/scripts/run-bounded-image-build.ts'
- 'tests/Shared/Docker/Dockerfile.e2e'
- 'tests/Shared/Docker/Dockerfile.e2e-polyglot-base'
- 'tests/Shared/Docker/Dockerfile.e2e-polyglot-go'
Expand Down
27 changes: 27 additions & 0 deletions docs/ci/cli-e2e-images.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,33 @@ Consumer workflows download image artifacts into `${{ github.workspace }}/cli-e2

The reusable image build workflow also has an `uploadImageArtifacts` input. It defaults to `true` for test workflows because each isolated test job needs the image tarballs. Cache-warming workflows set it to `false` so they only build the images and export BuildKit cache layers.

### Bounded apt mirror fallback

Both test builds and the cache warmer run `eng/scripts/build-cli-e2e-images.sh`.
Each image (.NET, polyglot base, Java and Go) first uses the Azure Ubuntu apt
mirror, with a five-minute limit. A failed or timed-out primary attempt retries
once with the Dockerfile's default Ubuntu apt sources. The fallback's failure
exit code and both attempts' output are preserved; there is no third attempt.

The job still has a 45-minute cap. Four primary attempts can consume at most
20 minutes plus two minutes of cancellation grace, leaving 23 minutes for
fallback builds, setup, saving and uploading images. This prevents a slow but
successful apt transfer from consuming the entire job before fallback runs.
Observed throttled transfers took 38-41 minutes for 82-108 MB.

The Node.js build supervisor signals the Docker client's process group with `TERM` so
BuildKit can cancel the build session before fallback starts. If the client
cannot stop within another 30 seconds, the supervisor kills the process group and the
script fails without fallback: a forced kill cannot establish that the remote
build acknowledged cancellation. Interrupts, termination and supervisor execution
errors also fail without retrying.

The mirror configuration script preserves the base image's original Ubuntu
sources before replacing them. An empty mirror restores those files, including
in derived Java/Go images, without changing third-party repositories. A
supervisor tracks external cancellation independently of its deadline, including
signals during cleanup and clients that exit with an ordinary failure code.

## Cross-PR BuildKit cache

The image build workflow exports BuildKit layers to the GitHub Actions cache with stable scopes:
Expand Down
5 changes: 5 additions & 0 deletions eng/github-ci/test-trigger-map.yml
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,11 @@ path_rules:
- eng/scripts/load-cli-e2e-images.sh
targets: [test:Aspire.Cli.EndToEnd.Tests]
reason: builds or loads the CLI E2E images consumed only by Aspire.Cli.EndToEnd.Tests
- paths:
- eng/scripts/build-cli-e2e-images.sh
- eng/scripts/run-bounded-image-build.ts
targets: [test:Aspire.Cli.EndToEnd.Tests, test:Infrastructure.Tests]
reason: shared image build entry point and its shell behavior tests
- paths:
- eng/scripts/pack-cli-npm-package.ps1
targets: [test:Aspire.Cli.Tests, test:Infrastructure.Tests]
Expand Down
76 changes: 76 additions & 0 deletions eng/scripts/build-cli-e2e-images.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
#!/usr/bin/env bash
set -euo pipefail

build_with_mirror_retry() {
local primary_timeout="$1"
local display_name="$2"
shift 2
local status

echo "Building $display_name with Azure Ubuntu apt mirror (limit: $primary_timeout)"
if node "$(dirname "${BASH_SOURCE[0]}")/run-bounded-image-build.ts" "$primary_timeout" \
"$@" --build-arg "UBUNTU_APT_MIRROR=http://azure.archive.ubuntu.com/ubuntu/" .; then
return 0
else
status=$?
fi

case "$status" in
124)
echo "$display_name primary mirror build exceeded $primary_timeout; retrying with default Ubuntu apt sources" >&2
;;
125|126|127|130|137|143)
# A forced kill cannot confirm BuildKit acknowledged cancellation. Do
# not overlap another build or turn workflow cancellation into a retry.
echo "$display_name primary build stopped with exit code $status; not retrying after cancellation or timeout execution failure" >&2
return "$status"
;;
*)
echo "$display_name build failed with the Azure Ubuntu apt mirror (exit code $status); retrying with default Ubuntu apt sources" >&2
Comment thread
IEvangelist marked this conversation as resolved.
Resolved
;;
esac

"$@" .
Comment thread
IEvangelist marked this conversation as resolved.
Resolved
}

build_cli_e2e_images() {
# Four primary attempts can spend at most 20m plus 2m cancellation grace,
# leaving 23m of the 45m job for default sources, setup and artifact handling.
# Throttled apt downloads have otherwise consumed 38-41m in a single image.
local primary_timeout=5m
local dotnet_cache_scope=cli-e2e-dotnet
local polyglot_cache_scope=cli-e2e-polyglot-base

docker buildx create --name cli-e2e-builder --use
docker buildx inspect --bootstrap

build_with_mirror_retry "$primary_timeout" ".NET image" docker buildx build \
--load \
--cache-from "type=gha,scope=$dotnet_cache_scope" \
--cache-to "type=gha,scope=$dotnet_cache_scope,mode=max,ignore-error=true" \
--build-arg SKIP_SOURCE_BUILD=true \
-f tests/Shared/Docker/Dockerfile.e2e \
-t "$CLI_E2E_DOTNET_IMAGE_TAG"

if [[ "$CLI_E2E_INCLUDE_POLYGLOT_IMAGES" == "true" ]]; then
build_with_mirror_retry "$primary_timeout" "polyglot base image" docker buildx build \
--load \
--cache-from "type=gha,scope=$polyglot_cache_scope" \
--cache-to "type=gha,scope=$polyglot_cache_scope,mode=max,ignore-error=true" \
--build-arg SKIP_SOURCE_BUILD=true \
-f tests/Shared/Docker/Dockerfile.e2e-polyglot-base \
-t "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG"
docker tag "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG" "$CLI_E2E_POLYGLOT_IMAGE_TAG"

DOCKER_BUILDKIT=1 build_with_mirror_retry "$primary_timeout" "Java polyglot image" docker build \
-f tests/Shared/Docker/Dockerfile.e2e-polyglot-java \
-t "$CLI_E2E_POLYGLOT_JAVA_IMAGE_TAG"
DOCKER_BUILDKIT=1 build_with_mirror_retry "$primary_timeout" "Go polyglot image" docker build \
-f tests/Shared/Docker/Dockerfile.e2e-polyglot-go \
-t "$CLI_E2E_POLYGLOT_GO_IMAGE_TAG"
fi
}

if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
build_cli_e2e_images
fi
80 changes: 80 additions & 0 deletions eng/scripts/run-bounded-image-build.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
#!/usr/bin/env node
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

import { spawn } from 'node:child_process';
import { constants } from 'node:os';

const [duration, executable, ...args] = process.argv.slice(2);
const match = /^([1-9]\d*)(s|m)$/.exec(duration ?? '');
if (!match || !executable || process.platform !== 'linux') {
console.error('Usage (Linux): node run-bounded-image-build.ts <positive duration in s or m> <command> [arguments...]');
process.exit(125);
}

const milliseconds = Number(match[1]) * (match[2] === 'm' ? 60000 : 1000);
if (!Number.isSafeInteger(milliseconds) || milliseconds > 2147483647) {
console.error(`Invalid image build duration: ${duration}`);
process.exit(125);
}

// A separate session lets us signal the client and all of its descendants,
// without signaling the workflow shell or other builds on the runner.
const child = spawn(executable, args, { detached: true, stdio: 'inherit' });
let deadlineReached = false;
let cancellationExitCode = 0;
let forcedKill = false;
let deadline: NodeJS.Timeout | undefined;
let grace: NodeJS.Timeout | undefined;

function signalGroup(signal: NodeJS.Signals): void {
if (child.pid === undefined) {
return;
}
try {
process.kill(-child.pid, signal);
} catch (error) {
// The client may finish between the signal event and the group kill.
if (!(error instanceof Error && 'code' in error && error.code === 'ESRCH')) {
throw error;
}
}
}

function stopClient(): void {
signalGroup('SIGTERM');
grace ??= setTimeout(() => {
forcedKill = true;
console.error('Image build did not acknowledge cancellation within 30s; killing its process group');
signalGroup('SIGKILL');
}, 30000);
}

function cancel(exitCode: number): void {
// Keep cancellation independent of the deadline, including signals arriving
// during the deadline's cleanup grace. GNU timeout returns 124 in that race.
cancellationExitCode = exitCode;
console.error('Image build externally cancelled; default-source retry is disabled');
stopClient();
}

process.on('SIGTERM', () => cancel(143));
process.on('SIGINT', () => cancel(130));
child.once('spawn', () => {
deadline = setTimeout(() => {
deadlineReached = true;
console.error(`Image build exceeded ${duration}; cancelling its process group`);
stopClient();
}, milliseconds);
});
child.once('error', error => {
console.error(`Could not start image build: ${error.message}`);
process.exit('code' in error && error.code === 'ENOENT' ? 127 : 125);
});
child.once('exit', (code, signal) => {
clearTimeout(deadline);
clearTimeout(grace);
const clientExitCode = code ?? (signal ? 128 + constants.signals[signal] : 1);
// An external kill/OOM during cleanup is not acknowledged cancellation.
process.exit(forcedKill || clientExitCode === 137 ? 137 : cancellationExitCode || (deadlineReached ? 124 : clientExitCode));

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[automated] [P2] Fail closed when the Buildx plugin is killed during cleanup

The supervised child is Docker, not Buildx itself. With Docker 28.0.4 (the version in this PR's CI), a SIGKILLed plugin makes Docker exit 255, not 137. Once the deadline has fired, this expression converts that to 124, so build_with_mirror_retry starts fallback and can return success without confirmed graceful cancellation.

I reproduced this through the production helper and supervisor using real Docker plus a fixture Buildx plugin that kills itself on deadline TERM: PLUGIN_KILLED_DURING_CLEANUP -> FALLBACK_RAN -> helper exit 0. This demonstrates the exit-classification bug, not actual surviving BuildKit work.

Handle the Docker/plugin termination path before converting cleanup exits to timeout, and cover that process boundary; the current direct-client SIGKILL regression only exercises exit 137.

});
47 changes: 47 additions & 0 deletions tests/Infrastructure.Tests/Pipelines/CliE2EImageBuildTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using Aspire.TestUtilities;
using Xunit;

namespace Infrastructure.Tests;

public sealed class CliE2EImageBuildTests(ITestOutputHelper output)
{
[Theory]
[InlineData("success")]
[InlineData("failure")]
[InlineData("fallback-failure")]
[InlineData("slow-success")]
[InlineData("stall")]
[InlineData("forced-kill")]
[InlineData("cancelled")]
[InlineData("external-cancel")]
[InlineData("cleanup-cancel")]
[InlineData("cleanup-kill")]
[InlineData("all-images")]
[InlineData("dotnet-only")]
[RequiresTools(["node", "bash"])]
public async Task ImageBuildPreservesFallbackAndCancellation(string scenario)
{
Assert.SkipWhen(!OperatingSystem.IsLinux(), "Image build process-group behavior requires Linux.");
using var command = new NodeCommand(output).WithTimeout(TimeSpan.FromMinutes(2));
var result = await command.ExecuteScriptAsync(
Path.Combine(RepoRoot.Path, "tests", "Infrastructure.Tests", "Pipelines", "cli-e2e-image-build.harness.ts"),
RepoRoot.Path, scenario);

result.EnsureSuccessful();
}

[Fact]
[RequiresTools(["node", "docker"])]
public async Task DerivedImageFallbackRestoresOriginalAptSources()
{
using var command = new NodeCommand(output).WithTimeout(TimeSpan.FromMinutes(2));
var result = await command.ExecuteScriptAsync(
Path.Combine(RepoRoot.Path, "tests", "Infrastructure.Tests", "Pipelines", "cli-e2e-image-build.harness.ts"),
RepoRoot.Path, "inherited-sources");

result.EnsureSuccessful();
}
}
Loading
Loading