Repository navigation
Bound primary apt mirror builds before CLI E2E fallback #21849
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
5d26690
4bb054c
296e3e9
b8274fc
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| #!/usr/bin/env bash | ||
| set -euo pipefail | ||
|
|
||
| build_with_mirror_retry() { | ||
| local primary_timeout="$1" | ||
| local display_name="$2" | ||
| shift 2 | ||
| local status | ||
|
|
||
| echo "Building $display_name with Azure Ubuntu apt mirror (limit: $primary_timeout)" | ||
| if node "$(dirname "${BASH_SOURCE[0]}")/run-bounded-image-build.ts" "$primary_timeout" \ | ||
| "$@" --build-arg "UBUNTU_APT_MIRROR=http://azure.archive.ubuntu.com/ubuntu/" .; then | ||
| return 0 | ||
| else | ||
| status=$? | ||
| fi | ||
|
|
||
| case "$status" in | ||
| 124) | ||
| echo "$display_name primary mirror build exceeded $primary_timeout; retrying with default Ubuntu apt sources" >&2 | ||
| ;; | ||
| 125|126|127|130|137|143) | ||
| # A forced kill cannot confirm BuildKit acknowledged cancellation. Do | ||
| # not overlap another build or turn workflow cancellation into a retry. | ||
| echo "$display_name primary build stopped with exit code $status; not retrying after cancellation or timeout execution failure" >&2 | ||
| return "$status" | ||
| ;; | ||
| *) | ||
| echo "$display_name build failed with the Azure Ubuntu apt mirror (exit code $status); retrying with default Ubuntu apt sources" >&2 | ||
| ;; | ||
| esac | ||
|
|
||
| "$@" . | ||
|
IEvangelist marked this conversation as resolved.
Resolved
|
||
| } | ||
|
|
||
| build_cli_e2e_images() { | ||
| # Four primary attempts can spend at most 20m plus 2m cancellation grace, | ||
| # leaving 23m of the 45m job for default sources, setup and artifact handling. | ||
| # Throttled apt downloads have otherwise consumed 38-41m in a single image. | ||
| local primary_timeout=5m | ||
| local dotnet_cache_scope=cli-e2e-dotnet | ||
| local polyglot_cache_scope=cli-e2e-polyglot-base | ||
|
|
||
| docker buildx create --name cli-e2e-builder --use | ||
| docker buildx inspect --bootstrap | ||
|
|
||
| build_with_mirror_retry "$primary_timeout" ".NET image" docker buildx build \ | ||
| --load \ | ||
| --cache-from "type=gha,scope=$dotnet_cache_scope" \ | ||
| --cache-to "type=gha,scope=$dotnet_cache_scope,mode=max,ignore-error=true" \ | ||
| --build-arg SKIP_SOURCE_BUILD=true \ | ||
| -f tests/Shared/Docker/Dockerfile.e2e \ | ||
| -t "$CLI_E2E_DOTNET_IMAGE_TAG" | ||
|
|
||
| if [[ "$CLI_E2E_INCLUDE_POLYGLOT_IMAGES" == "true" ]]; then | ||
| build_with_mirror_retry "$primary_timeout" "polyglot base image" docker buildx build \ | ||
| --load \ | ||
| --cache-from "type=gha,scope=$polyglot_cache_scope" \ | ||
| --cache-to "type=gha,scope=$polyglot_cache_scope,mode=max,ignore-error=true" \ | ||
| --build-arg SKIP_SOURCE_BUILD=true \ | ||
| -f tests/Shared/Docker/Dockerfile.e2e-polyglot-base \ | ||
| -t "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG" | ||
| docker tag "$CLI_E2E_POLYGLOT_BASE_IMAGE_TAG" "$CLI_E2E_POLYGLOT_IMAGE_TAG" | ||
|
|
||
| DOCKER_BUILDKIT=1 build_with_mirror_retry "$primary_timeout" "Java polyglot image" docker build \ | ||
| -f tests/Shared/Docker/Dockerfile.e2e-polyglot-java \ | ||
| -t "$CLI_E2E_POLYGLOT_JAVA_IMAGE_TAG" | ||
| DOCKER_BUILDKIT=1 build_with_mirror_retry "$primary_timeout" "Go polyglot image" docker build \ | ||
| -f tests/Shared/Docker/Dockerfile.e2e-polyglot-go \ | ||
| -t "$CLI_E2E_POLYGLOT_GO_IMAGE_TAG" | ||
| fi | ||
| } | ||
|
|
||
| if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then | ||
| build_cli_e2e_images | ||
| fi | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| #!/usr/bin/env node | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| import { spawn } from 'node:child_process'; | ||
| import { constants } from 'node:os'; | ||
|
|
||
| const [duration, executable, ...args] = process.argv.slice(2); | ||
| const match = /^([1-9]\d*)(s|m)$/.exec(duration ?? ''); | ||
| if (!match || !executable || process.platform !== 'linux') { | ||
| console.error('Usage (Linux): node run-bounded-image-build.ts <positive duration in s or m> <command> [arguments...]'); | ||
| process.exit(125); | ||
| } | ||
|
|
||
| const milliseconds = Number(match[1]) * (match[2] === 'm' ? 60000 : 1000); | ||
| if (!Number.isSafeInteger(milliseconds) || milliseconds > 2147483647) { | ||
| console.error(`Invalid image build duration: ${duration}`); | ||
| process.exit(125); | ||
| } | ||
|
|
||
| // A separate session lets us signal the client and all of its descendants, | ||
| // without signaling the workflow shell or other builds on the runner. | ||
| const child = spawn(executable, args, { detached: true, stdio: 'inherit' }); | ||
| let deadlineReached = false; | ||
| let cancellationExitCode = 0; | ||
| let forcedKill = false; | ||
| let deadline: NodeJS.Timeout | undefined; | ||
| let grace: NodeJS.Timeout | undefined; | ||
|
|
||
| function signalGroup(signal: NodeJS.Signals): void { | ||
| if (child.pid === undefined) { | ||
| return; | ||
| } | ||
| try { | ||
| process.kill(-child.pid, signal); | ||
| } catch (error) { | ||
| // The client may finish between the signal event and the group kill. | ||
| if (!(error instanceof Error && 'code' in error && error.code === 'ESRCH')) { | ||
| throw error; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| function stopClient(): void { | ||
| signalGroup('SIGTERM'); | ||
| grace ??= setTimeout(() => { | ||
| forcedKill = true; | ||
| console.error('Image build did not acknowledge cancellation within 30s; killing its process group'); | ||
| signalGroup('SIGKILL'); | ||
| }, 30000); | ||
| } | ||
|
|
||
| function cancel(exitCode: number): void { | ||
| // Keep cancellation independent of the deadline, including signals arriving | ||
| // during the deadline's cleanup grace. GNU timeout returns 124 in that race. | ||
| cancellationExitCode = exitCode; | ||
| console.error('Image build externally cancelled; default-source retry is disabled'); | ||
| stopClient(); | ||
| } | ||
|
|
||
| process.on('SIGTERM', () => cancel(143)); | ||
| process.on('SIGINT', () => cancel(130)); | ||
| child.once('spawn', () => { | ||
| deadline = setTimeout(() => { | ||
| deadlineReached = true; | ||
| console.error(`Image build exceeded ${duration}; cancelling its process group`); | ||
| stopClient(); | ||
| }, milliseconds); | ||
| }); | ||
| child.once('error', error => { | ||
| console.error(`Could not start image build: ${error.message}`); | ||
| process.exit('code' in error && error.code === 'ENOENT' ? 127 : 125); | ||
| }); | ||
| child.once('exit', (code, signal) => { | ||
| clearTimeout(deadline); | ||
| clearTimeout(grace); | ||
| const clientExitCode = code ?? (signal ? 128 + constants.signals[signal] : 1); | ||
| // An external kill/OOM during cleanup is not acknowledged cancellation. | ||
| process.exit(forcedKill || clientExitCode === 137 ? 137 : cancellationExitCode || (deadlineReached ? 124 : clientExitCode)); | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [automated] [P2] Fail closed when the Buildx plugin is killed during cleanup The supervised child is Docker, not Buildx itself. With Docker 28.0.4 (the version in this PR's CI), a SIGKILLed plugin makes Docker exit 255, not 137. Once the deadline has fired, this expression converts that to 124, so I reproduced this through the production helper and supervisor using real Docker plus a fixture Buildx plugin that kills itself on deadline TERM: Handle the Docker/plugin termination path before converting cleanup exits to timeout, and cover that process boundary; the current direct-client SIGKILL regression only exercises exit 137. |
||
| }); | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| // Licensed to the .NET Foundation under one or more agreements. | ||
| // The .NET Foundation licenses this file to you under the MIT license. | ||
|
|
||
| using Aspire.TestUtilities; | ||
| using Xunit; | ||
|
|
||
| namespace Infrastructure.Tests; | ||
|
|
||
| public sealed class CliE2EImageBuildTests(ITestOutputHelper output) | ||
| { | ||
| [Theory] | ||
| [InlineData("success")] | ||
| [InlineData("failure")] | ||
| [InlineData("fallback-failure")] | ||
| [InlineData("slow-success")] | ||
| [InlineData("stall")] | ||
| [InlineData("forced-kill")] | ||
| [InlineData("cancelled")] | ||
| [InlineData("external-cancel")] | ||
| [InlineData("cleanup-cancel")] | ||
| [InlineData("cleanup-kill")] | ||
| [InlineData("all-images")] | ||
| [InlineData("dotnet-only")] | ||
| [RequiresTools(["node", "bash"])] | ||
| public async Task ImageBuildPreservesFallbackAndCancellation(string scenario) | ||
| { | ||
| Assert.SkipWhen(!OperatingSystem.IsLinux(), "Image build process-group behavior requires Linux."); | ||
| using var command = new NodeCommand(output).WithTimeout(TimeSpan.FromMinutes(2)); | ||
| var result = await command.ExecuteScriptAsync( | ||
| Path.Combine(RepoRoot.Path, "tests", "Infrastructure.Tests", "Pipelines", "cli-e2e-image-build.harness.ts"), | ||
| RepoRoot.Path, scenario); | ||
|
|
||
| result.EnsureSuccessful(); | ||
| } | ||
|
|
||
| [Fact] | ||
| [RequiresTools(["node", "docker"])] | ||
| public async Task DerivedImageFallbackRestoresOriginalAptSources() | ||
| { | ||
| using var command = new NodeCommand(output).WithTimeout(TimeSpan.FromMinutes(2)); | ||
| var result = await command.ExecuteScriptAsync( | ||
| Path.Combine(RepoRoot.Path, "tests", "Infrastructure.Tests", "Pipelines", "cli-e2e-image-build.harness.ts"), | ||
| RepoRoot.Path, "inherited-sources"); | ||
|
|
||
| result.EnsureSuccessful(); | ||
| } | ||
| } |
Uh oh!
There was an error while loading. Please reload this page.