diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4efd15..aa62a01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,6 +193,15 @@ jobs: PSR_TEST_WORKER_IMAGE: psr-worker:ci PSR_TEST_DASHBOARD_IMAGE: psr-dashboard:ci + # A failed test keeps the logs of its containers (hosts, PostgreSQL, RabbitMQ, compose services). + - name: Upload container logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: container-logs + path: tests/PythonScriptRunner.Containers.Tests/bin/Debug/net10.0/TestResults/container-logs/ + if-no-files-found: ignore + # Spec 009, AC-1/AC-2: the Aspire AppHost brings everything up and the author runs CS-1 and CS-2. apphost: name: Aspire AppHost diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9adac16..e2155f9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,7 +65,8 @@ A pull request is merged only when every job of `.github/workflows/ci.yml` is gr them in an incompatible way needs a new schema or protocol version. - **Changelog:** user-visible changes get a line in [CHANGELOG.md](CHANGELOG.md), under the version being prepared. - **Commits:** [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `docs:`, `test:`, - `build:`, `ci:`, `chore:`). + `build:`, `ci:`, `chore:`). Commit messages and pull requests describe what changed and why, so that they read on their + own; link the issue when there is one. ## Releases diff --git a/README.md b/README.md index 55a8c33..ccf6b9d 100644 --- a/README.md +++ b/README.md @@ -569,6 +569,10 @@ the memory limit through cgroup v2 and the compose from `aspire publish`; the co dotnet test --project tests/PythonScriptRunner.Containers.Tests # needs Docker ``` +When a container test fails, it saves the logs of its containers to `TestResults/container-logs//` next to the test +assembly, and the end of the worker and dashboard logs goes to the test output. The CI uploads that folder as the +`container-logs` artifact. + The dashboard performance test uses about 200,000 runs; `PSR_PERF_RUNS_PER_DAY=50000` runs the full average volume (90 days). The dashboard's OpenAPI document is compared with `tests/PythonScriptRunner.Dashboard.Tests/Snapshots/openapi-v1.verified.json`. After you review an intended change, diff --git a/tests/PythonScriptRunner.Containers.Tests/CgroupTests.cs b/tests/PythonScriptRunner.Containers.Tests/CgroupTests.cs index ae1b423..7d58bc8 100644 --- a/tests/PythonScriptRunner.Containers.Tests/CgroupTests.cs +++ b/tests/PythonScriptRunner.Containers.Tests/CgroupTests.cs @@ -8,7 +8,7 @@ namespace PythonScriptRunner.Containers.Tests; /// Spec 001, AC-30 (D-180): with a writable cgroup v2 (a privileged container) the memory limit of an attempt is the /// memory really used, enforced by the kernel, and the numeric libraries keep their threads. /// -public sealed class CgroupTests(HostStack stack) +public sealed class CgroupTests(HostStack stack) : ContainerTest { private static readonly Dictionary Stages = new() { diff --git a/tests/PythonScriptRunner.Containers.Tests/ComposeTests.cs b/tests/PythonScriptRunner.Containers.Tests/ComposeTests.cs index 5eec07d..cc5be0e 100644 --- a/tests/PythonScriptRunner.Containers.Tests/ComposeTests.cs +++ b/tests/PythonScriptRunner.Containers.Tests/ComposeTests.cs @@ -8,7 +8,7 @@ namespace PythonScriptRunner.Containers.Tests; /// Spec 009, AC-3: the docker-compose generated by aspire publish runs CS-1 with the host images. -public sealed class ComposeTests(HostStack stack) +public sealed class ComposeTests(HostStack stack) : ContainerTest { private const string Version = "compose-test"; @@ -57,6 +57,13 @@ await File.WriteAllTextAsync(Path.Combine(output, ".env"), $""" } finally { + // `down` removes the services: their logs are kept for the report of a failed test. + foreach (var service in new[] { "worker", "dashboard", "postgres", "rabbitmq" }) + { + var (_, log) = await RunAsync("docker", output, "compose", "-p", project, "logs", "--no-color", "--no-log-prefix", service); + ContainerLogs.Add($"compose-{service}", log, host: service is "worker" or "dashboard"); + } + await RunAsync("docker", output, "compose", "-p", project, "down", "-v"); } } diff --git a/tests/PythonScriptRunner.Containers.Tests/HostImageTests.cs b/tests/PythonScriptRunner.Containers.Tests/HostImageTests.cs index f7de94a..8c3b086 100644 --- a/tests/PythonScriptRunner.Containers.Tests/HostImageTests.cs +++ b/tests/PythonScriptRunner.Containers.Tests/HostImageTests.cs @@ -7,7 +7,7 @@ namespace PythonScriptRunner.Containers.Tests; /// Spec 009: the worker and dashboard images as they run in production. -public sealed class HostImageTests(HostStack stack) +public sealed class HostImageTests(HostStack stack) : ContainerTest { private const long MaxWorkerImageBytes = 400L * 1024 * 1024; diff --git a/tests/PythonScriptRunner.Containers.Tests/Infrastructure/ContainerLogs.cs b/tests/PythonScriptRunner.Containers.Tests/Infrastructure/ContainerLogs.cs new file mode 100644 index 0000000..c29569a --- /dev/null +++ b/tests/PythonScriptRunner.Containers.Tests/Infrastructure/ContainerLogs.cs @@ -0,0 +1,151 @@ +using System.Collections.Concurrent; +using System.Text; +using DotNet.Testcontainers.Configurations; + +namespace PythonScriptRunner.Containers.Tests.Infrastructure; + +/// +/// The output of the containers, captured while they run: a test removes its containers when it ends, and their logs +/// with them. When a test fails, saves every log to TestResults/container-logs/<test>/ +/// next to the test assembly (the CI uploads that folder) and writes the end of the host logs to the test output. +/// +public static class ContainerLogs +{ + private const int TailLines = 80; + private const string SharedKey = ""; + + private static readonly ConcurrentDictionary> Logs = new(); + + /// An output consumer that keeps what the container writes, for the current test. + /// File name of the log (made unique within the test). + /// Whether the end of the log also goes to the test output (the worker and the dashboard). + public static IOutputConsumer Capture(string name, bool host = true) => Capture(CurrentTest(), name, host); + + /// An output consumer for a container shared by every test (PostgreSQL, RabbitMQ). + public static IOutputConsumer CaptureShared(string name) => Capture(SharedKey, name, host: false); + + /// Adds a log read some other way (docker compose logs) to the current test. + public static void Add(string name, string text, bool host = true) => Register(CurrentTest(), name, () => text, host); + + /// Called when a test class is disposed: writes the logs if the test failed and forgets them. + public static async Task ReportAsync() + { + if (TestContext.Current.Test is not { } test) + { + return; + } + + Logs.TryRemove(test.UniqueID, out var own); + if (TestContext.Current.TestState?.Result != TestResult.Failed) + { + return; + } + + var name = TestContext.Current.TestMethod?.MethodName ?? test.UniqueID; + var folder = Path.Combine(AppContext.BaseDirectory, "TestResults", "container-logs", name); + Directory.CreateDirectory(folder); + var output = TestContext.Current.TestOutputHelper; + var shared = Logs.TryGetValue(SharedKey, out var common) ? common : []; + foreach (var entry in (own ?? []).Concat(shared)) + { + var text = entry.Read(); + await File.WriteAllTextAsync(Path.Combine(folder, $"{entry.Name}.log"), text); + if (entry.Host && output is not null) + { + var lines = text.Split('\n'); + output.WriteLine($"----- {entry.Name}: last {Math.Min(TailLines, lines.Length)} of {lines.Length} lines -----"); + output.WriteLine(string.Join('\n', lines.TakeLast(TailLines))); + } + } + + output?.WriteLine($"Container logs: {folder}"); + } + + private static IOutputConsumer Capture(string key, string name, bool host) + { + var stdout = new LogStream(); + var stderr = new LogStream(); + Register(key, name, () => stderr.Length == 0 ? stdout.Text() : $"{stdout.Text()}\n----- stderr -----\n{stderr.Text()}", host); + return Consume.RedirectStdoutAndStderrToStream(stdout, stderr); + } + + private static void Register(string key, string name, Func read, bool host) + { + var entries = Logs.GetOrAdd(key, _ => new ConcurrentQueue()); + lock (entries) + { + var unique = name; + for (var i = 2; entries.Any(e => e.Name == unique); i++) + { + unique = $"{name}-{i}"; + } + + entries.Enqueue(new Entry(unique, read, host)); + } + } + + private static string CurrentTest() => + TestContext.Current.Test?.UniqueID ?? throw new InvalidOperationException("Container logs are captured inside a test."); + + private sealed record Entry(string Name, Func Read, bool Host); + + /// A write-only buffer that Testcontainers fills from a background task and that is read after the container is gone. + private sealed class LogStream : Stream + { + private readonly MemoryStream _buffer = new(); + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length + { + get + { + lock (_buffer) + { + return _buffer.Length; + } + } + } + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public string Text() + { + lock (_buffer) + { + return Encoding.UTF8.GetString(_buffer.GetBuffer(), 0, (int)_buffer.Length); + } + } + + public override void Write(byte[] buffer, int offset, int count) + { + lock (_buffer) + { + _buffer.Write(buffer, offset, count); + } + } + + public override void Flush() + { + } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + // Testcontainers disposes the consumer with the container; the captured text stays readable. + protected override void Dispose(bool disposing) + { + } + } +} diff --git a/tests/PythonScriptRunner.Containers.Tests/Infrastructure/ContainerTest.cs b/tests/PythonScriptRunner.Containers.Tests/Infrastructure/ContainerTest.cs new file mode 100644 index 0000000..f193251 --- /dev/null +++ b/tests/PythonScriptRunner.Containers.Tests/Infrastructure/ContainerTest.cs @@ -0,0 +1,13 @@ +namespace PythonScriptRunner.Containers.Tests.Infrastructure; + +/// Base of the container tests: when a test fails, the logs of its containers are kept (). +public abstract class ContainerTest : IAsyncLifetime +{ + public ValueTask InitializeAsync() => ValueTask.CompletedTask; + + public async ValueTask DisposeAsync() + { + await ContainerLogs.ReportAsync(); + GC.SuppressFinalize(this); + } +} diff --git a/tests/PythonScriptRunner.Containers.Tests/Infrastructure/HostStack.cs b/tests/PythonScriptRunner.Containers.Tests/Infrastructure/HostStack.cs index 2769a6f..c18ad69 100644 --- a/tests/PythonScriptRunner.Containers.Tests/Infrastructure/HostStack.cs +++ b/tests/PythonScriptRunner.Containers.Tests/Infrastructure/HostStack.cs @@ -37,10 +37,12 @@ public async ValueTask InitializeAsync() _postgres = new PostgreSqlBuilder("postgres:16-alpine") .WithNetwork(_network).WithNetworkAliases("pg") .WithUsername("psr").WithPassword("psr").WithDatabase("psr") + .WithOutputConsumer(ContainerLogs.CaptureShared("postgres")) .Build(); _rabbit = new RabbitMqBuilder("rabbitmq:3.13-alpine") .WithNetwork(_network).WithNetworkAliases("rabbit") .WithUsername("psr").WithPassword("psr") + .WithOutputConsumer(ContainerLogs.CaptureShared("rabbitmq")) .Build(); var images = Task.WhenAll(HostImages.ResolveAsync(HostImages.Worker, Ct), HostImages.ResolveAsync(HostImages.Dashboard, Ct)); await Task.WhenAll(_postgres.StartAsync(Ct), _rabbit.StartAsync(Ct), images); @@ -66,12 +68,15 @@ public async Task ScalarAsync(string database, string sql) /// /// A host container (not started) with migrations on, waiting until /health/ready answers 200; /// runs it privileged and as root, so it owns a writable /sys/fs/cgroup: a - /// delegable cgroup v2 (D-180), like a systemd service with Delegate=yes. + /// delegable cgroup v2 (D-180), like a systemd service with Delegate=yes. Its output is kept for the report of a + /// failed test (). /// public IContainer Host(string image, string database, IReadOnlyDictionary? environment = null, bool privileged = false) { + var logName = image == WorkerImage ? "worker" : image == DashboardImage ? "dashboard" : "host"; var builder = new ContainerBuilder(image) .WithNetwork(_network) + .WithOutputConsumer(ContainerLogs.Capture(logName)) .WithPrivileged(privileged) .WithEnvironment("ConnectionStrings__psr", $"Host=pg;Database={database};Username=psr;Password=psr") .WithEnvironment("ConnectionStrings__rabbitmq", "amqp://psr:psr@rabbit:5672/")