Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,15 @@ jobs:
PSR_TEST_WORKER_IMAGE: psr-worker:ci
PSR_TEST_DASHBOARD_IMAGE: psr-dashboard:ci

# A failed test keeps the logs of its containers (hosts, PostgreSQL, RabbitMQ, compose services).
- name: Upload container logs
if: failure()
uses: actions/upload-artifact@v7
with:
name: container-logs
path: tests/PythonScriptRunner.Containers.Tests/bin/Debug/net10.0/TestResults/container-logs/
if-no-files-found: ignore

# Spec 009, AC-1/AC-2: the Aspire AppHost brings everything up and the author runs CS-1 and CS-2.
apphost:
name: Aspire AppHost
Expand Down
3 changes: 2 additions & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,8 @@ A pull request is merged only when every job of `.github/workflows/ci.yml` is gr
them in an incompatible way needs a new schema or protocol version.
- **Changelog:** user-visible changes get a line in [CHANGELOG.md](CHANGELOG.md), under the version being prepared.
- **Commits:** [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `docs:`, `test:`,
`build:`, `ci:`, `chore:`).
`build:`, `ci:`, `chore:`). Commit messages and pull requests describe what changed and why, so that they read on their
own; link the issue when there is one.

## Releases

Expand Down
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -569,6 +569,10 @@ the memory limit through cgroup v2 and the compose from `aspire publish`; the co
dotnet test --project tests/PythonScriptRunner.Containers.Tests # needs Docker
```

When a container test fails, it saves the logs of its containers to `TestResults/container-logs/<test>/` next to the test
assembly, and the end of the worker and dashboard logs goes to the test output. The CI uploads that folder as the
`container-logs` artifact.

The dashboard performance test uses about 200,000 runs; `PSR_PERF_RUNS_PER_DAY=50000` runs the full average volume (90
days). The dashboard's OpenAPI document is compared with
`tests/PythonScriptRunner.Dashboard.Tests/Snapshots/openapi-v1.verified.json`. After you review an intended change,
Expand Down
2 changes: 1 addition & 1 deletion tests/PythonScriptRunner.Containers.Tests/CgroupTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ namespace PythonScriptRunner.Containers.Tests;
/// Spec 001, AC-30 (D-180): with a writable cgroup v2 (a privileged container) the memory limit of an attempt is the
/// memory really used, enforced by the kernel, and the numeric libraries keep their threads.
/// </summary>
public sealed class CgroupTests(HostStack stack)
public sealed class CgroupTests(HostStack stack) : ContainerTest
{
private static readonly Dictionary<string, string> Stages = new()
{
Expand Down
9 changes: 8 additions & 1 deletion tests/PythonScriptRunner.Containers.Tests/ComposeTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
namespace PythonScriptRunner.Containers.Tests;

/// <summary>Spec 009, AC-3: the docker-compose generated by <c>aspire publish</c> runs CS-1 with the host images.</summary>
public sealed class ComposeTests(HostStack stack)
public sealed class ComposeTests(HostStack stack) : ContainerTest
{
private const string Version = "compose-test";

Expand Down Expand Up @@ -57,6 +57,13 @@ await File.WriteAllTextAsync(Path.Combine(output, ".env"), $"""
}
finally
{
// `down` removes the services: their logs are kept for the report of a failed test.
foreach (var service in new[] { "worker", "dashboard", "postgres", "rabbitmq" })
{
var (_, log) = await RunAsync("docker", output, "compose", "-p", project, "logs", "--no-color", "--no-log-prefix", service);
ContainerLogs.Add($"compose-{service}", log, host: service is "worker" or "dashboard");
}

await RunAsync("docker", output, "compose", "-p", project, "down", "-v");
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
namespace PythonScriptRunner.Containers.Tests;

/// <summary>Spec 009: the worker and dashboard images as they run in production.</summary>
public sealed class HostImageTests(HostStack stack)
public sealed class HostImageTests(HostStack stack) : ContainerTest
{
private const long MaxWorkerImageBytes = 400L * 1024 * 1024;

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
using System.Collections.Concurrent;
using System.Text;
using DotNet.Testcontainers.Configurations;

namespace PythonScriptRunner.Containers.Tests.Infrastructure;

/// <summary>
/// The output of the containers, captured while they run: a test removes its containers when it ends, and their logs
/// with them. When a test fails, <see cref="ReportAsync"/> saves every log to <c>TestResults/container-logs/&lt;test&gt;/</c>
/// next to the test assembly (the CI uploads that folder) and writes the end of the host logs to the test output.
/// </summary>
public static class ContainerLogs
{
private const int TailLines = 80;
private const string SharedKey = "";

private static readonly ConcurrentDictionary<string, ConcurrentQueue<Entry>> Logs = new();

/// <summary>An output consumer that keeps what the container writes, for the current test.</summary>
/// <param name="name">File name of the log (made unique within the test).</param>
/// <param name="host">Whether the end of the log also goes to the test output (the worker and the dashboard).</param>
public static IOutputConsumer Capture(string name, bool host = true) => Capture(CurrentTest(), name, host);

/// <summary>An output consumer for a container shared by every test (PostgreSQL, RabbitMQ).</summary>
public static IOutputConsumer CaptureShared(string name) => Capture(SharedKey, name, host: false);

/// <summary>Adds a log read some other way (<c>docker compose logs</c>) to the current test.</summary>
public static void Add(string name, string text, bool host = true) => Register(CurrentTest(), name, () => text, host);

/// <summary>Called when a test class is disposed: writes the logs if the test failed and forgets them.</summary>
public static async Task ReportAsync()
{
if (TestContext.Current.Test is not { } test)
{
return;
}

Logs.TryRemove(test.UniqueID, out var own);
if (TestContext.Current.TestState?.Result != TestResult.Failed)
{
return;
}

var name = TestContext.Current.TestMethod?.MethodName ?? test.UniqueID;
var folder = Path.Combine(AppContext.BaseDirectory, "TestResults", "container-logs", name);
Directory.CreateDirectory(folder);
var output = TestContext.Current.TestOutputHelper;
var shared = Logs.TryGetValue(SharedKey, out var common) ? common : [];
foreach (var entry in (own ?? []).Concat(shared))
{
var text = entry.Read();
await File.WriteAllTextAsync(Path.Combine(folder, $"{entry.Name}.log"), text);
if (entry.Host && output is not null)
{
var lines = text.Split('\n');
output.WriteLine($"----- {entry.Name}: last {Math.Min(TailLines, lines.Length)} of {lines.Length} lines -----");
output.WriteLine(string.Join('\n', lines.TakeLast(TailLines)));
}
}

output?.WriteLine($"Container logs: {folder}");
}

private static IOutputConsumer Capture(string key, string name, bool host)
{
var stdout = new LogStream();
var stderr = new LogStream();
Register(key, name, () => stderr.Length == 0 ? stdout.Text() : $"{stdout.Text()}\n----- stderr -----\n{stderr.Text()}", host);
return Consume.RedirectStdoutAndStderrToStream(stdout, stderr);
}

private static void Register(string key, string name, Func<string> read, bool host)
{
var entries = Logs.GetOrAdd(key, _ => new ConcurrentQueue<Entry>());
lock (entries)
{
var unique = name;
for (var i = 2; entries.Any(e => e.Name == unique); i++)
{
unique = $"{name}-{i}";
}

entries.Enqueue(new Entry(unique, read, host));
}
}

private static string CurrentTest() =>
TestContext.Current.Test?.UniqueID ?? throw new InvalidOperationException("Container logs are captured inside a test.");

private sealed record Entry(string Name, Func<string> Read, bool Host);

/// <summary>A write-only buffer that Testcontainers fills from a background task and that is read after the container is gone.</summary>
private sealed class LogStream : Stream
{
private readonly MemoryStream _buffer = new();

public override bool CanRead => false;

public override bool CanSeek => false;

public override bool CanWrite => true;

public override long Length
{
get
{
lock (_buffer)
{
return _buffer.Length;
}
}
}

public override long Position
{
get => throw new NotSupportedException();
set => throw new NotSupportedException();
}

public string Text()
{
lock (_buffer)
{
return Encoding.UTF8.GetString(_buffer.GetBuffer(), 0, (int)_buffer.Length);
}
}

public override void Write(byte[] buffer, int offset, int count)
{
lock (_buffer)
{
_buffer.Write(buffer, offset, count);
}
}

public override void Flush()
{
}

public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();

public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();

public override void SetLength(long value) => throw new NotSupportedException();

// Testcontainers disposes the consumer with the container; the captured text stays readable.
protected override void Dispose(bool disposing)
{
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
namespace PythonScriptRunner.Containers.Tests.Infrastructure;

/// <summary>Base of the container tests: when a test fails, the logs of its containers are kept (<see cref="ContainerLogs"/>).</summary>
public abstract class ContainerTest : IAsyncLifetime
{
public ValueTask InitializeAsync() => ValueTask.CompletedTask;

public async ValueTask DisposeAsync()
{
await ContainerLogs.ReportAsync();
GC.SuppressFinalize(this);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,12 @@ public async ValueTask InitializeAsync()
_postgres = new PostgreSqlBuilder("postgres:16-alpine")
.WithNetwork(_network).WithNetworkAliases("pg")
.WithUsername("psr").WithPassword("psr").WithDatabase("psr")
.WithOutputConsumer(ContainerLogs.CaptureShared("postgres"))
.Build();
_rabbit = new RabbitMqBuilder("rabbitmq:3.13-alpine")
.WithNetwork(_network).WithNetworkAliases("rabbit")
.WithUsername("psr").WithPassword("psr")
.WithOutputConsumer(ContainerLogs.CaptureShared("rabbitmq"))
.Build();
var images = Task.WhenAll(HostImages.ResolveAsync(HostImages.Worker, Ct), HostImages.ResolveAsync(HostImages.Dashboard, Ct));
await Task.WhenAll(_postgres.StartAsync(Ct), _rabbit.StartAsync(Ct), images);
Expand All @@ -66,12 +68,15 @@ public async Task<string> ScalarAsync(string database, string sql)
/// <summary>
/// A host container (not started) with migrations on, waiting until <c>/health/ready</c> answers 200;
/// <paramref name="privileged"/> runs it privileged and as root, so it owns a writable <c>/sys/fs/cgroup</c>: a
/// delegable cgroup v2 (D-180), like a systemd service with <c>Delegate=yes</c>.
/// delegable cgroup v2 (D-180), like a systemd service with <c>Delegate=yes</c>. Its output is kept for the report of a
/// failed test (<see cref="ContainerLogs"/>).
/// </summary>
public IContainer Host(string image, string database, IReadOnlyDictionary<string, string>? environment = null, bool privileged = false)
{
var logName = image == WorkerImage ? "worker" : image == DashboardImage ? "dashboard" : "host";
var builder = new ContainerBuilder(image)
.WithNetwork(_network)
.WithOutputConsumer(ContainerLogs.Capture(logName))
.WithPrivileged(privileged)
.WithEnvironment("ConnectionStrings__psr", $"Host=pg;Database={database};Username=psr;Password=psr")
.WithEnvironment("ConnectionStrings__rabbitmq", "amqp://psr:psr@rabbit:5672/")
Expand Down
Loading