diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10df6f2..f4efd15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,16 @@ jobs: # Testcontainers (Docker); GitHub's Windows runners cannot run Linux containers, so Windows runs the Core tests only. - name: Test if: runner.os == 'Linux' - run: dotnet test --no-build --configuration Release --solution tests/PythonScriptRunner.SemE2E.slnf + run: dotnet test --no-build --configuration Release --solution tests/PythonScriptRunner.SemE2E.slnf --filter-not-trait "Category=Timing" + + # The tests that assert a duration (timeout, cancellation, latency, p95) run alone and one at a time: next to the + # other test projects, the runner's shared CPU made them miss their limits by a few milliseconds (D-188). + - name: Test (timing) + if: runner.os == 'Linux' + run: | + for project in Core Dashboard Worker; do + dotnet test --no-build --configuration Release --project "tests/PythonScriptRunner.$project.Tests" --filter-trait "Category=Timing" --parallel none + done # The panel E2E tests run alone: containers starting or stopping in the other test projects change the host's network # interfaces, and Chromium then aborts requests in flight with ERR_NETWORK_CHANGED. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4d4843c..9adac16 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,8 @@ A pull request is merged only when every job of `.github/workflows/ci.yml` is gr ## Conventions - **Tests:** every acceptance criterion and every fixed bug has a test. Integration tests run against real Python (uv), - PostgreSQL and RabbitMQ (Testcontainers). + PostgreSQL and RabbitMQ (Testcontainers). A test that asserts a duration gets `[Trait("Category", "Timing")]`: the CI + runs those alone, one at a time (`--filter-trait "Category=Timing" --parallel none`). - **Public API:** public types and members have XML documentation (the build generates it and fails on warnings). - **Packages:** versions live in `Directory.Packages.props` (Central Package Management). Dependabot proposes updates every Monday. diff --git a/tests/PythonScriptRunner.Core.Tests/Integration/IsolationTests.cs b/tests/PythonScriptRunner.Core.Tests/Integration/IsolationTests.cs index 0152342..0c9330e 100644 --- a/tests/PythonScriptRunner.Core.Tests/Integration/IsolationTests.cs +++ b/tests/PythonScriptRunner.Core.Tests/Integration/IsolationTests.cs @@ -8,6 +8,7 @@ namespace PythonScriptRunner.Core.Tests.Integration; public sealed class IsolationTests(PythonFixture fixture) { [Fact] + [Trait("Category", "Timing")] public async Task Spec001_AC07_Timeout_KillsTheProcessTree() { // Arrange @@ -44,6 +45,7 @@ with open(inputs["pid_file"], "w") as f: } [Fact] + [Trait("Category", "Timing")] public async Task Spec001_AC08_Cancellation_StopsTheRun() { await using var host = RunnerHost.Create(fixture, b => b.AddStage("espera", s => s.Script(""" diff --git a/tests/PythonScriptRunner.Dashboard.Tests/PerformanceTests.cs b/tests/PythonScriptRunner.Dashboard.Tests/PerformanceTests.cs index 360e654..81b22f3 100644 --- a/tests/PythonScriptRunner.Dashboard.Tests/PerformanceTests.cs +++ b/tests/PythonScriptRunner.Dashboard.Tests/PerformanceTests.cs @@ -16,6 +16,7 @@ public sealed class PerformanceTests(PostgresFixture postgres, BrokerFixture bro private static CancellationToken Ct => TestContext.Current.CancellationToken; [Fact] + [Trait("Category", "Timing")] public async Task Spec007_AC13_MainReadsWith90DaysOfRuns_AnswerWithP95Under500Ms() { var perDay = int.TryParse(Environment.GetEnvironmentVariable("PSR_PERF_RUNS_PER_DAY"), CultureInfo.InvariantCulture, out var configured) diff --git a/tests/PythonScriptRunner.Dashboard.Tests/RealtimeTests.cs b/tests/PythonScriptRunner.Dashboard.Tests/RealtimeTests.cs index 1d588a5..67319f3 100644 --- a/tests/PythonScriptRunner.Dashboard.Tests/RealtimeTests.cs +++ b/tests/PythonScriptRunner.Dashboard.Tests/RealtimeTests.cs @@ -14,6 +14,7 @@ public sealed class RealtimeTests(PostgresFixture postgres, BrokerFixture broker private static CancellationToken Ct => TestContext.Current.CancellationToken; [Fact] + [Trait("Category", "Timing")] public async Task Spec007_AC11_SubscribedClientsOfTwoInstances_ReceiveStepAndRunUpdatesInUnderOneSecond() { await using var env = await PsrEnvironment.CreateAsync(postgres, broker, python); diff --git a/tests/PythonScriptRunner.Worker.Tests/RecoveryTests.cs b/tests/PythonScriptRunner.Worker.Tests/RecoveryTests.cs index a702501..d187d98 100644 --- a/tests/PythonScriptRunner.Worker.Tests/RecoveryTests.cs +++ b/tests/PythonScriptRunner.Worker.Tests/RecoveryTests.cs @@ -10,6 +10,7 @@ public sealed class RecoveryTests(PostgresFixture postgres, BrokerFixture broker private static CancellationToken Ct => TestContext.Current.CancellationToken; [Fact] + [Trait("Category", "Timing")] public async Task Spec006_AC05_LostWorker_IdempotentStepIsRecoveredByAnother() { await using var env = await PsrEnvironment.CreateAsync(postgres, broker, python); @@ -49,6 +50,7 @@ public async Task Spec006_AC06_LostWorker_NonIdempotentStepFailsAndReprocessComp } [Fact] + [Trait("Category", "Timing")] public async Task Spec006_AC09_Cancel_StopsTheProcessInTheWorker() { await using var env = await PsrEnvironment.CreateAsync(postgres, broker, python);