Repository navigation
Expand file tree
/
Copy pathRecoveryTests.cs
More file actions
193 lines (163 loc) · 10.4 KB
/
Copy pathRecoveryTests.cs
File metadata and controls
193 lines (163 loc) · 10.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
using System.Diagnostics;
namespace PythonScriptRunner.Worker.Tests;
/// <summary>Spec 006: lost workers (leases), cancellation, graceful shutdown and queued reprocessing.</summary>
public sealed class RecoveryTests(PostgresFixture postgres, BrokerFixture broker, PythonFixture python)
{
private static readonly TimeSpan ShortLease = TimeSpan.FromSeconds(4);
private static CancellationToken Ct => TestContext.Current.CancellationToken;
[Fact]
[Trait("Category", "Timing")]
public async Task Spec006_AC05_LostWorker_IdempotentStepIsRecoveredByAnother()
{
await using var env = await PsrEnvironment.CreateAsync(postgres, broker, python);
var (doomed, survivor, runId) = await StartAndLoseAWorkerAsync(env, idempotent: true);
var lostAt = Stopwatch.GetTimestamp();
var run = await survivor.WaitForRunAsync(runId, timeout: TimeSpan.FromSeconds(60));
run.Status.ShouldBe(RunStatus.Succeeded);
var attempts = run.Steps.Single().Attempts;
attempts.Select(a => (a.Status, a.Cause, a.WorkerId)).ShouldBe(
[
(AttemptStatus.WorkerLost, AttemptCause.Initial, doomed.InstanceId),
(AttemptStatus.Succeeded, AttemptCause.LeaseRecovery, survivor.InstanceId),
]);
// Lease + reaper interval (the spec bound is LeaseDuration + 15 s with the default 60 s lease).
Stopwatch.GetElapsedTime(lostAt).ShouldBeLessThan(ShortLease + TimeSpan.FromSeconds(15));
}
[Fact]
public async Task Spec006_AC06_LostWorker_NonIdempotentStepFailsAndReprocessCompletesIt()
{
await using var env = await PsrEnvironment.CreateAsync(postgres, broker, python);
var (_, survivor, runId) = await StartAndLoseAWorkerAsync(env, idempotent: false);
var failed = await survivor.WaitForRunAsync(runId, timeout: TimeSpan.FromSeconds(60));
failed.Status.ShouldBe(RunStatus.Failed);
failed.Steps.Single().Error!.Kind.ShouldBe(ErrorKind.WorkerLost);
failed.Steps.Single().Attempts.Single().Status.ShouldBe(AttemptStatus.WorkerLost);
await survivor.Runs.ReprocessAsync(runId, PsrEnvironment.Author, ExecutionMode.Async, Ct);
var run = await survivor.WaitForRunAsync(runId, r => r.Status != RunStatus.Running);
run.Status.ShouldBe(RunStatus.Succeeded);
run.Steps.Single().Attempts.Select(a => a.Cause).ShouldBe([AttemptCause.Initial, AttemptCause.Reprocess]);
}
[Fact]
[Trait("Category", "Timing")]
public async Task Spec006_AC09_Cancel_StopsTheProcessInTheWorker()
{
await using var env = await PsrEnvironment.CreateAsync(postgres, broker, python);
var worker = await env.StartHostAsync();
var app = await env.StartHostAsync(s => s.Worker = false);
await PsrEnvironment.PublishStageAsync(app, "dorme", Scripts.Sleep, [Scripts.Number("segundos"), Scripts.Text("texto")], [Scripts.Text("texto")]);
var handle = await app.Queue.EnqueueStageAsync("dorme", new { segundos = 120, texto = "longo" }, cancellationToken: Ct);
await app.WaitForRunAsync(handle.RunId, r => r.Steps[0].Attempts.Count == 1);
await Task.Delay(500, Ct); // the Python process is running
var started = Stopwatch.GetTimestamp();
await app.Runs.CancelAsync(handle.RunId, PsrEnvironment.Author, Ct);
var run = await app.WaitForRunAsync(handle.RunId, timeout: TimeSpan.FromSeconds(10));
Stopwatch.GetElapsedTime(started).ShouldBeLessThan(TimeSpan.FromSeconds(3));
run.Status.ShouldBe(RunStatus.Cancelled);
run.Steps.Single().Status.ShouldBe(StepStatus.Cancelled);
run.Steps.Single().Attempts.Single().Status.ShouldBe(AttemptStatus.Cancelled);
await worker.Events.WaitForAsync(RunEventTypes.RunCancelled, e => e.RunId == handle.RunId);
}
[Fact]
public async Task Cancel_QueuedRun_EndsAtOnceAndFreesThePipelineSlot()
{
await using var env = await PsrEnvironment.CreateAsync(postgres, broker, python);
var app = await env.StartHostAsync(s => s.Worker = false);
await PsrEnvironment.PublishStageAsync(app, "eco", Scripts.Echo, [Scripts.Text("texto")], [Scripts.Text("texto")]);
await PsrEnvironment.PublishPipelineAsync(app, "unica", Scripts.SingleStep("eco", 1, [Scripts.Text("texto")]));
var pipeline = (await app.Admin.GetPipelineAsync("unica", Ct))!;
await app.Admin.UpdatePipelineSettingsAsync("unica", new PipelineSettings(1), pipeline.RowVersion, PsrEnvironment.Author, Ct);
var first = await app.Queue.EnqueuePipelineAsync("unica", new { texto = "a" }, cancellationToken: Ct);
var second = await app.Queue.EnqueuePipelineAsync("unica", new { texto = "b" }, cancellationToken: Ct);
second.Status.ShouldBe(RunStatus.Pending);
await app.Runs.CancelAsync(first.RunId, PsrEnvironment.Author, Ct);
(await app.Runs.GetAsync(first.RunId, Ct))!.Status.ShouldBe(RunStatus.Cancelled);
(await app.Runs.GetAsync(second.RunId, Ct))!.Status.ShouldBe(RunStatus.Queued);
await env.StartHostAsync();
(await app.WaitForRunAsync(second.RunId)).Status.ShouldBe(RunStatus.Succeeded);
await Should.ThrowAsync<PsrDomainException>(() => app.Runs.CancelAsync(first.RunId, PsrEnvironment.Author, Ct));
}
[Fact]
public async Task Spec006_AC15_Shutdown_FinishesTheStepInProgressAndStopsConsuming()
{
await using var env = await PsrEnvironment.CreateAsync(postgres, broker, python);
var worker = await env.StartHostAsync(s => s.ShutdownGracePeriod = TimeSpan.FromSeconds(60));
var app = await env.StartHostAsync(s => s.Worker = false);
await PsrEnvironment.PublishStageAsync(app, "dorme", Scripts.Sleep, [Scripts.Number("segundos"), Scripts.Text("texto")], [Scripts.Text("texto")]);
var running = await app.Queue.EnqueueStageAsync("dorme", new { segundos = 5, texto = "em andamento" }, cancellationToken: Ct);
await app.WaitForRunAsync(running.RunId, r => r.Steps[0].Attempts.Count == 1);
var stopping = worker.StopAsync(Ct);
await Task.Delay(500, Ct);
var later = await app.Queue.EnqueueStageAsync("dorme", new { segundos = 0, texto = "depois" }, cancellationToken: Ct);
await stopping;
var finished = (await app.Runs.GetAsync(running.RunId, Ct))!;
finished.Status.ShouldBe(RunStatus.Succeeded);
finished.Steps.Single().Attempts.Single().Status.ShouldBe(AttemptStatus.Succeeded);
(await env.ScalarAsync<long>("SELECT count(*) FROM psr.step_attempts WHERE status = 'Running'")).ShouldBe(0);
await Task.Delay(1000, Ct);
(await app.Runs.GetAsync(later.RunId, Ct))!.Status.ShouldBe(RunStatus.Queued);
}
[Fact]
public async Task Spec003_AC10_ReprocessAsync_ResumesFromTheFailedStep()
{
await using var env = await PsrEnvironment.CreateAsync(postgres, broker, python);
var host = await env.StartHostAsync();
await PsrEnvironment.PublishStageAsync(host, "conta", Scripts.Count, [Scripts.Text("marcador")], [Scripts.Integer("execucoes")]);
await PsrEnvironment.PublishStageAsync(host, "falha-uma-vez", FailOnce, [Scripts.Text("marcador")], [Scripts.Text("texto")]);
await PsrEnvironment.PublishPipelineAsync(host, "retoma", new PipelineDraftData
{
Inputs = [Scripts.Text("contador"), Scripts.Text("marcador")],
Steps =
[
new PipelineStepDefinition("conta", "conta", 1) { Bindings = [new InputBinding("marcador", new PipelineInputSource("contador"))] },
new PipelineStepDefinition("falha", "falha-uma-vez", 1) { Bindings = [new InputBinding("marcador", new PipelineInputSource("marcador"))] },
],
Outputs = [new PipelineOutputDefinition("texto", new StepOutputSource("falha", "texto"))],
});
var folder = Path.Combine(env.WorkRoot, "markers");
Directory.CreateDirectory(folder);
var inputs = new { contador = Path.Combine(folder, "conta.txt"), marcador = Path.Combine(folder, "falha.txt") };
var handle = await host.Queue.EnqueuePipelineAsync("retoma", inputs, cancellationToken: Ct);
(await host.WaitForRunAsync(handle.RunId)).Status.ShouldBe(RunStatus.Failed);
var queued = await host.Runs.ReprocessAsync(handle.RunId, PsrEnvironment.Author, ExecutionMode.Async, Ct);
queued.Status.ShouldBe(RunStatus.Running);
var run = await host.WaitForRunAsync(handle.RunId, r => r.Status != RunStatus.Running);
run.Status.ShouldBe(RunStatus.Succeeded);
run.Id.ShouldBe(handle.RunId);
run.Steps[0].Attempts.Count.ShouldBe(1);
run.Steps[0].Outputs!["execucoes"]!.GetValue<int>().ShouldBe(1);
run.Steps[1].Attempts.Select(a => (a.Cause, a.Status)).ShouldBe([(AttemptCause.Initial, AttemptStatus.Failed), (AttemptCause.Reprocess, AttemptStatus.Succeeded)]);
run.Steps[1].Attempts[1].TriggeredBy.ShouldBe(PsrEnvironment.Author.Subject);
}
private const string FailOnce = """
import os
from psrunner import StageError, stage
@stage
def run(inputs, ctx):
if not os.path.exists(inputs["marcador"]):
open(inputs["marcador"], "w").close()
raise StageError("primeira tentativa falha", code="PRIMEIRA")
return {"texto": "ok"}
""";
/// <summary>A worker takes the first attempt of a long step and is "killed" (no completion, no ack; D-109); a second worker keeps running.</summary>
private static async Task<(PsrHost Doomed, PsrHost Survivor, RunId RunId)> StartAndLoseAWorkerAsync(PsrEnvironment env, bool idempotent)
{
void Leases(HostSettings s)
{
s.LeaseDuration = ShortLease;
s.LeaseRenewInterval = TimeSpan.FromSeconds(1);
s.ReaperInterval = TimeSpan.FromSeconds(1);
}
var doomed = await env.StartHostAsync(s =>
{
Leases(s);
s.ShutdownGracePeriod = TimeSpan.Zero;
});
await PsrEnvironment.PublishStageAsync(doomed, "lenta", Scripts.SlowFirstAttempt, [], [Scripts.Integer("tentativa")], idempotent: idempotent);
var handle = await doomed.Queue.EnqueueStageAsync("lenta", null, cancellationToken: Ct);
await doomed.WaitForRunAsync(handle.RunId, r => r.Steps[0].Attempts.Count == 1);
await Task.Delay(500, Ct);
var survivor = await env.StartHostAsync(Leases);
await doomed.StopAsync(Ct);
return (doomed, survivor, handle.RunId);
}
}