Skip to content

Commit 804edd3

Browse files
net0wellclaude
andcommitted
feat(worker): worker com lease, retry, concorrência, cancelamento e cron (spec 006)
- StepWorker consome psr.step.execute (uma mensagem por passo), renova o lease e conclui o passo na mesma transação do próximo; mensagens duplicadas ou obsoletas são confirmadas sem efeito. - Reaper de leases, scheduler cron e limpeza de pastas de trabalho. - Graceful shutdown com ShutdownGracePeriod; métricas psr.worker.*, psr.queue.wait, psr.lease.recovered, psr.schedule.* e psr.runs.pending; AddPythonScriptRunnerChecks (banco, broker, uv). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent c55239a commit 804edd3

7 files changed

Lines changed: 627 additions & 0 deletions

File tree

‎PythonScriptRunner.slnx‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,13 @@
33
<Project Path="src/PythonScriptRunner.Abstractions/PythonScriptRunner.Abstractions.csproj" />
44
<Project Path="src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj" />
55
<Project Path="src/PythonScriptRunner.EntityFrameworkCore/PythonScriptRunner.EntityFrameworkCore.csproj" />
6+
<Project Path="src/PythonScriptRunner.RabbitMQ/PythonScriptRunner.RabbitMQ.csproj" />
7+
<Project Path="src/PythonScriptRunner.Worker/PythonScriptRunner.Worker.csproj" />
68
</Folder>
79
<Folder Name="/tests/">
810
<Project Path="tests/PythonScriptRunner.Core.Tests/PythonScriptRunner.Core.Tests.csproj" />
911
<Project Path="tests/PythonScriptRunner.EntityFrameworkCore.Tests/PythonScriptRunner.EntityFrameworkCore.Tests.csproj" />
12+
<Project Path="tests/PythonScriptRunner.RabbitMQ.Tests/PythonScriptRunner.RabbitMQ.Tests.csproj" />
13+
<Project Path="tests/PythonScriptRunner.Worker.Tests/PythonScriptRunner.Worker.Tests.csproj" />
1014
</Folder>
1115
</Solution>
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<RootNamespace>PythonScriptRunner.Worker</RootNamespace>
5+
<Description>Worker for PythonScriptRunner: executes queued steps with leases and retries, cron schedules, cancellation, graceful shutdown and health checks.</Description>
6+
</PropertyGroup>
7+
8+
<ItemGroup>
9+
<ProjectReference Include="..\PythonScriptRunner.RabbitMQ\PythonScriptRunner.RabbitMQ.csproj" />
10+
</ItemGroup>
11+
12+
<ItemGroup>
13+
<InternalsVisibleTo Include="PythonScriptRunner.Worker.Tests" />
14+
<InternalsVisibleTo Include="PythonScriptRunner.RabbitMQ.Tests" />
15+
</ItemGroup>
16+
17+
<ItemGroup>
18+
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" />
19+
<PackageReference Include="Microsoft.Extensions.Hosting" />
20+
</ItemGroup>
21+
22+
</Project>
Lines changed: 236 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,236 @@
1+
using System.Data.Common;
2+
using System.Diagnostics;
3+
using System.Globalization;
4+
using System.Text.Json;
5+
using Microsoft.Extensions.Hosting;
6+
using Microsoft.Extensions.Logging;
7+
using PythonScriptRunner.EntityFrameworkCore;
8+
using PythonScriptRunner.RabbitMQ;
9+
10+
namespace PythonScriptRunner.Worker;
11+
12+
/// <summary>
13+
/// Executes queued steps, one <c>ExecuteStep</c> message per step (D-34; spec 006): claims the step with a lease, runs the
14+
/// Python process while renewing the lease, records the outcome and the next message in one transaction, then acknowledges.
15+
/// Duplicate or stale messages are acknowledged without effect. On shutdown it stops receiving and lets steps in progress
16+
/// finish within <see cref="WorkerOptions.ShutdownGracePeriod"/>; past it, processes are stopped and their attempts are left
17+
/// to the reaper.
18+
/// </summary>
19+
internal sealed class StepWorker(
20+
PsrBroker broker,
21+
AsyncRunLifecycle lifecycle,
22+
IRunEngine engine,
23+
WorkerOptions options,
24+
WorkerMetrics metrics,
25+
TimeProvider time,
26+
ILogger<StepWorker> logger) : IHostedService, IDisposable
27+
{
28+
private readonly CancellationTokenSource _stopping = new();
29+
private QueueConsumer? _consumer;
30+
private Task? _running;
31+
32+
/// <summary>Identifier recorded on the attempts.</summary>
33+
public string WorkerId { get; } = options.WorkerId ?? broker.InstanceId;
34+
35+
/// <summary>The consumer (for tests and health).</summary>
36+
public QueueConsumer? Consumer => _consumer;
37+
38+
public Task StartAsync(CancellationToken cancellationToken)
39+
{
40+
_consumer = new QueueConsumer(broker, new QueueConsumerSettings
41+
{
42+
Queue = PsrTopology.ExecuteStepQueue,
43+
Prefetch = (ushort)options.MaxConcurrency,
44+
}, HandleAsync, logger);
45+
_running = Task.Run(() => _consumer.RunAsync(_stopping.Token), CancellationToken.None);
46+
Log.Started(logger, WorkerId, PsrTopology.ExecuteStepQueue, options.MaxConcurrency);
47+
return Task.CompletedTask;
48+
}
49+
50+
public async Task StopAsync(CancellationToken cancellationToken)
51+
{
52+
// A host can be stopped more than once (e.g. stop, then dispose).
53+
if (Interlocked.Exchange(ref _consumer, null) is not { } consumer || _running is not { } running)
54+
{
55+
return;
56+
}
57+
58+
await consumer.StopAsync(options.ShutdownGracePeriod, cancellationToken).ConfigureAwait(false);
59+
await _stopping.CancelAsync().ConfigureAwait(false);
60+
try
61+
{
62+
await running.WaitAsync(cancellationToken).ConfigureAwait(false);
63+
}
64+
catch (OperationCanceledException)
65+
{
66+
// Host shutdown timeout.
67+
}
68+
69+
await consumer.DisposeAsync().ConfigureAwait(false);
70+
}
71+
72+
public void Dispose() => _stopping.Dispose();
73+
74+
/// <param name="delivery">The message.</param>
75+
/// <param name="abandon">Cancelled when the shutdown grace period is over.</param>
76+
internal async Task<DeliveryOutcome> HandleAsync(PsrDelivery delivery, CancellationToken abandon)
77+
{
78+
ExecuteStepCommand command;
79+
try
80+
{
81+
if (delivery.Type is { Length: > 0 } type && type != PsrMessageTypes.ExecuteStep)
82+
{
83+
throw new JsonException($"unknown message type '{type}'");
84+
}
85+
86+
command = JsonSerializer.Deserialize<ExecuteStepCommand>(delivery.Body.Span, PsrMessageJson.Options) ?? throw new JsonException("The body is null.");
87+
}
88+
catch (JsonException ex)
89+
{
90+
Log.InvalidCommand(logger, delivery.MessageId, ex.Message);
91+
return DeliveryOutcome.DeadLetter;
92+
}
93+
94+
using var activity = StartActivity(delivery, command);
95+
var claimed = await RetryAsync("claim step", () => lifecycle.ClaimStepAsync(command, WorkerId, options.LeaseDuration, abandon), abandon)
96+
.ConfigureAwait(false);
97+
if (claimed is null)
98+
{
99+
// Duplicate, stale (already executed or recovered elsewhere) or cancelled: nothing to do.
100+
return DeliveryOutcome.Ack;
101+
}
102+
103+
RecordQueueWait(delivery);
104+
activity?.SetTag("psr.step.key", claimed.StepKey);
105+
activity?.SetTag("psr.stage.key", claimed.Stage.Key);
106+
metrics.StepStarted();
107+
try
108+
{
109+
return await ExecuteAsync(claimed, abandon).ConfigureAwait(false);
110+
}
111+
finally
112+
{
113+
metrics.StepEnded();
114+
}
115+
}
116+
117+
private async Task<DeliveryOutcome> ExecuteAsync(ClaimedStep claimed, CancellationToken abandon)
118+
{
119+
using var execution = CancellationTokenSource.CreateLinkedTokenSource(abandon);
120+
using var renewing = new CancellationTokenSource();
121+
var leaseLost = false;
122+
var renewal = Task.Run(async () =>
123+
{
124+
using var timer = new PeriodicTimer(options.LeaseRenewInterval, time);
125+
try
126+
{
127+
while (await timer.WaitForNextTickAsync(renewing.Token).ConfigureAwait(false))
128+
{
129+
try
130+
{
131+
if (!await lifecycle.RenewLeaseAsync(claimed, options.LeaseDuration, renewing.Token).ConfigureAwait(false))
132+
{
133+
// The reaper took the attempt: stop the process and record nothing.
134+
leaseLost = true;
135+
await execution.CancelAsync().ConfigureAwait(false);
136+
return;
137+
}
138+
}
139+
catch (Exception ex) when (IsTransient(ex))
140+
{
141+
Log.DatabaseRetry(logger, "renew lease", options.LeaseRenewInterval, ex);
142+
}
143+
}
144+
}
145+
catch (OperationCanceledException)
146+
{
147+
// Step finished.
148+
}
149+
}, CancellationToken.None);
150+
151+
StepExecution result;
152+
try
153+
{
154+
result = await engine.ExecuteStepAsync(claimed.ToExecutionRequest(), execution.Token).ConfigureAwait(false);
155+
}
156+
finally
157+
{
158+
await renewing.CancelAsync().ConfigureAwait(false);
159+
await renewal.ConfigureAwait(false);
160+
}
161+
162+
await using (result.ConfigureAwait(false))
163+
{
164+
if (leaseLost)
165+
{
166+
Log.LeaseLost(logger, claimed.StepKey, claimed.RunId);
167+
return DeliveryOutcome.Ack;
168+
}
169+
170+
if (abandon.IsCancellationRequested)
171+
{
172+
Log.Abandoned(logger, claimed.StepKey, claimed.RunId);
173+
return DeliveryOutcome.Abandon;
174+
}
175+
176+
// Retried while the lease lasts (database blips); afterwards the reaper decides (D-85).
177+
var completion = await RetryAsync("complete step", () => lifecycle.CompleteStepAsync(claimed, result, CancellationToken.None),
178+
CancellationToken.None, options.LeaseDuration).ConfigureAwait(false);
179+
if (completion == StepCompletion.LeaseLost)
180+
{
181+
Log.LeaseLost(logger, claimed.StepKey, claimed.RunId);
182+
}
183+
else
184+
{
185+
Log.StepCompleted(logger, claimed.StepKey, claimed.Attempt, claimed.RunId, completion.ToString());
186+
}
187+
188+
return DeliveryOutcome.Ack;
189+
}
190+
}
191+
192+
/// <summary>Retries transient database failures with backoff (up to <paramref name="limit"/>; default 30 seconds).</summary>
193+
private async Task<T> RetryAsync<T>(string operation, Func<Task<T>> action, CancellationToken cancellationToken, TimeSpan? limit = null)
194+
{
195+
var started = Stopwatch.GetTimestamp();
196+
var delay = TimeSpan.FromMilliseconds(200);
197+
while (true)
198+
{
199+
try
200+
{
201+
return await action().ConfigureAwait(false);
202+
}
203+
catch (Exception ex) when (IsTransient(ex) && Stopwatch.GetElapsedTime(started) + delay < (limit ?? TimeSpan.FromSeconds(30)))
204+
{
205+
Log.DatabaseRetry(logger, operation, delay, ex);
206+
await Task.Delay(delay, cancellationToken).ConfigureAwait(false);
207+
delay = TimeSpan.FromMilliseconds(Math.Min(5000, delay.TotalMilliseconds * 2));
208+
}
209+
}
210+
}
211+
212+
private static bool IsTransient(Exception exception) =>
213+
exception is DbException or TimeoutException || exception.InnerException is DbException or TimeoutException;
214+
215+
private static Activity? StartActivity(PsrDelivery delivery, ExecuteStepCommand command)
216+
{
217+
var activity = ActivityContext.TryParse(delivery.TraceParent, null, out var parent)
218+
? WorkerDiagnostics.Source.StartActivity("psr.message.consume", ActivityKind.Consumer, parent)
219+
: WorkerDiagnostics.Source.StartActivity("psr.message.consume", ActivityKind.Consumer);
220+
activity?.SetTag("messaging.system", "rabbitmq");
221+
activity?.SetTag("messaging.destination.name", delivery.Queue);
222+
activity?.SetTag("psr.run.id", command.RunId.ToString());
223+
activity?.SetTag("psr.attempt", command.Attempt);
224+
return activity;
225+
}
226+
227+
/// <summary><c>psr.queue.wait</c>: from the moment the message became due (header set by the relay) to the claim.</summary>
228+
private void RecordQueueWait(PsrDelivery delivery)
229+
{
230+
if (delivery.Headers.TryGetValue("psr-available-at", out var text)
231+
&& long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var milliseconds))
232+
{
233+
metrics.QueueWait(time.GetUtcNow() - DateTimeOffset.FromUnixTimeMilliseconds(milliseconds));
234+
}
235+
}
236+
}
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
using System.Diagnostics;
2+
using System.Diagnostics.Metrics;
3+
using Microsoft.Extensions.Logging;
4+
5+
namespace PythonScriptRunner.Worker;
6+
7+
/// <summary>Tracing of the worker (source <c>PythonScriptRunner</c>).</summary>
8+
internal static class WorkerDiagnostics
9+
{
10+
public static readonly ActivitySource Source = new("PythonScriptRunner");
11+
}
12+
13+
/// <summary>Metrics of the worker (meter <c>PythonScriptRunner</c>; spec 006).</summary>
14+
internal sealed class WorkerMetrics
15+
{
16+
private readonly UpDownCounter<int> _activeSteps;
17+
private readonly Histogram<double> _queueWait;
18+
private readonly Counter<long> _leaseRecovered;
19+
private readonly Counter<long> _scheduleFired;
20+
private readonly Counter<long> _scheduleRejected;
21+
private int _pendingRuns;
22+
23+
public WorkerMetrics(IMeterFactory meterFactory)
24+
{
25+
var meter = meterFactory.Create("PythonScriptRunner");
26+
_activeSteps = meter.CreateUpDownCounter<int>("psr.worker.active_steps", description: "Steps executing in this worker.");
27+
_queueWait = meter.CreateHistogram<double>("psr.queue.wait", "ms", "Time from queuing a step to its start in a worker.");
28+
_leaseRecovered = meter.CreateCounter<long>("psr.lease.recovered", description: "Attempts restarted after their worker was lost.");
29+
_scheduleFired = meter.CreateCounter<long>("psr.schedule.fired", description: "Runs created by schedules.");
30+
_scheduleRejected = meter.CreateCounter<long>("psr.schedule.rejected", description: "Schedule firings rejected.");
31+
meter.CreateObservableGauge("psr.runs.pending", () => Volatile.Read(ref _pendingRuns), description: "Runs waiting for a free slot of their pipeline.");
32+
}
33+
34+
public void StepStarted() => _activeSteps.Add(1);
35+
36+
public void StepEnded() => _activeSteps.Add(-1);
37+
38+
public void QueueWait(TimeSpan wait) => _queueWait.Record(Math.Max(0, wait.TotalMilliseconds));
39+
40+
public void LeasesRecovered(int count) => _leaseRecovered.Add(count);
41+
42+
public void Schedules(int fired, int rejected)
43+
{
44+
_scheduleFired.Add(fired);
45+
_scheduleRejected.Add(rejected);
46+
}
47+
48+
public void PendingRuns(int count) => Volatile.Write(ref _pendingRuns, count);
49+
}
50+
51+
/// <summary>Source-generated log messages.</summary>
52+
internal static partial class Log
53+
{
54+
[LoggerMessage(200, LogLevel.Information, "Worker {WorkerId} consuming {Queue} with {MaxConcurrency} concurrent steps")]
55+
public static partial void Started(ILogger logger, string workerId, string queue, int maxConcurrency);
56+
57+
[LoggerMessage(201, LogLevel.Information, "Step {StepKey} (attempt {Attempt}) of run {RunId}: {Completion}")]
58+
public static partial void StepCompleted(ILogger logger, string stepKey, int attempt, RunId runId, string completion);
59+
60+
[LoggerMessage(202, LogLevel.Warning, "The lease of step {StepKey} of run {RunId} was taken by another worker; the process was stopped")]
61+
public static partial void LeaseLost(ILogger logger, string stepKey, RunId runId);
62+
63+
[LoggerMessage(203, LogLevel.Warning, "Shutdown grace period over: step {StepKey} of run {RunId} was stopped and will be recovered by another worker")]
64+
public static partial void Abandoned(ILogger logger, string stepKey, RunId runId);
65+
66+
[LoggerMessage(204, LogLevel.Warning, "Database operation failed ({Operation}); retrying in {Delay}")]
67+
public static partial void DatabaseRetry(ILogger logger, string operation, TimeSpan delay, Exception exception);
68+
69+
[LoggerMessage(205, LogLevel.Information, "Lease reaper: {Recovered} recovered, {Failed} failed (worker lost), {Cancelled} cancelled")]
70+
public static partial void Reaped(ILogger logger, int recovered, int failed, int cancelled);
71+
72+
[LoggerMessage(206, LogLevel.Information, "Scheduler: {Fired} runs created, {Rejected} firings rejected")]
73+
public static partial void SchedulesFired(ILogger logger, int fired, int rejected);
74+
75+
[LoggerMessage(207, LogLevel.Error, "{Loop} failed")]
76+
public static partial void LoopFailed(ILogger logger, string loop, Exception exception);
77+
78+
[LoggerMessage(208, LogLevel.Error, "ExecuteStep message {MessageId} is invalid and goes to the dead-letter queue: {Reason}")]
79+
public static partial void InvalidCommand(ILogger logger, string? messageId, string reason);
80+
}

0 commit comments

Comments
 (0)