|
| 1 | +using System.Data.Common; |
| 2 | +using System.Diagnostics; |
| 3 | +using System.Globalization; |
| 4 | +using System.Text.Json; |
| 5 | +using Microsoft.Extensions.Hosting; |
| 6 | +using Microsoft.Extensions.Logging; |
| 7 | +using PythonScriptRunner.EntityFrameworkCore; |
| 8 | +using PythonScriptRunner.RabbitMQ; |
| 9 | + |
| 10 | +namespace PythonScriptRunner.Worker; |
| 11 | + |
| 12 | +/// <summary> |
| 13 | +/// Executes queued steps, one <c>ExecuteStep</c> message per step (D-34; spec 006): claims the step with a lease, runs the |
| 14 | +/// Python process while renewing the lease, records the outcome and the next message in one transaction, then acknowledges. |
| 15 | +/// Duplicate or stale messages are acknowledged without effect. On shutdown it stops receiving and lets steps in progress |
| 16 | +/// finish within <see cref="WorkerOptions.ShutdownGracePeriod"/>; past it, processes are stopped and their attempts are left |
| 17 | +/// to the reaper. |
| 18 | +/// </summary> |
| 19 | +internal sealed class StepWorker( |
| 20 | + PsrBroker broker, |
| 21 | + AsyncRunLifecycle lifecycle, |
| 22 | + IRunEngine engine, |
| 23 | + WorkerOptions options, |
| 24 | + WorkerMetrics metrics, |
| 25 | + TimeProvider time, |
| 26 | + ILogger<StepWorker> logger) : IHostedService, IDisposable |
| 27 | +{ |
| 28 | + private readonly CancellationTokenSource _stopping = new(); |
| 29 | + private QueueConsumer? _consumer; |
| 30 | + private Task? _running; |
| 31 | + |
| 32 | + /// <summary>Identifier recorded on the attempts.</summary> |
| 33 | + public string WorkerId { get; } = options.WorkerId ?? broker.InstanceId; |
| 34 | + |
| 35 | + /// <summary>The consumer (for tests and health).</summary> |
| 36 | + public QueueConsumer? Consumer => _consumer; |
| 37 | + |
| 38 | + public Task StartAsync(CancellationToken cancellationToken) |
| 39 | + { |
| 40 | + _consumer = new QueueConsumer(broker, new QueueConsumerSettings |
| 41 | + { |
| 42 | + Queue = PsrTopology.ExecuteStepQueue, |
| 43 | + Prefetch = (ushort)options.MaxConcurrency, |
| 44 | + }, HandleAsync, logger); |
| 45 | + _running = Task.Run(() => _consumer.RunAsync(_stopping.Token), CancellationToken.None); |
| 46 | + Log.Started(logger, WorkerId, PsrTopology.ExecuteStepQueue, options.MaxConcurrency); |
| 47 | + return Task.CompletedTask; |
| 48 | + } |
| 49 | + |
| 50 | + public async Task StopAsync(CancellationToken cancellationToken) |
| 51 | + { |
| 52 | + // A host can be stopped more than once (e.g. stop, then dispose). |
| 53 | + if (Interlocked.Exchange(ref _consumer, null) is not { } consumer || _running is not { } running) |
| 54 | + { |
| 55 | + return; |
| 56 | + } |
| 57 | + |
| 58 | + await consumer.StopAsync(options.ShutdownGracePeriod, cancellationToken).ConfigureAwait(false); |
| 59 | + await _stopping.CancelAsync().ConfigureAwait(false); |
| 60 | + try |
| 61 | + { |
| 62 | + await running.WaitAsync(cancellationToken).ConfigureAwait(false); |
| 63 | + } |
| 64 | + catch (OperationCanceledException) |
| 65 | + { |
| 66 | + // Host shutdown timeout. |
| 67 | + } |
| 68 | + |
| 69 | + await consumer.DisposeAsync().ConfigureAwait(false); |
| 70 | + } |
| 71 | + |
| 72 | + public void Dispose() => _stopping.Dispose(); |
| 73 | + |
| 74 | + /// <param name="delivery">The message.</param> |
| 75 | + /// <param name="abandon">Cancelled when the shutdown grace period is over.</param> |
| 76 | + internal async Task<DeliveryOutcome> HandleAsync(PsrDelivery delivery, CancellationToken abandon) |
| 77 | + { |
| 78 | + ExecuteStepCommand command; |
| 79 | + try |
| 80 | + { |
| 81 | + if (delivery.Type is { Length: > 0 } type && type != PsrMessageTypes.ExecuteStep) |
| 82 | + { |
| 83 | + throw new JsonException($"unknown message type '{type}'"); |
| 84 | + } |
| 85 | + |
| 86 | + command = JsonSerializer.Deserialize<ExecuteStepCommand>(delivery.Body.Span, PsrMessageJson.Options) ?? throw new JsonException("The body is null."); |
| 87 | + } |
| 88 | + catch (JsonException ex) |
| 89 | + { |
| 90 | + Log.InvalidCommand(logger, delivery.MessageId, ex.Message); |
| 91 | + return DeliveryOutcome.DeadLetter; |
| 92 | + } |
| 93 | + |
| 94 | + using var activity = StartActivity(delivery, command); |
| 95 | + var claimed = await RetryAsync("claim step", () => lifecycle.ClaimStepAsync(command, WorkerId, options.LeaseDuration, abandon), abandon) |
| 96 | + .ConfigureAwait(false); |
| 97 | + if (claimed is null) |
| 98 | + { |
| 99 | + // Duplicate, stale (already executed or recovered elsewhere) or cancelled: nothing to do. |
| 100 | + return DeliveryOutcome.Ack; |
| 101 | + } |
| 102 | + |
| 103 | + RecordQueueWait(delivery); |
| 104 | + activity?.SetTag("psr.step.key", claimed.StepKey); |
| 105 | + activity?.SetTag("psr.stage.key", claimed.Stage.Key); |
| 106 | + metrics.StepStarted(); |
| 107 | + try |
| 108 | + { |
| 109 | + return await ExecuteAsync(claimed, abandon).ConfigureAwait(false); |
| 110 | + } |
| 111 | + finally |
| 112 | + { |
| 113 | + metrics.StepEnded(); |
| 114 | + } |
| 115 | + } |
| 116 | + |
| 117 | + private async Task<DeliveryOutcome> ExecuteAsync(ClaimedStep claimed, CancellationToken abandon) |
| 118 | + { |
| 119 | + using var execution = CancellationTokenSource.CreateLinkedTokenSource(abandon); |
| 120 | + using var renewing = new CancellationTokenSource(); |
| 121 | + var leaseLost = false; |
| 122 | + var renewal = Task.Run(async () => |
| 123 | + { |
| 124 | + using var timer = new PeriodicTimer(options.LeaseRenewInterval, time); |
| 125 | + try |
| 126 | + { |
| 127 | + while (await timer.WaitForNextTickAsync(renewing.Token).ConfigureAwait(false)) |
| 128 | + { |
| 129 | + try |
| 130 | + { |
| 131 | + if (!await lifecycle.RenewLeaseAsync(claimed, options.LeaseDuration, renewing.Token).ConfigureAwait(false)) |
| 132 | + { |
| 133 | + // The reaper took the attempt: stop the process and record nothing. |
| 134 | + leaseLost = true; |
| 135 | + await execution.CancelAsync().ConfigureAwait(false); |
| 136 | + return; |
| 137 | + } |
| 138 | + } |
| 139 | + catch (Exception ex) when (IsTransient(ex)) |
| 140 | + { |
| 141 | + Log.DatabaseRetry(logger, "renew lease", options.LeaseRenewInterval, ex); |
| 142 | + } |
| 143 | + } |
| 144 | + } |
| 145 | + catch (OperationCanceledException) |
| 146 | + { |
| 147 | + // Step finished. |
| 148 | + } |
| 149 | + }, CancellationToken.None); |
| 150 | + |
| 151 | + StepExecution result; |
| 152 | + try |
| 153 | + { |
| 154 | + result = await engine.ExecuteStepAsync(claimed.ToExecutionRequest(), execution.Token).ConfigureAwait(false); |
| 155 | + } |
| 156 | + finally |
| 157 | + { |
| 158 | + await renewing.CancelAsync().ConfigureAwait(false); |
| 159 | + await renewal.ConfigureAwait(false); |
| 160 | + } |
| 161 | + |
| 162 | + await using (result.ConfigureAwait(false)) |
| 163 | + { |
| 164 | + if (leaseLost) |
| 165 | + { |
| 166 | + Log.LeaseLost(logger, claimed.StepKey, claimed.RunId); |
| 167 | + return DeliveryOutcome.Ack; |
| 168 | + } |
| 169 | + |
| 170 | + if (abandon.IsCancellationRequested) |
| 171 | + { |
| 172 | + Log.Abandoned(logger, claimed.StepKey, claimed.RunId); |
| 173 | + return DeliveryOutcome.Abandon; |
| 174 | + } |
| 175 | + |
| 176 | + // Retried while the lease lasts (database blips); afterwards the reaper decides (D-85). |
| 177 | + var completion = await RetryAsync("complete step", () => lifecycle.CompleteStepAsync(claimed, result, CancellationToken.None), |
| 178 | + CancellationToken.None, options.LeaseDuration).ConfigureAwait(false); |
| 179 | + if (completion == StepCompletion.LeaseLost) |
| 180 | + { |
| 181 | + Log.LeaseLost(logger, claimed.StepKey, claimed.RunId); |
| 182 | + } |
| 183 | + else |
| 184 | + { |
| 185 | + Log.StepCompleted(logger, claimed.StepKey, claimed.Attempt, claimed.RunId, completion.ToString()); |
| 186 | + } |
| 187 | + |
| 188 | + return DeliveryOutcome.Ack; |
| 189 | + } |
| 190 | + } |
| 191 | + |
| 192 | + /// <summary>Retries transient database failures with backoff (up to <paramref name="limit"/>; default 30 seconds).</summary> |
| 193 | + private async Task<T> RetryAsync<T>(string operation, Func<Task<T>> action, CancellationToken cancellationToken, TimeSpan? limit = null) |
| 194 | + { |
| 195 | + var started = Stopwatch.GetTimestamp(); |
| 196 | + var delay = TimeSpan.FromMilliseconds(200); |
| 197 | + while (true) |
| 198 | + { |
| 199 | + try |
| 200 | + { |
| 201 | + return await action().ConfigureAwait(false); |
| 202 | + } |
| 203 | + catch (Exception ex) when (IsTransient(ex) && Stopwatch.GetElapsedTime(started) + delay < (limit ?? TimeSpan.FromSeconds(30))) |
| 204 | + { |
| 205 | + Log.DatabaseRetry(logger, operation, delay, ex); |
| 206 | + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); |
| 207 | + delay = TimeSpan.FromMilliseconds(Math.Min(5000, delay.TotalMilliseconds * 2)); |
| 208 | + } |
| 209 | + } |
| 210 | + } |
| 211 | + |
| 212 | + private static bool IsTransient(Exception exception) => |
| 213 | + exception is DbException or TimeoutException || exception.InnerException is DbException or TimeoutException; |
| 214 | + |
| 215 | + private static Activity? StartActivity(PsrDelivery delivery, ExecuteStepCommand command) |
| 216 | + { |
| 217 | + var activity = ActivityContext.TryParse(delivery.TraceParent, null, out var parent) |
| 218 | + ? WorkerDiagnostics.Source.StartActivity("psr.message.consume", ActivityKind.Consumer, parent) |
| 219 | + : WorkerDiagnostics.Source.StartActivity("psr.message.consume", ActivityKind.Consumer); |
| 220 | + activity?.SetTag("messaging.system", "rabbitmq"); |
| 221 | + activity?.SetTag("messaging.destination.name", delivery.Queue); |
| 222 | + activity?.SetTag("psr.run.id", command.RunId.ToString()); |
| 223 | + activity?.SetTag("psr.attempt", command.Attempt); |
| 224 | + return activity; |
| 225 | + } |
| 226 | + |
| 227 | + /// <summary><c>psr.queue.wait</c>: from the moment the message became due (header set by the relay) to the claim.</summary> |
| 228 | + private void RecordQueueWait(PsrDelivery delivery) |
| 229 | + { |
| 230 | + if (delivery.Headers.TryGetValue("psr-available-at", out var text) |
| 231 | + && long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var milliseconds)) |
| 232 | + { |
| 233 | + metrics.QueueWait(time.GetUtcNow() - DateTimeOffset.FromUnixTimeMilliseconds(milliseconds)); |
| 234 | + } |
| 235 | + } |
| 236 | +} |
0 commit comments