Skip to content

Commit c55239a

Browse files
net0wellclaude
andcommitted
feat(rabbitmq): transporte RabbitMQ com relay do outbox, comando público e eventos (spec 005)
- Topologia declarada ou verificada (psr.commands, psr.public, psr.events, psr.control, psr.dlx; filas quorum com delivery-limit e DLX). - Relay do outbox com publisher confirms e backoff; IRunQueue transacional. - Consumidor de StartPipelineRun (run.queued / run.rejected, deduplicação por idempotencyKey ou message_id), cancelamento por psr.control e assinaturas IRunEventHandler duráveis ou por instância (D-110). - Propagação de traceparent, métricas e health check (Degraded sem broker). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 22bf099 commit c55239a

14 files changed

Lines changed: 1489 additions & 0 deletions

‎Directory.Packages.props‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@
33
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
44
</PropertyGroup>
55
<ItemGroup>
6+
<PackageVersion Include="Cronos" Version="0.13.0" />
67
<PackageVersion Include="JsonSchema.Net" Version="9.4.1" />
8+
<PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="10.0.13" />
79
<PackageVersion Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.13" />
810
<PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="10.0.13" />
911
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="10.0.13" />
@@ -13,13 +15,22 @@
1315
<PackageVersion Include="Microsoft.Extensions.Diagnostics" Version="10.0.13" />
1416
<PackageVersion Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="10.0.13" />
1517
<PackageVersion Include="Microsoft.Extensions.Diagnostics.Testing" Version="10.11.0" />
18+
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="10.0.13" />
1619
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.13" />
1720
<PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.13" />
1821
<PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.13" />
1922
<PackageVersion Include="Microsoft.Extensions.TimeProvider.Testing" Version="10.11.0" />
2023
<PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.3" />
24+
<PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.20.1" />
25+
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.20.1" />
26+
<PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.20.0" />
27+
<PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.20.0" />
28+
<PackageVersion Include="RabbitMQ.Client" Version="7.2.2" />
29+
<PackageVersion Include="Serilog.AspNetCore" Version="10.0.0" />
30+
<PackageVersion Include="Serilog.Formatting.Compact" Version="3.0.0" />
2131
<PackageVersion Include="Shouldly" Version="4.3.0" />
2232
<PackageVersion Include="Testcontainers.PostgreSql" Version="4.16.0" />
33+
<PackageVersion Include="Testcontainers.RabbitMq" Version="4.16.0" />
2334
<PackageVersion Include="xunit.v3" Version="4.0.2" />
2435
</ItemGroup>
2536
</Project>
Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
using System.Diagnostics;
2+
using System.Text.Json;
3+
using System.Text.Json.Nodes;
4+
using Microsoft.Extensions.Hosting;
5+
using Microsoft.Extensions.Logging;
6+
using PythonScriptRunner.EntityFrameworkCore;
7+
8+
namespace PythonScriptRunner.RabbitMQ;
9+
10+
/// <summary>Runs one <see cref="QueueConsumer"/> for the lifetime of the host.</summary>
11+
internal abstract class ConsumerService(PsrBroker broker, ILogger logger) : IHostedService, IDisposable
12+
{
13+
private readonly CancellationTokenSource _stopping = new();
14+
private QueueConsumer? _consumer;
15+
private Task? _running;
16+
17+
protected PsrBroker Broker { get; } = broker;
18+
19+
protected ILogger Logger { get; } = logger;
20+
21+
/// <summary>How long deliveries in progress may finish when the host stops.</summary>
22+
protected virtual TimeSpan GracePeriod => TimeSpan.FromSeconds(30);
23+
24+
public Task StartAsync(CancellationToken cancellationToken)
25+
{
26+
_consumer = new QueueConsumer(Broker, Settings(), HandleAsync, Logger);
27+
_running = Task.Run(() => _consumer.RunAsync(_stopping.Token), CancellationToken.None);
28+
return Task.CompletedTask;
29+
}
30+
31+
public async Task StopAsync(CancellationToken cancellationToken)
32+
{
33+
// A host can be stopped more than once (e.g. stop, then dispose).
34+
if (Interlocked.Exchange(ref _consumer, null) is not { } consumer || _running is not { } running)
35+
{
36+
return;
37+
}
38+
39+
await consumer.StopAsync(GracePeriod, cancellationToken).ConfigureAwait(false);
40+
await _stopping.CancelAsync().ConfigureAwait(false);
41+
try
42+
{
43+
await running.WaitAsync(cancellationToken).ConfigureAwait(false);
44+
}
45+
catch (OperationCanceledException)
46+
{
47+
// Host shutdown timeout.
48+
}
49+
50+
await consumer.DisposeAsync().ConfigureAwait(false);
51+
}
52+
53+
public void Dispose() => _stopping.Dispose();
54+
55+
protected abstract QueueConsumerSettings Settings();
56+
57+
protected abstract Task<DeliveryOutcome> HandleAsync(PsrDelivery delivery, CancellationToken cancellationToken);
58+
59+
/// <summary>Reads the JSON body; <c>null</c> (logged) when it is not valid.</summary>
60+
protected T? Read<T>(PsrDelivery delivery, string expectedType)
61+
where T : class
62+
{
63+
if (delivery.Type is { Length: > 0 } type && type != expectedType)
64+
{
65+
Log.InvalidMessage(Logger, delivery.MessageId, delivery.Queue, $"unknown message type '{type}'");
66+
return null;
67+
}
68+
69+
try
70+
{
71+
return JsonSerializer.Deserialize<T>(delivery.Body.Span, PsrMessageJson.Options)
72+
?? throw new JsonException("The body is null.");
73+
}
74+
catch (JsonException ex)
75+
{
76+
Log.InvalidMessage(Logger, delivery.MessageId, delivery.Queue, ex.Message);
77+
return null;
78+
}
79+
}
80+
}
81+
82+
/// <summary>
83+
/// Consumes the public <c>StartPipelineRun</c> command (D-72): valid commands create a run (<c>run.queued</c>); unknown
84+
/// pipelines, invalid inputs and file inputs produce <c>run.rejected</c> with the reasons. Both carry the caller's
85+
/// <c>correlationId</c>.
86+
/// </summary>
87+
internal sealed class StartPipelineRunService(
88+
PsrBroker broker, ICatalog catalog, IRunEngine engine, AsyncRunLifecycle lifecycle, ILogger<StartPipelineRunService> logger)
89+
: ConsumerService(broker, logger)
90+
{
91+
private const int MaxTextLength = 200;
92+
93+
protected override QueueConsumerSettings Settings() => new() { Queue = PsrTopology.StartPipelineQueue, Prefetch = 8 };
94+
95+
protected override async Task<DeliveryOutcome> HandleAsync(PsrDelivery delivery, CancellationToken cancellationToken)
96+
{
97+
var command = Read<StartPipelineRunCommand>(delivery, PsrMessageTypes.StartPipelineRun);
98+
if (command is null || string.IsNullOrWhiteSpace(command.PipelineKey))
99+
{
100+
return DeliveryOutcome.DeadLetter;
101+
}
102+
103+
using var activity = RabbitMqDiagnostics.StartFrom("psr.message.consume", ActivityKind.Consumer, delivery.TraceParent);
104+
activity?.SetTag("messaging.destination.name", delivery.Queue);
105+
activity?.SetTag("psr.pipeline.key", command.PipelineKey);
106+
try
107+
{
108+
if (new[] { command.RequestedBy, command.CorrelationId, command.IdempotencyKey }.Any(text => text?.Length > MaxTextLength - 10))
109+
{
110+
throw new PsrDomainException("TEXT_TOO_LONG", $"requestedBy, correlationId and idempotencyKey must have up to {MaxTextLength - 10} chars.");
111+
}
112+
113+
var pipeline = await catalog.FindPipelineAsync(command.PipelineKey, command.Version, cancellationToken).ConfigureAwait(false)
114+
?? throw (command.Version is { } version ? new VersionNotFoundException(command.PipelineKey, version) : new PipelineNotFoundException(command.PipelineKey));
115+
var inputs = command.Inputs ?? [];
116+
var files = pipeline.Inputs.Where(f => f.Type.IsFile && (f.Required || inputs.ContainsKey(f.Name))).Select(f => f.Name).ToList();
117+
if (files.Count > 0)
118+
{
119+
throw new PsrDomainException(PsrErrorCodes.FileInputNotAllowed,
120+
$"Pipeline '{pipeline.Key}' has file inputs ({string.Join(", ", files)}); the queue command cannot send files.");
121+
}
122+
123+
await lifecycle.CreateAsync(new NewAsyncRun
124+
{
125+
Kind = RunKind.Pipeline,
126+
Pipeline = pipeline,
127+
Inputs = engine.BindInputs(pipeline.Inputs, (JsonObject)inputs.DeepClone()),
128+
Trigger = RunTrigger.QueueCommand,
129+
TriggeredBy = command.RequestedBy is { Length: > 0 } requestedBy ? $"queue:{requestedBy}" : "system:queue",
130+
IdempotencyKey = command.IdempotencyKey,
131+
ImplicitIdempotencyKey = delivery.MessageId is { Length: > 0 } messageId ? $"message:{messageId}" : null,
132+
CorrelationId = command.CorrelationId,
133+
TraceParent = Activity.Current?.Id ?? delivery.TraceParent,
134+
AnnounceDuplicate = true,
135+
}, cancellationToken).ConfigureAwait(false);
136+
}
137+
catch (Exception ex) when (ex is PsrException and not PsrConfigurationException)
138+
{
139+
Log.CommandRejected(Logger, command.PipelineKey, ex.Message);
140+
await lifecycle.PublishRejectedAsync(new RunEvent
141+
{
142+
EventId = Guid.CreateVersion7(),
143+
Type = RunEventTypes.RunRejected,
144+
OccurredAt = DateTimeOffset.UtcNow,
145+
Kind = RunKind.Pipeline,
146+
PipelineKey = command.PipelineKey,
147+
CorrelationId = command.CorrelationId,
148+
Trigger = RunTrigger.QueueCommand,
149+
Error = AsyncRunLifecycle.Rejection(ex),
150+
}, cancellationToken).ConfigureAwait(false);
151+
}
152+
153+
return DeliveryOutcome.Ack;
154+
}
155+
}
156+
157+
/// <summary>Consumes the control queue of this instance: <c>CancelRun</c> stops the processes of the run if it executes here.</summary>
158+
internal sealed class ControlService(PsrBroker broker, IRunEngine engine, ILogger<ControlService> logger) : ConsumerService(broker, logger)
159+
{
160+
protected override TimeSpan GracePeriod => TimeSpan.FromSeconds(5);
161+
162+
protected override QueueConsumerSettings Settings()
163+
{
164+
var queue = PsrTopology.ControlQueue(Broker.InstanceId);
165+
return new()
166+
{
167+
Queue = queue,
168+
Prefetch = 4,
169+
Declare = async (channel, cancellationToken) =>
170+
{
171+
await channel.QueueDeclareAsync(queue, durable: false, exclusive: true, autoDelete: true, cancellationToken: cancellationToken)
172+
.ConfigureAwait(false);
173+
await channel.QueueBindAsync(queue, PsrTopology.ControlExchange, string.Empty, cancellationToken: cancellationToken).ConfigureAwait(false);
174+
},
175+
};
176+
}
177+
178+
protected override Task<DeliveryOutcome> HandleAsync(PsrDelivery delivery, CancellationToken cancellationToken)
179+
{
180+
if (Read<CancelRunCommand>(delivery, PsrMessageTypes.CancelRun) is not { } command)
181+
{
182+
return Task.FromResult(DeliveryOutcome.DeadLetter);
183+
}
184+
185+
engine.RequestCancellation(command.RunId);
186+
return Task.FromResult(DeliveryOutcome.Ack);
187+
}
188+
}
Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
using System.Diagnostics;
2+
using System.Diagnostics.Metrics;
3+
using Microsoft.Extensions.Logging;
4+
5+
namespace PythonScriptRunner.RabbitMQ;
6+
7+
/// <summary>Tracing of the transport (source <c>PythonScriptRunner</c>, shared by every package).</summary>
8+
internal static class RabbitMqDiagnostics
9+
{
10+
public static readonly ActivitySource Source = new("PythonScriptRunner");
11+
12+
/// <summary>Starts an activity whose parent is the W3C <c>traceparent</c> carried by a message (the trace continues across processes).</summary>
13+
public static Activity? StartFrom(string name, ActivityKind kind, string? traceParent) =>
14+
ActivityContext.TryParse(traceParent, null, out var parent)
15+
? Source.StartActivity(name, kind, parent)
16+
: Source.StartActivity(name, kind);
17+
}
18+
19+
/// <summary>Metrics of the transport (meter <c>PythonScriptRunner</c>).</summary>
20+
internal sealed class RabbitMqMetrics
21+
{
22+
private readonly Histogram<double> _publishDuration;
23+
private int _pending;
24+
25+
public RabbitMqMetrics(IMeterFactory meterFactory)
26+
{
27+
var meter = meterFactory.Create("PythonScriptRunner");
28+
_publishDuration = meter.CreateHistogram<double>("psr.outbox.publish.duration", "ms", "Time to publish one outbox batch with confirms.");
29+
meter.CreateObservableGauge("psr.outbox.pending", () => Volatile.Read(ref _pending), description: "Outbox messages not published yet.");
30+
}
31+
32+
public void BatchPublished(TimeSpan elapsed, int count) =>
33+
_publishDuration.Record(elapsed.TotalMilliseconds, new KeyValuePair<string, object?>("psr.messages", count));
34+
35+
public void Pending(int count) => Volatile.Write(ref _pending, count);
36+
}
37+
38+
/// <summary>Source-generated log messages.</summary>
39+
internal static partial class Log
40+
{
41+
[LoggerMessage(100, LogLevel.Information, "Connected to RabbitMQ at {Endpoint} (declare topology: {DeclareTopology})")]
42+
public static partial void Connected(ILogger logger, string endpoint, bool declareTopology);
43+
44+
[LoggerMessage(101, LogLevel.Warning, "RabbitMQ is unreachable ({Reason}); queued runs stay in the outbox until it comes back")]
45+
public static partial void BrokerUnreachable(ILogger logger, string reason);
46+
47+
[LoggerMessage(102, LogLevel.Critical, "The RabbitMQ topology is invalid; stopping the application")]
48+
public static partial void TopologyInvalid(ILogger logger, Exception exception);
49+
50+
[LoggerMessage(103, LogLevel.Warning, "Outbox message {MessageId} ({MessageType}) was not published: {Error}")]
51+
public static partial void PublishFailed(ILogger logger, Guid messageId, string messageType, string error);
52+
53+
[LoggerMessage(104, LogLevel.Error, "Message {MessageId} on {Queue} is invalid and goes to the dead-letter queue: {Reason}")]
54+
public static partial void InvalidMessage(ILogger logger, string? messageId, string queue, string reason);
55+
56+
[LoggerMessage(105, LogLevel.Warning, "Handling message {MessageId} on {Queue} failed; it will be delivered again")]
57+
public static partial void HandlerFailed(ILogger logger, string? messageId, string queue, Exception exception);
58+
59+
[LoggerMessage(106, LogLevel.Warning, "Consumer of {Queue} stopped ({Reason}); restarting")]
60+
public static partial void ConsumerRestarting(ILogger logger, string queue, string reason);
61+
62+
[LoggerMessage(107, LogLevel.Information, "StartPipelineRun for '{PipelineKey}' rejected: {Reason}")]
63+
public static partial void CommandRejected(ILogger logger, string pipelineKey, string reason);
64+
65+
[LoggerMessage(108, LogLevel.Error, "The outbox relay failed")]
66+
public static partial void RelayFailed(ILogger logger, Exception exception);
67+
}

0 commit comments

Comments
 (0)