Skip to content

Commit 22bf099

Browse files
net0wellclaude
andcommitted
feat(ef): ciclo de vida das execuções enfileiradas, agendamentos e eventos no outbox (specs 004, 005, 006)
- AsyncRunLifecycle (D-112): criação transacional com outbox e limite por trilha (Pending FIFO), claim com lease, conclusão (próximo passo, retry atrasado, fim), reaper de leases e disparo do cron com SKIP LOCKED e coalescência (D-45, D-77, D-85). - IRunMessageOutbox: ponto de extensão das mensagens (no-op sem fila); toda execução síncrona também grava eventos quando a fila está registrada (D-105). - PsrOutbox para o relay; PsrClock alinhado ao relógio do PostgreSQL. - CRUD de agendamentos com Cronos, validação de cron/fuso/versão/entradas e SCHEDULE_FILE_INPUT; aviso na publicação da trilha (D-77). - Cancelamento com CancelRun para todas as instâncias e reprocesso assíncrono (QA-003-3). - Migration AsyncRuns: coluna runs.trigger_details (D-108). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent e211510 commit 22bf099

23 files changed

Lines changed: 3464 additions & 242 deletions

‎src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Pipelines.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -235,12 +235,14 @@ public async Task<PipelinePublication> PublishPipelineAsync(
235235
}
236236

237237
var steps = await EfCatalog.LoadStepsAsync(db, row.Id, cancellationToken).ConfigureAwait(false);
238-
var report = await pipelineValidator.ValidateAsync(CatalogMapping.ToPipeline(row.Pipeline, row, steps), cancellationToken).ConfigureAwait(false);
238+
var definition = CatalogMapping.ToPipeline(row.Pipeline, row, steps);
239+
var report = await pipelineValidator.ValidateAsync(definition, cancellationToken).ConfigureAwait(false);
239240
if (!report.IsValid)
240241
{
241242
throw new DefinitionInvalidException(PsrErrorCodes.PipelineInvalid, report);
242243
}
243244

245+
List<ValidationIssue> warnings = [.. report.Warnings, .. await ScheduleWarningsAsync(db, row.PipelineId, definition, cancellationToken).ConfigureAwait(false)];
244246
var now = time.GetUtcNow();
245247
row.Status = VersionStatus.Published;
246248
row.PublishedAt = now;
@@ -252,7 +254,7 @@ public async Task<PipelinePublication> PublishPipelineAsync(
252254
AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelinePublished, "PipelineVersion", row.Id, $"{key}@{version}"), now);
253255
await SaveAsync(db, cancellationToken, conflict: Current(() => GetPipelineDraftAsync(key, CancellationToken.None))).ConfigureAwait(false);
254256
catalog.InvalidatePipeline(key);
255-
return new PipelinePublication(CatalogMapping.ToPipelineVersionDetail(row.Pipeline, row, steps), report.Warnings);
257+
return new PipelinePublication(CatalogMapping.ToPipelineVersionDetail(row.Pipeline, row, steps), warnings);
256258
}
257259
}
258260

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
using System.Text.Json.Nodes;
2+
using Microsoft.EntityFrameworkCore;
3+
using PythonScriptRunner.EntityFrameworkCore.Entities;
4+
using PythonScriptRunner.EntityFrameworkCore.Infrastructure;
5+
using PythonScriptRunner.EntityFrameworkCore.Scheduling;
6+
7+
namespace PythonScriptRunner.EntityFrameworkCore.Catalog;
8+
9+
/// <summary>Cron schedules of pipelines (D-41, D-45, D-77, D-89).</summary>
10+
internal sealed partial class EfCatalogAdmin
11+
{
12+
public async Task<Page<ScheduleDetail>> ListSchedulesAsync(ScheduleQuery query, CancellationToken cancellationToken = default)
13+
{
14+
ArgumentNullException.ThrowIfNull(query);
15+
var (pageNumber, pageSize) = (Math.Max(1, query.PageNumber), Math.Clamp(query.PageSize, 1, 100));
16+
var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
17+
await using (db.ConfigureAwait(false))
18+
{
19+
var rows = db.Schedules.AsNoTracking().Join(db.Pipelines, s => s.PipelineId, p => p.Id, (s, p) => new { Schedule = s, p.Key });
20+
if (query.PipelineKey is { } pipelineKey)
21+
{
22+
rows = rows.Where(r => r.Key == pipelineKey);
23+
}
24+
25+
var total = await rows.CountAsync(cancellationToken).ConfigureAwait(false);
26+
var items = await rows.OrderBy(r => r.Schedule.Name).ThenBy(r => r.Schedule.Id)
27+
.Skip((pageNumber - 1) * pageSize).Take(pageSize)
28+
.ToListAsync(cancellationToken).ConfigureAwait(false);
29+
return new Page<ScheduleDetail>([.. items.Select(r => ToScheduleDetail(r.Schedule, r.Key))], pageNumber, pageSize, total);
30+
}
31+
}
32+
33+
public async Task<ScheduleDetail?> GetScheduleAsync(ScheduleId id, CancellationToken cancellationToken = default)
34+
{
35+
var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
36+
await using (db.ConfigureAwait(false))
37+
{
38+
var row = await db.Schedules.AsNoTracking().Join(db.Pipelines, s => s.PipelineId, p => p.Id, (s, p) => new { Schedule = s, p.Key })
39+
.SingleOrDefaultAsync(r => r.Schedule.Id == id.Value, cancellationToken).ConfigureAwait(false);
40+
return row is null ? null : ToScheduleDetail(row.Schedule, row.Key);
41+
}
42+
}
43+
44+
public async Task<ScheduleDetail> CreateScheduleAsync(string pipelineKey, ScheduleData schedule, Actor actor, CancellationToken cancellationToken = default)
45+
{
46+
ArgumentNullException.ThrowIfNull(schedule);
47+
ArgumentNullException.ThrowIfNull(actor);
48+
var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
49+
await using (db.ConfigureAwait(false))
50+
{
51+
var pipeline = await db.Pipelines.AsNoTracking().SingleOrDefaultAsync(p => p.Key == pipelineKey, cancellationToken).ConfigureAwait(false)
52+
?? throw new PipelineNotFoundException(pipelineKey);
53+
var cron = await ValidateScheduleAsync(pipelineKey, schedule, cancellationToken).ConfigureAwait(false);
54+
var now = time.GetUtcNow();
55+
var row = new ScheduleEntity
56+
{
57+
Id = Guid.CreateVersion7(now),
58+
PipelineId = pipeline.Id,
59+
Name = schedule.Name,
60+
Cron = schedule.Cron.Trim(),
61+
TimeZone = schedule.TimeZone,
62+
PinnedVersion = schedule.PinnedVersion,
63+
Inputs = schedule.Inputs.ToJsonString(),
64+
IsEnabled = schedule.IsEnabled,
65+
NextRunAt = schedule.IsEnabled ? cron.Next(now) : null,
66+
CreatedAt = now,
67+
CreatedBy = actor.Subject,
68+
UpdatedAt = now,
69+
UpdatedBy = actor.Subject,
70+
};
71+
db.Schedules.Add(row);
72+
AuditLog.Add(db, new AuditEntry(actor, AuditActions.ScheduleCreated, "Schedule", row.Id, ScheduleKey(pipelineKey, row.Name), Describe(row)), now);
73+
await SaveAsync(db, cancellationToken).ConfigureAwait(false);
74+
return ToScheduleDetail(row, pipelineKey);
75+
}
76+
}
77+
78+
public async Task<ScheduleDetail> UpdateScheduleAsync(
79+
ScheduleId id, ScheduleData schedule, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default)
80+
{
81+
ArgumentNullException.ThrowIfNull(schedule);
82+
ArgumentNullException.ThrowIfNull(actor);
83+
var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
84+
await using (db.ConfigureAwait(false))
85+
{
86+
var (row, pipelineKey) = await LoadScheduleAsync(db, id, expectedRowVersion, cancellationToken).ConfigureAwait(false);
87+
var cron = await ValidateScheduleAsync(pipelineKey, schedule, cancellationToken).ConfigureAwait(false);
88+
var before = Describe(row);
89+
var now = time.GetUtcNow();
90+
row.Name = schedule.Name;
91+
row.Cron = schedule.Cron.Trim();
92+
row.TimeZone = schedule.TimeZone;
93+
row.PinnedVersion = schedule.PinnedVersion;
94+
row.Inputs = schedule.Inputs.ToJsonString();
95+
row.IsEnabled = schedule.IsEnabled;
96+
row.NextRunAt = schedule.IsEnabled ? cron.Next(now) : null;
97+
row.UpdatedAt = now;
98+
row.UpdatedBy = actor.Subject;
99+
var after = Describe(row);
100+
var changes = new JsonObject();
101+
foreach (var (name, value) in after)
102+
{
103+
if (!JsonNode.DeepEquals(before[name], value))
104+
{
105+
changes[name] = new JsonArray(before[name]?.DeepClone(), value?.DeepClone());
106+
}
107+
}
108+
109+
AuditLog.Add(db, new AuditEntry(actor, AuditActions.ScheduleUpdated, "Schedule", row.Id, ScheduleKey(pipelineKey, row.Name), changes), now);
110+
await SaveAsync(db, cancellationToken, conflict: Current(() => GetScheduleAsync(id, CancellationToken.None))).ConfigureAwait(false);
111+
return ToScheduleDetail(row, pipelineKey);
112+
}
113+
}
114+
115+
public async Task<ScheduleDetail> SetScheduleEnabledAsync(
116+
ScheduleId id, bool enabled, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default)
117+
{
118+
ArgumentNullException.ThrowIfNull(actor);
119+
var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
120+
await using (db.ConfigureAwait(false))
121+
{
122+
var (row, pipelineKey) = await LoadScheduleAsync(db, id, expectedRowVersion, cancellationToken).ConfigureAwait(false);
123+
if (row.IsEnabled == enabled)
124+
{
125+
return ToScheduleDetail(row, pipelineKey);
126+
}
127+
128+
var now = time.GetUtcNow();
129+
row.IsEnabled = enabled;
130+
row.NextRunAt = enabled ? CronSchedule.Parse(row.Cron, row.TimeZone).Next(now) : null;
131+
row.UpdatedAt = now;
132+
row.UpdatedBy = actor.Subject;
133+
AuditLog.Add(db, new AuditEntry(actor, enabled ? AuditActions.ScheduleEnabled : AuditActions.ScheduleDisabled, "Schedule", row.Id,
134+
ScheduleKey(pipelineKey, row.Name)), now);
135+
await SaveAsync(db, cancellationToken, conflict: Current(() => GetScheduleAsync(id, CancellationToken.None))).ConfigureAwait(false);
136+
return ToScheduleDetail(row, pipelineKey);
137+
}
138+
}
139+
140+
public async Task DeleteScheduleAsync(ScheduleId id, Actor actor, CancellationToken cancellationToken = default)
141+
{
142+
ArgumentNullException.ThrowIfNull(actor);
143+
var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
144+
await using (db.ConfigureAwait(false))
145+
{
146+
var row = await db.Schedules.Join(db.Pipelines, s => s.PipelineId, p => p.Id, (s, p) => new { Schedule = s, p.Key })
147+
.SingleOrDefaultAsync(r => r.Schedule.Id == id.Value, cancellationToken).ConfigureAwait(false)
148+
?? throw ScheduleNotFound(id);
149+
db.Schedules.Remove(row.Schedule);
150+
AuditLog.Add(db, new AuditEntry(actor, AuditActions.ScheduleDeleted, "Schedule", row.Schedule.Id, ScheduleKey(row.Key, row.Schedule.Name),
151+
Describe(row.Schedule)), time.GetUtcNow());
152+
await SaveAsync(db, cancellationToken).ConfigureAwait(false);
153+
}
154+
}
155+
156+
/// <summary>
157+
/// Warnings of D-77: enabled schedules that follow the latest version and do not provide a required input of the new one
158+
/// (their next firings will be rejected).
159+
/// </summary>
160+
private static async Task<List<ValidationIssue>> ScheduleWarningsAsync(
161+
PsrDbContext db, Guid pipelineId, PipelineDefinition published, CancellationToken cancellationToken)
162+
{
163+
var schedules = await db.Schedules.AsNoTracking()
164+
.Where(s => s.PipelineId == pipelineId && s.IsEnabled && s.PinnedVersion == null)
165+
.Select(s => new { s.Name, s.Inputs })
166+
.ToListAsync(cancellationToken).ConfigureAwait(false);
167+
var warnings = new List<ValidationIssue>();
168+
foreach (var schedule in schedules)
169+
{
170+
var inputs = StoredJson.Object(schedule.Inputs) ?? [];
171+
var missing = ScheduleRules.MissingInputs(published, inputs)
172+
.Concat(published.Inputs.Where(f => f.Type.IsFile && f.Required).Select(f => f.Name))
173+
.Distinct(StringComparer.Ordinal)
174+
.ToList();
175+
if (missing.Count > 0)
176+
{
177+
warnings.Add(new ValidationIssue("SCHEDULE_MISSING_INPUT",
178+
$"schedule '{schedule.Name}' does not provide the required input(s) {string.Join(", ", missing)}; its firings will be rejected"));
179+
}
180+
}
181+
182+
return warnings;
183+
}
184+
185+
/// <summary>Validates name, cron, time zone, version and inputs; returns the parsed expression.</summary>
186+
private async Task<CronSchedule> ValidateScheduleAsync(string pipelineKey, ScheduleData schedule, CancellationToken cancellationToken)
187+
{
188+
ValidateName(schedule.Name);
189+
ArgumentNullException.ThrowIfNull(schedule.Inputs);
190+
var cron = CronSchedule.Parse(schedule.Cron, schedule.TimeZone);
191+
if (schedule.PinnedVersion is < 1)
192+
{
193+
throw new PsrDomainException(PsrErrorCodes.ScheduleInvalid, "The pinned version must be at least 1 (or null for the latest).");
194+
}
195+
196+
var pipeline = await ScheduleRules.ResolveAsync(catalog, pipelineKey, schedule.PinnedVersion, cancellationToken).ConfigureAwait(false);
197+
ScheduleRules.EnsureNoFileInputs(pipeline, schedule.Inputs);
198+
engine.BindInputs(pipeline.Inputs, schedule.Inputs);
199+
return cron;
200+
}
201+
202+
private static async Task<(ScheduleEntity Row, string PipelineKey)> LoadScheduleAsync(
203+
PsrDbContext db, ScheduleId id, uint expectedRowVersion, CancellationToken cancellationToken)
204+
{
205+
var loaded = await db.Schedules.Join(db.Pipelines, s => s.PipelineId, p => p.Id, (s, p) => new { Schedule = s, p.Key })
206+
.SingleOrDefaultAsync(r => r.Schedule.Id == id.Value, cancellationToken).ConfigureAwait(false)
207+
?? throw ScheduleNotFound(id);
208+
if (loaded.Schedule.RowVersion != expectedRowVersion)
209+
{
210+
throw new ConcurrencyConflictException($"Schedule {id} was changed by someone else.", ToScheduleDetail(loaded.Schedule, loaded.Key));
211+
}
212+
213+
db.Entry(loaded.Schedule).Property(s => s.RowVersion).OriginalValue = expectedRowVersion;
214+
return (loaded.Schedule, loaded.Key);
215+
}
216+
217+
private static PsrDomainException ScheduleNotFound(ScheduleId id) => new(PsrErrorCodes.ScheduleNotFound, $"Schedule {id} was not found.");
218+
219+
private static string ScheduleKey(string pipelineKey, string name)
220+
{
221+
var key = $"{pipelineKey}/{name}";
222+
return key.Length <= 100 ? key : key[..100];
223+
}
224+
225+
private static JsonObject Describe(ScheduleEntity row) => new()
226+
{
227+
["name"] = row.Name,
228+
["cron"] = row.Cron,
229+
["timeZone"] = row.TimeZone,
230+
["pinnedVersion"] = row.PinnedVersion,
231+
["inputs"] = StoredJson.Node(row.Inputs),
232+
["isEnabled"] = row.IsEnabled,
233+
};
234+
235+
private static ScheduleDetail ToScheduleDetail(ScheduleEntity row, string pipelineKey) => new()
236+
{
237+
Id = new ScheduleId(row.Id),
238+
PipelineKey = pipelineKey,
239+
Data = new ScheduleData
240+
{
241+
Name = row.Name,
242+
Cron = row.Cron,
243+
TimeZone = row.TimeZone,
244+
PinnedVersion = row.PinnedVersion,
245+
Inputs = StoredJson.Object(row.Inputs) ?? [],
246+
IsEnabled = row.IsEnabled,
247+
},
248+
NextRunAt = row.NextRunAt,
249+
LastFiredAt = row.LastFiredAt,
250+
LastRunId = row.LastRunId is { } runId ? new RunId(runId) : null,
251+
LastError = row.LastError,
252+
RowVersion = row.RowVersion,
253+
Authorship = new Authorship(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy),
254+
};
255+
}

‎src/PythonScriptRunner.EntityFrameworkCore/Entities/RunEntities.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ public sealed class RunEntity
3333

3434
public Guid? ScheduleId { get; set; }
3535

36+
/// <summary>Details of the trigger as JSON (e.g. scheduled time and missed occurrences of a schedule recovery).</summary>
37+
public string? TriggerDetails { get; set; }
38+
3639
public RunStatus Status { get; set; }
3740

3841
/// <summary>Validated inputs as JSON (files as <c>{"$artifact": id}</c>).</summary>
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
using System.Diagnostics;
2+
using Npgsql;
3+
using PythonScriptRunner.EntityFrameworkCore.Runs;
4+
5+
namespace PythonScriptRunner.EntityFrameworkCore;
6+
7+
/// <summary>
8+
/// Current time of queued runs: the host clock corrected by its offset to the PostgreSQL clock (measured every minute), so
9+
/// lease expiry, retry delays and cron decisions agree across hosts (spec 006). With
10+
/// <see cref="PsrEntityFrameworkOptions.UseDatabaseClock"/> off it is the registered <see cref="TimeProvider"/>.
11+
/// </summary>
12+
public sealed class PsrClock
13+
{
14+
private static readonly TimeSpan SyncInterval = TimeSpan.FromMinutes(1);
15+
16+
private readonly PsrDataSource _dataSource;
17+
private readonly PsrEntityFrameworkOptions _options;
18+
private readonly TimeProvider _time;
19+
private readonly SemaphoreSlim _sync = new(1, 1);
20+
private TimeSpan _offset;
21+
private long _syncedAt;
22+
23+
internal PsrClock(PsrDataSource dataSource, PsrEntityFrameworkOptions options, TimeProvider time)
24+
{
25+
_dataSource = dataSource;
26+
_options = options;
27+
_time = time;
28+
}
29+
30+
/// <summary>Current UTC time aligned to the database.</summary>
31+
/// <param name="cancellationToken">Cancellation token.</param>
32+
public async ValueTask<DateTimeOffset> GetUtcNowAsync(CancellationToken cancellationToken = default)
33+
{
34+
if (!_options.UseDatabaseClock)
35+
{
36+
return _time.GetUtcNow();
37+
}
38+
39+
var syncedAt = Volatile.Read(ref _syncedAt);
40+
if (syncedAt == 0 || Stopwatch.GetElapsedTime(syncedAt) > SyncInterval)
41+
{
42+
await SyncAsync(cancellationToken).ConfigureAwait(false);
43+
}
44+
45+
return _time.GetUtcNow() + _offset;
46+
}
47+
48+
private async Task SyncAsync(CancellationToken cancellationToken)
49+
{
50+
await _sync.WaitAsync(cancellationToken).ConfigureAwait(false);
51+
try
52+
{
53+
if (_syncedAt != 0 && Stopwatch.GetElapsedTime(_syncedAt) <= SyncInterval)
54+
{
55+
return;
56+
}
57+
58+
var connection = await _dataSource.DataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
59+
await using (connection.ConfigureAwait(false))
60+
{
61+
var command = new NpgsqlCommand("SELECT clock_timestamp()", connection);
62+
await using (command.ConfigureAwait(false))
63+
{
64+
var before = _time.GetUtcNow();
65+
var started = Stopwatch.GetTimestamp();
66+
var database = (DateTime)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!;
67+
var local = before + (Stopwatch.GetElapsedTime(started) / 2);
68+
_offset = new DateTimeOffset(DateTime.SpecifyKind(database, DateTimeKind.Utc)) - local;
69+
}
70+
}
71+
72+
Volatile.Write(ref _syncedAt, Stopwatch.GetTimestamp());
73+
}
74+
finally
75+
{
76+
_sync.Release();
77+
}
78+
}
79+
}

0 commit comments

Comments
 (0)