diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3affef9..d72fea1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -41,8 +41,13 @@ jobs:
- name: Build
run: dotnet build --no-restore --configuration Release
- # The persistence tests start PostgreSQL 16 with Testcontainers (Docker); GitHub's Windows runners
- # cannot run Linux containers, so Windows runs the Core tests only.
+ # weasyprint (CS-2 fixture, html-para-pdf) renders with pango.
+ - name: Install pango
+ if: runner.os == 'Linux'
+ run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz0b fonts-dejavu-core
+
+ # The persistence, queue and worker tests start PostgreSQL 16, RabbitMQ 3.13 and Mailpit with Testcontainers
+ # (Docker); GitHub's Windows runners cannot run Linux containers, so Windows runs the Core tests only.
- name: Test
if: runner.os == 'Linux'
run: dotnet test --no-build --configuration Release
diff --git a/Directory.Packages.props b/Directory.Packages.props
index b891480..d1093a0 100644
--- a/Directory.Packages.props
+++ b/Directory.Packages.props
@@ -3,7 +3,9 @@
true
+
+
@@ -13,13 +15,22 @@
+
+
+
+
+
+
+
+
+
diff --git a/PythonScriptRunner.slnx b/PythonScriptRunner.slnx
index 091dc73..0d7529d 100644
--- a/PythonScriptRunner.slnx
+++ b/PythonScriptRunner.slnx
@@ -1,11 +1,19 @@
+
+
+
+
+
+
+
+
diff --git a/README.md b/README.md
index aeea566..b328f0d 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,8 @@ com contrato de entrada/saída, trilhas (**Pipelines**), execução assíncrona
cron e painel Vue opcional.
> Projeto em desenvolvimento. Esta versão entrega o **Core Runner** (execução síncrona de Stages e de
-> trilhas), o SDK Python **`psrunner`** e a **persistência em PostgreSQL** (catálogo versionado e histórico).
+> trilhas), o SDK Python **`psrunner`**, a **persistência em PostgreSQL** (catálogo versionado e histórico) e a
+> **execução assíncrona** com RabbitMQ, worker e agendamento cron.
## Como funciona
@@ -120,11 +121,45 @@ await app.Services.MigratePythonScriptRunnerAsync(); // explícito; ou Generat
um job de manutenção cria partições à frente e remove as antigas. Uma trilha pode encurtar a retenção.
- **Auditoria:** ações humanas (`Actor.User(...)`) ficam em `psr.audit_events`; disparos automáticos ficam na própria execução.
+## Fila, worker e agendamento (RabbitMQ)
+
+Com `PythonScriptRunner.RabbitMQ` e `PythonScriptRunner.Worker`, execuções podem ser enfileiradas. Cada passo é uma
+mensagem: o worker assume o passo com um *lease*, executa o processo Python e grava o resultado e a mensagem do próximo
+passo na mesma transação (outbox no PostgreSQL). Enfileirar funciona mesmo com o broker fora do ar.
+
+```csharp
+builder.Services
+ .AddPythonScriptRunner()
+ .AddEntityFrameworkCore(o => o.UseNpgsql(builder.Configuration.GetConnectionString("psr")!))
+ .AddRabbitMq(o => o.ConnectionString = builder.Configuration.GetConnectionString("rabbitmq")) // RabbitMQ 3.13+
+ .AddWorker(o => o.MaxConcurrency = 8) // só nos processos que executam passos
+ .AddRunEventHandler(s => s.Subscription("erp").On("run.succeeded.calculo-progressao", "run.failed.*"));
+builder.Services.AddHealthChecks().AddPythonScriptRunnerChecks(); // banco, broker e uv
+
+// ...
+RunHandle run = await queue.EnqueuePipelineAsync("calculo-progressao", new { a1 = 1, razao = 2, n = 4, coeficientes = new[] { 1, 0, 1 } });
+```
+
+- **Confiabilidade:** retry automático por Stage com atraso exponencial; worker perdido é detectado pelo *lease* e o
+ passo é retomado (Stage idempotente ou com retry restante) ou falha com `WorkerLost` para reprocesso manual;
+ mensagens duplicadas não executam o passo duas vezes.
+- **Concorrência:** `MaxConcurrency` por worker e `max_concurrent_runs` por trilha (as demais esperam como `Pending`, em ordem).
+- **Cancelamento:** `IRunManager.CancelAsync` encerra a árvore de processos no worker que executa o passo.
+- **Agendamento:** `ICatalogAdmin.CreateScheduleAsync` com expressão cron e fuso IANA (padrão `America/Sao_Paulo`);
+ ocorrências perdidas viram uma execução de recuperação; vários workers nunca disparam a mesma ocorrência.
+- **Sistemas externos:** o comando público `StartPipelineRun` (exchange `psr.public`) inicia trilhas e os eventos
+ `run.*`/`step.*` (exchange `psr.events`) informam o andamento e as saídas. Contratos em `schemas/messages`.
+- **Observabilidade:** `ActivitySource` e `Meter` `PythonScriptRunner`; o `traceparent` acompanha a execução até o worker.
+
+O host `hosts/PythonScriptRunner.WorkerHost` é um worker pronto, configurado por variáveis de ambiente
+(`ConnectionStrings__psr`, `ConnectionStrings__rabbitmq`, `PythonScriptRunner__Worker__MaxConcurrency`,
+`PythonScriptRunner__Secrets__`, `PSR_AUTO_MIGRATE=true`), com `/health/live`, `/health/ready`, logs JSON e OTLP.
+
## Requisitos
- .NET 10 SDK
- [uv](https://docs.astral.sh/uv/) no PATH (cria e cacheia um ambiente Python por conjunto de dependências)
-- PostgreSQL 16+ apenas para o pacote de persistência (os testes usam Docker/Testcontainers)
+- PostgreSQL 16+ apenas para o pacote de persistência e RabbitMQ 3.13+ para a fila (os testes usam Docker/Testcontainers)
## Estrutura
@@ -132,9 +167,12 @@ await app.Services.MigratePythonScriptRunnerAsync(); // explícito; ou Generat
|-------|----------|
| `src/PythonScriptRunner.Abstractions` | contratos públicos (tipos de campo, `StageDefinition`, `RunResult`, `PsrFile`, interfaces) |
| `src/PythonScriptRunner.Core` | catálogos (código e pasta), validação, ambientes uv, executor de processos, runner |
-| `src/PythonScriptRunner.EntityFrameworkCore` | PostgreSQL: catálogo versionado, histórico, arquivos, idempotência, outbox, auditoria, retenção |
+| `src/PythonScriptRunner.EntityFrameworkCore` | PostgreSQL: catálogo versionado, histórico, arquivos, idempotência, outbox, auditoria, retenção, agendamentos e ciclo das execuções enfileiradas |
+| `src/PythonScriptRunner.RabbitMQ` | topologia, relay do outbox, `IRunQueue`, comando público, eventos e assinaturas |
+| `src/PythonScriptRunner.Worker` | execução dos passos enfileirados, *lease*, retry, cancelamento, cron |
+| `hosts` | `WorkerHost` e `ServiceDefaults` (Serilog, OpenTelemetry, health checks) |
| `python/psrunner` | SDK Python (`@stage`, `Context`, `File`, `StageError`, `psrunner.testing`, CLI) |
-| `schemas` | JSON Schemas do `stage.json` e do `*.pipeline.json` |
+| `schemas` | JSON Schemas do `stage.json`, do `*.pipeline.json` e das mensagens públicas |
| `tests` | testes .NET (unitários e de integração com Python real) e fixtures de trilha |
## Desenvolvimento
@@ -143,6 +181,8 @@ await app.Services.MigratePythonScriptRunnerAsync(); // explícito; ou Generat
dotnet build
dotnet test --project tests/PythonScriptRunner.Core.Tests
dotnet test --project tests/PythonScriptRunner.EntityFrameworkCore.Tests # precisa de Docker
+dotnet test --project tests/PythonScriptRunner.RabbitMQ.Tests # precisa de Docker
+dotnet test --project tests/PythonScriptRunner.Worker.Tests # precisa de Docker
cd python/psrunner
uv sync
diff --git a/hosts/Directory.Build.props b/hosts/Directory.Build.props
new file mode 100644
index 0000000..a8d6751
--- /dev/null
+++ b/hosts/Directory.Build.props
@@ -0,0 +1,11 @@
+
+
+
+
+
+ false
+
+ true
+ $(NoWarn);CS1591
+
+
diff --git a/hosts/PythonScriptRunner.ServiceDefaults/PythonScriptRunner.ServiceDefaults.csproj b/hosts/PythonScriptRunner.ServiceDefaults/PythonScriptRunner.ServiceDefaults.csproj
new file mode 100644
index 0000000..b2fe11a
--- /dev/null
+++ b/hosts/PythonScriptRunner.ServiceDefaults/PythonScriptRunner.ServiceDefaults.csproj
@@ -0,0 +1,20 @@
+
+
+
+ PythonScriptRunner.ServiceDefaults
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/hosts/PythonScriptRunner.ServiceDefaults/ServiceDefaultsExtensions.cs b/hosts/PythonScriptRunner.ServiceDefaults/ServiceDefaultsExtensions.cs
new file mode 100644
index 0000000..6cf5c7a
--- /dev/null
+++ b/hosts/PythonScriptRunner.ServiceDefaults/ServiceDefaultsExtensions.cs
@@ -0,0 +1,65 @@
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Diagnostics.HealthChecks;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using OpenTelemetry;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
+using Serilog;
+using Serilog.Formatting.Compact;
+
+namespace Microsoft.Extensions.Hosting;
+
+///
+/// Defaults of the standalone hosts (spec 009; D-54): Serilog writing JSON to the console, OpenTelemetry traces and metrics of
+/// the PythonScriptRunner source/meter (exported with OTLP when OTEL_EXPORTER_OTLP_ENDPOINT is set) and the
+/// /health/live and /health/ready endpoints.
+///
+public static class ServiceDefaultsExtensions
+{
+ /// Tag of the checks behind /health/live.
+ public const string LiveTag = "live";
+
+ /// Tag of the checks behind /health/ready.
+ public const string ReadyTag = "ready";
+
+ /// Adds logging, telemetry and the liveness check.
+ /// Host builder.
+ /// Service name in logs and telemetry.
+ public static TBuilder AddServiceDefaults(this TBuilder builder, string serviceName)
+ where TBuilder : IHostApplicationBuilder
+ {
+ ArgumentNullException.ThrowIfNull(builder);
+ builder.Services.AddSerilog((services, logger) => logger
+ .ReadFrom.Configuration(builder.Configuration)
+ .Enrich.FromLogContext()
+ .Enrich.WithProperty("service", serviceName)
+ .WriteTo.Console(new RenderedCompactJsonFormatter()));
+
+ var telemetry = builder.Services.AddOpenTelemetry()
+ .ConfigureResource(resource => resource.AddService(serviceName))
+ .WithTracing(tracing => tracing.AddSource("PythonScriptRunner").AddAspNetCoreInstrumentation())
+ .WithMetrics(metrics => metrics.AddMeter("PythonScriptRunner").AddAspNetCoreInstrumentation().AddRuntimeInstrumentation());
+ if (!string.IsNullOrWhiteSpace(builder.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"]))
+ {
+ telemetry.UseOtlpExporter();
+ }
+
+ builder.Services.AddHealthChecks().AddCheck("self", () => HealthCheckResult.Healthy(), [LiveTag]);
+ return builder;
+ }
+
+ ///
+ /// Maps /health/live (the process answers) and /health/ready (checks tagged : 200 when
+ /// healthy or degraded, 503 when unhealthy).
+ ///
+ /// Application.
+ public static WebApplication MapDefaultEndpoints(this WebApplication app)
+ {
+ ArgumentNullException.ThrowIfNull(app);
+ app.MapHealthChecks("/health/live", new HealthCheckOptions { Predicate = check => check.Tags.Contains(LiveTag) });
+ app.MapHealthChecks("/health/ready", new HealthCheckOptions { Predicate = check => check.Tags.Contains(ReadyTag) });
+ return app;
+ }
+}
diff --git a/hosts/PythonScriptRunner.WorkerHost/Program.cs b/hosts/PythonScriptRunner.WorkerHost/Program.cs
new file mode 100644
index 0000000..99937a6
--- /dev/null
+++ b/hosts/PythonScriptRunner.WorkerHost/Program.cs
@@ -0,0 +1,36 @@
+// Standalone worker of PythonScriptRunner (spec 009): runs queued steps, cron schedules and the maintenance job.
+// Configuration comes from environment variables, e.g. ConnectionStrings__psr, ConnectionStrings__rabbitmq,
+// PythonScriptRunner__Worker__MaxConcurrency, PythonScriptRunner__Secrets__ and PSR_AUTO_MIGRATE.
+var builder = WebApplication.CreateBuilder(args);
+builder.AddServiceDefaults("psr-worker");
+
+var configuration = builder.Configuration;
+builder.Services
+ .AddPythonScriptRunner(o => configuration.GetSection("PythonScriptRunner").Bind(o))
+ .AddEntityFrameworkCore(o =>
+ {
+ configuration.GetSection("PythonScriptRunner:EntityFrameworkCore").Bind(o);
+ o.UseNpgsql(configuration.GetConnectionString("psr")
+ ?? throw new InvalidOperationException("Set the PostgreSQL connection string ConnectionStrings__psr."));
+ })
+ .AddRabbitMq(o =>
+ {
+ configuration.GetSection("PythonScriptRunner:RabbitMQ").Bind(o);
+ o.ConnectionString = configuration.GetConnectionString("rabbitmq")
+ ?? throw new InvalidOperationException("Set the RabbitMQ connection string ConnectionStrings__rabbitmq.");
+ })
+ .AddWorker(o => configuration.GetSection("PythonScriptRunner:Worker").Bind(o));
+builder.Services.AddHealthChecks().AddPythonScriptRunnerChecks();
+
+var app = builder.Build();
+if (configuration.GetValue("PSR_AUTO_MIGRATE"))
+{
+ // EF Core serializes concurrent migrations with a PostgreSQL advisory lock, so replicas can start together (spec 009, AC-8).
+ await app.Services.MigratePythonScriptRunnerAsync();
+}
+
+app.MapDefaultEndpoints();
+await app.RunAsync();
+
+/// Entry point (public for the integration tests).
+public partial class Program;
diff --git a/hosts/PythonScriptRunner.WorkerHost/PythonScriptRunner.WorkerHost.csproj b/hosts/PythonScriptRunner.WorkerHost/PythonScriptRunner.WorkerHost.csproj
new file mode 100644
index 0000000..5334549
--- /dev/null
+++ b/hosts/PythonScriptRunner.WorkerHost/PythonScriptRunner.WorkerHost.csproj
@@ -0,0 +1,12 @@
+
+
+
+ PythonScriptRunner.WorkerHost
+
+
+
+
+
+
+
+
diff --git a/schemas/messages/run-event.v1.json b/schemas/messages/run-event.v1.json
new file mode 100644
index 0000000..8655b82
--- /dev/null
+++ b/schemas/messages/run-event.v1.json
@@ -0,0 +1,62 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://raw.githubusercontent.com/net0well/PythonScriptRunner/main/schemas/messages/run-event.v1.json",
+ "title": "PythonScriptRunner run event (psr.run-event.v1)",
+ "description": "Published on the topic exchange psr.events with the routing key . or .stage: (AMQP type psr.run-event.v1). Fields are only added within v1; consumers must ignore unknown fields.",
+ "type": "object",
+ "required": ["eventId", "type", "occurredAt"],
+ "properties": {
+ "eventId": { "type": "string", "format": "uuid" },
+ "type": {
+ "enum": [
+ "run.queued", "run.rejected", "run.started", "step.started", "step.succeeded", "step.retry-scheduled", "step.failed",
+ "run.succeeded", "run.failed", "run.cancelled"
+ ]
+ },
+ "occurredAt": { "type": "string", "format": "date-time" },
+ "runId": {
+ "description": "Null when a request was rejected before a run existed.",
+ "type": ["string", "null"],
+ "format": "uuid"
+ },
+ "kind": { "enum": ["Pipeline", "PipelineTest", "Stage", "StageTest", null] },
+ "pipelineKey": { "type": ["string", "null"] },
+ "pipelineVersion": { "type": ["integer", "null"] },
+ "stageKey": { "type": ["string", "null"] },
+ "stageVersion": { "type": ["integer", "null"] },
+ "correlationId": { "type": ["string", "null"] },
+ "trigger": { "enum": ["Library", "Dashboard", "Schedule", "QueueCommand", null] },
+ "status": { "enum": ["Pending", "Queued", "Running", "Succeeded", "Failed", "Cancelling", "Cancelled", null] },
+ "step": {
+ "description": "The step of step.* events.",
+ "type": ["object", "null"],
+ "required": ["key", "order", "attempt"],
+ "properties": {
+ "key": { "type": "string" },
+ "order": { "type": "integer", "minimum": 1 },
+ "attempt": { "type": "integer", "minimum": 0 }
+ }
+ },
+ "error": {
+ "type": ["object", "null"],
+ "required": ["kind", "message"],
+ "properties": {
+ "kind": { "description": "An error kind of the data mapping (e.g. ScriptError) or Rejected.", "type": "string" },
+ "code": { "type": ["string", "null"] },
+ "message": { "type": "string" },
+ "step": { "type": ["string", "null"] },
+ "reasons": { "description": "Why a request was rejected.", "type": ["array", "null"], "items": { "type": "string" } }
+ }
+ },
+ "outputs": {
+ "description": "Declared outputs of run.succeeded: values inline; files as {\"$artifact\": {id, name, contentType, size}}. Null above 256 KB.",
+ "type": ["object", "null"]
+ },
+ "outputsTruncated": { "type": "boolean" },
+ "retryAt": { "type": ["string", "null"], "format": "date-time" },
+ "triggerDetails": {
+ "description": "Details of the trigger, e.g. scheduleId, scheduledFor and missedOccurrences of a schedule.",
+ "type": ["object", "null"]
+ }
+ }
+}
diff --git a/schemas/messages/start-pipeline-run.v1.json b/schemas/messages/start-pipeline-run.v1.json
new file mode 100644
index 0000000..3d4acf3
--- /dev/null
+++ b/schemas/messages/start-pipeline-run.v1.json
@@ -0,0 +1,41 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://raw.githubusercontent.com/net0well/PythonScriptRunner/main/schemas/messages/start-pipeline-run.v1.json",
+ "title": "PythonScriptRunner StartPipelineRun command (psr.start-pipeline-run.v1)",
+ "description": "Publish to the exchange psr.public with the routing key pipeline.start (AMQP type psr.start-pipeline-run.v1, content type application/json). The run is announced with run.queued, or refused with run.rejected; both carry the correlationId.",
+ "type": "object",
+ "required": ["pipelineKey"],
+ "properties": {
+ "pipelineKey": {
+ "description": "Key of the pipeline.",
+ "type": "string",
+ "minLength": 3,
+ "maxLength": 64,
+ "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"
+ },
+ "version": {
+ "description": "Version to run; null runs the latest published version.",
+ "type": ["integer", "null"],
+ "minimum": 1
+ },
+ "inputs": {
+ "description": "Pipeline inputs in the wire format of the data mapping. Files are not accepted: pipelines with file inputs are rejected.",
+ "type": ["object", "null"]
+ },
+ "idempotencyKey": {
+ "description": "Repeated commands with the same key within 24 hours return the same run. Without it, the AMQP message_id is used.",
+ "type": ["string", "null"],
+ "maxLength": 190
+ },
+ "correlationId": {
+ "description": "Returned in every event of the run.",
+ "type": ["string", "null"],
+ "maxLength": 190
+ },
+ "requestedBy": {
+ "description": "Name of the calling system; the run is triggered by queue:.",
+ "type": ["string", "null"],
+ "maxLength": 190
+ }
+ }
+}
diff --git a/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs b/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs
index a4f7742..e1e4476 100644
--- a/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs
+++ b/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs
@@ -107,6 +107,12 @@ public sealed record RunStarting
/// Deduplication key given by the caller.
public string? IdempotencyKey { get; init; }
+ /// Correlation given by the caller.
+ public string? CorrelationId { get; init; }
+
+ /// W3C trace context of the caller (traceparent).
+ public string? TraceParent { get; init; }
+
/// Validated inputs in wire format (files carry metadata only).
public required JsonObject Inputs { get; init; }
diff --git a/src/PythonScriptRunner.Abstractions/Ids.cs b/src/PythonScriptRunner.Abstractions/Ids.cs
index e935398..8c53214 100644
--- a/src/PythonScriptRunner.Abstractions/Ids.cs
+++ b/src/PythonScriptRunner.Abstractions/Ids.cs
@@ -1,7 +1,10 @@
+using System.Text.Json.Serialization;
+
namespace PythonScriptRunner;
/// Identifier of a run (time-ordered UUID v7). Its timestamp is the creation time of the run.
/// The underlying GUID.
+[JsonConverter(typeof(RunIdJsonConverter))]
public readonly record struct RunId(Guid Value)
{
/// Creates a new time-ordered identifier.
@@ -35,6 +38,7 @@ public readonly record struct AttemptId(Guid Value)
/// Identifier of a step within a run (time-ordered UUID v7).
/// The underlying GUID.
+[JsonConverter(typeof(StepRunIdJsonConverter))]
public readonly record struct StepRunId(Guid Value)
{
/// Creates a new time-ordered identifier.
diff --git a/src/PythonScriptRunner.Abstractions/Management/CatalogAdmin.cs b/src/PythonScriptRunner.Abstractions/Management/CatalogAdmin.cs
index 2f8d846..a2a1839 100644
--- a/src/PythonScriptRunner.Abstractions/Management/CatalogAdmin.cs
+++ b/src/PythonScriptRunner.Abstractions/Management/CatalogAdmin.cs
@@ -238,6 +238,52 @@ Task UpdatePipelineRetentionAsync(
/// Author.
/// Cancellation token.
Task ImportAsync(Stream zip, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Lists schedules ordered by name.
+ /// Pipeline filter and page.
+ /// Cancellation token.
+ Task> ListSchedulesAsync(ScheduleQuery query, CancellationToken cancellationToken = default);
+
+ /// Reads a schedule.
+ /// Schedule identifier.
+ /// Cancellation token.
+ /// The schedule, or null when it does not exist.
+ Task GetScheduleAsync(ScheduleId id, CancellationToken cancellationToken = default);
+
+ ///
+ /// Creates a cron schedule of a pipeline. Validates the cron expression, the IANA time zone, the version (pinned or the latest
+ /// published) and the fixed inputs: SCHEDULE_INVALID, SCHEDULE_FILE_INPUT (pipelines with file inputs cannot be
+ /// scheduled) or .
+ ///
+ /// Pipeline key.
+ /// Schedule settings.
+ /// Author.
+ /// Cancellation token.
+ Task CreateScheduleAsync(string pipelineKey, ScheduleData schedule, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Changes a schedule (same validation as ); the next run is recomputed.
+ /// Schedule identifier.
+ /// New settings.
+ /// Row version read by the caller.
+ /// Author.
+ /// Cancellation token.
+ Task UpdateScheduleAsync(
+ ScheduleId id, ScheduleData schedule, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Enables or disables a schedule; enabling recomputes the next run from now.
+ /// Schedule identifier.
+ /// Whether it fires.
+ /// Row version read by the caller.
+ /// Author.
+ /// Cancellation token.
+ Task SetScheduleEnabledAsync(
+ ScheduleId id, bool enabled, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Deletes a schedule (its runs stay in the history).
+ /// Schedule identifier.
+ /// Author.
+ /// Cancellation token.
+ Task DeleteScheduleAsync(ScheduleId id, Actor actor, CancellationToken cancellationToken = default);
}
/// Search and page of a catalog list.
diff --git a/src/PythonScriptRunner.Abstractions/Management/DomainErrors.cs b/src/PythonScriptRunner.Abstractions/Management/DomainErrors.cs
index 6997f2d..2f1baa9 100644
--- a/src/PythonScriptRunner.Abstractions/Management/DomainErrors.cs
+++ b/src/PythonScriptRunner.Abstractions/Management/DomainErrors.cs
@@ -56,6 +56,18 @@ public static class PsrErrorCodes
/// The imported archive is not a valid catalog folder.
public const string ImportInvalid = "IMPORT_INVALID";
+
+ /// The schedule does not exist.
+ public const string ScheduleNotFound = "SCHEDULE_NOT_FOUND";
+
+ /// The schedule has an invalid cron expression, time zone or version.
+ public const string ScheduleInvalid = "SCHEDULE_INVALID";
+
+ /// Pipelines with file inputs cannot be scheduled nor started by the public queue command (D-89).
+ public const string ScheduleFileInput = "SCHEDULE_FILE_INPUT";
+
+ /// The pipeline has file inputs, which the public queue command cannot send (D-89).
+ public const string FileInputNotAllowed = "FILE_INPUT_NOT_ALLOWED";
}
/// A business rule rejected the operation; is stable (see ).
diff --git a/src/PythonScriptRunner.Abstractions/Management/RunManagement.cs b/src/PythonScriptRunner.Abstractions/Management/RunManagement.cs
index efff25f..1e954c1 100644
--- a/src/PythonScriptRunner.Abstractions/Management/RunManagement.cs
+++ b/src/PythonScriptRunner.Abstractions/Management/RunManagement.cs
@@ -38,7 +38,10 @@ public interface IRunManager
///
/// Run identifier.
/// Who reprocesses.
- /// runs it now in this process; needs a run queue.
+ ///
+ /// runs it now in this process; queues the failed step for the
+ /// workers (requires the RabbitMQ package).
+ ///
/// Cancellation token (cancels a synchronous reprocess).
/// The run after reprocessing (sync) or after it was queued (async).
Task ReprocessAsync(RunId runId, Actor actor, ExecutionMode mode = ExecutionMode.Async, CancellationToken cancellationToken = default);
@@ -141,6 +144,15 @@ public sealed record RunSnapshot
/// Idempotency key given by the caller.
public string? IdempotencyKey { get; init; }
+ /// Correlation given by the caller (returned in the events).
+ public string? CorrelationId { get; init; }
+
+ /// Schedule that fired the run.
+ public ScheduleId? ScheduleId { get; init; }
+
+ /// Details of the trigger, e.g. {"scheduledFor": "...", "missedOccurrences": 10} for a schedule recovery.
+ public JsonObject? TriggerDetails { get; init; }
+
/// Creation time.
public required DateTimeOffset CreatedAt { get; init; }
diff --git a/src/PythonScriptRunner.Abstractions/Management/Schedules.cs b/src/PythonScriptRunner.Abstractions/Management/Schedules.cs
new file mode 100644
index 0000000..9b1c789
--- /dev/null
+++ b/src/PythonScriptRunner.Abstractions/Management/Schedules.cs
@@ -0,0 +1,65 @@
+using System.Text.Json.Nodes;
+
+namespace PythonScriptRunner;
+
+/// Settings of a cron schedule of a pipeline.
+public sealed record ScheduleData
+{
+ /// Default IANA time zone of schedules.
+ public const string DefaultTimeZone = "America/Sao_Paulo";
+
+ /// Display name (1-200 chars).
+ public required string Name { get; init; }
+
+ /// Cron expression with 5 fields (minute precision) or 6 fields (with seconds).
+ public required string Cron { get; init; }
+
+ /// IANA time zone in which the expression is evaluated (daylight saving time handled).
+ public string TimeZone { get; init; } = DefaultTimeZone;
+
+ /// Version to run; null runs the latest published version at each firing.
+ public int? PinnedVersion { get; init; }
+
+ /// Fixed inputs of the pipeline (no files).
+ public JsonObject Inputs { get; init; } = [];
+
+ /// Whether the schedule fires.
+ public bool IsEnabled { get; init; } = true;
+}
+
+/// A stored schedule.
+public sealed record ScheduleDetail
+{
+ /// Identifier.
+ public required ScheduleId Id { get; init; }
+
+ /// Scheduled pipeline.
+ public required string PipelineKey { get; init; }
+
+ /// Settings.
+ public required ScheduleData Data { get; init; }
+
+ /// Next firing (UTC); null when disabled.
+ public DateTimeOffset? NextRunAt { get; init; }
+
+ /// Last firing.
+ public DateTimeOffset? LastFiredAt { get; init; }
+
+ /// Run created by the last successful firing.
+ public RunId? LastRunId { get; init; }
+
+ /// Why the last firing was rejected (cleared by the next successful one).
+ public string? LastError { get; init; }
+
+ /// Row version for optimistic concurrency.
+ public required uint RowVersion { get; init; }
+
+ /// Who created and last changed it.
+ public required Authorship Authorship { get; init; }
+}
+
+/// Filter and page of a schedule list.
+/// Only schedules of this pipeline.
+/// 1-based page number.
+/// Items per page (1-100).
+public sealed record ScheduleQuery(string? PipelineKey = null, int PageNumber = 1, int PageSize = 25);
diff --git a/src/PythonScriptRunner.Abstractions/Messaging/Commands.cs b/src/PythonScriptRunner.Abstractions/Messaging/Commands.cs
new file mode 100644
index 0000000..d9961cf
--- /dev/null
+++ b/src/PythonScriptRunner.Abstractions/Messaging/Commands.cs
@@ -0,0 +1,102 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+namespace PythonScriptRunner;
+
+///
+/// Internal command psr.execute-step.v1: execute one attempt of a step. It carries identifiers only; inputs and files
+/// stay in the database.
+///
+/// Run identifier.
+/// Creation time of the run (its partition).
+/// Step identifier.
+/// Attempt number to start.
+public sealed record ExecuteStepCommand(RunId RunId, DateTimeOffset RunCreatedAt, StepRunId StepRunId, int Attempt)
+{
+ /// Why the attempt starts.
+ public AttemptCause Cause { get; init; } = AttemptCause.Initial;
+
+ /// Who started the attempt (the reprocessing user, or the run's caller).
+ public string? TriggeredBy { get; init; }
+}
+
+///
+/// Public command psr.start-pipeline-run.v1 (exchange psr.public): lets systems that are not .NET start a
+/// pipeline. Inputs cannot contain files.
+///
+public sealed record StartPipelineRunCommand
+{
+ /// Pipeline key.
+ public required string PipelineKey { get; init; }
+
+ /// Version to run; null runs the latest published version.
+ public int? Version { get; init; }
+
+ /// Pipeline inputs (no files).
+ public JsonObject? Inputs { get; init; }
+
+ /// Deduplicates repeated commands for 24 hours (default: the AMQP message_id).
+ public string? IdempotencyKey { get; init; }
+
+ /// Returned in every event of the run.
+ public string? CorrelationId { get; init; }
+
+ /// Name of the calling system (the run is triggered by queue:<name>).
+ public string? RequestedBy { get; init; }
+}
+
+/// Control message psr.cancel-run.v1 (fanout to every instance): stop the processes of a run.
+/// Run identifier.
+/// Running step, when known.
+/// Who cancelled.
+public sealed record CancelRunCommand(RunId RunId, StepRunId? StepRunId, string RequestedBy);
+
+/// Versioned message types (AMQP type property).
+public static class PsrMessageTypes
+{
+ /// .
+ public const string ExecuteStep = "psr.execute-step.v1";
+
+ /// .
+ public const string StartPipelineRun = "psr.start-pipeline-run.v1";
+
+ /// .
+ public const string CancelRun = "psr.cancel-run.v1";
+
+ /// .
+ public const string RunEvent = "psr.run-event.v1";
+}
+
+/// JSON settings of the messages: camelCase properties and enum names as strings.
+public static class PsrMessageJson
+{
+ /// Shared options (do not modify).
+ public static JsonSerializerOptions Options { get; } = CreateOptions();
+
+ private static JsonSerializerOptions CreateOptions()
+ {
+ var options = new JsonSerializerOptions(JsonSerializerDefaults.Web)
+ {
+ Converters = { new JsonStringEnumConverter() },
+ };
+ options.MakeReadOnly(populateMissingResolver: true);
+ return options;
+ }
+}
+
+/// Writes as its GUID text.
+internal sealed class RunIdJsonConverter : JsonConverter
+{
+ public override RunId Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => new(reader.GetGuid());
+
+ public override void Write(Utf8JsonWriter writer, RunId value, JsonSerializerOptions options) => writer.WriteStringValue(value.Value);
+}
+
+/// Writes as its GUID text.
+internal sealed class StepRunIdJsonConverter : JsonConverter
+{
+ public override StepRunId Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => new(reader.GetGuid());
+
+ public override void Write(Utf8JsonWriter writer, StepRunId value, JsonSerializerOptions options) => writer.WriteStringValue(value.Value);
+}
diff --git a/src/PythonScriptRunner.Abstractions/Messaging/RunEvents.cs b/src/PythonScriptRunner.Abstractions/Messaging/RunEvents.cs
new file mode 100644
index 0000000..45e07e0
--- /dev/null
+++ b/src/PythonScriptRunner.Abstractions/Messaging/RunEvents.cs
@@ -0,0 +1,137 @@
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
+namespace PythonScriptRunner;
+
+///
+/// Event of a run (psr.run-event.v1), published on the psr.events exchange with the routing key
+/// <type>.<pipeline key> or <type>.stage:<stage key>. Fields are only added within v1.
+///
+public sealed record RunEvent
+{
+ /// Unique event identifier.
+ public required Guid EventId { get; init; }
+
+ /// Event type (see ).
+ public required string Type { get; init; }
+
+ /// When it happened.
+ public required DateTimeOffset OccurredAt { get; init; }
+
+ /// The run (null when a request was rejected before a run existed).
+ public RunId? RunId { get; init; }
+
+ /// What the run executes.
+ public RunKind? Kind { get; init; }
+
+ /// Pipeline key, for pipeline runs (and rejected pipeline requests).
+ public string? PipelineKey { get; init; }
+
+ /// Pipeline version, for pipeline runs.
+ public int? PipelineVersion { get; init; }
+
+ /// Stage key, for single-stage runs.
+ public string? StageKey { get; init; }
+
+ /// Stage version, for single-stage runs.
+ public int? StageVersion { get; init; }
+
+ /// Correlation given by the caller (e.g. the correlationId of StartPipelineRun).
+ public string? CorrelationId { get; init; }
+
+ /// How the run started.
+ public RunTrigger? Trigger { get; init; }
+
+ /// Status of the run after the event.
+ public RunStatus? Status { get; init; }
+
+ /// The step, for step.* events.
+ public RunEventStep? Step { get; init; }
+
+ /// Failure (step.failed, run.failed, run.rejected).
+ public RunEventError? Error { get; init; }
+
+ ///
+ /// Declared outputs, for run.succeeded: values inline, files as {"$artifact": {id, name, contentType, size}}.
+ /// null when larger than 256 KB (see ).
+ ///
+ public JsonObject? Outputs { get; init; }
+
+ /// Whether was omitted because it was too large.
+ public bool OutputsTruncated { get; init; }
+
+ /// When the next attempt starts, for step.retry-scheduled.
+ public DateTimeOffset? RetryAt { get; init; }
+
+ /// Details of the trigger (schedule id, scheduled time, missed occurrences).
+ public JsonObject? TriggerDetails { get; init; }
+
+ /// The target used in the routing key: the pipeline key or stage:<stage key>.
+ [JsonIgnore]
+ public string RoutingTarget => PipelineKey ?? (StageKey is { } stage ? $"stage:{stage}" : "unknown");
+}
+
+/// The step of a step.* event.
+/// Step key.
+/// 1-based position.
+/// Attempt number.
+public sealed record RunEventStep(string Key, int Order, int Attempt);
+
+/// Failure carried by an event (no traceback).
+/// Error kind (an name, or Rejected).
+/// Stable code.
+/// Human-readable message.
+/// Key of the step that failed.
+public sealed record RunEventError(string Kind, string? Code, string Message, string? Step)
+{
+ /// Why a request was rejected (run.rejected).
+ public IReadOnlyList? Reasons { get; init; }
+
+ /// Converts a run error.
+ /// The error.
+ public static RunEventError? From(RunError? error) =>
+ error is null ? null : new(error.Kind.ToString(), error.Code, error.Message, error.StepKey);
+}
+
+/// Types of .
+public static class RunEventTypes
+{
+ /// A run was queued (or a duplicate request returned it).
+ public const string RunQueued = "run.queued";
+
+ /// A queue command or a schedule was rejected; no run was created.
+ public const string RunRejected = "run.rejected";
+
+ /// The first step of a run started.
+ public const string RunStarted = "run.started";
+
+ /// An attempt of a step started.
+ public const string StepStarted = "step.started";
+
+ /// A step succeeded.
+ public const string StepSucceeded = "step.succeeded";
+
+ /// A step failed and will be retried automatically.
+ public const string StepRetryScheduled = "step.retry-scheduled";
+
+ /// A step failed for good.
+ public const string StepFailed = "step.failed";
+
+ /// A run succeeded.
+ public const string RunSucceeded = "run.succeeded";
+
+ /// A run failed.
+ public const string RunFailed = "run.failed";
+
+ /// A run was cancelled.
+ public const string RunCancelled = "run.cancelled";
+}
+
+/// Receives run events from a RabbitMQ subscription (registered with AddRunEventHandler).
+public interface IRunEventHandler
+{
+ /// Handles one event. Throwing makes the broker deliver it again (up to the delivery limit).
+ /// The event.
+ /// Cancellation token.
+ Task HandleAsync(RunEvent runEvent, CancellationToken cancellationToken);
+}
diff --git a/src/PythonScriptRunner.Abstractions/Messaging/RunQueue.cs b/src/PythonScriptRunner.Abstractions/Messaging/RunQueue.cs
new file mode 100644
index 0000000..713b4e9
--- /dev/null
+++ b/src/PythonScriptRunner.Abstractions/Messaging/RunQueue.cs
@@ -0,0 +1,36 @@
+namespace PythonScriptRunner;
+
+///
+/// Queues runs for the workers (requires the persistence and RabbitMQ packages). The run, its first step and the message
+/// are stored in one transaction, so a run is queued even while the broker is down.
+///
+public interface IRunQueue
+{
+ /// Queues a published pipeline.
+ /// Pipeline key.
+ /// Pipeline inputs (same forms as ; files are stored in the database).
+ /// Version, idempotency key, correlation and caller identity.
+ /// Cancellation token.
+ /// The run: , or when the pipeline is at its concurrency limit.
+ /// The pipeline does not exist.
+ /// The requested version does not exist.
+ /// The inputs do not match the contract (no run is created).
+ Task EnqueuePipelineAsync(string pipelineKey, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default);
+
+ /// Queues a published stage as a single-step run.
+ /// Stage key.
+ /// Stage inputs.
+ /// Version, idempotency key, correlation and caller identity.
+ /// Cancellation token.
+ /// The queued run.
+ /// The stage does not exist.
+ /// The requested version does not exist.
+ /// The inputs do not match the contract (no run is created).
+ Task EnqueueStageAsync(string stageKey, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default);
+}
+
+/// A queued run.
+/// Run identifier.
+/// or (the current status when deduplicated).
+/// Whether an earlier run with the same idempotency key was returned.
+public sealed record RunHandle(RunId RunId, RunStatus Status, bool Deduplicated);
diff --git a/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs b/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs
index 2fc069c..c84ac5b 100644
--- a/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs
+++ b/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs
@@ -42,4 +42,7 @@ public sealed record RunOptions
/// How the run was started. Default: .
public RunTrigger Trigger { get; init; } = RunTrigger.Library;
+
+ /// Correlation of the caller, stored with the run and returned in its events (requires persistence).
+ public string? CorrelationId { get; init; }
}
diff --git a/src/PythonScriptRunner.Abstractions/Running/IRunEngine.cs b/src/PythonScriptRunner.Abstractions/Running/IRunEngine.cs
index 92a03e7..3d931fb 100644
--- a/src/PythonScriptRunner.Abstractions/Running/IRunEngine.cs
+++ b/src/PythonScriptRunner.Abstractions/Running/IRunEngine.cs
@@ -37,10 +37,123 @@ Task RunPipelineAsync(
/// Cancels the run.
Task ResumeAsync(ResumeRequest request, CancellationToken cancellationToken = default);
- /// Cancels a run that is executing in this process.
+ /// Cancels a run that is executing in this process (including a step started by ).
/// Run identifier.
/// Whether the run was executing here.
bool RequestCancellation(RunId runId);
+
+ /// Validates inputs against declared fields and converts them to the wire format, without running anything.
+ /// Declared inputs (of a stage or a pipeline).
+ /// Inputs (same forms as ).
+ /// Wire values (files carry metadata only) and the files by field name.
+ /// The inputs do not match the contract.
+ BoundRunInputs BindInputs(IReadOnlyList fields, object? inputs);
+
+ ///
+ /// Executes one attempt of one step of a run whose state is kept by the caller (the asynchronous worker): resolves the
+ /// bindings, downloads the files it needs, runs the Python process and validates the outputs. Nothing is recorded and
+ /// there is no automatic retry: the caller stores the result and decides what comes next. Dispose the result to delete
+ /// the local files.
+ ///
+ /// The step, its run and the outputs of the steps already completed.
+ /// Cancels the attempt (the process tree is stopped and the step ends Cancelled).
+ Task ExecuteStepAsync(StepExecutionRequest request, CancellationToken cancellationToken = default);
+
+ /// Removes work folders left behind by interrupted runs (older than OrphanRetention).
+ /// How many folders were removed.
+ int RemoveOrphanWorkFolders();
+}
+
+/// Validated inputs.
+/// Inputs in wire format; file values carry metadata only.
+/// Files by field name (in list order for list<file>).
+public sealed record BoundRunInputs(JsonObject Wire, IReadOnlyDictionary> Files);
+
+/// One attempt of one step to execute with .
+public sealed record StepExecutionRequest
+{
+ /// Run identifier.
+ public required RunId RunId { get; init; }
+
+ /// What the run executes.
+ public required RunKind Kind { get; init; }
+
+ /// Exact stage version of the step (any status).
+ public required StageDefinition Stage { get; init; }
+
+ /// The pipeline, for pipeline runs (its bindings give the step inputs); null for single-stage runs.
+ public PipelineDefinition? Pipeline { get; init; }
+
+ /// 1-based position of the step.
+ public required int Order { get; init; }
+
+ /// Validated inputs of the run in wire format.
+ public required JsonObject RunInputs { get; init; }
+
+ /// Input files of the run by field name.
+ public IReadOnlyDictionary> RunInputFiles { get; init; } = new Dictionary>();
+
+ /// Earlier steps; the succeeded ones provide outputs and files to the bindings.
+ public IReadOnlyList CompletedSteps { get; init; } = [];
+
+ /// Identifier of the attempt (already recorded by the caller).
+ public required AttemptId AttemptId { get; init; }
+
+ /// 1-based attempt number.
+ public required int Attempt { get; init; }
+
+ /// Why the attempt starts.
+ public required AttemptCause Cause { get; init; }
+
+ /// Who started the attempt.
+ public required string TriggeredBy { get; init; }
+}
+
+/// Result of . Dispose it to delete the local files.
+///
+/// The step: holds this attempt only (empty when the bindings could not be resolved).
+///
+/// Resolved step inputs with file metadata; null when the bindings failed.
+/// Input files of the step by field name.
+/// Declared outputs of the run, when this was its last step and it succeeded.
+/// Declared file outputs of the run, under the same condition.
+/// Deletes the local files.
+public sealed class StepExecution(
+ StepResult step,
+ JsonObject? inputs,
+ IReadOnlyDictionary> inputFiles,
+ JsonObject? runOutputs,
+ IReadOnlyDictionary>? runOutputFiles,
+ Func? cleanup = null) : IAsyncDisposable
+{
+ private int _disposed;
+
+ /// The step and its attempt.
+ public StepResult Step { get; } = step;
+
+ /// The attempt, or null when no process ran because the bindings could not be resolved.
+ public AttemptResult? Attempt => Step.Attempts.Count > 0 ? Step.Attempts[^1] : null;
+
+ /// Resolved step inputs with file metadata.
+ public JsonObject? Inputs { get; } = inputs;
+
+ /// Input files of the step by field name.
+ public IReadOnlyDictionary> InputFiles { get; } = inputFiles;
+
+ /// Declared outputs of the run, when this was its last step and it succeeded.
+ public JsonObject? RunOutputs { get; } = runOutputs;
+
+ /// Declared file outputs of the run, when this was its last step and it succeeded.
+ public IReadOnlyDictionary>? RunOutputFiles { get; } = runOutputFiles;
+
+ ///
+ public async ValueTask DisposeAsync()
+ {
+ if (Interlocked.Exchange(ref _disposed, 1) == 0 && cleanup is not null)
+ {
+ await cleanup().ConfigureAwait(false);
+ }
+ }
}
/// State of a failed run to resume.
diff --git a/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj b/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj
index 70f4fef..7395d79 100644
--- a/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj
+++ b/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj
@@ -18,6 +18,8 @@
+
+
diff --git a/src/PythonScriptRunner.Core/Running/PythonRunner.cs b/src/PythonScriptRunner.Core/Running/PythonRunner.cs
index 78ec229..3dd8a0f 100644
--- a/src/PythonScriptRunner.Core/Running/PythonRunner.cs
+++ b/src/PythonScriptRunner.Core/Running/PythonRunner.cs
@@ -185,6 +185,134 @@ public bool RequestCancellation(RunId runId)
}
}
+ public BoundRunInputs BindInputs(IReadOnlyList fields, object? inputs)
+ {
+ ArgumentNullException.ThrowIfNull(fields);
+ var bound = InputBinder.Bind(fields, inputs, Settings.MaxFileBytes);
+ var files = bound.Files.GroupBy(f => f.Field, StringComparer.Ordinal)
+ .ToDictionary(g => g.Key, g => (IReadOnlyList)[.. g.OrderBy(f => f.Index).Select(f => f.File)], StringComparer.Ordinal);
+ return new BoundRunInputs(bound.Wire, files);
+ }
+
+ public int RemoveOrphanWorkFolders() => _janitor.Clean();
+
+ public async Task ExecuteStepAsync(StepExecutionRequest request, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(request);
+ var stage = request.Stage;
+ var settings = Settings;
+ _janitor.EnsureCleaned();
+ var context = new RunContext(new RunWorkspace(settings.WorkRoot, request.RunId), settings, request.Kind, request.TriggeredBy);
+ var stepKey = request.Pipeline is { } definition ? definition.Steps[request.Order - 1].Key : stage.Key;
+ var completed = request.CompletedSteps.Where(s => s.Status == StepStatus.Succeeded).ToDictionary(s => s.StepKey, StringComparer.Ordinal);
+ async ValueTask Cleanup()
+ {
+ context.Workspace.DeleteRun();
+ await context.Workspace.DeleteResultsAsync().ConfigureAwait(false);
+ }
+
+ using var cancellation = Register(request.RunId, cancellationToken);
+ var token = cancellation.Token;
+ try
+ {
+ var resolved = request.Pipeline is { } pipeline
+ ? StepInputResolver.Resolve(stage, pipeline, request.Order - 1, request.RunInputs, request.RunInputFiles, completed)
+ : new StepInputs((JsonObject)request.RunInputs.DeepClone(), request.RunInputFiles, []);
+ if (resolved.Errors.Count > 0)
+ {
+ return new StepExecution(InvalidInputs(stage, stepKey, request.Order, resolved.Errors, prior: null), null, NoFiles, null, null, Cleanup);
+ }
+
+ var startedAt = _time.GetUtcNow();
+ context.UsedBytes = TotalBytes(request.RunInputFiles) + completed.Values.Sum(s => TotalBytes(s.Files));
+ AttemptOutcome outcome;
+ if (context.UsedBytes > settings.MaxRunFilesBytes)
+ {
+ outcome = new AttemptOutcome(
+ NotStarted(request, startedAt, new RunError(ErrorKind.ArtifactLimitExceeded, "The files of the run exceed the run limit.")
+ {
+ Code = "RUN_FILES_TOO_LARGE",
+ }),
+ null,
+ NoFiles);
+ }
+ else
+ {
+ try
+ {
+ var local = await MaterializeStoredAsync(resolved.Files, Path.Combine(context.Workspace.RunDirectory, "step-inputs", stepKey), token)
+ .ConfigureAwait(false);
+ outcome = await RunAttemptAsync(stage, stepKey, request.Attempt, request.Cause, request.AttemptId, startedAt, resolved.Wire,
+ ToCopies(local), context, token).ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (token.IsCancellationRequested)
+ {
+ outcome = new AttemptOutcome(
+ NotStarted(request, startedAt, new RunError(ErrorKind.Cancelled, "The run was cancelled.")) with { Status = AttemptStatus.Cancelled },
+ null,
+ NoFiles);
+ }
+ }
+
+ var attempt = outcome.Attempt;
+ var succeeded = attempt.Status == AttemptStatus.Succeeded;
+ var step = new StepResult
+ {
+ StepKey = stepKey,
+ Order = request.Order,
+ StageKey = stage.Key,
+ StageVersion = stage.Version,
+ Status = attempt.Status switch
+ {
+ AttemptStatus.Succeeded => StepStatus.Succeeded,
+ AttemptStatus.TimedOut => StepStatus.TimedOut,
+ AttemptStatus.Cancelled => StepStatus.Cancelled,
+ _ => StepStatus.Failed,
+ },
+ Attempts = [attempt],
+ Outputs = succeeded ? outcome.Outputs : null,
+ Files = succeeded ? outcome.Files : NoFiles,
+ Error = attempt.Error is null ? null : attempt.Error with { StepKey = stepKey },
+ };
+
+ JsonObject? runOutputs = null;
+ IReadOnlyDictionary>? runFiles = null;
+ if (succeeded && request.Pipeline is null)
+ {
+ (runOutputs, runFiles) = ((JsonObject)step.Outputs!.DeepClone(), step.Files);
+ }
+ else if (succeeded && request.Pipeline is { } last && request.Order == last.Steps.Count)
+ {
+ completed[stepKey] = step;
+ (runOutputs, runFiles) = PipelineOutputs(last, completed);
+ }
+
+ return new StepExecution(step, WithFilePlaceholders(resolved.Wire, resolved.Files, stage), resolved.Files, runOutputs, runFiles, Cleanup);
+ }
+ catch
+ {
+ await Cleanup().ConfigureAwait(false);
+ throw;
+ }
+ finally
+ {
+ _active.TryRemove(request.RunId, out _);
+ }
+ }
+
+ /// An attempt that failed before its process started.
+ private static AttemptResult NotStarted(StepExecutionRequest request, DateTimeOffset startedAt, RunError error) => new()
+ {
+ Id = request.AttemptId,
+ Number = request.Attempt,
+ Cause = request.Cause,
+ Status = AttemptStatus.Failed,
+ StartedAt = startedAt,
+ Duration = TimeSpan.Zero,
+ Overhead = TimeSpan.Zero,
+ Error = error,
+ };
+
public void Dispose()
{
if (_concurrency.IsValueCreated)
@@ -223,6 +351,8 @@ private async Task RunNewAsync(
Trigger = options?.Trigger ?? RunTrigger.Library,
TriggeredBy = context.TriggeredBy,
IdempotencyKey = options?.IdempotencyKey,
+ CorrelationId = options?.CorrelationId,
+ TraceParent = Activity.Current?.Id,
Inputs = (JsonObject)bound.Wire.DeepClone(),
InputFiles = overBudget ? NoFiles : inputFiles,
Steps = target.Steps,
@@ -365,6 +495,14 @@ private async Task ExecutePipelineAsync(
return new RunOutcome(status, steps, null, null, error);
}
+ var (outputs, files) = PipelineOutputs(pipeline, completed);
+ return new RunOutcome(RunStatus.Succeeded, steps, outputs, files, null);
+ }
+
+ /// The declared outputs of a pipeline, read from its completed steps.
+ private static (JsonObject Outputs, Dictionary> Files) PipelineOutputs(
+ PipelineDefinition pipeline, IReadOnlyDictionary completed)
+ {
var outputs = new JsonObject();
var files = new Dictionary>(StringComparer.Ordinal);
foreach (var output in pipeline.Outputs)
@@ -381,7 +519,7 @@ private async Task ExecutePipelineAsync(
}
}
- return new RunOutcome(RunStatus.Succeeded, steps, outputs, files, null);
+ return (outputs, files);
}
private async Task ResolveStageAsync(string key, int? version, CancellationToken cancellationToken)
@@ -450,7 +588,13 @@ await _recorder.StepStartedAsync(
for (var number = first; ; number++)
{
var cause = number == first ? firstCause : AttemptCause.AutoRetry;
- var outcome = await RunAttemptAsync(stage, stepKey, order, number, cause, inputs, copies, context, cancellationToken).ConfigureAwait(false);
+ var attemptId = AttemptId.New();
+ var startedAt = _time.GetUtcNow();
+ await _recorder.AttemptStartedAsync(
+ new AttemptStarted(context.RunId, stepKey, order, attemptId, number, cause, context.TriggeredBy, startedAt),
+ CancellationToken.None).ConfigureAwait(false);
+ var outcome = await RunAttemptAsync(stage, stepKey, number, cause, attemptId, startedAt, inputs, copies, context, cancellationToken)
+ .ConfigureAwait(false);
attempts.Add(outcome.Attempt);
await _recorder.AttemptFinishedAsync(new AttemptFinished(context.RunId, stepKey, order, outcome.Attempt), CancellationToken.None)
.ConfigureAwait(false);
@@ -498,27 +642,24 @@ private async Task RecordStepAsync(RunContext context, StepResult st
return step;
}
+ /// Runs one Python process (the attempt is already reported by the caller when it is recorded).
private async Task RunAttemptAsync(
StageDefinition stage,
string stepKey,
- int order,
int number,
AttemptCause cause,
+ AttemptId attemptId,
+ DateTimeOffset startedAt,
JsonObject inputs,
IReadOnlyList files,
RunContext context,
CancellationToken cancellationToken)
{
var settings = context.Settings;
- var attemptId = AttemptId.New();
- var startedAt = _time.GetUtcNow();
var started = _time.GetTimestamp();
var queued = TimeSpan.Zero;
string? environmentHash = null;
var workDirectory = context.Workspace.AttemptDirectory(stepKey, number);
- await _recorder.AttemptStartedAsync(
- new AttemptStarted(context.RunId, stepKey, order, attemptId, number, cause, context.TriggeredBy, startedAt),
- CancellationToken.None).ConfigureAwait(false);
using var activity = PsrDiagnostics.Source.StartActivity("psr.stage.execute");
activity?.SetTag("psr.stage.key", stage.Key);
diff --git a/src/PythonScriptRunner.Core/Running/RunWorkspace.cs b/src/PythonScriptRunner.Core/Running/RunWorkspace.cs
index d303ac5..a8751e4 100644
--- a/src/PythonScriptRunner.Core/Running/RunWorkspace.cs
+++ b/src/PythonScriptRunner.Core/Running/RunWorkspace.cs
@@ -61,13 +61,19 @@ internal sealed class WorkspaceJanitor(IOptions optio
{
private int _cleaned;
+ /// Cleans once per process (the worker also calls periodically).
public void EnsureCleaned()
{
- if (Interlocked.Exchange(ref _cleaned, 1) == 1)
+ if (Interlocked.Exchange(ref _cleaned, 1) == 0)
{
- return;
+ Clean();
}
+ }
+ /// Removes leftover folders now.
+ /// How many folders were removed.
+ public int Clean()
+ {
var settings = options.Value;
var cutoff = (time.GetUtcNow() - settings.OrphanRetention).UtcDateTime;
var removed = 0;
@@ -93,5 +99,7 @@ public void EnsureCleaned()
{
Log.OrphansRemoved(logger, removed, settings.OrphanRetention);
}
+
+ return removed;
}
}
diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Pipelines.cs b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Pipelines.cs
index d0da7af..e14ea6c 100644
--- a/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Pipelines.cs
+++ b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Pipelines.cs
@@ -235,12 +235,14 @@ public async Task PublishPipelineAsync(
}
var steps = await EfCatalog.LoadStepsAsync(db, row.Id, cancellationToken).ConfigureAwait(false);
- var report = await pipelineValidator.ValidateAsync(CatalogMapping.ToPipeline(row.Pipeline, row, steps), cancellationToken).ConfigureAwait(false);
+ var definition = CatalogMapping.ToPipeline(row.Pipeline, row, steps);
+ var report = await pipelineValidator.ValidateAsync(definition, cancellationToken).ConfigureAwait(false);
if (!report.IsValid)
{
throw new DefinitionInvalidException(PsrErrorCodes.PipelineInvalid, report);
}
+ List warnings = [.. report.Warnings, .. await ScheduleWarningsAsync(db, row.PipelineId, definition, cancellationToken).ConfigureAwait(false)];
var now = time.GetUtcNow();
row.Status = VersionStatus.Published;
row.PublishedAt = now;
@@ -252,7 +254,7 @@ public async Task PublishPipelineAsync(
AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelinePublished, "PipelineVersion", row.Id, $"{key}@{version}"), now);
await SaveAsync(db, cancellationToken, conflict: Current(() => GetPipelineDraftAsync(key, CancellationToken.None))).ConfigureAwait(false);
catalog.InvalidatePipeline(key);
- return new PipelinePublication(CatalogMapping.ToPipelineVersionDetail(row.Pipeline, row, steps), report.Warnings);
+ return new PipelinePublication(CatalogMapping.ToPipelineVersionDetail(row.Pipeline, row, steps), warnings);
}
}
diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Schedules.cs b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Schedules.cs
new file mode 100644
index 0000000..64ec2fe
--- /dev/null
+++ b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Schedules.cs
@@ -0,0 +1,255 @@
+using System.Text.Json.Nodes;
+using Microsoft.EntityFrameworkCore;
+using PythonScriptRunner.EntityFrameworkCore.Entities;
+using PythonScriptRunner.EntityFrameworkCore.Infrastructure;
+using PythonScriptRunner.EntityFrameworkCore.Scheduling;
+
+namespace PythonScriptRunner.EntityFrameworkCore.Catalog;
+
+/// Cron schedules of pipelines (D-41, D-45, D-77, D-89).
+internal sealed partial class EfCatalogAdmin
+{
+ public async Task> ListSchedulesAsync(ScheduleQuery query, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(query);
+ var (pageNumber, pageSize) = (Math.Max(1, query.PageNumber), Math.Clamp(query.PageSize, 1, 100));
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var rows = db.Schedules.AsNoTracking().Join(db.Pipelines, s => s.PipelineId, p => p.Id, (s, p) => new { Schedule = s, p.Key });
+ if (query.PipelineKey is { } pipelineKey)
+ {
+ rows = rows.Where(r => r.Key == pipelineKey);
+ }
+
+ var total = await rows.CountAsync(cancellationToken).ConfigureAwait(false);
+ var items = await rows.OrderBy(r => r.Schedule.Name).ThenBy(r => r.Schedule.Id)
+ .Skip((pageNumber - 1) * pageSize).Take(pageSize)
+ .ToListAsync(cancellationToken).ConfigureAwait(false);
+ return new Page([.. items.Select(r => ToScheduleDetail(r.Schedule, r.Key))], pageNumber, pageSize, total);
+ }
+ }
+
+ public async Task GetScheduleAsync(ScheduleId id, CancellationToken cancellationToken = default)
+ {
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var row = await db.Schedules.AsNoTracking().Join(db.Pipelines, s => s.PipelineId, p => p.Id, (s, p) => new { Schedule = s, p.Key })
+ .SingleOrDefaultAsync(r => r.Schedule.Id == id.Value, cancellationToken).ConfigureAwait(false);
+ return row is null ? null : ToScheduleDetail(row.Schedule, row.Key);
+ }
+ }
+
+ public async Task CreateScheduleAsync(string pipelineKey, ScheduleData schedule, Actor actor, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(schedule);
+ ArgumentNullException.ThrowIfNull(actor);
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var pipeline = await db.Pipelines.AsNoTracking().SingleOrDefaultAsync(p => p.Key == pipelineKey, cancellationToken).ConfigureAwait(false)
+ ?? throw new PipelineNotFoundException(pipelineKey);
+ var cron = await ValidateScheduleAsync(pipelineKey, schedule, cancellationToken).ConfigureAwait(false);
+ var now = time.GetUtcNow();
+ var row = new ScheduleEntity
+ {
+ Id = Guid.CreateVersion7(now),
+ PipelineId = pipeline.Id,
+ Name = schedule.Name,
+ Cron = schedule.Cron.Trim(),
+ TimeZone = schedule.TimeZone,
+ PinnedVersion = schedule.PinnedVersion,
+ Inputs = schedule.Inputs.ToJsonString(),
+ IsEnabled = schedule.IsEnabled,
+ NextRunAt = schedule.IsEnabled ? cron.Next(now) : null,
+ CreatedAt = now,
+ CreatedBy = actor.Subject,
+ UpdatedAt = now,
+ UpdatedBy = actor.Subject,
+ };
+ db.Schedules.Add(row);
+ AuditLog.Add(db, new AuditEntry(actor, AuditActions.ScheduleCreated, "Schedule", row.Id, ScheduleKey(pipelineKey, row.Name), Describe(row)), now);
+ await SaveAsync(db, cancellationToken).ConfigureAwait(false);
+ return ToScheduleDetail(row, pipelineKey);
+ }
+ }
+
+ public async Task UpdateScheduleAsync(
+ ScheduleId id, ScheduleData schedule, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(schedule);
+ ArgumentNullException.ThrowIfNull(actor);
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var (row, pipelineKey) = await LoadScheduleAsync(db, id, expectedRowVersion, cancellationToken).ConfigureAwait(false);
+ var cron = await ValidateScheduleAsync(pipelineKey, schedule, cancellationToken).ConfigureAwait(false);
+ var before = Describe(row);
+ var now = time.GetUtcNow();
+ row.Name = schedule.Name;
+ row.Cron = schedule.Cron.Trim();
+ row.TimeZone = schedule.TimeZone;
+ row.PinnedVersion = schedule.PinnedVersion;
+ row.Inputs = schedule.Inputs.ToJsonString();
+ row.IsEnabled = schedule.IsEnabled;
+ row.NextRunAt = schedule.IsEnabled ? cron.Next(now) : null;
+ row.UpdatedAt = now;
+ row.UpdatedBy = actor.Subject;
+ var after = Describe(row);
+ var changes = new JsonObject();
+ foreach (var (name, value) in after)
+ {
+ if (!JsonNode.DeepEquals(before[name], value))
+ {
+ changes[name] = new JsonArray(before[name]?.DeepClone(), value?.DeepClone());
+ }
+ }
+
+ AuditLog.Add(db, new AuditEntry(actor, AuditActions.ScheduleUpdated, "Schedule", row.Id, ScheduleKey(pipelineKey, row.Name), changes), now);
+ await SaveAsync(db, cancellationToken, conflict: Current(() => GetScheduleAsync(id, CancellationToken.None))).ConfigureAwait(false);
+ return ToScheduleDetail(row, pipelineKey);
+ }
+ }
+
+ public async Task SetScheduleEnabledAsync(
+ ScheduleId id, bool enabled, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(actor);
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var (row, pipelineKey) = await LoadScheduleAsync(db, id, expectedRowVersion, cancellationToken).ConfigureAwait(false);
+ if (row.IsEnabled == enabled)
+ {
+ return ToScheduleDetail(row, pipelineKey);
+ }
+
+ var now = time.GetUtcNow();
+ row.IsEnabled = enabled;
+ row.NextRunAt = enabled ? CronSchedule.Parse(row.Cron, row.TimeZone).Next(now) : null;
+ row.UpdatedAt = now;
+ row.UpdatedBy = actor.Subject;
+ AuditLog.Add(db, new AuditEntry(actor, enabled ? AuditActions.ScheduleEnabled : AuditActions.ScheduleDisabled, "Schedule", row.Id,
+ ScheduleKey(pipelineKey, row.Name)), now);
+ await SaveAsync(db, cancellationToken, conflict: Current(() => GetScheduleAsync(id, CancellationToken.None))).ConfigureAwait(false);
+ return ToScheduleDetail(row, pipelineKey);
+ }
+ }
+
+ public async Task DeleteScheduleAsync(ScheduleId id, Actor actor, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(actor);
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var row = await db.Schedules.Join(db.Pipelines, s => s.PipelineId, p => p.Id, (s, p) => new { Schedule = s, p.Key })
+ .SingleOrDefaultAsync(r => r.Schedule.Id == id.Value, cancellationToken).ConfigureAwait(false)
+ ?? throw ScheduleNotFound(id);
+ db.Schedules.Remove(row.Schedule);
+ AuditLog.Add(db, new AuditEntry(actor, AuditActions.ScheduleDeleted, "Schedule", row.Schedule.Id, ScheduleKey(row.Key, row.Schedule.Name),
+ Describe(row.Schedule)), time.GetUtcNow());
+ await SaveAsync(db, cancellationToken).ConfigureAwait(false);
+ }
+ }
+
+ ///
+ /// Warnings of D-77: enabled schedules that follow the latest version and do not provide a required input of the new one
+ /// (their next firings will be rejected).
+ ///
+ private static async Task> ScheduleWarningsAsync(
+ PsrDbContext db, Guid pipelineId, PipelineDefinition published, CancellationToken cancellationToken)
+ {
+ var schedules = await db.Schedules.AsNoTracking()
+ .Where(s => s.PipelineId == pipelineId && s.IsEnabled && s.PinnedVersion == null)
+ .Select(s => new { s.Name, s.Inputs })
+ .ToListAsync(cancellationToken).ConfigureAwait(false);
+ var warnings = new List();
+ foreach (var schedule in schedules)
+ {
+ var inputs = StoredJson.Object(schedule.Inputs) ?? [];
+ var missing = ScheduleRules.MissingInputs(published, inputs)
+ .Concat(published.Inputs.Where(f => f.Type.IsFile && f.Required).Select(f => f.Name))
+ .Distinct(StringComparer.Ordinal)
+ .ToList();
+ if (missing.Count > 0)
+ {
+ warnings.Add(new ValidationIssue("SCHEDULE_MISSING_INPUT",
+ $"schedule '{schedule.Name}' does not provide the required input(s) {string.Join(", ", missing)}; its firings will be rejected"));
+ }
+ }
+
+ return warnings;
+ }
+
+ /// Validates name, cron, time zone, version and inputs; returns the parsed expression.
+ private async Task ValidateScheduleAsync(string pipelineKey, ScheduleData schedule, CancellationToken cancellationToken)
+ {
+ ValidateName(schedule.Name);
+ ArgumentNullException.ThrowIfNull(schedule.Inputs);
+ var cron = CronSchedule.Parse(schedule.Cron, schedule.TimeZone);
+ if (schedule.PinnedVersion is < 1)
+ {
+ throw new PsrDomainException(PsrErrorCodes.ScheduleInvalid, "The pinned version must be at least 1 (or null for the latest).");
+ }
+
+ var pipeline = await ScheduleRules.ResolveAsync(catalog, pipelineKey, schedule.PinnedVersion, cancellationToken).ConfigureAwait(false);
+ ScheduleRules.EnsureNoFileInputs(pipeline, schedule.Inputs);
+ engine.BindInputs(pipeline.Inputs, schedule.Inputs);
+ return cron;
+ }
+
+ private static async Task<(ScheduleEntity Row, string PipelineKey)> LoadScheduleAsync(
+ PsrDbContext db, ScheduleId id, uint expectedRowVersion, CancellationToken cancellationToken)
+ {
+ var loaded = await db.Schedules.Join(db.Pipelines, s => s.PipelineId, p => p.Id, (s, p) => new { Schedule = s, p.Key })
+ .SingleOrDefaultAsync(r => r.Schedule.Id == id.Value, cancellationToken).ConfigureAwait(false)
+ ?? throw ScheduleNotFound(id);
+ if (loaded.Schedule.RowVersion != expectedRowVersion)
+ {
+ throw new ConcurrencyConflictException($"Schedule {id} was changed by someone else.", ToScheduleDetail(loaded.Schedule, loaded.Key));
+ }
+
+ db.Entry(loaded.Schedule).Property(s => s.RowVersion).OriginalValue = expectedRowVersion;
+ return (loaded.Schedule, loaded.Key);
+ }
+
+ private static PsrDomainException ScheduleNotFound(ScheduleId id) => new(PsrErrorCodes.ScheduleNotFound, $"Schedule {id} was not found.");
+
+ private static string ScheduleKey(string pipelineKey, string name)
+ {
+ var key = $"{pipelineKey}/{name}";
+ return key.Length <= 100 ? key : key[..100];
+ }
+
+ private static JsonObject Describe(ScheduleEntity row) => new()
+ {
+ ["name"] = row.Name,
+ ["cron"] = row.Cron,
+ ["timeZone"] = row.TimeZone,
+ ["pinnedVersion"] = row.PinnedVersion,
+ ["inputs"] = StoredJson.Node(row.Inputs),
+ ["isEnabled"] = row.IsEnabled,
+ };
+
+ private static ScheduleDetail ToScheduleDetail(ScheduleEntity row, string pipelineKey) => new()
+ {
+ Id = new ScheduleId(row.Id),
+ PipelineKey = pipelineKey,
+ Data = new ScheduleData
+ {
+ Name = row.Name,
+ Cron = row.Cron,
+ TimeZone = row.TimeZone,
+ PinnedVersion = row.PinnedVersion,
+ Inputs = StoredJson.Object(row.Inputs) ?? [],
+ IsEnabled = row.IsEnabled,
+ },
+ NextRunAt = row.NextRunAt,
+ LastFiredAt = row.LastFiredAt,
+ LastRunId = row.LastRunId is { } runId ? new RunId(runId) : null,
+ LastError = row.LastError,
+ RowVersion = row.RowVersion,
+ Authorship = new Authorship(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy),
+ };
+}
diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Entities/RunEntities.cs b/src/PythonScriptRunner.EntityFrameworkCore/Entities/RunEntities.cs
index 2da8f17..deb99a7 100644
--- a/src/PythonScriptRunner.EntityFrameworkCore/Entities/RunEntities.cs
+++ b/src/PythonScriptRunner.EntityFrameworkCore/Entities/RunEntities.cs
@@ -33,6 +33,9 @@ public sealed class RunEntity
public Guid? ScheduleId { get; set; }
+ /// Details of the trigger as JSON (e.g. scheduled time and missed occurrences of a schedule recovery).
+ public string? TriggerDetails { get; set; }
+
public RunStatus Status { get; set; }
/// Validated inputs as JSON (files as {"$artifact": id}).
diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/PsrClock.cs b/src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/PsrClock.cs
new file mode 100644
index 0000000..382d6f1
--- /dev/null
+++ b/src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/PsrClock.cs
@@ -0,0 +1,79 @@
+using System.Diagnostics;
+using Npgsql;
+using PythonScriptRunner.EntityFrameworkCore.Runs;
+
+namespace PythonScriptRunner.EntityFrameworkCore;
+
+///
+/// Current time of queued runs: the host clock corrected by its offset to the PostgreSQL clock (measured every minute), so
+/// lease expiry, retry delays and cron decisions agree across hosts (spec 006). With
+/// off it is the registered .
+///
+public sealed class PsrClock
+{
+ private static readonly TimeSpan SyncInterval = TimeSpan.FromMinutes(1);
+
+ private readonly PsrDataSource _dataSource;
+ private readonly PsrEntityFrameworkOptions _options;
+ private readonly TimeProvider _time;
+ private readonly SemaphoreSlim _sync = new(1, 1);
+ private TimeSpan _offset;
+ private long _syncedAt;
+
+ internal PsrClock(PsrDataSource dataSource, PsrEntityFrameworkOptions options, TimeProvider time)
+ {
+ _dataSource = dataSource;
+ _options = options;
+ _time = time;
+ }
+
+ /// Current UTC time aligned to the database.
+ /// Cancellation token.
+ public async ValueTask GetUtcNowAsync(CancellationToken cancellationToken = default)
+ {
+ if (!_options.UseDatabaseClock)
+ {
+ return _time.GetUtcNow();
+ }
+
+ var syncedAt = Volatile.Read(ref _syncedAt);
+ if (syncedAt == 0 || Stopwatch.GetElapsedTime(syncedAt) > SyncInterval)
+ {
+ await SyncAsync(cancellationToken).ConfigureAwait(false);
+ }
+
+ return _time.GetUtcNow() + _offset;
+ }
+
+ private async Task SyncAsync(CancellationToken cancellationToken)
+ {
+ await _sync.WaitAsync(cancellationToken).ConfigureAwait(false);
+ try
+ {
+ if (_syncedAt != 0 && Stopwatch.GetElapsedTime(_syncedAt) <= SyncInterval)
+ {
+ return;
+ }
+
+ var connection = await _dataSource.DataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
+ await using (connection.ConfigureAwait(false))
+ {
+ var command = new NpgsqlCommand("SELECT clock_timestamp()", connection);
+ await using (command.ConfigureAwait(false))
+ {
+ var before = _time.GetUtcNow();
+ var started = Stopwatch.GetTimestamp();
+ var database = (DateTime)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!;
+ var local = before + (Stopwatch.GetElapsedTime(started) / 2);
+ _offset = new DateTimeOffset(DateTime.SpecifyKind(database, DateTimeKind.Utc)) - local;
+ }
+ }
+
+ Volatile.Write(ref _syncedAt, Stopwatch.GetTimestamp());
+ }
+ finally
+ {
+ _sync.Release();
+ }
+ }
+}
diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/PsrOutbox.cs b/src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/PsrOutbox.cs
new file mode 100644
index 0000000..a854b9e
--- /dev/null
+++ b/src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/PsrOutbox.cs
@@ -0,0 +1,119 @@
+using System.Text.Json.Nodes;
+using Microsoft.EntityFrameworkCore;
+
+namespace PythonScriptRunner.EntityFrameworkCore;
+
+///
+/// Reads psr.outbox_messages for the relay (spec 005): batches due now are locked with FOR UPDATE SKIP LOCKED,
+/// so relays in several instances never publish the same row at the same time.
+///
+public sealed class PsrOutbox
+{
+ private static readonly TimeSpan MaxBackoff = TimeSpan.FromSeconds(60);
+
+ private readonly IDbContextFactory _factory;
+ private readonly PsrClock _clock;
+
+ internal PsrOutbox(IDbContextFactory factory, PsrClock clock)
+ {
+ _factory = factory;
+ _clock = clock;
+ }
+
+ ///
+ /// Locks up to due messages, hands them to and records the outcome:
+ /// published rows get published_at; failed ones get attempts + 1, the error and a backoff (1 s doubling up to
+ /// 60 s). When throws (broker down) nothing changes.
+ ///
+ /// Messages per batch.
+ /// Publishes the messages and returns the outcome of each one.
+ /// Cancellation token.
+ /// How many messages were locked.
+ public async Task DispatchAsync(
+ int batchSize,
+ Func, CancellationToken, Task>> publish,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(publish);
+ var db = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var transaction = await db.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
+ await using (transaction.ConfigureAwait(false))
+ {
+ var now = await _clock.GetUtcNowAsync(cancellationToken).ConfigureAwait(false);
+ var rows = await db.OutboxMessages.FromSql($"""
+ SELECT * FROM psr.outbox_messages
+ WHERE published_at IS NULL AND available_at <= {now}
+ ORDER BY available_at
+ LIMIT {batchSize}
+ FOR UPDATE SKIP LOCKED
+ """).AsNoTracking().ToListAsync(cancellationToken).ConfigureAwait(false);
+ if (rows.Count == 0)
+ {
+ return 0;
+ }
+
+ var envelopes = rows.Select(r => new OutboxEnvelope(
+ r.Id, r.MessageType, r.Exchange, r.RoutingKey, r.Payload,
+ (JsonNode.Parse(r.Headers) as JsonObject ?? []).ToDictionary(h => h.Key, h => h.Value?.GetValue() ?? string.Empty),
+ r.AvailableAt, r.Attempts)).ToList();
+ var deliveries = await publish(envelopes, cancellationToken).ConfigureAwait(false);
+
+ var published = deliveries.Where(d => d.Error is null).Select(d => d.Id).ToList();
+ var publishedAt = await _clock.GetUtcNowAsync(cancellationToken).ConfigureAwait(false);
+ await db.OutboxMessages.Where(m => published.Contains(m.Id))
+ .ExecuteUpdateAsync(s => s.SetProperty(m => m.PublishedAt, publishedAt).SetProperty(m => m.LastError, (string?)null), cancellationToken)
+ .ConfigureAwait(false);
+ foreach (var failure in deliveries.Where(d => d.Error is not null))
+ {
+ var attempts = rows.Single(r => r.Id == failure.Id).Attempts + 1;
+ var retryAt = publishedAt + Backoff(attempts);
+ var error = failure.Error!.Length <= 2000 ? failure.Error : failure.Error[..2000];
+ await db.OutboxMessages.Where(m => m.Id == failure.Id)
+ .ExecuteUpdateAsync(s => s
+ .SetProperty(m => m.Attempts, attempts)
+ .SetProperty(m => m.LastError, error)
+ .SetProperty(m => m.AvailableAt, retryAt), cancellationToken).ConfigureAwait(false);
+ }
+
+ await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
+ return rows.Count;
+ }
+ }
+ }
+
+ /// Messages not published yet (gauge psr.outbox.pending).
+ /// Cancellation token.
+ public async Task CountPendingAsync(CancellationToken cancellationToken = default)
+ {
+ var db = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ return await db.OutboxMessages.CountAsync(m => m.PublishedAt == null, cancellationToken).ConfigureAwait(false);
+ }
+ }
+
+ /// Delay before publishing again after failures: 1 s, 2 s, 4 s... up to 60 s.
+ /// Failed publications so far.
+ public static TimeSpan Backoff(int attempts) =>
+ attempts >= 7 ? MaxBackoff : TimeSpan.FromSeconds(Math.Min(MaxBackoff.TotalSeconds, Math.Pow(2, Math.Max(0, attempts - 1))));
+}
+
+/// A message to publish.
+/// Outbox row id (the AMQP message_id).
+/// Versioned type (AMQP type).
+/// Destination exchange.
+/// Routing key.
+/// JSON body.
+/// Headers (e.g. traceparent).
+/// When it became due.
+/// Failed publications so far.
+public sealed record OutboxEnvelope(
+ Guid Id, string MessageType, string Exchange, string RoutingKey, string Payload, IReadOnlyDictionary Headers, DateTimeOffset AvailableAt,
+ int Attempts);
+
+/// Outcome of publishing one message.
+/// Outbox row id.
+/// null when the broker confirmed it; otherwise why it failed.
+public sealed record OutboxDelivery(Guid Id, string? Error);
diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Migrations/20261020143224_AsyncRuns.Designer.cs b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/20261020143224_AsyncRuns.Designer.cs
new file mode 100644
index 0000000..81e1d25
--- /dev/null
+++ b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/20261020143224_AsyncRuns.Designer.cs
@@ -0,0 +1,1351 @@
+//
+using System;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+using PythonScriptRunner.EntityFrameworkCore;
+
+#nullable disable
+
+namespace PythonScriptRunner.EntityFrameworkCore.Migrations
+{
+ [DbContext(typeof(PsrDbContext))]
+ [Migration("20261020143224_AsyncRuns")]
+ partial class AsyncRuns
+ {
+ ///
+ protected override void BuildTargetModel(ModelBuilder modelBuilder)
+ {
+#pragma warning disable 612, 618
+ modelBuilder
+ .HasDefaultSchema("psr")
+ .HasAnnotation("ProductVersion", "10.0.13")
+ .HasAnnotation("Relational:MaxIdentifierLength", 63);
+
+ NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.ArtifactContentEntity", b =>
+ {
+ b.Property("ArtifactId")
+ .HasColumnType("uuid")
+ .HasColumnName("artifact_id");
+
+ b.Property("CreatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("created_at");
+
+ b.Property("Content")
+ .IsRequired()
+ .HasColumnType("bytea")
+ .HasColumnName("content");
+
+ b.HasKey("ArtifactId", "CreatedAt")
+ .HasName("pk_artifact_contents");
+
+ b.ToTable("artifact_contents", "psr");
+ });
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.ArtifactEntity", b =>
+ {
+ b.Property("Id")
+ .HasColumnType("uuid")
+ .HasColumnName("id");
+
+ b.Property("RunCreatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("run_created_at");
+
+ b.Property("ContentCreatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("content_created_at");
+
+ b.Property("ContentDeletedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("content_deleted_at");
+
+ b.Property("ContentExpiresAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("content_expires_at");
+
+ b.Property("ContentType")
+ .IsRequired()
+ .HasMaxLength(255)
+ .HasColumnType("character varying(255)")
+ .HasColumnName("content_type");
+
+ b.Property("Direction")
+ .IsRequired()
+ .HasMaxLength(8)
+ .HasColumnType("character varying(8)")
+ .HasColumnName("direction");
+
+ b.Property("FieldName")
+ .IsRequired()
+ .HasMaxLength(64)
+ .HasColumnType("character varying(64)")
+ .HasColumnName("field_name");
+
+ b.Property("FileName")
+ .IsRequired()
+ .HasMaxLength(255)
+ .HasColumnType("character varying(255)")
+ .HasColumnName("file_name");
+
+ b.Property("ListIndex")
+ .HasColumnType("integer")
+ .HasColumnName("list_index");
+
+ b.Property("RunId")
+ .HasColumnType("uuid")
+ .HasColumnName("run_id");
+
+ b.Property("Sha256")
+ .IsRequired()
+ .HasMaxLength(64)
+ .HasColumnType("character(64)")
+ .HasColumnName("sha256")
+ .IsFixedLength();
+
+ b.Property("SizeBytes")
+ .HasColumnType("bigint")
+ .HasColumnName("size_bytes");
+
+ b.Property("StepRunId")
+ .HasColumnType("uuid")
+ .HasColumnName("step_run_id");
+
+ b.HasKey("Id", "RunCreatedAt")
+ .HasName("pk_artifacts");
+
+ b.HasIndex("ContentExpiresAt")
+ .HasDatabaseName("ix_artifacts_content_expires_at")
+ .HasFilter("content_deleted_at IS NULL");
+
+ b.HasIndex("RunId", "RunCreatedAt")
+ .HasDatabaseName("ix_artifacts_run_id_run_created_at");
+
+ b.HasIndex("StepRunId", "RunCreatedAt")
+ .HasDatabaseName("ix_artifacts_step_run_id_run_created_at");
+
+ b.ToTable("artifacts", "psr", t =>
+ {
+ t.HasCheckConstraint("ck_artifacts_direction", "direction IN ('Input', 'Output')");
+ });
+ });
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.AttemptLogEntity", b =>
+ {
+ b.Property("AttemptId")
+ .HasColumnType("uuid")
+ .HasColumnName("attempt_id");
+
+ b.Property("RunCreatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("run_created_at");
+
+ b.Property("Stream")
+ .HasMaxLength(8)
+ .HasColumnType("character varying(8)")
+ .HasColumnName("stream");
+
+ b.Property("Content")
+ .IsRequired()
+ .HasColumnType("text")
+ .HasColumnName("content");
+
+ b.Property("OriginalBytes")
+ .HasColumnType("bigint")
+ .HasColumnName("original_bytes");
+
+ b.Property("Truncated")
+ .HasColumnType("boolean")
+ .HasColumnName("truncated");
+
+ b.HasKey("AttemptId", "RunCreatedAt", "Stream")
+ .HasName("pk_attempt_logs");
+
+ b.ToTable("attempt_logs", "psr", t =>
+ {
+ t.HasCheckConstraint("ck_attempt_logs_stream", "stream IN ('stdout', 'stderr')");
+ });
+ });
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.AuditEventEntity", b =>
+ {
+ b.Property("Id")
+ .HasColumnType("uuid")
+ .HasColumnName("id");
+
+ b.Property("OccurredAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("occurred_at");
+
+ b.Property("Action")
+ .IsRequired()
+ .HasMaxLength(60)
+ .HasColumnType("character varying(60)")
+ .HasColumnName("action");
+
+ b.Property("Actor")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("actor");
+
+ b.Property("Details")
+ .HasColumnType("jsonb")
+ .HasColumnName("details");
+
+ b.Property("EntityId")
+ .HasColumnType("uuid")
+ .HasColumnName("entity_id");
+
+ b.Property("EntityKey")
+ .HasMaxLength(100)
+ .HasColumnType("character varying(100)")
+ .HasColumnName("entity_key");
+
+ b.Property("EntityType")
+ .IsRequired()
+ .HasMaxLength(30)
+ .HasColumnType("character varying(30)")
+ .HasColumnName("entity_type");
+
+ b.Property("TraceId")
+ .HasMaxLength(32)
+ .HasColumnType("character varying(32)")
+ .HasColumnName("trace_id");
+
+ b.HasKey("Id", "OccurredAt")
+ .HasName("pk_audit_events");
+
+ b.HasIndex("OccurredAt", "Id")
+ .IsDescending()
+ .HasDatabaseName("ix_audit_events_occurred_at_id");
+
+ b.ToTable("audit_events", "psr");
+ });
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.IdempotencyKeyEntity", b =>
+ {
+ b.Property("Scope")
+ .HasMaxLength(80)
+ .HasColumnType("character varying(80)")
+ .HasColumnName("scope");
+
+ b.Property("Key")
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("key");
+
+ b.Property("ExpiresAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("expires_at");
+
+ b.Property("RunCreatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("run_created_at");
+
+ b.Property("RunId")
+ .HasColumnType("uuid")
+ .HasColumnName("run_id");
+
+ b.HasKey("Scope", "Key")
+ .HasName("pk_idempotency_keys");
+
+ b.HasIndex("ExpiresAt")
+ .HasDatabaseName("ix_idempotency_keys_expires_at");
+
+ b.ToTable("idempotency_keys", "psr");
+ });
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.OutboxMessageEntity", b =>
+ {
+ b.Property("Id")
+ .HasColumnType("uuid")
+ .HasColumnName("id");
+
+ b.Property("Attempts")
+ .HasColumnType("integer")
+ .HasColumnName("attempts");
+
+ b.Property("AvailableAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("available_at");
+
+ b.Property("CreatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("created_at");
+
+ b.Property("Exchange")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("exchange");
+
+ b.Property("Headers")
+ .IsRequired()
+ .HasColumnType("jsonb")
+ .HasColumnName("headers");
+
+ b.Property("LastError")
+ .HasColumnType("text")
+ .HasColumnName("last_error");
+
+ b.Property("MessageType")
+ .IsRequired()
+ .HasMaxLength(100)
+ .HasColumnType("character varying(100)")
+ .HasColumnName("message_type");
+
+ b.Property("Payload")
+ .IsRequired()
+ .HasColumnType("jsonb")
+ .HasColumnName("payload");
+
+ b.Property("PublishedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("published_at");
+
+ b.Property("RoutingKey")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("routing_key");
+
+ b.HasKey("Id")
+ .HasName("pk_outbox_messages");
+
+ b.HasIndex("AvailableAt")
+ .HasDatabaseName("ix_outbox_messages_available_at")
+ .HasFilter("published_at IS NULL");
+
+ b.ToTable("outbox_messages", "psr");
+ });
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineEntity", b =>
+ {
+ b.Property("Id")
+ .HasColumnType("uuid")
+ .HasColumnName("id");
+
+ b.Property("ArtifactRetentionDays")
+ .HasColumnType("integer")
+ .HasColumnName("artifact_retention_days");
+
+ b.Property("CreatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("created_at");
+
+ b.Property("CreatedBy")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("created_by");
+
+ b.Property("Description")
+ .HasColumnType("text")
+ .HasColumnName("description");
+
+ b.Property("Key")
+ .IsRequired()
+ .HasMaxLength(64)
+ .HasColumnType("character varying(64)")
+ .HasColumnName("key");
+
+ b.Property("LatestPublishedVersion")
+ .HasColumnType("integer")
+ .HasColumnName("latest_published_version");
+
+ b.Property("MaxConcurrentRuns")
+ .HasColumnType("integer")
+ .HasColumnName("max_concurrent_runs");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("name");
+
+ b.Property("RowVersion")
+ .IsConcurrencyToken()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("xid")
+ .HasColumnName("xmin");
+
+ b.Property("RunRetentionDays")
+ .HasColumnType("integer")
+ .HasColumnName("run_retention_days");
+
+ b.Property("UpdatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("updated_at");
+
+ b.Property("UpdatedBy")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("updated_by");
+
+ b.HasKey("Id")
+ .HasName("pk_pipelines");
+
+ b.HasIndex("Key")
+ .IsUnique()
+ .HasDatabaseName("ux_pipelines_key");
+
+ b.ToTable("pipelines", "psr");
+ });
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineStepEntity", b =>
+ {
+ b.Property("Id")
+ .HasColumnType("uuid")
+ .HasColumnName("id");
+
+ b.Property("Bindings")
+ .IsRequired()
+ .HasColumnType("jsonb")
+ .HasColumnName("bindings");
+
+ b.Property("DisplayName")
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("display_name");
+
+ b.Property("Key")
+ .IsRequired()
+ .HasMaxLength(64)
+ .HasColumnType("character varying(64)")
+ .HasColumnName("key");
+
+ b.Property("Order")
+ .HasColumnType("integer")
+ .HasColumnName("order");
+
+ b.Property("PipelineVersionId")
+ .HasColumnType("uuid")
+ .HasColumnName("pipeline_version_id");
+
+ b.Property("StageVersionId")
+ .HasColumnType("uuid")
+ .HasColumnName("stage_version_id");
+
+ b.HasKey("Id")
+ .HasName("pk_pipeline_steps");
+
+ b.HasIndex("StageVersionId")
+ .HasDatabaseName("ix_pipeline_steps_stage_version_id");
+
+ b.HasIndex("PipelineVersionId", "Key")
+ .IsUnique()
+ .HasDatabaseName("ux_pipeline_steps_pipeline_version_id_key");
+
+ b.HasIndex("PipelineVersionId", "Order")
+ .IsUnique()
+ .HasDatabaseName("ux_pipeline_steps_pipeline_version_id_order");
+
+ b.ToTable("pipeline_steps", "psr");
+ });
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineVersionEntity", b =>
+ {
+ b.Property("Id")
+ .HasColumnType("uuid")
+ .HasColumnName("id");
+
+ b.Property("ArchivedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("archived_at");
+
+ b.Property("ArchivedBy")
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("archived_by");
+
+ b.Property("CreatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("created_at");
+
+ b.Property("CreatedBy")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("created_by");
+
+ b.Property("InputFields")
+ .IsRequired()
+ .HasColumnType("jsonb")
+ .HasColumnName("input_fields");
+
+ b.Property("Layout")
+ .HasColumnType("jsonb")
+ .HasColumnName("layout");
+
+ b.Property("Outputs")
+ .IsRequired()
+ .HasColumnType("jsonb")
+ .HasColumnName("outputs");
+
+ b.Property("PipelineId")
+ .HasColumnType("uuid")
+ .HasColumnName("pipeline_id");
+
+ b.Property("PublishedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("published_at");
+
+ b.Property("PublishedBy")
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("published_by");
+
+ b.Property("RowVersion")
+ .IsConcurrencyToken()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("xid")
+ .HasColumnName("xmin");
+
+ b.Property("Status")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasMaxLength(16)
+ .HasColumnType("character varying(16)")
+ .HasDefaultValue("Draft")
+ .HasColumnName("status");
+
+ b.Property("UpdatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("updated_at");
+
+ b.Property("UpdatedBy")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("updated_by");
+
+ b.Property("Version")
+ .HasColumnType("integer")
+ .HasColumnName("version");
+
+ b.HasKey("Id")
+ .HasName("pk_pipeline_versions");
+
+ b.HasIndex("PipelineId")
+ .IsUnique()
+ .HasDatabaseName("ux_pipeline_versions_draft")
+ .HasFilter("status = 'Draft'");
+
+ b.HasIndex("PipelineId", "Version")
+ .IsUnique()
+ .HasDatabaseName("ux_pipeline_versions_pipeline_id_version");
+
+ b.ToTable("pipeline_versions", "psr", t =>
+ {
+ t.HasCheckConstraint("ck_pipeline_versions_status", "status IN ('Draft', 'Published', 'Archived')");
+ });
+ });
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.RunEntity", b =>
+ {
+ b.Property("Id")
+ .HasColumnType("uuid")
+ .HasColumnName("id");
+
+ b.Property("CreatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("created_at");
+
+ b.Property("CancelRequestedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("cancel_requested_at");
+
+ b.Property("CancelRequestedBy")
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("cancel_requested_by");
+
+ b.Property("CorrelationId")
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("correlation_id");
+
+ b.Property("Error")
+ .HasColumnType("jsonb")
+ .HasColumnName("error");
+
+ b.Property("ExpiresAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("expires_at");
+
+ b.Property("FinishedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("finished_at");
+
+ b.Property("IdempotencyKey")
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("idempotency_key");
+
+ b.Property("Inputs")
+ .IsRequired()
+ .HasColumnType("jsonb")
+ .HasColumnName("inputs");
+
+ b.Property("Kind")
+ .IsRequired()
+ .HasMaxLength(16)
+ .HasColumnType("character varying(16)")
+ .HasColumnName("kind");
+
+ b.Property("Mode")
+ .IsRequired()
+ .HasMaxLength(8)
+ .HasColumnType("character varying(8)")
+ .HasColumnName("mode");
+
+ b.Property("Outputs")
+ .HasColumnType("jsonb")
+ .HasColumnName("outputs");
+
+ b.Property("PipelineId")
+ .HasColumnType("uuid")
+ .HasColumnName("pipeline_id");
+
+ b.Property("PipelineKey")
+ .HasMaxLength(64)
+ .HasColumnType("character varying(64)")
+ .HasColumnName("pipeline_key");
+
+ b.Property("PipelineVersion")
+ .HasColumnType("integer")
+ .HasColumnName("pipeline_version");
+
+ b.Property("PipelineVersionId")
+ .HasColumnType("uuid")
+ .HasColumnName("pipeline_version_id");
+
+ b.Property("QueuedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("queued_at");
+
+ b.Property("ScheduleId")
+ .HasColumnType("uuid")
+ .HasColumnName("schedule_id");
+
+ b.Property("StageKey")
+ .HasMaxLength(64)
+ .HasColumnType("character varying(64)")
+ .HasColumnName("stage_key");
+
+ b.Property("StageVersion")
+ .HasColumnType("integer")
+ .HasColumnName("stage_version");
+
+ b.Property("StageVersionId")
+ .HasColumnType("uuid")
+ .HasColumnName("stage_version_id");
+
+ b.Property("StartedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("started_at");
+
+ b.Property("Status")
+ .IsRequired()
+ .HasMaxLength(16)
+ .HasColumnType("character varying(16)")
+ .HasColumnName("status");
+
+ b.Property("TraceParent")
+ .HasMaxLength(55)
+ .HasColumnType("character varying(55)")
+ .HasColumnName("trace_parent");
+
+ b.Property("Trigger")
+ .IsRequired()
+ .HasMaxLength(16)
+ .HasColumnType("character varying(16)")
+ .HasColumnName("trigger");
+
+ b.Property("TriggerDetails")
+ .HasColumnType("jsonb")
+ .HasColumnName("trigger_details");
+
+ b.Property("TriggeredBy")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("triggered_by");
+
+ b.HasKey("Id", "CreatedAt")
+ .HasName("pk_runs");
+
+ b.HasIndex("ExpiresAt")
+ .HasDatabaseName("ix_runs_expires_at")
+ .HasFilter("expires_at IS NOT NULL");
+
+ b.HasIndex("CreatedAt", "Id")
+ .IsDescending()
+ .HasDatabaseName("ix_runs_created_at_id");
+
+ b.HasIndex("Status", "CreatedAt")
+ .HasDatabaseName("ix_runs_status_created_at")
+ .HasFilter("status IN ('Pending', 'Queued', 'Running', 'Cancelling')");
+
+ b.HasIndex("PipelineId", "CreatedAt", "Id")
+ .IsDescending(false, true, true)
+ .HasDatabaseName("ix_runs_pipeline_id_created_at_id");
+
+ b.HasIndex("StageKey", "CreatedAt", "Id")
+ .IsDescending(false, true, true)
+ .HasDatabaseName("ix_runs_stage_key_created_at_id")
+ .HasFilter("stage_key IS NOT NULL");
+
+ b.ToTable("runs", "psr", t =>
+ {
+ t.HasCheckConstraint("ck_runs_kind", "kind IN ('Pipeline', 'PipelineTest', 'Stage', 'StageTest')");
+
+ t.HasCheckConstraint("ck_runs_mode", "mode IN ('Sync', 'Async')");
+
+ t.HasCheckConstraint("ck_runs_status", "status IN ('Pending', 'Queued', 'Running', 'Succeeded', 'Failed', 'Cancelling', 'Cancelled')");
+
+ t.HasCheckConstraint("ck_runs_trigger", "trigger IN ('Library', 'Dashboard', 'Schedule', 'QueueCommand')");
+ });
+ });
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.ScheduleEntity", b =>
+ {
+ b.Property("Id")
+ .HasColumnType("uuid")
+ .HasColumnName("id");
+
+ b.Property("CreatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("created_at");
+
+ b.Property("CreatedBy")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("created_by");
+
+ b.Property("Cron")
+ .IsRequired()
+ .HasMaxLength(100)
+ .HasColumnType("character varying(100)")
+ .HasColumnName("cron");
+
+ b.Property("Inputs")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasColumnType("jsonb")
+ .HasColumnName("inputs")
+ .HasDefaultValueSql("'{}'::jsonb");
+
+ b.Property("IsEnabled")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("boolean")
+ .HasDefaultValue(true)
+ .HasColumnName("is_enabled");
+
+ b.Property("LastError")
+ .HasColumnType("text")
+ .HasColumnName("last_error");
+
+ b.Property("LastFiredAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("last_fired_at");
+
+ b.Property("LastRunId")
+ .HasColumnType("uuid")
+ .HasColumnName("last_run_id");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("name");
+
+ b.Property("NextRunAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("next_run_at");
+
+ b.Property("PinnedVersion")
+ .HasColumnType("integer")
+ .HasColumnName("pinned_version");
+
+ b.Property("PipelineId")
+ .HasColumnType("uuid")
+ .HasColumnName("pipeline_id");
+
+ b.Property("RowVersion")
+ .IsConcurrencyToken()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("xid")
+ .HasColumnName("xmin");
+
+ b.Property("TimeZone")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasMaxLength(64)
+ .HasColumnType("character varying(64)")
+ .HasDefaultValue("America/Sao_Paulo")
+ .HasColumnName("time_zone");
+
+ b.Property("UpdatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("updated_at");
+
+ b.Property("UpdatedBy")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("updated_by");
+
+ b.HasKey("Id")
+ .HasName("pk_schedules");
+
+ b.HasIndex("NextRunAt")
+ .HasDatabaseName("ix_schedules_next_run_at")
+ .HasFilter("is_enabled");
+
+ b.HasIndex("PipelineId")
+ .HasDatabaseName("ix_schedules_pipeline_id");
+
+ b.ToTable("schedules", "psr");
+ });
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StageEntity", b =>
+ {
+ b.Property("Id")
+ .HasColumnType("uuid")
+ .HasColumnName("id");
+
+ b.Property("CreatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("created_at");
+
+ b.Property("CreatedBy")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("created_by");
+
+ b.Property("Description")
+ .HasColumnType("text")
+ .HasColumnName("description");
+
+ b.Property("Key")
+ .IsRequired()
+ .HasMaxLength(64)
+ .HasColumnType("character varying(64)")
+ .HasColumnName("key");
+
+ b.Property("LatestPublishedVersion")
+ .HasColumnType("integer")
+ .HasColumnName("latest_published_version");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("name");
+
+ b.Property("RowVersion")
+ .IsConcurrencyToken()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("xid")
+ .HasColumnName("xmin");
+
+ b.Property("UpdatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("updated_at");
+
+ b.Property("UpdatedBy")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("updated_by");
+
+ b.HasKey("Id")
+ .HasName("pk_stages");
+
+ b.HasIndex("Key")
+ .IsUnique()
+ .HasDatabaseName("ux_stages_key");
+
+ b.ToTable("stages", "psr");
+ });
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StageVersionEntity", b =>
+ {
+ b.Property("Id")
+ .HasColumnType("uuid")
+ .HasColumnName("id");
+
+ b.Property("ArchivedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("archived_at");
+
+ b.Property("ArchivedBy")
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("archived_by");
+
+ b.Property("CreatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("created_at");
+
+ b.Property("CreatedBy")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("created_by");
+
+ b.Property("EnvironmentHash")
+ .HasMaxLength(64)
+ .HasColumnType("character(64)")
+ .HasColumnName("environment_hash")
+ .IsFixedLength();
+
+ b.Property("Idempotent")
+ .HasColumnType("boolean")
+ .HasColumnName("idempotent");
+
+ b.Property("InputFields")
+ .IsRequired()
+ .HasColumnType("jsonb")
+ .HasColumnName("input_fields");
+
+ b.Property("LockedRequirements")
+ .HasColumnType("text")
+ .HasColumnName("locked_requirements");
+
+ b.Property("MemoryLimitMb")
+ .HasColumnType("integer")
+ .HasColumnName("memory_limit_mb");
+
+ b.Property("OutputFields")
+ .IsRequired()
+ .HasColumnType("jsonb")
+ .HasColumnName("output_fields");
+
+ b.Property("PublishedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("published_at");
+
+ b.Property("PublishedBy")
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("published_by");
+
+ b.Property("PythonVersion")
+ .IsRequired()
+ .HasMaxLength(10)
+ .HasColumnType("character varying(10)")
+ .HasColumnName("python_version");
+
+ b.PrimitiveCollection("RequiredSecrets")
+ .IsRequired()
+ .HasColumnType("text[]")
+ .HasColumnName("required_secrets");
+
+ b.Property("Requirements")
+ .HasColumnType("text")
+ .HasColumnName("requirements");
+
+ b.Property("RetryBackoffMultiplier")
+ .HasPrecision(4, 2)
+ .HasColumnType("numeric(4,2)")
+ .HasColumnName("retry_backoff_multiplier");
+
+ b.Property("RetryInitialDelaySeconds")
+ .HasColumnType("integer")
+ .HasColumnName("retry_initial_delay_seconds");
+
+ b.Property("RetryMaxAttempts")
+ .HasColumnType("integer")
+ .HasColumnName("retry_max_attempts");
+
+ b.Property("RowVersion")
+ .IsConcurrencyToken()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("xid")
+ .HasColumnName("xmin");
+
+ b.Property("Script")
+ .IsRequired()
+ .HasColumnType("text")
+ .HasColumnName("script");
+
+ b.Property("ScriptSha256")
+ .IsRequired()
+ .HasMaxLength(64)
+ .HasColumnType("character(64)")
+ .HasColumnName("script_sha256")
+ .IsFixedLength();
+
+ b.Property("StageId")
+ .HasColumnType("uuid")
+ .HasColumnName("stage_id");
+
+ b.Property("Status")
+ .IsRequired()
+ .ValueGeneratedOnAdd()
+ .HasMaxLength(16)
+ .HasColumnType("character varying(16)")
+ .HasDefaultValue("Draft")
+ .HasColumnName("status");
+
+ b.Property("TimeoutSeconds")
+ .HasColumnType("integer")
+ .HasColumnName("timeout_seconds");
+
+ b.Property("UpdatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("updated_at");
+
+ b.Property("UpdatedBy")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)")
+ .HasColumnName("updated_by");
+
+ b.Property("Version")
+ .HasColumnType("integer")
+ .HasColumnName("version");
+
+ b.HasKey("Id")
+ .HasName("pk_stage_versions");
+
+ b.HasIndex("StageId")
+ .IsUnique()
+ .HasDatabaseName("ux_stage_versions_draft")
+ .HasFilter("status = 'Draft'");
+
+ b.HasIndex("StageId", "Version")
+ .IsUnique()
+ .HasDatabaseName("ux_stage_versions_stage_id_version");
+
+ b.ToTable("stage_versions", "psr", t =>
+ {
+ t.HasCheckConstraint("ck_stage_versions_status", "status IN ('Draft', 'Published', 'Archived')");
+ });
+ });
+
+ modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StepAttemptEntity", b =>
+ {
+ b.Property("Id")
+ .HasColumnType("uuid")
+ .HasColumnName("id");
+
+ b.Property("RunCreatedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("run_created_at");
+
+ b.Property("Attempt")
+ .HasColumnType("integer")
+ .HasColumnName("attempt");
+
+ b.Property("Cause")
+ .IsRequired()
+ .HasMaxLength(16)
+ .HasColumnType("character varying(16)")
+ .HasColumnName("cause");
+
+ b.Property("DurationMs")
+ .HasColumnType("integer")
+ .HasColumnName("duration_ms");
+
+ b.Property("EnvironmentHash")
+ .HasMaxLength(64)
+ .HasColumnType("character(64)")
+ .HasColumnName("environment_hash")
+ .IsFixedLength();
+
+ b.Property("Error")
+ .HasColumnType("jsonb")
+ .HasColumnName("error");
+
+ b.Property("ErrorKind")
+ .HasMaxLength(40)
+ .HasColumnType("character varying(40)")
+ .HasColumnName("error_kind");
+
+ b.Property("ExitCode")
+ .HasColumnType("integer")
+ .HasColumnName("exit_code");
+
+ b.Property("FinishedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("finished_at");
+
+ b.Property("LeaseExpiresAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("lease_expires_at");
+
+ b.Property("OverheadMs")
+ .HasColumnType("integer")
+ .HasColumnName("overhead_ms");
+
+ b.Property("RunId")
+ .HasColumnType("uuid")
+ .HasColumnName("run_id");
+
+ b.Property("StartedAt")
+ .HasColumnType("timestamp with time zone")
+ .HasColumnName("started_at");
+
+ b.Property