diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4bdb406..3affef9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -41,9 +41,16 @@ jobs:
- name: Build
run: dotnet build --no-restore --configuration Release
+ # The persistence tests start PostgreSQL 16 with Testcontainers (Docker); GitHub's Windows runners
+ # cannot run Linux containers, so Windows runs the Core tests only.
- name: Test
+ if: runner.os == 'Linux'
run: dotnet test --no-build --configuration Release
+ - name: Test (Core)
+ if: runner.os == 'Windows'
+ run: dotnet test --no-build --configuration Release --project tests/PythonScriptRunner.Core.Tests
+
format:
name: .NET format
runs-on: ubuntu-latest
diff --git a/Directory.Packages.props b/Directory.Packages.props
index 8f016f7..b891480 100644
--- a/Directory.Packages.props
+++ b/Directory.Packages.props
@@ -4,17 +4,22 @@
+
+
+
+
+
diff --git a/PythonScriptRunner.slnx b/PythonScriptRunner.slnx
index a691e0f..091dc73 100644
--- a/PythonScriptRunner.slnx
+++ b/PythonScriptRunner.slnx
@@ -2,8 +2,10 @@
+
+
diff --git a/README.md b/README.md
index 1a9384d..aeea566 100644
--- a/README.md
+++ b/README.md
@@ -5,7 +5,7 @@ com contrato de entrada/saída, trilhas (**Pipelines**), execução assíncrona
cron e painel Vue opcional.
> Projeto em desenvolvimento. Esta versão entrega o **Core Runner** (execução síncrona de Stages e de
-> trilhas) e o SDK Python **`psrunner`**.
+> trilhas), o SDK Python **`psrunner`** e a **persistência em PostgreSQL** (catálogo versionado e histórico).
## Como funciona
@@ -89,10 +89,42 @@ double[] termos = result.GetStepOutput("termos", "termos"); // depur
A mesma trilha pode ser definida em C# com `AddPipeline("chave", p => p.Input(...).Step(...).Output(...))`.
Se um passo falhar, os seguintes ficam como `Skipped` e a execução termina `Failed`.
+## Persistência (PostgreSQL)
+
+Com o pacote `PythonScriptRunner.EntityFrameworkCore`, Stages e trilhas passam a viver no banco (schema `psr`) com
+versões imutáveis (rascunho → publicado → arquivado), e **toda** execução fica registrada: passos, tentativas, cauda
+de stdout/stderr e arquivos (em `bytea`, lidos e gravados em streaming).
+
+```csharp
+builder.Services
+ .AddPythonScriptRunner()
+ .AddEntityFrameworkCore(o =>
+ {
+ o.UseNpgsql(builder.Configuration.GetConnectionString("psr")!); // PostgreSQL 16+
+ o.Retention.Artifacts = TimeSpan.FromDays(7); // arquivos
+ o.Retention.Runs = TimeSpan.FromDays(90); // histórico
+ });
+builder.Services.AddHealthChecks().AddPythonScriptRunnerDatabase();
+
+var app = builder.Build();
+await app.Services.MigratePythonScriptRunnerAsync(); // explícito; ou GeneratePythonScriptRunnerMigrationScriptAsync()
+```
+
+- **Catálogo:** `ICatalogAdmin` cria rascunhos (concorrência otimista), publica (verifica a sintaxe com o Python
+ declarado e trava as dependências com `uv pip compile --generate-hashes`), arquiva, exclui, testa rascunhos e
+ importa/exporta zip no formato de pasta.
+- **Execuções:** `IRunManager` consulta (paginação keyset), cancela e reprocessa uma execução que falhou — os passos
+ que deram certo reaproveitam as saídas e o que falhou ganha uma nova tentativa. `RunOptions.IdempotencyKey` devolve
+ a mesma execução por 24 h.
+- **Retenção:** o histórico é particionado (semana para execuções, dia para conteúdo de arquivos, mês para auditoria);
+ um job de manutenção cria partições à frente e remove as antigas. Uma trilha pode encurtar a retenção.
+- **Auditoria:** ações humanas (`Actor.User(...)`) ficam em `psr.audit_events`; disparos automáticos ficam na própria execução.
+
## Requisitos
- .NET 10 SDK
- [uv](https://docs.astral.sh/uv/) no PATH (cria e cacheia um ambiente Python por conjunto de dependências)
+- PostgreSQL 16+ apenas para o pacote de persistência (os testes usam Docker/Testcontainers)
## Estrutura
@@ -100,6 +132,7 @@ Se um passo falhar, os seguintes ficam como `Skipped` e a execução termina `Fa
|-------|----------|
| `src/PythonScriptRunner.Abstractions` | contratos públicos (tipos de campo, `StageDefinition`, `RunResult`, `PsrFile`, interfaces) |
| `src/PythonScriptRunner.Core` | catálogos (código e pasta), validação, ambientes uv, executor de processos, runner |
+| `src/PythonScriptRunner.EntityFrameworkCore` | PostgreSQL: catálogo versionado, histórico, arquivos, idempotência, outbox, auditoria, retenção |
| `python/psrunner` | SDK Python (`@stage`, `Context`, `File`, `StageError`, `psrunner.testing`, CLI) |
| `schemas` | JSON Schemas do `stage.json` e do `*.pipeline.json` |
| `tests` | testes .NET (unitários e de integração com Python real) e fixtures de trilha |
@@ -109,6 +142,7 @@ Se um passo falhar, os seguintes ficam como `Skipped` e a execução termina `Fa
```bash
dotnet build
dotnet test --project tests/PythonScriptRunner.Core.Tests
+dotnet test --project tests/PythonScriptRunner.EntityFrameworkCore.Tests # precisa de Docker
cd python/psrunner
uv sync
diff --git a/src/PythonScriptRunner.Abstractions/Exceptions.cs b/src/PythonScriptRunner.Abstractions/Exceptions.cs
index ab38d86..df77067 100644
--- a/src/PythonScriptRunner.Abstractions/Exceptions.cs
+++ b/src/PythonScriptRunner.Abstractions/Exceptions.cs
@@ -47,6 +47,61 @@ public VersionNotFoundException(string key, int version)
public int Version { get; }
}
+/// The version exists but cannot run (it is a draft or was archived).
+public sealed class VersionNotRunnableException : PsrException
+{
+ /// Creates the exception.
+ /// Stage or pipeline key.
+ /// Requested version.
+ /// Current status of the version.
+ public VersionNotRunnableException(string key, int version, VersionStatus status)
+ : base($"Version {version} of '{key}' is {status} and cannot run; only published versions run.")
+ {
+ Key = key;
+ Version = version;
+ Status = status;
+ }
+
+ /// Stage or pipeline key.
+ public string Key { get; }
+
+ /// Requested version.
+ public int Version { get; }
+
+ /// Current status of the version.
+ public VersionStatus Status { get; }
+}
+
+/// The run does not exist (or is beyond the retention).
+public sealed class RunNotFoundException : PsrException
+{
+ /// Creates the exception.
+ /// Requested run.
+ public RunNotFoundException(RunId runId)
+ : base($"Run {runId} was not found.")
+ {
+ RunId = runId;
+ }
+
+ /// Requested run.
+ public RunId RunId { get; }
+}
+
+/// The stored file does not exist (or is beyond the retention).
+public sealed class ArtifactNotFoundException : PsrException
+{
+ /// Creates the exception.
+ /// Requested file.
+ public ArtifactNotFoundException(ArtifactId artifactId)
+ : base($"Artifact {artifactId} was not found.")
+ {
+ ArtifactId = artifactId;
+ }
+
+ /// Requested file.
+ public ArtifactId ArtifactId { get; }
+}
+
/// One invalid input.
/// Field path (e.g. coeficientes[2]).
/// Stable code (e.g. MISSING_REQUIRED_INPUT).
diff --git a/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs b/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs
index 9a57725..a4f7742 100644
--- a/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs
+++ b/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs
@@ -2,7 +2,7 @@
namespace PythonScriptRunner;
-/// Source of stage definitions (memory, folder or database).
+/// Source of stage and pipeline definitions (memory, folder or database).
public interface ICatalog
{
/// Finds a stage version.
@@ -10,6 +10,7 @@ public interface ICatalog
/// Exact version, or null for the latest published version.
/// Cancellation token.
/// The definition, or null when not found.
+ /// The version exists but is a draft or archived.
ValueTask FindStageAsync(string key, int? version, CancellationToken cancellationToken = default);
/// Finds a pipeline version.
@@ -17,9 +18,19 @@ public interface ICatalog
/// Exact version, or null for the latest published version.
/// Cancellation token.
/// The definition, or null when not found.
+ /// The version exists but is a draft or archived.
ValueTask FindPipelineAsync(string key, int? version, CancellationToken cancellationToken = default);
}
+/// Validates stage definitions (contract, limits and secret names).
+public interface IStageValidator
+{
+ /// Validates a stage definition.
+ /// The definition.
+ /// Errors (code STAGE_DEFINITION) and warnings.
+ ValidationReport Validate(StageDefinition stage);
+}
+
/// Resolves the secrets declared by stages.
public interface ISecretProvider
{
@@ -29,46 +40,145 @@ public interface ISecretProvider
ValueTask GetSecretAsync(string name, CancellationToken cancellationToken = default);
}
-/// Receives run transitions so they can be persisted. The default implementation does nothing.
+///
+/// Receives every transition of a run so it can be persisted. The default implementation does nothing; the
+/// EntityFrameworkCore package stores runs, steps, attempts, logs and files.
+///
public interface IRunRecorder
{
- /// A run started.
+ /// A run is about to start (inputs already validated). Reserves the idempotency key.
+ /// Run data.
+ /// Cancellation token.
+ /// The existing run when the idempotency key was already used (the new run must not start); otherwise null.
+ ValueTask RunStartingAsync(RunStarting run, CancellationToken cancellationToken);
+
+ /// A failed run is being reprocessed: it is running again and skipped steps are pending.
/// Run data.
/// Cancellation token.
- ValueTask RunStartedAsync(RunStarted run, CancellationToken cancellationToken);
+ ValueTask RunResumingAsync(RunResuming run, CancellationToken cancellationToken);
+
+ /// A step started with its resolved inputs.
+ /// Step data.
+ /// Cancellation token.
+ ValueTask StepStartedAsync(StepStarted step, CancellationToken cancellationToken);
- /// An attempt finished.
+ /// An attempt (one Python process) started.
+ /// Attempt data.
+ /// Cancellation token.
+ ValueTask AttemptStartedAsync(AttemptStarted attempt, CancellationToken cancellationToken);
+
+ /// An attempt finished (with its logs).
/// Attempt data.
/// Cancellation token.
ValueTask AttemptFinishedAsync(AttemptFinished attempt, CancellationToken cancellationToken);
+ /// A step finished (succeeded with outputs and files, failed, cancelled or skipped).
+ /// Step data.
+ /// Cancellation token.
+ ValueTask StepFinishedAsync(StepFinished step, CancellationToken cancellationToken);
+
/// A run finished.
/// Final data.
/// Cancellation token.
ValueTask RunFinishedAsync(RunFinished run, CancellationToken cancellationToken);
}
-/// A run started.
+/// A run is about to start.
+public sealed record RunStarting
+{
+ /// Run identifier (its timestamp is the creation time).
+ public required RunId RunId { get; init; }
+
+ /// What is executed.
+ public required RunKind Kind { get; init; }
+
+ /// Stage or pipeline key.
+ public required string TargetKey { get; init; }
+
+ /// Stage or pipeline version.
+ public required int TargetVersion { get; init; }
+
+ /// How the run started.
+ public required RunTrigger Trigger { get; init; }
+
+ /// Who requested the run.
+ public required string TriggeredBy { get; init; }
+
+ /// Deduplication key given by the caller.
+ public string? IdempotencyKey { get; init; }
+
+ /// Validated inputs in wire format (files carry metadata only).
+ public required JsonObject Inputs { get; init; }
+
+ /// Input files by field name.
+ public IReadOnlyDictionary> InputFiles { get; init; } = new Dictionary>();
+
+ /// Steps that will run, in order.
+ public required IReadOnlyList Steps { get; init; }
+
+ /// Start time.
+ public required DateTimeOffset StartedAt { get; init; }
+}
+
+/// A step of a run that is about to start.
+/// Step key (the stage key for single-stage runs).
+/// 1-based position.
+/// Stage key.
+/// Exact stage version.
+public sealed record PlannedStep(string StepKey, int Order, string StageKey, int StageVersion);
+
+/// A failed run is being reprocessed.
+/// Run identifier.
+/// Who reprocesses.
+/// When.
+public sealed record RunResuming(RunId RunId, string TriggeredBy, DateTimeOffset ResumedAt);
+
+/// A step started.
/// Run identifier.
-/// What is executed.
-/// Stage or pipeline key.
-/// Stage or pipeline version.
-/// Who requested the run.
+/// Step key.
+/// 1-based position.
+/// Resolved inputs in wire format.
+/// Input files by field name.
/// Start time.
-/// Validated inputs in wire format.
-public sealed record RunStarted(
- RunId RunId, RunKind Kind, string TargetKey, int TargetVersion, string TriggeredBy, DateTimeOffset StartedAt, JsonObject Inputs);
+public sealed record StepStarted(
+ RunId RunId, string StepKey, int Order, JsonObject Inputs, IReadOnlyDictionary> InputFiles, DateTimeOffset StartedAt);
+
+/// An attempt started.
+/// Run identifier.
+/// Step key.
+/// 1-based position of the step.
+/// Attempt identifier.
+/// 1-based attempt number.
+/// Why it started.
+/// Who started it.
+/// Start time.
+public sealed record AttemptStarted(
+ RunId RunId, string StepKey, int Order, AttemptId AttemptId, int Number, AttemptCause Cause, string TriggeredBy, DateTimeOffset StartedAt);
/// An attempt finished.
/// Run identifier.
/// Step key.
+/// 1-based position of the step.
/// Attempt outcome.
-public sealed record AttemptFinished(RunId RunId, string StepKey, AttemptResult Attempt);
+public sealed record AttemptFinished(RunId RunId, string StepKey, int Order, AttemptResult Attempt);
+
+/// A step finished.
+/// Run identifier.
+/// Step outcome (outputs and files when it succeeded).
+/// End time.
+public sealed record StepFinished(RunId RunId, StepResult Step, DateTimeOffset FinishedAt);
/// A run finished.
/// Run identifier.
/// Final status.
/// End time.
/// Declared outputs in wire format.
+/// Declared file outputs by name.
/// Failure, when the run did not succeed.
-public sealed record RunFinished(RunId RunId, RunStatus Status, DateTimeOffset FinishedAt, JsonObject? Outputs, RunError? Error);
+public sealed record RunFinished(
+ RunId RunId,
+ RunStatus Status,
+ DateTimeOffset FinishedAt,
+ JsonObject? Outputs,
+ IReadOnlyDictionary>? OutputFiles,
+ RunError? Error);
diff --git a/src/PythonScriptRunner.Abstractions/Files/PsrFile.cs b/src/PythonScriptRunner.Abstractions/Files/PsrFile.cs
index da44e0c..f520e55 100644
--- a/src/PythonScriptRunner.Abstractions/Files/PsrFile.cs
+++ b/src/PythonScriptRunner.Abstractions/Files/PsrFile.cs
@@ -27,10 +27,20 @@ public sealed class PsrFile
private readonly string? _path;
private readonly ReadOnlyMemory? _bytes;
+ private readonly Func>? _opener;
+ private readonly long? _knownLength;
private Stream? _stream;
private string? _sha256;
- private PsrFile(string fileName, string contentType, string? path, ReadOnlyMemory? bytes, Stream? stream, string? sha256)
+ private PsrFile(
+ string fileName,
+ string contentType,
+ string? path,
+ ReadOnlyMemory? bytes,
+ Stream? stream,
+ string? sha256,
+ Func>? opener = null,
+ long? knownLength = null)
{
ArgumentException.ThrowIfNullOrWhiteSpace(fileName);
ArgumentException.ThrowIfNullOrWhiteSpace(contentType);
@@ -40,6 +50,8 @@ private PsrFile(string fileName, string contentType, string? path, ReadOnlyMemor
_bytes = bytes;
_stream = stream;
_sha256 = sha256;
+ _opener = opener;
+ _knownLength = knownLength;
}
/// A file on disk.
@@ -90,6 +102,38 @@ public static PsrFile FromText(string text, string fileName, string contentType
public static PsrFile FromProducedFile(string path, string fileName, string contentType, string sha256) =>
new(fileName, contentType, Path.GetFullPath(path), null, null, sha256);
+ /// A stored file (e.g. in the database) that is streamed on demand.
+ /// Identifier of the stored file.
+ /// File name.
+ /// MIME type.
+ /// Size in bytes.
+ /// SHA-256 as lowercase hex.
+ /// Opens the content; called once per read.
+ public static PsrFile FromArtifact(
+ ArtifactId artifactId, string fileName, string contentType, long length, string sha256, Func> open)
+ {
+ ArgumentNullException.ThrowIfNull(open);
+ var file = new PsrFile(fileName, contentType, null, null, null, sha256, open, length);
+ file.AttachArtifact(artifactId);
+ return file;
+ }
+
+ /// Identifier of the stored copy of this file, when a persistence package stored it.
+ public ArtifactId? ArtifactId { get; private set; }
+
+ /// Associates the file with its stored copy (used by persistence packages).
+ /// Identifier of the stored file.
+ /// The file is already associated with another stored file.
+ public void AttachArtifact(ArtifactId artifactId)
+ {
+ if (ArtifactId is { } current && current != artifactId)
+ {
+ throw new InvalidOperationException($"File '{FileName}' is already stored as artifact {current}.");
+ }
+
+ ArtifactId = artifactId;
+ }
+
/// File name (without directories).
public string FileName { get; }
@@ -114,6 +158,11 @@ public long? Length
return bytes.Length;
}
+ if (_knownLength is { } known)
+ {
+ return known;
+ }
+
return _stream is { CanSeek: true } stream ? stream.Length - stream.Position : null;
}
}
@@ -159,6 +208,11 @@ public Task OpenReadAsync(CancellationToken cancellationToken = default)
return Task.FromResult(new MemoryStream(bytes.ToArray(), writable: false));
}
+ if (_opener is not null)
+ {
+ return _opener(cancellationToken);
+ }
+
var stream = Interlocked.Exchange(ref _stream, null)
?? throw new InvalidOperationException($"The stream of file '{FileName}' was already consumed.");
return Task.FromResult(stream);
diff --git a/src/PythonScriptRunner.Abstractions/Ids.cs b/src/PythonScriptRunner.Abstractions/Ids.cs
index 961efe9..e935398 100644
--- a/src/PythonScriptRunner.Abstractions/Ids.cs
+++ b/src/PythonScriptRunner.Abstractions/Ids.cs
@@ -1,12 +1,23 @@
namespace PythonScriptRunner;
-/// Identifier of a run (time-ordered UUID v7).
+/// Identifier of a run (time-ordered UUID v7). Its timestamp is the creation time of the run.
/// The underlying GUID.
public readonly record struct RunId(Guid Value)
{
/// Creates a new time-ordered identifier.
public static RunId New() => new(Guid.CreateVersion7());
+ /// Creates a new identifier whose timestamp is (millisecond precision).
+ /// Creation time of the run.
+ public static RunId New(DateTimeOffset createdAt) => new(Guid.CreateVersion7(createdAt));
+
+ /// Parses the canonical text form.
+ /// GUID text.
+ public static RunId Parse(string text) => new(Guid.Parse(text));
+
+ /// Creation time embedded in the identifier (UTC, millisecond precision).
+ public DateTimeOffset CreatedAt => UuidV7.Timestamp(Value);
+
///
public override string ToString() => Value.ToString();
}
@@ -21,3 +32,101 @@ public readonly record struct AttemptId(Guid Value)
///
public override string ToString() => Value.ToString();
}
+
+/// Identifier of a step within a run (time-ordered UUID v7).
+/// The underlying GUID.
+public readonly record struct StepRunId(Guid Value)
+{
+ /// Creates a new time-ordered identifier.
+ public static StepRunId New() => new(Guid.CreateVersion7());
+
+ ///
+ public override string ToString() => Value.ToString();
+}
+
+/// Identifier of a stored file (time-ordered UUID v7).
+/// The underlying GUID.
+public readonly record struct ArtifactId(Guid Value)
+{
+ /// Creates a new time-ordered identifier.
+ public static ArtifactId New() => new(Guid.CreateVersion7());
+
+ /// Parses the canonical text form.
+ /// GUID text.
+ public static ArtifactId Parse(string text) => new(Guid.Parse(text));
+
+ ///
+ public override string ToString() => Value.ToString();
+}
+
+/// Identifier of a stage (UUID v7).
+/// The underlying GUID.
+public readonly record struct StageId(Guid Value)
+{
+ /// Creates a new time-ordered identifier.
+ public static StageId New() => new(Guid.CreateVersion7());
+
+ ///
+ public override string ToString() => Value.ToString();
+}
+
+/// Identifier of a stage version (UUID v7).
+/// The underlying GUID.
+public readonly record struct StageVersionId(Guid Value)
+{
+ /// Creates a new time-ordered identifier.
+ public static StageVersionId New() => new(Guid.CreateVersion7());
+
+ ///
+ public override string ToString() => Value.ToString();
+}
+
+/// Identifier of a pipeline (UUID v7).
+/// The underlying GUID.
+public readonly record struct PipelineId(Guid Value)
+{
+ /// Creates a new time-ordered identifier.
+ public static PipelineId New() => new(Guid.CreateVersion7());
+
+ ///
+ public override string ToString() => Value.ToString();
+}
+
+/// Identifier of a pipeline version (UUID v7).
+/// The underlying GUID.
+public readonly record struct PipelineVersionId(Guid Value)
+{
+ /// Creates a new time-ordered identifier.
+ public static PipelineVersionId New() => new(Guid.CreateVersion7());
+
+ ///
+ public override string ToString() => Value.ToString();
+}
+
+/// Identifier of a schedule (UUID v7).
+/// The underlying GUID.
+public readonly record struct ScheduleId(Guid Value)
+{
+ /// Creates a new time-ordered identifier.
+ public static ScheduleId New() => new(Guid.CreateVersion7());
+
+ ///
+ public override string ToString() => Value.ToString();
+}
+
+/// Reads the timestamp of a UUID v7.
+internal static class UuidV7
+{
+ public static DateTimeOffset Timestamp(Guid value)
+ {
+ Span bytes = stackalloc byte[16];
+ value.TryWriteBytes(bytes, bigEndian: true, out _);
+ var milliseconds = 0L;
+ for (var i = 0; i < 6; i++)
+ {
+ milliseconds = (milliseconds << 8) | bytes[i];
+ }
+
+ return DateTimeOffset.FromUnixTimeMilliseconds(milliseconds);
+ }
+}
diff --git a/src/PythonScriptRunner.Abstractions/Management/Actor.cs b/src/PythonScriptRunner.Abstractions/Management/Actor.cs
new file mode 100644
index 0000000..7a132bd
--- /dev/null
+++ b/src/PythonScriptRunner.Abstractions/Management/Actor.cs
@@ -0,0 +1,42 @@
+namespace PythonScriptRunner;
+
+/// Who performs an operation: used for authorship columns and to decide auditing (only humans are audited).
+/// User subject, or system:<origin> for automatic callers.
+/// Whether a person performed the action (panel); library, queue and cron are not.
+public sealed record Actor(string Subject, bool IsHuman)
+{
+ /// Code calling the library (system:library).
+ public static Actor Library { get; } = System("library");
+
+ /// An automatic caller.
+ /// Origin (e.g. queue or schedule:<id>).
+ public static Actor System(string origin) => new($"system:{origin}", IsHuman: false);
+
+ /// A person (e.g. the subject of the authenticated user in the panel).
+ /// User subject.
+ public static Actor User(string subject) => new(subject, IsHuman: true);
+
+ ///
+ public override string ToString() => Subject;
+}
+
+/// Creation and last change of a row.
+/// Creation time.
+/// Creator subject.
+/// Last change.
+/// Subject of the last change.
+public sealed record Authorship(DateTimeOffset CreatedAt, string CreatedBy, DateTimeOffset UpdatedAt, string UpdatedBy);
+
+/// One page of an offset-paginated list.
+/// Item type.
+/// Items of the page.
+/// 1-based page number.
+/// Requested page size.
+/// Number of items in every page.
+public sealed record Page(IReadOnlyList Items, int PageNumber, int PageSize, int Total);
+
+/// One page of a keyset-paginated list (newest first).
+/// Item type.
+/// Items of the page.
+/// Opaque cursor of the next page; null on the last page.
+public sealed record KeysetPage(IReadOnlyList Items, string? NextCursor);
diff --git a/src/PythonScriptRunner.Abstractions/Management/Audit.cs b/src/PythonScriptRunner.Abstractions/Management/Audit.cs
new file mode 100644
index 0000000..cc93a6d
--- /dev/null
+++ b/src/PythonScriptRunner.Abstractions/Management/Audit.cs
@@ -0,0 +1,100 @@
+using System.Text.Json.Nodes;
+
+namespace PythonScriptRunner;
+
+/// Records human actions (publish, archive, delete, trigger, cancel, reprocess...). Automatic callers are not audited.
+public interface IAuditWriter
+{
+ /// Writes an event when is human; does nothing otherwise.
+ /// The event.
+ /// Cancellation token.
+ Task WriteAsync(AuditEntry entry, CancellationToken cancellationToken = default);
+}
+
+/// Reads audit events.
+public interface IAuditReader
+{
+ /// Lists events newest first inside a time window (default: last 24 hours; at most 90 days).
+ /// Filters, window and cursor.
+ /// Cancellation token.
+ Task> ListAsync(AuditQuery query, CancellationToken cancellationToken = default);
+}
+
+/// An action to audit.
+/// Who acted.
+/// What was done (see ).
+/// Stage, StageVersion, Pipeline, PipelineVersion, Schedule, Run or Settings.
+/// Identifier of the entity.
+/// Readable reference (e.g. html-to-pdf@3).
+/// Diff ({field: [before, after]}) or action parameters.
+public sealed record AuditEntry(Actor Actor, string Action, string EntityType, Guid? EntityId, string? EntityKey, JsonObject? Details = null);
+
+/// A stored audit event.
+/// Event identifier.
+/// When it happened.
+/// Subject of the user.
+/// What was done.
+/// Entity type.
+/// Entity identifier.
+/// Readable reference.
+/// Diff or parameters.
+/// OpenTelemetry trace.
+public sealed record AuditEvent(
+ Guid Id, DateTimeOffset OccurredAt, string Actor, string Action, string EntityType, Guid? EntityId, string? EntityKey, JsonObject? Details, string? TraceId);
+
+/// Filters of an audit list.
+public sealed record AuditQuery
+{
+ /// Start of the window (inclusive); default: minus 24 hours.
+ public DateTimeOffset? From { get; init; }
+
+ /// End of the window (exclusive); default: now.
+ public DateTimeOffset? To { get; init; }
+
+ /// Only events of this user.
+ public string? Actor { get; init; }
+
+ /// Only events of this entity type.
+ public string? EntityType { get; init; }
+
+ /// Cursor returned by the previous page.
+ public string? Cursor { get; init; }
+
+ /// Items per page (1-100). Default: 50.
+ public int Limit { get; init; } = 50;
+}
+
+/// Audited actions.
+public static class AuditActions
+{
+#pragma warning disable CS1591 // The names are self-explanatory.
+ public const string StageCreated = nameof(StageCreated);
+ public const string StageUpdated = nameof(StageUpdated);
+ public const string StageDraftCreated = nameof(StageDraftCreated);
+ public const string StageDraftSaved = nameof(StageDraftSaved);
+ public const string StageDraftDiscarded = nameof(StageDraftDiscarded);
+ public const string StagePublished = nameof(StagePublished);
+ public const string StageVersionArchived = nameof(StageVersionArchived);
+ public const string StageDeleted = nameof(StageDeleted);
+ public const string StageTested = nameof(StageTested);
+ public const string PipelineCreated = nameof(PipelineCreated);
+ public const string PipelineUpdated = nameof(PipelineUpdated);
+ public const string PipelineDraftCreated = nameof(PipelineDraftCreated);
+ public const string PipelineDraftSaved = nameof(PipelineDraftSaved);
+ public const string PipelineDraftDiscarded = nameof(PipelineDraftDiscarded);
+ public const string PipelinePublished = nameof(PipelinePublished);
+ public const string PipelineVersionArchived = nameof(PipelineVersionArchived);
+ public const string PipelineDeleted = nameof(PipelineDeleted);
+ public const string PipelineSettingsChanged = nameof(PipelineSettingsChanged);
+ public const string PipelineTested = nameof(PipelineTested);
+ public const string ScheduleCreated = nameof(ScheduleCreated);
+ public const string ScheduleUpdated = nameof(ScheduleUpdated);
+ public const string ScheduleDeleted = nameof(ScheduleDeleted);
+ public const string ScheduleEnabled = nameof(ScheduleEnabled);
+ public const string ScheduleDisabled = nameof(ScheduleDisabled);
+ public const string RunTriggered = nameof(RunTriggered);
+ public const string RunCancelled = nameof(RunCancelled);
+ public const string RunReprocessed = nameof(RunReprocessed);
+ public const string CatalogImported = nameof(CatalogImported);
+#pragma warning restore CS1591
+}
diff --git a/src/PythonScriptRunner.Abstractions/Management/CatalogAdmin.cs b/src/PythonScriptRunner.Abstractions/Management/CatalogAdmin.cs
new file mode 100644
index 0000000..2f8d846
--- /dev/null
+++ b/src/PythonScriptRunner.Abstractions/Management/CatalogAdmin.cs
@@ -0,0 +1,499 @@
+using System.Text.Json.Nodes;
+
+namespace PythonScriptRunner;
+
+///
+/// Edits the catalog stored in a database: drafts, publication, archiving, deletion, settings and import/export.
+/// Every change receives the that performs it (authorship and audit of human actions).
+///
+/// Errors are reported with codes ().
+public interface ICatalogAdmin
+{
+ /// Lists stages.
+ /// Search and page.
+ /// Cancellation token.
+ Task> ListStagesAsync(CatalogQuery query, CancellationToken cancellationToken = default);
+
+ /// Reads a stage with its versions and the pipelines that use it.
+ /// Stage key.
+ /// Cancellation token.
+ /// The stage, or null when it does not exist.
+ Task GetStageAsync(string key, CancellationToken cancellationToken = default);
+
+ /// Reads one version of a stage (any status).
+ /// Stage key.
+ /// Version number.
+ /// Cancellation token.
+ Task GetStageVersionAsync(string key, int version, CancellationToken cancellationToken = default);
+
+ /// Reads the current draft of a stage.
+ /// Stage key.
+ /// Cancellation token.
+ Task GetStageDraftAsync(string key, CancellationToken cancellationToken = default);
+
+ /// Creates a stage with an initial draft (version 1). STAGE_KEY_TAKEN when the key exists.
+ /// Key, name and description.
+ /// Author.
+ /// Cancellation token.
+ Task CreateStageAsync(NewCatalogItem stage, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Changes the name or description of a stage.
+ /// Stage key.
+ /// New values (null keeps the current one).
+ /// Row version read by the caller.
+ /// Author.
+ /// Cancellation token.
+ Task UpdateStageAsync(
+ string key, CatalogItemChanges changes, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Creates a draft (next version) with the content of an existing version. DRAFT_ALREADY_EXISTS when there is one.
+ /// Stage key.
+ /// Version to copy.
+ /// Author.
+ /// Cancellation token.
+ Task CreateStageDraftFromAsync(string key, int fromVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Saves a draft. VERSION_IMMUTABLE when the version is not a draft; CONCURRENCY_CONFLICT when it changed.
+ /// Stage key.
+ /// Draft version.
+ /// New content.
+ /// Row version read by the caller.
+ /// Author.
+ /// Cancellation token.
+ Task SaveStageDraftAsync(
+ string key, int version, StageDraftData data, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Deletes a draft.
+ /// Stage key.
+ /// Draft version.
+ /// Row version read by the caller.
+ /// Author.
+ /// Cancellation token.
+ Task DiscardStageDraftAsync(string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Validates the draft contract (fields, limits, secrets) without compiling or locking.
+ /// Stage key.
+ /// Cancellation token.
+ Task ValidateStageDraftAsync(string key, CancellationToken cancellationToken = default);
+
+ ///
+ /// Publishes a draft: validates the contract, compiles the script with its Python version and locks the dependencies with uv.
+ /// Errors: STAGE_INVALID, SCRIPT_SYNTAX_ERROR, DEPENDENCY_RESOLUTION_FAILED, CONCURRENCY_CONFLICT.
+ ///
+ /// Stage key.
+ /// Draft version.
+ /// Row version read by the caller.
+ /// Author.
+ /// Cancellation token.
+ Task PublishStageAsync(
+ string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Archives a published version. VERSION_REFERENCED when a published pipeline uses it.
+ /// Stage key.
+ /// Published version.
+ /// Author.
+ /// Cancellation token.
+ Task ArchiveStageVersionAsync(string key, int version, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Deletes a stage and its versions. DELETE_NOT_ALLOWED when referenced by a pipeline step or present in the retained history.
+ /// Stage key.
+ /// Administrator.
+ /// Cancellation token.
+ Task DeleteStageAsync(string key, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Runs the current draft as a test () and waits for the result.
+ /// Stage key.
+ /// Inputs (same forms as ).
+ /// Author.
+ /// Cancels the run.
+ Task TestStageDraftAsync(string key, object? inputs, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Writes a version as a zip in the folder catalog format (stages/<key>@<version>/stage.json + files).
+ /// Stage key.
+ /// Version.
+ /// Writable stream that receives the zip.
+ /// Cancellation token.
+ Task ExportStageVersionAsync(string key, int version, Stream destination, CancellationToken cancellationToken = default);
+
+ /// Lists pipelines.
+ /// Search and page.
+ /// Cancellation token.
+ Task> ListPipelinesAsync(CatalogQuery query, CancellationToken cancellationToken = default);
+
+ /// Reads a pipeline with its versions and settings.
+ /// Pipeline key.
+ /// Cancellation token.
+ Task GetPipelineAsync(string key, CancellationToken cancellationToken = default);
+
+ /// Reads one version of a pipeline (any status).
+ /// Pipeline key.
+ /// Version number.
+ /// Cancellation token.
+ Task GetPipelineVersionAsync(string key, int version, CancellationToken cancellationToken = default);
+
+ /// Reads the current draft of a pipeline.
+ /// Pipeline key.
+ /// Cancellation token.
+ Task GetPipelineDraftAsync(string key, CancellationToken cancellationToken = default);
+
+ /// Creates a pipeline with an empty draft (version 1). PIPELINE_KEY_TAKEN when the key exists.
+ /// Key, name and description.
+ /// Author.
+ /// Cancellation token.
+ Task CreatePipelineAsync(NewCatalogItem pipeline, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Changes the name or description of a pipeline.
+ /// Pipeline key.
+ /// New values (null keeps the current one).
+ /// Row version read by the caller.
+ /// Author.
+ /// Cancellation token.
+ Task UpdatePipelineAsync(
+ string key, CatalogItemChanges changes, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Creates a draft (next version) with the content of an existing version.
+ /// Pipeline key.
+ /// Version to copy.
+ /// Author.
+ /// Cancellation token.
+ Task CreatePipelineDraftFromAsync(string key, int fromVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Saves a draft. Steps must reference existing stage versions (published to be publishable).
+ /// Pipeline key.
+ /// Draft version.
+ /// New content.
+ /// Row version read by the caller.
+ /// Author.
+ /// Cancellation token.
+ Task SavePipelineDraftAsync(
+ string key, int version, PipelineDraftData data, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Deletes a draft.
+ /// Pipeline key.
+ /// Draft version.
+ /// Row version read by the caller.
+ /// Author.
+ /// Cancellation token.
+ Task DiscardPipelineDraftAsync(string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Publishes a draft after validation (PIPELINE_INVALID with the report); warnings are returned.
+ /// Pipeline key.
+ /// Draft version.
+ /// Row version read by the caller.
+ /// Author.
+ /// Cancellation token.
+ Task PublishPipelineAsync(
+ string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Archives a published version. VERSION_SCHEDULED when an enabled schedule pins it.
+ /// Pipeline key.
+ /// Published version.
+ /// Author.
+ /// Cancellation token.
+ Task ArchivePipelineVersionAsync(string key, int version, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Deletes a pipeline, its versions and schedules. DELETE_NOT_ALLOWED when it has runs in the retained history.
+ /// Pipeline key.
+ /// Administrator.
+ /// Cancellation token.
+ Task DeletePipelineAsync(string key, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Changes operational settings (not versioned).
+ /// Pipeline key.
+ /// New settings.
+ /// Row version read by the caller.
+ /// Author.
+ /// Cancellation token.
+ Task UpdatePipelineSettingsAsync(
+ string key, PipelineSettings settings, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Shortens the retention of the pipeline runs and files. RETENTION_EXCEEDS_GLOBAL when longer than the global one.
+ /// Pipeline key.
+ /// New retention (null uses the global value).
+ /// Row version read by the caller.
+ /// Administrator.
+ /// Cancellation token.
+ Task UpdatePipelineRetentionAsync(
+ string key, PipelineRetention retention, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Validates and runs the current draft as a test (); PIPELINE_INVALID when invalid.
+ /// Pipeline key.
+ /// Pipeline inputs.
+ /// Author.
+ /// Cancels the run.
+ Task TestPipelineDraftAsync(string key, object? inputs, Actor actor, CancellationToken cancellationToken = default);
+
+ /// Writes a version and the stage versions it uses as a zip in the folder catalog format.
+ /// Pipeline key.
+ /// Version.
+ /// Writable stream that receives the zip.
+ /// Cancellation token.
+ Task ExportPipelineVersionAsync(string key, int version, Stream destination, CancellationToken cancellationToken = default);
+
+ ///
+ /// Imports a zip in the folder catalog format as drafts: new keys get version 1, existing keys get a draft of a new version
+ /// (published versions are never overwritten). All or nothing: IMPORT_INVALID lists every problem.
+ ///
+ /// Readable zip stream.
+ /// Author.
+ /// Cancellation token.
+ Task ImportAsync(Stream zip, Actor actor, CancellationToken cancellationToken = default);
+}
+
+/// Search and page of a catalog list.
+/// Text contained in the key or the name.
+/// 1-based page number.
+/// Items per page (1-100).
+public sealed record CatalogQuery(string? Search = null, int PageNumber = 1, int PageSize = 25);
+
+/// A new stage or pipeline.
+/// Slug (immutable).
+/// Display name.
+/// Description.
+public sealed record NewCatalogItem(string Key, string Name, string? Description = null);
+
+/// Changes to the name or description; null keeps the current value.
+/// New name.
+/// New description.
+public sealed record CatalogItemChanges(string? Name = null, string? Description = null);
+
+/// A version in a list.
+/// Version number.
+/// Status.
+/// Publication time.
+/// Last change.
+public sealed record VersionSummary(int Version, VersionStatus Status, DateTimeOffset? PublishedAt, DateTimeOffset UpdatedAt);
+
+/// Editable content of a stage version.
+public sealed record StageDraftData
+{
+ /// Python source of the single script.
+ public required string Script { get; init; }
+
+ /// Python version (e.g. 3.12).
+ public required string PythonVersion { get; init; }
+
+ /// Dependencies in requirements.txt format.
+ public string? Requirements { get; init; }
+
+ /// Declared inputs.
+ public IReadOnlyList Inputs { get; init; } = [];
+
+ /// Declared outputs.
+ public IReadOnlyList Outputs { get; init; } = [];
+
+ /// Names of the secrets the script needs.
+ public IReadOnlyList RequiredSecrets { get; init; } = [];
+
+ /// Maximum duration of one attempt.
+ public TimeSpan Timeout { get; init; } = TimeSpan.FromMinutes(5);
+
+ /// Memory limit in MB; null uses the runner default.
+ public int? MemoryLimitMb { get; init; }
+
+ /// Automatic retry policy.
+ public RetryPolicy Retry { get; init; } = RetryPolicy.None;
+
+ /// Whether running the script twice with the same inputs is safe.
+ public bool Idempotent { get; init; }
+}
+
+/// The current draft of a stage.
+/// Stage key.
+/// Draft version.
+/// Content.
+/// Concurrency token (send it back when saving).
+/// Creation and last change.
+public sealed record StageDraft(string Key, int Version, StageDraftData Data, uint RowVersion, Authorship Authorship);
+
+/// One version of a stage.
+public sealed record StageVersionDetail
+{
+ /// Stage key.
+ public required string Key { get; init; }
+
+ /// Version number.
+ public required int Version { get; init; }
+
+ /// Status.
+ public required VersionStatus Status { get; init; }
+
+ /// Content.
+ public required StageDraftData Data { get; init; }
+
+ /// SHA-256 of the script.
+ public required string ScriptSha256 { get; init; }
+
+ /// Locked dependencies (published versions).
+ public string? LockedRequirements { get; init; }
+
+ /// Hash of the Python environment (published versions).
+ public string? EnvironmentHash { get; init; }
+
+ /// Publication time.
+ public DateTimeOffset? PublishedAt { get; init; }
+
+ /// Who published.
+ public string? PublishedBy { get; init; }
+
+ /// Archiving time.
+ public DateTimeOffset? ArchivedAt { get; init; }
+
+ /// Who archived.
+ public string? ArchivedBy { get; init; }
+
+ /// Concurrency token.
+ public required uint RowVersion { get; init; }
+
+ /// Creation and last change.
+ public required Authorship Authorship { get; init; }
+}
+
+/// A stage in a list.
+/// Stage key.
+/// Display name.
+/// Description.
+/// Latest published version.
+/// Version of the current draft.
+/// Last change.
+public sealed record StageSummary(string Key, string Name, string? Description, int? LatestPublishedVersion, int? DraftVersion, DateTimeOffset UpdatedAt);
+
+/// A pipeline step that uses a stage version.
+/// Pipeline key.
+/// Pipeline version.
+/// Status of the pipeline version.
+/// Step key.
+/// Stage version used by the step.
+public sealed record StageUsage(string PipelineKey, int PipelineVersion, VersionStatus PipelineStatus, string StepKey, int StageVersion);
+
+/// A stage with its versions and usages.
+/// Stage key.
+/// Display name.
+/// Description.
+/// Latest published version.
+/// Every version, newest first.
+/// Pipeline steps that use a version of the stage.
+/// Concurrency token of the stage row.
+/// Creation and last change.
+public sealed record StageDetail(
+ string Key,
+ string Name,
+ string? Description,
+ int? LatestPublishedVersion,
+ IReadOnlyList Versions,
+ IReadOnlyList UsedBy,
+ uint RowVersion,
+ Authorship Authorship);
+
+/// Editable content of a pipeline version.
+public sealed record PipelineDraftData
+{
+ /// Inputs received by the pipeline.
+ public IReadOnlyList Inputs { get; init; } = [];
+
+ /// Steps in execution order.
+ public IReadOnlyList Steps { get; init; } = [];
+
+ /// Declared outputs.
+ public IReadOnlyList Outputs { get; init; } = [];
+
+ /// Positions of nodes and edges in the visual editor (does not affect execution).
+ public JsonNode? Layout { get; init; }
+}
+
+/// The current draft of a pipeline.
+/// Pipeline key.
+/// Draft version.
+/// Content.
+/// Concurrency token (send it back when saving).
+/// Creation and last change.
+public sealed record PipelineDraft(string Key, int Version, PipelineDraftData Data, uint RowVersion, Authorship Authorship);
+
+/// One version of a pipeline.
+public sealed record PipelineVersionDetail
+{
+ /// Pipeline key.
+ public required string Key { get; init; }
+
+ /// Version number.
+ public required int Version { get; init; }
+
+ /// Status.
+ public required VersionStatus Status { get; init; }
+
+ /// Content.
+ public required PipelineDraftData Data { get; init; }
+
+ /// Publication time.
+ public DateTimeOffset? PublishedAt { get; init; }
+
+ /// Who published.
+ public string? PublishedBy { get; init; }
+
+ /// Archiving time.
+ public DateTimeOffset? ArchivedAt { get; init; }
+
+ /// Who archived.
+ public string? ArchivedBy { get; init; }
+
+ /// Concurrency token.
+ public required uint RowVersion { get; init; }
+
+ /// Creation and last change.
+ public required Authorship Authorship { get; init; }
+}
+
+/// A pipeline in a list.
+/// Pipeline key.
+/// Display name.
+/// Description.
+/// Latest published version.
+/// Version of the current draft.
+/// Last change.
+public sealed record PipelineSummary(string Key, string Name, string? Description, int? LatestPublishedVersion, int? DraftVersion, DateTimeOffset UpdatedAt);
+
+/// Operational settings of a pipeline (not versioned).
+/// Maximum simultaneous asynchronous runs; null = no limit.
+public sealed record PipelineSettings(int? MaxConcurrentRuns);
+
+/// Retention of a pipeline; it can only be shorter than the global retention.
+/// Days the files are kept; null = global.
+/// Days the runs are kept; null = global.
+public sealed record PipelineRetention(int? ArtifactRetentionDays, int? RunRetentionDays);
+
+/// A pipeline with its versions and settings.
+/// Pipeline key.
+/// Display name.
+/// Description.
+/// Latest published version.
+/// Every version, newest first.
+/// Operational settings.
+/// Retention.
+/// Concurrency token of the pipeline row.
+/// Creation and last change.
+public sealed record PipelineDetail(
+ string Key,
+ string Name,
+ string? Description,
+ int? LatestPublishedVersion,
+ IReadOnlyList Versions,
+ PipelineSettings Settings,
+ PipelineRetention Retention,
+ uint RowVersion,
+ Authorship Authorship);
+
+/// Result of publishing a pipeline.
+/// The published version.
+/// Validation warnings (they do not block publication).
+public sealed record PipelinePublication(PipelineVersionDetail Version, IReadOnlyList Warnings);
+
+/// A draft created by an import.
+/// Stage or Pipeline.
+/// Key.
+/// Version of the created draft.
+/// Whether the key did not exist before.
+public sealed record ImportedItem(string Kind, string Key, int Version, bool NewKey);
+
+/// Result of an import.
+/// Drafts created.
+public sealed record ImportReport(IReadOnlyList Items);
diff --git a/src/PythonScriptRunner.Abstractions/Management/DomainErrors.cs b/src/PythonScriptRunner.Abstractions/Management/DomainErrors.cs
new file mode 100644
index 0000000..6997f2d
--- /dev/null
+++ b/src/PythonScriptRunner.Abstractions/Management/DomainErrors.cs
@@ -0,0 +1,146 @@
+namespace PythonScriptRunner;
+
+/// Stable codes of the domain errors raised by catalog and run management (mapped to HTTP by the dashboard).
+public static class PsrErrorCodes
+{
+ /// A stage with this key already exists.
+ public const string StageKeyTaken = "STAGE_KEY_TAKEN";
+
+ /// A pipeline with this key already exists.
+ public const string PipelineKeyTaken = "PIPELINE_KEY_TAKEN";
+
+ /// The row changed since it was read (optimistic concurrency).
+ public const string ConcurrencyConflict = "CONCURRENCY_CONFLICT";
+
+ /// Published and archived versions cannot be edited.
+ public const string VersionImmutable = "VERSION_IMMUTABLE";
+
+ /// The stage version is used by a published pipeline.
+ public const string VersionReferenced = "VERSION_REFERENCED";
+
+ /// The pipeline version is pinned by an enabled schedule.
+ public const string VersionScheduled = "VERSION_SCHEDULED";
+
+ /// Only one draft per stage or pipeline is allowed.
+ public const string DraftAlreadyExists = "DRAFT_ALREADY_EXISTS";
+
+ /// The stage definition is invalid (fields, limits, secrets).
+ public const string StageInvalid = "STAGE_INVALID";
+
+ /// The script does not compile.
+ public const string ScriptSyntaxError = "SCRIPT_SYNTAX_ERROR";
+
+ /// uv could not resolve the dependencies.
+ public const string DependencyResolutionFailed = "DEPENDENCY_RESOLUTION_FAILED";
+
+ /// The pipeline does not pass validation.
+ public const string PipelineInvalid = "PIPELINE_INVALID";
+
+ /// The stage or pipeline is referenced or has runs in the retained history.
+ public const string DeleteNotAllowed = "DELETE_NOT_ALLOWED";
+
+ /// A pipeline retention cannot exceed the global retention.
+ public const string RetentionExceedsGlobal = "RETENTION_EXCEEDS_GLOBAL";
+
+ /// Only failed runs can be reprocessed.
+ public const string RunNotFailed = "RUN_NOT_FAILED";
+
+ /// The run already finished.
+ public const string RunNotCancellable = "RUN_NOT_CANCELLABLE";
+
+ /// Files needed to reprocess the run are beyond the retention.
+ public const string ArtifactsExpired = "ARTIFACTS_EXPIRED";
+
+ /// The file content is beyond the retention.
+ public const string ArtifactExpired = "ARTIFACT_EXPIRED";
+
+ /// The imported archive is not a valid catalog folder.
+ public const string ImportInvalid = "IMPORT_INVALID";
+}
+
+/// A business rule rejected the operation; is stable (see ).
+public class PsrDomainException : PsrException
+{
+ /// Creates the exception.
+ /// Stable code.
+ /// Human-readable message.
+ /// Cause.
+ public PsrDomainException(string code, string message, Exception? innerException = null)
+ : base(message, innerException)
+ {
+ Code = code;
+ }
+
+ /// Stable code (see ).
+ public string Code { get; }
+}
+
+/// The row changed since it was read; holds the current state.
+public sealed class ConcurrencyConflictException : PsrDomainException
+{
+ /// Creates the exception.
+ /// What conflicted.
+ /// Current state (e.g. the current draft), when it still exists.
+ public ConcurrencyConflictException(string message, object? current)
+ : base(PsrErrorCodes.ConcurrencyConflict, message)
+ {
+ Current = current;
+ }
+
+ /// Current state (e.g. ), or null when it no longer exists.
+ public object? Current { get; }
+}
+
+/// The script of a stage does not compile.
+public sealed class ScriptSyntaxException : PsrDomainException
+{
+ /// Creates the exception.
+ /// Python error message.
+ /// 1-based line, when known.
+ /// 1-based column, when known.
+ public ScriptSyntaxException(string message, int? line, int? column)
+ : base(PsrErrorCodes.ScriptSyntaxError, line is null ? message : $"{message} (line {line}, column {column})")
+ {
+ Line = line;
+ Column = column;
+ }
+
+ /// 1-based line, when known.
+ public int? Line { get; }
+
+ /// 1-based column, when known.
+ public int? Column { get; }
+}
+
+/// A definition does not pass validation; lists the problems.
+public sealed class DefinitionInvalidException : PsrDomainException
+{
+ /// Creates the exception.
+ /// or .
+ /// Validation report.
+ public DefinitionInvalidException(string code, ValidationReport report)
+ : base(code, $"Validation failed: {report}")
+ {
+ Report = report;
+ }
+
+ /// Validation report.
+ public ValidationReport Report { get; }
+}
+
+/// The stage version is used by published pipelines and cannot be archived.
+public sealed class VersionReferencedException : PsrDomainException
+{
+ /// Creates the exception.
+ /// Stage key.
+ /// Stage version.
+ /// Pipelines that use it (key@version).
+ public VersionReferencedException(string key, int version, IReadOnlyList pipelines)
+ : base(PsrErrorCodes.VersionReferenced, $"Stage '{key}@{version}' is used by published pipelines: {string.Join(", ", pipelines)}.")
+ {
+ Pipelines = pipelines;
+ }
+
+ /// Pipelines that use the version (key@version).
+ public IReadOnlyList Pipelines { get; }
+}
diff --git a/src/PythonScriptRunner.Abstractions/Management/RunManagement.cs b/src/PythonScriptRunner.Abstractions/Management/RunManagement.cs
new file mode 100644
index 0000000..efff25f
--- /dev/null
+++ b/src/PythonScriptRunner.Abstractions/Management/RunManagement.cs
@@ -0,0 +1,358 @@
+using System.Text.Json.Nodes;
+
+namespace PythonScriptRunner;
+
+/// Queries, cancels and reprocesses persisted runs.
+public interface IRunManager
+{
+ /// Reads a run with its steps, attempts and files (logs are read with ).
+ /// Run identifier.
+ /// Cancellation token.
+ /// The run, or null when it does not exist or is beyond the retention.
+ Task GetAsync(RunId runId, CancellationToken cancellationToken = default);
+
+ /// Lists runs newest first inside a time window (default: last 24 hours; at most 90 days).
+ /// Filters, window and cursor.
+ /// Cancellation token.
+ Task> ListAsync(RunQuery query, CancellationToken cancellationToken = default);
+
+ /// Reads the stdout or stderr tail of an attempt.
+ /// Run identifier.
+ /// Attempt identifier.
+ /// Which stream.
+ /// Cancellation token.
+ Task GetAttemptLogAsync(RunId runId, AttemptId attemptId, LogStream stream, CancellationToken cancellationToken = default);
+
+ ///
+ /// Cancels a run: pending/queued runs end Cancelled; running runs move to Cancelling and their processes are stopped.
+ /// RUN_NOT_CANCELLABLE when the run already finished.
+ ///
+ /// Run identifier.
+ /// Who cancels.
+ /// Cancellation token.
+ Task CancelAsync(RunId runId, Actor actor, CancellationToken cancellationToken = default);
+
+ ///
+ /// Reprocesses a failed run with the same version and inputs: successful steps keep their outputs and the failed step gets a
+ /// new attempt (). Errors: RUN_NOT_FAILED, ARTIFACTS_EXPIRED.
+ ///
+ /// Run identifier.
+ /// Who reprocesses.
+ /// runs it now in this process; needs a run queue.
+ /// Cancellation token (cancels a synchronous reprocess).
+ /// The run after reprocessing (sync) or after it was queued (async).
+ Task ReprocessAsync(RunId runId, Actor actor, ExecutionMode mode = ExecutionMode.Async, CancellationToken cancellationToken = default);
+}
+
+/// Filters of a run list. The window is required to keep queries on recent partitions.
+public sealed record RunQuery
+{
+ /// Start of the window (inclusive); default: minus 24 hours.
+ public DateTimeOffset? From { get; init; }
+
+ /// End of the window (exclusive); default: now.
+ public DateTimeOffset? To { get; init; }
+
+ /// Only runs of this pipeline.
+ public string? PipelineKey { get; init; }
+
+ /// Only single-stage runs of this stage.
+ public string? StageKey { get; init; }
+
+ /// Only runs of this kind.
+ public RunKind? Kind { get; init; }
+
+ /// Only runs with this status.
+ public RunStatus? Status { get; init; }
+
+ /// Only runs started this way.
+ public RunTrigger? Trigger { get; init; }
+
+ /// Cursor returned by the previous page.
+ public string? Cursor { get; init; }
+
+ /// Items per page (1-100). Default: 50.
+ public int Limit { get; init; } = 50;
+}
+
+/// A run in a list.
+/// Run identifier.
+/// What was executed.
+/// Sync or async.
+/// Status.
+/// Pipeline or stage key.
+/// Pipeline or stage version.
+/// How the run started.
+/// Who started it.
+/// Creation time.
+/// Start time.
+/// End time.
+/// Failure.
+public sealed record RunSummary(
+ RunId Id,
+ RunKind Kind,
+ ExecutionMode Mode,
+ RunStatus Status,
+ string TargetKey,
+ int TargetVersion,
+ RunTrigger Trigger,
+ string TriggeredBy,
+ DateTimeOffset CreatedAt,
+ DateTimeOffset? StartedAt,
+ DateTimeOffset? FinishedAt,
+ RunError? Error);
+
+/// A persisted run with its steps, attempts and files.
+public sealed record RunSnapshot
+{
+ /// Run identifier.
+ public required RunId Id { get; init; }
+
+ /// What was executed.
+ public required RunKind Kind { get; init; }
+
+ /// Sync or async.
+ public required ExecutionMode Mode { get; init; }
+
+ /// Status.
+ public required RunStatus Status { get; init; }
+
+ /// Pipeline or stage key.
+ public required string TargetKey { get; init; }
+
+ /// Pipeline or stage version.
+ public required int TargetVersion { get; init; }
+
+ /// How the run started.
+ public required RunTrigger Trigger { get; init; }
+
+ /// Who started it.
+ public required string TriggeredBy { get; init; }
+
+ /// Validated inputs; files as {"$artifact": "<id>"}.
+ public required JsonObject Inputs { get; init; }
+
+ /// Declared outputs; files as {"$artifact": "<id>"}.
+ public JsonObject? Outputs { get; init; }
+
+ /// Failure.
+ public RunError? Error { get; init; }
+
+ /// Idempotency key given by the caller.
+ public string? IdempotencyKey { get; init; }
+
+ /// Creation time.
+ public required DateTimeOffset CreatedAt { get; init; }
+
+ /// When it was queued.
+ public DateTimeOffset? QueuedAt { get; init; }
+
+ /// Start time.
+ public DateTimeOffset? StartedAt { get; init; }
+
+ /// End time.
+ public DateTimeOffset? FinishedAt { get; init; }
+
+ /// When cancellation was requested.
+ public DateTimeOffset? CancelRequestedAt { get; init; }
+
+ /// Who requested cancellation.
+ public string? CancelRequestedBy { get; init; }
+
+ /// When the run is deleted (pipelines with a shorter retention).
+ public DateTimeOffset? ExpiresAt { get; init; }
+
+ /// Steps in order.
+ public required IReadOnlyList Steps { get; init; }
+
+ /// Input and output files.
+ public required IReadOnlyList Artifacts { get; init; }
+}
+
+/// A persisted step of a run.
+public sealed record StepRunSnapshot
+{
+ /// Step identifier.
+ public required StepRunId Id { get; init; }
+
+ /// Step key (the stage key for single-stage runs).
+ public required string StepKey { get; init; }
+
+ /// 1-based position.
+ public required int Order { get; init; }
+
+ /// Executed stage.
+ public required string StageKey { get; init; }
+
+ /// Exact stage version.
+ public required int StageVersion { get; init; }
+
+ /// Status.
+ public required StepStatus Status { get; init; }
+
+ /// Number of the last attempt (0 = not executed).
+ public required int CurrentAttempt { get; init; }
+
+ /// Inputs after resolving the bindings.
+ public JsonObject? Inputs { get; init; }
+
+ /// Validated outputs.
+ public JsonObject? Outputs { get; init; }
+
+ /// Failure of the last attempt.
+ public RunError? Error { get; init; }
+
+ /// Start time.
+ public DateTimeOffset? StartedAt { get; init; }
+
+ /// End time.
+ public DateTimeOffset? FinishedAt { get; init; }
+
+ /// Attempts in order.
+ public required IReadOnlyList Attempts { get; init; }
+}
+
+/// A persisted attempt (one Python process).
+public sealed record AttemptSnapshot
+{
+ /// Attempt identifier.
+ public required AttemptId Id { get; init; }
+
+ /// 1-based number within the step.
+ public required int Number { get; init; }
+
+ /// Why it started.
+ public required AttemptCause Cause { get; init; }
+
+ /// Who started it (user for reprocessing, system:* otherwise).
+ public required string TriggeredBy { get; init; }
+
+ /// Status.
+ public required AttemptStatus Status { get; init; }
+
+ /// Worker that executed it (asynchronous runs).
+ public string? WorkerId { get; init; }
+
+ /// Python environment used.
+ public string? EnvironmentHash { get; init; }
+
+ /// Exit code of the process.
+ public int? ExitCode { get; init; }
+
+ /// Failure.
+ public RunError? Error { get; init; }
+
+ /// Start time.
+ public required DateTimeOffset StartedAt { get; init; }
+
+ /// End time.
+ public DateTimeOffset? FinishedAt { get; init; }
+
+ /// Total duration.
+ public TimeSpan? Duration { get; init; }
+
+ /// Time spent by the runner outside the Python process.
+ public TimeSpan? Overhead { get; init; }
+
+ /// Logs captured (content is read with ).
+ public IReadOnlyList Logs { get; init; } = [];
+}
+
+/// The stdout or stderr tail of an attempt.
+/// Which stream.
+/// Captured text (secret values masked).
+/// Whether older output was discarded.
+/// Total bytes written by the process.
+public sealed record AttemptLog(LogStream Stream, string Content, bool Truncated, long OriginalBytes);
+
+/// Metadata of a stored file.
+public sealed record ArtifactInfo
+{
+ /// File identifier.
+ public required ArtifactId Id { get; init; }
+
+ /// Run that owns the file.
+ public required RunId RunId { get; init; }
+
+ /// Step that produced the file (null for run inputs).
+ public string? StepKey { get; init; }
+
+ /// Input or output.
+ public required ArtifactDirection Direction { get; init; }
+
+ /// Field that carries the file.
+ public required string FieldName { get; init; }
+
+ /// Position in a list<file>.
+ public int? ListIndex { get; init; }
+
+ /// File name.
+ public required string FileName { get; init; }
+
+ /// MIME type.
+ public required string ContentType { get; init; }
+
+ /// Size in bytes.
+ public required long SizeBytes { get; init; }
+
+ /// SHA-256 as lowercase hex.
+ public required string Sha256 { get; init; }
+
+ /// When the content is deleted.
+ public required DateTimeOffset ContentExpiresAt { get; init; }
+
+ /// When the content was deleted (downloads then fail with ARTIFACT_EXPIRED).
+ public DateTimeOffset? ContentDeletedAt { get; init; }
+}
+
+/// Stores and streams run files.
+public interface IArtifactStore
+{
+ /// Stores a file (streamed; never fully loaded in memory).
+ /// Metadata.
+ /// Readable content.
+ /// Cancellation token.
+ Task SaveAsync(NewArtifact artifact, Stream content, CancellationToken cancellationToken = default);
+
+ /// Reads the metadata of a file.
+ /// File identifier.
+ /// Cancellation token.
+ Task GetAsync(ArtifactId artifactId, CancellationToken cancellationToken = default);
+
+ /// Opens the content for streaming. ARTIFACT_EXPIRED when the content was deleted by the retention.
+ /// File identifier.
+ /// Cancellation token.
+ /// The file does not exist.
+ Task OpenReadAsync(ArtifactId artifactId, CancellationToken cancellationToken = default);
+}
+
+/// Metadata of a file to store.
+public sealed record NewArtifact
+{
+ /// Identifier to use (new when omitted).
+ public ArtifactId? Id { get; init; }
+
+ /// Run that owns the file.
+ public required RunId RunId { get; init; }
+
+ /// Step that produced the file (null for run inputs).
+ public StepRunId? StepRunId { get; init; }
+
+ /// Input or output.
+ public required ArtifactDirection Direction { get; init; }
+
+ /// Field that carries the file.
+ public required string FieldName { get; init; }
+
+ /// Position in a list<file>.
+ public int? ListIndex { get; init; }
+
+ /// File name.
+ public required string FileName { get; init; }
+
+ /// MIME type.
+ public required string ContentType { get; init; }
+
+ /// How long the content is kept; null uses the global retention.
+ public TimeSpan? Retention { get; init; }
+}
diff --git a/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs b/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs
index 5f649f7..2fc069c 100644
--- a/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs
+++ b/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs
@@ -39,4 +39,7 @@ public sealed record RunOptions
/// Who requested the run. Default: system:library.
public string? TriggeredBy { get; init; }
+
+ /// How the run was started. Default: .
+ public RunTrigger Trigger { get; init; } = RunTrigger.Library;
}
diff --git a/src/PythonScriptRunner.Abstractions/Running/IRunEngine.cs b/src/PythonScriptRunner.Abstractions/Running/IRunEngine.cs
new file mode 100644
index 0000000..92a03e7
--- /dev/null
+++ b/src/PythonScriptRunner.Abstractions/Running/IRunEngine.cs
@@ -0,0 +1,72 @@
+using System.Text.Json.Nodes;
+
+namespace PythonScriptRunner;
+
+///
+/// Advanced API of the execution engine, used by the persistence, queue and dashboard packages: runs definitions that do
+/// not come from the catalog (drafts), resumes failed runs and cancels runs of this process. Applications normally use
+/// .
+///
+public interface IRunEngine
+{
+ /// Runs a stage definition (any status) as a single-step run and waits for its result.
+ /// The definition (e.g. a draft).
+ /// or .
+ /// Inputs (same forms as ).
+ /// Idempotency key and caller identity ( is ignored).
+ /// Cancels the run.
+ /// The inputs do not match the contract.
+ Task RunStageAsync(
+ StageDefinition stage, RunKind kind, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default);
+
+ /// Runs a pipeline definition (any status) and waits for its result. Its stages are resolved in the catalog.
+ /// The definition (e.g. a draft).
+ /// or .
+ /// Pipeline inputs.
+ /// Idempotency key and caller identity ( is ignored).
+ /// Cancels the run.
+ /// The inputs do not match the contract.
+ Task RunPipelineAsync(
+ PipelineDefinition pipeline, RunKind kind, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default);
+
+ ///
+ /// Resumes a failed run in this process: successful steps keep their outputs, the first unsuccessful step gets a new
+ /// attempt () and the following steps run normally. The run keeps its identifier.
+ ///
+ /// State of the run to resume.
+ /// Cancels the run.
+ Task ResumeAsync(ResumeRequest request, CancellationToken cancellationToken = default);
+
+ /// Cancels a run that is executing in this process.
+ /// Run identifier.
+ /// Whether the run was executing here.
+ bool RequestCancellation(RunId runId);
+}
+
+/// State of a failed run to resume.
+public sealed record ResumeRequest
+{
+ /// Run identifier (kept).
+ public required RunId RunId { get; init; }
+
+ /// What the run executes.
+ public required RunKind Kind { get; init; }
+
+ /// The stage, for single-stage runs.
+ public StageDefinition? Stage { get; init; }
+
+ /// The pipeline, for pipeline runs.
+ public PipelineDefinition? Pipeline { get; init; }
+
+ /// Validated inputs of the run in wire format.
+ public required JsonObject Inputs { get; init; }
+
+ /// Input files of the run by field name.
+ public IReadOnlyDictionary> InputFiles { get; init; } = new Dictionary>();
+
+ /// Steps as recorded: succeeded steps (with outputs and files) are reused; attempts of the others are counted.
+ public required IReadOnlyList PreviousSteps { get; init; }
+
+ /// Who reprocesses.
+ public required string TriggeredBy { get; init; }
+}
diff --git a/src/PythonScriptRunner.Abstractions/Running/Results.cs b/src/PythonScriptRunner.Abstractions/Running/Results.cs
index 14bbff3..4f3abd6 100644
--- a/src/PythonScriptRunner.Abstractions/Running/Results.cs
+++ b/src/PythonScriptRunner.Abstractions/Running/Results.cs
@@ -93,6 +93,7 @@ public sealed class RunResult : IAsyncDisposable
/// Output files of the run by field name.
/// Failure, when the run did not succeed.
/// Deletes local files when the result is disposed.
+ /// Whether this is an earlier run returned for the same idempotency key.
public RunResult(
RunId id,
RunKind kind,
@@ -102,7 +103,8 @@ public RunResult(
JsonObject? outputs,
IReadOnlyDictionary>? files,
RunError? error,
- Func? cleanup = null)
+ Func? cleanup = null,
+ bool deduplicated = false)
{
Id = id;
Kind = kind;
@@ -113,6 +115,7 @@ public RunResult(
_files = files ?? new Dictionary>();
Error = error;
_cleanup = cleanup;
+ Deduplicated = deduplicated;
}
/// Run identifier.
@@ -121,9 +124,18 @@ public RunResult(
/// What was executed.
public RunKind Kind { get; }
- /// Final status: Succeeded, Failed or Cancelled.
+ ///
+ /// Final status: Succeeded, Failed or Cancelled. A deduplicated result reports the current status of the earlier run,
+ /// which may still be running.
+ ///
public RunStatus Status { get; }
+ /// Whether this is an earlier run returned because the same idempotency key was used within 24 hours.
+ public bool Deduplicated { get; }
+
+ /// Output files of the run by declared output name.
+ public IReadOnlyDictionary> Files => _files;
+
/// Failure, when the run did not succeed.
public RunError? Error { get; }
diff --git a/src/PythonScriptRunner.Abstractions/Running/Statuses.cs b/src/PythonScriptRunner.Abstractions/Running/Statuses.cs
index 54a9fa2..7aab167 100644
--- a/src/PythonScriptRunner.Abstractions/Running/Statuses.cs
+++ b/src/PythonScriptRunner.Abstractions/Running/Statuses.cs
@@ -83,6 +83,57 @@ public enum AttemptCause
LeaseRecovery,
}
+/// Who or what started a run.
+public enum RunTrigger
+{
+ /// Code calling the library.
+ Library,
+ /// A user in the panel.
+ Dashboard,
+ /// A cron schedule.
+ Schedule,
+ /// The public queue command.
+ QueueCommand,
+}
+
+/// How a run is executed.
+public enum ExecutionMode
+{
+ /// In the calling process, waiting for the result.
+ Sync,
+ /// By a worker, through the queue.
+ Async,
+}
+
+/// Life cycle of a stage or pipeline version.
+public enum VersionStatus
+{
+ /// Editable; can be tested but not used by published pipelines.
+ Draft,
+ /// Immutable and runnable.
+ Published,
+ /// Immutable and no longer runnable (terminal).
+ Archived,
+}
+
+/// Whether a stored file was given to a run or produced by it.
+public enum ArtifactDirection
+{
+ /// Input of the run.
+ Input,
+ /// Output of a step.
+ Output,
+}
+
+/// Output stream of a Python process.
+public enum LogStream
+{
+ /// Standard output.
+ Stdout,
+ /// Standard error.
+ Stderr,
+}
+
/// Classification of a failure.
public enum ErrorKind
{
diff --git a/src/PythonScriptRunner.Core/Catalog/CatalogManifests.cs b/src/PythonScriptRunner.Core/Catalog/CatalogManifests.cs
new file mode 100644
index 0000000..73ac004
--- /dev/null
+++ b/src/PythonScriptRunner.Core/Catalog/CatalogManifests.cs
@@ -0,0 +1,258 @@
+using System.Globalization;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Json.Schema;
+using PythonScriptRunner.Core.Catalog;
+using PythonScriptRunner.Core.Pipelines;
+
+namespace PythonScriptRunner;
+
+///
+/// Reads and writes the folder catalog format: stages/<folder>/stage.json (+ script.py,
+/// requirements.txt, requirements.lock) and pipelines/<key>.pipeline.json. Used by the folder
+/// catalog and by import/export.
+///
+public static class CatalogManifests
+{
+ /// Manifest file of a stage folder.
+ public const string StageManifestName = "stage.json";
+
+ /// Default script file of a stage folder.
+ public const string ScriptName = "script.py";
+
+ /// Default requirements file of a stage folder.
+ public const string RequirementsName = "requirements.txt";
+
+ /// Locked requirements of a stage folder (used as-is when present).
+ public const string LockName = "requirements.lock";
+
+ /// Suffix of pipeline manifests.
+ public const string PipelineSuffix = ".pipeline.json";
+
+ private const string SchemaBase = "https://raw.githubusercontent.com/net0well/PythonScriptRunner/main/schemas/";
+
+ private static readonly Lazy StageSchema = new(() => LoadSchema("stage.v1.json"), LazyThreadSafetyMode.ExecutionAndPublication);
+ private static readonly Lazy PipelineSchema = new(() => LoadSchema("pipeline.v1.json"), LazyThreadSafetyMode.ExecutionAndPublication);
+
+ /// Writes the stage.json of a stage (script in script.py, requirements in requirements.txt).
+ /// The stage.
+ public static JsonObject WriteStage(StageDefinition stage)
+ {
+ ArgumentNullException.ThrowIfNull(stage);
+ var json = new JsonObject
+ {
+ ["$schema"] = SchemaBase + "stage.v1.json",
+ ["key"] = stage.Key,
+ ["version"] = stage.Version,
+ ["name"] = stage.Name,
+ };
+ if (stage.Description is not null)
+ {
+ json["description"] = stage.Description;
+ }
+
+ if (stage.PythonVersion is not null)
+ {
+ json["python"] = stage.PythonVersion;
+ }
+
+ json["script"] = ScriptName;
+ if (!string.IsNullOrWhiteSpace(stage.Requirements))
+ {
+ json["requirements"] = RequirementsName;
+ }
+
+ if (stage.Timeout is { } timeout)
+ {
+ json["timeoutSeconds"] = (int)Math.Ceiling(timeout.TotalSeconds);
+ }
+
+ if (stage.MemoryLimitMb is { } memory)
+ {
+ json["memoryLimitMb"] = memory;
+ }
+
+ json["retry"] = new JsonObject
+ {
+ ["maxAttempts"] = stage.Retry.MaxAttempts,
+ ["initialDelaySeconds"] = (int)stage.Retry.InitialDelay.TotalSeconds,
+ ["backoffMultiplier"] = stage.Retry.BackoffMultiplier,
+ };
+ json["idempotent"] = stage.Idempotent;
+ json["secrets"] = new JsonArray([.. stage.RequiredSecrets.Select(s => (JsonNode)JsonValue.Create(s))]);
+ json["inputs"] = FieldDefinitionJson.ToJson(stage.Inputs);
+ json["outputs"] = FieldDefinitionJson.ToJson(stage.Outputs);
+ return json;
+ }
+
+ /// Writes the *.pipeline.json of a pipeline (compact binding syntax).
+ /// The pipeline.
+ public static JsonObject WritePipeline(PipelineDefinition pipeline)
+ {
+ ArgumentNullException.ThrowIfNull(pipeline);
+ var json = new JsonObject
+ {
+ ["$schema"] = SchemaBase + "pipeline.v1.json",
+ ["key"] = pipeline.Key,
+ ["version"] = pipeline.Version,
+ ["name"] = pipeline.Name,
+ };
+ if (pipeline.Description is not null)
+ {
+ json["description"] = pipeline.Description;
+ }
+
+ json["inputs"] = FieldDefinitionJson.ToJson(pipeline.Inputs);
+ json["steps"] = new JsonArray([.. pipeline.Steps.Select(step =>
+ {
+ var node = new JsonObject
+ {
+ ["key"] = step.Key,
+ ["stage"] = $"{step.StageKey}@{step.StageVersion.ToString(CultureInfo.InvariantCulture)}",
+ };
+ if (step.DisplayName is not null)
+ {
+ node["displayName"] = step.DisplayName;
+ }
+
+ if (step.Bindings.Count > 0)
+ {
+ node["bindings"] = new JsonObject(step.Bindings.Select(b => KeyValuePair.Create(b.Input, (JsonNode?)WriteSource(b.Source))));
+ }
+
+ return (JsonNode)node;
+ })]);
+ json["outputs"] = new JsonArray([.. pipeline.Outputs.Select(output =>
+ {
+ var node = new JsonObject { ["name"] = output.Name, ["source"] = output.Source.ToString() };
+ if (output.Description is not null)
+ {
+ node["description"] = output.Description;
+ }
+
+ return (JsonNode)node;
+ })]);
+ return json;
+ }
+
+ /// Writes field declarations in the manifest format (camelCase; see the data mapping).
+ /// Fields.
+ public static JsonArray WriteFields(IEnumerable fields) => FieldDefinitionJson.ToJson(fields);
+
+ /// Reads field declarations written by .
+ /// JSON array of fields.
+ /// A field is malformed.
+ public static IReadOnlyList ReadFields(JsonArray fields)
+ {
+ ArgumentNullException.ThrowIfNull(fields);
+ return [.. fields.Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))];
+ }
+
+ /// Reads a stage.json validated against schemas/stage.v1.json.
+ /// Content of stage.json.
+ /// The script of the stage.
+ /// Content of the requirements file, if any.
+ /// Content of requirements.lock, if any.
+ /// The manifest is not valid JSON or does not match the schema.
+ public static StageDefinition ReadStage(string manifest, ScriptSource script, string? requirements, string? lockedRequirements)
+ {
+ var json = ParseValidated(manifest, StageSchema.Value, "schemas/stage.v1.json");
+ return StageFromJson(json, script, requirements, lockedRequirements);
+ }
+
+ /// Reads a *.pipeline.json validated against schemas/pipeline.v1.json.
+ /// Content of the manifest.
+ /// The manifest is not valid JSON or does not match the schema.
+ public static PipelineDefinition ReadPipeline(string manifest)
+ {
+ var json = ParseValidated(manifest, PipelineSchema.Value, "schemas/pipeline.v1.json");
+ try
+ {
+ return PipelineJson.FromJson(json);
+ }
+ catch (ArgumentException ex)
+ {
+ throw new PsrConfigurationException(ex.Message, ex);
+ }
+ }
+
+ /// Builds a stage from a manifest already validated by the schema.
+ internal static StageDefinition StageFromJson(JsonObject json, ScriptSource script, string? requirements, string? lockedRequirements)
+ {
+ var retry = json["retry"]?.AsObject();
+ return new StageDefinition
+ {
+ Key = json["key"]!.GetValue(),
+ Version = json["version"]!.GetValue(),
+ Name = json["name"]!.GetValue(),
+ Description = json["description"]?.GetValue(),
+ Script = script,
+ PythonVersion = json["python"]?.GetValue(),
+ Requirements = requirements,
+ LockedRequirements = lockedRequirements,
+ Inputs = [.. json["inputs"]!.AsArray().Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))],
+ Outputs = [.. json["outputs"]!.AsArray().Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))],
+ RequiredSecrets = json["secrets"] is JsonArray secrets ? [.. secrets.Select(s => s!.GetValue())] : [],
+ Timeout = json["timeoutSeconds"] is { } seconds ? TimeSpan.FromSeconds(seconds.GetValue()) : null,
+ MemoryLimitMb = json["memoryLimitMb"]?.GetValue(),
+ Retry = retry is null
+ ? RetryPolicy.None
+ : new RetryPolicy(
+ retry["maxAttempts"]?.GetValue() ?? 0,
+ TimeSpan.FromSeconds(retry["initialDelaySeconds"]?.GetValue() ?? 10),
+ retry["backoffMultiplier"]?.GetValue() ?? 2.0),
+ Idempotent = json["idempotent"]?.GetValue() ?? false,
+ };
+ }
+
+ /// Parses a manifest and validates it against the schema; errors cite the JSON location.
+ internal static JsonObject ParseValidated(string manifest, bool pipeline) =>
+ ParseValidated(manifest, pipeline ? PipelineSchema.Value : StageSchema.Value, pipeline ? "schemas/pipeline.v1.json" : "schemas/stage.v1.json");
+
+ private static JsonObject ParseValidated(string manifest, JsonSchema schema, string schemaName)
+ {
+ JsonDocument document;
+ try
+ {
+ document = JsonDocument.Parse(manifest, new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true });
+ }
+ catch (JsonException ex)
+ {
+ throw new PsrConfigurationException($"invalid JSON: {ex.Message}", ex);
+ }
+
+ using (document)
+ {
+ var results = schema.Evaluate(document.RootElement, new EvaluationOptions { OutputFormat = OutputFormat.List });
+ if (results.IsValid)
+ {
+ return JsonNode.Parse(document.RootElement.GetRawText())!.AsObject();
+ }
+
+ var errors = new List();
+ foreach (var detail in (IEnumerable)[results, .. results.Details ?? []])
+ {
+ foreach (var message in detail.Errors?.Values ?? Enumerable.Empty())
+ {
+ errors.Add($"{(detail.InstanceLocation.ToString() is { Length: > 0 } at ? at : "/")}: {message}");
+ }
+ }
+
+ throw new PsrConfigurationException(errors.Count > 0 ? string.Join("; ", errors) : $"does not match {schemaName}");
+ }
+ }
+
+ private static JsonNode WriteSource(BindingSource source) => source switch
+ {
+ ConstantSource constant => new JsonObject { ["const"] = constant.Value?.DeepClone() },
+ _ => JsonValue.Create(source.ToString())!,
+ };
+
+ private static JsonSchema LoadSchema(string name)
+ {
+ using var stream = typeof(CatalogManifests).Assembly.GetManifestResourceStream($"PythonScriptRunner.Schemas.{name}")
+ ?? throw new InvalidOperationException($"Embedded schema {name} not found.");
+ using var reader = new StreamReader(stream);
+ return JsonSchema.FromText(reader.ReadToEnd());
+ }
+}
diff --git a/src/PythonScriptRunner.Core/Catalog/FileCatalogSource.cs b/src/PythonScriptRunner.Core/Catalog/FileCatalogSource.cs
index fe30434..94b0efe 100644
--- a/src/PythonScriptRunner.Core/Catalog/FileCatalogSource.cs
+++ b/src/PythonScriptRunner.Core/Catalog/FileCatalogSource.cs
@@ -1,7 +1,4 @@
using System.Text.Json;
-using System.Text.Json.Nodes;
-using Json.Schema;
-using PythonScriptRunner.Core.Pipelines;
namespace PythonScriptRunner.Core.Catalog;
@@ -11,11 +8,6 @@ namespace PythonScriptRunner.Core.Catalog;
///
internal sealed class FileCatalogSource(string root) : ICatalogSource
{
- private const string LockFileName = "requirements.lock";
-
- private static readonly Lazy StageSchema = new(() => LoadSchema("stage.v1.json"), LazyThreadSafetyMode.ExecutionAndPublication);
- private static readonly Lazy PipelineSchema = new(() => LoadSchema("pipeline.v1.json"), LazyThreadSafetyMode.ExecutionAndPublication);
-
public string Description => $"folder '{root}'";
public IReadOnlyList LoadStages()
@@ -26,7 +18,7 @@ public IReadOnlyList LoadStages()
return [];
}
- var manifests = Directory.EnumerateDirectories(folder).Order(StringComparer.Ordinal).Select(d => Path.Combine(d, "stage.json"));
+ var manifests = Directory.EnumerateDirectories(folder).Order(StringComparer.Ordinal).Select(d => Path.Combine(d, CatalogManifests.StageManifestName));
return LoadAll(manifests, "stage catalog", (manifest, errors) =>
File.Exists(manifest) ? LoadStage(manifest, errors) : Missing(manifest, errors));
}
@@ -39,8 +31,8 @@ public IReadOnlyList LoadPipelines()
return [];
}
- var manifests = Directory.EnumerateFiles(folder, "*.pipeline.json").Order(StringComparer.Ordinal);
- return LoadAll(manifests, "pipeline catalog", LoadPipeline);
+ var manifests = Directory.EnumerateFiles(folder, "*" + CatalogManifests.PipelineSuffix).Order(StringComparer.Ordinal);
+ return LoadAll(manifests, "pipeline catalog", (manifest, _) => CatalogManifests.ReadPipeline(File.ReadAllText(manifest)));
}
private string? Subfolder(string name)
@@ -88,14 +80,10 @@ private static List LoadAll(IEnumerable manifests, string what, Fu
private static StageDefinition? LoadStage(string manifest, List errors)
{
- if (ParseAndValidate(manifest, StageSchema.Value, "schemas/stage.v1.json", errors) is not { } json)
- {
- return null;
- }
-
+ var json = CatalogManifests.ParseValidated(File.ReadAllText(manifest), pipeline: false);
var folder = Path.GetDirectoryName(manifest)!;
var requirementsName = json["requirements"]?.GetValue();
- var requirementsPath = Path.Combine(folder, requirementsName ?? "requirements.txt");
+ var requirementsPath = Path.Combine(folder, requirementsName ?? CatalogManifests.RequirementsName);
string? requirements = null;
if (File.Exists(requirementsPath))
{
@@ -107,72 +95,8 @@ private static List LoadAll(IEnumerable manifests, string what, Fu
return null;
}
- var lockPath = Path.Combine(folder, LockFileName);
- var retry = json["retry"]?.AsObject();
- return new StageDefinition
- {
- Key = json["key"]!.GetValue(),
- Version = json["version"]!.GetValue(),
- Name = json["name"]!.GetValue(),
- Description = json["description"]?.GetValue(),
- Script = ScriptSource.FromFile(Path.Combine(folder, json["script"]?.GetValue() ?? "script.py")),
- PythonVersion = json["python"]?.GetValue(),
- Requirements = requirements,
- LockedRequirements = File.Exists(lockPath) ? File.ReadAllText(lockPath) : null,
- Inputs = [.. json["inputs"]!.AsArray().Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))],
- Outputs = [.. json["outputs"]!.AsArray().Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))],
- RequiredSecrets = json["secrets"] is JsonArray secrets ? [.. secrets.Select(s => s!.GetValue())] : [],
- Timeout = json["timeoutSeconds"] is { } seconds ? TimeSpan.FromSeconds(seconds.GetValue()) : null,
- MemoryLimitMb = json["memoryLimitMb"]?.GetValue(),
- Retry = retry is null
- ? RetryPolicy.None
- : new RetryPolicy(
- retry["maxAttempts"]?.GetValue() ?? 0,
- TimeSpan.FromSeconds(retry["initialDelaySeconds"]?.GetValue() ?? 10),
- retry["backoffMultiplier"]?.GetValue() ?? 2.0),
- Idempotent = json["idempotent"]?.GetValue() ?? false,
- };
- }
-
- private static PipelineDefinition? LoadPipeline(string manifest, List errors) =>
- ParseAndValidate(manifest, PipelineSchema.Value, "schemas/pipeline.v1.json", errors) is { } json
- ? PipelineJson.FromJson(json)
- : null;
-
- private static JsonObject? ParseAndValidate(string manifest, JsonSchema schema, string schemaName, List errors)
- {
- using var document = JsonDocument.Parse(
- File.ReadAllText(manifest),
- new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true });
-
- var results = schema.Evaluate(document.RootElement, new EvaluationOptions { OutputFormat = OutputFormat.List });
- if (results.IsValid)
- {
- return JsonNode.Parse(document.RootElement.GetRawText())!.AsObject();
- }
-
- var before = errors.Count;
- foreach (var detail in (IEnumerable)[results, .. results.Details ?? []])
- {
- foreach (var message in detail.Errors?.Values ?? Enumerable.Empty())
- {
- errors.Add($"{manifest}: {(detail.InstanceLocation.ToString() is { Length: > 0 } at ? at : "/")}: {message}");
- }
- }
-
- if (errors.Count == before)
- {
- errors.Add($"{manifest}: does not match {schemaName}");
- }
-
- return null;
- }
-
- private static JsonSchema LoadSchema(string name)
- {
- using var stream = typeof(FileCatalogSource).Assembly.GetManifestResourceStream($"PythonScriptRunner.Schemas.{name}")
- ?? throw new InvalidOperationException($"Embedded schema {name} not found.");
- using var reader = new StreamReader(stream);
- return JsonSchema.FromText(reader.ReadToEnd());
+ var lockPath = Path.Combine(folder, CatalogManifests.LockName);
+ var script = ScriptSource.FromFile(Path.Combine(folder, json["script"]?.GetValue() ?? CatalogManifests.ScriptName));
+ return CatalogManifests.StageFromJson(json, script, requirements, File.Exists(lockPath) ? File.ReadAllText(lockPath) : null);
}
}
diff --git a/src/PythonScriptRunner.Core/Catalog/StageDefinitionValidator.cs b/src/PythonScriptRunner.Core/Catalog/StageDefinitionValidator.cs
index 60fcf09..624806b 100644
--- a/src/PythonScriptRunner.Core/Catalog/StageDefinitionValidator.cs
+++ b/src/PythonScriptRunner.Core/Catalog/StageDefinitionValidator.cs
@@ -3,6 +3,18 @@
namespace PythonScriptRunner.Core.Catalog;
+/// Public validator of stage definitions (used by the panel and by publication).
+internal sealed class StageValidator : IStageValidator
+{
+ public const string Code = "STAGE_DEFINITION";
+
+ public ValidationReport Validate(StageDefinition stage)
+ {
+ ArgumentNullException.ThrowIfNull(stage);
+ return new ValidationReport([.. StageDefinitionValidator.Validate(stage).Select(e => new ValidationIssue(Code, e))], []);
+ }
+}
+
/// Validates a stage definition (same rules as schemas/stage.v1.json plus semantic checks).
internal static partial class StageDefinitionValidator
{
diff --git a/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs b/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs
index bea51a8..f9d248d 100644
--- a/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs
+++ b/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs
@@ -35,13 +35,16 @@ public static IPythonScriptRunnerBuilder AddPythonScriptRunner(
services.TryAddSingleton(TimeProvider.System);
services.TryAddSingleton();
services.TryAddSingleton();
+ services.TryAddSingleton();
services.TryAddSingleton();
services.TryAddSingleton();
services.TryAddSingleton();
services.TryAddSingleton();
services.TryAddSingleton();
services.TryAddSingleton();
- services.TryAddSingleton();
+ services.TryAddSingleton();
+ services.TryAddSingleton(sp => sp.GetRequiredService());
+ services.TryAddSingleton(sp => sp.GetRequiredService());
services.TryAddEnumerable(ServiceDescriptor.Singleton());
return new PythonScriptRunnerBuilder(services);
}
diff --git a/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs b/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs
index a29269c..1c9fce8 100644
--- a/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs
+++ b/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs
@@ -73,4 +73,7 @@ internal static partial class Log
[LoggerMessage(9, LogLevel.Warning, "Pipeline {PipelineKey}@{PipelineVersion}: {Warning}")]
public static partial void PipelineWarning(ILogger logger, string pipelineKey, int pipelineVersion, string warning);
+
+ [LoggerMessage(10, LogLevel.Error, "Could not record the failure of run {RunId}")]
+ public static partial void RecordingFailed(ILogger logger, RunId runId, Exception exception);
}
diff --git a/src/PythonScriptRunner.Core/Environments/IEnvironmentManager.cs b/src/PythonScriptRunner.Core/Environments/IEnvironmentManager.cs
index 5ee278f..3cb1f70 100644
--- a/src/PythonScriptRunner.Core/Environments/IEnvironmentManager.cs
+++ b/src/PythonScriptRunner.Core/Environments/IEnvironmentManager.cs
@@ -11,6 +11,19 @@ public interface IEnvironmentManager
/// Resolution failed.
Task LockAsync(string? pythonVersion, string? requirements, CancellationToken cancellationToken = default);
+ /// Compiles a script with the given Python version (no dependencies needed).
+ /// Python version; null uses the runner default.
+ /// Python source.
+ /// Cancellation token.
+ /// The syntax error, or null when the script compiles.
+ /// uv or the Python version is not available.
+ Task CheckSyntaxAsync(string? pythonVersion, string script, CancellationToken cancellationToken = default);
+
+ /// Hash that identifies the environment of a Python version and locked requirements (the venv cache key).
+ /// Python version; null uses the runner default.
+ /// Locked requirements (empty when there are none).
+ string ComputeHash(string? pythonVersion, string? lockedRequirements);
+
/// Returns the environment of a stage, creating it on first use.
/// The stage.
/// Cancellation token.
@@ -25,6 +38,12 @@ public interface IEnvironmentManager
/// Whether it was created by this call.
public sealed record PythonEnvironment(string Hash, string Directory, string PythonExecutable, bool Created);
+/// A script that does not compile.
+/// Python error message.
+/// 1-based line, when known.
+/// 1-based column, when known.
+public sealed record ScriptSyntaxError(string Message, int? Line, int? Column);
+
/// A Python environment could not be resolved or created.
public sealed class PythonEnvironmentException : PsrException
{
diff --git a/src/PythonScriptRunner.Core/Environments/UvEnvironmentManager.cs b/src/PythonScriptRunner.Core/Environments/UvEnvironmentManager.cs
index a2275de..701abeb 100644
--- a/src/PythonScriptRunner.Core/Environments/UvEnvironmentManager.cs
+++ b/src/PythonScriptRunner.Core/Environments/UvEnvironmentManager.cs
@@ -2,6 +2,7 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
+using System.Text.Json.Nodes;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using PythonScriptRunner.Core.Diagnostics;
@@ -18,6 +19,62 @@ internal sealed class UvEnvironmentManager(
private const string ReadyMarker = ".psr-ready";
private static readonly TimeSpan LockTimeout = TimeSpan.FromMinutes(20);
+ /// Compiles the script given as argument and prints the outcome as JSON (same check as py_compile).
+ private const string SyntaxChecker = """
+ import json
+ import sys
+
+ with open(sys.argv[1], encoding="utf-8") as source_file:
+ source = source_file.read()
+ try:
+ compile(source, "script.py", "exec")
+ except (SyntaxError, ValueError) as error:
+ print(json.dumps({
+ "ok": False,
+ "message": getattr(error, "msg", None) or str(error),
+ "line": getattr(error, "lineno", None),
+ "column": getattr(error, "offset", None),
+ }))
+ else:
+ print(json.dumps({"ok": True}))
+ """;
+
+ public async Task CheckSyntaxAsync(string? pythonVersion, string script, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(script);
+ var folder = Path.Combine(Path.GetTempPath(), "psr-syntax", Guid.NewGuid().ToString("N"));
+ Directory.CreateDirectory(folder);
+ try
+ {
+ var checker = Path.Combine(folder, "check_syntax.py");
+ var target = Path.Combine(folder, "script.py");
+ await File.WriteAllTextAsync(checker, SyntaxChecker, cancellationToken).ConfigureAwait(false);
+ await File.WriteAllTextAsync(target, script, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), cancellationToken).ConfigureAwait(false);
+ var version = pythonVersion ?? options.Value.DefaultPythonVersion;
+ var output = await CreateCli().RunAsync(
+ ["run", "--no-project", "--python", version, "--directory", folder, "python", "-I", checker, target], null, cancellationToken)
+ .ConfigureAwait(false);
+ var result = JsonNode.Parse(output.Trim().Split('\n')[^1])!.AsObject();
+ return result["ok"]!.GetValue()
+ ? null
+ : new ScriptSyntaxError(result["message"]?.GetValue() ?? "invalid syntax", result["line"]?.GetValue(), result["column"]?.GetValue());
+ }
+ finally
+ {
+ try
+ {
+ Directory.Delete(folder, recursive: true);
+ }
+ catch (IOException)
+ {
+ // Best effort.
+ }
+ }
+ }
+
+ public string ComputeHash(string? pythonVersion, string? lockedRequirements) =>
+ Sha256Hex($"{pythonVersion ?? options.Value.DefaultPythonVersion}\n{Normalize(lockedRequirements ?? string.Empty)}\n{PsrunnerIdentity(options.Value.PsrunnerRequirement)}");
+
public async Task LockAsync(string? pythonVersion, string? requirements, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(requirements))
@@ -40,7 +97,7 @@ public async Task EnsureAsync(StageDefinition stage, Cancella
var settings = options.Value;
var version = stage.PythonVersion ?? settings.DefaultPythonVersion;
var locked = stage.LockedRequirements ?? await GetCachedLockAsync(stage, version, cancellationToken).ConfigureAwait(false);
- var hash = Sha256Hex($"{version}\n{Normalize(locked)}\n{PsrunnerIdentity(settings.PsrunnerRequirement)}");
+ var hash = ComputeHash(version, locked);
var directory = Path.Combine(settings.EnvironmentCacheRoot, "envs", hash[..32]);
var python = OperatingSystem.IsWindows()
? Path.Combine(directory, "Scripts", "python.exe")
@@ -113,7 +170,16 @@ private async Task GetCachedLockAsync(StageDefinition stage, string vers
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
var temporary = $"{path}.{Guid.NewGuid():N}.tmp";
await File.WriteAllTextAsync(temporary, locked, cancellationToken).ConfigureAwait(false);
- File.Move(temporary, path, overwrite: true);
+ try
+ {
+ File.Move(temporary, path, overwrite: true);
+ }
+ catch (Exception ex) when ((ex is IOException or UnauthorizedAccessException) && File.Exists(path))
+ {
+ // Another process stored the same lock at the same moment (Windows denies replacing a file in use); keep theirs.
+ File.Delete(temporary);
+ }
+
return locked;
}
diff --git a/src/PythonScriptRunner.Core/Pipelines/PipelineValidator.cs b/src/PythonScriptRunner.Core/Pipelines/PipelineValidator.cs
index ed156a1..7954045 100644
--- a/src/PythonScriptRunner.Core/Pipelines/PipelineValidator.cs
+++ b/src/PythonScriptRunner.Core/Pipelines/PipelineValidator.cs
@@ -12,7 +12,14 @@ public async ValueTask ValidateAsync(PipelineDefinition pipeli
var reference = (step.StageKey, step.StageVersion);
if (!stages.ContainsKey(reference))
{
- stages[reference] = await catalog.FindStageAsync(step.StageKey, step.StageVersion, cancellationToken).ConfigureAwait(false);
+ try
+ {
+ stages[reference] = await catalog.FindStageAsync(step.StageKey, step.StageVersion, cancellationToken).ConfigureAwait(false);
+ }
+ catch (VersionNotRunnableException)
+ {
+ stages[reference] = null;
+ }
}
}
diff --git a/src/PythonScriptRunner.Core/Pipelines/StepInputResolver.cs b/src/PythonScriptRunner.Core/Pipelines/StepInputResolver.cs
index 8854dd9..d8f78c3 100644
--- a/src/PythonScriptRunner.Core/Pipelines/StepInputResolver.cs
+++ b/src/PythonScriptRunner.Core/Pipelines/StepInputResolver.cs
@@ -1,11 +1,10 @@
using System.Text.Json.Nodes;
-using PythonScriptRunner.Core.Execution;
using PythonScriptRunner.Core.Values;
namespace PythonScriptRunner.Core.Pipelines;
-/// Inputs of one pipeline step after resolving its bindings.
-internal sealed record StepInputs(JsonObject Wire, IReadOnlyList Files, IReadOnlyList Errors);
+/// Inputs of one pipeline step after resolving its bindings (files are shared with their source, never copied).
+internal sealed record StepInputs(JsonObject Wire, IReadOnlyDictionary> Files, IReadOnlyList Errors);
/// Resolves the inputs of a step from pipeline inputs, earlier step outputs, constants and implicit bindings.
internal static class StepInputResolver
@@ -20,7 +19,7 @@ public static StepInputs Resolve(
{
var step = pipeline.Steps[index];
var wire = new JsonObject();
- var files = new List();
+ var files = new Dictionary>(StringComparer.Ordinal);
var errors = new List();
foreach (var target in stage.Inputs)
{
@@ -30,8 +29,7 @@ public static StepInputs Resolve(
{
if (sourceFiles is { Count: > 0 })
{
- files.AddRange(sourceFiles.Select((file, i) =>
- new InputFileCopy(target.Name, i, file.LocalPath!, file.FileName, file.ContentType, file.Length ?? 0, file.Sha256!)));
+ files[target.Name] = sourceFiles;
}
else if (target.Required)
{
diff --git a/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj b/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj
index e5b04ba..70f4fef 100644
--- a/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj
+++ b/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj
@@ -1,4 +1,4 @@
-
+
PythonScriptRunner
@@ -16,6 +16,8 @@
+
+
diff --git a/src/PythonScriptRunner.Core/Running/Infrastructure.cs b/src/PythonScriptRunner.Core/Running/Infrastructure.cs
index 1133cbc..7c747ba 100644
--- a/src/PythonScriptRunner.Core/Running/Infrastructure.cs
+++ b/src/PythonScriptRunner.Core/Running/Infrastructure.cs
@@ -8,10 +8,18 @@ namespace PythonScriptRunner.Core.Running;
/// Default recorder: nothing is persisted without the EF package.
internal sealed class NullRunRecorder : IRunRecorder
{
- public ValueTask RunStartedAsync(RunStarted run, CancellationToken cancellationToken) => ValueTask.CompletedTask;
+ public ValueTask RunStartingAsync(RunStarting run, CancellationToken cancellationToken) => ValueTask.FromResult(null);
+
+ public ValueTask RunResumingAsync(RunResuming run, CancellationToken cancellationToken) => ValueTask.CompletedTask;
+
+ public ValueTask StepStartedAsync(StepStarted step, CancellationToken cancellationToken) => ValueTask.CompletedTask;
+
+ public ValueTask AttemptStartedAsync(AttemptStarted attempt, CancellationToken cancellationToken) => ValueTask.CompletedTask;
public ValueTask AttemptFinishedAsync(AttemptFinished attempt, CancellationToken cancellationToken) => ValueTask.CompletedTask;
+ public ValueTask StepFinishedAsync(StepFinished step, CancellationToken cancellationToken) => ValueTask.CompletedTask;
+
public ValueTask RunFinishedAsync(RunFinished run, CancellationToken cancellationToken) => ValueTask.CompletedTask;
}
@@ -26,12 +34,22 @@ internal sealed class ConfigurationSecretProvider(IConfiguration? configuration
/// Fails fast when the host starts: validates options, loads catalogs and removes leftover folders.
internal sealed class PythonScriptRunnerStartup(
- IOptions options, ICatalog catalog, WorkspaceJanitor janitor) : IHostedService
+ IOptions options, ICatalog catalog, IEnumerable sources, WorkspaceJanitor janitor) : IHostedService
{
public Task StartAsync(CancellationToken cancellationToken)
{
_ = options.Value;
- (catalog as CompositeCatalog)?.EnsureLoaded();
+ if (catalog is CompositeCatalog composite)
+ {
+ composite.EnsureLoaded();
+ }
+ else if (sources.FirstOrDefault() is { } ignored)
+ {
+ throw new PsrConfigurationException(
+ $"The catalog is provided by {catalog.GetType().Name}, so stages and pipelines added with AddStage, AddPipeline or " +
+ $"AddFileCatalog ({ignored.Description}) would be ignored. Import them into that catalog instead.");
+ }
+
janitor.EnsureCleaned();
return Task.CompletedTask;
}
diff --git a/src/PythonScriptRunner.Core/Running/PythonRunner.cs b/src/PythonScriptRunner.Core/Running/PythonRunner.cs
index a3b32b8..78ec229 100644
--- a/src/PythonScriptRunner.Core/Running/PythonRunner.cs
+++ b/src/PythonScriptRunner.Core/Running/PythonRunner.cs
@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
using System.Diagnostics;
using System.Globalization;
using System.Security.Cryptography;
@@ -12,11 +13,13 @@
namespace PythonScriptRunner.Core.Running;
///
-/// Runs stages and pipelines in the current process: validation, secrets, environment,
-/// attempts with retry and, for pipelines, bindings between steps.
+/// Runs stages and pipelines in the current process: validation, secrets, environment, attempts with retry and, for
+/// pipelines, bindings between steps. Every transition is reported to the .
///
-internal sealed class PythonRunner : IPythonRunner, IDisposable
+internal sealed class PythonRunner : IPythonRunner, IRunEngine, IDisposable
{
+ private static readonly IReadOnlyDictionary> NoFiles = new Dictionary>();
+
private readonly ICatalog _catalog;
private readonly IEnvironmentManager _environments;
private readonly IPythonExecutor _executor;
@@ -29,6 +32,7 @@ internal sealed class PythonRunner : IPythonRunner, IDisposable
private readonly ILogger _logger;
private readonly ILogger _scriptLogger;
private readonly Lazy _concurrency;
+ private readonly ConcurrentDictionary _active = new();
public PythonRunner(
ICatalog catalog,
@@ -75,34 +79,112 @@ public async Task RunStageAsync(
string stageKey, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(stageKey);
+ var stage = await ResolveStageAsync(stageKey, options?.Version, cancellationToken).ConfigureAwait(false);
+ return await RunStageAsync(stage, RunKind.Stage, inputs, options, cancellationToken).ConfigureAwait(false);
+ }
+
+ public async Task RunPipelineAsync(
+ string pipelineKey, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default)
+ {
+ ArgumentException.ThrowIfNullOrWhiteSpace(pipelineKey);
+ var pipeline = await ResolvePipelineAsync(pipelineKey, options?.Version, cancellationToken).ConfigureAwait(false);
+ return await RunPipelineAsync(pipeline, RunKind.Pipeline, inputs, options, cancellationToken).ConfigureAwait(false);
+ }
+
+ public async Task RunStageAsync(
+ StageDefinition stage, RunKind kind, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(stage);
var settings = Settings;
_janitor.EnsureCleaned();
- var stage = await ResolveStageAsync(stageKey, options?.Version, cancellationToken).ConfigureAwait(false);
var bound = InputBinder.Bind(stage.Inputs, inputs, settings.MaxFileBytes);
cancellationToken.ThrowIfCancellationRequested();
-
- return await RunAsync(RunKind.Stage, stage.Key, stage.Version, bound, options, settings, async (context, inputFiles) =>
- {
- var step = await ExecuteStepAsync(stage, stage.Key, 1, bound.Wire, ToCopies(inputFiles), context, cancellationToken).ConfigureAwait(false);
- return new RunOutcome(StatusOf(step.Status), [step], step.Outputs, step.Files, step.Error);
- }, cancellationToken).ConfigureAwait(false);
+ var target = new RunTarget(kind, stage.Key, stage.Version, [new PlannedStep(stage.Key, 1, stage.Key, stage.Version)]);
+ return await RunNewAsync(target, bound, options, settings,
+ (context, inputFiles, token) => ExecuteStageAsync(stage, bound.Wire, inputFiles, context, prior: null, token),
+ cancellationToken).ConfigureAwait(false);
}
public async Task RunPipelineAsync(
- string pipelineKey, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default)
+ PipelineDefinition pipeline, RunKind kind, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default)
{
- ArgumentException.ThrowIfNullOrWhiteSpace(pipelineKey);
+ ArgumentNullException.ThrowIfNull(pipeline);
var settings = Settings;
_janitor.EnsureCleaned();
- var pipeline = await ResolvePipelineAsync(pipelineKey, options?.Version, cancellationToken).ConfigureAwait(false);
var bound = InputBinder.Bind(pipeline.Inputs, inputs, settings.MaxFileBytes);
cancellationToken.ThrowIfCancellationRequested();
-
- return await RunAsync(RunKind.Pipeline, pipeline.Key, pipeline.Version, bound, options, settings,
- (context, inputFiles) => ExecutePipelineAsync(pipeline, bound.Wire, inputFiles, context, cancellationToken),
+ var steps = pipeline.Steps.Select((s, i) => new PlannedStep(s.Key, i + 1, s.StageKey, s.StageVersion)).ToList();
+ var target = new RunTarget(kind, pipeline.Key, pipeline.Version, steps);
+ return await RunNewAsync(target, bound, options, settings,
+ (context, inputFiles, token) => ExecutePipelineAsync(pipeline, bound.Wire, inputFiles, context, NoPrevious, token),
cancellationToken).ConfigureAwait(false);
}
+ public async Task ResumeAsync(ResumeRequest request, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(request);
+ if ((request.Stage is null) == (request.Pipeline is null))
+ {
+ throw new ArgumentException("Set exactly one of Stage or Pipeline.", nameof(request));
+ }
+
+ var settings = Settings;
+ _janitor.EnsureCleaned();
+ var context = new RunContext(new RunWorkspace(settings.WorkRoot, request.RunId), settings, request.Kind, request.TriggeredBy);
+ var started = _time.GetTimestamp();
+ using var cancellation = Register(request.RunId, cancellationToken);
+ var token = cancellation.Token;
+ try
+ {
+ var inputFiles = await MaterializeStoredAsync(request.InputFiles, context.Workspace.InputsDirectory, token).ConfigureAwait(false);
+ var previous = new Dictionary(StringComparer.Ordinal);
+ foreach (var step in request.PreviousSteps)
+ {
+ previous[step.StepKey] = step.Status == StepStatus.Succeeded
+ ? step with
+ {
+ Files = await MaterializeStoredAsync(step.Files, Path.Combine(context.Workspace.ResultsDirectory, step.StepKey), token)
+ .ConfigureAwait(false),
+ }
+ : step;
+ }
+
+ context.UsedBytes = TotalBytes(inputFiles) + previous.Values.Sum(s => TotalBytes(s.Files));
+ await _recorder.RunResumingAsync(new RunResuming(request.RunId, request.TriggeredBy, _time.GetUtcNow()), token).ConfigureAwait(false);
+ return await CompleteAsync(context, started, () => request.Stage is { } stage
+ ? ExecuteStageAsync(stage, request.Inputs, inputFiles, context, previous.GetValueOrDefault(stage.Key), token)
+ : ExecutePipelineAsync(request.Pipeline!, request.Inputs, inputFiles, context, previous, token), token).ConfigureAwait(false);
+ }
+ catch
+ {
+ await context.Workspace.DeleteResultsAsync().ConfigureAwait(false);
+ throw;
+ }
+ finally
+ {
+ _active.TryRemove(request.RunId, out _);
+ context.Workspace.DeleteRun();
+ }
+ }
+
+ public bool RequestCancellation(RunId runId)
+ {
+ if (!_active.TryGetValue(runId, out var source))
+ {
+ return false;
+ }
+
+ try
+ {
+ source.Cancel();
+ return true;
+ }
+ catch (ObjectDisposedException)
+ {
+ return false;
+ }
+ }
+
public void Dispose()
{
if (_concurrency.IsValueCreated)
@@ -111,43 +193,54 @@ public void Dispose()
}
}
- private async Task RunAsync(
- RunKind kind,
- string key,
- int version,
+ private static IReadOnlyDictionary NoPrevious { get; } = new Dictionary();
+
+ private async Task RunNewAsync(
+ RunTarget target,
BoundInputs bound,
RunOptions? options,
PythonScriptRunnerOptions settings,
- Func>, Task> body,
+ Func>, CancellationToken, Task> body,
CancellationToken cancellationToken)
{
- var runId = RunId.New();
+ var runId = RunId.New(_time.GetUtcNow());
+ var context = new RunContext(new RunWorkspace(settings.WorkRoot, runId), settings, target.Kind, options?.TriggeredBy ?? Actor.Library.Subject);
var started = _time.GetTimestamp();
- await _recorder.RunStartedAsync(
- new RunStarted(runId, kind, key, version, options?.TriggeredBy ?? "system:library", _time.GetUtcNow(), (JsonObject)bound.Wire.DeepClone()),
- cancellationToken).ConfigureAwait(false);
-
- var context = new RunContext(new RunWorkspace(settings.WorkRoot, runId), settings);
+ using var cancellation = Register(runId, cancellationToken);
+ var token = cancellation.Token;
try
{
- RunOutcome outcome;
- try
- {
- var inputFiles = await MaterializeInputsAsync(bound.Files, context.Workspace, cancellationToken).ConfigureAwait(false);
- context.UsedBytes = inputFiles.Values.SelectMany(files => files).Sum(file => file.Length ?? 0);
- outcome = context.UsedBytes > settings.MaxRunFilesBytes
- ? RunOutcome.Aborted(RunStatus.Failed, new RunError(ErrorKind.ArtifactLimitExceeded, "The input files exceed the run limit.") { Code = "RUN_FILES_TOO_LARGE" })
- : await body(context, inputFiles).ConfigureAwait(false);
- }
- catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ var inputFiles = await MaterializeInputsAsync(bound.Files, context.Workspace, token).ConfigureAwait(false);
+ context.UsedBytes = TotalBytes(inputFiles);
+ var overBudget = context.UsedBytes > settings.MaxRunFilesBytes;
+ var existing = await _recorder.RunStartingAsync(
+ new RunStarting
+ {
+ RunId = runId,
+ Kind = target.Kind,
+ TargetKey = target.Key,
+ TargetVersion = target.Version,
+ Trigger = options?.Trigger ?? RunTrigger.Library,
+ TriggeredBy = context.TriggeredBy,
+ IdempotencyKey = options?.IdempotencyKey,
+ Inputs = (JsonObject)bound.Wire.DeepClone(),
+ InputFiles = overBudget ? NoFiles : inputFiles,
+ Steps = target.Steps,
+ StartedAt = runId.CreatedAt,
+ },
+ token).ConfigureAwait(false);
+ if (existing is not null)
{
- outcome = RunOutcome.Aborted(RunStatus.Cancelled, new RunError(ErrorKind.Cancelled, "The run was cancelled."));
+ await context.Workspace.DeleteResultsAsync().ConfigureAwait(false);
+ return existing;
}
- await _recorder.RunFinishedAsync(new RunFinished(runId, outcome.Status, _time.GetUtcNow(), outcome.Outputs, outcome.Error), CancellationToken.None)
- .ConfigureAwait(false);
- return new RunResult(runId, kind, outcome.Status, _time.GetElapsedTime(started), outcome.Steps, outcome.Outputs, outcome.Files, outcome.Error,
- context.Workspace.DeleteResultsAsync);
+ return await CompleteAsync(context, started, () => overBudget
+ ? Task.FromResult(RunOutcome.Aborted(RunStatus.Failed, new RunError(ErrorKind.ArtifactLimitExceeded, "The input files exceed the run limit.")
+ {
+ Code = "RUN_FILES_TOO_LARGE",
+ }))
+ : body(context, inputFiles, token), token).ConfigureAwait(false);
}
catch
{
@@ -156,15 +249,76 @@ await _recorder.RunFinishedAsync(new RunFinished(runId, outcome.Status, _time.Ge
}
finally
{
+ _active.TryRemove(runId, out _);
context.Workspace.DeleteRun();
}
}
+ /// Executes the body, records the end of the run and builds the result.
+ private async Task CompleteAsync(RunContext context, long started, Func> body, CancellationToken cancellationToken)
+ {
+ RunOutcome outcome;
+ try
+ {
+ outcome = await body().ConfigureAwait(false);
+ }
+ catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+ {
+ outcome = RunOutcome.Aborted(RunStatus.Cancelled, new RunError(ErrorKind.Cancelled, "The run was cancelled."));
+ }
+ catch (Exception ex) when (ex is not OperationCanceledException)
+ {
+ await RecordCrashAsync(context.RunId, ex).ConfigureAwait(false);
+ throw;
+ }
+
+ await _recorder.RunFinishedAsync(
+ new RunFinished(context.RunId, outcome.Status, _time.GetUtcNow(), outcome.Outputs, outcome.Files, outcome.Error),
+ CancellationToken.None).ConfigureAwait(false);
+ return new RunResult(context.RunId, context.Kind, outcome.Status, _time.GetElapsedTime(started), outcome.Steps, outcome.Outputs, outcome.Files,
+ outcome.Error, context.Workspace.DeleteResultsAsync);
+ }
+
+ /// Best effort: a run interrupted by an unexpected error must not stay "running" in the history.
+ private async Task RecordCrashAsync(RunId runId, Exception exception)
+ {
+ try
+ {
+ var error = new RunError(ErrorKind.ProcessCrashed, $"The runner failed: {exception.Message}") { Code = "RUNNER_ERROR" };
+ await _recorder.RunFinishedAsync(new RunFinished(runId, RunStatus.Failed, _time.GetUtcNow(), null, null, error), CancellationToken.None)
+ .ConfigureAwait(false);
+ }
+ catch (Exception recordError) when (recordError is not OutOfMemoryException)
+ {
+ Log.RecordingFailed(_logger, runId, recordError);
+ }
+ }
+
+ private CancellationTokenSource Register(RunId runId, CancellationToken cancellationToken)
+ {
+ var source = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ _active[runId] = source;
+ return source;
+ }
+
+ private async Task ExecuteStageAsync(
+ StageDefinition stage,
+ JsonObject inputs,
+ IReadOnlyDictionary> files,
+ RunContext context,
+ StepResult? prior,
+ CancellationToken cancellationToken)
+ {
+ var step = await ExecuteStepAsync(stage, stage.Key, 1, inputs, files, context, prior, cancellationToken).ConfigureAwait(false);
+ return new RunOutcome(StatusOf(step.Status), [step], step.Outputs, step.Files, step.Error);
+ }
+
private async Task ExecutePipelineAsync(
PipelineDefinition pipeline,
JsonObject pipelineInputs,
IReadOnlyDictionary> pipelineFiles,
RunContext context,
+ IReadOnlyDictionary previous,
CancellationToken cancellationToken)
{
var steps = new List();
@@ -174,9 +328,17 @@ private async Task ExecutePipelineAsync(
for (var i = 0; i < pipeline.Steps.Count; i++)
{
var definition = pipeline.Steps[i];
+ var prior = previous.GetValueOrDefault(definition.Key);
if (status != RunStatus.Succeeded)
{
- steps.Add(Skipped(definition, i + 1));
+ steps.Add(await RecordStepAsync(context, Skipped(definition, i + 1, prior)).ConfigureAwait(false));
+ continue;
+ }
+
+ if (prior is { Status: StepStatus.Succeeded })
+ {
+ steps.Add(prior);
+ completed[definition.Key] = prior;
continue;
}
@@ -184,8 +346,8 @@ private async Task ExecutePipelineAsync(
?? throw new PsrConfigurationException($"Stage '{definition.StageKey}@{definition.StageVersion}' of pipeline '{pipeline.Key}' is not in the catalog.");
var inputs = StepInputResolver.Resolve(stage, pipeline, i, pipelineInputs, pipelineFiles, completed);
var result = inputs.Errors.Count > 0
- ? InvalidInputs(stage, definition.Key, i + 1, inputs.Errors)
- : await ExecuteStepAsync(stage, definition.Key, i + 1, inputs.Wire, inputs.Files, context, cancellationToken).ConfigureAwait(false);
+ ? await RecordStepAsync(context, InvalidInputs(stage, definition.Key, i + 1, inputs.Errors, prior)).ConfigureAwait(false)
+ : await ExecuteStepAsync(stage, definition.Key, i + 1, inputs.Wire, inputs.Files, context, prior, cancellationToken).ConfigureAwait(false);
steps.Add(result);
if (result.Status == StepStatus.Succeeded)
{
@@ -252,16 +414,25 @@ private async Task ResolvePipelineAsync(string key, int? ver
throw new PipelineNotFoundException(key);
}
+ /// Runs the attempts of one step; a step resumed after a failure continues the attempt numbering.
private async Task ExecuteStepAsync(
StageDefinition stage,
string stepKey,
int order,
JsonObject inputs,
- IReadOnlyList files,
+ IReadOnlyDictionary> files,
RunContext context,
+ StepResult? prior,
CancellationToken cancellationToken)
{
- var attempts = new List();
+ await _recorder.StepStartedAsync(
+ new StepStarted(context.RunId, stepKey, order, WithFilePlaceholders(inputs, files, stage), files, _time.GetUtcNow()),
+ CancellationToken.None).ConfigureAwait(false);
+
+ var attempts = new List(prior?.Attempts ?? []);
+ var first = attempts.Count + 1;
+ var firstCause = prior is { Status: not (StepStatus.Pending or StepStatus.Skipped) } ? AttemptCause.Reprocess : AttemptCause.Initial;
+ var copies = ToCopies(files);
StepResult Finish(StepStatus status, RunError? error, AttemptOutcome? success = null) => new()
{
StepKey = stepKey,
@@ -271,35 +442,41 @@ private async Task ExecuteStepAsync(
Status = status,
Attempts = attempts,
Outputs = success?.Outputs,
- Files = success?.Files ?? new Dictionary>(),
+ Files = success?.Files ?? NoFiles,
Error = error is null ? null : error with { StepKey = stepKey },
};
- for (var number = 1; ; number++)
+ StepResult result;
+ for (var number = first; ; number++)
{
- var cause = number == 1 ? AttemptCause.Initial : AttemptCause.AutoRetry;
- var outcome = await RunAttemptAsync(stage, stepKey, number, cause, inputs, files, context, cancellationToken).ConfigureAwait(false);
+ var cause = number == first ? firstCause : AttemptCause.AutoRetry;
+ var outcome = await RunAttemptAsync(stage, stepKey, order, number, cause, inputs, copies, context, cancellationToken).ConfigureAwait(false);
attempts.Add(outcome.Attempt);
- await _recorder.AttemptFinishedAsync(new AttemptFinished(context.Workspace.RunId, stepKey, outcome.Attempt), CancellationToken.None)
+ await _recorder.AttemptFinishedAsync(new AttemptFinished(context.RunId, stepKey, order, outcome.Attempt), CancellationToken.None)
.ConfigureAwait(false);
- switch (outcome.Attempt.Status)
+ if (outcome.Attempt.Status == AttemptStatus.Succeeded)
{
- case AttemptStatus.Succeeded:
- context.UsedBytes += outcome.Files.Values.SelectMany(f => f).Sum(f => f.Length ?? 0);
- return Finish(StepStatus.Succeeded, null, outcome);
- case AttemptStatus.Cancelled:
- return Finish(StepStatus.Cancelled, outcome.Attempt.Error);
+ context.UsedBytes += TotalBytes(outcome.Files);
+ result = Finish(StepStatus.Succeeded, null, outcome);
+ break;
+ }
+
+ if (outcome.Attempt.Status == AttemptStatus.Cancelled)
+ {
+ result = Finish(StepStatus.Cancelled, outcome.Attempt.Error);
+ break;
}
var error = outcome.Attempt.Error!;
Log.AttemptFailed(_logger, stage.Key, number, error.Kind, error.Message);
- if (number > stage.Retry.MaxAttempts)
+ if (number - first >= stage.Retry.MaxAttempts)
{
- return Finish(outcome.Attempt.Status == AttemptStatus.TimedOut ? StepStatus.TimedOut : StepStatus.Failed, error);
+ result = Finish(outcome.Attempt.Status == AttemptStatus.TimedOut ? StepStatus.TimedOut : StepStatus.Failed, error);
+ break;
}
- var delay = stage.Retry.GetDelay(number);
+ var delay = stage.Retry.GetDelay(number - first + 1);
Log.RetryScheduled(_logger, stage.Key, delay, number + 1);
try
{
@@ -307,14 +484,24 @@ await _recorder.AttemptFinishedAsync(new AttemptFinished(context.Workspace.RunId
}
catch (OperationCanceledException)
{
- return Finish(StepStatus.Cancelled, new RunError(ErrorKind.Cancelled, "The run was cancelled while waiting to retry."));
+ result = Finish(StepStatus.Cancelled, new RunError(ErrorKind.Cancelled, "The run was cancelled while waiting to retry."));
+ break;
}
}
+
+ return await RecordStepAsync(context, result).ConfigureAwait(false);
+ }
+
+ private async Task RecordStepAsync(RunContext context, StepResult step)
+ {
+ await _recorder.StepFinishedAsync(new StepFinished(context.RunId, step, _time.GetUtcNow()), CancellationToken.None).ConfigureAwait(false);
+ return step;
}
private async Task RunAttemptAsync(
StageDefinition stage,
string stepKey,
+ int order,
int number,
AttemptCause cause,
JsonObject inputs,
@@ -323,16 +510,20 @@ private async Task RunAttemptAsync(
CancellationToken cancellationToken)
{
var settings = context.Settings;
+ var attemptId = AttemptId.New();
var startedAt = _time.GetUtcNow();
var started = _time.GetTimestamp();
var queued = TimeSpan.Zero;
string? environmentHash = null;
var workDirectory = context.Workspace.AttemptDirectory(stepKey, number);
+ await _recorder.AttemptStartedAsync(
+ new AttemptStarted(context.RunId, stepKey, order, attemptId, number, cause, context.TriggeredBy, startedAt),
+ CancellationToken.None).ConfigureAwait(false);
using var activity = PsrDiagnostics.Source.StartActivity("psr.stage.execute");
activity?.SetTag("psr.stage.key", stage.Key);
activity?.SetTag("psr.stage.version", stage.Version);
- activity?.SetTag("psr.run.id", context.Workspace.RunId.ToString());
+ activity?.SetTag("psr.run.id", context.RunId.ToString());
activity?.SetTag("psr.step.key", stepKey);
activity?.SetTag("psr.attempt", number);
@@ -358,9 +549,10 @@ private async Task RunAttemptAsync(
{
Stage = stage,
Environment = environment,
- RunId = context.Workspace.RunId,
+ RunId = context.RunId,
StepKey = stepKey,
Attempt = number,
+ IsTest = context.Kind is RunKind.StageTest or RunKind.PipelineTest,
WorkDirectory = workDirectory,
Inputs = inputs,
Files = files,
@@ -397,7 +589,7 @@ private async Task RunAttemptAsync(
var overhead = duration - execution.ProcessDuration - queued;
var attempt = new AttemptResult
{
- Id = AttemptId.New(),
+ Id = attemptId,
Number = number,
Cause = cause,
Status = execution.Status,
@@ -453,26 +645,80 @@ private static async Task>> M
var list = new List();
foreach (var bound in group.OrderBy(f => f.Index))
{
- var name = RunWorkspace.SafeFileName(bound.File.FileName);
- var target = Path.Combine(workspace.InputsDirectory, bound.Field, bound.Index.ToString(CultureInfo.InvariantCulture), name);
- Directory.CreateDirectory(Path.GetDirectoryName(target)!);
- await bound.File.SaveAsAsync(target, cancellationToken).ConfigureAwait(false);
+ list.Add(await CopyToAsync(bound.File, Path.Combine(workspace.InputsDirectory, bound.Field, bound.Index.ToString(CultureInfo.InvariantCulture)),
+ cancellationToken).ConfigureAwait(false));
+ }
- string sha256;
- await using (var stream = File.OpenRead(target))
- {
- sha256 = Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false));
- }
+ result[group.Key] = list;
+ }
+
+ return result;
+ }
- list.Add(PsrFile.FromProducedFile(target, name, bound.File.ContentType, sha256));
+ /// Downloads stored files (e.g. from the database) to ; files already on disk are kept.
+ private static async Task>> MaterializeStoredAsync(
+ IReadOnlyDictionary> files, string directory, CancellationToken cancellationToken)
+ {
+ var result = new Dictionary>(StringComparer.Ordinal);
+ foreach (var (field, list) in files)
+ {
+ var local = new List();
+ for (var i = 0; i < list.Count; i++)
+ {
+ var file = list[i];
+ local.Add(file.LocalPath is not null
+ ? file
+ : await CopyToAsync(file, Path.Combine(directory, field, i.ToString(CultureInfo.InvariantCulture)), cancellationToken).ConfigureAwait(false));
}
- result[group.Key] = list;
+ result[field] = local;
}
return result;
}
+ private static async Task CopyToAsync(PsrFile file, string directory, CancellationToken cancellationToken)
+ {
+ var name = RunWorkspace.SafeFileName(file.FileName);
+ var target = Path.Combine(directory, name);
+ Directory.CreateDirectory(directory);
+ await file.SaveAsAsync(target, cancellationToken).ConfigureAwait(false);
+
+ string sha256;
+ await using (var stream = File.OpenRead(target))
+ {
+ sha256 = Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false));
+ }
+
+ var copy = PsrFile.FromProducedFile(target, name, file.ContentType, sha256);
+ if (file.ArtifactId is { } artifactId)
+ {
+ copy.AttachArtifact(artifactId);
+ }
+
+ return copy;
+ }
+
+ /// Copy of the step inputs with {"$file": {...}} metadata for file fields (a list for list<file>), as recorded.
+ private static JsonObject WithFilePlaceholders(JsonObject inputs, IReadOnlyDictionary> files, StageDefinition stage)
+ {
+ var copy = (JsonObject)inputs.DeepClone();
+ foreach (var (field, list) in files)
+ {
+ var references = list.Select(file => (JsonNode)new JsonObject
+ {
+ [WireValidator.FileMarker] = new JsonObject { ["name"] = file.FileName, ["contentType"] = file.ContentType },
+ }).ToList();
+ var isList = stage.Inputs.FirstOrDefault(f => f.Name == field)?.Type.Kind == FieldKind.List;
+ copy[field] = isList ? new JsonArray([.. references]) : references.FirstOrDefault();
+ }
+
+ return copy;
+ }
+
+ private static long TotalBytes(IReadOnlyDictionary> files) =>
+ files.Values.SelectMany(list => list).Sum(file => file.Length ?? 0);
+
private static List ToCopies(IReadOnlyDictionary> files) =>
[.. files.SelectMany(group => group.Value.Select((file, index) =>
new InputFileCopy(group.Key, index, file.LocalPath!, file.FileName, file.ContentType, file.Length ?? 0, file.Sha256!)))];
@@ -505,24 +751,24 @@ private static Dictionary> MoveOutputs(IReadOnlyL
_ => RunStatus.Failed,
};
- private static StepResult Skipped(PipelineStepDefinition definition, int order) => new()
+ private static StepResult Skipped(PipelineStepDefinition definition, int order, StepResult? prior) => new()
{
StepKey = definition.Key,
Order = order,
StageKey = definition.StageKey,
StageVersion = definition.StageVersion,
Status = StepStatus.Skipped,
- Attempts = [],
+ Attempts = prior?.Attempts ?? [],
};
- private static StepResult InvalidInputs(StageDefinition stage, string stepKey, int order, IReadOnlyList errors) => new()
+ private static StepResult InvalidInputs(StageDefinition stage, string stepKey, int order, IReadOnlyList errors, StepResult? prior) => new()
{
StepKey = stepKey,
Order = order,
StageKey = stage.Key,
StageVersion = stage.Version,
Status = StepStatus.Failed,
- Attempts = [],
+ Attempts = prior?.Attempts ?? [],
Error = new RunError(ErrorKind.InputValidation, string.Join("; ", errors.Select(e => $"'{e.Field}': {e.Message}")))
{
Code = errors[0].Code,
@@ -543,6 +789,8 @@ private void ForwardOutput(string stageKey, AttemptResult attempt)
}
}
+ private sealed record RunTarget(RunKind Kind, string Key, int Version, IReadOnlyList Steps);
+
private sealed record AttemptOutcome(AttemptResult Attempt, JsonObject? Outputs, IReadOnlyDictionary> Files);
private sealed record RunOutcome(
@@ -555,13 +803,19 @@ private sealed record RunOutcome(
public static RunOutcome Aborted(RunStatus status, RunError error) => new(status, [], null, null, error);
}
- /// State shared by the steps of one run (folders, settings and the file budget).
- private sealed class RunContext(RunWorkspace workspace, PythonScriptRunnerOptions settings)
+ /// State shared by the steps of one run (folders, settings, caller and the file budget).
+ private sealed class RunContext(RunWorkspace workspace, PythonScriptRunnerOptions settings, RunKind kind, string triggeredBy)
{
public RunWorkspace Workspace { get; } = workspace;
+ public RunId RunId => Workspace.RunId;
+
public PythonScriptRunnerOptions Settings { get; } = settings;
+ public RunKind Kind { get; } = kind;
+
+ public string TriggeredBy { get; } = triggeredBy;
+
public long UsedBytes { get; set; }
}
}
diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Catalog/CatalogMapping.cs b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/CatalogMapping.cs
new file mode 100644
index 0000000..fa4b149
--- /dev/null
+++ b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/CatalogMapping.cs
@@ -0,0 +1,149 @@
+using System.Security.Cryptography;
+using System.Text;
+using PythonScriptRunner.EntityFrameworkCore.Entities;
+
+namespace PythonScriptRunner.EntityFrameworkCore.Catalog;
+
+/// A step row with the stage version it references.
+internal sealed record StepRow(PipelineStepEntity Step, string StageKey, int StageVersion);
+
+/// Maps catalog rows to definitions and DTOs, and back.
+internal static class CatalogMapping
+{
+ public static StageDefinition ToStage(StageEntity stage, StageVersionEntity version) => new()
+ {
+ Key = stage.Key,
+ Version = version.Version,
+ Name = stage.Name,
+ Description = stage.Description,
+ Script = ScriptSource.Inline(version.Script),
+ PythonVersion = version.PythonVersion,
+ Requirements = version.Requirements,
+ LockedRequirements = version.LockedRequirements,
+ Inputs = StoredJson.Fields(version.InputFields),
+ Outputs = StoredJson.Fields(version.OutputFields),
+ RequiredSecrets = version.RequiredSecrets,
+ Timeout = TimeSpan.FromSeconds(version.TimeoutSeconds),
+ MemoryLimitMb = version.MemoryLimitMb,
+ Retry = new RetryPolicy(version.RetryMaxAttempts, TimeSpan.FromSeconds(version.RetryInitialDelaySeconds), (double)version.RetryBackoffMultiplier),
+ Idempotent = version.Idempotent,
+ };
+
+ public static StageDraftData ToStageData(StageVersionEntity version) => new()
+ {
+ Script = version.Script,
+ PythonVersion = version.PythonVersion,
+ Requirements = version.Requirements,
+ Inputs = StoredJson.Fields(version.InputFields),
+ Outputs = StoredJson.Fields(version.OutputFields),
+ RequiredSecrets = version.RequiredSecrets,
+ Timeout = TimeSpan.FromSeconds(version.TimeoutSeconds),
+ MemoryLimitMb = version.MemoryLimitMb,
+ Retry = new RetryPolicy(version.RetryMaxAttempts, TimeSpan.FromSeconds(version.RetryInitialDelaySeconds), (double)version.RetryBackoffMultiplier),
+ Idempotent = version.Idempotent,
+ };
+
+ /// Copies editable content into a version row.
+ public static void Apply(StageVersionEntity version, StageDraftData data)
+ {
+ version.Script = data.Script;
+ version.ScriptSha256 = Sha256(data.Script);
+ version.PythonVersion = data.PythonVersion;
+ version.Requirements = string.IsNullOrWhiteSpace(data.Requirements) ? null : data.Requirements;
+ version.InputFields = StoredJson.Fields(data.Inputs);
+ version.OutputFields = StoredJson.Fields(data.Outputs);
+ version.RequiredSecrets = [.. data.RequiredSecrets];
+ version.TimeoutSeconds = (int)Math.Ceiling(data.Timeout.TotalSeconds);
+ version.MemoryLimitMb = data.MemoryLimitMb;
+ version.RetryMaxAttempts = data.Retry.MaxAttempts;
+ version.RetryInitialDelaySeconds = (int)data.Retry.InitialDelay.TotalSeconds;
+ version.RetryBackoffMultiplier = (decimal)data.Retry.BackoffMultiplier;
+ version.Idempotent = data.Idempotent;
+ }
+
+ public static StageDraftData ToStageData(StageDefinition stage, string defaultPythonVersion) => new()
+ {
+ Script = stage.Script switch
+ {
+ InlineScriptSource inline => inline.Content,
+ FileScriptSource file => File.ReadAllText(file.Path),
+ _ => throw new InvalidOperationException("Unknown script source."),
+ },
+ PythonVersion = stage.PythonVersion ?? defaultPythonVersion,
+ Requirements = stage.Requirements,
+ Inputs = stage.Inputs,
+ Outputs = stage.Outputs,
+ RequiredSecrets = stage.RequiredSecrets,
+ Timeout = stage.Timeout ?? TimeSpan.FromMinutes(5),
+ MemoryLimitMb = stage.MemoryLimitMb,
+ Retry = stage.Retry,
+ Idempotent = stage.Idempotent,
+ };
+
+ public static StageDraft ToStageDraft(StageEntity stage, StageVersionEntity version) =>
+ new(stage.Key, version.Version, ToStageData(version), version.RowVersion, Authorship(version));
+
+ public static StageVersionDetail ToStageVersionDetail(StageEntity stage, StageVersionEntity version) => new()
+ {
+ Key = stage.Key,
+ Version = version.Version,
+ Status = version.Status,
+ Data = ToStageData(version),
+ ScriptSha256 = version.ScriptSha256,
+ LockedRequirements = version.LockedRequirements,
+ EnvironmentHash = version.EnvironmentHash,
+ PublishedAt = version.PublishedAt,
+ PublishedBy = version.PublishedBy,
+ ArchivedAt = version.ArchivedAt,
+ ArchivedBy = version.ArchivedBy,
+ RowVersion = version.RowVersion,
+ Authorship = Authorship(version),
+ };
+
+ public static PipelineDefinition ToPipeline(PipelineEntity pipeline, PipelineVersionEntity version, IEnumerable steps) => new()
+ {
+ Key = pipeline.Key,
+ Version = version.Version,
+ Name = pipeline.Name,
+ Description = pipeline.Description,
+ Inputs = StoredJson.Fields(version.InputFields),
+ Steps = [.. steps.OrderBy(s => s.Step.Order).Select(s => new PipelineStepDefinition(s.Step.Key, s.StageKey, s.StageVersion)
+ {
+ DisplayName = s.Step.DisplayName,
+ Bindings = StoredJson.Bindings(s.Step.Bindings),
+ })],
+ Outputs = StoredJson.Outputs(version.Outputs),
+ };
+
+ public static PipelineDraftData ToPipelineData(PipelineDefinition pipeline, string? layout) => new()
+ {
+ Inputs = pipeline.Inputs,
+ Steps = pipeline.Steps,
+ Outputs = pipeline.Outputs,
+ Layout = StoredJson.Node(layout),
+ };
+
+ public static PipelineVersionDetail ToPipelineVersionDetail(PipelineEntity pipeline, PipelineVersionEntity version, IEnumerable steps) => new()
+ {
+ Key = pipeline.Key,
+ Version = version.Version,
+ Status = version.Status,
+ Data = ToPipelineData(ToPipeline(pipeline, version, steps), version.Layout),
+ PublishedAt = version.PublishedAt,
+ PublishedBy = version.PublishedBy,
+ ArchivedAt = version.ArchivedAt,
+ ArchivedBy = version.ArchivedBy,
+ RowVersion = version.RowVersion,
+ Authorship = Authorship(version),
+ };
+
+ public static Authorship Authorship(StageEntity row) => new(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy);
+
+ public static Authorship Authorship(StageVersionEntity row) => new(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy);
+
+ public static Authorship Authorship(PipelineEntity row) => new(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy);
+
+ public static Authorship Authorship(PipelineVersionEntity row) => new(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy);
+
+ public static string Sha256(string text) => Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text)));
+}
diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalog.cs b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalog.cs
new file mode 100644
index 0000000..df52f76
--- /dev/null
+++ b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalog.cs
@@ -0,0 +1,106 @@
+using System.Collections.Concurrent;
+using Microsoft.EntityFrameworkCore;
+
+namespace PythonScriptRunner.EntityFrameworkCore.Catalog;
+
+///
+/// Catalog stored in PostgreSQL. Only published versions run; definitions and "latest published" lookups are cached for
+/// and invalidated in this process on every change.
+///
+internal sealed class EfCatalog(IDbContextFactory factory, PsrEntityFrameworkOptions options, TimeProvider time) : ICatalog
+{
+ private readonly ConcurrentDictionary<(string Key, int? Version), Cached> _stages = new();
+ private readonly ConcurrentDictionary<(string Key, int? Version), Cached> _pipelines = new();
+
+ public async ValueTask FindStageAsync(string key, int? version, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(key);
+ if (_stages.TryGetValue((key, version), out var cached) && cached.ExpiresAt > time.GetUtcNow())
+ {
+ return cached.Value;
+ }
+
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var query = db.StageVersions.AsNoTracking().Include(v => v.Stage).Where(v => v.Stage.Key == key);
+ query = version is { } number
+ ? query.Where(v => v.Version == number)
+ : query.Where(v => v.Version == v.Stage.LatestPublishedVersion);
+ var row = await query.SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false);
+ if (row is null)
+ {
+ return null;
+ }
+
+ if (row.Status != VersionStatus.Published)
+ {
+ throw new VersionNotRunnableException(key, row.Version, row.Status);
+ }
+
+ var stage = CatalogMapping.ToStage(row.Stage, row);
+ _stages[(key, version)] = new(stage, time.GetUtcNow() + options.LatestVersionCacheDuration);
+ return stage;
+ }
+ }
+
+ public async ValueTask FindPipelineAsync(string key, int? version, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(key);
+ if (_pipelines.TryGetValue((key, version), out var cached) && cached.ExpiresAt > time.GetUtcNow())
+ {
+ return cached.Value;
+ }
+
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var query = db.PipelineVersions.AsNoTracking().Include(v => v.Pipeline).Where(v => v.Pipeline.Key == key);
+ query = version is { } number
+ ? query.Where(v => v.Version == number)
+ : query.Where(v => v.Version == v.Pipeline.LatestPublishedVersion);
+ var row = await query.SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false);
+ if (row is null)
+ {
+ return null;
+ }
+
+ if (row.Status != VersionStatus.Published)
+ {
+ throw new VersionNotRunnableException(key, row.Version, row.Status);
+ }
+
+ var steps = await LoadStepsAsync(db, row.Id, cancellationToken).ConfigureAwait(false);
+ var pipeline = CatalogMapping.ToPipeline(row.Pipeline, row, steps);
+ _pipelines[(key, version)] = new(pipeline, time.GetUtcNow() + options.LatestVersionCacheDuration);
+ return pipeline;
+ }
+ }
+
+ /// Forgets cached definitions of a stage (called after every change in this process).
+ public void InvalidateStage(string key)
+ {
+ foreach (var entry in _stages.Keys.Where(k => k.Key == key))
+ {
+ _stages.TryRemove(entry, out _);
+ }
+ }
+
+ /// Forgets cached definitions of a pipeline.
+ public void InvalidatePipeline(string key)
+ {
+ foreach (var entry in _pipelines.Keys.Where(k => k.Key == key))
+ {
+ _pipelines.TryRemove(entry, out _);
+ }
+ }
+
+ public static async Task> LoadStepsAsync(PsrDbContext db, Guid pipelineVersionId, CancellationToken cancellationToken) =>
+ await db.PipelineSteps.AsNoTracking()
+ .Where(s => s.PipelineVersionId == pipelineVersionId)
+ .OrderBy(s => s.Order)
+ .Select(s => new StepRow(s, s.StageVersion.Stage.Key, s.StageVersion.Version))
+ .ToListAsync(cancellationToken).ConfigureAwait(false);
+
+ private sealed record Cached(T Value, DateTimeOffset ExpiresAt);
+}
diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Pipelines.cs b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Pipelines.cs
new file mode 100644
index 0000000..d0da7af
--- /dev/null
+++ b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Pipelines.cs
@@ -0,0 +1,530 @@
+using System.Text.Json.Nodes;
+using Microsoft.EntityFrameworkCore;
+using PythonScriptRunner.EntityFrameworkCore.Entities;
+using PythonScriptRunner.EntityFrameworkCore.Infrastructure;
+
+namespace PythonScriptRunner.EntityFrameworkCore.Catalog;
+
+internal sealed partial class EfCatalogAdmin
+{
+ public async Task> ListPipelinesAsync(CatalogQuery query, CancellationToken cancellationToken = default)
+ {
+ var (pageNumber, pageSize) = Paging(query);
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var pipelines = db.Pipelines.AsNoTracking();
+ if (!string.IsNullOrWhiteSpace(query.Search))
+ {
+ var pattern = $"%{query.Search.Trim()}%";
+ pipelines = pipelines.Where(p => EF.Functions.ILike(p.Key, pattern) || EF.Functions.ILike(p.Name, pattern));
+ }
+
+ var total = await pipelines.CountAsync(cancellationToken).ConfigureAwait(false);
+ var items = await pipelines.OrderBy(p => p.Key)
+ .Skip((pageNumber - 1) * pageSize).Take(pageSize)
+ .Select(p => new PipelineSummary(
+ p.Key, p.Name, p.Description, p.LatestPublishedVersion,
+ p.Versions.Where(v => v.Status == VersionStatus.Draft).Select(v => (int?)v.Version).FirstOrDefault(),
+ p.UpdatedAt))
+ .ToListAsync(cancellationToken).ConfigureAwait(false);
+ return new Page(items, pageNumber, pageSize, total);
+ }
+ }
+
+ public async Task GetPipelineAsync(string key, CancellationToken cancellationToken = default)
+ {
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var pipeline = await db.Pipelines.AsNoTracking().Include(p => p.Versions).SingleOrDefaultAsync(p => p.Key == key, cancellationToken).ConfigureAwait(false);
+ return pipeline is null ? null : ToPipelineDetail(pipeline);
+ }
+ }
+
+ public async Task GetPipelineVersionAsync(string key, int version, CancellationToken cancellationToken = default)
+ {
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var row = await db.PipelineVersions.AsNoTracking().Include(v => v.Pipeline)
+ .SingleOrDefaultAsync(v => v.Pipeline.Key == key && v.Version == version, cancellationToken).ConfigureAwait(false);
+ return row is null
+ ? null
+ : CatalogMapping.ToPipelineVersionDetail(row.Pipeline, row, await EfCatalog.LoadStepsAsync(db, row.Id, cancellationToken).ConfigureAwait(false));
+ }
+ }
+
+ public async Task GetPipelineDraftAsync(string key, CancellationToken cancellationToken = default)
+ {
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var row = await db.PipelineVersions.AsNoTracking().Include(v => v.Pipeline)
+ .SingleOrDefaultAsync(v => v.Pipeline.Key == key && v.Status == VersionStatus.Draft, cancellationToken).ConfigureAwait(false);
+ return row is null ? null : await ToPipelineDraftAsync(db, row, cancellationToken).ConfigureAwait(false);
+ }
+ }
+
+ public async Task CreatePipelineAsync(NewCatalogItem pipeline, Actor actor, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(pipeline);
+ ArgumentNullException.ThrowIfNull(actor);
+ ValidateItem(pipeline);
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ if (await db.Pipelines.AnyAsync(p => p.Key == pipeline.Key, cancellationToken).ConfigureAwait(false))
+ {
+ throw new PsrDomainException(PsrErrorCodes.PipelineKeyTaken, $"Pipeline '{pipeline.Key}' already exists.");
+ }
+
+ var now = time.GetUtcNow();
+ var row = new PipelineEntity
+ {
+ Id = Guid.CreateVersion7(now),
+ Key = pipeline.Key,
+ Name = pipeline.Name,
+ Description = pipeline.Description,
+ CreatedAt = now,
+ CreatedBy = actor.Subject,
+ UpdatedAt = now,
+ UpdatedBy = actor.Subject,
+ };
+ var draft = NewPipelineVersion(row, 1, actor, now);
+ row.Versions.Add(draft);
+ db.Pipelines.Add(row);
+ AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelineCreated, "Pipeline", row.Id, row.Key), now);
+ await SaveAsync(db, cancellationToken, unique: _ => new PsrDomainException(PsrErrorCodes.PipelineKeyTaken, $"Pipeline '{pipeline.Key}' already exists."))
+ .ConfigureAwait(false);
+ return new PipelineDraft(row.Key, 1, new PipelineDraftData(), draft.RowVersion, CatalogMapping.Authorship(draft));
+ }
+ }
+
+ public async Task UpdatePipelineAsync(
+ string key, CatalogItemChanges changes, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(changes);
+ return await ChangePipelineAsync(key, expectedRowVersion, actor, AuditActions.PipelineUpdated, (pipeline, details) =>
+ {
+ if (changes.Name is { } name && name != pipeline.Name)
+ {
+ ValidateName(name);
+ details["name"] = new JsonArray(pipeline.Name, name);
+ pipeline.Name = name;
+ }
+
+ if (changes.Description is { } description && description != pipeline.Description)
+ {
+ details["description"] = new JsonArray(pipeline.Description, description);
+ pipeline.Description = description;
+ }
+ }, cancellationToken).ConfigureAwait(false);
+ }
+
+ public async Task CreatePipelineDraftFromAsync(string key, int fromVersion, Actor actor, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(actor);
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var pipeline = await db.Pipelines.Include(p => p.Versions).ThenInclude(v => v.Steps)
+ .SingleOrDefaultAsync(p => p.Key == key, cancellationToken).ConfigureAwait(false)
+ ?? throw new PipelineNotFoundException(key);
+ EnsureNoDraft(pipeline.Versions.Any(v => v.Status == VersionStatus.Draft), key);
+ var source = pipeline.Versions.SingleOrDefault(v => v.Version == fromVersion) ?? throw new VersionNotFoundException(key, fromVersion);
+ var now = time.GetUtcNow();
+ var draft = NewPipelineVersion(pipeline, pipeline.Versions.Max(v => v.Version) + 1, actor, now);
+ draft.InputFields = source.InputFields;
+ draft.Outputs = source.Outputs;
+ draft.Layout = source.Layout;
+ draft.Steps.AddRange(source.Steps.Select(s => new PipelineStepEntity
+ {
+ Id = Guid.CreateVersion7(now),
+ PipelineVersionId = draft.Id,
+ Order = s.Order,
+ Key = s.Key,
+ DisplayName = s.DisplayName,
+ StageVersionId = s.StageVersionId,
+ Bindings = s.Bindings,
+ }));
+ pipeline.Versions.Add(draft);
+ AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelineDraftCreated, "PipelineVersion", draft.Id, $"{key}@{draft.Version}",
+ new JsonObject { ["fromVersion"] = fromVersion }), now);
+ await SaveAsync(db, cancellationToken, unique: _ => DraftExists(key)).ConfigureAwait(false);
+ return await ToPipelineDraftAsync(db, draft, cancellationToken).ConfigureAwait(false);
+ }
+ }
+
+ public async Task SavePipelineDraftAsync(
+ string key, int version, PipelineDraftData data, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(data);
+ ArgumentNullException.ThrowIfNull(actor);
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var row = await LoadPipelineVersionAsync(db, key, version, cancellationToken).ConfigureAwait(false);
+ EnsureDraft(row.Status, key, version);
+ if (row.RowVersion != expectedRowVersion)
+ {
+ throw new ConcurrencyConflictException($"The draft {key}@{version} was changed by someone else.",
+ await ToPipelineDraftAsync(db, row, cancellationToken).ConfigureAwait(false));
+ }
+
+ var steps = await BuildStepsAsync(db, row.Id, data.Steps, cancellationToken).ConfigureAwait(false);
+ var now = time.GetUtcNow();
+ var transaction = await db.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
+ await using var rollback = transaction.ConfigureAwait(false);
+ await db.PipelineSteps.Where(s => s.PipelineVersionId == row.Id).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false);
+ db.PipelineSteps.AddRange(steps);
+ row.InputFields = StoredJson.Fields(data.Inputs);
+ row.Outputs = StoredJson.Outputs(data.Outputs);
+ row.Layout = StoredJson.Node(data.Layout);
+ Touch(row, actor, now);
+ db.Entry(row).Property(v => v.RowVersion).OriginalValue = expectedRowVersion;
+ AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelineDraftSaved, "PipelineVersion", row.Id, $"{key}@{version}"), now);
+ await SaveAsync(db, cancellationToken, conflict: Current(() => GetPipelineDraftAsync(key, CancellationToken.None))).ConfigureAwait(false);
+ await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
+ return await ToPipelineDraftAsync(db, row, cancellationToken).ConfigureAwait(false);
+ }
+ }
+
+ public async Task DiscardPipelineDraftAsync(string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(actor);
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var row = await LoadPipelineVersionAsync(db, key, version, cancellationToken).ConfigureAwait(false);
+ EnsureDraft(row.Status, key, version);
+ if (row.RowVersion != expectedRowVersion)
+ {
+ throw new ConcurrencyConflictException($"The draft {key}@{version} was changed by someone else.",
+ await ToPipelineDraftAsync(db, row, cancellationToken).ConfigureAwait(false));
+ }
+
+ var onlyVersion = !await db.PipelineVersions.AnyAsync(v => v.PipelineId == row.PipelineId && v.Id != row.Id, cancellationToken).ConfigureAwait(false);
+ db.Entry(row).Property(v => v.RowVersion).OriginalValue = expectedRowVersion;
+ db.PipelineVersions.Remove(row);
+ if (onlyVersion && !await db.Runs.AnyAsync(r => r.PipelineId == row.PipelineId, cancellationToken).ConfigureAwait(false))
+ {
+ // A pipeline without versions cannot get a new draft; discarding its first draft removes it.
+ db.Pipelines.Remove(row.Pipeline);
+ }
+
+ AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelineDraftDiscarded, "PipelineVersion", row.Id, $"{key}@{version}"), time.GetUtcNow());
+ await SaveAsync(db, cancellationToken, conflict: Current(() => GetPipelineDraftAsync(key, CancellationToken.None))).ConfigureAwait(false);
+ catalog.InvalidatePipeline(key);
+ }
+ }
+
+ public async Task PublishPipelineAsync(
+ string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(actor);
+ var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
+ await using (db.ConfigureAwait(false))
+ {
+ var row = await LoadPipelineVersionAsync(db, key, version, cancellationToken).ConfigureAwait(false);
+ EnsureDraft(row.Status, key, version);
+ if (row.RowVersion != expectedRowVersion)
+ {
+ throw new ConcurrencyConflictException($"The draft {key}@{version} was changed by someone else.",
+ await ToPipelineDraftAsync(db, row, cancellationToken).ConfigureAwait(false));
+ }
+
+ var steps = await EfCatalog.LoadStepsAsync(db, row.Id, cancellationToken).ConfigureAwait(false);
+ var report = await pipelineValidator.ValidateAsync(CatalogMapping.ToPipeline(row.Pipeline, row, steps), cancellationToken).ConfigureAwait(false);
+ if (!report.IsValid)
+ {
+ throw new DefinitionInvalidException(PsrErrorCodes.PipelineInvalid, report);
+ }
+
+ var now = time.GetUtcNow();
+ row.Status = VersionStatus.Published;
+ row.PublishedAt = now;
+ row.PublishedBy = actor.Subject;
+ Touch(row, actor, now);
+ row.Pipeline.LatestPublishedVersion = Math.Max(row.Pipeline.LatestPublishedVersion ?? 0, version);
+ Touch(row.Pipeline, actor, now);
+ db.Entry(row).Property(v => v.RowVersion).OriginalValue = expectedRowVersion;
+ AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelinePublished, "PipelineVersion", row.Id, $"{key}@{version}"), now);
+ await SaveAsync(db, cancellationToken, conflict: Current(() => GetPipelineDraftAsync(key, CancellationToken.None))).ConfigureAwait(false);
+ catalog.InvalidatePipeline(key);
+ return new PipelinePublication(CatalogMapping.ToPipelineVersionDetail(row.Pipeline, row, steps), report.Warnings);
+ }
+ }
+
+ public async Task