From 0b2f51af640b8a0979cc9adcb4217d766724166e Mon Sep 17 00:00:00 2001 From: net0well Date: Mon, 19 Oct 2026 19:29:50 -0300 Subject: [PATCH 1/4] =?UTF-8?q?feat(core):=20contratos=20de=20gest=C3=A3o?= =?UTF-8?q?=20e=20motor=20IRunEngine=20(spec=20004)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Abstractions: ICatalogAdmin, IRunManager, IArtifactStore, IAuditWriter/IAuditReader, Actor, erros de domínio com códigos estáveis e IDs UUID v7 tipados. - IRunRecorder recebe todas as transições (execução, passo, tentativa, arquivos) e reserva a chave de idempotência; RunResult.Deduplicated. - IRunEngine: roda definições (rascunhos), retoma execuções que falharam e cancela execuções em andamento no processo. - CatalogManifests público para importar e exportar o formato de pasta; IStageValidator; verificação de sintaxe e hash de ambiente no IEnvironmentManager. Co-Authored-By: Claude Opus 5.5 --- .../Exceptions.cs | 55 ++ .../Extensibility/Interfaces.cs | 140 ++++- .../Files/PsrFile.cs | 56 +- src/PythonScriptRunner.Abstractions/Ids.cs | 111 +++- .../Management/Actor.cs | 42 ++ .../Management/Audit.cs | 100 ++++ .../Management/CatalogAdmin.cs | 499 ++++++++++++++++++ .../Management/DomainErrors.cs | 146 +++++ .../Management/RunManagement.cs | 358 +++++++++++++ .../Running/IPythonRunner.cs | 3 + .../Running/IRunEngine.cs | 72 +++ .../Running/Results.cs | 16 +- .../Running/Statuses.cs | 51 ++ .../Catalog/CatalogManifests.cs | 258 +++++++++ .../Catalog/FileCatalogSource.cs | 92 +--- .../Catalog/StageDefinitionValidator.cs | 12 + ...ScriptRunnerServiceCollectionExtensions.cs | 5 +- .../Diagnostics/PsrDiagnostics.cs | 3 + .../Environments/IEnvironmentManager.cs | 19 + .../Environments/UvEnvironmentManager.cs | 70 ++- .../Pipelines/PipelineValidator.cs | 9 +- .../Pipelines/StepInputResolver.cs | 10 +- .../PythonScriptRunner.Core.csproj | 4 +- .../Running/Infrastructure.cs | 24 +- .../Running/PythonRunner.cs | 418 ++++++++++++--- 25 files changed, 2374 insertions(+), 199 deletions(-) create mode 100644 src/PythonScriptRunner.Abstractions/Management/Actor.cs create mode 100644 src/PythonScriptRunner.Abstractions/Management/Audit.cs create mode 100644 src/PythonScriptRunner.Abstractions/Management/CatalogAdmin.cs create mode 100644 src/PythonScriptRunner.Abstractions/Management/DomainErrors.cs create mode 100644 src/PythonScriptRunner.Abstractions/Management/RunManagement.cs create mode 100644 src/PythonScriptRunner.Abstractions/Running/IRunEngine.cs create mode 100644 src/PythonScriptRunner.Core/Catalog/CatalogManifests.cs diff --git a/src/PythonScriptRunner.Abstractions/Exceptions.cs b/src/PythonScriptRunner.Abstractions/Exceptions.cs index ab38d86..df77067 100644 --- a/src/PythonScriptRunner.Abstractions/Exceptions.cs +++ b/src/PythonScriptRunner.Abstractions/Exceptions.cs @@ -47,6 +47,61 @@ public VersionNotFoundException(string key, int version) public int Version { get; } } +/// The version exists but cannot run (it is a draft or was archived). +public sealed class VersionNotRunnableException : PsrException +{ + /// Creates the exception. + /// Stage or pipeline key. + /// Requested version. + /// Current status of the version. + public VersionNotRunnableException(string key, int version, VersionStatus status) + : base($"Version {version} of '{key}' is {status} and cannot run; only published versions run.") + { + Key = key; + Version = version; + Status = status; + } + + /// Stage or pipeline key. + public string Key { get; } + + /// Requested version. + public int Version { get; } + + /// Current status of the version. + public VersionStatus Status { get; } +} + +/// The run does not exist (or is beyond the retention). +public sealed class RunNotFoundException : PsrException +{ + /// Creates the exception. + /// Requested run. + public RunNotFoundException(RunId runId) + : base($"Run {runId} was not found.") + { + RunId = runId; + } + + /// Requested run. + public RunId RunId { get; } +} + +/// The stored file does not exist (or is beyond the retention). +public sealed class ArtifactNotFoundException : PsrException +{ + /// Creates the exception. + /// Requested file. + public ArtifactNotFoundException(ArtifactId artifactId) + : base($"Artifact {artifactId} was not found.") + { + ArtifactId = artifactId; + } + + /// Requested file. + public ArtifactId ArtifactId { get; } +} + /// One invalid input. /// Field path (e.g. coeficientes[2]). /// Stable code (e.g. MISSING_REQUIRED_INPUT). diff --git a/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs b/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs index 9a57725..a4f7742 100644 --- a/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs +++ b/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs @@ -2,7 +2,7 @@ namespace PythonScriptRunner; -/// Source of stage definitions (memory, folder or database). +/// Source of stage and pipeline definitions (memory, folder or database). public interface ICatalog { /// Finds a stage version. @@ -10,6 +10,7 @@ public interface ICatalog /// Exact version, or null for the latest published version. /// Cancellation token. /// The definition, or null when not found. + /// The version exists but is a draft or archived. ValueTask FindStageAsync(string key, int? version, CancellationToken cancellationToken = default); /// Finds a pipeline version. @@ -17,9 +18,19 @@ public interface ICatalog /// Exact version, or null for the latest published version. /// Cancellation token. /// The definition, or null when not found. + /// The version exists but is a draft or archived. ValueTask FindPipelineAsync(string key, int? version, CancellationToken cancellationToken = default); } +/// Validates stage definitions (contract, limits and secret names). +public interface IStageValidator +{ + /// Validates a stage definition. + /// The definition. + /// Errors (code STAGE_DEFINITION) and warnings. + ValidationReport Validate(StageDefinition stage); +} + /// Resolves the secrets declared by stages. public interface ISecretProvider { @@ -29,46 +40,145 @@ public interface ISecretProvider ValueTask GetSecretAsync(string name, CancellationToken cancellationToken = default); } -/// Receives run transitions so they can be persisted. The default implementation does nothing. +/// +/// Receives every transition of a run so it can be persisted. The default implementation does nothing; the +/// EntityFrameworkCore package stores runs, steps, attempts, logs and files. +/// public interface IRunRecorder { - /// A run started. + /// A run is about to start (inputs already validated). Reserves the idempotency key. + /// Run data. + /// Cancellation token. + /// The existing run when the idempotency key was already used (the new run must not start); otherwise null. + ValueTask RunStartingAsync(RunStarting run, CancellationToken cancellationToken); + + /// A failed run is being reprocessed: it is running again and skipped steps are pending. /// Run data. /// Cancellation token. - ValueTask RunStartedAsync(RunStarted run, CancellationToken cancellationToken); + ValueTask RunResumingAsync(RunResuming run, CancellationToken cancellationToken); + + /// A step started with its resolved inputs. + /// Step data. + /// Cancellation token. + ValueTask StepStartedAsync(StepStarted step, CancellationToken cancellationToken); - /// An attempt finished. + /// An attempt (one Python process) started. + /// Attempt data. + /// Cancellation token. + ValueTask AttemptStartedAsync(AttemptStarted attempt, CancellationToken cancellationToken); + + /// An attempt finished (with its logs). /// Attempt data. /// Cancellation token. ValueTask AttemptFinishedAsync(AttemptFinished attempt, CancellationToken cancellationToken); + /// A step finished (succeeded with outputs and files, failed, cancelled or skipped). + /// Step data. + /// Cancellation token. + ValueTask StepFinishedAsync(StepFinished step, CancellationToken cancellationToken); + /// A run finished. /// Final data. /// Cancellation token. ValueTask RunFinishedAsync(RunFinished run, CancellationToken cancellationToken); } -/// A run started. +/// A run is about to start. +public sealed record RunStarting +{ + /// Run identifier (its timestamp is the creation time). + public required RunId RunId { get; init; } + + /// What is executed. + public required RunKind Kind { get; init; } + + /// Stage or pipeline key. + public required string TargetKey { get; init; } + + /// Stage or pipeline version. + public required int TargetVersion { get; init; } + + /// How the run started. + public required RunTrigger Trigger { get; init; } + + /// Who requested the run. + public required string TriggeredBy { get; init; } + + /// Deduplication key given by the caller. + public string? IdempotencyKey { get; init; } + + /// Validated inputs in wire format (files carry metadata only). + public required JsonObject Inputs { get; init; } + + /// Input files by field name. + public IReadOnlyDictionary> InputFiles { get; init; } = new Dictionary>(); + + /// Steps that will run, in order. + public required IReadOnlyList Steps { get; init; } + + /// Start time. + public required DateTimeOffset StartedAt { get; init; } +} + +/// A step of a run that is about to start. +/// Step key (the stage key for single-stage runs). +/// 1-based position. +/// Stage key. +/// Exact stage version. +public sealed record PlannedStep(string StepKey, int Order, string StageKey, int StageVersion); + +/// A failed run is being reprocessed. +/// Run identifier. +/// Who reprocesses. +/// When. +public sealed record RunResuming(RunId RunId, string TriggeredBy, DateTimeOffset ResumedAt); + +/// A step started. /// Run identifier. -/// What is executed. -/// Stage or pipeline key. -/// Stage or pipeline version. -/// Who requested the run. +/// Step key. +/// 1-based position. +/// Resolved inputs in wire format. +/// Input files by field name. /// Start time. -/// Validated inputs in wire format. -public sealed record RunStarted( - RunId RunId, RunKind Kind, string TargetKey, int TargetVersion, string TriggeredBy, DateTimeOffset StartedAt, JsonObject Inputs); +public sealed record StepStarted( + RunId RunId, string StepKey, int Order, JsonObject Inputs, IReadOnlyDictionary> InputFiles, DateTimeOffset StartedAt); + +/// An attempt started. +/// Run identifier. +/// Step key. +/// 1-based position of the step. +/// Attempt identifier. +/// 1-based attempt number. +/// Why it started. +/// Who started it. +/// Start time. +public sealed record AttemptStarted( + RunId RunId, string StepKey, int Order, AttemptId AttemptId, int Number, AttemptCause Cause, string TriggeredBy, DateTimeOffset StartedAt); /// An attempt finished. /// Run identifier. /// Step key. +/// 1-based position of the step. /// Attempt outcome. -public sealed record AttemptFinished(RunId RunId, string StepKey, AttemptResult Attempt); +public sealed record AttemptFinished(RunId RunId, string StepKey, int Order, AttemptResult Attempt); + +/// A step finished. +/// Run identifier. +/// Step outcome (outputs and files when it succeeded). +/// End time. +public sealed record StepFinished(RunId RunId, StepResult Step, DateTimeOffset FinishedAt); /// A run finished. /// Run identifier. /// Final status. /// End time. /// Declared outputs in wire format. +/// Declared file outputs by name. /// Failure, when the run did not succeed. -public sealed record RunFinished(RunId RunId, RunStatus Status, DateTimeOffset FinishedAt, JsonObject? Outputs, RunError? Error); +public sealed record RunFinished( + RunId RunId, + RunStatus Status, + DateTimeOffset FinishedAt, + JsonObject? Outputs, + IReadOnlyDictionary>? OutputFiles, + RunError? Error); diff --git a/src/PythonScriptRunner.Abstractions/Files/PsrFile.cs b/src/PythonScriptRunner.Abstractions/Files/PsrFile.cs index da44e0c..f520e55 100644 --- a/src/PythonScriptRunner.Abstractions/Files/PsrFile.cs +++ b/src/PythonScriptRunner.Abstractions/Files/PsrFile.cs @@ -27,10 +27,20 @@ public sealed class PsrFile private readonly string? _path; private readonly ReadOnlyMemory? _bytes; + private readonly Func>? _opener; + private readonly long? _knownLength; private Stream? _stream; private string? _sha256; - private PsrFile(string fileName, string contentType, string? path, ReadOnlyMemory? bytes, Stream? stream, string? sha256) + private PsrFile( + string fileName, + string contentType, + string? path, + ReadOnlyMemory? bytes, + Stream? stream, + string? sha256, + Func>? opener = null, + long? knownLength = null) { ArgumentException.ThrowIfNullOrWhiteSpace(fileName); ArgumentException.ThrowIfNullOrWhiteSpace(contentType); @@ -40,6 +50,8 @@ private PsrFile(string fileName, string contentType, string? path, ReadOnlyMemor _bytes = bytes; _stream = stream; _sha256 = sha256; + _opener = opener; + _knownLength = knownLength; } /// A file on disk. @@ -90,6 +102,38 @@ public static PsrFile FromText(string text, string fileName, string contentType public static PsrFile FromProducedFile(string path, string fileName, string contentType, string sha256) => new(fileName, contentType, Path.GetFullPath(path), null, null, sha256); + /// A stored file (e.g. in the database) that is streamed on demand. + /// Identifier of the stored file. + /// File name. + /// MIME type. + /// Size in bytes. + /// SHA-256 as lowercase hex. + /// Opens the content; called once per read. + public static PsrFile FromArtifact( + ArtifactId artifactId, string fileName, string contentType, long length, string sha256, Func> open) + { + ArgumentNullException.ThrowIfNull(open); + var file = new PsrFile(fileName, contentType, null, null, null, sha256, open, length); + file.AttachArtifact(artifactId); + return file; + } + + /// Identifier of the stored copy of this file, when a persistence package stored it. + public ArtifactId? ArtifactId { get; private set; } + + /// Associates the file with its stored copy (used by persistence packages). + /// Identifier of the stored file. + /// The file is already associated with another stored file. + public void AttachArtifact(ArtifactId artifactId) + { + if (ArtifactId is { } current && current != artifactId) + { + throw new InvalidOperationException($"File '{FileName}' is already stored as artifact {current}."); + } + + ArtifactId = artifactId; + } + /// File name (without directories). public string FileName { get; } @@ -114,6 +158,11 @@ public long? Length return bytes.Length; } + if (_knownLength is { } known) + { + return known; + } + return _stream is { CanSeek: true } stream ? stream.Length - stream.Position : null; } } @@ -159,6 +208,11 @@ public Task OpenReadAsync(CancellationToken cancellationToken = default) return Task.FromResult(new MemoryStream(bytes.ToArray(), writable: false)); } + if (_opener is not null) + { + return _opener(cancellationToken); + } + var stream = Interlocked.Exchange(ref _stream, null) ?? throw new InvalidOperationException($"The stream of file '{FileName}' was already consumed."); return Task.FromResult(stream); diff --git a/src/PythonScriptRunner.Abstractions/Ids.cs b/src/PythonScriptRunner.Abstractions/Ids.cs index 961efe9..e935398 100644 --- a/src/PythonScriptRunner.Abstractions/Ids.cs +++ b/src/PythonScriptRunner.Abstractions/Ids.cs @@ -1,12 +1,23 @@ namespace PythonScriptRunner; -/// Identifier of a run (time-ordered UUID v7). +/// Identifier of a run (time-ordered UUID v7). Its timestamp is the creation time of the run. /// The underlying GUID. public readonly record struct RunId(Guid Value) { /// Creates a new time-ordered identifier. public static RunId New() => new(Guid.CreateVersion7()); + /// Creates a new identifier whose timestamp is (millisecond precision). + /// Creation time of the run. + public static RunId New(DateTimeOffset createdAt) => new(Guid.CreateVersion7(createdAt)); + + /// Parses the canonical text form. + /// GUID text. + public static RunId Parse(string text) => new(Guid.Parse(text)); + + /// Creation time embedded in the identifier (UTC, millisecond precision). + public DateTimeOffset CreatedAt => UuidV7.Timestamp(Value); + /// public override string ToString() => Value.ToString(); } @@ -21,3 +32,101 @@ public readonly record struct AttemptId(Guid Value) /// public override string ToString() => Value.ToString(); } + +/// Identifier of a step within a run (time-ordered UUID v7). +/// The underlying GUID. +public readonly record struct StepRunId(Guid Value) +{ + /// Creates a new time-ordered identifier. + public static StepRunId New() => new(Guid.CreateVersion7()); + + /// + public override string ToString() => Value.ToString(); +} + +/// Identifier of a stored file (time-ordered UUID v7). +/// The underlying GUID. +public readonly record struct ArtifactId(Guid Value) +{ + /// Creates a new time-ordered identifier. + public static ArtifactId New() => new(Guid.CreateVersion7()); + + /// Parses the canonical text form. + /// GUID text. + public static ArtifactId Parse(string text) => new(Guid.Parse(text)); + + /// + public override string ToString() => Value.ToString(); +} + +/// Identifier of a stage (UUID v7). +/// The underlying GUID. +public readonly record struct StageId(Guid Value) +{ + /// Creates a new time-ordered identifier. + public static StageId New() => new(Guid.CreateVersion7()); + + /// + public override string ToString() => Value.ToString(); +} + +/// Identifier of a stage version (UUID v7). +/// The underlying GUID. +public readonly record struct StageVersionId(Guid Value) +{ + /// Creates a new time-ordered identifier. + public static StageVersionId New() => new(Guid.CreateVersion7()); + + /// + public override string ToString() => Value.ToString(); +} + +/// Identifier of a pipeline (UUID v7). +/// The underlying GUID. +public readonly record struct PipelineId(Guid Value) +{ + /// Creates a new time-ordered identifier. + public static PipelineId New() => new(Guid.CreateVersion7()); + + /// + public override string ToString() => Value.ToString(); +} + +/// Identifier of a pipeline version (UUID v7). +/// The underlying GUID. +public readonly record struct PipelineVersionId(Guid Value) +{ + /// Creates a new time-ordered identifier. + public static PipelineVersionId New() => new(Guid.CreateVersion7()); + + /// + public override string ToString() => Value.ToString(); +} + +/// Identifier of a schedule (UUID v7). +/// The underlying GUID. +public readonly record struct ScheduleId(Guid Value) +{ + /// Creates a new time-ordered identifier. + public static ScheduleId New() => new(Guid.CreateVersion7()); + + /// + public override string ToString() => Value.ToString(); +} + +/// Reads the timestamp of a UUID v7. +internal static class UuidV7 +{ + public static DateTimeOffset Timestamp(Guid value) + { + Span bytes = stackalloc byte[16]; + value.TryWriteBytes(bytes, bigEndian: true, out _); + var milliseconds = 0L; + for (var i = 0; i < 6; i++) + { + milliseconds = (milliseconds << 8) | bytes[i]; + } + + return DateTimeOffset.FromUnixTimeMilliseconds(milliseconds); + } +} diff --git a/src/PythonScriptRunner.Abstractions/Management/Actor.cs b/src/PythonScriptRunner.Abstractions/Management/Actor.cs new file mode 100644 index 0000000..7a132bd --- /dev/null +++ b/src/PythonScriptRunner.Abstractions/Management/Actor.cs @@ -0,0 +1,42 @@ +namespace PythonScriptRunner; + +/// Who performs an operation: used for authorship columns and to decide auditing (only humans are audited). +/// User subject, or system:<origin> for automatic callers. +/// Whether a person performed the action (panel); library, queue and cron are not. +public sealed record Actor(string Subject, bool IsHuman) +{ + /// Code calling the library (system:library). + public static Actor Library { get; } = System("library"); + + /// An automatic caller. + /// Origin (e.g. queue or schedule:<id>). + public static Actor System(string origin) => new($"system:{origin}", IsHuman: false); + + /// A person (e.g. the subject of the authenticated user in the panel). + /// User subject. + public static Actor User(string subject) => new(subject, IsHuman: true); + + /// + public override string ToString() => Subject; +} + +/// Creation and last change of a row. +/// Creation time. +/// Creator subject. +/// Last change. +/// Subject of the last change. +public sealed record Authorship(DateTimeOffset CreatedAt, string CreatedBy, DateTimeOffset UpdatedAt, string UpdatedBy); + +/// One page of an offset-paginated list. +/// Item type. +/// Items of the page. +/// 1-based page number. +/// Requested page size. +/// Number of items in every page. +public sealed record Page(IReadOnlyList Items, int PageNumber, int PageSize, int Total); + +/// One page of a keyset-paginated list (newest first). +/// Item type. +/// Items of the page. +/// Opaque cursor of the next page; null on the last page. +public sealed record KeysetPage(IReadOnlyList Items, string? NextCursor); diff --git a/src/PythonScriptRunner.Abstractions/Management/Audit.cs b/src/PythonScriptRunner.Abstractions/Management/Audit.cs new file mode 100644 index 0000000..cc93a6d --- /dev/null +++ b/src/PythonScriptRunner.Abstractions/Management/Audit.cs @@ -0,0 +1,100 @@ +using System.Text.Json.Nodes; + +namespace PythonScriptRunner; + +/// Records human actions (publish, archive, delete, trigger, cancel, reprocess...). Automatic callers are not audited. +public interface IAuditWriter +{ + /// Writes an event when is human; does nothing otherwise. + /// The event. + /// Cancellation token. + Task WriteAsync(AuditEntry entry, CancellationToken cancellationToken = default); +} + +/// Reads audit events. +public interface IAuditReader +{ + /// Lists events newest first inside a time window (default: last 24 hours; at most 90 days). + /// Filters, window and cursor. + /// Cancellation token. + Task> ListAsync(AuditQuery query, CancellationToken cancellationToken = default); +} + +/// An action to audit. +/// Who acted. +/// What was done (see ). +/// Stage, StageVersion, Pipeline, PipelineVersion, Schedule, Run or Settings. +/// Identifier of the entity. +/// Readable reference (e.g. html-to-pdf@3). +/// Diff ({field: [before, after]}) or action parameters. +public sealed record AuditEntry(Actor Actor, string Action, string EntityType, Guid? EntityId, string? EntityKey, JsonObject? Details = null); + +/// A stored audit event. +/// Event identifier. +/// When it happened. +/// Subject of the user. +/// What was done. +/// Entity type. +/// Entity identifier. +/// Readable reference. +/// Diff or parameters. +/// OpenTelemetry trace. +public sealed record AuditEvent( + Guid Id, DateTimeOffset OccurredAt, string Actor, string Action, string EntityType, Guid? EntityId, string? EntityKey, JsonObject? Details, string? TraceId); + +/// Filters of an audit list. +public sealed record AuditQuery +{ + /// Start of the window (inclusive); default: minus 24 hours. + public DateTimeOffset? From { get; init; } + + /// End of the window (exclusive); default: now. + public DateTimeOffset? To { get; init; } + + /// Only events of this user. + public string? Actor { get; init; } + + /// Only events of this entity type. + public string? EntityType { get; init; } + + /// Cursor returned by the previous page. + public string? Cursor { get; init; } + + /// Items per page (1-100). Default: 50. + public int Limit { get; init; } = 50; +} + +/// Audited actions. +public static class AuditActions +{ +#pragma warning disable CS1591 // The names are self-explanatory. + public const string StageCreated = nameof(StageCreated); + public const string StageUpdated = nameof(StageUpdated); + public const string StageDraftCreated = nameof(StageDraftCreated); + public const string StageDraftSaved = nameof(StageDraftSaved); + public const string StageDraftDiscarded = nameof(StageDraftDiscarded); + public const string StagePublished = nameof(StagePublished); + public const string StageVersionArchived = nameof(StageVersionArchived); + public const string StageDeleted = nameof(StageDeleted); + public const string StageTested = nameof(StageTested); + public const string PipelineCreated = nameof(PipelineCreated); + public const string PipelineUpdated = nameof(PipelineUpdated); + public const string PipelineDraftCreated = nameof(PipelineDraftCreated); + public const string PipelineDraftSaved = nameof(PipelineDraftSaved); + public const string PipelineDraftDiscarded = nameof(PipelineDraftDiscarded); + public const string PipelinePublished = nameof(PipelinePublished); + public const string PipelineVersionArchived = nameof(PipelineVersionArchived); + public const string PipelineDeleted = nameof(PipelineDeleted); + public const string PipelineSettingsChanged = nameof(PipelineSettingsChanged); + public const string PipelineTested = nameof(PipelineTested); + public const string ScheduleCreated = nameof(ScheduleCreated); + public const string ScheduleUpdated = nameof(ScheduleUpdated); + public const string ScheduleDeleted = nameof(ScheduleDeleted); + public const string ScheduleEnabled = nameof(ScheduleEnabled); + public const string ScheduleDisabled = nameof(ScheduleDisabled); + public const string RunTriggered = nameof(RunTriggered); + public const string RunCancelled = nameof(RunCancelled); + public const string RunReprocessed = nameof(RunReprocessed); + public const string CatalogImported = nameof(CatalogImported); +#pragma warning restore CS1591 +} diff --git a/src/PythonScriptRunner.Abstractions/Management/CatalogAdmin.cs b/src/PythonScriptRunner.Abstractions/Management/CatalogAdmin.cs new file mode 100644 index 0000000..2f8d846 --- /dev/null +++ b/src/PythonScriptRunner.Abstractions/Management/CatalogAdmin.cs @@ -0,0 +1,499 @@ +using System.Text.Json.Nodes; + +namespace PythonScriptRunner; + +/// +/// Edits the catalog stored in a database: drafts, publication, archiving, deletion, settings and import/export. +/// Every change receives the that performs it (authorship and audit of human actions). +/// +/// Errors are reported with codes (). +public interface ICatalogAdmin +{ + /// Lists stages. + /// Search and page. + /// Cancellation token. + Task> ListStagesAsync(CatalogQuery query, CancellationToken cancellationToken = default); + + /// Reads a stage with its versions and the pipelines that use it. + /// Stage key. + /// Cancellation token. + /// The stage, or null when it does not exist. + Task GetStageAsync(string key, CancellationToken cancellationToken = default); + + /// Reads one version of a stage (any status). + /// Stage key. + /// Version number. + /// Cancellation token. + Task GetStageVersionAsync(string key, int version, CancellationToken cancellationToken = default); + + /// Reads the current draft of a stage. + /// Stage key. + /// Cancellation token. + Task GetStageDraftAsync(string key, CancellationToken cancellationToken = default); + + /// Creates a stage with an initial draft (version 1). STAGE_KEY_TAKEN when the key exists. + /// Key, name and description. + /// Author. + /// Cancellation token. + Task CreateStageAsync(NewCatalogItem stage, Actor actor, CancellationToken cancellationToken = default); + + /// Changes the name or description of a stage. + /// Stage key. + /// New values (null keeps the current one). + /// Row version read by the caller. + /// Author. + /// Cancellation token. + Task UpdateStageAsync( + string key, CatalogItemChanges changes, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default); + + /// Creates a draft (next version) with the content of an existing version. DRAFT_ALREADY_EXISTS when there is one. + /// Stage key. + /// Version to copy. + /// Author. + /// Cancellation token. + Task CreateStageDraftFromAsync(string key, int fromVersion, Actor actor, CancellationToken cancellationToken = default); + + /// Saves a draft. VERSION_IMMUTABLE when the version is not a draft; CONCURRENCY_CONFLICT when it changed. + /// Stage key. + /// Draft version. + /// New content. + /// Row version read by the caller. + /// Author. + /// Cancellation token. + Task SaveStageDraftAsync( + string key, int version, StageDraftData data, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default); + + /// Deletes a draft. + /// Stage key. + /// Draft version. + /// Row version read by the caller. + /// Author. + /// Cancellation token. + Task DiscardStageDraftAsync(string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default); + + /// Validates the draft contract (fields, limits, secrets) without compiling or locking. + /// Stage key. + /// Cancellation token. + Task ValidateStageDraftAsync(string key, CancellationToken cancellationToken = default); + + /// + /// Publishes a draft: validates the contract, compiles the script with its Python version and locks the dependencies with uv. + /// Errors: STAGE_INVALID, SCRIPT_SYNTAX_ERROR, DEPENDENCY_RESOLUTION_FAILED, CONCURRENCY_CONFLICT. + /// + /// Stage key. + /// Draft version. + /// Row version read by the caller. + /// Author. + /// Cancellation token. + Task PublishStageAsync( + string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default); + + /// Archives a published version. VERSION_REFERENCED when a published pipeline uses it. + /// Stage key. + /// Published version. + /// Author. + /// Cancellation token. + Task ArchiveStageVersionAsync(string key, int version, Actor actor, CancellationToken cancellationToken = default); + + /// Deletes a stage and its versions. DELETE_NOT_ALLOWED when referenced by a pipeline step or present in the retained history. + /// Stage key. + /// Administrator. + /// Cancellation token. + Task DeleteStageAsync(string key, Actor actor, CancellationToken cancellationToken = default); + + /// Runs the current draft as a test () and waits for the result. + /// Stage key. + /// Inputs (same forms as ). + /// Author. + /// Cancels the run. + Task TestStageDraftAsync(string key, object? inputs, Actor actor, CancellationToken cancellationToken = default); + + /// Writes a version as a zip in the folder catalog format (stages/<key>@<version>/stage.json + files). + /// Stage key. + /// Version. + /// Writable stream that receives the zip. + /// Cancellation token. + Task ExportStageVersionAsync(string key, int version, Stream destination, CancellationToken cancellationToken = default); + + /// Lists pipelines. + /// Search and page. + /// Cancellation token. + Task> ListPipelinesAsync(CatalogQuery query, CancellationToken cancellationToken = default); + + /// Reads a pipeline with its versions and settings. + /// Pipeline key. + /// Cancellation token. + Task GetPipelineAsync(string key, CancellationToken cancellationToken = default); + + /// Reads one version of a pipeline (any status). + /// Pipeline key. + /// Version number. + /// Cancellation token. + Task GetPipelineVersionAsync(string key, int version, CancellationToken cancellationToken = default); + + /// Reads the current draft of a pipeline. + /// Pipeline key. + /// Cancellation token. + Task GetPipelineDraftAsync(string key, CancellationToken cancellationToken = default); + + /// Creates a pipeline with an empty draft (version 1). PIPELINE_KEY_TAKEN when the key exists. + /// Key, name and description. + /// Author. + /// Cancellation token. + Task CreatePipelineAsync(NewCatalogItem pipeline, Actor actor, CancellationToken cancellationToken = default); + + /// Changes the name or description of a pipeline. + /// Pipeline key. + /// New values (null keeps the current one). + /// Row version read by the caller. + /// Author. + /// Cancellation token. + Task UpdatePipelineAsync( + string key, CatalogItemChanges changes, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default); + + /// Creates a draft (next version) with the content of an existing version. + /// Pipeline key. + /// Version to copy. + /// Author. + /// Cancellation token. + Task CreatePipelineDraftFromAsync(string key, int fromVersion, Actor actor, CancellationToken cancellationToken = default); + + /// Saves a draft. Steps must reference existing stage versions (published to be publishable). + /// Pipeline key. + /// Draft version. + /// New content. + /// Row version read by the caller. + /// Author. + /// Cancellation token. + Task SavePipelineDraftAsync( + string key, int version, PipelineDraftData data, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default); + + /// Deletes a draft. + /// Pipeline key. + /// Draft version. + /// Row version read by the caller. + /// Author. + /// Cancellation token. + Task DiscardPipelineDraftAsync(string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default); + + /// Publishes a draft after validation (PIPELINE_INVALID with the report); warnings are returned. + /// Pipeline key. + /// Draft version. + /// Row version read by the caller. + /// Author. + /// Cancellation token. + Task PublishPipelineAsync( + string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default); + + /// Archives a published version. VERSION_SCHEDULED when an enabled schedule pins it. + /// Pipeline key. + /// Published version. + /// Author. + /// Cancellation token. + Task ArchivePipelineVersionAsync(string key, int version, Actor actor, CancellationToken cancellationToken = default); + + /// Deletes a pipeline, its versions and schedules. DELETE_NOT_ALLOWED when it has runs in the retained history. + /// Pipeline key. + /// Administrator. + /// Cancellation token. + Task DeletePipelineAsync(string key, Actor actor, CancellationToken cancellationToken = default); + + /// Changes operational settings (not versioned). + /// Pipeline key. + /// New settings. + /// Row version read by the caller. + /// Author. + /// Cancellation token. + Task UpdatePipelineSettingsAsync( + string key, PipelineSettings settings, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default); + + /// Shortens the retention of the pipeline runs and files. RETENTION_EXCEEDS_GLOBAL when longer than the global one. + /// Pipeline key. + /// New retention (null uses the global value). + /// Row version read by the caller. + /// Administrator. + /// Cancellation token. + Task UpdatePipelineRetentionAsync( + string key, PipelineRetention retention, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default); + + /// Validates and runs the current draft as a test (); PIPELINE_INVALID when invalid. + /// Pipeline key. + /// Pipeline inputs. + /// Author. + /// Cancels the run. + Task TestPipelineDraftAsync(string key, object? inputs, Actor actor, CancellationToken cancellationToken = default); + + /// Writes a version and the stage versions it uses as a zip in the folder catalog format. + /// Pipeline key. + /// Version. + /// Writable stream that receives the zip. + /// Cancellation token. + Task ExportPipelineVersionAsync(string key, int version, Stream destination, CancellationToken cancellationToken = default); + + /// + /// Imports a zip in the folder catalog format as drafts: new keys get version 1, existing keys get a draft of a new version + /// (published versions are never overwritten). All or nothing: IMPORT_INVALID lists every problem. + /// + /// Readable zip stream. + /// Author. + /// Cancellation token. + Task ImportAsync(Stream zip, Actor actor, CancellationToken cancellationToken = default); +} + +/// Search and page of a catalog list. +/// Text contained in the key or the name. +/// 1-based page number. +/// Items per page (1-100). +public sealed record CatalogQuery(string? Search = null, int PageNumber = 1, int PageSize = 25); + +/// A new stage or pipeline. +/// Slug (immutable). +/// Display name. +/// Description. +public sealed record NewCatalogItem(string Key, string Name, string? Description = null); + +/// Changes to the name or description; null keeps the current value. +/// New name. +/// New description. +public sealed record CatalogItemChanges(string? Name = null, string? Description = null); + +/// A version in a list. +/// Version number. +/// Status. +/// Publication time. +/// Last change. +public sealed record VersionSummary(int Version, VersionStatus Status, DateTimeOffset? PublishedAt, DateTimeOffset UpdatedAt); + +/// Editable content of a stage version. +public sealed record StageDraftData +{ + /// Python source of the single script. + public required string Script { get; init; } + + /// Python version (e.g. 3.12). + public required string PythonVersion { get; init; } + + /// Dependencies in requirements.txt format. + public string? Requirements { get; init; } + + /// Declared inputs. + public IReadOnlyList Inputs { get; init; } = []; + + /// Declared outputs. + public IReadOnlyList Outputs { get; init; } = []; + + /// Names of the secrets the script needs. + public IReadOnlyList RequiredSecrets { get; init; } = []; + + /// Maximum duration of one attempt. + public TimeSpan Timeout { get; init; } = TimeSpan.FromMinutes(5); + + /// Memory limit in MB; null uses the runner default. + public int? MemoryLimitMb { get; init; } + + /// Automatic retry policy. + public RetryPolicy Retry { get; init; } = RetryPolicy.None; + + /// Whether running the script twice with the same inputs is safe. + public bool Idempotent { get; init; } +} + +/// The current draft of a stage. +/// Stage key. +/// Draft version. +/// Content. +/// Concurrency token (send it back when saving). +/// Creation and last change. +public sealed record StageDraft(string Key, int Version, StageDraftData Data, uint RowVersion, Authorship Authorship); + +/// One version of a stage. +public sealed record StageVersionDetail +{ + /// Stage key. + public required string Key { get; init; } + + /// Version number. + public required int Version { get; init; } + + /// Status. + public required VersionStatus Status { get; init; } + + /// Content. + public required StageDraftData Data { get; init; } + + /// SHA-256 of the script. + public required string ScriptSha256 { get; init; } + + /// Locked dependencies (published versions). + public string? LockedRequirements { get; init; } + + /// Hash of the Python environment (published versions). + public string? EnvironmentHash { get; init; } + + /// Publication time. + public DateTimeOffset? PublishedAt { get; init; } + + /// Who published. + public string? PublishedBy { get; init; } + + /// Archiving time. + public DateTimeOffset? ArchivedAt { get; init; } + + /// Who archived. + public string? ArchivedBy { get; init; } + + /// Concurrency token. + public required uint RowVersion { get; init; } + + /// Creation and last change. + public required Authorship Authorship { get; init; } +} + +/// A stage in a list. +/// Stage key. +/// Display name. +/// Description. +/// Latest published version. +/// Version of the current draft. +/// Last change. +public sealed record StageSummary(string Key, string Name, string? Description, int? LatestPublishedVersion, int? DraftVersion, DateTimeOffset UpdatedAt); + +/// A pipeline step that uses a stage version. +/// Pipeline key. +/// Pipeline version. +/// Status of the pipeline version. +/// Step key. +/// Stage version used by the step. +public sealed record StageUsage(string PipelineKey, int PipelineVersion, VersionStatus PipelineStatus, string StepKey, int StageVersion); + +/// A stage with its versions and usages. +/// Stage key. +/// Display name. +/// Description. +/// Latest published version. +/// Every version, newest first. +/// Pipeline steps that use a version of the stage. +/// Concurrency token of the stage row. +/// Creation and last change. +public sealed record StageDetail( + string Key, + string Name, + string? Description, + int? LatestPublishedVersion, + IReadOnlyList Versions, + IReadOnlyList UsedBy, + uint RowVersion, + Authorship Authorship); + +/// Editable content of a pipeline version. +public sealed record PipelineDraftData +{ + /// Inputs received by the pipeline. + public IReadOnlyList Inputs { get; init; } = []; + + /// Steps in execution order. + public IReadOnlyList Steps { get; init; } = []; + + /// Declared outputs. + public IReadOnlyList Outputs { get; init; } = []; + + /// Positions of nodes and edges in the visual editor (does not affect execution). + public JsonNode? Layout { get; init; } +} + +/// The current draft of a pipeline. +/// Pipeline key. +/// Draft version. +/// Content. +/// Concurrency token (send it back when saving). +/// Creation and last change. +public sealed record PipelineDraft(string Key, int Version, PipelineDraftData Data, uint RowVersion, Authorship Authorship); + +/// One version of a pipeline. +public sealed record PipelineVersionDetail +{ + /// Pipeline key. + public required string Key { get; init; } + + /// Version number. + public required int Version { get; init; } + + /// Status. + public required VersionStatus Status { get; init; } + + /// Content. + public required PipelineDraftData Data { get; init; } + + /// Publication time. + public DateTimeOffset? PublishedAt { get; init; } + + /// Who published. + public string? PublishedBy { get; init; } + + /// Archiving time. + public DateTimeOffset? ArchivedAt { get; init; } + + /// Who archived. + public string? ArchivedBy { get; init; } + + /// Concurrency token. + public required uint RowVersion { get; init; } + + /// Creation and last change. + public required Authorship Authorship { get; init; } +} + +/// A pipeline in a list. +/// Pipeline key. +/// Display name. +/// Description. +/// Latest published version. +/// Version of the current draft. +/// Last change. +public sealed record PipelineSummary(string Key, string Name, string? Description, int? LatestPublishedVersion, int? DraftVersion, DateTimeOffset UpdatedAt); + +/// Operational settings of a pipeline (not versioned). +/// Maximum simultaneous asynchronous runs; null = no limit. +public sealed record PipelineSettings(int? MaxConcurrentRuns); + +/// Retention of a pipeline; it can only be shorter than the global retention. +/// Days the files are kept; null = global. +/// Days the runs are kept; null = global. +public sealed record PipelineRetention(int? ArtifactRetentionDays, int? RunRetentionDays); + +/// A pipeline with its versions and settings. +/// Pipeline key. +/// Display name. +/// Description. +/// Latest published version. +/// Every version, newest first. +/// Operational settings. +/// Retention. +/// Concurrency token of the pipeline row. +/// Creation and last change. +public sealed record PipelineDetail( + string Key, + string Name, + string? Description, + int? LatestPublishedVersion, + IReadOnlyList Versions, + PipelineSettings Settings, + PipelineRetention Retention, + uint RowVersion, + Authorship Authorship); + +/// Result of publishing a pipeline. +/// The published version. +/// Validation warnings (they do not block publication). +public sealed record PipelinePublication(PipelineVersionDetail Version, IReadOnlyList Warnings); + +/// A draft created by an import. +/// Stage or Pipeline. +/// Key. +/// Version of the created draft. +/// Whether the key did not exist before. +public sealed record ImportedItem(string Kind, string Key, int Version, bool NewKey); + +/// Result of an import. +/// Drafts created. +public sealed record ImportReport(IReadOnlyList Items); diff --git a/src/PythonScriptRunner.Abstractions/Management/DomainErrors.cs b/src/PythonScriptRunner.Abstractions/Management/DomainErrors.cs new file mode 100644 index 0000000..6997f2d --- /dev/null +++ b/src/PythonScriptRunner.Abstractions/Management/DomainErrors.cs @@ -0,0 +1,146 @@ +namespace PythonScriptRunner; + +/// Stable codes of the domain errors raised by catalog and run management (mapped to HTTP by the dashboard). +public static class PsrErrorCodes +{ + /// A stage with this key already exists. + public const string StageKeyTaken = "STAGE_KEY_TAKEN"; + + /// A pipeline with this key already exists. + public const string PipelineKeyTaken = "PIPELINE_KEY_TAKEN"; + + /// The row changed since it was read (optimistic concurrency). + public const string ConcurrencyConflict = "CONCURRENCY_CONFLICT"; + + /// Published and archived versions cannot be edited. + public const string VersionImmutable = "VERSION_IMMUTABLE"; + + /// The stage version is used by a published pipeline. + public const string VersionReferenced = "VERSION_REFERENCED"; + + /// The pipeline version is pinned by an enabled schedule. + public const string VersionScheduled = "VERSION_SCHEDULED"; + + /// Only one draft per stage or pipeline is allowed. + public const string DraftAlreadyExists = "DRAFT_ALREADY_EXISTS"; + + /// The stage definition is invalid (fields, limits, secrets). + public const string StageInvalid = "STAGE_INVALID"; + + /// The script does not compile. + public const string ScriptSyntaxError = "SCRIPT_SYNTAX_ERROR"; + + /// uv could not resolve the dependencies. + public const string DependencyResolutionFailed = "DEPENDENCY_RESOLUTION_FAILED"; + + /// The pipeline does not pass validation. + public const string PipelineInvalid = "PIPELINE_INVALID"; + + /// The stage or pipeline is referenced or has runs in the retained history. + public const string DeleteNotAllowed = "DELETE_NOT_ALLOWED"; + + /// A pipeline retention cannot exceed the global retention. + public const string RetentionExceedsGlobal = "RETENTION_EXCEEDS_GLOBAL"; + + /// Only failed runs can be reprocessed. + public const string RunNotFailed = "RUN_NOT_FAILED"; + + /// The run already finished. + public const string RunNotCancellable = "RUN_NOT_CANCELLABLE"; + + /// Files needed to reprocess the run are beyond the retention. + public const string ArtifactsExpired = "ARTIFACTS_EXPIRED"; + + /// The file content is beyond the retention. + public const string ArtifactExpired = "ARTIFACT_EXPIRED"; + + /// The imported archive is not a valid catalog folder. + public const string ImportInvalid = "IMPORT_INVALID"; +} + +/// A business rule rejected the operation; is stable (see ). +public class PsrDomainException : PsrException +{ + /// Creates the exception. + /// Stable code. + /// Human-readable message. + /// Cause. + public PsrDomainException(string code, string message, Exception? innerException = null) + : base(message, innerException) + { + Code = code; + } + + /// Stable code (see ). + public string Code { get; } +} + +/// The row changed since it was read; holds the current state. +public sealed class ConcurrencyConflictException : PsrDomainException +{ + /// Creates the exception. + /// What conflicted. + /// Current state (e.g. the current draft), when it still exists. + public ConcurrencyConflictException(string message, object? current) + : base(PsrErrorCodes.ConcurrencyConflict, message) + { + Current = current; + } + + /// Current state (e.g. ), or null when it no longer exists. + public object? Current { get; } +} + +/// The script of a stage does not compile. +public sealed class ScriptSyntaxException : PsrDomainException +{ + /// Creates the exception. + /// Python error message. + /// 1-based line, when known. + /// 1-based column, when known. + public ScriptSyntaxException(string message, int? line, int? column) + : base(PsrErrorCodes.ScriptSyntaxError, line is null ? message : $"{message} (line {line}, column {column})") + { + Line = line; + Column = column; + } + + /// 1-based line, when known. + public int? Line { get; } + + /// 1-based column, when known. + public int? Column { get; } +} + +/// A definition does not pass validation; lists the problems. +public sealed class DefinitionInvalidException : PsrDomainException +{ + /// Creates the exception. + /// or . + /// Validation report. + public DefinitionInvalidException(string code, ValidationReport report) + : base(code, $"Validation failed: {report}") + { + Report = report; + } + + /// Validation report. + public ValidationReport Report { get; } +} + +/// The stage version is used by published pipelines and cannot be archived. +public sealed class VersionReferencedException : PsrDomainException +{ + /// Creates the exception. + /// Stage key. + /// Stage version. + /// Pipelines that use it (key@version). + public VersionReferencedException(string key, int version, IReadOnlyList pipelines) + : base(PsrErrorCodes.VersionReferenced, $"Stage '{key}@{version}' is used by published pipelines: {string.Join(", ", pipelines)}.") + { + Pipelines = pipelines; + } + + /// Pipelines that use the version (key@version). + public IReadOnlyList Pipelines { get; } +} diff --git a/src/PythonScriptRunner.Abstractions/Management/RunManagement.cs b/src/PythonScriptRunner.Abstractions/Management/RunManagement.cs new file mode 100644 index 0000000..efff25f --- /dev/null +++ b/src/PythonScriptRunner.Abstractions/Management/RunManagement.cs @@ -0,0 +1,358 @@ +using System.Text.Json.Nodes; + +namespace PythonScriptRunner; + +/// Queries, cancels and reprocesses persisted runs. +public interface IRunManager +{ + /// Reads a run with its steps, attempts and files (logs are read with ). + /// Run identifier. + /// Cancellation token. + /// The run, or null when it does not exist or is beyond the retention. + Task GetAsync(RunId runId, CancellationToken cancellationToken = default); + + /// Lists runs newest first inside a time window (default: last 24 hours; at most 90 days). + /// Filters, window and cursor. + /// Cancellation token. + Task> ListAsync(RunQuery query, CancellationToken cancellationToken = default); + + /// Reads the stdout or stderr tail of an attempt. + /// Run identifier. + /// Attempt identifier. + /// Which stream. + /// Cancellation token. + Task GetAttemptLogAsync(RunId runId, AttemptId attemptId, LogStream stream, CancellationToken cancellationToken = default); + + /// + /// Cancels a run: pending/queued runs end Cancelled; running runs move to Cancelling and their processes are stopped. + /// RUN_NOT_CANCELLABLE when the run already finished. + /// + /// Run identifier. + /// Who cancels. + /// Cancellation token. + Task CancelAsync(RunId runId, Actor actor, CancellationToken cancellationToken = default); + + /// + /// Reprocesses a failed run with the same version and inputs: successful steps keep their outputs and the failed step gets a + /// new attempt (). Errors: RUN_NOT_FAILED, ARTIFACTS_EXPIRED. + /// + /// Run identifier. + /// Who reprocesses. + /// runs it now in this process; needs a run queue. + /// Cancellation token (cancels a synchronous reprocess). + /// The run after reprocessing (sync) or after it was queued (async). + Task ReprocessAsync(RunId runId, Actor actor, ExecutionMode mode = ExecutionMode.Async, CancellationToken cancellationToken = default); +} + +/// Filters of a run list. The window is required to keep queries on recent partitions. +public sealed record RunQuery +{ + /// Start of the window (inclusive); default: minus 24 hours. + public DateTimeOffset? From { get; init; } + + /// End of the window (exclusive); default: now. + public DateTimeOffset? To { get; init; } + + /// Only runs of this pipeline. + public string? PipelineKey { get; init; } + + /// Only single-stage runs of this stage. + public string? StageKey { get; init; } + + /// Only runs of this kind. + public RunKind? Kind { get; init; } + + /// Only runs with this status. + public RunStatus? Status { get; init; } + + /// Only runs started this way. + public RunTrigger? Trigger { get; init; } + + /// Cursor returned by the previous page. + public string? Cursor { get; init; } + + /// Items per page (1-100). Default: 50. + public int Limit { get; init; } = 50; +} + +/// A run in a list. +/// Run identifier. +/// What was executed. +/// Sync or async. +/// Status. +/// Pipeline or stage key. +/// Pipeline or stage version. +/// How the run started. +/// Who started it. +/// Creation time. +/// Start time. +/// End time. +/// Failure. +public sealed record RunSummary( + RunId Id, + RunKind Kind, + ExecutionMode Mode, + RunStatus Status, + string TargetKey, + int TargetVersion, + RunTrigger Trigger, + string TriggeredBy, + DateTimeOffset CreatedAt, + DateTimeOffset? StartedAt, + DateTimeOffset? FinishedAt, + RunError? Error); + +/// A persisted run with its steps, attempts and files. +public sealed record RunSnapshot +{ + /// Run identifier. + public required RunId Id { get; init; } + + /// What was executed. + public required RunKind Kind { get; init; } + + /// Sync or async. + public required ExecutionMode Mode { get; init; } + + /// Status. + public required RunStatus Status { get; init; } + + /// Pipeline or stage key. + public required string TargetKey { get; init; } + + /// Pipeline or stage version. + public required int TargetVersion { get; init; } + + /// How the run started. + public required RunTrigger Trigger { get; init; } + + /// Who started it. + public required string TriggeredBy { get; init; } + + /// Validated inputs; files as {"$artifact": "<id>"}. + public required JsonObject Inputs { get; init; } + + /// Declared outputs; files as {"$artifact": "<id>"}. + public JsonObject? Outputs { get; init; } + + /// Failure. + public RunError? Error { get; init; } + + /// Idempotency key given by the caller. + public string? IdempotencyKey { get; init; } + + /// Creation time. + public required DateTimeOffset CreatedAt { get; init; } + + /// When it was queued. + public DateTimeOffset? QueuedAt { get; init; } + + /// Start time. + public DateTimeOffset? StartedAt { get; init; } + + /// End time. + public DateTimeOffset? FinishedAt { get; init; } + + /// When cancellation was requested. + public DateTimeOffset? CancelRequestedAt { get; init; } + + /// Who requested cancellation. + public string? CancelRequestedBy { get; init; } + + /// When the run is deleted (pipelines with a shorter retention). + public DateTimeOffset? ExpiresAt { get; init; } + + /// Steps in order. + public required IReadOnlyList Steps { get; init; } + + /// Input and output files. + public required IReadOnlyList Artifacts { get; init; } +} + +/// A persisted step of a run. +public sealed record StepRunSnapshot +{ + /// Step identifier. + public required StepRunId Id { get; init; } + + /// Step key (the stage key for single-stage runs). + public required string StepKey { get; init; } + + /// 1-based position. + public required int Order { get; init; } + + /// Executed stage. + public required string StageKey { get; init; } + + /// Exact stage version. + public required int StageVersion { get; init; } + + /// Status. + public required StepStatus Status { get; init; } + + /// Number of the last attempt (0 = not executed). + public required int CurrentAttempt { get; init; } + + /// Inputs after resolving the bindings. + public JsonObject? Inputs { get; init; } + + /// Validated outputs. + public JsonObject? Outputs { get; init; } + + /// Failure of the last attempt. + public RunError? Error { get; init; } + + /// Start time. + public DateTimeOffset? StartedAt { get; init; } + + /// End time. + public DateTimeOffset? FinishedAt { get; init; } + + /// Attempts in order. + public required IReadOnlyList Attempts { get; init; } +} + +/// A persisted attempt (one Python process). +public sealed record AttemptSnapshot +{ + /// Attempt identifier. + public required AttemptId Id { get; init; } + + /// 1-based number within the step. + public required int Number { get; init; } + + /// Why it started. + public required AttemptCause Cause { get; init; } + + /// Who started it (user for reprocessing, system:* otherwise). + public required string TriggeredBy { get; init; } + + /// Status. + public required AttemptStatus Status { get; init; } + + /// Worker that executed it (asynchronous runs). + public string? WorkerId { get; init; } + + /// Python environment used. + public string? EnvironmentHash { get; init; } + + /// Exit code of the process. + public int? ExitCode { get; init; } + + /// Failure. + public RunError? Error { get; init; } + + /// Start time. + public required DateTimeOffset StartedAt { get; init; } + + /// End time. + public DateTimeOffset? FinishedAt { get; init; } + + /// Total duration. + public TimeSpan? Duration { get; init; } + + /// Time spent by the runner outside the Python process. + public TimeSpan? Overhead { get; init; } + + /// Logs captured (content is read with ). + public IReadOnlyList Logs { get; init; } = []; +} + +/// The stdout or stderr tail of an attempt. +/// Which stream. +/// Captured text (secret values masked). +/// Whether older output was discarded. +/// Total bytes written by the process. +public sealed record AttemptLog(LogStream Stream, string Content, bool Truncated, long OriginalBytes); + +/// Metadata of a stored file. +public sealed record ArtifactInfo +{ + /// File identifier. + public required ArtifactId Id { get; init; } + + /// Run that owns the file. + public required RunId RunId { get; init; } + + /// Step that produced the file (null for run inputs). + public string? StepKey { get; init; } + + /// Input or output. + public required ArtifactDirection Direction { get; init; } + + /// Field that carries the file. + public required string FieldName { get; init; } + + /// Position in a list<file>. + public int? ListIndex { get; init; } + + /// File name. + public required string FileName { get; init; } + + /// MIME type. + public required string ContentType { get; init; } + + /// Size in bytes. + public required long SizeBytes { get; init; } + + /// SHA-256 as lowercase hex. + public required string Sha256 { get; init; } + + /// When the content is deleted. + public required DateTimeOffset ContentExpiresAt { get; init; } + + /// When the content was deleted (downloads then fail with ARTIFACT_EXPIRED). + public DateTimeOffset? ContentDeletedAt { get; init; } +} + +/// Stores and streams run files. +public interface IArtifactStore +{ + /// Stores a file (streamed; never fully loaded in memory). + /// Metadata. + /// Readable content. + /// Cancellation token. + Task SaveAsync(NewArtifact artifact, Stream content, CancellationToken cancellationToken = default); + + /// Reads the metadata of a file. + /// File identifier. + /// Cancellation token. + Task GetAsync(ArtifactId artifactId, CancellationToken cancellationToken = default); + + /// Opens the content for streaming. ARTIFACT_EXPIRED when the content was deleted by the retention. + /// File identifier. + /// Cancellation token. + /// The file does not exist. + Task OpenReadAsync(ArtifactId artifactId, CancellationToken cancellationToken = default); +} + +/// Metadata of a file to store. +public sealed record NewArtifact +{ + /// Identifier to use (new when omitted). + public ArtifactId? Id { get; init; } + + /// Run that owns the file. + public required RunId RunId { get; init; } + + /// Step that produced the file (null for run inputs). + public StepRunId? StepRunId { get; init; } + + /// Input or output. + public required ArtifactDirection Direction { get; init; } + + /// Field that carries the file. + public required string FieldName { get; init; } + + /// Position in a list<file>. + public int? ListIndex { get; init; } + + /// File name. + public required string FileName { get; init; } + + /// MIME type. + public required string ContentType { get; init; } + + /// How long the content is kept; null uses the global retention. + public TimeSpan? Retention { get; init; } +} diff --git a/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs b/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs index 5f649f7..2fc069c 100644 --- a/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs +++ b/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs @@ -39,4 +39,7 @@ public sealed record RunOptions /// Who requested the run. Default: system:library. public string? TriggeredBy { get; init; } + + /// How the run was started. Default: . + public RunTrigger Trigger { get; init; } = RunTrigger.Library; } diff --git a/src/PythonScriptRunner.Abstractions/Running/IRunEngine.cs b/src/PythonScriptRunner.Abstractions/Running/IRunEngine.cs new file mode 100644 index 0000000..92a03e7 --- /dev/null +++ b/src/PythonScriptRunner.Abstractions/Running/IRunEngine.cs @@ -0,0 +1,72 @@ +using System.Text.Json.Nodes; + +namespace PythonScriptRunner; + +/// +/// Advanced API of the execution engine, used by the persistence, queue and dashboard packages: runs definitions that do +/// not come from the catalog (drafts), resumes failed runs and cancels runs of this process. Applications normally use +/// . +/// +public interface IRunEngine +{ + /// Runs a stage definition (any status) as a single-step run and waits for its result. + /// The definition (e.g. a draft). + /// or . + /// Inputs (same forms as ). + /// Idempotency key and caller identity ( is ignored). + /// Cancels the run. + /// The inputs do not match the contract. + Task RunStageAsync( + StageDefinition stage, RunKind kind, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default); + + /// Runs a pipeline definition (any status) and waits for its result. Its stages are resolved in the catalog. + /// The definition (e.g. a draft). + /// or . + /// Pipeline inputs. + /// Idempotency key and caller identity ( is ignored). + /// Cancels the run. + /// The inputs do not match the contract. + Task RunPipelineAsync( + PipelineDefinition pipeline, RunKind kind, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default); + + /// + /// Resumes a failed run in this process: successful steps keep their outputs, the first unsuccessful step gets a new + /// attempt () and the following steps run normally. The run keeps its identifier. + /// + /// State of the run to resume. + /// Cancels the run. + Task ResumeAsync(ResumeRequest request, CancellationToken cancellationToken = default); + + /// Cancels a run that is executing in this process. + /// Run identifier. + /// Whether the run was executing here. + bool RequestCancellation(RunId runId); +} + +/// State of a failed run to resume. +public sealed record ResumeRequest +{ + /// Run identifier (kept). + public required RunId RunId { get; init; } + + /// What the run executes. + public required RunKind Kind { get; init; } + + /// The stage, for single-stage runs. + public StageDefinition? Stage { get; init; } + + /// The pipeline, for pipeline runs. + public PipelineDefinition? Pipeline { get; init; } + + /// Validated inputs of the run in wire format. + public required JsonObject Inputs { get; init; } + + /// Input files of the run by field name. + public IReadOnlyDictionary> InputFiles { get; init; } = new Dictionary>(); + + /// Steps as recorded: succeeded steps (with outputs and files) are reused; attempts of the others are counted. + public required IReadOnlyList PreviousSteps { get; init; } + + /// Who reprocesses. + public required string TriggeredBy { get; init; } +} diff --git a/src/PythonScriptRunner.Abstractions/Running/Results.cs b/src/PythonScriptRunner.Abstractions/Running/Results.cs index 14bbff3..4f3abd6 100644 --- a/src/PythonScriptRunner.Abstractions/Running/Results.cs +++ b/src/PythonScriptRunner.Abstractions/Running/Results.cs @@ -93,6 +93,7 @@ public sealed class RunResult : IAsyncDisposable /// Output files of the run by field name. /// Failure, when the run did not succeed. /// Deletes local files when the result is disposed. + /// Whether this is an earlier run returned for the same idempotency key. public RunResult( RunId id, RunKind kind, @@ -102,7 +103,8 @@ public RunResult( JsonObject? outputs, IReadOnlyDictionary>? files, RunError? error, - Func? cleanup = null) + Func? cleanup = null, + bool deduplicated = false) { Id = id; Kind = kind; @@ -113,6 +115,7 @@ public RunResult( _files = files ?? new Dictionary>(); Error = error; _cleanup = cleanup; + Deduplicated = deduplicated; } /// Run identifier. @@ -121,9 +124,18 @@ public RunResult( /// What was executed. public RunKind Kind { get; } - /// Final status: Succeeded, Failed or Cancelled. + /// + /// Final status: Succeeded, Failed or Cancelled. A deduplicated result reports the current status of the earlier run, + /// which may still be running. + /// public RunStatus Status { get; } + /// Whether this is an earlier run returned because the same idempotency key was used within 24 hours. + public bool Deduplicated { get; } + + /// Output files of the run by declared output name. + public IReadOnlyDictionary> Files => _files; + /// Failure, when the run did not succeed. public RunError? Error { get; } diff --git a/src/PythonScriptRunner.Abstractions/Running/Statuses.cs b/src/PythonScriptRunner.Abstractions/Running/Statuses.cs index 54a9fa2..7aab167 100644 --- a/src/PythonScriptRunner.Abstractions/Running/Statuses.cs +++ b/src/PythonScriptRunner.Abstractions/Running/Statuses.cs @@ -83,6 +83,57 @@ public enum AttemptCause LeaseRecovery, } +/// Who or what started a run. +public enum RunTrigger +{ + /// Code calling the library. + Library, + /// A user in the panel. + Dashboard, + /// A cron schedule. + Schedule, + /// The public queue command. + QueueCommand, +} + +/// How a run is executed. +public enum ExecutionMode +{ + /// In the calling process, waiting for the result. + Sync, + /// By a worker, through the queue. + Async, +} + +/// Life cycle of a stage or pipeline version. +public enum VersionStatus +{ + /// Editable; can be tested but not used by published pipelines. + Draft, + /// Immutable and runnable. + Published, + /// Immutable and no longer runnable (terminal). + Archived, +} + +/// Whether a stored file was given to a run or produced by it. +public enum ArtifactDirection +{ + /// Input of the run. + Input, + /// Output of a step. + Output, +} + +/// Output stream of a Python process. +public enum LogStream +{ + /// Standard output. + Stdout, + /// Standard error. + Stderr, +} + /// Classification of a failure. public enum ErrorKind { diff --git a/src/PythonScriptRunner.Core/Catalog/CatalogManifests.cs b/src/PythonScriptRunner.Core/Catalog/CatalogManifests.cs new file mode 100644 index 0000000..73ac004 --- /dev/null +++ b/src/PythonScriptRunner.Core/Catalog/CatalogManifests.cs @@ -0,0 +1,258 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Nodes; +using Json.Schema; +using PythonScriptRunner.Core.Catalog; +using PythonScriptRunner.Core.Pipelines; + +namespace PythonScriptRunner; + +/// +/// Reads and writes the folder catalog format: stages/<folder>/stage.json (+ script.py, +/// requirements.txt, requirements.lock) and pipelines/<key>.pipeline.json. Used by the folder +/// catalog and by import/export. +/// +public static class CatalogManifests +{ + /// Manifest file of a stage folder. + public const string StageManifestName = "stage.json"; + + /// Default script file of a stage folder. + public const string ScriptName = "script.py"; + + /// Default requirements file of a stage folder. + public const string RequirementsName = "requirements.txt"; + + /// Locked requirements of a stage folder (used as-is when present). + public const string LockName = "requirements.lock"; + + /// Suffix of pipeline manifests. + public const string PipelineSuffix = ".pipeline.json"; + + private const string SchemaBase = "https://raw.githubusercontent.com/net0well/PythonScriptRunner/main/schemas/"; + + private static readonly Lazy StageSchema = new(() => LoadSchema("stage.v1.json"), LazyThreadSafetyMode.ExecutionAndPublication); + private static readonly Lazy PipelineSchema = new(() => LoadSchema("pipeline.v1.json"), LazyThreadSafetyMode.ExecutionAndPublication); + + /// Writes the stage.json of a stage (script in script.py, requirements in requirements.txt). + /// The stage. + public static JsonObject WriteStage(StageDefinition stage) + { + ArgumentNullException.ThrowIfNull(stage); + var json = new JsonObject + { + ["$schema"] = SchemaBase + "stage.v1.json", + ["key"] = stage.Key, + ["version"] = stage.Version, + ["name"] = stage.Name, + }; + if (stage.Description is not null) + { + json["description"] = stage.Description; + } + + if (stage.PythonVersion is not null) + { + json["python"] = stage.PythonVersion; + } + + json["script"] = ScriptName; + if (!string.IsNullOrWhiteSpace(stage.Requirements)) + { + json["requirements"] = RequirementsName; + } + + if (stage.Timeout is { } timeout) + { + json["timeoutSeconds"] = (int)Math.Ceiling(timeout.TotalSeconds); + } + + if (stage.MemoryLimitMb is { } memory) + { + json["memoryLimitMb"] = memory; + } + + json["retry"] = new JsonObject + { + ["maxAttempts"] = stage.Retry.MaxAttempts, + ["initialDelaySeconds"] = (int)stage.Retry.InitialDelay.TotalSeconds, + ["backoffMultiplier"] = stage.Retry.BackoffMultiplier, + }; + json["idempotent"] = stage.Idempotent; + json["secrets"] = new JsonArray([.. stage.RequiredSecrets.Select(s => (JsonNode)JsonValue.Create(s))]); + json["inputs"] = FieldDefinitionJson.ToJson(stage.Inputs); + json["outputs"] = FieldDefinitionJson.ToJson(stage.Outputs); + return json; + } + + /// Writes the *.pipeline.json of a pipeline (compact binding syntax). + /// The pipeline. + public static JsonObject WritePipeline(PipelineDefinition pipeline) + { + ArgumentNullException.ThrowIfNull(pipeline); + var json = new JsonObject + { + ["$schema"] = SchemaBase + "pipeline.v1.json", + ["key"] = pipeline.Key, + ["version"] = pipeline.Version, + ["name"] = pipeline.Name, + }; + if (pipeline.Description is not null) + { + json["description"] = pipeline.Description; + } + + json["inputs"] = FieldDefinitionJson.ToJson(pipeline.Inputs); + json["steps"] = new JsonArray([.. pipeline.Steps.Select(step => + { + var node = new JsonObject + { + ["key"] = step.Key, + ["stage"] = $"{step.StageKey}@{step.StageVersion.ToString(CultureInfo.InvariantCulture)}", + }; + if (step.DisplayName is not null) + { + node["displayName"] = step.DisplayName; + } + + if (step.Bindings.Count > 0) + { + node["bindings"] = new JsonObject(step.Bindings.Select(b => KeyValuePair.Create(b.Input, (JsonNode?)WriteSource(b.Source)))); + } + + return (JsonNode)node; + })]); + json["outputs"] = new JsonArray([.. pipeline.Outputs.Select(output => + { + var node = new JsonObject { ["name"] = output.Name, ["source"] = output.Source.ToString() }; + if (output.Description is not null) + { + node["description"] = output.Description; + } + + return (JsonNode)node; + })]); + return json; + } + + /// Writes field declarations in the manifest format (camelCase; see the data mapping). + /// Fields. + public static JsonArray WriteFields(IEnumerable fields) => FieldDefinitionJson.ToJson(fields); + + /// Reads field declarations written by . + /// JSON array of fields. + /// A field is malformed. + public static IReadOnlyList ReadFields(JsonArray fields) + { + ArgumentNullException.ThrowIfNull(fields); + return [.. fields.Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))]; + } + + /// Reads a stage.json validated against schemas/stage.v1.json. + /// Content of stage.json. + /// The script of the stage. + /// Content of the requirements file, if any. + /// Content of requirements.lock, if any. + /// The manifest is not valid JSON or does not match the schema. + public static StageDefinition ReadStage(string manifest, ScriptSource script, string? requirements, string? lockedRequirements) + { + var json = ParseValidated(manifest, StageSchema.Value, "schemas/stage.v1.json"); + return StageFromJson(json, script, requirements, lockedRequirements); + } + + /// Reads a *.pipeline.json validated against schemas/pipeline.v1.json. + /// Content of the manifest. + /// The manifest is not valid JSON or does not match the schema. + public static PipelineDefinition ReadPipeline(string manifest) + { + var json = ParseValidated(manifest, PipelineSchema.Value, "schemas/pipeline.v1.json"); + try + { + return PipelineJson.FromJson(json); + } + catch (ArgumentException ex) + { + throw new PsrConfigurationException(ex.Message, ex); + } + } + + /// Builds a stage from a manifest already validated by the schema. + internal static StageDefinition StageFromJson(JsonObject json, ScriptSource script, string? requirements, string? lockedRequirements) + { + var retry = json["retry"]?.AsObject(); + return new StageDefinition + { + Key = json["key"]!.GetValue(), + Version = json["version"]!.GetValue(), + Name = json["name"]!.GetValue(), + Description = json["description"]?.GetValue(), + Script = script, + PythonVersion = json["python"]?.GetValue(), + Requirements = requirements, + LockedRequirements = lockedRequirements, + Inputs = [.. json["inputs"]!.AsArray().Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))], + Outputs = [.. json["outputs"]!.AsArray().Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))], + RequiredSecrets = json["secrets"] is JsonArray secrets ? [.. secrets.Select(s => s!.GetValue())] : [], + Timeout = json["timeoutSeconds"] is { } seconds ? TimeSpan.FromSeconds(seconds.GetValue()) : null, + MemoryLimitMb = json["memoryLimitMb"]?.GetValue(), + Retry = retry is null + ? RetryPolicy.None + : new RetryPolicy( + retry["maxAttempts"]?.GetValue() ?? 0, + TimeSpan.FromSeconds(retry["initialDelaySeconds"]?.GetValue() ?? 10), + retry["backoffMultiplier"]?.GetValue() ?? 2.0), + Idempotent = json["idempotent"]?.GetValue() ?? false, + }; + } + + /// Parses a manifest and validates it against the schema; errors cite the JSON location. + internal static JsonObject ParseValidated(string manifest, bool pipeline) => + ParseValidated(manifest, pipeline ? PipelineSchema.Value : StageSchema.Value, pipeline ? "schemas/pipeline.v1.json" : "schemas/stage.v1.json"); + + private static JsonObject ParseValidated(string manifest, JsonSchema schema, string schemaName) + { + JsonDocument document; + try + { + document = JsonDocument.Parse(manifest, new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true }); + } + catch (JsonException ex) + { + throw new PsrConfigurationException($"invalid JSON: {ex.Message}", ex); + } + + using (document) + { + var results = schema.Evaluate(document.RootElement, new EvaluationOptions { OutputFormat = OutputFormat.List }); + if (results.IsValid) + { + return JsonNode.Parse(document.RootElement.GetRawText())!.AsObject(); + } + + var errors = new List(); + foreach (var detail in (IEnumerable)[results, .. results.Details ?? []]) + { + foreach (var message in detail.Errors?.Values ?? Enumerable.Empty()) + { + errors.Add($"{(detail.InstanceLocation.ToString() is { Length: > 0 } at ? at : "/")}: {message}"); + } + } + + throw new PsrConfigurationException(errors.Count > 0 ? string.Join("; ", errors) : $"does not match {schemaName}"); + } + } + + private static JsonNode WriteSource(BindingSource source) => source switch + { + ConstantSource constant => new JsonObject { ["const"] = constant.Value?.DeepClone() }, + _ => JsonValue.Create(source.ToString())!, + }; + + private static JsonSchema LoadSchema(string name) + { + using var stream = typeof(CatalogManifests).Assembly.GetManifestResourceStream($"PythonScriptRunner.Schemas.{name}") + ?? throw new InvalidOperationException($"Embedded schema {name} not found."); + using var reader = new StreamReader(stream); + return JsonSchema.FromText(reader.ReadToEnd()); + } +} diff --git a/src/PythonScriptRunner.Core/Catalog/FileCatalogSource.cs b/src/PythonScriptRunner.Core/Catalog/FileCatalogSource.cs index fe30434..94b0efe 100644 --- a/src/PythonScriptRunner.Core/Catalog/FileCatalogSource.cs +++ b/src/PythonScriptRunner.Core/Catalog/FileCatalogSource.cs @@ -1,7 +1,4 @@ using System.Text.Json; -using System.Text.Json.Nodes; -using Json.Schema; -using PythonScriptRunner.Core.Pipelines; namespace PythonScriptRunner.Core.Catalog; @@ -11,11 +8,6 @@ namespace PythonScriptRunner.Core.Catalog; /// internal sealed class FileCatalogSource(string root) : ICatalogSource { - private const string LockFileName = "requirements.lock"; - - private static readonly Lazy StageSchema = new(() => LoadSchema("stage.v1.json"), LazyThreadSafetyMode.ExecutionAndPublication); - private static readonly Lazy PipelineSchema = new(() => LoadSchema("pipeline.v1.json"), LazyThreadSafetyMode.ExecutionAndPublication); - public string Description => $"folder '{root}'"; public IReadOnlyList LoadStages() @@ -26,7 +18,7 @@ public IReadOnlyList LoadStages() return []; } - var manifests = Directory.EnumerateDirectories(folder).Order(StringComparer.Ordinal).Select(d => Path.Combine(d, "stage.json")); + var manifests = Directory.EnumerateDirectories(folder).Order(StringComparer.Ordinal).Select(d => Path.Combine(d, CatalogManifests.StageManifestName)); return LoadAll(manifests, "stage catalog", (manifest, errors) => File.Exists(manifest) ? LoadStage(manifest, errors) : Missing(manifest, errors)); } @@ -39,8 +31,8 @@ public IReadOnlyList LoadPipelines() return []; } - var manifests = Directory.EnumerateFiles(folder, "*.pipeline.json").Order(StringComparer.Ordinal); - return LoadAll(manifests, "pipeline catalog", LoadPipeline); + var manifests = Directory.EnumerateFiles(folder, "*" + CatalogManifests.PipelineSuffix).Order(StringComparer.Ordinal); + return LoadAll(manifests, "pipeline catalog", (manifest, _) => CatalogManifests.ReadPipeline(File.ReadAllText(manifest))); } private string? Subfolder(string name) @@ -88,14 +80,10 @@ private static List LoadAll(IEnumerable manifests, string what, Fu private static StageDefinition? LoadStage(string manifest, List errors) { - if (ParseAndValidate(manifest, StageSchema.Value, "schemas/stage.v1.json", errors) is not { } json) - { - return null; - } - + var json = CatalogManifests.ParseValidated(File.ReadAllText(manifest), pipeline: false); var folder = Path.GetDirectoryName(manifest)!; var requirementsName = json["requirements"]?.GetValue(); - var requirementsPath = Path.Combine(folder, requirementsName ?? "requirements.txt"); + var requirementsPath = Path.Combine(folder, requirementsName ?? CatalogManifests.RequirementsName); string? requirements = null; if (File.Exists(requirementsPath)) { @@ -107,72 +95,8 @@ private static List LoadAll(IEnumerable manifests, string what, Fu return null; } - var lockPath = Path.Combine(folder, LockFileName); - var retry = json["retry"]?.AsObject(); - return new StageDefinition - { - Key = json["key"]!.GetValue(), - Version = json["version"]!.GetValue(), - Name = json["name"]!.GetValue(), - Description = json["description"]?.GetValue(), - Script = ScriptSource.FromFile(Path.Combine(folder, json["script"]?.GetValue() ?? "script.py")), - PythonVersion = json["python"]?.GetValue(), - Requirements = requirements, - LockedRequirements = File.Exists(lockPath) ? File.ReadAllText(lockPath) : null, - Inputs = [.. json["inputs"]!.AsArray().Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))], - Outputs = [.. json["outputs"]!.AsArray().Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))], - RequiredSecrets = json["secrets"] is JsonArray secrets ? [.. secrets.Select(s => s!.GetValue())] : [], - Timeout = json["timeoutSeconds"] is { } seconds ? TimeSpan.FromSeconds(seconds.GetValue()) : null, - MemoryLimitMb = json["memoryLimitMb"]?.GetValue(), - Retry = retry is null - ? RetryPolicy.None - : new RetryPolicy( - retry["maxAttempts"]?.GetValue() ?? 0, - TimeSpan.FromSeconds(retry["initialDelaySeconds"]?.GetValue() ?? 10), - retry["backoffMultiplier"]?.GetValue() ?? 2.0), - Idempotent = json["idempotent"]?.GetValue() ?? false, - }; - } - - private static PipelineDefinition? LoadPipeline(string manifest, List errors) => - ParseAndValidate(manifest, PipelineSchema.Value, "schemas/pipeline.v1.json", errors) is { } json - ? PipelineJson.FromJson(json) - : null; - - private static JsonObject? ParseAndValidate(string manifest, JsonSchema schema, string schemaName, List errors) - { - using var document = JsonDocument.Parse( - File.ReadAllText(manifest), - new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true }); - - var results = schema.Evaluate(document.RootElement, new EvaluationOptions { OutputFormat = OutputFormat.List }); - if (results.IsValid) - { - return JsonNode.Parse(document.RootElement.GetRawText())!.AsObject(); - } - - var before = errors.Count; - foreach (var detail in (IEnumerable)[results, .. results.Details ?? []]) - { - foreach (var message in detail.Errors?.Values ?? Enumerable.Empty()) - { - errors.Add($"{manifest}: {(detail.InstanceLocation.ToString() is { Length: > 0 } at ? at : "/")}: {message}"); - } - } - - if (errors.Count == before) - { - errors.Add($"{manifest}: does not match {schemaName}"); - } - - return null; - } - - private static JsonSchema LoadSchema(string name) - { - using var stream = typeof(FileCatalogSource).Assembly.GetManifestResourceStream($"PythonScriptRunner.Schemas.{name}") - ?? throw new InvalidOperationException($"Embedded schema {name} not found."); - using var reader = new StreamReader(stream); - return JsonSchema.FromText(reader.ReadToEnd()); + var lockPath = Path.Combine(folder, CatalogManifests.LockName); + var script = ScriptSource.FromFile(Path.Combine(folder, json["script"]?.GetValue() ?? CatalogManifests.ScriptName)); + return CatalogManifests.StageFromJson(json, script, requirements, File.Exists(lockPath) ? File.ReadAllText(lockPath) : null); } } diff --git a/src/PythonScriptRunner.Core/Catalog/StageDefinitionValidator.cs b/src/PythonScriptRunner.Core/Catalog/StageDefinitionValidator.cs index 60fcf09..624806b 100644 --- a/src/PythonScriptRunner.Core/Catalog/StageDefinitionValidator.cs +++ b/src/PythonScriptRunner.Core/Catalog/StageDefinitionValidator.cs @@ -3,6 +3,18 @@ namespace PythonScriptRunner.Core.Catalog; +/// Public validator of stage definitions (used by the panel and by publication). +internal sealed class StageValidator : IStageValidator +{ + public const string Code = "STAGE_DEFINITION"; + + public ValidationReport Validate(StageDefinition stage) + { + ArgumentNullException.ThrowIfNull(stage); + return new ValidationReport([.. StageDefinitionValidator.Validate(stage).Select(e => new ValidationIssue(Code, e))], []); + } +} + /// Validates a stage definition (same rules as schemas/stage.v1.json plus semantic checks). internal static partial class StageDefinitionValidator { diff --git a/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs b/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs index bea51a8..f9d248d 100644 --- a/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs +++ b/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs @@ -35,13 +35,16 @@ public static IPythonScriptRunnerBuilder AddPythonScriptRunner( services.TryAddSingleton(TimeProvider.System); services.TryAddSingleton(); services.TryAddSingleton(); + services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); - services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(sp => sp.GetRequiredService()); + services.TryAddSingleton(sp => sp.GetRequiredService()); services.TryAddEnumerable(ServiceDescriptor.Singleton()); return new PythonScriptRunnerBuilder(services); } diff --git a/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs b/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs index a29269c..1c9fce8 100644 --- a/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs +++ b/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs @@ -73,4 +73,7 @@ internal static partial class Log [LoggerMessage(9, LogLevel.Warning, "Pipeline {PipelineKey}@{PipelineVersion}: {Warning}")] public static partial void PipelineWarning(ILogger logger, string pipelineKey, int pipelineVersion, string warning); + + [LoggerMessage(10, LogLevel.Error, "Could not record the failure of run {RunId}")] + public static partial void RecordingFailed(ILogger logger, RunId runId, Exception exception); } diff --git a/src/PythonScriptRunner.Core/Environments/IEnvironmentManager.cs b/src/PythonScriptRunner.Core/Environments/IEnvironmentManager.cs index 5ee278f..3cb1f70 100644 --- a/src/PythonScriptRunner.Core/Environments/IEnvironmentManager.cs +++ b/src/PythonScriptRunner.Core/Environments/IEnvironmentManager.cs @@ -11,6 +11,19 @@ public interface IEnvironmentManager /// Resolution failed. Task LockAsync(string? pythonVersion, string? requirements, CancellationToken cancellationToken = default); + /// Compiles a script with the given Python version (no dependencies needed). + /// Python version; null uses the runner default. + /// Python source. + /// Cancellation token. + /// The syntax error, or null when the script compiles. + /// uv or the Python version is not available. + Task CheckSyntaxAsync(string? pythonVersion, string script, CancellationToken cancellationToken = default); + + /// Hash that identifies the environment of a Python version and locked requirements (the venv cache key). + /// Python version; null uses the runner default. + /// Locked requirements (empty when there are none). + string ComputeHash(string? pythonVersion, string? lockedRequirements); + /// Returns the environment of a stage, creating it on first use. /// The stage. /// Cancellation token. @@ -25,6 +38,12 @@ public interface IEnvironmentManager /// Whether it was created by this call. public sealed record PythonEnvironment(string Hash, string Directory, string PythonExecutable, bool Created); +/// A script that does not compile. +/// Python error message. +/// 1-based line, when known. +/// 1-based column, when known. +public sealed record ScriptSyntaxError(string Message, int? Line, int? Column); + /// A Python environment could not be resolved or created. public sealed class PythonEnvironmentException : PsrException { diff --git a/src/PythonScriptRunner.Core/Environments/UvEnvironmentManager.cs b/src/PythonScriptRunner.Core/Environments/UvEnvironmentManager.cs index a2275de..701abeb 100644 --- a/src/PythonScriptRunner.Core/Environments/UvEnvironmentManager.cs +++ b/src/PythonScriptRunner.Core/Environments/UvEnvironmentManager.cs @@ -2,6 +2,7 @@ using System.Globalization; using System.Security.Cryptography; using System.Text; +using System.Text.Json.Nodes; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using PythonScriptRunner.Core.Diagnostics; @@ -18,6 +19,62 @@ internal sealed class UvEnvironmentManager( private const string ReadyMarker = ".psr-ready"; private static readonly TimeSpan LockTimeout = TimeSpan.FromMinutes(20); + /// Compiles the script given as argument and prints the outcome as JSON (same check as py_compile). + private const string SyntaxChecker = """ + import json + import sys + + with open(sys.argv[1], encoding="utf-8") as source_file: + source = source_file.read() + try: + compile(source, "script.py", "exec") + except (SyntaxError, ValueError) as error: + print(json.dumps({ + "ok": False, + "message": getattr(error, "msg", None) or str(error), + "line": getattr(error, "lineno", None), + "column": getattr(error, "offset", None), + })) + else: + print(json.dumps({"ok": True})) + """; + + public async Task CheckSyntaxAsync(string? pythonVersion, string script, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(script); + var folder = Path.Combine(Path.GetTempPath(), "psr-syntax", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(folder); + try + { + var checker = Path.Combine(folder, "check_syntax.py"); + var target = Path.Combine(folder, "script.py"); + await File.WriteAllTextAsync(checker, SyntaxChecker, cancellationToken).ConfigureAwait(false); + await File.WriteAllTextAsync(target, script, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), cancellationToken).ConfigureAwait(false); + var version = pythonVersion ?? options.Value.DefaultPythonVersion; + var output = await CreateCli().RunAsync( + ["run", "--no-project", "--python", version, "--directory", folder, "python", "-I", checker, target], null, cancellationToken) + .ConfigureAwait(false); + var result = JsonNode.Parse(output.Trim().Split('\n')[^1])!.AsObject(); + return result["ok"]!.GetValue() + ? null + : new ScriptSyntaxError(result["message"]?.GetValue() ?? "invalid syntax", result["line"]?.GetValue(), result["column"]?.GetValue()); + } + finally + { + try + { + Directory.Delete(folder, recursive: true); + } + catch (IOException) + { + // Best effort. + } + } + } + + public string ComputeHash(string? pythonVersion, string? lockedRequirements) => + Sha256Hex($"{pythonVersion ?? options.Value.DefaultPythonVersion}\n{Normalize(lockedRequirements ?? string.Empty)}\n{PsrunnerIdentity(options.Value.PsrunnerRequirement)}"); + public async Task LockAsync(string? pythonVersion, string? requirements, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(requirements)) @@ -40,7 +97,7 @@ public async Task EnsureAsync(StageDefinition stage, Cancella var settings = options.Value; var version = stage.PythonVersion ?? settings.DefaultPythonVersion; var locked = stage.LockedRequirements ?? await GetCachedLockAsync(stage, version, cancellationToken).ConfigureAwait(false); - var hash = Sha256Hex($"{version}\n{Normalize(locked)}\n{PsrunnerIdentity(settings.PsrunnerRequirement)}"); + var hash = ComputeHash(version, locked); var directory = Path.Combine(settings.EnvironmentCacheRoot, "envs", hash[..32]); var python = OperatingSystem.IsWindows() ? Path.Combine(directory, "Scripts", "python.exe") @@ -113,7 +170,16 @@ private async Task GetCachedLockAsync(StageDefinition stage, string vers Directory.CreateDirectory(Path.GetDirectoryName(path)!); var temporary = $"{path}.{Guid.NewGuid():N}.tmp"; await File.WriteAllTextAsync(temporary, locked, cancellationToken).ConfigureAwait(false); - File.Move(temporary, path, overwrite: true); + try + { + File.Move(temporary, path, overwrite: true); + } + catch (Exception ex) when ((ex is IOException or UnauthorizedAccessException) && File.Exists(path)) + { + // Another process stored the same lock at the same moment (Windows denies replacing a file in use); keep theirs. + File.Delete(temporary); + } + return locked; } diff --git a/src/PythonScriptRunner.Core/Pipelines/PipelineValidator.cs b/src/PythonScriptRunner.Core/Pipelines/PipelineValidator.cs index ed156a1..7954045 100644 --- a/src/PythonScriptRunner.Core/Pipelines/PipelineValidator.cs +++ b/src/PythonScriptRunner.Core/Pipelines/PipelineValidator.cs @@ -12,7 +12,14 @@ public async ValueTask ValidateAsync(PipelineDefinition pipeli var reference = (step.StageKey, step.StageVersion); if (!stages.ContainsKey(reference)) { - stages[reference] = await catalog.FindStageAsync(step.StageKey, step.StageVersion, cancellationToken).ConfigureAwait(false); + try + { + stages[reference] = await catalog.FindStageAsync(step.StageKey, step.StageVersion, cancellationToken).ConfigureAwait(false); + } + catch (VersionNotRunnableException) + { + stages[reference] = null; + } } } diff --git a/src/PythonScriptRunner.Core/Pipelines/StepInputResolver.cs b/src/PythonScriptRunner.Core/Pipelines/StepInputResolver.cs index 8854dd9..d8f78c3 100644 --- a/src/PythonScriptRunner.Core/Pipelines/StepInputResolver.cs +++ b/src/PythonScriptRunner.Core/Pipelines/StepInputResolver.cs @@ -1,11 +1,10 @@ using System.Text.Json.Nodes; -using PythonScriptRunner.Core.Execution; using PythonScriptRunner.Core.Values; namespace PythonScriptRunner.Core.Pipelines; -/// Inputs of one pipeline step after resolving its bindings. -internal sealed record StepInputs(JsonObject Wire, IReadOnlyList Files, IReadOnlyList Errors); +/// Inputs of one pipeline step after resolving its bindings (files are shared with their source, never copied). +internal sealed record StepInputs(JsonObject Wire, IReadOnlyDictionary> Files, IReadOnlyList Errors); /// Resolves the inputs of a step from pipeline inputs, earlier step outputs, constants and implicit bindings. internal static class StepInputResolver @@ -20,7 +19,7 @@ public static StepInputs Resolve( { var step = pipeline.Steps[index]; var wire = new JsonObject(); - var files = new List(); + var files = new Dictionary>(StringComparer.Ordinal); var errors = new List(); foreach (var target in stage.Inputs) { @@ -30,8 +29,7 @@ public static StepInputs Resolve( { if (sourceFiles is { Count: > 0 }) { - files.AddRange(sourceFiles.Select((file, i) => - new InputFileCopy(target.Name, i, file.LocalPath!, file.FileName, file.ContentType, file.Length ?? 0, file.Sha256!))); + files[target.Name] = sourceFiles; } else if (target.Required) { diff --git a/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj b/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj index e5b04ba..70f4fef 100644 --- a/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj +++ b/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj @@ -1,4 +1,4 @@ - + PythonScriptRunner @@ -16,6 +16,8 @@ + + diff --git a/src/PythonScriptRunner.Core/Running/Infrastructure.cs b/src/PythonScriptRunner.Core/Running/Infrastructure.cs index 1133cbc..7c747ba 100644 --- a/src/PythonScriptRunner.Core/Running/Infrastructure.cs +++ b/src/PythonScriptRunner.Core/Running/Infrastructure.cs @@ -8,10 +8,18 @@ namespace PythonScriptRunner.Core.Running; /// Default recorder: nothing is persisted without the EF package. internal sealed class NullRunRecorder : IRunRecorder { - public ValueTask RunStartedAsync(RunStarted run, CancellationToken cancellationToken) => ValueTask.CompletedTask; + public ValueTask RunStartingAsync(RunStarting run, CancellationToken cancellationToken) => ValueTask.FromResult(null); + + public ValueTask RunResumingAsync(RunResuming run, CancellationToken cancellationToken) => ValueTask.CompletedTask; + + public ValueTask StepStartedAsync(StepStarted step, CancellationToken cancellationToken) => ValueTask.CompletedTask; + + public ValueTask AttemptStartedAsync(AttemptStarted attempt, CancellationToken cancellationToken) => ValueTask.CompletedTask; public ValueTask AttemptFinishedAsync(AttemptFinished attempt, CancellationToken cancellationToken) => ValueTask.CompletedTask; + public ValueTask StepFinishedAsync(StepFinished step, CancellationToken cancellationToken) => ValueTask.CompletedTask; + public ValueTask RunFinishedAsync(RunFinished run, CancellationToken cancellationToken) => ValueTask.CompletedTask; } @@ -26,12 +34,22 @@ internal sealed class ConfigurationSecretProvider(IConfiguration? configuration /// Fails fast when the host starts: validates options, loads catalogs and removes leftover folders. internal sealed class PythonScriptRunnerStartup( - IOptions options, ICatalog catalog, WorkspaceJanitor janitor) : IHostedService + IOptions options, ICatalog catalog, IEnumerable sources, WorkspaceJanitor janitor) : IHostedService { public Task StartAsync(CancellationToken cancellationToken) { _ = options.Value; - (catalog as CompositeCatalog)?.EnsureLoaded(); + if (catalog is CompositeCatalog composite) + { + composite.EnsureLoaded(); + } + else if (sources.FirstOrDefault() is { } ignored) + { + throw new PsrConfigurationException( + $"The catalog is provided by {catalog.GetType().Name}, so stages and pipelines added with AddStage, AddPipeline or " + + $"AddFileCatalog ({ignored.Description}) would be ignored. Import them into that catalog instead."); + } + janitor.EnsureCleaned(); return Task.CompletedTask; } diff --git a/src/PythonScriptRunner.Core/Running/PythonRunner.cs b/src/PythonScriptRunner.Core/Running/PythonRunner.cs index a3b32b8..78ec229 100644 --- a/src/PythonScriptRunner.Core/Running/PythonRunner.cs +++ b/src/PythonScriptRunner.Core/Running/PythonRunner.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Diagnostics; using System.Globalization; using System.Security.Cryptography; @@ -12,11 +13,13 @@ namespace PythonScriptRunner.Core.Running; /// -/// Runs stages and pipelines in the current process: validation, secrets, environment, -/// attempts with retry and, for pipelines, bindings between steps. +/// Runs stages and pipelines in the current process: validation, secrets, environment, attempts with retry and, for +/// pipelines, bindings between steps. Every transition is reported to the . /// -internal sealed class PythonRunner : IPythonRunner, IDisposable +internal sealed class PythonRunner : IPythonRunner, IRunEngine, IDisposable { + private static readonly IReadOnlyDictionary> NoFiles = new Dictionary>(); + private readonly ICatalog _catalog; private readonly IEnvironmentManager _environments; private readonly IPythonExecutor _executor; @@ -29,6 +32,7 @@ internal sealed class PythonRunner : IPythonRunner, IDisposable private readonly ILogger _logger; private readonly ILogger _scriptLogger; private readonly Lazy _concurrency; + private readonly ConcurrentDictionary _active = new(); public PythonRunner( ICatalog catalog, @@ -75,34 +79,112 @@ public async Task RunStageAsync( string stageKey, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default) { ArgumentException.ThrowIfNullOrWhiteSpace(stageKey); + var stage = await ResolveStageAsync(stageKey, options?.Version, cancellationToken).ConfigureAwait(false); + return await RunStageAsync(stage, RunKind.Stage, inputs, options, cancellationToken).ConfigureAwait(false); + } + + public async Task RunPipelineAsync( + string pipelineKey, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(pipelineKey); + var pipeline = await ResolvePipelineAsync(pipelineKey, options?.Version, cancellationToken).ConfigureAwait(false); + return await RunPipelineAsync(pipeline, RunKind.Pipeline, inputs, options, cancellationToken).ConfigureAwait(false); + } + + public async Task RunStageAsync( + StageDefinition stage, RunKind kind, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(stage); var settings = Settings; _janitor.EnsureCleaned(); - var stage = await ResolveStageAsync(stageKey, options?.Version, cancellationToken).ConfigureAwait(false); var bound = InputBinder.Bind(stage.Inputs, inputs, settings.MaxFileBytes); cancellationToken.ThrowIfCancellationRequested(); - - return await RunAsync(RunKind.Stage, stage.Key, stage.Version, bound, options, settings, async (context, inputFiles) => - { - var step = await ExecuteStepAsync(stage, stage.Key, 1, bound.Wire, ToCopies(inputFiles), context, cancellationToken).ConfigureAwait(false); - return new RunOutcome(StatusOf(step.Status), [step], step.Outputs, step.Files, step.Error); - }, cancellationToken).ConfigureAwait(false); + var target = new RunTarget(kind, stage.Key, stage.Version, [new PlannedStep(stage.Key, 1, stage.Key, stage.Version)]); + return await RunNewAsync(target, bound, options, settings, + (context, inputFiles, token) => ExecuteStageAsync(stage, bound.Wire, inputFiles, context, prior: null, token), + cancellationToken).ConfigureAwait(false); } public async Task RunPipelineAsync( - string pipelineKey, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default) + PipelineDefinition pipeline, RunKind kind, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default) { - ArgumentException.ThrowIfNullOrWhiteSpace(pipelineKey); + ArgumentNullException.ThrowIfNull(pipeline); var settings = Settings; _janitor.EnsureCleaned(); - var pipeline = await ResolvePipelineAsync(pipelineKey, options?.Version, cancellationToken).ConfigureAwait(false); var bound = InputBinder.Bind(pipeline.Inputs, inputs, settings.MaxFileBytes); cancellationToken.ThrowIfCancellationRequested(); - - return await RunAsync(RunKind.Pipeline, pipeline.Key, pipeline.Version, bound, options, settings, - (context, inputFiles) => ExecutePipelineAsync(pipeline, bound.Wire, inputFiles, context, cancellationToken), + var steps = pipeline.Steps.Select((s, i) => new PlannedStep(s.Key, i + 1, s.StageKey, s.StageVersion)).ToList(); + var target = new RunTarget(kind, pipeline.Key, pipeline.Version, steps); + return await RunNewAsync(target, bound, options, settings, + (context, inputFiles, token) => ExecutePipelineAsync(pipeline, bound.Wire, inputFiles, context, NoPrevious, token), cancellationToken).ConfigureAwait(false); } + public async Task ResumeAsync(ResumeRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + if ((request.Stage is null) == (request.Pipeline is null)) + { + throw new ArgumentException("Set exactly one of Stage or Pipeline.", nameof(request)); + } + + var settings = Settings; + _janitor.EnsureCleaned(); + var context = new RunContext(new RunWorkspace(settings.WorkRoot, request.RunId), settings, request.Kind, request.TriggeredBy); + var started = _time.GetTimestamp(); + using var cancellation = Register(request.RunId, cancellationToken); + var token = cancellation.Token; + try + { + var inputFiles = await MaterializeStoredAsync(request.InputFiles, context.Workspace.InputsDirectory, token).ConfigureAwait(false); + var previous = new Dictionary(StringComparer.Ordinal); + foreach (var step in request.PreviousSteps) + { + previous[step.StepKey] = step.Status == StepStatus.Succeeded + ? step with + { + Files = await MaterializeStoredAsync(step.Files, Path.Combine(context.Workspace.ResultsDirectory, step.StepKey), token) + .ConfigureAwait(false), + } + : step; + } + + context.UsedBytes = TotalBytes(inputFiles) + previous.Values.Sum(s => TotalBytes(s.Files)); + await _recorder.RunResumingAsync(new RunResuming(request.RunId, request.TriggeredBy, _time.GetUtcNow()), token).ConfigureAwait(false); + return await CompleteAsync(context, started, () => request.Stage is { } stage + ? ExecuteStageAsync(stage, request.Inputs, inputFiles, context, previous.GetValueOrDefault(stage.Key), token) + : ExecutePipelineAsync(request.Pipeline!, request.Inputs, inputFiles, context, previous, token), token).ConfigureAwait(false); + } + catch + { + await context.Workspace.DeleteResultsAsync().ConfigureAwait(false); + throw; + } + finally + { + _active.TryRemove(request.RunId, out _); + context.Workspace.DeleteRun(); + } + } + + public bool RequestCancellation(RunId runId) + { + if (!_active.TryGetValue(runId, out var source)) + { + return false; + } + + try + { + source.Cancel(); + return true; + } + catch (ObjectDisposedException) + { + return false; + } + } + public void Dispose() { if (_concurrency.IsValueCreated) @@ -111,43 +193,54 @@ public void Dispose() } } - private async Task RunAsync( - RunKind kind, - string key, - int version, + private static IReadOnlyDictionary NoPrevious { get; } = new Dictionary(); + + private async Task RunNewAsync( + RunTarget target, BoundInputs bound, RunOptions? options, PythonScriptRunnerOptions settings, - Func>, Task> body, + Func>, CancellationToken, Task> body, CancellationToken cancellationToken) { - var runId = RunId.New(); + var runId = RunId.New(_time.GetUtcNow()); + var context = new RunContext(new RunWorkspace(settings.WorkRoot, runId), settings, target.Kind, options?.TriggeredBy ?? Actor.Library.Subject); var started = _time.GetTimestamp(); - await _recorder.RunStartedAsync( - new RunStarted(runId, kind, key, version, options?.TriggeredBy ?? "system:library", _time.GetUtcNow(), (JsonObject)bound.Wire.DeepClone()), - cancellationToken).ConfigureAwait(false); - - var context = new RunContext(new RunWorkspace(settings.WorkRoot, runId), settings); + using var cancellation = Register(runId, cancellationToken); + var token = cancellation.Token; try { - RunOutcome outcome; - try - { - var inputFiles = await MaterializeInputsAsync(bound.Files, context.Workspace, cancellationToken).ConfigureAwait(false); - context.UsedBytes = inputFiles.Values.SelectMany(files => files).Sum(file => file.Length ?? 0); - outcome = context.UsedBytes > settings.MaxRunFilesBytes - ? RunOutcome.Aborted(RunStatus.Failed, new RunError(ErrorKind.ArtifactLimitExceeded, "The input files exceed the run limit.") { Code = "RUN_FILES_TOO_LARGE" }) - : await body(context, inputFiles).ConfigureAwait(false); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + var inputFiles = await MaterializeInputsAsync(bound.Files, context.Workspace, token).ConfigureAwait(false); + context.UsedBytes = TotalBytes(inputFiles); + var overBudget = context.UsedBytes > settings.MaxRunFilesBytes; + var existing = await _recorder.RunStartingAsync( + new RunStarting + { + RunId = runId, + Kind = target.Kind, + TargetKey = target.Key, + TargetVersion = target.Version, + Trigger = options?.Trigger ?? RunTrigger.Library, + TriggeredBy = context.TriggeredBy, + IdempotencyKey = options?.IdempotencyKey, + Inputs = (JsonObject)bound.Wire.DeepClone(), + InputFiles = overBudget ? NoFiles : inputFiles, + Steps = target.Steps, + StartedAt = runId.CreatedAt, + }, + token).ConfigureAwait(false); + if (existing is not null) { - outcome = RunOutcome.Aborted(RunStatus.Cancelled, new RunError(ErrorKind.Cancelled, "The run was cancelled.")); + await context.Workspace.DeleteResultsAsync().ConfigureAwait(false); + return existing; } - await _recorder.RunFinishedAsync(new RunFinished(runId, outcome.Status, _time.GetUtcNow(), outcome.Outputs, outcome.Error), CancellationToken.None) - .ConfigureAwait(false); - return new RunResult(runId, kind, outcome.Status, _time.GetElapsedTime(started), outcome.Steps, outcome.Outputs, outcome.Files, outcome.Error, - context.Workspace.DeleteResultsAsync); + return await CompleteAsync(context, started, () => overBudget + ? Task.FromResult(RunOutcome.Aborted(RunStatus.Failed, new RunError(ErrorKind.ArtifactLimitExceeded, "The input files exceed the run limit.") + { + Code = "RUN_FILES_TOO_LARGE", + })) + : body(context, inputFiles, token), token).ConfigureAwait(false); } catch { @@ -156,15 +249,76 @@ await _recorder.RunFinishedAsync(new RunFinished(runId, outcome.Status, _time.Ge } finally { + _active.TryRemove(runId, out _); context.Workspace.DeleteRun(); } } + /// Executes the body, records the end of the run and builds the result. + private async Task CompleteAsync(RunContext context, long started, Func> body, CancellationToken cancellationToken) + { + RunOutcome outcome; + try + { + outcome = await body().ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + outcome = RunOutcome.Aborted(RunStatus.Cancelled, new RunError(ErrorKind.Cancelled, "The run was cancelled.")); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + await RecordCrashAsync(context.RunId, ex).ConfigureAwait(false); + throw; + } + + await _recorder.RunFinishedAsync( + new RunFinished(context.RunId, outcome.Status, _time.GetUtcNow(), outcome.Outputs, outcome.Files, outcome.Error), + CancellationToken.None).ConfigureAwait(false); + return new RunResult(context.RunId, context.Kind, outcome.Status, _time.GetElapsedTime(started), outcome.Steps, outcome.Outputs, outcome.Files, + outcome.Error, context.Workspace.DeleteResultsAsync); + } + + /// Best effort: a run interrupted by an unexpected error must not stay "running" in the history. + private async Task RecordCrashAsync(RunId runId, Exception exception) + { + try + { + var error = new RunError(ErrorKind.ProcessCrashed, $"The runner failed: {exception.Message}") { Code = "RUNNER_ERROR" }; + await _recorder.RunFinishedAsync(new RunFinished(runId, RunStatus.Failed, _time.GetUtcNow(), null, null, error), CancellationToken.None) + .ConfigureAwait(false); + } + catch (Exception recordError) when (recordError is not OutOfMemoryException) + { + Log.RecordingFailed(_logger, runId, recordError); + } + } + + private CancellationTokenSource Register(RunId runId, CancellationToken cancellationToken) + { + var source = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _active[runId] = source; + return source; + } + + private async Task ExecuteStageAsync( + StageDefinition stage, + JsonObject inputs, + IReadOnlyDictionary> files, + RunContext context, + StepResult? prior, + CancellationToken cancellationToken) + { + var step = await ExecuteStepAsync(stage, stage.Key, 1, inputs, files, context, prior, cancellationToken).ConfigureAwait(false); + return new RunOutcome(StatusOf(step.Status), [step], step.Outputs, step.Files, step.Error); + } + private async Task ExecutePipelineAsync( PipelineDefinition pipeline, JsonObject pipelineInputs, IReadOnlyDictionary> pipelineFiles, RunContext context, + IReadOnlyDictionary previous, CancellationToken cancellationToken) { var steps = new List(); @@ -174,9 +328,17 @@ private async Task ExecutePipelineAsync( for (var i = 0; i < pipeline.Steps.Count; i++) { var definition = pipeline.Steps[i]; + var prior = previous.GetValueOrDefault(definition.Key); if (status != RunStatus.Succeeded) { - steps.Add(Skipped(definition, i + 1)); + steps.Add(await RecordStepAsync(context, Skipped(definition, i + 1, prior)).ConfigureAwait(false)); + continue; + } + + if (prior is { Status: StepStatus.Succeeded }) + { + steps.Add(prior); + completed[definition.Key] = prior; continue; } @@ -184,8 +346,8 @@ private async Task ExecutePipelineAsync( ?? throw new PsrConfigurationException($"Stage '{definition.StageKey}@{definition.StageVersion}' of pipeline '{pipeline.Key}' is not in the catalog."); var inputs = StepInputResolver.Resolve(stage, pipeline, i, pipelineInputs, pipelineFiles, completed); var result = inputs.Errors.Count > 0 - ? InvalidInputs(stage, definition.Key, i + 1, inputs.Errors) - : await ExecuteStepAsync(stage, definition.Key, i + 1, inputs.Wire, inputs.Files, context, cancellationToken).ConfigureAwait(false); + ? await RecordStepAsync(context, InvalidInputs(stage, definition.Key, i + 1, inputs.Errors, prior)).ConfigureAwait(false) + : await ExecuteStepAsync(stage, definition.Key, i + 1, inputs.Wire, inputs.Files, context, prior, cancellationToken).ConfigureAwait(false); steps.Add(result); if (result.Status == StepStatus.Succeeded) { @@ -252,16 +414,25 @@ private async Task ResolvePipelineAsync(string key, int? ver throw new PipelineNotFoundException(key); } + /// Runs the attempts of one step; a step resumed after a failure continues the attempt numbering. private async Task ExecuteStepAsync( StageDefinition stage, string stepKey, int order, JsonObject inputs, - IReadOnlyList files, + IReadOnlyDictionary> files, RunContext context, + StepResult? prior, CancellationToken cancellationToken) { - var attempts = new List(); + await _recorder.StepStartedAsync( + new StepStarted(context.RunId, stepKey, order, WithFilePlaceholders(inputs, files, stage), files, _time.GetUtcNow()), + CancellationToken.None).ConfigureAwait(false); + + var attempts = new List(prior?.Attempts ?? []); + var first = attempts.Count + 1; + var firstCause = prior is { Status: not (StepStatus.Pending or StepStatus.Skipped) } ? AttemptCause.Reprocess : AttemptCause.Initial; + var copies = ToCopies(files); StepResult Finish(StepStatus status, RunError? error, AttemptOutcome? success = null) => new() { StepKey = stepKey, @@ -271,35 +442,41 @@ private async Task ExecuteStepAsync( Status = status, Attempts = attempts, Outputs = success?.Outputs, - Files = success?.Files ?? new Dictionary>(), + Files = success?.Files ?? NoFiles, Error = error is null ? null : error with { StepKey = stepKey }, }; - for (var number = 1; ; number++) + StepResult result; + for (var number = first; ; number++) { - var cause = number == 1 ? AttemptCause.Initial : AttemptCause.AutoRetry; - var outcome = await RunAttemptAsync(stage, stepKey, number, cause, inputs, files, context, cancellationToken).ConfigureAwait(false); + var cause = number == first ? firstCause : AttemptCause.AutoRetry; + var outcome = await RunAttemptAsync(stage, stepKey, order, number, cause, inputs, copies, context, cancellationToken).ConfigureAwait(false); attempts.Add(outcome.Attempt); - await _recorder.AttemptFinishedAsync(new AttemptFinished(context.Workspace.RunId, stepKey, outcome.Attempt), CancellationToken.None) + await _recorder.AttemptFinishedAsync(new AttemptFinished(context.RunId, stepKey, order, outcome.Attempt), CancellationToken.None) .ConfigureAwait(false); - switch (outcome.Attempt.Status) + if (outcome.Attempt.Status == AttemptStatus.Succeeded) { - case AttemptStatus.Succeeded: - context.UsedBytes += outcome.Files.Values.SelectMany(f => f).Sum(f => f.Length ?? 0); - return Finish(StepStatus.Succeeded, null, outcome); - case AttemptStatus.Cancelled: - return Finish(StepStatus.Cancelled, outcome.Attempt.Error); + context.UsedBytes += TotalBytes(outcome.Files); + result = Finish(StepStatus.Succeeded, null, outcome); + break; + } + + if (outcome.Attempt.Status == AttemptStatus.Cancelled) + { + result = Finish(StepStatus.Cancelled, outcome.Attempt.Error); + break; } var error = outcome.Attempt.Error!; Log.AttemptFailed(_logger, stage.Key, number, error.Kind, error.Message); - if (number > stage.Retry.MaxAttempts) + if (number - first >= stage.Retry.MaxAttempts) { - return Finish(outcome.Attempt.Status == AttemptStatus.TimedOut ? StepStatus.TimedOut : StepStatus.Failed, error); + result = Finish(outcome.Attempt.Status == AttemptStatus.TimedOut ? StepStatus.TimedOut : StepStatus.Failed, error); + break; } - var delay = stage.Retry.GetDelay(number); + var delay = stage.Retry.GetDelay(number - first + 1); Log.RetryScheduled(_logger, stage.Key, delay, number + 1); try { @@ -307,14 +484,24 @@ await _recorder.AttemptFinishedAsync(new AttemptFinished(context.Workspace.RunId } catch (OperationCanceledException) { - return Finish(StepStatus.Cancelled, new RunError(ErrorKind.Cancelled, "The run was cancelled while waiting to retry.")); + result = Finish(StepStatus.Cancelled, new RunError(ErrorKind.Cancelled, "The run was cancelled while waiting to retry.")); + break; } } + + return await RecordStepAsync(context, result).ConfigureAwait(false); + } + + private async Task RecordStepAsync(RunContext context, StepResult step) + { + await _recorder.StepFinishedAsync(new StepFinished(context.RunId, step, _time.GetUtcNow()), CancellationToken.None).ConfigureAwait(false); + return step; } private async Task RunAttemptAsync( StageDefinition stage, string stepKey, + int order, int number, AttemptCause cause, JsonObject inputs, @@ -323,16 +510,20 @@ private async Task RunAttemptAsync( CancellationToken cancellationToken) { var settings = context.Settings; + var attemptId = AttemptId.New(); var startedAt = _time.GetUtcNow(); var started = _time.GetTimestamp(); var queued = TimeSpan.Zero; string? environmentHash = null; var workDirectory = context.Workspace.AttemptDirectory(stepKey, number); + await _recorder.AttemptStartedAsync( + new AttemptStarted(context.RunId, stepKey, order, attemptId, number, cause, context.TriggeredBy, startedAt), + CancellationToken.None).ConfigureAwait(false); using var activity = PsrDiagnostics.Source.StartActivity("psr.stage.execute"); activity?.SetTag("psr.stage.key", stage.Key); activity?.SetTag("psr.stage.version", stage.Version); - activity?.SetTag("psr.run.id", context.Workspace.RunId.ToString()); + activity?.SetTag("psr.run.id", context.RunId.ToString()); activity?.SetTag("psr.step.key", stepKey); activity?.SetTag("psr.attempt", number); @@ -358,9 +549,10 @@ private async Task RunAttemptAsync( { Stage = stage, Environment = environment, - RunId = context.Workspace.RunId, + RunId = context.RunId, StepKey = stepKey, Attempt = number, + IsTest = context.Kind is RunKind.StageTest or RunKind.PipelineTest, WorkDirectory = workDirectory, Inputs = inputs, Files = files, @@ -397,7 +589,7 @@ private async Task RunAttemptAsync( var overhead = duration - execution.ProcessDuration - queued; var attempt = new AttemptResult { - Id = AttemptId.New(), + Id = attemptId, Number = number, Cause = cause, Status = execution.Status, @@ -453,26 +645,80 @@ private static async Task>> M var list = new List(); foreach (var bound in group.OrderBy(f => f.Index)) { - var name = RunWorkspace.SafeFileName(bound.File.FileName); - var target = Path.Combine(workspace.InputsDirectory, bound.Field, bound.Index.ToString(CultureInfo.InvariantCulture), name); - Directory.CreateDirectory(Path.GetDirectoryName(target)!); - await bound.File.SaveAsAsync(target, cancellationToken).ConfigureAwait(false); + list.Add(await CopyToAsync(bound.File, Path.Combine(workspace.InputsDirectory, bound.Field, bound.Index.ToString(CultureInfo.InvariantCulture)), + cancellationToken).ConfigureAwait(false)); + } - string sha256; - await using (var stream = File.OpenRead(target)) - { - sha256 = Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false)); - } + result[group.Key] = list; + } + + return result; + } - list.Add(PsrFile.FromProducedFile(target, name, bound.File.ContentType, sha256)); + /// Downloads stored files (e.g. from the database) to ; files already on disk are kept. + private static async Task>> MaterializeStoredAsync( + IReadOnlyDictionary> files, string directory, CancellationToken cancellationToken) + { + var result = new Dictionary>(StringComparer.Ordinal); + foreach (var (field, list) in files) + { + var local = new List(); + for (var i = 0; i < list.Count; i++) + { + var file = list[i]; + local.Add(file.LocalPath is not null + ? file + : await CopyToAsync(file, Path.Combine(directory, field, i.ToString(CultureInfo.InvariantCulture)), cancellationToken).ConfigureAwait(false)); } - result[group.Key] = list; + result[field] = local; } return result; } + private static async Task CopyToAsync(PsrFile file, string directory, CancellationToken cancellationToken) + { + var name = RunWorkspace.SafeFileName(file.FileName); + var target = Path.Combine(directory, name); + Directory.CreateDirectory(directory); + await file.SaveAsAsync(target, cancellationToken).ConfigureAwait(false); + + string sha256; + await using (var stream = File.OpenRead(target)) + { + sha256 = Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false)); + } + + var copy = PsrFile.FromProducedFile(target, name, file.ContentType, sha256); + if (file.ArtifactId is { } artifactId) + { + copy.AttachArtifact(artifactId); + } + + return copy; + } + + /// Copy of the step inputs with {"$file": {...}} metadata for file fields (a list for list<file>), as recorded. + private static JsonObject WithFilePlaceholders(JsonObject inputs, IReadOnlyDictionary> files, StageDefinition stage) + { + var copy = (JsonObject)inputs.DeepClone(); + foreach (var (field, list) in files) + { + var references = list.Select(file => (JsonNode)new JsonObject + { + [WireValidator.FileMarker] = new JsonObject { ["name"] = file.FileName, ["contentType"] = file.ContentType }, + }).ToList(); + var isList = stage.Inputs.FirstOrDefault(f => f.Name == field)?.Type.Kind == FieldKind.List; + copy[field] = isList ? new JsonArray([.. references]) : references.FirstOrDefault(); + } + + return copy; + } + + private static long TotalBytes(IReadOnlyDictionary> files) => + files.Values.SelectMany(list => list).Sum(file => file.Length ?? 0); + private static List ToCopies(IReadOnlyDictionary> files) => [.. files.SelectMany(group => group.Value.Select((file, index) => new InputFileCopy(group.Key, index, file.LocalPath!, file.FileName, file.ContentType, file.Length ?? 0, file.Sha256!)))]; @@ -505,24 +751,24 @@ private static Dictionary> MoveOutputs(IReadOnlyL _ => RunStatus.Failed, }; - private static StepResult Skipped(PipelineStepDefinition definition, int order) => new() + private static StepResult Skipped(PipelineStepDefinition definition, int order, StepResult? prior) => new() { StepKey = definition.Key, Order = order, StageKey = definition.StageKey, StageVersion = definition.StageVersion, Status = StepStatus.Skipped, - Attempts = [], + Attempts = prior?.Attempts ?? [], }; - private static StepResult InvalidInputs(StageDefinition stage, string stepKey, int order, IReadOnlyList errors) => new() + private static StepResult InvalidInputs(StageDefinition stage, string stepKey, int order, IReadOnlyList errors, StepResult? prior) => new() { StepKey = stepKey, Order = order, StageKey = stage.Key, StageVersion = stage.Version, Status = StepStatus.Failed, - Attempts = [], + Attempts = prior?.Attempts ?? [], Error = new RunError(ErrorKind.InputValidation, string.Join("; ", errors.Select(e => $"'{e.Field}': {e.Message}"))) { Code = errors[0].Code, @@ -543,6 +789,8 @@ private void ForwardOutput(string stageKey, AttemptResult attempt) } } + private sealed record RunTarget(RunKind Kind, string Key, int Version, IReadOnlyList Steps); + private sealed record AttemptOutcome(AttemptResult Attempt, JsonObject? Outputs, IReadOnlyDictionary> Files); private sealed record RunOutcome( @@ -555,13 +803,19 @@ private sealed record RunOutcome( public static RunOutcome Aborted(RunStatus status, RunError error) => new(status, [], null, null, error); } - /// State shared by the steps of one run (folders, settings and the file budget). - private sealed class RunContext(RunWorkspace workspace, PythonScriptRunnerOptions settings) + /// State shared by the steps of one run (folders, settings, caller and the file budget). + private sealed class RunContext(RunWorkspace workspace, PythonScriptRunnerOptions settings, RunKind kind, string triggeredBy) { public RunWorkspace Workspace { get; } = workspace; + public RunId RunId => Workspace.RunId; + public PythonScriptRunnerOptions Settings { get; } = settings; + public RunKind Kind { get; } = kind; + + public string TriggeredBy { get; } = triggeredBy; + public long UsedBytes { get; set; } } } From bdec5254554ebec1644566c0310eda0e75794320 Mon Sep 17 00:00:00 2001 From: net0well Date: Mon, 19 Oct 2026 19:29:50 -0300 Subject: [PATCH 2/4] =?UTF-8?q?feat(ef):=20persist=C3=AAncia=20em=20Postgr?= =?UTF-8?q?eSQL=20(spec=20004)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - PsrDbContext no schema psr com migration inicial: histórico particionado por semana, conteúdo de arquivos por dia e auditoria por mês. - Catálogo no banco: rascunhos com concorrência otimista, publicação com verificação de sintaxe e lock do uv, arquivamento, exclusão, teste de rascunho e importação/exportação em zip. - Histórico de toda execução síncrona, arquivos em bytea com streaming, idempotência de 24 h, outbox transacional e auditoria de ações humanas. - IRunManager: consulta, listagem keyset, cancelamento e reprocesso síncrono. - Manutenção com advisory lock (partições, retenção, limpezas) e health check. Co-Authored-By: Claude Opus 5.5 --- Directory.Packages.props | 5 + PythonScriptRunner.slnx | 2 + .../Catalog/CatalogMapping.cs | 149 ++ .../Catalog/EfCatalog.cs | 106 ++ .../Catalog/EfCatalogAdmin.Pipelines.cs | 530 +++++++ .../Catalog/EfCatalogAdmin.Transfer.cs | 366 +++++ .../Catalog/EfCatalogAdmin.cs | 611 ++++++++ .../Catalog/StoredJson.cs | 106 ++ .../Entities/CatalogEntities.cs | 229 +++ .../Entities/InfrastructureEntities.cs | 67 + .../Entities/RunEntities.cs | 200 +++ .../Infrastructure/AuditLog.cs | 137 ++ .../Infrastructure/Transactional.cs | 134 ++ .../Maintenance/DatabaseHealthCheck.cs | 78 + .../Maintenance/MaintenanceService.cs | 32 + .../Maintenance/PsrMaintenance.cs | 227 +++ .../Migrations/.editorconfig | 4 + .../20261019221743_InitialSchema.Designer.cs | 1347 +++++++++++++++++ .../20261019221743_InitialSchema.cs | 506 +++++++ .../Migrations/DesignTimeFactory.cs | 13 + .../Migrations/PartitioningSql.cs | 265 ++++ .../Migrations/PsrDbContextModelSnapshot.cs | 1344 ++++++++++++++++ .../PsrDbContext.cs | 373 +++++ .../PsrDbContextOptions.cs | 10 + .../PsrEntityFrameworkCoreExtensions.cs | 80 + .../PsrEntityFrameworkOptions.cs | 78 + .../PsrMigrations.cs | 41 + ...honScriptRunner.EntityFrameworkCore.csproj | 23 + .../Runs/ArtifactRefs.cs | 80 + .../Runs/ArtifactStore.cs | 316 ++++ .../Runs/EfRunManager.cs | 264 ++++ .../Runs/EfRunRecorder.cs | 347 +++++ .../Runs/RunLoader.cs | 147 ++ 33 files changed, 8217 insertions(+) create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Catalog/CatalogMapping.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalog.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Pipelines.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Transfer.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Catalog/StoredJson.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Entities/CatalogEntities.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Entities/InfrastructureEntities.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Entities/RunEntities.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/AuditLog.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/Transactional.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Maintenance/DatabaseHealthCheck.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Maintenance/MaintenanceService.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Maintenance/PsrMaintenance.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Migrations/.editorconfig create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Migrations/20261019221743_InitialSchema.Designer.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Migrations/20261019221743_InitialSchema.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Migrations/DesignTimeFactory.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Migrations/PartitioningSql.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Migrations/PsrDbContextModelSnapshot.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/PsrDbContext.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/PsrDbContextOptions.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/PsrEntityFrameworkCoreExtensions.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/PsrEntityFrameworkOptions.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/PsrMigrations.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/PythonScriptRunner.EntityFrameworkCore.csproj create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Runs/ArtifactRefs.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Runs/ArtifactStore.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Runs/EfRunManager.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Runs/EfRunRecorder.cs create mode 100644 src/PythonScriptRunner.EntityFrameworkCore/Runs/RunLoader.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 8f016f7..b891480 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,17 +4,22 @@ + + + + + diff --git a/PythonScriptRunner.slnx b/PythonScriptRunner.slnx index a691e0f..091dc73 100644 --- a/PythonScriptRunner.slnx +++ b/PythonScriptRunner.slnx @@ -2,8 +2,10 @@ + + diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Catalog/CatalogMapping.cs b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/CatalogMapping.cs new file mode 100644 index 0000000..fa4b149 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/CatalogMapping.cs @@ -0,0 +1,149 @@ +using System.Security.Cryptography; +using System.Text; +using PythonScriptRunner.EntityFrameworkCore.Entities; + +namespace PythonScriptRunner.EntityFrameworkCore.Catalog; + +/// A step row with the stage version it references. +internal sealed record StepRow(PipelineStepEntity Step, string StageKey, int StageVersion); + +/// Maps catalog rows to definitions and DTOs, and back. +internal static class CatalogMapping +{ + public static StageDefinition ToStage(StageEntity stage, StageVersionEntity version) => new() + { + Key = stage.Key, + Version = version.Version, + Name = stage.Name, + Description = stage.Description, + Script = ScriptSource.Inline(version.Script), + PythonVersion = version.PythonVersion, + Requirements = version.Requirements, + LockedRequirements = version.LockedRequirements, + Inputs = StoredJson.Fields(version.InputFields), + Outputs = StoredJson.Fields(version.OutputFields), + RequiredSecrets = version.RequiredSecrets, + Timeout = TimeSpan.FromSeconds(version.TimeoutSeconds), + MemoryLimitMb = version.MemoryLimitMb, + Retry = new RetryPolicy(version.RetryMaxAttempts, TimeSpan.FromSeconds(version.RetryInitialDelaySeconds), (double)version.RetryBackoffMultiplier), + Idempotent = version.Idempotent, + }; + + public static StageDraftData ToStageData(StageVersionEntity version) => new() + { + Script = version.Script, + PythonVersion = version.PythonVersion, + Requirements = version.Requirements, + Inputs = StoredJson.Fields(version.InputFields), + Outputs = StoredJson.Fields(version.OutputFields), + RequiredSecrets = version.RequiredSecrets, + Timeout = TimeSpan.FromSeconds(version.TimeoutSeconds), + MemoryLimitMb = version.MemoryLimitMb, + Retry = new RetryPolicy(version.RetryMaxAttempts, TimeSpan.FromSeconds(version.RetryInitialDelaySeconds), (double)version.RetryBackoffMultiplier), + Idempotent = version.Idempotent, + }; + + /// Copies editable content into a version row. + public static void Apply(StageVersionEntity version, StageDraftData data) + { + version.Script = data.Script; + version.ScriptSha256 = Sha256(data.Script); + version.PythonVersion = data.PythonVersion; + version.Requirements = string.IsNullOrWhiteSpace(data.Requirements) ? null : data.Requirements; + version.InputFields = StoredJson.Fields(data.Inputs); + version.OutputFields = StoredJson.Fields(data.Outputs); + version.RequiredSecrets = [.. data.RequiredSecrets]; + version.TimeoutSeconds = (int)Math.Ceiling(data.Timeout.TotalSeconds); + version.MemoryLimitMb = data.MemoryLimitMb; + version.RetryMaxAttempts = data.Retry.MaxAttempts; + version.RetryInitialDelaySeconds = (int)data.Retry.InitialDelay.TotalSeconds; + version.RetryBackoffMultiplier = (decimal)data.Retry.BackoffMultiplier; + version.Idempotent = data.Idempotent; + } + + public static StageDraftData ToStageData(StageDefinition stage, string defaultPythonVersion) => new() + { + Script = stage.Script switch + { + InlineScriptSource inline => inline.Content, + FileScriptSource file => File.ReadAllText(file.Path), + _ => throw new InvalidOperationException("Unknown script source."), + }, + PythonVersion = stage.PythonVersion ?? defaultPythonVersion, + Requirements = stage.Requirements, + Inputs = stage.Inputs, + Outputs = stage.Outputs, + RequiredSecrets = stage.RequiredSecrets, + Timeout = stage.Timeout ?? TimeSpan.FromMinutes(5), + MemoryLimitMb = stage.MemoryLimitMb, + Retry = stage.Retry, + Idempotent = stage.Idempotent, + }; + + public static StageDraft ToStageDraft(StageEntity stage, StageVersionEntity version) => + new(stage.Key, version.Version, ToStageData(version), version.RowVersion, Authorship(version)); + + public static StageVersionDetail ToStageVersionDetail(StageEntity stage, StageVersionEntity version) => new() + { + Key = stage.Key, + Version = version.Version, + Status = version.Status, + Data = ToStageData(version), + ScriptSha256 = version.ScriptSha256, + LockedRequirements = version.LockedRequirements, + EnvironmentHash = version.EnvironmentHash, + PublishedAt = version.PublishedAt, + PublishedBy = version.PublishedBy, + ArchivedAt = version.ArchivedAt, + ArchivedBy = version.ArchivedBy, + RowVersion = version.RowVersion, + Authorship = Authorship(version), + }; + + public static PipelineDefinition ToPipeline(PipelineEntity pipeline, PipelineVersionEntity version, IEnumerable steps) => new() + { + Key = pipeline.Key, + Version = version.Version, + Name = pipeline.Name, + Description = pipeline.Description, + Inputs = StoredJson.Fields(version.InputFields), + Steps = [.. steps.OrderBy(s => s.Step.Order).Select(s => new PipelineStepDefinition(s.Step.Key, s.StageKey, s.StageVersion) + { + DisplayName = s.Step.DisplayName, + Bindings = StoredJson.Bindings(s.Step.Bindings), + })], + Outputs = StoredJson.Outputs(version.Outputs), + }; + + public static PipelineDraftData ToPipelineData(PipelineDefinition pipeline, string? layout) => new() + { + Inputs = pipeline.Inputs, + Steps = pipeline.Steps, + Outputs = pipeline.Outputs, + Layout = StoredJson.Node(layout), + }; + + public static PipelineVersionDetail ToPipelineVersionDetail(PipelineEntity pipeline, PipelineVersionEntity version, IEnumerable steps) => new() + { + Key = pipeline.Key, + Version = version.Version, + Status = version.Status, + Data = ToPipelineData(ToPipeline(pipeline, version, steps), version.Layout), + PublishedAt = version.PublishedAt, + PublishedBy = version.PublishedBy, + ArchivedAt = version.ArchivedAt, + ArchivedBy = version.ArchivedBy, + RowVersion = version.RowVersion, + Authorship = Authorship(version), + }; + + public static Authorship Authorship(StageEntity row) => new(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy); + + public static Authorship Authorship(StageVersionEntity row) => new(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy); + + public static Authorship Authorship(PipelineEntity row) => new(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy); + + public static Authorship Authorship(PipelineVersionEntity row) => new(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy); + + public static string Sha256(string text) => Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text))); +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalog.cs b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalog.cs new file mode 100644 index 0000000..df52f76 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalog.cs @@ -0,0 +1,106 @@ +using System.Collections.Concurrent; +using Microsoft.EntityFrameworkCore; + +namespace PythonScriptRunner.EntityFrameworkCore.Catalog; + +/// +/// Catalog stored in PostgreSQL. Only published versions run; definitions and "latest published" lookups are cached for +/// and invalidated in this process on every change. +/// +internal sealed class EfCatalog(IDbContextFactory factory, PsrEntityFrameworkOptions options, TimeProvider time) : ICatalog +{ + private readonly ConcurrentDictionary<(string Key, int? Version), Cached> _stages = new(); + private readonly ConcurrentDictionary<(string Key, int? Version), Cached> _pipelines = new(); + + public async ValueTask FindStageAsync(string key, int? version, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(key); + if (_stages.TryGetValue((key, version), out var cached) && cached.ExpiresAt > time.GetUtcNow()) + { + return cached.Value; + } + + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var query = db.StageVersions.AsNoTracking().Include(v => v.Stage).Where(v => v.Stage.Key == key); + query = version is { } number + ? query.Where(v => v.Version == number) + : query.Where(v => v.Version == v.Stage.LatestPublishedVersion); + var row = await query.SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false); + if (row is null) + { + return null; + } + + if (row.Status != VersionStatus.Published) + { + throw new VersionNotRunnableException(key, row.Version, row.Status); + } + + var stage = CatalogMapping.ToStage(row.Stage, row); + _stages[(key, version)] = new(stage, time.GetUtcNow() + options.LatestVersionCacheDuration); + return stage; + } + } + + public async ValueTask FindPipelineAsync(string key, int? version, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(key); + if (_pipelines.TryGetValue((key, version), out var cached) && cached.ExpiresAt > time.GetUtcNow()) + { + return cached.Value; + } + + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var query = db.PipelineVersions.AsNoTracking().Include(v => v.Pipeline).Where(v => v.Pipeline.Key == key); + query = version is { } number + ? query.Where(v => v.Version == number) + : query.Where(v => v.Version == v.Pipeline.LatestPublishedVersion); + var row = await query.SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false); + if (row is null) + { + return null; + } + + if (row.Status != VersionStatus.Published) + { + throw new VersionNotRunnableException(key, row.Version, row.Status); + } + + var steps = await LoadStepsAsync(db, row.Id, cancellationToken).ConfigureAwait(false); + var pipeline = CatalogMapping.ToPipeline(row.Pipeline, row, steps); + _pipelines[(key, version)] = new(pipeline, time.GetUtcNow() + options.LatestVersionCacheDuration); + return pipeline; + } + } + + /// Forgets cached definitions of a stage (called after every change in this process). + public void InvalidateStage(string key) + { + foreach (var entry in _stages.Keys.Where(k => k.Key == key)) + { + _stages.TryRemove(entry, out _); + } + } + + /// Forgets cached definitions of a pipeline. + public void InvalidatePipeline(string key) + { + foreach (var entry in _pipelines.Keys.Where(k => k.Key == key)) + { + _pipelines.TryRemove(entry, out _); + } + } + + public static async Task> LoadStepsAsync(PsrDbContext db, Guid pipelineVersionId, CancellationToken cancellationToken) => + await db.PipelineSteps.AsNoTracking() + .Where(s => s.PipelineVersionId == pipelineVersionId) + .OrderBy(s => s.Order) + .Select(s => new StepRow(s, s.StageVersion.Stage.Key, s.StageVersion.Version)) + .ToListAsync(cancellationToken).ConfigureAwait(false); + + private sealed record Cached(T Value, DateTimeOffset ExpiresAt); +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Pipelines.cs b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Pipelines.cs new file mode 100644 index 0000000..d0da7af --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Pipelines.cs @@ -0,0 +1,530 @@ +using System.Text.Json.Nodes; +using Microsoft.EntityFrameworkCore; +using PythonScriptRunner.EntityFrameworkCore.Entities; +using PythonScriptRunner.EntityFrameworkCore.Infrastructure; + +namespace PythonScriptRunner.EntityFrameworkCore.Catalog; + +internal sealed partial class EfCatalogAdmin +{ + public async Task> ListPipelinesAsync(CatalogQuery query, CancellationToken cancellationToken = default) + { + var (pageNumber, pageSize) = Paging(query); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var pipelines = db.Pipelines.AsNoTracking(); + if (!string.IsNullOrWhiteSpace(query.Search)) + { + var pattern = $"%{query.Search.Trim()}%"; + pipelines = pipelines.Where(p => EF.Functions.ILike(p.Key, pattern) || EF.Functions.ILike(p.Name, pattern)); + } + + var total = await pipelines.CountAsync(cancellationToken).ConfigureAwait(false); + var items = await pipelines.OrderBy(p => p.Key) + .Skip((pageNumber - 1) * pageSize).Take(pageSize) + .Select(p => new PipelineSummary( + p.Key, p.Name, p.Description, p.LatestPublishedVersion, + p.Versions.Where(v => v.Status == VersionStatus.Draft).Select(v => (int?)v.Version).FirstOrDefault(), + p.UpdatedAt)) + .ToListAsync(cancellationToken).ConfigureAwait(false); + return new Page(items, pageNumber, pageSize, total); + } + } + + public async Task GetPipelineAsync(string key, CancellationToken cancellationToken = default) + { + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var pipeline = await db.Pipelines.AsNoTracking().Include(p => p.Versions).SingleOrDefaultAsync(p => p.Key == key, cancellationToken).ConfigureAwait(false); + return pipeline is null ? null : ToPipelineDetail(pipeline); + } + } + + public async Task GetPipelineVersionAsync(string key, int version, CancellationToken cancellationToken = default) + { + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await db.PipelineVersions.AsNoTracking().Include(v => v.Pipeline) + .SingleOrDefaultAsync(v => v.Pipeline.Key == key && v.Version == version, cancellationToken).ConfigureAwait(false); + return row is null + ? null + : CatalogMapping.ToPipelineVersionDetail(row.Pipeline, row, await EfCatalog.LoadStepsAsync(db, row.Id, cancellationToken).ConfigureAwait(false)); + } + } + + public async Task GetPipelineDraftAsync(string key, CancellationToken cancellationToken = default) + { + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await db.PipelineVersions.AsNoTracking().Include(v => v.Pipeline) + .SingleOrDefaultAsync(v => v.Pipeline.Key == key && v.Status == VersionStatus.Draft, cancellationToken).ConfigureAwait(false); + return row is null ? null : await ToPipelineDraftAsync(db, row, cancellationToken).ConfigureAwait(false); + } + } + + public async Task CreatePipelineAsync(NewCatalogItem pipeline, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(pipeline); + ArgumentNullException.ThrowIfNull(actor); + ValidateItem(pipeline); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + if (await db.Pipelines.AnyAsync(p => p.Key == pipeline.Key, cancellationToken).ConfigureAwait(false)) + { + throw new PsrDomainException(PsrErrorCodes.PipelineKeyTaken, $"Pipeline '{pipeline.Key}' already exists."); + } + + var now = time.GetUtcNow(); + var row = new PipelineEntity + { + Id = Guid.CreateVersion7(now), + Key = pipeline.Key, + Name = pipeline.Name, + Description = pipeline.Description, + CreatedAt = now, + CreatedBy = actor.Subject, + UpdatedAt = now, + UpdatedBy = actor.Subject, + }; + var draft = NewPipelineVersion(row, 1, actor, now); + row.Versions.Add(draft); + db.Pipelines.Add(row); + AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelineCreated, "Pipeline", row.Id, row.Key), now); + await SaveAsync(db, cancellationToken, unique: _ => new PsrDomainException(PsrErrorCodes.PipelineKeyTaken, $"Pipeline '{pipeline.Key}' already exists.")) + .ConfigureAwait(false); + return new PipelineDraft(row.Key, 1, new PipelineDraftData(), draft.RowVersion, CatalogMapping.Authorship(draft)); + } + } + + public async Task UpdatePipelineAsync( + string key, CatalogItemChanges changes, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(changes); + return await ChangePipelineAsync(key, expectedRowVersion, actor, AuditActions.PipelineUpdated, (pipeline, details) => + { + if (changes.Name is { } name && name != pipeline.Name) + { + ValidateName(name); + details["name"] = new JsonArray(pipeline.Name, name); + pipeline.Name = name; + } + + if (changes.Description is { } description && description != pipeline.Description) + { + details["description"] = new JsonArray(pipeline.Description, description); + pipeline.Description = description; + } + }, cancellationToken).ConfigureAwait(false); + } + + public async Task CreatePipelineDraftFromAsync(string key, int fromVersion, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var pipeline = await db.Pipelines.Include(p => p.Versions).ThenInclude(v => v.Steps) + .SingleOrDefaultAsync(p => p.Key == key, cancellationToken).ConfigureAwait(false) + ?? throw new PipelineNotFoundException(key); + EnsureNoDraft(pipeline.Versions.Any(v => v.Status == VersionStatus.Draft), key); + var source = pipeline.Versions.SingleOrDefault(v => v.Version == fromVersion) ?? throw new VersionNotFoundException(key, fromVersion); + var now = time.GetUtcNow(); + var draft = NewPipelineVersion(pipeline, pipeline.Versions.Max(v => v.Version) + 1, actor, now); + draft.InputFields = source.InputFields; + draft.Outputs = source.Outputs; + draft.Layout = source.Layout; + draft.Steps.AddRange(source.Steps.Select(s => new PipelineStepEntity + { + Id = Guid.CreateVersion7(now), + PipelineVersionId = draft.Id, + Order = s.Order, + Key = s.Key, + DisplayName = s.DisplayName, + StageVersionId = s.StageVersionId, + Bindings = s.Bindings, + })); + pipeline.Versions.Add(draft); + AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelineDraftCreated, "PipelineVersion", draft.Id, $"{key}@{draft.Version}", + new JsonObject { ["fromVersion"] = fromVersion }), now); + await SaveAsync(db, cancellationToken, unique: _ => DraftExists(key)).ConfigureAwait(false); + return await ToPipelineDraftAsync(db, draft, cancellationToken).ConfigureAwait(false); + } + } + + public async Task SavePipelineDraftAsync( + string key, int version, PipelineDraftData data, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(data); + ArgumentNullException.ThrowIfNull(actor); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await LoadPipelineVersionAsync(db, key, version, cancellationToken).ConfigureAwait(false); + EnsureDraft(row.Status, key, version); + if (row.RowVersion != expectedRowVersion) + { + throw new ConcurrencyConflictException($"The draft {key}@{version} was changed by someone else.", + await ToPipelineDraftAsync(db, row, cancellationToken).ConfigureAwait(false)); + } + + var steps = await BuildStepsAsync(db, row.Id, data.Steps, cancellationToken).ConfigureAwait(false); + var now = time.GetUtcNow(); + var transaction = await db.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + await using var rollback = transaction.ConfigureAwait(false); + await db.PipelineSteps.Where(s => s.PipelineVersionId == row.Id).ExecuteDeleteAsync(cancellationToken).ConfigureAwait(false); + db.PipelineSteps.AddRange(steps); + row.InputFields = StoredJson.Fields(data.Inputs); + row.Outputs = StoredJson.Outputs(data.Outputs); + row.Layout = StoredJson.Node(data.Layout); + Touch(row, actor, now); + db.Entry(row).Property(v => v.RowVersion).OriginalValue = expectedRowVersion; + AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelineDraftSaved, "PipelineVersion", row.Id, $"{key}@{version}"), now); + await SaveAsync(db, cancellationToken, conflict: Current(() => GetPipelineDraftAsync(key, CancellationToken.None))).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return await ToPipelineDraftAsync(db, row, cancellationToken).ConfigureAwait(false); + } + } + + public async Task DiscardPipelineDraftAsync(string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await LoadPipelineVersionAsync(db, key, version, cancellationToken).ConfigureAwait(false); + EnsureDraft(row.Status, key, version); + if (row.RowVersion != expectedRowVersion) + { + throw new ConcurrencyConflictException($"The draft {key}@{version} was changed by someone else.", + await ToPipelineDraftAsync(db, row, cancellationToken).ConfigureAwait(false)); + } + + var onlyVersion = !await db.PipelineVersions.AnyAsync(v => v.PipelineId == row.PipelineId && v.Id != row.Id, cancellationToken).ConfigureAwait(false); + db.Entry(row).Property(v => v.RowVersion).OriginalValue = expectedRowVersion; + db.PipelineVersions.Remove(row); + if (onlyVersion && !await db.Runs.AnyAsync(r => r.PipelineId == row.PipelineId, cancellationToken).ConfigureAwait(false)) + { + // A pipeline without versions cannot get a new draft; discarding its first draft removes it. + db.Pipelines.Remove(row.Pipeline); + } + + AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelineDraftDiscarded, "PipelineVersion", row.Id, $"{key}@{version}"), time.GetUtcNow()); + await SaveAsync(db, cancellationToken, conflict: Current(() => GetPipelineDraftAsync(key, CancellationToken.None))).ConfigureAwait(false); + catalog.InvalidatePipeline(key); + } + } + + public async Task PublishPipelineAsync( + string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await LoadPipelineVersionAsync(db, key, version, cancellationToken).ConfigureAwait(false); + EnsureDraft(row.Status, key, version); + if (row.RowVersion != expectedRowVersion) + { + throw new ConcurrencyConflictException($"The draft {key}@{version} was changed by someone else.", + await ToPipelineDraftAsync(db, row, cancellationToken).ConfigureAwait(false)); + } + + var steps = await EfCatalog.LoadStepsAsync(db, row.Id, cancellationToken).ConfigureAwait(false); + var report = await pipelineValidator.ValidateAsync(CatalogMapping.ToPipeline(row.Pipeline, row, steps), cancellationToken).ConfigureAwait(false); + if (!report.IsValid) + { + throw new DefinitionInvalidException(PsrErrorCodes.PipelineInvalid, report); + } + + var now = time.GetUtcNow(); + row.Status = VersionStatus.Published; + row.PublishedAt = now; + row.PublishedBy = actor.Subject; + Touch(row, actor, now); + row.Pipeline.LatestPublishedVersion = Math.Max(row.Pipeline.LatestPublishedVersion ?? 0, version); + Touch(row.Pipeline, actor, now); + db.Entry(row).Property(v => v.RowVersion).OriginalValue = expectedRowVersion; + AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelinePublished, "PipelineVersion", row.Id, $"{key}@{version}"), now); + await SaveAsync(db, cancellationToken, conflict: Current(() => GetPipelineDraftAsync(key, CancellationToken.None))).ConfigureAwait(false); + catalog.InvalidatePipeline(key); + return new PipelinePublication(CatalogMapping.ToPipelineVersionDetail(row.Pipeline, row, steps), report.Warnings); + } + } + + public async Task ArchivePipelineVersionAsync(string key, int version, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await LoadPipelineVersionAsync(db, key, version, cancellationToken).ConfigureAwait(false); + var steps = await EfCatalog.LoadStepsAsync(db, row.Id, cancellationToken).ConfigureAwait(false); + if (row.Status == VersionStatus.Archived) + { + return CatalogMapping.ToPipelineVersionDetail(row.Pipeline, row, steps); + } + + if (row.Status != VersionStatus.Published) + { + throw new PsrDomainException(PsrErrorCodes.VersionImmutable, $"Only published versions can be archived; discard the draft {key}@{version} instead."); + } + + var schedules = await db.Schedules.AsNoTracking() + .Where(s => s.PipelineId == row.PipelineId && s.IsEnabled && s.PinnedVersion == version) + .Select(s => s.Name) + .ToListAsync(cancellationToken).ConfigureAwait(false); + if (schedules.Count > 0) + { + throw new PsrDomainException(PsrErrorCodes.VersionScheduled, + $"Version {key}@{version} is pinned by enabled schedules: {string.Join(", ", schedules)}."); + } + + var now = time.GetUtcNow(); + row.Status = VersionStatus.Archived; + row.ArchivedAt = now; + row.ArchivedBy = actor.Subject; + Touch(row, actor, now); + if (row.Pipeline.LatestPublishedVersion == version) + { + row.Pipeline.LatestPublishedVersion = await db.PipelineVersions + .Where(v => v.PipelineId == row.PipelineId && v.Status == VersionStatus.Published && v.Id != row.Id) + .MaxAsync(v => (int?)v.Version, cancellationToken).ConfigureAwait(false); + Touch(row.Pipeline, actor, now); + } + + AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelineVersionArchived, "PipelineVersion", row.Id, $"{key}@{version}"), now); + await SaveAsync(db, cancellationToken).ConfigureAwait(false); + catalog.InvalidatePipeline(key); + return CatalogMapping.ToPipelineVersionDetail(row.Pipeline, row, steps); + } + } + + public async Task DeletePipelineAsync(string key, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var pipeline = await db.Pipelines.SingleOrDefaultAsync(p => p.Key == key, cancellationToken).ConfigureAwait(false) + ?? throw new PipelineNotFoundException(key); + if (await db.Runs.AnyAsync(r => r.PipelineId == pipeline.Id, cancellationToken).ConfigureAwait(false)) + { + throw new PsrDomainException(PsrErrorCodes.DeleteNotAllowed, $"Pipeline '{key}' has runs in the retained history."); + } + + db.Pipelines.Remove(pipeline); + AuditLog.Add(db, new AuditEntry(actor, AuditActions.PipelineDeleted, "Pipeline", pipeline.Id, key), time.GetUtcNow()); + await SaveAsync(db, cancellationToken).ConfigureAwait(false); + catalog.InvalidatePipeline(key); + } + } + + public async Task UpdatePipelineSettingsAsync( + string key, PipelineSettings settings, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(settings); + if (settings.MaxConcurrentRuns is < 1) + { + throw new ArgumentOutOfRangeException(nameof(settings), "MaxConcurrentRuns must be at least 1 (or null for no limit)."); + } + + return await ChangePipelineAsync(key, expectedRowVersion, actor, AuditActions.PipelineSettingsChanged, (pipeline, details) => + { + if (pipeline.MaxConcurrentRuns != settings.MaxConcurrentRuns) + { + details["maxConcurrentRuns"] = new JsonArray(pipeline.MaxConcurrentRuns, settings.MaxConcurrentRuns); + pipeline.MaxConcurrentRuns = settings.MaxConcurrentRuns; + } + }, cancellationToken).ConfigureAwait(false); + } + + public async Task UpdatePipelineRetentionAsync( + string key, PipelineRetention retention, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(retention); + CheckRetention(retention.ArtifactRetentionDays, options.Retention.Artifacts, "artifacts"); + CheckRetention(retention.RunRetentionDays, options.Retention.Runs, "runs"); + return await ChangePipelineAsync(key, expectedRowVersion, actor, AuditActions.PipelineSettingsChanged, (pipeline, details) => + { + if (pipeline.ArtifactRetentionDays != retention.ArtifactRetentionDays) + { + details["artifactRetentionDays"] = new JsonArray(pipeline.ArtifactRetentionDays, retention.ArtifactRetentionDays); + pipeline.ArtifactRetentionDays = retention.ArtifactRetentionDays; + } + + if (pipeline.RunRetentionDays != retention.RunRetentionDays) + { + details["runRetentionDays"] = new JsonArray(pipeline.RunRetentionDays, retention.RunRetentionDays); + pipeline.RunRetentionDays = retention.RunRetentionDays; + } + }, cancellationToken).ConfigureAwait(false); + } + + public async Task TestPipelineDraftAsync(string key, object? inputs, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + PipelineDefinition pipeline; + Guid versionId; + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await db.PipelineVersions.AsNoTracking().Include(v => v.Pipeline) + .SingleOrDefaultAsync(v => v.Pipeline.Key == key && v.Status == VersionStatus.Draft, cancellationToken).ConfigureAwait(false) + ?? throw (await db.Pipelines.AnyAsync(p => p.Key == key, cancellationToken).ConfigureAwait(false) + ? new PsrDomainException(PsrErrorCodes.VersionImmutable, $"Pipeline '{key}' has no draft; create one from a version first.") + : new PipelineNotFoundException(key)); + pipeline = CatalogMapping.ToPipeline(row.Pipeline, row, await EfCatalog.LoadStepsAsync(db, row.Id, cancellationToken).ConfigureAwait(false)); + versionId = row.Id; + } + + var report = await pipelineValidator.ValidateAsync(pipeline, cancellationToken).ConfigureAwait(false); + if (!report.IsValid) + { + throw new DefinitionInvalidException(PsrErrorCodes.PipelineInvalid, report); + } + + await WriteAuditAsync(new AuditEntry(actor, AuditActions.PipelineTested, "PipelineVersion", versionId, $"{key}@{pipeline.Version}"), cancellationToken) + .ConfigureAwait(false); + return await engine.RunPipelineAsync(pipeline, RunKind.PipelineTest, inputs, RunOptionsFor(actor), cancellationToken).ConfigureAwait(false); + } + + private async Task ChangePipelineAsync( + string key, uint expectedRowVersion, Actor actor, string action, Action change, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(actor); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var pipeline = await db.Pipelines.Include(p => p.Versions).SingleOrDefaultAsync(p => p.Key == key, cancellationToken).ConfigureAwait(false) + ?? throw new PipelineNotFoundException(key); + if (pipeline.RowVersion != expectedRowVersion) + { + throw new ConcurrencyConflictException($"Pipeline '{key}' was changed by someone else.", ToPipelineDetail(pipeline)); + } + + var details = new JsonObject(); + change(pipeline, details); + var now = time.GetUtcNow(); + Touch(pipeline, actor, now); + db.Entry(pipeline).Property(p => p.RowVersion).OriginalValue = expectedRowVersion; + AuditLog.Add(db, new AuditEntry(actor, action, "Pipeline", pipeline.Id, key, details), now); + await SaveAsync(db, cancellationToken, conflict: Current(() => GetPipelineAsync(key, CancellationToken.None))).ConfigureAwait(false); + catalog.InvalidatePipeline(key); + return ToPipelineDetail(pipeline); + } + } + + /// Builds the step rows of a draft; every step must reference an existing stage version. + private static async Task> BuildStepsAsync( + PsrDbContext db, Guid pipelineVersionId, IReadOnlyList steps, CancellationToken cancellationToken) + { + var issues = new List(); + foreach (var duplicate in steps.GroupBy(s => s.Key, StringComparer.Ordinal).Where(g => g.Count() > 1)) + { + issues.Add(new ValidationIssue("PIPE_STEP_KEY", $"step key '{duplicate.Key}' is used more than once") { Step = duplicate.Key }); + } + + var keys = steps.Select(s => s.StageKey).Distinct().ToList(); + var versions = await db.StageVersions.AsNoTracking() + .Where(v => keys.Contains(v.Stage.Key)) + .Select(v => new { v.Id, v.Stage.Key, v.Version }) + .ToListAsync(cancellationToken).ConfigureAwait(false); + var result = new List(); + for (var i = 0; i < steps.Count; i++) + { + var step = steps[i]; + if (step.Key.Length is 0 or > 64 || !KeyPattern().IsMatch(step.Key)) + { + issues.Add(new ValidationIssue("PIPE_STEP_KEY", $"step key '{step.Key}' must be a slug of up to 64 chars") { Step = step.Key }); + } + + var stageVersion = versions.FirstOrDefault(v => v.Key == step.StageKey && v.Version == step.StageVersion); + if (stageVersion is null) + { + issues.Add(new ValidationIssue("PIPE_STAGE_NOT_PUBLISHED", $"stage '{step.StageKey}@{step.StageVersion}' does not exist") { Step = step.Key }); + continue; + } + + result.Add(new PipelineStepEntity + { + Id = Guid.CreateVersion7(), + PipelineVersionId = pipelineVersionId, + Order = i + 1, + Key = step.Key, + DisplayName = step.DisplayName, + StageVersionId = stageVersion.Id, + Bindings = StoredJson.Bindings(step.Bindings), + }); + } + + return issues.Count > 0 ? throw new DefinitionInvalidException(PsrErrorCodes.PipelineInvalid, new ValidationReport(issues, [])) : result; + } + + private static async Task LoadPipelineVersionAsync(PsrDbContext db, string key, int version, CancellationToken cancellationToken) + { + var row = await db.PipelineVersions.Include(v => v.Pipeline) + .SingleOrDefaultAsync(v => v.Pipeline.Key == key && v.Version == version, cancellationToken).ConfigureAwait(false); + if (row is not null) + { + return row; + } + + return await db.Pipelines.AnyAsync(p => p.Key == key, cancellationToken).ConfigureAwait(false) + ? throw new VersionNotFoundException(key, version) + : throw new PipelineNotFoundException(key); + } + + private static async Task ToPipelineDraftAsync(PsrDbContext db, PipelineVersionEntity row, CancellationToken cancellationToken) + { + var steps = await EfCatalog.LoadStepsAsync(db, row.Id, cancellationToken).ConfigureAwait(false); + var definition = CatalogMapping.ToPipeline(row.Pipeline, row, steps); + return new PipelineDraft(row.Pipeline.Key, row.Version, CatalogMapping.ToPipelineData(definition, row.Layout), row.RowVersion, CatalogMapping.Authorship(row)); + } + + private static PipelineDetail ToPipelineDetail(PipelineEntity pipeline) => new( + pipeline.Key, + pipeline.Name, + pipeline.Description, + pipeline.LatestPublishedVersion, + [.. pipeline.Versions.OrderByDescending(v => v.Version).Select(v => new VersionSummary(v.Version, v.Status, v.PublishedAt, v.UpdatedAt))], + new PipelineSettings(pipeline.MaxConcurrentRuns), + new PipelineRetention(pipeline.ArtifactRetentionDays, pipeline.RunRetentionDays), + pipeline.RowVersion, + CatalogMapping.Authorship(pipeline)); + + private static PipelineVersionEntity NewPipelineVersion(PipelineEntity pipeline, int version, Actor actor, DateTimeOffset now) => new() + { + Id = Guid.CreateVersion7(now), + PipelineId = pipeline.Id, + Pipeline = pipeline, + Version = version, + Status = VersionStatus.Draft, + InputFields = "[]", + Outputs = "[]", + CreatedAt = now, + CreatedBy = actor.Subject, + UpdatedAt = now, + UpdatedBy = actor.Subject, + }; + + private static void CheckRetention(int? days, TimeSpan global, string what) + { + if (days is < 1) + { + throw new ArgumentOutOfRangeException(nameof(days), $"The {what} retention must be at least 1 day."); + } + + if (days is { } value && TimeSpan.FromDays(value) > global) + { + throw new PsrDomainException(PsrErrorCodes.RetentionExceedsGlobal, + $"The {what} retention ({value} days) exceeds the global retention ({global.TotalDays:0} days); pipelines can only shorten it."); + } + } + + private static Func> Current(Func> load) => async () => await load().ConfigureAwait(false); +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Transfer.cs b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Transfer.cs new file mode 100644 index 0000000..861ee6f --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.Transfer.cs @@ -0,0 +1,366 @@ +using System.IO.Compression; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.EntityFrameworkCore; +using PythonScriptRunner.EntityFrameworkCore.Entities; +using PythonScriptRunner.EntityFrameworkCore.Infrastructure; + +namespace PythonScriptRunner.EntityFrameworkCore.Catalog; + +internal sealed partial class EfCatalogAdmin +{ + private const long MaxImportBytes = 50L * 1024 * 1024; + private static readonly JsonSerializerOptions Indented = new() { WriteIndented = true }; + + public async Task ExportStageVersionAsync(string key, int version, Stream destination, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(destination); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await LoadStageVersionAsync(db, key, version, cancellationToken).ConfigureAwait(false); + await WriteZipAsync(destination, zip => AddStage(zip, CatalogMapping.ToStage(row.Stage, row)), cancellationToken).ConfigureAwait(false); + } + } + + public async Task ExportPipelineVersionAsync(string key, int version, Stream destination, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(destination); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await LoadPipelineVersionAsync(db, key, version, cancellationToken).ConfigureAwait(false); + var steps = await EfCatalog.LoadStepsAsync(db, row.Id, cancellationToken).ConfigureAwait(false); + var pipeline = CatalogMapping.ToPipeline(row.Pipeline, row, steps); + var stageIds = steps.Select(s => s.Step.StageVersionId).Distinct().ToList(); + var stages = await db.StageVersions.AsNoTracking().Include(v => v.Stage).Where(v => stageIds.Contains(v.Id)) + .ToListAsync(cancellationToken).ConfigureAwait(false); + await WriteZipAsync(destination, zip => + { + AddText(zip, $"pipelines/{pipeline.Key}{CatalogManifests.PipelineSuffix}", CatalogManifests.WritePipeline(pipeline).ToJsonString(Indented)); + foreach (var stage in stages.OrderBy(s => s.Stage.Key).ThenBy(s => s.Version)) + { + AddStage(zip, CatalogMapping.ToStage(stage.Stage, stage)); + } + }, cancellationToken).ConfigureAwait(false); + } + } + + public async Task ImportAsync(Stream zip, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(zip); + ArgumentNullException.ThrowIfNull(actor); + var buffer = new MemoryStream(); + await using (buffer.ConfigureAwait(false)) + { + await zip.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + if (buffer.Length > MaxImportBytes) + { + throw Invalid([new ValidationIssue("IMPORT_TOO_LARGE", "the archive is larger than 50 MB")]); + } + + buffer.Position = 0; + var (stages, pipelines, issues) = ReadArchive(buffer); + return await ImportAsync(stages, pipelines, issues, actor, cancellationToken).ConfigureAwait(false); + } + } + + private async Task ImportAsync( + List stages, List pipelines, List issues, Actor actor, CancellationToken cancellationToken) + { + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var now = time.GetUtcNow(); + var items = new List(); + var created = new Dictionary<(string Key, int Version), StageVersionEntity>(); + var stageKeys = stages.Select(s => s.Key).ToList(); + var existingStages = await db.Stages.Include(s => s.Versions).Where(s => stageKeys.Contains(s.Key)) + .ToDictionaryAsync(s => s.Key, cancellationToken).ConfigureAwait(false); + foreach (var definition in stages) + { + var stage = existingStages.GetValueOrDefault(definition.Key); + if (stage is not null && stage.Versions.Any(v => v.Status == VersionStatus.Draft)) + { + issues.Add(new ValidationIssue(PsrErrorCodes.DraftAlreadyExists, $"stage '{definition.Key}' already has a draft") { Field = definition.Key }); + continue; + } + + stage ??= NewImportedStage(db, definition, actor, now); + var draft = NewStageVersion(stage, stage.Versions.Count == 0 ? 1 : stage.Versions.Max(v => v.Version) + 1, actor, now); + CatalogMapping.Apply(draft, CatalogMapping.ToStageData(definition, runnerOptions.Value.DefaultPythonVersion)); + stage.Versions.Add(draft); + created[(definition.Key, definition.Version)] = draft; + items.Add(new ImportedItem("Stage", definition.Key, draft.Version, !existingStages.ContainsKey(definition.Key))); + } + + var pipelineKeys = pipelines.Select(p => p.Key).ToList(); + var existingPipelines = await db.Pipelines.Include(p => p.Versions).Where(p => pipelineKeys.Contains(p.Key)) + .ToDictionaryAsync(p => p.Key, cancellationToken).ConfigureAwait(false); + var references = pipelines.SelectMany(p => p.Steps).Select(s => s.StageKey).Distinct().ToList(); + var dbVersions = await db.StageVersions.AsNoTracking().Where(v => references.Contains(v.Stage.Key)) + .Select(v => new { v.Id, v.Stage.Key, v.Version }) + .ToListAsync(cancellationToken).ConfigureAwait(false); + foreach (var definition in pipelines) + { + var pipeline = existingPipelines.GetValueOrDefault(definition.Key); + if (pipeline is not null && pipeline.Versions.Any(v => v.Status == VersionStatus.Draft)) + { + issues.Add(new ValidationIssue(PsrErrorCodes.DraftAlreadyExists, $"pipeline '{definition.Key}' already has a draft") { Field = definition.Key }); + continue; + } + + pipeline ??= NewImportedPipeline(db, definition, actor, now); + var draft = NewPipelineVersion(pipeline, pipeline.Versions.Count == 0 ? 1 : pipeline.Versions.Max(v => v.Version) + 1, actor, now); + draft.InputFields = StoredJson.Fields(definition.Inputs); + draft.Outputs = StoredJson.Outputs(definition.Outputs); + for (var i = 0; i < definition.Steps.Count; i++) + { + var step = definition.Steps[i]; + var entity = new PipelineStepEntity + { + Id = Guid.CreateVersion7(now), + PipelineVersionId = draft.Id, + Order = i + 1, + Key = step.Key, + DisplayName = step.DisplayName, + Bindings = StoredJson.Bindings(step.Bindings), + }; + + // A stage imported in the same archive may get another version number here: follow it. + if (created.TryGetValue((step.StageKey, step.StageVersion), out var imported)) + { + entity.StageVersion = imported; + } + else if (dbVersions.FirstOrDefault(v => v.Key == step.StageKey && v.Version == step.StageVersion) is { } existing) + { + entity.StageVersionId = existing.Id; + } + else + { + issues.Add(new ValidationIssue("PIPE_STAGE_NOT_PUBLISHED", $"stage '{step.StageKey}@{step.StageVersion}' is not in the archive nor in the catalog") + { + Step = step.Key, + Field = definition.Key, + }); + } + + draft.Steps.Add(entity); + } + + pipeline.Versions.Add(draft); + items.Add(new ImportedItem("Pipeline", definition.Key, draft.Version, !existingPipelines.ContainsKey(definition.Key))); + } + + if (issues.Count > 0) + { + throw Invalid(issues); + } + + AuditLog.Add(db, new AuditEntry(actor, AuditActions.CatalogImported, "Settings", null, null, new JsonObject + { + ["items"] = new JsonArray([.. items.Select(i => (JsonNode)JsonValue.Create($"{i.Kind}:{i.Key}@{i.Version}"))]), + }), now); + await SaveAsync(db, cancellationToken, unique: _ => Invalid([new ValidationIssue(PsrErrorCodes.DraftAlreadyExists, "a draft was created concurrently")])) + .ConfigureAwait(false); + foreach (var item in items) + { + if (item.Kind == "Stage") + { + catalog.InvalidateStage(item.Key); + } + else + { + catalog.InvalidatePipeline(item.Key); + } + } + + return new ImportReport(items); + } + } + + private (List Stages, List Pipelines, List Issues) ReadArchive(Stream buffer) + { + var stages = new List(); + var pipelines = new List(); + var issues = new List(); + ZipArchive archive; + try + { + archive = new ZipArchive(buffer, ZipArchiveMode.Read, leaveOpen: true); + } + catch (InvalidDataException ex) + { + throw Invalid([new ValidationIssue("IMPORT_NOT_ZIP", ex.Message)]); + } + + using (archive) + { + var entries = archive.Entries.Where(e => !string.IsNullOrEmpty(e.Name)) + .ToDictionary(e => e.FullName.Replace('\\', '/'), StringComparer.Ordinal); + foreach (var (path, entry) in entries.Where(e => IsStageManifest(e.Key)).OrderBy(e => e.Key, StringComparer.Ordinal)) + { + var folder = path[..path.LastIndexOf('/')]; + try + { + var manifest = Read(entry); + var json = JsonNode.Parse(manifest, documentOptions: new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true }); + var scriptPath = $"{folder}/{json?["script"]?.GetValue() ?? CatalogManifests.ScriptName}"; + var requirementsName = json?["requirements"]?.GetValue(); + var requirementsPath = $"{folder}/{requirementsName ?? CatalogManifests.RequirementsName}"; + if (!entries.TryGetValue(scriptPath, out var script)) + { + issues.Add(new ValidationIssue("IMPORT_SCRIPT_MISSING", $"script not found: {scriptPath}") { Field = path }); + continue; + } + + string? requirements = entries.TryGetValue(requirementsPath, out var requirementsEntry) ? Read(requirementsEntry) : null; + if (requirements is null && requirementsName is not null) + { + issues.Add(new ValidationIssue("IMPORT_REQUIREMENTS_MISSING", $"requirements not found: {requirementsPath}") { Field = path }); + continue; + } + + var stage = CatalogManifests.ReadStage(manifest, ScriptSource.Inline(Read(script)), requirements, null); + var report = stageValidator.Validate(stage); + issues.AddRange(report.Errors.Select(e => e with { Field = path })); + if (stages.Any(s => s.Key == stage.Key && s.Version == stage.Version)) + { + issues.Add(new ValidationIssue("IMPORT_DUPLICATE", $"stage '{stage.Key}@{stage.Version}' appears twice") { Field = path }); + } + + stages.Add(stage); + } + catch (Exception ex) when (ex is PsrConfigurationException or JsonException or ArgumentException or InvalidDataException) + { + issues.Add(new ValidationIssue("IMPORT_MANIFEST_INVALID", ex.Message) { Field = path }); + } + } + + foreach (var (path, entry) in entries.Where(e => IsPipelineManifest(e.Key)).OrderBy(e => e.Key, StringComparer.Ordinal)) + { + try + { + var pipeline = CatalogManifests.ReadPipeline(Read(entry)); + if (pipelines.Any(p => p.Key == pipeline.Key)) + { + issues.Add(new ValidationIssue("IMPORT_DUPLICATE", $"pipeline '{pipeline.Key}' appears twice") { Field = path }); + } + + pipelines.Add(pipeline); + } + catch (Exception ex) when (ex is PsrConfigurationException or JsonException or ArgumentException or InvalidDataException) + { + issues.Add(new ValidationIssue("IMPORT_MANIFEST_INVALID", ex.Message) { Field = path }); + } + } + } + + if (stages.Count == 0 && pipelines.Count == 0 && issues.Count == 0) + { + issues.Add(new ValidationIssue("IMPORT_EMPTY", "the archive has no stages/*/stage.json nor pipelines/*.pipeline.json")); + } + + if (stages.GroupBy(s => s.Key).FirstOrDefault(g => g.Count() > 1) is { } repeated) + { + issues.Add(new ValidationIssue("IMPORT_DUPLICATE", $"stage '{repeated.Key}' appears in more than one version; import one version per archive")); + } + + return (stages, pipelines, issues); + } + + private static StageEntity NewImportedStage(PsrDbContext db, StageDefinition definition, Actor actor, DateTimeOffset now) + { + var stage = new StageEntity + { + Id = Guid.CreateVersion7(now), + Key = definition.Key, + Name = definition.Name, + Description = definition.Description, + CreatedAt = now, + CreatedBy = actor.Subject, + UpdatedAt = now, + UpdatedBy = actor.Subject, + }; + db.Stages.Add(stage); + return stage; + } + + private static PipelineEntity NewImportedPipeline(PsrDbContext db, PipelineDefinition definition, Actor actor, DateTimeOffset now) + { + var pipeline = new PipelineEntity + { + Id = Guid.CreateVersion7(now), + Key = definition.Key, + Name = definition.Name, + Description = definition.Description, + CreatedAt = now, + CreatedBy = actor.Subject, + UpdatedAt = now, + UpdatedBy = actor.Subject, + }; + db.Pipelines.Add(pipeline); + return pipeline; + } + + private static bool IsStageManifest(string path) + { + var parts = path.Split('/'); + return parts.Length == 3 && parts[0] == "stages" && parts[2] == CatalogManifests.StageManifestName; + } + + private static bool IsPipelineManifest(string path) + { + var parts = path.Split('/'); + return parts.Length == 2 && parts[0] == "pipelines" && parts[1].EndsWith(CatalogManifests.PipelineSuffix, StringComparison.Ordinal); + } + + private static string Read(ZipArchiveEntry entry) + { + using var reader = new StreamReader(entry.Open(), Encoding.UTF8); + return reader.ReadToEnd(); + } + + private static void AddStage(ZipArchive zip, StageDefinition stage) + { + var folder = $"stages/{stage.Key}@{stage.Version}"; + AddText(zip, $"{folder}/{CatalogManifests.StageManifestName}", CatalogManifests.WriteStage(stage).ToJsonString(Indented)); + AddText(zip, $"{folder}/{CatalogManifests.ScriptName}", ((InlineScriptSource)stage.Script).Content); + if (!string.IsNullOrWhiteSpace(stage.Requirements)) + { + AddText(zip, $"{folder}/{CatalogManifests.RequirementsName}", stage.Requirements); + } + + if (!string.IsNullOrWhiteSpace(stage.LockedRequirements)) + { + AddText(zip, $"{folder}/{CatalogManifests.LockName}", stage.LockedRequirements); + } + } + + private static void AddText(ZipArchive zip, string path, string content) + { + var entry = zip.CreateEntry(path, CompressionLevel.Optimal); + using var writer = new StreamWriter(entry.Open(), new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + writer.Write(content); + } + + /// Builds the zip in memory (catalog files are small) and copies it asynchronously to the destination. + private static async Task WriteZipAsync(Stream destination, Action write, CancellationToken cancellationToken) + { + var buffer = new MemoryStream(); + await using (buffer.ConfigureAwait(false)) + { + using (var zip = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + { + write(zip); + } + + buffer.Position = 0; + await buffer.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); + } + } + + private static DefinitionInvalidException Invalid(IReadOnlyList issues) => + new(PsrErrorCodes.ImportInvalid, new ValidationReport(issues, [])); +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.cs b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.cs new file mode 100644 index 0000000..d6b83c2 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/EfCatalogAdmin.cs @@ -0,0 +1,611 @@ +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Options; +using Npgsql; +using PythonScriptRunner.EntityFrameworkCore.Entities; +using PythonScriptRunner.EntityFrameworkCore.Infrastructure; + +namespace PythonScriptRunner.EntityFrameworkCore.Catalog; + +/// Edits the catalog stored in PostgreSQL (drafts with optimistic concurrency, publication, archiving, deletion, import/export). +internal sealed partial class EfCatalogAdmin( + IDbContextFactory factory, + EfCatalog catalog, + IStageValidator stageValidator, + IPipelineValidator pipelineValidator, + IEnvironmentManager environments, + IRunEngine engine, + IOptions runnerOptions, + PsrEntityFrameworkOptions options, + TimeProvider time) : ICatalogAdmin +{ + private const int MaxScriptBytes = 1024 * 1024; + + private const string StageTemplate = """ + from psrunner import stage + + + @stage + def run(inputs, ctx): + return {} + """; + + [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] + private static partial Regex KeyPattern(); + + public async Task> ListStagesAsync(CatalogQuery query, CancellationToken cancellationToken = default) + { + var (pageNumber, pageSize) = Paging(query); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var stages = db.Stages.AsNoTracking(); + if (!string.IsNullOrWhiteSpace(query.Search)) + { + var pattern = $"%{query.Search.Trim()}%"; + stages = stages.Where(s => EF.Functions.ILike(s.Key, pattern) || EF.Functions.ILike(s.Name, pattern)); + } + + var total = await stages.CountAsync(cancellationToken).ConfigureAwait(false); + var items = await stages.OrderBy(s => s.Key) + .Skip((pageNumber - 1) * pageSize).Take(pageSize) + .Select(s => new StageSummary( + s.Key, s.Name, s.Description, s.LatestPublishedVersion, + s.Versions.Where(v => v.Status == VersionStatus.Draft).Select(v => (int?)v.Version).FirstOrDefault(), + s.UpdatedAt)) + .ToListAsync(cancellationToken).ConfigureAwait(false); + return new Page(items, pageNumber, pageSize, total); + } + } + + public async Task GetStageAsync(string key, CancellationToken cancellationToken = default) + { + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var stage = await db.Stages.AsNoTracking().Include(s => s.Versions).SingleOrDefaultAsync(s => s.Key == key, cancellationToken).ConfigureAwait(false); + return stage is null ? null : await ToStageDetailAsync(db, stage, cancellationToken).ConfigureAwait(false); + } + } + + public async Task GetStageVersionAsync(string key, int version, CancellationToken cancellationToken = default) + { + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await db.StageVersions.AsNoTracking().Include(v => v.Stage) + .SingleOrDefaultAsync(v => v.Stage.Key == key && v.Version == version, cancellationToken).ConfigureAwait(false); + return row is null ? null : CatalogMapping.ToStageVersionDetail(row.Stage, row); + } + } + + public async Task GetStageDraftAsync(string key, CancellationToken cancellationToken = default) + { + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await db.StageVersions.AsNoTracking().Include(v => v.Stage) + .SingleOrDefaultAsync(v => v.Stage.Key == key && v.Status == VersionStatus.Draft, cancellationToken).ConfigureAwait(false); + return row is null ? null : CatalogMapping.ToStageDraft(row.Stage, row); + } + } + + public async Task CreateStageAsync(NewCatalogItem stage, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(stage); + ArgumentNullException.ThrowIfNull(actor); + ValidateItem(stage); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + if (await db.Stages.AnyAsync(s => s.Key == stage.Key, cancellationToken).ConfigureAwait(false)) + { + throw new PsrDomainException(PsrErrorCodes.StageKeyTaken, $"Stage '{stage.Key}' already exists."); + } + + var now = time.GetUtcNow(); + var row = new StageEntity + { + Id = Guid.CreateVersion7(now), + Key = stage.Key, + Name = stage.Name, + Description = stage.Description, + CreatedAt = now, + CreatedBy = actor.Subject, + UpdatedAt = now, + UpdatedBy = actor.Subject, + }; + var draft = NewStageVersion(row, 1, actor, now); + CatalogMapping.Apply(draft, new StageDraftData { Script = StageTemplate, PythonVersion = runnerOptions.Value.DefaultPythonVersion }); + row.Versions.Add(draft); + db.Stages.Add(row); + AuditLog.Add(db, new AuditEntry(actor, AuditActions.StageCreated, "Stage", row.Id, row.Key), now); + await SaveAsync(db, cancellationToken, unique: _ => new PsrDomainException(PsrErrorCodes.StageKeyTaken, $"Stage '{stage.Key}' already exists.")) + .ConfigureAwait(false); + return CatalogMapping.ToStageDraft(row, draft); + } + } + + public async Task UpdateStageAsync( + string key, CatalogItemChanges changes, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(changes); + ArgumentNullException.ThrowIfNull(actor); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var stage = await db.Stages.Include(s => s.Versions).SingleOrDefaultAsync(s => s.Key == key, cancellationToken).ConfigureAwait(false) + ?? throw new StageNotFoundException(key); + if (stage.RowVersion != expectedRowVersion) + { + throw new ConcurrencyConflictException($"Stage '{key}' was changed by someone else.", await ToStageDetailAsync(db, stage, cancellationToken).ConfigureAwait(false)); + } + + var details = new JsonObject(); + if (changes.Name is { } name && name != stage.Name) + { + ValidateName(name); + details["name"] = new JsonArray(stage.Name, name); + stage.Name = name; + } + + if (changes.Description is { } description && description != stage.Description) + { + details["description"] = new JsonArray(stage.Description, description); + stage.Description = description; + } + + var now = time.GetUtcNow(); + Touch(stage, actor, now); + db.Entry(stage).Property(s => s.RowVersion).OriginalValue = expectedRowVersion; + AuditLog.Add(db, new AuditEntry(actor, AuditActions.StageUpdated, "Stage", stage.Id, stage.Key, details), now); + await SaveAsync(db, cancellationToken, conflict: Current(() => GetStageAsync(key, CancellationToken.None))).ConfigureAwait(false); + catalog.InvalidateStage(key); + return await ToStageDetailAsync(db, stage, cancellationToken).ConfigureAwait(false); + } + } + + public async Task CreateStageDraftFromAsync(string key, int fromVersion, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var stage = await db.Stages.Include(s => s.Versions).SingleOrDefaultAsync(s => s.Key == key, cancellationToken).ConfigureAwait(false) + ?? throw new StageNotFoundException(key); + EnsureNoDraft(stage.Versions.Any(v => v.Status == VersionStatus.Draft), key); + var source = stage.Versions.SingleOrDefault(v => v.Version == fromVersion) ?? throw new VersionNotFoundException(key, fromVersion); + var now = time.GetUtcNow(); + var draft = NewStageVersion(stage, stage.Versions.Max(v => v.Version) + 1, actor, now); + CatalogMapping.Apply(draft, CatalogMapping.ToStageData(source)); + stage.Versions.Add(draft); + AuditLog.Add(db, new AuditEntry(actor, AuditActions.StageDraftCreated, "StageVersion", draft.Id, $"{key}@{draft.Version}", + new JsonObject { ["fromVersion"] = fromVersion }), now); + await SaveAsync(db, cancellationToken, unique: _ => DraftExists(key)).ConfigureAwait(false); + return CatalogMapping.ToStageDraft(stage, draft); + } + } + + public async Task SaveStageDraftAsync( + string key, int version, StageDraftData data, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(data); + ArgumentNullException.ThrowIfNull(actor); + if (System.Text.Encoding.UTF8.GetByteCount(data.Script) > MaxScriptBytes) + { + throw new DefinitionInvalidException(PsrErrorCodes.StageInvalid, Report("STAGE_DEFINITION", "script is larger than 1 MB", field: "script")); + } + + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await LoadStageVersionAsync(db, key, version, cancellationToken).ConfigureAwait(false); + EnsureDraft(row.Status, key, version); + if (row.RowVersion != expectedRowVersion) + { + throw new ConcurrencyConflictException($"The draft {key}@{version} was changed by someone else.", CatalogMapping.ToStageDraft(row.Stage, row)); + } + + var before = CatalogMapping.ToStageData(row); + CatalogMapping.Apply(row, data); + var now = time.GetUtcNow(); + Touch(row, actor, now); + db.Entry(row).Property(v => v.RowVersion).OriginalValue = expectedRowVersion; + AuditLog.Add(db, new AuditEntry(actor, AuditActions.StageDraftSaved, "StageVersion", row.Id, $"{key}@{version}", + new JsonObject { ["changed"] = ChangedFields(before, data) }), now); + await SaveAsync(db, cancellationToken, conflict: Current(() => GetStageDraftAsync(key, CancellationToken.None))).ConfigureAwait(false); + return CatalogMapping.ToStageDraft(row.Stage, row); + } + } + + public async Task DiscardStageDraftAsync(string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await LoadStageVersionAsync(db, key, version, cancellationToken).ConfigureAwait(false); + EnsureDraft(row.Status, key, version); + if (row.RowVersion != expectedRowVersion) + { + throw new ConcurrencyConflictException($"The draft {key}@{version} was changed by someone else.", CatalogMapping.ToStageDraft(row.Stage, row)); + } + + if (await db.PipelineSteps.AnyAsync(s => s.StageVersionId == row.Id, cancellationToken).ConfigureAwait(false)) + { + throw new PsrDomainException(PsrErrorCodes.DeleteNotAllowed, $"The draft {key}@{version} is used by a pipeline draft."); + } + + var now = time.GetUtcNow(); + var onlyVersion = !await db.StageVersions.AnyAsync(v => v.StageId == row.StageId && v.Id != row.Id, cancellationToken).ConfigureAwait(false); + db.Entry(row).Property(v => v.RowVersion).OriginalValue = expectedRowVersion; + db.StageVersions.Remove(row); + if (onlyVersion) + { + // A stage without versions cannot get a new draft; discarding its first draft removes it. + db.Stages.Remove(row.Stage); + } + + AuditLog.Add(db, new AuditEntry(actor, AuditActions.StageDraftDiscarded, "StageVersion", row.Id, $"{key}@{version}"), now); + await SaveAsync(db, cancellationToken, conflict: Current(() => GetStageDraftAsync(key, CancellationToken.None))).ConfigureAwait(false); + catalog.InvalidateStage(key); + } + } + + public async Task ValidateStageDraftAsync(string key, CancellationToken cancellationToken = default) + { + var (stage, _) = await LoadStageDraftDefinitionAsync(key, cancellationToken).ConfigureAwait(false); + return stageValidator.Validate(stage); + } + + public async Task PublishStageAsync( + string key, int version, uint expectedRowVersion, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await LoadStageVersionAsync(db, key, version, cancellationToken).ConfigureAwait(false); + EnsureDraft(row.Status, key, version); + if (row.RowVersion != expectedRowVersion) + { + throw new ConcurrencyConflictException($"The draft {key}@{version} was changed by someone else.", CatalogMapping.ToStageDraft(row.Stage, row)); + } + + var definition = CatalogMapping.ToStage(row.Stage, row); + var report = stageValidator.Validate(definition); + if (!report.IsValid) + { + throw new DefinitionInvalidException(PsrErrorCodes.StageInvalid, report); + } + + string locked; + try + { + if (await environments.CheckSyntaxAsync(row.PythonVersion, row.Script, cancellationToken).ConfigureAwait(false) is { } syntax) + { + throw new ScriptSyntaxException(syntax.Message, syntax.Line, syntax.Column); + } + + locked = await environments.LockAsync(row.PythonVersion, row.Requirements, cancellationToken).ConfigureAwait(false); + } + catch (PythonEnvironmentException ex) + { + throw new PsrDomainException(PsrErrorCodes.DependencyResolutionFailed, ex.Message, ex); + } + + var now = time.GetUtcNow(); + row.LockedRequirements = locked; + row.EnvironmentHash = environments.ComputeHash(row.PythonVersion, locked); + row.Status = VersionStatus.Published; + row.PublishedAt = now; + row.PublishedBy = actor.Subject; + Touch(row, actor, now); + row.Stage.LatestPublishedVersion = Math.Max(row.Stage.LatestPublishedVersion ?? 0, version); + Touch(row.Stage, actor, now); + db.Entry(row).Property(v => v.RowVersion).OriginalValue = expectedRowVersion; + AuditLog.Add(db, new AuditEntry(actor, AuditActions.StagePublished, "StageVersion", row.Id, $"{key}@{version}", + new JsonObject { ["environmentHash"] = row.EnvironmentHash }), now); + await SaveAsync(db, cancellationToken, conflict: Current(() => GetStageDraftAsync(key, CancellationToken.None))).ConfigureAwait(false); + catalog.InvalidateStage(key); + return CatalogMapping.ToStageVersionDetail(row.Stage, row); + } + } + + public async Task ArchiveStageVersionAsync(string key, int version, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await LoadStageVersionAsync(db, key, version, cancellationToken).ConfigureAwait(false); + if (row.Status == VersionStatus.Archived) + { + return CatalogMapping.ToStageVersionDetail(row.Stage, row); + } + + if (row.Status != VersionStatus.Published) + { + throw new PsrDomainException(PsrErrorCodes.VersionImmutable, $"Only published versions can be archived; discard the draft {key}@{version} instead."); + } + + var users = await db.PipelineSteps.AsNoTracking() + .Where(s => s.StageVersionId == row.Id) + .Join(db.PipelineVersions.Where(p => p.Status == VersionStatus.Published), s => s.PipelineVersionId, p => p.Id, + (s, p) => new { p.Pipeline.Key, p.Version }) + .Distinct() + .ToListAsync(cancellationToken).ConfigureAwait(false); + if (users.Count > 0) + { + throw new VersionReferencedException(key, version, [.. users.OrderBy(u => u.Key).ThenBy(u => u.Version).Select(u => $"{u.Key}@{u.Version}")]); + } + + var now = time.GetUtcNow(); + row.Status = VersionStatus.Archived; + row.ArchivedAt = now; + row.ArchivedBy = actor.Subject; + Touch(row, actor, now); + if (row.Stage.LatestPublishedVersion == version) + { + row.Stage.LatestPublishedVersion = await db.StageVersions + .Where(v => v.StageId == row.StageId && v.Status == VersionStatus.Published && v.Id != row.Id) + .MaxAsync(v => (int?)v.Version, cancellationToken).ConfigureAwait(false); + Touch(row.Stage, actor, now); + } + + AuditLog.Add(db, new AuditEntry(actor, AuditActions.StageVersionArchived, "StageVersion", row.Id, $"{key}@{version}"), now); + await SaveAsync(db, cancellationToken).ConfigureAwait(false); + catalog.InvalidateStage(key); + return CatalogMapping.ToStageVersionDetail(row.Stage, row); + } + } + + public async Task DeleteStageAsync(string key, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var stage = await db.Stages.SingleOrDefaultAsync(s => s.Key == key, cancellationToken).ConfigureAwait(false) + ?? throw new StageNotFoundException(key); + var users = await db.PipelineSteps.AsNoTracking() + .Where(s => s.StageVersion.StageId == stage.Id) + .Join(db.PipelineVersions, s => s.PipelineVersionId, p => p.Id, (s, p) => new { p.Pipeline.Key, p.Version }) + .Distinct() + .ToListAsync(cancellationToken).ConfigureAwait(false); + if (users.Count > 0) + { + throw new PsrDomainException(PsrErrorCodes.DeleteNotAllowed, + $"Stage '{key}' is used by pipelines: {string.Join(", ", users.Select(u => $"{u.Key}@{u.Version}"))}."); + } + + var hasRuns = await db.Runs.AnyAsync(r => r.StageKey == key, cancellationToken).ConfigureAwait(false) + || await db.StepRuns.AnyAsync(s => s.StageKey == key, cancellationToken).ConfigureAwait(false); + if (hasRuns) + { + throw new PsrDomainException(PsrErrorCodes.DeleteNotAllowed, $"Stage '{key}' has runs in the retained history."); + } + + db.Stages.Remove(stage); + AuditLog.Add(db, new AuditEntry(actor, AuditActions.StageDeleted, "Stage", stage.Id, key), time.GetUtcNow()); + await SaveAsync(db, cancellationToken).ConfigureAwait(false); + catalog.InvalidateStage(key); + } + } + + public async Task TestStageDraftAsync(string key, object? inputs, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + var (stage, versionId) = await LoadStageDraftDefinitionAsync(key, cancellationToken).ConfigureAwait(false); + var report = stageValidator.Validate(stage); + if (!report.IsValid) + { + throw new DefinitionInvalidException(PsrErrorCodes.StageInvalid, report); + } + + await WriteAuditAsync(new AuditEntry(actor, AuditActions.StageTested, "StageVersion", versionId, $"{key}@{stage.Version}"), cancellationToken) + .ConfigureAwait(false); + return await engine.RunStageAsync(stage, RunKind.StageTest, inputs, RunOptionsFor(actor), cancellationToken).ConfigureAwait(false); + } + + private async Task<(StageDefinition Stage, Guid VersionId)> LoadStageDraftDefinitionAsync(string key, CancellationToken cancellationToken) + { + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await db.StageVersions.AsNoTracking().Include(v => v.Stage) + .SingleOrDefaultAsync(v => v.Stage.Key == key && v.Status == VersionStatus.Draft, cancellationToken).ConfigureAwait(false) + ?? throw (await db.Stages.AnyAsync(s => s.Key == key, cancellationToken).ConfigureAwait(false) + ? new PsrDomainException(PsrErrorCodes.VersionImmutable, $"Stage '{key}' has no draft; create one from a version first.") + : new StageNotFoundException(key)); + return (CatalogMapping.ToStage(row.Stage, row), row.Id); + } + } + + private static async Task LoadStageVersionAsync(PsrDbContext db, string key, int version, CancellationToken cancellationToken) + { + var row = await db.StageVersions.Include(v => v.Stage) + .SingleOrDefaultAsync(v => v.Stage.Key == key && v.Version == version, cancellationToken).ConfigureAwait(false); + if (row is not null) + { + return row; + } + + return await db.Stages.AnyAsync(s => s.Key == key, cancellationToken).ConfigureAwait(false) + ? throw new VersionNotFoundException(key, version) + : throw new StageNotFoundException(key); + } + + private static async Task ToStageDetailAsync(PsrDbContext db, StageEntity stage, CancellationToken cancellationToken) + { + var versions = stage.Versions.Count > 0 + ? stage.Versions + : await db.StageVersions.AsNoTracking().Where(v => v.StageId == stage.Id).ToListAsync(cancellationToken).ConfigureAwait(false); + var usages = await db.PipelineSteps.AsNoTracking() + .Where(s => s.StageVersion.StageId == stage.Id) + .Join(db.PipelineVersions, s => s.PipelineVersionId, p => p.Id, + (s, p) => new StageUsage(p.Pipeline.Key, p.Version, p.Status, s.Key, s.StageVersion.Version)) + .ToListAsync(cancellationToken).ConfigureAwait(false); + return new StageDetail( + stage.Key, + stage.Name, + stage.Description, + stage.LatestPublishedVersion, + [.. versions.OrderByDescending(v => v.Version).Select(v => new VersionSummary(v.Version, v.Status, v.PublishedAt, v.UpdatedAt))], + [.. usages.OrderBy(u => u.PipelineKey).ThenBy(u => u.PipelineVersion)], + stage.RowVersion, + CatalogMapping.Authorship(stage)); + } + + private static StageVersionEntity NewStageVersion(StageEntity stage, int version, Actor actor, DateTimeOffset now) => new() + { + Id = Guid.CreateVersion7(now), + StageId = stage.Id, + Stage = stage, + Version = version, + Status = VersionStatus.Draft, + Script = string.Empty, + ScriptSha256 = string.Empty, + PythonVersion = string.Empty, + InputFields = "[]", + OutputFields = "[]", + CreatedAt = now, + CreatedBy = actor.Subject, + UpdatedAt = now, + UpdatedBy = actor.Subject, + }; + + private static JsonArray ChangedFields(StageDraftData before, StageDraftData after) + { + var changed = new JsonArray(); + void Check(string name, bool same) + { + if (!same) + { + changed.Add(name); + } + } + + Check("script", before.Script == after.Script); + Check("pythonVersion", before.PythonVersion == after.PythonVersion); + Check("requirements", (before.Requirements ?? string.Empty) == (after.Requirements ?? string.Empty)); + Check("inputs", StoredJson.Fields(before.Inputs) == StoredJson.Fields(after.Inputs)); + Check("outputs", StoredJson.Fields(before.Outputs) == StoredJson.Fields(after.Outputs)); + Check("secrets", before.RequiredSecrets.SequenceEqual(after.RequiredSecrets)); + Check("limits", before.Timeout == after.Timeout && before.MemoryLimitMb == after.MemoryLimitMb); + Check("retry", before.Retry == after.Retry && before.Idempotent == after.Idempotent); + return changed; + } + + private static void Touch(StageEntity row, Actor actor, DateTimeOffset now) + { + row.UpdatedAt = now; + row.UpdatedBy = actor.Subject; + } + + private static void Touch(StageVersionEntity row, Actor actor, DateTimeOffset now) + { + row.UpdatedAt = now; + row.UpdatedBy = actor.Subject; + } + + private static void Touch(PipelineEntity row, Actor actor, DateTimeOffset now) + { + row.UpdatedAt = now; + row.UpdatedBy = actor.Subject; + } + + private static void Touch(PipelineVersionEntity row, Actor actor, DateTimeOffset now) + { + row.UpdatedAt = now; + row.UpdatedBy = actor.Subject; + } + + private static void EnsureDraft(VersionStatus status, string key, int version) + { + if (status != VersionStatus.Draft) + { + throw new PsrDomainException(PsrErrorCodes.VersionImmutable, + $"Version {key}@{version} is {status} and cannot change; create a draft from it instead."); + } + } + + private static void EnsureNoDraft(bool exists, string key) + { + if (exists) + { + throw DraftExists(key); + } + } + + private static PsrDomainException DraftExists(string key) => + new(PsrErrorCodes.DraftAlreadyExists, $"'{key}' already has a draft; edit or discard it first."); + + private static void ValidateItem(NewCatalogItem item) + { + if (item.Key.Length is < 3 or > 64 || !KeyPattern().IsMatch(item.Key)) + { + throw new ArgumentException($"Key '{item.Key}' must be a slug of 3-64 chars (a-z, 0-9, '-').", nameof(item)); + } + + ValidateName(item.Name); + } + + private static void ValidateName(string name) + { + if (string.IsNullOrWhiteSpace(name) || name.Length > 200) + { + throw new ArgumentException("Name must have 1-200 chars.", nameof(name)); + } + } + + private static (int PageNumber, int PageSize) Paging(CatalogQuery query) + { + ArgumentNullException.ThrowIfNull(query); + return (Math.Max(1, query.PageNumber), Math.Clamp(query.PageSize, 1, 100)); + } + + private static ValidationReport Report(string code, string message, string? step = null, string? field = null) => + new([new ValidationIssue(code, message) { Step = step, Field = field }], []); + + private static RunOptions RunOptionsFor(Actor actor) => + new() { TriggeredBy = actor.Subject, Trigger = actor.IsHuman ? RunTrigger.Dashboard : RunTrigger.Library }; + + private async Task WriteAuditAsync(AuditEntry entry, CancellationToken cancellationToken) + { + if (!entry.Actor.IsHuman) + { + return; + } + + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + AuditLog.Add(db, entry, time.GetUtcNow()); + await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// Saves and maps unique violations and concurrency conflicts to domain errors. + private static async Task SaveAsync( + PsrDbContext db, + CancellationToken cancellationToken, + Func>? conflict = null, + Func? unique = null) + { + try + { + await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + catch (DbUpdateConcurrencyException ex) + { + var current = conflict is null ? null : await conflict().ConfigureAwait(false); + throw new ConcurrencyConflictException($"The row was changed by someone else ({ex.Entries.Count} entries).", current); + } + catch (DbUpdateException ex) when (unique is not null && ex.InnerException is PostgresException { SqlState: PostgresErrorCodes.UniqueViolation } postgres) + { + throw unique(postgres); + } + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Catalog/StoredJson.cs b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/StoredJson.cs new file mode 100644 index 0000000..2a2df4e --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Catalog/StoredJson.cs @@ -0,0 +1,106 @@ +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace PythonScriptRunner.EntityFrameworkCore.Catalog; + +/// JSON stored in the jsonb columns (formats of the data mapping §2.1, §4.4, §4.5 and §5.1). +internal static class StoredJson +{ + public static string Fields(IEnumerable fields) => CatalogManifests.WriteFields(fields).ToJsonString(); + + public static IReadOnlyList Fields(string json) => CatalogManifests.ReadFields(JsonNode.Parse(json)!.AsArray()); + + public static string Bindings(IEnumerable bindings) => + new JsonArray([.. bindings.Select(b => (JsonNode)new JsonObject { ["input"] = b.Input, ["source"] = Source(b.Source) })]).ToJsonString(); + + public static IReadOnlyList Bindings(string json) => + [.. JsonNode.Parse(json)!.AsArray().Select(b => new InputBinding(b!["input"]!.GetValue(), Source(b["source"]!.AsObject())))]; + + public static string Outputs(IEnumerable outputs) => + new JsonArray([.. outputs.Select(o => + { + var node = new JsonObject + { + ["name"] = o.Name, + ["source"] = new JsonObject { ["step"] = o.Source.Step, ["field"] = o.Source.Field }, + }; + if (o.Description is not null) + { + node["description"] = o.Description; + } + + return (JsonNode)node; + })]).ToJsonString(); + + public static IReadOnlyList Outputs(string json) => + [.. JsonNode.Parse(json)!.AsArray().Select(o => new PipelineOutputDefinition( + o!["name"]!.GetValue(), + new StepOutputSource(o["source"]!["step"]!.GetValue(), o["source"]!["field"]!.GetValue())) + { + Description = o["description"]?.GetValue(), + })]; + + public static string? Error(RunError? error) + { + if (error is null) + { + return null; + } + + var node = new JsonObject { ["kind"] = error.Kind.ToString(), ["code"] = error.Code, ["message"] = error.Message, ["step"] = error.StepKey }; + if (error.Traceback is not null) + { + node["traceback"] = error.Traceback; + } + + if (error.Details is not null) + { + node["details"] = error.Details.DeepClone(); + } + + return node.ToJsonString(); + } + + public static RunError? Error(string? json) + { + if (json is null) + { + return null; + } + + var node = JsonNode.Parse(json)!; + return new RunError(Enum.Parse(node["kind"]!.GetValue()), node["message"]?.GetValue() ?? string.Empty) + { + Code = node["code"]?.GetValue(), + StepKey = node["step"]?.GetValue(), + Traceback = node["traceback"]?.GetValue(), + Details = node["details"]?.DeepClone(), + }; + } + + public static string? Node(JsonNode? node) => node?.ToJsonString(); + + public static JsonNode? Node(string? json) => json is null ? null : JsonNode.Parse(json); + + public static JsonObject? Object(string? json) => json is null ? null : JsonNode.Parse(json)!.AsObject(); + + /// Whether two JSON texts hold the same value (ignores formatting). + public static bool SameJson(string? left, string? right) => + left == right || (left is not null && right is not null && JsonNode.DeepEquals(JsonNode.Parse(left), JsonNode.Parse(right))); + + private static JsonObject Source(BindingSource source) => source switch + { + PipelineInputSource input => new JsonObject { ["kind"] = "pipeline", ["field"] = input.Field }, + StepOutputSource output => new JsonObject { ["kind"] = "step", ["step"] = output.Step, ["field"] = output.Field }, + ConstantSource constant => new JsonObject { ["kind"] = "constant", ["value"] = constant.Value?.DeepClone() }, + _ => throw new JsonException($"Unknown binding source {source.GetType().Name}."), + }; + + private static BindingSource Source(JsonObject json) => json["kind"]!.GetValue() switch + { + "pipeline" => new PipelineInputSource(json["field"]!.GetValue()), + "step" => new StepOutputSource(json["step"]!.GetValue(), json["field"]!.GetValue()), + "constant" => new ConstantSource(json["value"]?.DeepClone()), + var kind => throw new JsonException($"Unknown binding kind '{kind}'."), + }; +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Entities/CatalogEntities.cs b/src/PythonScriptRunner.EntityFrameworkCore/Entities/CatalogEntities.cs new file mode 100644 index 0000000..8bd2eb7 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Entities/CatalogEntities.cs @@ -0,0 +1,229 @@ +namespace PythonScriptRunner.EntityFrameworkCore.Entities; + +#pragma warning disable CS1591 // Persistence rows mirror the documented schema (psr.*); see the data mapping. + +/// Row of psr.stages. +public sealed class StageEntity +{ + public Guid Id { get; set; } + + public required string Key { get; set; } + + public required string Name { get; set; } + + public string? Description { get; set; } + + public int? LatestPublishedVersion { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public required string CreatedBy { get; set; } + + public DateTimeOffset UpdatedAt { get; set; } + + public required string UpdatedBy { get; set; } + + /// PostgreSQL xmin (optimistic concurrency). + public uint RowVersion { get; set; } + + public List Versions { get; set; } = []; +} + +/// Row of psr.stage_versions. +public sealed class StageVersionEntity +{ + public Guid Id { get; set; } + + public Guid StageId { get; set; } + + public StageEntity Stage { get; set; } = null!; + + public int Version { get; set; } + + public VersionStatus Status { get; set; } + + public required string Script { get; set; } + + public required string ScriptSha256 { get; set; } + + public required string PythonVersion { get; set; } + + public string? Requirements { get; set; } + + public string? LockedRequirements { get; set; } + + public string? EnvironmentHash { get; set; } + + /// FieldDefinition[] as JSON. + public required string InputFields { get; set; } + + /// FieldDefinition[] as JSON. + public required string OutputFields { get; set; } + + public string[] RequiredSecrets { get; set; } = []; + + public int TimeoutSeconds { get; set; } + + public int? MemoryLimitMb { get; set; } + + public int RetryMaxAttempts { get; set; } + + public int RetryInitialDelaySeconds { get; set; } + + public decimal RetryBackoffMultiplier { get; set; } + + public bool Idempotent { get; set; } + + public DateTimeOffset? PublishedAt { get; set; } + + public string? PublishedBy { get; set; } + + public DateTimeOffset? ArchivedAt { get; set; } + + public string? ArchivedBy { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public required string CreatedBy { get; set; } + + public DateTimeOffset UpdatedAt { get; set; } + + public required string UpdatedBy { get; set; } + + public uint RowVersion { get; set; } +} + +/// Row of psr.pipelines. +public sealed class PipelineEntity +{ + public Guid Id { get; set; } + + public required string Key { get; set; } + + public required string Name { get; set; } + + public string? Description { get; set; } + + public int? LatestPublishedVersion { get; set; } + + public int? MaxConcurrentRuns { get; set; } + + public int? ArtifactRetentionDays { get; set; } + + public int? RunRetentionDays { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public required string CreatedBy { get; set; } + + public DateTimeOffset UpdatedAt { get; set; } + + public required string UpdatedBy { get; set; } + + public uint RowVersion { get; set; } + + public List Versions { get; set; } = []; +} + +/// Row of psr.pipeline_versions. +public sealed class PipelineVersionEntity +{ + public Guid Id { get; set; } + + public Guid PipelineId { get; set; } + + public PipelineEntity Pipeline { get; set; } = null!; + + public int Version { get; set; } + + public VersionStatus Status { get; set; } + + /// FieldDefinition[] as JSON. + public required string InputFields { get; set; } + + /// PipelineOutput[] as JSON. + public required string Outputs { get; set; } + + /// Visual editor layout as JSON. + public string? Layout { get; set; } + + public DateTimeOffset? PublishedAt { get; set; } + + public string? PublishedBy { get; set; } + + public DateTimeOffset? ArchivedAt { get; set; } + + public string? ArchivedBy { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public required string CreatedBy { get; set; } + + public DateTimeOffset UpdatedAt { get; set; } + + public required string UpdatedBy { get; set; } + + public uint RowVersion { get; set; } + + public List Steps { get; set; } = []; +} + +/// Row of psr.pipeline_steps. +public sealed class PipelineStepEntity +{ + public Guid Id { get; set; } + + public Guid PipelineVersionId { get; set; } + + public int Order { get; set; } + + public required string Key { get; set; } + + public string? DisplayName { get; set; } + + public Guid StageVersionId { get; set; } + + public StageVersionEntity StageVersion { get; set; } = null!; + + /// InputBinding[] as JSON. + public required string Bindings { get; set; } +} + +/// Row of psr.schedules (managed by the worker package). +public sealed class ScheduleEntity +{ + public Guid Id { get; set; } + + public Guid PipelineId { get; set; } + + public required string Name { get; set; } + + public required string Cron { get; set; } + + public required string TimeZone { get; set; } + + public int? PinnedVersion { get; set; } + + /// Fixed inputs as JSON. + public required string Inputs { get; set; } + + public bool IsEnabled { get; set; } + + public DateTimeOffset? NextRunAt { get; set; } + + public DateTimeOffset? LastFiredAt { get; set; } + + public Guid? LastRunId { get; set; } + + public string? LastError { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public required string CreatedBy { get; set; } + + public DateTimeOffset UpdatedAt { get; set; } + + public required string UpdatedBy { get; set; } + + public uint RowVersion { get; set; } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Entities/InfrastructureEntities.cs b/src/PythonScriptRunner.EntityFrameworkCore/Entities/InfrastructureEntities.cs new file mode 100644 index 0000000..54cfcf9 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Entities/InfrastructureEntities.cs @@ -0,0 +1,67 @@ +namespace PythonScriptRunner.EntityFrameworkCore.Entities; + +#pragma warning disable CS1591 // Persistence rows mirror the documented schema (psr.*); see the data mapping. + +/// Row of psr.outbox_messages: written in the same transaction as the change, published by the relay. +public sealed class OutboxMessageEntity +{ + public Guid Id { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public DateTimeOffset AvailableAt { get; set; } + + public required string MessageType { get; set; } + + public required string Exchange { get; set; } + + public required string RoutingKey { get; set; } + + /// Body as JSON. + public required string Payload { get; set; } + + /// Headers as JSON. + public required string Headers { get; set; } + + public int Attempts { get; set; } + + public string? LastError { get; set; } + + public DateTimeOffset? PublishedAt { get; set; } +} + +/// Row of psr.idempotency_keys. +public sealed class IdempotencyKeyEntity +{ + public required string Scope { get; set; } + + public required string Key { get; set; } + + public Guid RunId { get; set; } + + public DateTimeOffset RunCreatedAt { get; set; } + + public DateTimeOffset ExpiresAt { get; set; } +} + +/// Row of psr.audit_events (partitioned by month). +public sealed class AuditEventEntity +{ + public Guid Id { get; set; } + + public DateTimeOffset OccurredAt { get; set; } + + public required string Actor { get; set; } + + public required string Action { get; set; } + + public required string EntityType { get; set; } + + public Guid? EntityId { get; set; } + + public string? EntityKey { get; set; } + + public string? Details { get; set; } + + public string? TraceId { get; set; } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Entities/RunEntities.cs b/src/PythonScriptRunner.EntityFrameworkCore/Entities/RunEntities.cs new file mode 100644 index 0000000..2da8f17 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Entities/RunEntities.cs @@ -0,0 +1,200 @@ +namespace PythonScriptRunner.EntityFrameworkCore.Entities; + +#pragma warning disable CS1591 // Persistence rows mirror the documented schema (psr.*); see the data mapping. + +/// Row of psr.runs (partitioned by week on ). +public sealed class RunEntity +{ + public Guid Id { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public RunKind Kind { get; set; } + + public ExecutionMode Mode { get; set; } + + public Guid? PipelineId { get; set; } + + public Guid? PipelineVersionId { get; set; } + + public string? PipelineKey { get; set; } + + public int? PipelineVersion { get; set; } + + public Guid? StageVersionId { get; set; } + + public string? StageKey { get; set; } + + public int? StageVersion { get; set; } + + public RunTrigger Trigger { get; set; } + + public required string TriggeredBy { get; set; } + + public Guid? ScheduleId { get; set; } + + public RunStatus Status { get; set; } + + /// Validated inputs as JSON (files as {"$artifact": id}). + public required string Inputs { get; set; } + + public string? Outputs { get; set; } + + public string? Error { get; set; } + + public string? IdempotencyKey { get; set; } + + public string? CorrelationId { get; set; } + + public string? TraceParent { get; set; } + + public DateTimeOffset? QueuedAt { get; set; } + + public DateTimeOffset? StartedAt { get; set; } + + public DateTimeOffset? FinishedAt { get; set; } + + public DateTimeOffset? CancelRequestedAt { get; set; } + + public string? CancelRequestedBy { get; set; } + + public DateTimeOffset? ExpiresAt { get; set; } + + public List Steps { get; set; } = []; +} + +/// Row of psr.step_runs (partitioned with its run). +public sealed class StepRunEntity +{ + public Guid Id { get; set; } + + public Guid RunId { get; set; } + + public DateTimeOffset RunCreatedAt { get; set; } + + public Guid? PipelineStepId { get; set; } + + public required string StepKey { get; set; } + + public int Order { get; set; } + + public Guid? StageVersionId { get; set; } + + public required string StageKey { get; set; } + + public int StageVersion { get; set; } + + public StepStatus Status { get; set; } + + public int CurrentAttempt { get; set; } + + public string? Inputs { get; set; } + + public string? Outputs { get; set; } + + public string? Error { get; set; } + + public DateTimeOffset? StartedAt { get; set; } + + public DateTimeOffset? FinishedAt { get; set; } + + public List Attempts { get; set; } = []; +} + +/// Row of psr.step_attempts (partitioned with its run). +public sealed class StepAttemptEntity +{ + public Guid Id { get; set; } + + public Guid StepRunId { get; set; } + + public Guid RunId { get; set; } + + public DateTimeOffset RunCreatedAt { get; set; } + + public int Attempt { get; set; } + + public AttemptCause Cause { get; set; } + + public required string TriggeredBy { get; set; } + + public AttemptStatus Status { get; set; } + + public string? WorkerId { get; set; } + + public DateTimeOffset? LeaseExpiresAt { get; set; } + + public string? EnvironmentHash { get; set; } + + public int? ExitCode { get; set; } + + public ErrorKind? ErrorKind { get; set; } + + public string? Error { get; set; } + + public DateTimeOffset StartedAt { get; set; } + + public DateTimeOffset? FinishedAt { get; set; } + + public int? DurationMs { get; set; } + + public int? OverheadMs { get; set; } +} + +/// Row of psr.attempt_logs (partitioned with its run). +public sealed class AttemptLogEntity +{ + public Guid AttemptId { get; set; } + + public DateTimeOffset RunCreatedAt { get; set; } + + public LogStream Stream { get; set; } + + public required string Content { get; set; } + + public bool Truncated { get; set; } + + public long OriginalBytes { get; set; } +} + +/// Row of psr.artifacts (metadata; partitioned with its run). +public sealed class ArtifactEntity +{ + public Guid Id { get; set; } + + public Guid RunId { get; set; } + + public DateTimeOffset RunCreatedAt { get; set; } + + public Guid? StepRunId { get; set; } + + public ArtifactDirection Direction { get; set; } + + public required string FieldName { get; set; } + + public int? ListIndex { get; set; } + + public required string FileName { get; set; } + + public required string ContentType { get; set; } + + public long SizeBytes { get; set; } + + public required string Sha256 { get; set; } + + public DateTimeOffset ContentCreatedAt { get; set; } + + public DateTimeOffset ContentExpiresAt { get; set; } + + public DateTimeOffset? ContentDeletedAt { get; set; } +} + +/// Row of psr.artifact_contents (blob; partitioned by day). Read and written with Npgsql streaming only. +public sealed class ArtifactContentEntity +{ + public Guid ArtifactId { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public byte[] Content { get; set; } = []; +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/AuditLog.cs b/src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/AuditLog.cs new file mode 100644 index 0000000..63c0472 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/AuditLog.cs @@ -0,0 +1,137 @@ +using System.Diagnostics; +using Microsoft.EntityFrameworkCore; +using PythonScriptRunner.EntityFrameworkCore.Catalog; +using PythonScriptRunner.EntityFrameworkCore.Entities; + +namespace PythonScriptRunner.EntityFrameworkCore.Infrastructure; + +/// Adds audit events to a context, so they are saved in the same transaction as the change they describe. +internal static class AuditLog +{ + /// Adds the event when the actor is human (automatic callers are recorded on the run itself; D-93). + public static void Add(PsrDbContext db, AuditEntry entry, DateTimeOffset now) + { + if (!entry.Actor.IsHuman) + { + return; + } + + db.AuditEvents.Add(new AuditEventEntity + { + Id = Guid.CreateVersion7(now), + OccurredAt = now, + Actor = entry.Actor.Subject, + Action = entry.Action, + EntityType = entry.EntityType, + EntityId = entry.EntityId, + EntityKey = entry.EntityKey, + Details = entry.Details is { Count: > 0 } details ? details.ToJsonString() : null, + TraceId = Activity.Current?.TraceId.ToHexString(), + }); + } +} + +/// Writes audit events in their own transaction. +internal sealed class AuditWriter(IDbContextFactory factory, TimeProvider time) : IAuditWriter +{ + public async Task WriteAsync(AuditEntry entry, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entry); + if (!entry.Actor.IsHuman) + { + return; + } + + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + AuditLog.Add(db, entry, time.GetUtcNow()); + await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + } +} + +/// Lists audit events newest first (keyset on occurred_at, id). +internal sealed class AuditReader(IDbContextFactory factory, TimeProvider time) : IAuditReader +{ + public async Task> ListAsync(AuditQuery query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + var (from, to) = TimeWindow.Resolve(query.From, query.To, time); + var limit = Math.Clamp(query.Limit, 1, 100); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var events = db.AuditEvents.AsNoTracking().Where(e => e.OccurredAt >= from && e.OccurredAt < to); + if (query.Actor is { } actor) + { + events = events.Where(e => e.Actor == actor); + } + + if (query.EntityType is { } entityType) + { + events = events.Where(e => e.EntityType == entityType); + } + + if (KeysetCursor.Decode(query.Cursor) is { } cursor) + { + events = events.Where(e => e.OccurredAt < cursor.At || (e.OccurredAt == cursor.At && e.Id < cursor.Id)); + } + + var rows = await events.OrderByDescending(e => e.OccurredAt).ThenByDescending(e => e.Id).Take(limit + 1) + .ToListAsync(cancellationToken).ConfigureAwait(false); + var page = rows.Take(limit) + .Select(e => new AuditEvent(e.Id, e.OccurredAt, e.Actor, e.Action, e.EntityType, e.EntityId, e.EntityKey, StoredJson.Object(e.Details), e.TraceId)) + .ToList(); + return new KeysetPage(page, rows.Count > limit ? KeysetCursor.Encode(rows[limit - 1].OccurredAt, rows[limit - 1].Id) : null); + } + } +} + +/// Time window of history queries: default last 24 hours, at most 90 days (keeps queries on recent partitions). +internal static class TimeWindow +{ + public static readonly TimeSpan Default = TimeSpan.FromHours(24); + public static readonly TimeSpan Maximum = TimeSpan.FromDays(90); + + public static (DateTimeOffset From, DateTimeOffset To) Resolve(DateTimeOffset? from, DateTimeOffset? to, TimeProvider time) + { + var end = to ?? time.GetUtcNow(); + var start = from ?? end - Default; + if (start >= end) + { + throw new ArgumentException("The window must start before it ends.", nameof(from)); + } + + return end - start > Maximum + ? throw new ArgumentException($"The window cannot exceed {Maximum.TotalDays:0} days.", nameof(from)) + : (start, end); + } +} + +/// Opaque cursor (timestamp, id) of keyset pagination. +internal static class KeysetCursor +{ + public static string Encode(DateTimeOffset at, Guid id) => + Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes($"{at.UtcTicks}:{id:N}")).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + + public static (DateTimeOffset At, Guid Id)? Decode(string? cursor) + { + if (string.IsNullOrEmpty(cursor)) + { + return null; + } + + try + { + var base64 = cursor.Replace('-', '+').Replace('_', '/'); + base64 = base64.PadRight(base64.Length + ((4 - (base64.Length % 4)) % 4), '='); + var parts = System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(base64)).Split(':'); + return (new DateTimeOffset(long.Parse(parts[0], System.Globalization.CultureInfo.InvariantCulture), TimeSpan.Zero), Guid.ParseExact(parts[1], "N")); + } + catch (Exception ex) when (ex is FormatException or IndexOutOfRangeException or ArgumentException or OverflowException) + { + throw new ArgumentException("Invalid cursor.", nameof(cursor), ex); + } + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/Transactional.cs b/src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/Transactional.cs new file mode 100644 index 0000000..1d0474f --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Infrastructure/Transactional.cs @@ -0,0 +1,134 @@ +using System.Text.Json.Nodes; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; +using Npgsql; +using PythonScriptRunner.EntityFrameworkCore.Entities; + +namespace PythonScriptRunner.EntityFrameworkCore; + +/// Deduplicates run requests by (scope, key) for . +public interface IIdempotencyStore +{ + /// + /// Reserves the key for in the current transaction of (a rollback releases it). + /// Concurrent callers with the same key wait for each other: only one run holds the key. + /// + /// Context with an open transaction. + /// Scope (see ). + /// Key given by the caller. + /// Run that wants the key. + /// Cancellation token. + /// null when reserved; otherwise the run that already holds the key. + Task ReserveAsync(PsrDbContext db, string scope, string key, RunId runId, CancellationToken cancellationToken = default); +} + +/// Scopes of idempotency keys. +public static class IdempotencyScopes +{ + /// Runs of a pipeline (including draft tests). + /// Pipeline key. + public static string Pipeline(string key) => $"pipeline:{key}"; + + /// Single-stage runs (including draft tests). + /// Stage key. + public static string Stage(string key) => $"stage:{key}"; + + /// Scope of a run kind and target. + /// Run kind. + /// Pipeline or stage key. + public static string For(RunKind kind, string key) => kind is RunKind.Pipeline or RunKind.PipelineTest ? Pipeline(key) : Stage(key); +} + +/// Writes messages to psr.outbox_messages in the same transaction as the change that produced them. +public interface IOutboxWriter +{ + /// Adds the message to the context; it is stored by the caller's SaveChanges (and discarded by a rollback). + /// Context of the change. + /// The message. + void Enqueue(PsrDbContext db, OutboxMessage message); +} + +/// A message to publish after the transaction commits. +/// Type (e.g. psr.execute-step.v1). +/// Destination exchange. +/// Routing key. +/// Body. +public sealed record OutboxMessage(string MessageType, string Exchange, string RoutingKey, JsonNode Payload) +{ + /// Message identifier (also the AMQP message_id); new when omitted. + public Guid? Id { get; init; } + + /// Publish from this instant on (retry delays); now when omitted. + public DateTimeOffset? AvailableAt { get; init; } + + /// Headers (e.g. traceparent). + public IReadOnlyDictionary Headers { get; init; } = new Dictionary(); +} + +/// Reserves keys with INSERT ... ON CONFLICT: an expired key is taken over, a live one is kept. +internal sealed class IdempotencyStore(PsrEntityFrameworkOptions options, TimeProvider time) : IIdempotencyStore +{ + public async Task ReserveAsync(PsrDbContext db, string scope, string key, RunId runId, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(db); + ArgumentException.ThrowIfNullOrWhiteSpace(key); + var now = time.GetUtcNow(); + var reserved = await ScalarAsync(db, + """ + INSERT INTO psr.idempotency_keys (scope, key, run_id, run_created_at, expires_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (scope, key) DO UPDATE + SET run_id = EXCLUDED.run_id, run_created_at = EXCLUDED.run_created_at, expires_at = EXCLUDED.expires_at + WHERE psr.idempotency_keys.expires_at <= $6 + RETURNING run_id + """, + cancellationToken, scope, key, runId.Value, runId.CreatedAt, now + options.IdempotencyWindow, now).ConfigureAwait(false); + if (reserved == runId.Value) + { + return null; + } + + var existing = await ScalarAsync(db, "SELECT run_id FROM psr.idempotency_keys WHERE scope = $1 AND key = $2", cancellationToken, scope, key) + .ConfigureAwait(false); + return existing is { } id ? new RunId(id) : null; + } + + /// Runs a scalar command on the connection and transaction of the context. + internal static async Task ScalarAsync(PsrDbContext db, string sql, CancellationToken cancellationToken, params object[] parameters) + { + await db.Database.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + var command = new NpgsqlCommand(sql, (NpgsqlConnection)db.Database.GetDbConnection(), db.Database.CurrentTransaction?.GetDbTransaction() as NpgsqlTransaction); + await using (command.ConfigureAwait(false)) + { + foreach (var parameter in parameters) + { + command.Parameters.Add(new NpgsqlParameter { Value = parameter }); + } + + var value = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false); + return value is null or DBNull ? default : (T)value; + } + } +} + +/// Adds outbox rows to the caller's context. +internal sealed class OutboxWriter(TimeProvider time) : IOutboxWriter +{ + public void Enqueue(PsrDbContext db, OutboxMessage message) + { + ArgumentNullException.ThrowIfNull(db); + ArgumentNullException.ThrowIfNull(message); + var now = time.GetUtcNow(); + db.OutboxMessages.Add(new OutboxMessageEntity + { + Id = message.Id ?? Guid.CreateVersion7(now), + CreatedAt = now, + AvailableAt = message.AvailableAt ?? now, + MessageType = message.MessageType, + Exchange = message.Exchange, + RoutingKey = message.RoutingKey, + Payload = message.Payload.ToJsonString(), + Headers = new JsonObject(message.Headers.Select(h => KeyValuePair.Create(h.Key, (JsonNode?)JsonValue.Create(h.Value)))).ToJsonString(), + }); + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Maintenance/DatabaseHealthCheck.cs b/src/PythonScriptRunner.EntityFrameworkCore/Maintenance/DatabaseHealthCheck.cs new file mode 100644 index 0000000..b7c9cb0 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Maintenance/DatabaseHealthCheck.cs @@ -0,0 +1,78 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Npgsql; +using PythonScriptRunner.EntityFrameworkCore.Runs; + +namespace PythonScriptRunner.EntityFrameworkCore.Maintenance; + +/// +/// Database health: reachable, schema migrated (never migrates by itself) and at least two periods of partitions ahead +/// (2 weeks of history, 2 days of file contents, 2 months of audit). +/// +internal sealed class DatabaseHealthCheck(IDbContextFactory factory, PsrDataSource dataSource, TimeProvider time) : IHealthCheck +{ + private const string LatestPartitions = """ + SELECT parent.relname::text, + max(substring(pg_get_expr(child.relpartbound, child.oid) FROM 'TO \(''([^'']+)''\)')::timestamptz) + FROM pg_inherits inh + JOIN pg_class child ON child.oid = inh.inhrelid + JOIN pg_class parent ON parent.oid = inh.inhparent + JOIN pg_namespace ns ON ns.oid = parent.relnamespace + WHERE ns.nspname = 'psr' AND parent.relkind = 'p' AND child.relkind IN ('r', 'p') + GROUP BY parent.relname + """; + + private static readonly string[] Weekly = ["runs", "step_runs", "step_attempts", "attempt_logs", "artifacts"]; + + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + try + { + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var pending = (await db.Database.GetPendingMigrationsAsync(cancellationToken).ConfigureAwait(false)).ToList(); + if (pending.Count > 0) + { + return new HealthCheckResult(context.Registration.FailureStatus, + $"psr schema pending: {pending.Count} migration(s) not applied ({string.Join(", ", pending)})."); + } + } + + var latest = new Dictionary(StringComparer.Ordinal); + var connection = await dataSource.DataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using (connection.ConfigureAwait(false)) + { + var command = new NpgsqlCommand(LatestPartitions, connection); + await using (command.ConfigureAwait(false)) + { + var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + await using (reader.ConfigureAwait(false)) + { + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + latest[reader.GetString(0)] = reader.GetFieldValue(1); + } + } + } + } + + var now = time.GetUtcNow(); + var missing = Weekly.Where(t => !Covers(latest, t, now.AddDays(14))) + .Concat(Covers(latest, "artifact_contents", now.AddDays(2)) ? [] : ["artifact_contents"]) + .Concat(Covers(latest, "audit_events", now.AddMonths(2)) ? [] : ["audit_events"]) + .ToList(); + return missing.Count > 0 + ? new HealthCheckResult(context.Registration.FailureStatus, + $"Fewer than two periods of partitions ahead for: {string.Join(", ", missing)} (is the maintenance job running?).") + : HealthCheckResult.Healthy("psr database ready."); + } + catch (Exception ex) when (ex is NpgsqlException or InvalidOperationException or TimeoutException) + { + return new HealthCheckResult(context.Registration.FailureStatus, "psr database unreachable.", ex); + } + } + + private static bool Covers(Dictionary latest, string table, DateTimeOffset until) => + latest.TryGetValue(table, out var upper) && upper >= until; +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Maintenance/MaintenanceService.cs b/src/PythonScriptRunner.EntityFrameworkCore/Maintenance/MaintenanceService.cs new file mode 100644 index 0000000..c2c02b4 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Maintenance/MaintenanceService.cs @@ -0,0 +1,32 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace PythonScriptRunner.EntityFrameworkCore.Maintenance; + +/// Runs at start and then every . +internal sealed class MaintenanceService( + PsrMaintenance maintenance, PsrEntityFrameworkOptions options, TimeProvider time, ILogger logger) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + using var timer = new PeriodicTimer(options.Maintenance.Interval, time); + do + { + try + { + await maintenance.RunOnceAsync(stoppingToken).ConfigureAwait(false); + } + catch (PostgresException ex) when (ex.SqlState is PostgresErrorCodes.UndefinedTable or PostgresErrorCodes.UndefinedFunction + or PostgresErrorCodes.InvalidSchemaName) + { + Log.SchemaMissing(logger); + } + catch (Exception ex) when (ex is NpgsqlException or InvalidOperationException or TimeoutException) + { + Log.Failed(logger, ex); + } + } + while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)); + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Maintenance/PsrMaintenance.cs b/src/PythonScriptRunner.EntityFrameworkCore/Maintenance/PsrMaintenance.cs new file mode 100644 index 0000000..61a53a4 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Maintenance/PsrMaintenance.cs @@ -0,0 +1,227 @@ +using System.Diagnostics.Metrics; +using Microsoft.Extensions.Logging; +using Npgsql; +using PythonScriptRunner.EntityFrameworkCore.Runs; + +namespace PythonScriptRunner.EntityFrameworkCore; + +/// Outcome of one maintenance cycle. +/// Partitions created ahead. +/// Partitions dropped beyond the global retention. +/// Unfinished runs marked Failed (Expired). +/// File contents deleted by a shortened retention. +/// Runs deleted by a shortened retention. +/// Published outbox messages removed. +/// Expired idempotency keys removed. +public sealed record MaintenanceReport( + int PartitionsCreated, + IReadOnlyList PartitionsDropped, + int RunsExpired, + int ContentsDeleted, + int RunsDeleted, + int OutboxDeleted, + int IdempotencyKeysDeleted); + +/// +/// Maintenance of the psr schema: creates partitions ahead (4 weeks, 7 days, 3 months), applies shortened retentions in +/// batches, ends unfinished runs that reached the file retention, drops partitions beyond the global retention and cleans the +/// outbox and idempotency keys. Only one instance runs at a time (PostgreSQL advisory lock). +/// +public sealed class PsrMaintenance +{ + /// Advisory lock key ("PSRMAINT"). + internal const long LockKey = 0x5053524D41494E54; + + private const string Terminal = "('Succeeded', 'Failed', 'Cancelled')"; + + private readonly PsrDataSource _dataSource; + private readonly PsrEntityFrameworkOptions _options; + private readonly TimeProvider _time; + private readonly ILogger _logger; + private readonly Counter _partitionsCreated; + private readonly Counter _partitionsDropped; + private readonly Counter _rowsDeleted; + + internal PsrMaintenance(PsrDataSource dataSource, PsrEntityFrameworkOptions options, TimeProvider time, IMeterFactory meters, ILogger logger) + { + _dataSource = dataSource; + _options = options; + _time = time; + _logger = logger; + var meter = meters.Create("PythonScriptRunner"); + _partitionsCreated = meter.CreateCounter("psr.maintenance.partitions.created", description: "Partitions created ahead."); + _partitionsDropped = meter.CreateCounter("psr.maintenance.partitions.dropped", description: "Partitions dropped by the retention."); + _rowsDeleted = meter.CreateCounter("psr.maintenance.rows.deleted", description: "Rows deleted by shortened retentions and cleanups."); + } + + /// Runs one cycle now. + /// Cancellation token. + /// The report, or null when another instance is running the cycle. + public async Task RunOnceAsync(CancellationToken cancellationToken = default) + { + var connection = await _dataSource.DataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + await using (connection.ConfigureAwait(false)) + { + if (!await ScalarAsync(connection, "SELECT pg_try_advisory_lock($1)", cancellationToken, LockKey).ConfigureAwait(false)) + { + Log.Skipped(_logger); + return null; + } + + try + { + var report = await RunCycleAsync(connection, _time.GetUtcNow(), cancellationToken).ConfigureAwait(false); + _partitionsCreated.Add(report.PartitionsCreated); + _partitionsDropped.Add(report.PartitionsDropped.Count); + _rowsDeleted.Add(report.ContentsDeleted + report.RunsDeleted + report.OutboxDeleted + report.IdempotencyKeysDeleted); + Log.Completed(_logger, report.PartitionsCreated, report.PartitionsDropped.Count, report.RunsExpired, report.ContentsDeleted, report.RunsDeleted); + return report; + } + finally + { + await ScalarAsync(connection, "SELECT pg_advisory_unlock($1)", CancellationToken.None, LockKey).ConfigureAwait(false); + } + } + } + + private async Task RunCycleAsync(NpgsqlConnection connection, DateTimeOffset now, CancellationToken cancellationToken) + { + var retention = _options.Retention; + var batch = _options.Maintenance.BatchSize; + var created = await ScalarAsync(connection, "SELECT psr.ensure_partitions($1, 4, 7, 3)", cancellationToken, now).ConfigureAwait(false); + + // QA-D6: an unfinished run that reaches the file retention (minus one day) ends Failed (Expired) before its files go. + var expired = await ScalarAsync(connection, + $""" + WITH expired AS ( + UPDATE psr.runs r + SET status = 'Failed', finished_at = $1, + error = jsonb_build_object('kind', 'Expired', 'code', 'RUN_EXPIRED', 'message', 'The run reached the file retention before finishing.', 'step', NULL) + FROM ( + SELECT r2.id, r2.created_at + FROM psr.runs r2 + LEFT JOIN psr.pipelines p ON p.id = r2.pipeline_id + WHERE r2.status IN ('Pending', 'Queued', 'Running', 'Cancelling') + AND r2.created_at <= $1 - (LEAST($2::interval, COALESCE(make_interval(days => p.artifact_retention_days), $2::interval)) - interval '1 day') + ) x + WHERE r.id = x.id AND r.created_at = x.created_at + RETURNING r.id, r.created_at), + steps AS ( + UPDATE psr.step_runs s + SET status = CASE WHEN s.status IN ('Pending', 'Queued') THEN 'Skipped' ELSE 'Failed' END, finished_at = $1 + FROM expired e + WHERE s.run_id = e.id AND s.run_created_at = e.created_at AND s.status IN ('Pending', 'Queued', 'Running')) + SELECT count(*) FROM expired + """, + cancellationToken, now, retention.Artifacts).ConfigureAwait(false); + + // Contents past their expiry (shortened by a pipeline, or in partitions about to be dropped) are deleted; metadata stays. + var contents = await RepeatAsync(connection, + """ + WITH expired AS ( + SELECT id, run_created_at, content_created_at FROM psr.artifacts + WHERE content_deleted_at IS NULL AND (content_expires_at <= $1 OR content_created_at <= $1 - $3::interval) + LIMIT $2), + removed AS ( + DELETE FROM psr.artifact_contents c USING expired e WHERE c.artifact_id = e.id AND c.created_at = e.content_created_at) + UPDATE psr.artifacts a SET content_deleted_at = $1 + FROM expired e WHERE a.id = e.id AND a.run_created_at = e.run_created_at + """, + batch, cancellationToken, now, batch, retention.Artifacts).ConfigureAwait(false); + + // Runs of pipelines with a shorter run retention (children go with the foreign keys). + var runs = await RepeatAsync(connection, + $""" + WITH expired AS ( + SELECT id, created_at FROM psr.runs WHERE expires_at <= $1 AND status IN {Terminal} LIMIT $2), + contents AS ( + DELETE FROM psr.artifact_contents c USING psr.artifacts a, expired e + WHERE a.run_id = e.id AND a.run_created_at = e.created_at AND c.artifact_id = a.id AND c.created_at = a.content_created_at) + DELETE FROM psr.runs r USING expired e WHERE r.id = e.id AND r.created_at = e.created_at + """, + batch, cancellationToken, now, batch).ConfigureAwait(false); + + var dropped = new List(); + var command = new NpgsqlCommand("SELECT * FROM psr.drop_expired_partitions($1, $2, $3, $4)", connection); + await using (command.ConfigureAwait(false)) + { + command.Parameters.Add(new NpgsqlParameter { Value = now }); + command.Parameters.Add(new NpgsqlParameter { Value = retention.Runs }); + command.Parameters.Add(new NpgsqlParameter { Value = retention.Artifacts }); + command.Parameters.Add(new NpgsqlParameter { Value = retention.Audit }); + var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false); + await using (reader.ConfigureAwait(false)) + { + while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + dropped.Add(reader.GetString(0)); + } + } + } + + var outbox = await ExecuteAsync(connection, "DELETE FROM psr.outbox_messages WHERE published_at <= $1", cancellationToken, now - TimeSpan.FromHours(24)) + .ConfigureAwait(false); + var keys = await ExecuteAsync(connection, "DELETE FROM psr.idempotency_keys WHERE expires_at <= $1", cancellationToken, now).ConfigureAwait(false); + return new MaintenanceReport(created, dropped, (int)expired, contents, runs, outbox, keys); + } + + /// Repeats a batched statement until it affects fewer rows than a batch. + private static async Task RepeatAsync(NpgsqlConnection connection, string sql, int batch, CancellationToken cancellationToken, params object[] parameters) + { + var total = 0; + int affected; + do + { + affected = await ExecuteAsync(connection, sql, cancellationToken, parameters).ConfigureAwait(false); + total += affected; + } + while (affected >= batch); + return total; + } + + private static async Task ExecuteAsync(NpgsqlConnection connection, string sql, CancellationToken cancellationToken, params object[] parameters) + { + var command = Command(connection, sql, parameters); + await using (command.ConfigureAwait(false)) + { + return await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + } + + private static async Task ScalarAsync(NpgsqlConnection connection, string sql, CancellationToken cancellationToken, params object[] parameters) + { + var command = Command(connection, sql, parameters); + await using (command.ConfigureAwait(false)) + { + return (T)(await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false))!; + } + } + + private static NpgsqlCommand Command(NpgsqlConnection connection, string sql, object[] parameters) + { + var command = new NpgsqlCommand(sql, connection); + foreach (var parameter in parameters) + { + command.Parameters.Add(new NpgsqlParameter { Value = parameter }); + } + + return command; + } +} + +/// Log messages of the persistence package. +internal static partial class Log +{ + [LoggerMessage(101, LogLevel.Debug, "Maintenance skipped: another instance holds the lock")] + public static partial void Skipped(ILogger logger); + + [LoggerMessage(102, LogLevel.Information, + "Maintenance: {Created} partitions created, {Dropped} dropped, {Expired} runs expired, {Contents} file contents and {Runs} runs deleted")] + public static partial void Completed(ILogger logger, int created, int dropped, int expired, int contents, int runs); + + [LoggerMessage(103, LogLevel.Error, "Maintenance cycle failed")] + public static partial void Failed(ILogger logger, Exception exception); + + [LoggerMessage(104, LogLevel.Warning, "Maintenance skipped: the psr schema is not migrated (call MigratePythonScriptRunnerAsync)")] + public static partial void SchemaMissing(ILogger logger); +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Migrations/.editorconfig b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/.editorconfig new file mode 100644 index 0000000..d1e578d --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/.editorconfig @@ -0,0 +1,4 @@ +# EF Core migrations are generated code (reviewed, but not restyled). +[*.cs] +generated_code = true +dotnet_analyzer_diagnostic.severity = none diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Migrations/20261019221743_InitialSchema.Designer.cs b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/20261019221743_InitialSchema.Designer.cs new file mode 100644 index 0000000..7050cf8 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/20261019221743_InitialSchema.Designer.cs @@ -0,0 +1,1347 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using PythonScriptRunner.EntityFrameworkCore; + +#nullable disable + +namespace PythonScriptRunner.EntityFrameworkCore.Migrations +{ + [DbContext(typeof(PsrDbContext))] + [Migration("20261019221743_InitialSchema")] + partial class InitialSchema + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("psr") + .HasAnnotation("ProductVersion", "10.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.ArtifactContentEntity", b => + { + b.Property("ArtifactId") + .HasColumnType("uuid") + .HasColumnName("artifact_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("Content") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("content"); + + b.HasKey("ArtifactId", "CreatedAt") + .HasName("pk_artifact_contents"); + + b.ToTable("artifact_contents", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.ArtifactEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("RunCreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("run_created_at"); + + b.Property("ContentCreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("content_created_at"); + + b.Property("ContentDeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("content_deleted_at"); + + b.Property("ContentExpiresAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("content_expires_at"); + + b.Property("ContentType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("content_type"); + + b.Property("Direction") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("character varying(8)") + .HasColumnName("direction"); + + b.Property("FieldName") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("field_name"); + + b.Property("FileName") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("file_name"); + + b.Property("ListIndex") + .HasColumnType("integer") + .HasColumnName("list_index"); + + b.Property("RunId") + .HasColumnType("uuid") + .HasColumnName("run_id"); + + b.Property("Sha256") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character(64)") + .HasColumnName("sha256") + .IsFixedLength(); + + b.Property("SizeBytes") + .HasColumnType("bigint") + .HasColumnName("size_bytes"); + + b.Property("StepRunId") + .HasColumnType("uuid") + .HasColumnName("step_run_id"); + + b.HasKey("Id", "RunCreatedAt") + .HasName("pk_artifacts"); + + b.HasIndex("ContentExpiresAt") + .HasDatabaseName("ix_artifacts_content_expires_at") + .HasFilter("content_deleted_at IS NULL"); + + b.HasIndex("RunId", "RunCreatedAt") + .HasDatabaseName("ix_artifacts_run_id_run_created_at"); + + b.HasIndex("StepRunId", "RunCreatedAt") + .HasDatabaseName("ix_artifacts_step_run_id_run_created_at"); + + b.ToTable("artifacts", "psr", t => + { + t.HasCheckConstraint("ck_artifacts_direction", "direction IN ('Input', 'Output')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.AttemptLogEntity", b => + { + b.Property("AttemptId") + .HasColumnType("uuid") + .HasColumnName("attempt_id"); + + b.Property("RunCreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("run_created_at"); + + b.Property("Stream") + .HasMaxLength(8) + .HasColumnType("character varying(8)") + .HasColumnName("stream"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property("OriginalBytes") + .HasColumnType("bigint") + .HasColumnName("original_bytes"); + + b.Property("Truncated") + .HasColumnType("boolean") + .HasColumnName("truncated"); + + b.HasKey("AttemptId", "RunCreatedAt", "Stream") + .HasName("pk_attempt_logs"); + + b.ToTable("attempt_logs", "psr", t => + { + t.HasCheckConstraint("ck_attempt_logs_stream", "stream IN ('stdout', 'stderr')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.AuditEventEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("OccurredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("occurred_at"); + + b.Property("Action") + .IsRequired() + .HasMaxLength(60) + .HasColumnType("character varying(60)") + .HasColumnName("action"); + + b.Property("Actor") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("actor"); + + b.Property("Details") + .HasColumnType("jsonb") + .HasColumnName("details"); + + b.Property("EntityId") + .HasColumnType("uuid") + .HasColumnName("entity_id"); + + b.Property("EntityKey") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("entity_key"); + + b.Property("EntityType") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("character varying(30)") + .HasColumnName("entity_type"); + + b.Property("TraceId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("trace_id"); + + b.HasKey("Id", "OccurredAt") + .HasName("pk_audit_events"); + + b.HasIndex("OccurredAt", "Id") + .IsDescending() + .HasDatabaseName("ix_audit_events_occurred_at_id"); + + b.ToTable("audit_events", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.IdempotencyKeyEntity", b => + { + b.Property("Scope") + .HasMaxLength(80) + .HasColumnType("character varying(80)") + .HasColumnName("scope"); + + b.Property("Key") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("key"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expires_at"); + + b.Property("RunCreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("run_created_at"); + + b.Property("RunId") + .HasColumnType("uuid") + .HasColumnName("run_id"); + + b.HasKey("Scope", "Key") + .HasName("pk_idempotency_keys"); + + b.HasIndex("ExpiresAt") + .HasDatabaseName("ix_idempotency_keys_expires_at"); + + b.ToTable("idempotency_keys", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.OutboxMessageEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Attempts") + .HasColumnType("integer") + .HasColumnName("attempts"); + + b.Property("AvailableAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("available_at"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("Exchange") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("exchange"); + + b.Property("Headers") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("headers"); + + b.Property("LastError") + .HasColumnType("text") + .HasColumnName("last_error"); + + b.Property("MessageType") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("message_type"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("payload"); + + b.Property("PublishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("published_at"); + + b.Property("RoutingKey") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("routing_key"); + + b.HasKey("Id") + .HasName("pk_outbox_messages"); + + b.HasIndex("AvailableAt") + .HasDatabaseName("ix_outbox_messages_available_at") + .HasFilter("published_at IS NULL"); + + b.ToTable("outbox_messages", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ArtifactRetentionDays") + .HasColumnType("integer") + .HasColumnName("artifact_retention_days"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("key"); + + b.Property("LatestPublishedVersion") + .HasColumnType("integer") + .HasColumnName("latest_published_version"); + + b.Property("MaxConcurrentRuns") + .HasColumnType("integer") + .HasColumnName("max_concurrent_runs"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + + b.Property("RunRetentionDays") + .HasColumnType("integer") + .HasColumnName("run_retention_days"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UpdatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("updated_by"); + + b.HasKey("Id") + .HasName("pk_pipelines"); + + b.HasIndex("Key") + .IsUnique() + .HasDatabaseName("ux_pipelines_key"); + + b.ToTable("pipelines", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineStepEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Bindings") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("bindings"); + + b.Property("DisplayName") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("display_name"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("key"); + + b.Property("Order") + .HasColumnType("integer") + .HasColumnName("order"); + + b.Property("PipelineVersionId") + .HasColumnType("uuid") + .HasColumnName("pipeline_version_id"); + + b.Property("StageVersionId") + .HasColumnType("uuid") + .HasColumnName("stage_version_id"); + + b.HasKey("Id") + .HasName("pk_pipeline_steps"); + + b.HasIndex("StageVersionId") + .HasDatabaseName("ix_pipeline_steps_stage_version_id"); + + b.HasIndex("PipelineVersionId", "Key") + .IsUnique() + .HasDatabaseName("ux_pipeline_steps_pipeline_version_id_key"); + + b.HasIndex("PipelineVersionId", "Order") + .IsUnique() + .HasDatabaseName("ux_pipeline_steps_pipeline_version_id_order"); + + b.ToTable("pipeline_steps", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineVersionEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ArchivedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("archived_at"); + + b.Property("ArchivedBy") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("archived_by"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("InputFields") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("input_fields"); + + b.Property("Layout") + .HasColumnType("jsonb") + .HasColumnName("layout"); + + b.Property("Outputs") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("outputs"); + + b.Property("PipelineId") + .HasColumnType("uuid") + .HasColumnName("pipeline_id"); + + b.Property("PublishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("published_at"); + + b.Property("PublishedBy") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("published_by"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + + b.Property("Status") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasDefaultValue("Draft") + .HasColumnName("status"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UpdatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("updated_by"); + + b.Property("Version") + .HasColumnType("integer") + .HasColumnName("version"); + + b.HasKey("Id") + .HasName("pk_pipeline_versions"); + + b.HasIndex("PipelineId") + .IsUnique() + .HasDatabaseName("ux_pipeline_versions_draft") + .HasFilter("status = 'Draft'"); + + b.HasIndex("PipelineId", "Version") + .IsUnique() + .HasDatabaseName("ux_pipeline_versions_pipeline_id_version"); + + b.ToTable("pipeline_versions", "psr", t => + { + t.HasCheckConstraint("ck_pipeline_versions_status", "status IN ('Draft', 'Published', 'Archived')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.RunEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CancelRequestedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("cancel_requested_at"); + + b.Property("CancelRequestedBy") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("cancel_requested_by"); + + b.Property("CorrelationId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("correlation_id"); + + b.Property("Error") + .HasColumnType("jsonb") + .HasColumnName("error"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expires_at"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_at"); + + b.Property("IdempotencyKey") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("idempotency_key"); + + b.Property("Inputs") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("inputs"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("kind"); + + b.Property("Mode") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("character varying(8)") + .HasColumnName("mode"); + + b.Property("Outputs") + .HasColumnType("jsonb") + .HasColumnName("outputs"); + + b.Property("PipelineId") + .HasColumnType("uuid") + .HasColumnName("pipeline_id"); + + b.Property("PipelineKey") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("pipeline_key"); + + b.Property("PipelineVersion") + .HasColumnType("integer") + .HasColumnName("pipeline_version"); + + b.Property("PipelineVersionId") + .HasColumnType("uuid") + .HasColumnName("pipeline_version_id"); + + b.Property("QueuedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("queued_at"); + + b.Property("ScheduleId") + .HasColumnType("uuid") + .HasColumnName("schedule_id"); + + b.Property("StageKey") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("stage_key"); + + b.Property("StageVersion") + .HasColumnType("integer") + .HasColumnName("stage_version"); + + b.Property("StageVersionId") + .HasColumnType("uuid") + .HasColumnName("stage_version_id"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("status"); + + b.Property("TraceParent") + .HasMaxLength(55) + .HasColumnType("character varying(55)") + .HasColumnName("trace_parent"); + + b.Property("Trigger") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("trigger"); + + b.Property("TriggeredBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("triggered_by"); + + b.HasKey("Id", "CreatedAt") + .HasName("pk_runs"); + + b.HasIndex("ExpiresAt") + .HasDatabaseName("ix_runs_expires_at") + .HasFilter("expires_at IS NOT NULL"); + + b.HasIndex("CreatedAt", "Id") + .IsDescending() + .HasDatabaseName("ix_runs_created_at_id"); + + b.HasIndex("Status", "CreatedAt") + .HasDatabaseName("ix_runs_status_created_at") + .HasFilter("status IN ('Pending', 'Queued', 'Running', 'Cancelling')"); + + b.HasIndex("PipelineId", "CreatedAt", "Id") + .IsDescending(false, true, true) + .HasDatabaseName("ix_runs_pipeline_id_created_at_id"); + + b.HasIndex("StageKey", "CreatedAt", "Id") + .IsDescending(false, true, true) + .HasDatabaseName("ix_runs_stage_key_created_at_id") + .HasFilter("stage_key IS NOT NULL"); + + b.ToTable("runs", "psr", t => + { + t.HasCheckConstraint("ck_runs_kind", "kind IN ('Pipeline', 'PipelineTest', 'Stage', 'StageTest')"); + + t.HasCheckConstraint("ck_runs_mode", "mode IN ('Sync', 'Async')"); + + t.HasCheckConstraint("ck_runs_status", "status IN ('Pending', 'Queued', 'Running', 'Succeeded', 'Failed', 'Cancelling', 'Cancelled')"); + + t.HasCheckConstraint("ck_runs_trigger", "trigger IN ('Library', 'Dashboard', 'Schedule', 'QueueCommand')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.ScheduleEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("Cron") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("cron"); + + b.Property("Inputs") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("jsonb") + .HasColumnName("inputs") + .HasDefaultValueSql("'{}'::jsonb"); + + b.Property("IsEnabled") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("is_enabled"); + + b.Property("LastError") + .HasColumnType("text") + .HasColumnName("last_error"); + + b.Property("LastFiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_fired_at"); + + b.Property("LastRunId") + .HasColumnType("uuid") + .HasColumnName("last_run_id"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.Property("NextRunAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("next_run_at"); + + b.Property("PinnedVersion") + .HasColumnType("integer") + .HasColumnName("pinned_version"); + + b.Property("PipelineId") + .HasColumnType("uuid") + .HasColumnName("pipeline_id"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + + b.Property("TimeZone") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasDefaultValue("America/Sao_Paulo") + .HasColumnName("time_zone"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UpdatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("updated_by"); + + b.HasKey("Id") + .HasName("pk_schedules"); + + b.HasIndex("NextRunAt") + .HasDatabaseName("ix_schedules_next_run_at") + .HasFilter("is_enabled"); + + b.HasIndex("PipelineId") + .HasDatabaseName("ix_schedules_pipeline_id"); + + b.ToTable("schedules", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StageEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("key"); + + b.Property("LatestPublishedVersion") + .HasColumnType("integer") + .HasColumnName("latest_published_version"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UpdatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("updated_by"); + + b.HasKey("Id") + .HasName("pk_stages"); + + b.HasIndex("Key") + .IsUnique() + .HasDatabaseName("ux_stages_key"); + + b.ToTable("stages", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StageVersionEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ArchivedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("archived_at"); + + b.Property("ArchivedBy") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("archived_by"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("EnvironmentHash") + .HasMaxLength(64) + .HasColumnType("character(64)") + .HasColumnName("environment_hash") + .IsFixedLength(); + + b.Property("Idempotent") + .HasColumnType("boolean") + .HasColumnName("idempotent"); + + b.Property("InputFields") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("input_fields"); + + b.Property("LockedRequirements") + .HasColumnType("text") + .HasColumnName("locked_requirements"); + + b.Property("MemoryLimitMb") + .HasColumnType("integer") + .HasColumnName("memory_limit_mb"); + + b.Property("OutputFields") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("output_fields"); + + b.Property("PublishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("published_at"); + + b.Property("PublishedBy") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("published_by"); + + b.Property("PythonVersion") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("python_version"); + + b.PrimitiveCollection("RequiredSecrets") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("required_secrets"); + + b.Property("Requirements") + .HasColumnType("text") + .HasColumnName("requirements"); + + b.Property("RetryBackoffMultiplier") + .HasPrecision(4, 2) + .HasColumnType("numeric(4,2)") + .HasColumnName("retry_backoff_multiplier"); + + b.Property("RetryInitialDelaySeconds") + .HasColumnType("integer") + .HasColumnName("retry_initial_delay_seconds"); + + b.Property("RetryMaxAttempts") + .HasColumnType("integer") + .HasColumnName("retry_max_attempts"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + + b.Property("Script") + .IsRequired() + .HasColumnType("text") + .HasColumnName("script"); + + b.Property("ScriptSha256") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character(64)") + .HasColumnName("script_sha256") + .IsFixedLength(); + + b.Property("StageId") + .HasColumnType("uuid") + .HasColumnName("stage_id"); + + b.Property("Status") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasDefaultValue("Draft") + .HasColumnName("status"); + + b.Property("TimeoutSeconds") + .HasColumnType("integer") + .HasColumnName("timeout_seconds"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UpdatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("updated_by"); + + b.Property("Version") + .HasColumnType("integer") + .HasColumnName("version"); + + b.HasKey("Id") + .HasName("pk_stage_versions"); + + b.HasIndex("StageId") + .IsUnique() + .HasDatabaseName("ux_stage_versions_draft") + .HasFilter("status = 'Draft'"); + + b.HasIndex("StageId", "Version") + .IsUnique() + .HasDatabaseName("ux_stage_versions_stage_id_version"); + + b.ToTable("stage_versions", "psr", t => + { + t.HasCheckConstraint("ck_stage_versions_status", "status IN ('Draft', 'Published', 'Archived')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StepAttemptEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("RunCreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("run_created_at"); + + b.Property("Attempt") + .HasColumnType("integer") + .HasColumnName("attempt"); + + b.Property("Cause") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("cause"); + + b.Property("DurationMs") + .HasColumnType("integer") + .HasColumnName("duration_ms"); + + b.Property("EnvironmentHash") + .HasMaxLength(64) + .HasColumnType("character(64)") + .HasColumnName("environment_hash") + .IsFixedLength(); + + b.Property("Error") + .HasColumnType("jsonb") + .HasColumnName("error"); + + b.Property("ErrorKind") + .HasMaxLength(40) + .HasColumnType("character varying(40)") + .HasColumnName("error_kind"); + + b.Property("ExitCode") + .HasColumnType("integer") + .HasColumnName("exit_code"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_at"); + + b.Property("LeaseExpiresAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("lease_expires_at"); + + b.Property("OverheadMs") + .HasColumnType("integer") + .HasColumnName("overhead_ms"); + + b.Property("RunId") + .HasColumnType("uuid") + .HasColumnName("run_id"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("status"); + + b.Property("StepRunId") + .HasColumnType("uuid") + .HasColumnName("step_run_id"); + + b.Property("TriggeredBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("triggered_by"); + + b.Property("WorkerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("worker_id"); + + b.HasKey("Id", "RunCreatedAt") + .HasName("pk_step_attempts"); + + b.HasIndex("LeaseExpiresAt") + .HasDatabaseName("ix_step_attempts_lease_expires_at") + .HasFilter("status = 'Running'"); + + b.HasIndex("StepRunId", "RunCreatedAt") + .HasDatabaseName("ix_step_attempts_step_run_id_run_created_at"); + + b.ToTable("step_attempts", "psr", t => + { + t.HasCheckConstraint("ck_step_attempts_cause", "cause IN ('Initial', 'AutoRetry', 'Reprocess', 'LeaseRecovery')"); + + t.HasCheckConstraint("ck_step_attempts_status", "status IN ('Running', 'Succeeded', 'Failed', 'TimedOut', 'Cancelled', 'WorkerLost')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StepRunEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("RunCreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("run_created_at"); + + b.Property("CurrentAttempt") + .HasColumnType("integer") + .HasColumnName("current_attempt"); + + b.Property("Error") + .HasColumnType("jsonb") + .HasColumnName("error"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_at"); + + b.Property("Inputs") + .HasColumnType("jsonb") + .HasColumnName("inputs"); + + b.Property("Order") + .HasColumnType("integer") + .HasColumnName("order"); + + b.Property("Outputs") + .HasColumnType("jsonb") + .HasColumnName("outputs"); + + b.Property("PipelineStepId") + .HasColumnType("uuid") + .HasColumnName("pipeline_step_id"); + + b.Property("RunId") + .HasColumnType("uuid") + .HasColumnName("run_id"); + + b.Property("StageKey") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("stage_key"); + + b.Property("StageVersion") + .HasColumnType("integer") + .HasColumnName("stage_version"); + + b.Property("StageVersionId") + .HasColumnType("uuid") + .HasColumnName("stage_version_id"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("status"); + + b.Property("StepKey") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("step_key"); + + b.HasKey("Id", "RunCreatedAt") + .HasName("pk_step_runs"); + + b.HasIndex("RunId", "RunCreatedAt", "Order") + .IsUnique() + .HasDatabaseName("ux_step_runs_run_id_run_created_at_order"); + + b.ToTable("step_runs", "psr", t => + { + t.HasCheckConstraint("ck_step_runs_status", "status IN ('Pending', 'Queued', 'Running', 'Succeeded', 'Failed', 'TimedOut', 'Cancelled', 'Skipped')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.ArtifactEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.RunEntity", null) + .WithMany() + .HasForeignKey("RunId", "RunCreatedAt") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_artifacts_runs_run_id_run_created_at"); + + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.StepRunEntity", null) + .WithMany() + .HasForeignKey("StepRunId", "RunCreatedAt") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_artifacts_step_runs_step_run_id_run_created_at"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.AttemptLogEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.StepAttemptEntity", null) + .WithMany() + .HasForeignKey("AttemptId", "RunCreatedAt") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_attempt_logs_step_attempts_attempt_id_run_created_at"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineStepEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineVersionEntity", null) + .WithMany("Steps") + .HasForeignKey("PipelineVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_pipeline_steps_pipeline_versions_pipeline_version_id"); + + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.StageVersionEntity", "StageVersion") + .WithMany() + .HasForeignKey("StageVersionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_pipeline_steps_stage_versions_stage_version_id"); + + b.Navigation("StageVersion"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineVersionEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineEntity", "Pipeline") + .WithMany("Versions") + .HasForeignKey("PipelineId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_pipeline_versions_pipelines_pipeline_id"); + + b.Navigation("Pipeline"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.ScheduleEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineEntity", null) + .WithMany() + .HasForeignKey("PipelineId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_schedules_pipelines_pipeline_id"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StageVersionEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.StageEntity", "Stage") + .WithMany("Versions") + .HasForeignKey("StageId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_stage_versions_stages_stage_id"); + + b.Navigation("Stage"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StepAttemptEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.StepRunEntity", null) + .WithMany("Attempts") + .HasForeignKey("StepRunId", "RunCreatedAt") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_step_attempts_step_runs_step_run_id_run_created_at"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StepRunEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.RunEntity", null) + .WithMany("Steps") + .HasForeignKey("RunId", "RunCreatedAt") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_step_runs_runs_run_id_run_created_at"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineEntity", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineVersionEntity", b => + { + b.Navigation("Steps"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.RunEntity", b => + { + b.Navigation("Steps"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StageEntity", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StepRunEntity", b => + { + b.Navigation("Attempts"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Migrations/20261019221743_InitialSchema.cs b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/20261019221743_InitialSchema.cs new file mode 100644 index 0000000..ed09cc0 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/20261019221743_InitialSchema.cs @@ -0,0 +1,506 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace PythonScriptRunner.EntityFrameworkCore.Migrations +{ + /// + public partial class InitialSchema : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.EnsureSchema( + name: "psr"); + + migrationBuilder.Sql(PartitioningSql.ArtifactContents); + + migrationBuilder.Sql(PartitioningSql.AuditEvents); + + migrationBuilder.CreateTable( + name: "idempotency_keys", + schema: "psr", + columns: table => new + { + scope = table.Column(type: "character varying(80)", maxLength: 80, nullable: false), + key = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + run_id = table.Column(type: "uuid", nullable: false), + run_created_at = table.Column(type: "timestamp with time zone", nullable: false), + expires_at = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_idempotency_keys", x => new { x.scope, x.key }); + }); + + migrationBuilder.CreateTable( + name: "outbox_messages", + schema: "psr", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + available_at = table.Column(type: "timestamp with time zone", nullable: false), + message_type = table.Column(type: "character varying(100)", maxLength: 100, nullable: false), + exchange = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + routing_key = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + payload = table.Column(type: "jsonb", nullable: false), + headers = table.Column(type: "jsonb", nullable: false), + attempts = table.Column(type: "integer", nullable: false), + last_error = table.Column(type: "text", nullable: true), + published_at = table.Column(type: "timestamp with time zone", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("pk_outbox_messages", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "pipelines", + schema: "psr", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + key = table.Column(type: "character varying(64)", maxLength: 64, nullable: false), + name = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + description = table.Column(type: "text", nullable: true), + latest_published_version = table.Column(type: "integer", nullable: true), + max_concurrent_runs = table.Column(type: "integer", nullable: true), + artifact_retention_days = table.Column(type: "integer", nullable: true), + run_retention_days = table.Column(type: "integer", nullable: true), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + created_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + updated_at = table.Column(type: "timestamp with time zone", nullable: false), + updated_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + xmin = table.Column(type: "xid", rowVersion: true, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_pipelines", x => x.id); + }); + + migrationBuilder.Sql(PartitioningSql.Runs); + + migrationBuilder.CreateTable( + name: "stages", + schema: "psr", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + key = table.Column(type: "character varying(64)", maxLength: 64, nullable: false), + name = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + description = table.Column(type: "text", nullable: true), + latest_published_version = table.Column(type: "integer", nullable: true), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + created_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + updated_at = table.Column(type: "timestamp with time zone", nullable: false), + updated_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + xmin = table.Column(type: "xid", rowVersion: true, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_stages", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "pipeline_versions", + schema: "psr", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + pipeline_id = table.Column(type: "uuid", nullable: false), + version = table.Column(type: "integer", nullable: false), + status = table.Column(type: "character varying(16)", maxLength: 16, nullable: false, defaultValue: "Draft"), + input_fields = table.Column(type: "jsonb", nullable: false), + outputs = table.Column(type: "jsonb", nullable: false), + layout = table.Column(type: "jsonb", nullable: true), + published_at = table.Column(type: "timestamp with time zone", nullable: true), + published_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: true), + archived_at = table.Column(type: "timestamp with time zone", nullable: true), + archived_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: true), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + created_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + updated_at = table.Column(type: "timestamp with time zone", nullable: false), + updated_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + xmin = table.Column(type: "xid", rowVersion: true, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_pipeline_versions", x => x.id); + table.CheckConstraint("ck_pipeline_versions_status", "status IN ('Draft', 'Published', 'Archived')"); + table.ForeignKey( + name: "fk_pipeline_versions_pipelines_pipeline_id", + column: x => x.pipeline_id, + principalSchema: "psr", + principalTable: "pipelines", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "schedules", + schema: "psr", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + pipeline_id = table.Column(type: "uuid", nullable: false), + name = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + cron = table.Column(type: "character varying(100)", maxLength: 100, nullable: false), + time_zone = table.Column(type: "character varying(64)", maxLength: 64, nullable: false, defaultValue: "America/Sao_Paulo"), + pinned_version = table.Column(type: "integer", nullable: true), + inputs = table.Column(type: "jsonb", nullable: false, defaultValueSql: "'{}'::jsonb"), + is_enabled = table.Column(type: "boolean", nullable: false, defaultValue: true), + next_run_at = table.Column(type: "timestamp with time zone", nullable: true), + last_fired_at = table.Column(type: "timestamp with time zone", nullable: true), + last_run_id = table.Column(type: "uuid", nullable: true), + last_error = table.Column(type: "text", nullable: true), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + created_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + updated_at = table.Column(type: "timestamp with time zone", nullable: false), + updated_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + xmin = table.Column(type: "xid", rowVersion: true, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_schedules", x => x.id); + table.ForeignKey( + name: "fk_schedules_pipelines_pipeline_id", + column: x => x.pipeline_id, + principalSchema: "psr", + principalTable: "pipelines", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.Sql(PartitioningSql.StepRuns); + + migrationBuilder.CreateTable( + name: "stage_versions", + schema: "psr", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + stage_id = table.Column(type: "uuid", nullable: false), + version = table.Column(type: "integer", nullable: false), + status = table.Column(type: "character varying(16)", maxLength: 16, nullable: false, defaultValue: "Draft"), + script = table.Column(type: "text", nullable: false), + script_sha256 = table.Column(type: "character(64)", fixedLength: true, maxLength: 64, nullable: false), + python_version = table.Column(type: "character varying(10)", maxLength: 10, nullable: false), + requirements = table.Column(type: "text", nullable: true), + locked_requirements = table.Column(type: "text", nullable: true), + environment_hash = table.Column(type: "character(64)", fixedLength: true, maxLength: 64, nullable: true), + input_fields = table.Column(type: "jsonb", nullable: false), + output_fields = table.Column(type: "jsonb", nullable: false), + required_secrets = table.Column(type: "text[]", nullable: false), + timeout_seconds = table.Column(type: "integer", nullable: false), + memory_limit_mb = table.Column(type: "integer", nullable: true), + retry_max_attempts = table.Column(type: "integer", nullable: false), + retry_initial_delay_seconds = table.Column(type: "integer", nullable: false), + retry_backoff_multiplier = table.Column(type: "numeric(4,2)", precision: 4, scale: 2, nullable: false), + idempotent = table.Column(type: "boolean", nullable: false), + published_at = table.Column(type: "timestamp with time zone", nullable: true), + published_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: true), + archived_at = table.Column(type: "timestamp with time zone", nullable: true), + archived_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: true), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + created_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + updated_at = table.Column(type: "timestamp with time zone", nullable: false), + updated_by = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + xmin = table.Column(type: "xid", rowVersion: true, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_stage_versions", x => x.id); + table.CheckConstraint("ck_stage_versions_status", "status IN ('Draft', 'Published', 'Archived')"); + table.ForeignKey( + name: "fk_stage_versions_stages_stage_id", + column: x => x.stage_id, + principalSchema: "psr", + principalTable: "stages", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.Sql(PartitioningSql.Artifacts); + + migrationBuilder.Sql(PartitioningSql.StepAttempts); + + migrationBuilder.CreateTable( + name: "pipeline_steps", + schema: "psr", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + pipeline_version_id = table.Column(type: "uuid", nullable: false), + order = table.Column(type: "integer", nullable: false), + key = table.Column(type: "character varying(64)", maxLength: 64, nullable: false), + display_name = table.Column(type: "character varying(200)", maxLength: 200, nullable: true), + stage_version_id = table.Column(type: "uuid", nullable: false), + bindings = table.Column(type: "jsonb", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_pipeline_steps", x => x.id); + table.ForeignKey( + name: "fk_pipeline_steps_pipeline_versions_pipeline_version_id", + column: x => x.pipeline_version_id, + principalSchema: "psr", + principalTable: "pipeline_versions", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "fk_pipeline_steps_stage_versions_stage_version_id", + column: x => x.stage_version_id, + principalSchema: "psr", + principalTable: "stage_versions", + principalColumn: "id", + onDelete: ReferentialAction.Restrict); + }); + + migrationBuilder.Sql(PartitioningSql.AttemptLogs); + + migrationBuilder.CreateIndex( + name: "ix_artifacts_content_expires_at", + schema: "psr", + table: "artifacts", + column: "content_expires_at", + filter: "content_deleted_at IS NULL"); + + migrationBuilder.CreateIndex( + name: "ix_artifacts_run_id_run_created_at", + schema: "psr", + table: "artifacts", + columns: new[] { "run_id", "run_created_at" }); + + migrationBuilder.CreateIndex( + name: "ix_artifacts_step_run_id_run_created_at", + schema: "psr", + table: "artifacts", + columns: new[] { "step_run_id", "run_created_at" }); + + migrationBuilder.CreateIndex( + name: "ix_audit_events_occurred_at_id", + schema: "psr", + table: "audit_events", + columns: new[] { "occurred_at", "id" }, + descending: new bool[0]); + + migrationBuilder.CreateIndex( + name: "ix_idempotency_keys_expires_at", + schema: "psr", + table: "idempotency_keys", + column: "expires_at"); + + migrationBuilder.CreateIndex( + name: "ix_outbox_messages_available_at", + schema: "psr", + table: "outbox_messages", + column: "available_at", + filter: "published_at IS NULL"); + + migrationBuilder.CreateIndex( + name: "ix_pipeline_steps_stage_version_id", + schema: "psr", + table: "pipeline_steps", + column: "stage_version_id"); + + migrationBuilder.CreateIndex( + name: "ux_pipeline_steps_pipeline_version_id_key", + schema: "psr", + table: "pipeline_steps", + columns: new[] { "pipeline_version_id", "key" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "ux_pipeline_steps_pipeline_version_id_order", + schema: "psr", + table: "pipeline_steps", + columns: new[] { "pipeline_version_id", "order" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "ux_pipeline_versions_draft", + schema: "psr", + table: "pipeline_versions", + column: "pipeline_id", + unique: true, + filter: "status = 'Draft'"); + + migrationBuilder.CreateIndex( + name: "ux_pipeline_versions_pipeline_id_version", + schema: "psr", + table: "pipeline_versions", + columns: new[] { "pipeline_id", "version" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "ux_pipelines_key", + schema: "psr", + table: "pipelines", + column: "key", + unique: true); + + migrationBuilder.CreateIndex( + name: "ix_runs_created_at_id", + schema: "psr", + table: "runs", + columns: new[] { "created_at", "id" }, + descending: new bool[0]); + + migrationBuilder.CreateIndex( + name: "ix_runs_expires_at", + schema: "psr", + table: "runs", + column: "expires_at", + filter: "expires_at IS NOT NULL"); + + migrationBuilder.CreateIndex( + name: "ix_runs_pipeline_id_created_at_id", + schema: "psr", + table: "runs", + columns: new[] { "pipeline_id", "created_at", "id" }, + descending: new[] { false, true, true }); + + migrationBuilder.CreateIndex( + name: "ix_runs_stage_key_created_at_id", + schema: "psr", + table: "runs", + columns: new[] { "stage_key", "created_at", "id" }, + descending: new[] { false, true, true }, + filter: "stage_key IS NOT NULL"); + + migrationBuilder.CreateIndex( + name: "ix_runs_status_created_at", + schema: "psr", + table: "runs", + columns: new[] { "status", "created_at" }, + filter: "status IN ('Pending', 'Queued', 'Running', 'Cancelling')"); + + migrationBuilder.CreateIndex( + name: "ix_schedules_next_run_at", + schema: "psr", + table: "schedules", + column: "next_run_at", + filter: "is_enabled"); + + migrationBuilder.CreateIndex( + name: "ix_schedules_pipeline_id", + schema: "psr", + table: "schedules", + column: "pipeline_id"); + + migrationBuilder.CreateIndex( + name: "ux_stage_versions_draft", + schema: "psr", + table: "stage_versions", + column: "stage_id", + unique: true, + filter: "status = 'Draft'"); + + migrationBuilder.CreateIndex( + name: "ux_stage_versions_stage_id_version", + schema: "psr", + table: "stage_versions", + columns: new[] { "stage_id", "version" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "ux_stages_key", + schema: "psr", + table: "stages", + column: "key", + unique: true); + + migrationBuilder.CreateIndex( + name: "ix_step_attempts_lease_expires_at", + schema: "psr", + table: "step_attempts", + column: "lease_expires_at", + filter: "status = 'Running'"); + + migrationBuilder.CreateIndex( + name: "ix_step_attempts_step_run_id_run_created_at", + schema: "psr", + table: "step_attempts", + columns: new[] { "step_run_id", "run_created_at" }); + + migrationBuilder.CreateIndex( + name: "ux_step_runs_run_id_run_created_at_order", + schema: "psr", + table: "step_runs", + columns: new[] { "run_id", "run_created_at", "order" }, + unique: true); + + migrationBuilder.Sql(PartitioningSql.EnsurePartitionsFunction); + migrationBuilder.Sql(PartitioningSql.DropExpiredPartitionsFunction); + migrationBuilder.Sql(PartitioningSql.CreateInitialPartitions); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(PartitioningSql.DropFunctions); + + migrationBuilder.DropTable( + name: "artifact_contents", + schema: "psr"); + + migrationBuilder.DropTable( + name: "artifacts", + schema: "psr"); + + migrationBuilder.DropTable( + name: "attempt_logs", + schema: "psr"); + + migrationBuilder.DropTable( + name: "audit_events", + schema: "psr"); + + migrationBuilder.DropTable( + name: "idempotency_keys", + schema: "psr"); + + migrationBuilder.DropTable( + name: "outbox_messages", + schema: "psr"); + + migrationBuilder.DropTable( + name: "pipeline_steps", + schema: "psr"); + + migrationBuilder.DropTable( + name: "schedules", + schema: "psr"); + + migrationBuilder.DropTable( + name: "step_attempts", + schema: "psr"); + + migrationBuilder.DropTable( + name: "pipeline_versions", + schema: "psr"); + + migrationBuilder.DropTable( + name: "stage_versions", + schema: "psr"); + + migrationBuilder.DropTable( + name: "step_runs", + schema: "psr"); + + migrationBuilder.DropTable( + name: "pipelines", + schema: "psr"); + + migrationBuilder.DropTable( + name: "stages", + schema: "psr"); + + migrationBuilder.DropTable( + name: "runs", + schema: "psr"); + } + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Migrations/DesignTimeFactory.cs b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/DesignTimeFactory.cs new file mode 100644 index 0000000..61b920e --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/DesignTimeFactory.cs @@ -0,0 +1,13 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; + +namespace PythonScriptRunner.EntityFrameworkCore.Migrations; + +/// Lets dotnet ef migrations add build the model without a host (no database connection is opened). +internal sealed class DesignTimeFactory : IDesignTimeDbContextFactory +{ + public PsrDbContext CreateDbContext(string[] args) => + new(new DbContextOptionsBuilder() + .UseNpgsql("Host=localhost;Database=psr_design", PsrDbContextOptions.ConfigureNpgsql) + .Options); +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Migrations/PartitioningSql.cs b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/PartitioningSql.cs new file mode 100644 index 0000000..c308ec9 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/PartitioningSql.cs @@ -0,0 +1,265 @@ +namespace PythonScriptRunner.EntityFrameworkCore.Migrations; + +/// +/// SQL that EF Core cannot model: history tables partitioned by range (weekly with the run, daily for file contents, +/// monthly for audit) and the functions that create and drop partitions (called by the migration and the maintenance job). +/// +internal static class PartitioningSql +{ + /// psr.runs, partitioned by range. + public const string Runs = """ + CREATE TABLE psr.runs ( + id uuid NOT NULL, + created_at timestamp with time zone NOT NULL, + kind character varying(16) NOT NULL, + mode character varying(8) NOT NULL, + pipeline_id uuid, + pipeline_version_id uuid, + pipeline_key character varying(64), + pipeline_version integer, + stage_version_id uuid, + stage_key character varying(64), + stage_version integer, + trigger character varying(16) NOT NULL, + triggered_by character varying(200) NOT NULL, + schedule_id uuid, + status character varying(16) NOT NULL, + inputs jsonb NOT NULL, + outputs jsonb, + error jsonb, + idempotency_key character varying(200), + correlation_id character varying(200), + trace_parent character varying(55), + queued_at timestamp with time zone, + started_at timestamp with time zone, + finished_at timestamp with time zone, + cancel_requested_at timestamp with time zone, + cancel_requested_by character varying(200), + expires_at timestamp with time zone, + CONSTRAINT pk_runs PRIMARY KEY (id, created_at), + CONSTRAINT ck_runs_kind CHECK (kind IN ('Pipeline', 'PipelineTest', 'Stage', 'StageTest')), + CONSTRAINT ck_runs_mode CHECK (mode IN ('Sync', 'Async')), + CONSTRAINT ck_runs_status CHECK (status IN ('Pending', 'Queued', 'Running', 'Succeeded', 'Failed', 'Cancelling', 'Cancelled')), + CONSTRAINT ck_runs_trigger CHECK (trigger IN ('Library', 'Dashboard', 'Schedule', 'QueueCommand')) + ) PARTITION BY RANGE (created_at); + """; + + /// psr.step_runs, partitioned by range. + public const string StepRuns = """ + CREATE TABLE psr.step_runs ( + id uuid NOT NULL, + run_created_at timestamp with time zone NOT NULL, + run_id uuid NOT NULL, + pipeline_step_id uuid, + step_key character varying(64) NOT NULL, + "order" integer NOT NULL, + stage_version_id uuid, + stage_key character varying(64) NOT NULL, + stage_version integer NOT NULL, + status character varying(16) NOT NULL, + current_attempt integer NOT NULL, + inputs jsonb, + outputs jsonb, + error jsonb, + started_at timestamp with time zone, + finished_at timestamp with time zone, + CONSTRAINT pk_step_runs PRIMARY KEY (id, run_created_at), + CONSTRAINT ck_step_runs_status CHECK (status IN ('Pending', 'Queued', 'Running', 'Succeeded', 'Failed', 'TimedOut', 'Cancelled', 'Skipped')), + CONSTRAINT fk_step_runs_runs_run_id_run_created_at FOREIGN KEY (run_id, run_created_at) REFERENCES psr.runs (id, created_at) ON DELETE CASCADE + ) PARTITION BY RANGE (run_created_at); + """; + + /// psr.step_attempts, partitioned by range. + public const string StepAttempts = """ + CREATE TABLE psr.step_attempts ( + id uuid NOT NULL, + run_created_at timestamp with time zone NOT NULL, + step_run_id uuid NOT NULL, + run_id uuid NOT NULL, + attempt integer NOT NULL, + cause character varying(16) NOT NULL, + triggered_by character varying(200) NOT NULL, + status character varying(16) NOT NULL, + worker_id character varying(200), + lease_expires_at timestamp with time zone, + environment_hash character(64), + exit_code integer, + error_kind character varying(40), + error jsonb, + started_at timestamp with time zone NOT NULL, + finished_at timestamp with time zone, + duration_ms integer, + overhead_ms integer, + CONSTRAINT pk_step_attempts PRIMARY KEY (id, run_created_at), + CONSTRAINT ck_step_attempts_cause CHECK (cause IN ('Initial', 'AutoRetry', 'Reprocess', 'LeaseRecovery')), + CONSTRAINT ck_step_attempts_status CHECK (status IN ('Running', 'Succeeded', 'Failed', 'TimedOut', 'Cancelled', 'WorkerLost')), + CONSTRAINT fk_step_attempts_step_runs_step_run_id_run_created_at FOREIGN KEY (step_run_id, run_created_at) REFERENCES psr.step_runs (id, run_created_at) ON DELETE CASCADE + ) PARTITION BY RANGE (run_created_at); + """; + + /// psr.attempt_logs, partitioned by range. + public const string AttemptLogs = """ + CREATE TABLE psr.attempt_logs ( + attempt_id uuid NOT NULL, + run_created_at timestamp with time zone NOT NULL, + stream character varying(8) NOT NULL, + content text NOT NULL, + truncated boolean NOT NULL, + original_bytes bigint NOT NULL, + CONSTRAINT pk_attempt_logs PRIMARY KEY (attempt_id, run_created_at, stream), + CONSTRAINT ck_attempt_logs_stream CHECK (stream IN ('stdout', 'stderr')), + CONSTRAINT fk_attempt_logs_step_attempts_attempt_id_run_created_at FOREIGN KEY (attempt_id, run_created_at) REFERENCES psr.step_attempts (id, run_created_at) ON DELETE CASCADE + ) PARTITION BY RANGE (run_created_at); + """; + + /// psr.artifacts, partitioned by range. + public const string Artifacts = """ + CREATE TABLE psr.artifacts ( + id uuid NOT NULL, + run_created_at timestamp with time zone NOT NULL, + run_id uuid NOT NULL, + step_run_id uuid, + direction character varying(8) NOT NULL, + field_name character varying(64) NOT NULL, + list_index integer, + file_name character varying(255) NOT NULL, + content_type character varying(255) NOT NULL, + size_bytes bigint NOT NULL, + sha256 character(64) NOT NULL, + content_created_at timestamp with time zone NOT NULL, + content_expires_at timestamp with time zone NOT NULL, + content_deleted_at timestamp with time zone, + CONSTRAINT pk_artifacts PRIMARY KEY (id, run_created_at), + CONSTRAINT ck_artifacts_direction CHECK (direction IN ('Input', 'Output')), + CONSTRAINT fk_artifacts_runs_run_id_run_created_at FOREIGN KEY (run_id, run_created_at) REFERENCES psr.runs (id, created_at) ON DELETE CASCADE, + CONSTRAINT fk_artifacts_step_runs_step_run_id_run_created_at FOREIGN KEY (step_run_id, run_created_at) REFERENCES psr.step_runs (id, run_created_at) ON DELETE CASCADE + ) PARTITION BY RANGE (run_created_at); + """; + + /// psr.artifact_contents, partitioned by range. + public const string ArtifactContents = """ + CREATE TABLE psr.artifact_contents ( + artifact_id uuid NOT NULL, + created_at timestamp with time zone NOT NULL, + content bytea NOT NULL, + CONSTRAINT pk_artifact_contents PRIMARY KEY (artifact_id, created_at) + ) PARTITION BY RANGE (created_at); + """; + + /// psr.audit_events, partitioned by range. + public const string AuditEvents = """ + CREATE TABLE psr.audit_events ( + id uuid NOT NULL, + occurred_at timestamp with time zone NOT NULL, + actor character varying(200) NOT NULL, + action character varying(60) NOT NULL, + entity_type character varying(30) NOT NULL, + entity_id uuid, + entity_key character varying(100), + details jsonb, + trace_id character varying(32), + CONSTRAINT pk_audit_events PRIMARY KEY (id, occurred_at) + ) PARTITION BY RANGE (occurred_at); + """; + + /// Creates the partitions of the current period and of the periods ahead (idempotent). + public const string EnsurePartitionsFunction = """ + CREATE OR REPLACE FUNCTION psr.ensure_partitions( + p_now timestamptz, p_weeks_ahead integer DEFAULT 4, p_days_ahead integer DEFAULT 7, p_months_ahead integer DEFAULT 3) + RETURNS integer + LANGUAGE plpgsql + AS $fn$ + DECLARE + v_created integer := 0; + v_table text; + v_start timestamptz; + v_end timestamptz; + v_name text; + v_week timestamp := date_trunc('week', p_now AT TIME ZONE 'UTC'); + v_day timestamp := date_trunc('day', p_now AT TIME ZONE 'UTC'); + v_month timestamp := date_trunc('month', p_now AT TIME ZONE 'UTC'); + i integer; + BEGIN + FOREACH v_table IN ARRAY ARRAY['runs', 'step_runs', 'step_attempts', 'attempt_logs', 'artifacts'] LOOP + FOR i IN 0..p_weeks_ahead LOOP + v_start := (v_week + make_interval(weeks => i)) AT TIME ZONE 'UTC'; + v_end := v_start + interval '7 days'; + v_name := v_table || '_p' || to_char(v_start AT TIME ZONE 'UTC', 'YYYYMMDD'); + IF to_regclass(format('psr.%I', v_name)) IS NULL THEN + EXECUTE format('CREATE TABLE psr.%I PARTITION OF psr.%I FOR VALUES FROM (%L) TO (%L)', v_name, v_table, v_start, v_end); + v_created := v_created + 1; + END IF; + END LOOP; + END LOOP; + + FOR i IN 0..p_days_ahead LOOP + v_start := (v_day + make_interval(days => i)) AT TIME ZONE 'UTC'; + v_end := v_start + interval '1 day'; + v_name := 'artifact_contents_p' || to_char(v_start AT TIME ZONE 'UTC', 'YYYYMMDD'); + IF to_regclass(format('psr.%I', v_name)) IS NULL THEN + EXECUTE format('CREATE TABLE psr.%I PARTITION OF psr.artifact_contents FOR VALUES FROM (%L) TO (%L)', v_name, v_start, v_end); + v_created := v_created + 1; + END IF; + END LOOP; + + FOR i IN 0..p_months_ahead LOOP + v_start := (v_month + make_interval(months => i)) AT TIME ZONE 'UTC'; + v_end := (v_month + make_interval(months => i + 1)) AT TIME ZONE 'UTC'; + v_name := 'audit_events_p' || to_char(v_start AT TIME ZONE 'UTC', 'YYYYMM'); + IF to_regclass(format('psr.%I', v_name)) IS NULL THEN + EXECUTE format('CREATE TABLE psr.%I PARTITION OF psr.audit_events FOR VALUES FROM (%L) TO (%L)', v_name, v_start, v_end); + v_created := v_created + 1; + END IF; + END LOOP; + + RETURN v_created; + END + $fn$; + """; + + /// Detaches and drops the partitions older than the global retention. + public const string DropExpiredPartitionsFunction = """ + CREATE OR REPLACE FUNCTION psr.drop_expired_partitions( + p_now timestamptz, p_runs interval, p_contents interval, p_audit interval) + RETURNS SETOF text + LANGUAGE plpgsql + AS $fn$ + DECLARE + v_row record; + v_upper timestamptz; + v_retention interval; + BEGIN + -- Children first: the foreign keys of the history tables point to runs and step_runs. + FOR v_row IN + SELECT child.relname::text AS name, parent.relname::text AS parent, pg_get_expr(child.relpartbound, child.oid) AS bound + FROM pg_inherits inh + JOIN pg_class child ON child.oid = inh.inhrelid + JOIN pg_class parent ON parent.oid = inh.inhparent + JOIN pg_namespace ns ON ns.oid = parent.relnamespace + WHERE ns.nspname = 'psr' + AND parent.relname IN ('attempt_logs', 'step_attempts', 'artifacts', 'step_runs', 'runs', 'artifact_contents', 'audit_events') + ORDER BY array_position( + ARRAY['attempt_logs', 'step_attempts', 'artifacts', 'step_runs', 'runs', 'artifact_contents', 'audit_events'], parent.relname::text), + child.relname + LOOP + v_upper := substring(v_row.bound FROM 'TO \(''([^'']+)''\)')::timestamptz; + v_retention := CASE v_row.parent WHEN 'artifact_contents' THEN p_contents WHEN 'audit_events' THEN p_audit ELSE p_runs END; + IF v_upper <= p_now - v_retention THEN + EXECUTE format('ALTER TABLE psr.%I DETACH PARTITION psr.%I', v_row.parent, v_row.name); + EXECUTE format('DROP TABLE psr.%I', v_row.name); + RETURN NEXT v_row.name; + END IF; + END LOOP; + END + $fn$; + """; + + /// Creates the partitions of the current period and the ones ahead. + public const string CreateInitialPartitions = "DO $do$ BEGIN PERFORM psr.ensure_partitions(now()); END $do$;"; + + /// Removes the partition functions. + public const string DropFunctions = """ + DROP FUNCTION IF EXISTS psr.drop_expired_partitions(timestamptz, interval, interval, interval); + DROP FUNCTION IF EXISTS psr.ensure_partitions(timestamptz, integer, integer, integer); + """; +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Migrations/PsrDbContextModelSnapshot.cs b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/PsrDbContextModelSnapshot.cs new file mode 100644 index 0000000..8cf4057 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Migrations/PsrDbContextModelSnapshot.cs @@ -0,0 +1,1344 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using PythonScriptRunner.EntityFrameworkCore; + +#nullable disable + +namespace PythonScriptRunner.EntityFrameworkCore.Migrations +{ + [DbContext(typeof(PsrDbContext))] + partial class PsrDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasDefaultSchema("psr") + .HasAnnotation("ProductVersion", "10.0.13") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.ArtifactContentEntity", b => + { + b.Property("ArtifactId") + .HasColumnType("uuid") + .HasColumnName("artifact_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("Content") + .IsRequired() + .HasColumnType("bytea") + .HasColumnName("content"); + + b.HasKey("ArtifactId", "CreatedAt") + .HasName("pk_artifact_contents"); + + b.ToTable("artifact_contents", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.ArtifactEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("RunCreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("run_created_at"); + + b.Property("ContentCreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("content_created_at"); + + b.Property("ContentDeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("content_deleted_at"); + + b.Property("ContentExpiresAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("content_expires_at"); + + b.Property("ContentType") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("content_type"); + + b.Property("Direction") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("character varying(8)") + .HasColumnName("direction"); + + b.Property("FieldName") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("field_name"); + + b.Property("FileName") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("character varying(255)") + .HasColumnName("file_name"); + + b.Property("ListIndex") + .HasColumnType("integer") + .HasColumnName("list_index"); + + b.Property("RunId") + .HasColumnType("uuid") + .HasColumnName("run_id"); + + b.Property("Sha256") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character(64)") + .HasColumnName("sha256") + .IsFixedLength(); + + b.Property("SizeBytes") + .HasColumnType("bigint") + .HasColumnName("size_bytes"); + + b.Property("StepRunId") + .HasColumnType("uuid") + .HasColumnName("step_run_id"); + + b.HasKey("Id", "RunCreatedAt") + .HasName("pk_artifacts"); + + b.HasIndex("ContentExpiresAt") + .HasDatabaseName("ix_artifacts_content_expires_at") + .HasFilter("content_deleted_at IS NULL"); + + b.HasIndex("RunId", "RunCreatedAt") + .HasDatabaseName("ix_artifacts_run_id_run_created_at"); + + b.HasIndex("StepRunId", "RunCreatedAt") + .HasDatabaseName("ix_artifacts_step_run_id_run_created_at"); + + b.ToTable("artifacts", "psr", t => + { + t.HasCheckConstraint("ck_artifacts_direction", "direction IN ('Input', 'Output')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.AttemptLogEntity", b => + { + b.Property("AttemptId") + .HasColumnType("uuid") + .HasColumnName("attempt_id"); + + b.Property("RunCreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("run_created_at"); + + b.Property("Stream") + .HasMaxLength(8) + .HasColumnType("character varying(8)") + .HasColumnName("stream"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text") + .HasColumnName("content"); + + b.Property("OriginalBytes") + .HasColumnType("bigint") + .HasColumnName("original_bytes"); + + b.Property("Truncated") + .HasColumnType("boolean") + .HasColumnName("truncated"); + + b.HasKey("AttemptId", "RunCreatedAt", "Stream") + .HasName("pk_attempt_logs"); + + b.ToTable("attempt_logs", "psr", t => + { + t.HasCheckConstraint("ck_attempt_logs_stream", "stream IN ('stdout', 'stderr')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.AuditEventEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("OccurredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("occurred_at"); + + b.Property("Action") + .IsRequired() + .HasMaxLength(60) + .HasColumnType("character varying(60)") + .HasColumnName("action"); + + b.Property("Actor") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("actor"); + + b.Property("Details") + .HasColumnType("jsonb") + .HasColumnName("details"); + + b.Property("EntityId") + .HasColumnType("uuid") + .HasColumnName("entity_id"); + + b.Property("EntityKey") + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("entity_key"); + + b.Property("EntityType") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("character varying(30)") + .HasColumnName("entity_type"); + + b.Property("TraceId") + .HasMaxLength(32) + .HasColumnType("character varying(32)") + .HasColumnName("trace_id"); + + b.HasKey("Id", "OccurredAt") + .HasName("pk_audit_events"); + + b.HasIndex("OccurredAt", "Id") + .IsDescending() + .HasDatabaseName("ix_audit_events_occurred_at_id"); + + b.ToTable("audit_events", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.IdempotencyKeyEntity", b => + { + b.Property("Scope") + .HasMaxLength(80) + .HasColumnType("character varying(80)") + .HasColumnName("scope"); + + b.Property("Key") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("key"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expires_at"); + + b.Property("RunCreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("run_created_at"); + + b.Property("RunId") + .HasColumnType("uuid") + .HasColumnName("run_id"); + + b.HasKey("Scope", "Key") + .HasName("pk_idempotency_keys"); + + b.HasIndex("ExpiresAt") + .HasDatabaseName("ix_idempotency_keys_expires_at"); + + b.ToTable("idempotency_keys", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.OutboxMessageEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Attempts") + .HasColumnType("integer") + .HasColumnName("attempts"); + + b.Property("AvailableAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("available_at"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("Exchange") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("exchange"); + + b.Property("Headers") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("headers"); + + b.Property("LastError") + .HasColumnType("text") + .HasColumnName("last_error"); + + b.Property("MessageType") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("message_type"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("payload"); + + b.Property("PublishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("published_at"); + + b.Property("RoutingKey") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("routing_key"); + + b.HasKey("Id") + .HasName("pk_outbox_messages"); + + b.HasIndex("AvailableAt") + .HasDatabaseName("ix_outbox_messages_available_at") + .HasFilter("published_at IS NULL"); + + b.ToTable("outbox_messages", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ArtifactRetentionDays") + .HasColumnType("integer") + .HasColumnName("artifact_retention_days"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("key"); + + b.Property("LatestPublishedVersion") + .HasColumnType("integer") + .HasColumnName("latest_published_version"); + + b.Property("MaxConcurrentRuns") + .HasColumnType("integer") + .HasColumnName("max_concurrent_runs"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + + b.Property("RunRetentionDays") + .HasColumnType("integer") + .HasColumnName("run_retention_days"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UpdatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("updated_by"); + + b.HasKey("Id") + .HasName("pk_pipelines"); + + b.HasIndex("Key") + .IsUnique() + .HasDatabaseName("ux_pipelines_key"); + + b.ToTable("pipelines", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineStepEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Bindings") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("bindings"); + + b.Property("DisplayName") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("display_name"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("key"); + + b.Property("Order") + .HasColumnType("integer") + .HasColumnName("order"); + + b.Property("PipelineVersionId") + .HasColumnType("uuid") + .HasColumnName("pipeline_version_id"); + + b.Property("StageVersionId") + .HasColumnType("uuid") + .HasColumnName("stage_version_id"); + + b.HasKey("Id") + .HasName("pk_pipeline_steps"); + + b.HasIndex("StageVersionId") + .HasDatabaseName("ix_pipeline_steps_stage_version_id"); + + b.HasIndex("PipelineVersionId", "Key") + .IsUnique() + .HasDatabaseName("ux_pipeline_steps_pipeline_version_id_key"); + + b.HasIndex("PipelineVersionId", "Order") + .IsUnique() + .HasDatabaseName("ux_pipeline_steps_pipeline_version_id_order"); + + b.ToTable("pipeline_steps", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineVersionEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ArchivedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("archived_at"); + + b.Property("ArchivedBy") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("archived_by"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("InputFields") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("input_fields"); + + b.Property("Layout") + .HasColumnType("jsonb") + .HasColumnName("layout"); + + b.Property("Outputs") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("outputs"); + + b.Property("PipelineId") + .HasColumnType("uuid") + .HasColumnName("pipeline_id"); + + b.Property("PublishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("published_at"); + + b.Property("PublishedBy") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("published_by"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + + b.Property("Status") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasDefaultValue("Draft") + .HasColumnName("status"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UpdatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("updated_by"); + + b.Property("Version") + .HasColumnType("integer") + .HasColumnName("version"); + + b.HasKey("Id") + .HasName("pk_pipeline_versions"); + + b.HasIndex("PipelineId") + .IsUnique() + .HasDatabaseName("ux_pipeline_versions_draft") + .HasFilter("status = 'Draft'"); + + b.HasIndex("PipelineId", "Version") + .IsUnique() + .HasDatabaseName("ux_pipeline_versions_pipeline_id_version"); + + b.ToTable("pipeline_versions", "psr", t => + { + t.HasCheckConstraint("ck_pipeline_versions_status", "status IN ('Draft', 'Published', 'Archived')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.RunEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CancelRequestedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("cancel_requested_at"); + + b.Property("CancelRequestedBy") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("cancel_requested_by"); + + b.Property("CorrelationId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("correlation_id"); + + b.Property("Error") + .HasColumnType("jsonb") + .HasColumnName("error"); + + b.Property("ExpiresAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("expires_at"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_at"); + + b.Property("IdempotencyKey") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("idempotency_key"); + + b.Property("Inputs") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("inputs"); + + b.Property("Kind") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("kind"); + + b.Property("Mode") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("character varying(8)") + .HasColumnName("mode"); + + b.Property("Outputs") + .HasColumnType("jsonb") + .HasColumnName("outputs"); + + b.Property("PipelineId") + .HasColumnType("uuid") + .HasColumnName("pipeline_id"); + + b.Property("PipelineKey") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("pipeline_key"); + + b.Property("PipelineVersion") + .HasColumnType("integer") + .HasColumnName("pipeline_version"); + + b.Property("PipelineVersionId") + .HasColumnType("uuid") + .HasColumnName("pipeline_version_id"); + + b.Property("QueuedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("queued_at"); + + b.Property("ScheduleId") + .HasColumnType("uuid") + .HasColumnName("schedule_id"); + + b.Property("StageKey") + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("stage_key"); + + b.Property("StageVersion") + .HasColumnType("integer") + .HasColumnName("stage_version"); + + b.Property("StageVersionId") + .HasColumnType("uuid") + .HasColumnName("stage_version_id"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("status"); + + b.Property("TraceParent") + .HasMaxLength(55) + .HasColumnType("character varying(55)") + .HasColumnName("trace_parent"); + + b.Property("Trigger") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("trigger"); + + b.Property("TriggeredBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("triggered_by"); + + b.HasKey("Id", "CreatedAt") + .HasName("pk_runs"); + + b.HasIndex("ExpiresAt") + .HasDatabaseName("ix_runs_expires_at") + .HasFilter("expires_at IS NOT NULL"); + + b.HasIndex("CreatedAt", "Id") + .IsDescending() + .HasDatabaseName("ix_runs_created_at_id"); + + b.HasIndex("Status", "CreatedAt") + .HasDatabaseName("ix_runs_status_created_at") + .HasFilter("status IN ('Pending', 'Queued', 'Running', 'Cancelling')"); + + b.HasIndex("PipelineId", "CreatedAt", "Id") + .IsDescending(false, true, true) + .HasDatabaseName("ix_runs_pipeline_id_created_at_id"); + + b.HasIndex("StageKey", "CreatedAt", "Id") + .IsDescending(false, true, true) + .HasDatabaseName("ix_runs_stage_key_created_at_id") + .HasFilter("stage_key IS NOT NULL"); + + b.ToTable("runs", "psr", t => + { + t.HasCheckConstraint("ck_runs_kind", "kind IN ('Pipeline', 'PipelineTest', 'Stage', 'StageTest')"); + + t.HasCheckConstraint("ck_runs_mode", "mode IN ('Sync', 'Async')"); + + t.HasCheckConstraint("ck_runs_status", "status IN ('Pending', 'Queued', 'Running', 'Succeeded', 'Failed', 'Cancelling', 'Cancelled')"); + + t.HasCheckConstraint("ck_runs_trigger", "trigger IN ('Library', 'Dashboard', 'Schedule', 'QueueCommand')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.ScheduleEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("Cron") + .IsRequired() + .HasMaxLength(100) + .HasColumnType("character varying(100)") + .HasColumnName("cron"); + + b.Property("Inputs") + .IsRequired() + .ValueGeneratedOnAdd() + .HasColumnType("jsonb") + .HasColumnName("inputs") + .HasDefaultValueSql("'{}'::jsonb"); + + b.Property("IsEnabled") + .ValueGeneratedOnAdd() + .HasColumnType("boolean") + .HasDefaultValue(true) + .HasColumnName("is_enabled"); + + b.Property("LastError") + .HasColumnType("text") + .HasColumnName("last_error"); + + b.Property("LastFiredAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_fired_at"); + + b.Property("LastRunId") + .HasColumnType("uuid") + .HasColumnName("last_run_id"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.Property("NextRunAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("next_run_at"); + + b.Property("PinnedVersion") + .HasColumnType("integer") + .HasColumnName("pinned_version"); + + b.Property("PipelineId") + .HasColumnType("uuid") + .HasColumnName("pipeline_id"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + + b.Property("TimeZone") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasDefaultValue("America/Sao_Paulo") + .HasColumnName("time_zone"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UpdatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("updated_by"); + + b.HasKey("Id") + .HasName("pk_schedules"); + + b.HasIndex("NextRunAt") + .HasDatabaseName("ix_schedules_next_run_at") + .HasFilter("is_enabled"); + + b.HasIndex("PipelineId") + .HasDatabaseName("ix_schedules_pipeline_id"); + + b.ToTable("schedules", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StageEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("Description") + .HasColumnType("text") + .HasColumnName("description"); + + b.Property("Key") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("key"); + + b.Property("LatestPublishedVersion") + .HasColumnType("integer") + .HasColumnName("latest_published_version"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("name"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UpdatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("updated_by"); + + b.HasKey("Id") + .HasName("pk_stages"); + + b.HasIndex("Key") + .IsUnique() + .HasDatabaseName("ux_stages_key"); + + b.ToTable("stages", "psr"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StageVersionEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ArchivedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("archived_at"); + + b.Property("ArchivedBy") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("archived_by"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("created_by"); + + b.Property("EnvironmentHash") + .HasMaxLength(64) + .HasColumnType("character(64)") + .HasColumnName("environment_hash") + .IsFixedLength(); + + b.Property("Idempotent") + .HasColumnType("boolean") + .HasColumnName("idempotent"); + + b.Property("InputFields") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("input_fields"); + + b.Property("LockedRequirements") + .HasColumnType("text") + .HasColumnName("locked_requirements"); + + b.Property("MemoryLimitMb") + .HasColumnType("integer") + .HasColumnName("memory_limit_mb"); + + b.Property("OutputFields") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("output_fields"); + + b.Property("PublishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("published_at"); + + b.Property("PublishedBy") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("published_by"); + + b.Property("PythonVersion") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("character varying(10)") + .HasColumnName("python_version"); + + b.PrimitiveCollection("RequiredSecrets") + .IsRequired() + .HasColumnType("text[]") + .HasColumnName("required_secrets"); + + b.Property("Requirements") + .HasColumnType("text") + .HasColumnName("requirements"); + + b.Property("RetryBackoffMultiplier") + .HasPrecision(4, 2) + .HasColumnType("numeric(4,2)") + .HasColumnName("retry_backoff_multiplier"); + + b.Property("RetryInitialDelaySeconds") + .HasColumnType("integer") + .HasColumnName("retry_initial_delay_seconds"); + + b.Property("RetryMaxAttempts") + .HasColumnType("integer") + .HasColumnName("retry_max_attempts"); + + b.Property("RowVersion") + .IsConcurrencyToken() + .ValueGeneratedOnAddOrUpdate() + .HasColumnType("xid") + .HasColumnName("xmin"); + + b.Property("Script") + .IsRequired() + .HasColumnType("text") + .HasColumnName("script"); + + b.Property("ScriptSha256") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character(64)") + .HasColumnName("script_sha256") + .IsFixedLength(); + + b.Property("StageId") + .HasColumnType("uuid") + .HasColumnName("stage_id"); + + b.Property("Status") + .IsRequired() + .ValueGeneratedOnAdd() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasDefaultValue("Draft") + .HasColumnName("status"); + + b.Property("TimeoutSeconds") + .HasColumnType("integer") + .HasColumnName("timeout_seconds"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UpdatedBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("updated_by"); + + b.Property("Version") + .HasColumnType("integer") + .HasColumnName("version"); + + b.HasKey("Id") + .HasName("pk_stage_versions"); + + b.HasIndex("StageId") + .IsUnique() + .HasDatabaseName("ux_stage_versions_draft") + .HasFilter("status = 'Draft'"); + + b.HasIndex("StageId", "Version") + .IsUnique() + .HasDatabaseName("ux_stage_versions_stage_id_version"); + + b.ToTable("stage_versions", "psr", t => + { + t.HasCheckConstraint("ck_stage_versions_status", "status IN ('Draft', 'Published', 'Archived')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StepAttemptEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("RunCreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("run_created_at"); + + b.Property("Attempt") + .HasColumnType("integer") + .HasColumnName("attempt"); + + b.Property("Cause") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("cause"); + + b.Property("DurationMs") + .HasColumnType("integer") + .HasColumnName("duration_ms"); + + b.Property("EnvironmentHash") + .HasMaxLength(64) + .HasColumnType("character(64)") + .HasColumnName("environment_hash") + .IsFixedLength(); + + b.Property("Error") + .HasColumnType("jsonb") + .HasColumnName("error"); + + b.Property("ErrorKind") + .HasMaxLength(40) + .HasColumnType("character varying(40)") + .HasColumnName("error_kind"); + + b.Property("ExitCode") + .HasColumnType("integer") + .HasColumnName("exit_code"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_at"); + + b.Property("LeaseExpiresAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("lease_expires_at"); + + b.Property("OverheadMs") + .HasColumnType("integer") + .HasColumnName("overhead_ms"); + + b.Property("RunId") + .HasColumnType("uuid") + .HasColumnName("run_id"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("status"); + + b.Property("StepRunId") + .HasColumnType("uuid") + .HasColumnName("step_run_id"); + + b.Property("TriggeredBy") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("triggered_by"); + + b.Property("WorkerId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("worker_id"); + + b.HasKey("Id", "RunCreatedAt") + .HasName("pk_step_attempts"); + + b.HasIndex("LeaseExpiresAt") + .HasDatabaseName("ix_step_attempts_lease_expires_at") + .HasFilter("status = 'Running'"); + + b.HasIndex("StepRunId", "RunCreatedAt") + .HasDatabaseName("ix_step_attempts_step_run_id_run_created_at"); + + b.ToTable("step_attempts", "psr", t => + { + t.HasCheckConstraint("ck_step_attempts_cause", "cause IN ('Initial', 'AutoRetry', 'Reprocess', 'LeaseRecovery')"); + + t.HasCheckConstraint("ck_step_attempts_status", "status IN ('Running', 'Succeeded', 'Failed', 'TimedOut', 'Cancelled', 'WorkerLost')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StepRunEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("RunCreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("run_created_at"); + + b.Property("CurrentAttempt") + .HasColumnType("integer") + .HasColumnName("current_attempt"); + + b.Property("Error") + .HasColumnType("jsonb") + .HasColumnName("error"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_at"); + + b.Property("Inputs") + .HasColumnType("jsonb") + .HasColumnName("inputs"); + + b.Property("Order") + .HasColumnType("integer") + .HasColumnName("order"); + + b.Property("Outputs") + .HasColumnType("jsonb") + .HasColumnName("outputs"); + + b.Property("PipelineStepId") + .HasColumnType("uuid") + .HasColumnName("pipeline_step_id"); + + b.Property("RunId") + .HasColumnType("uuid") + .HasColumnName("run_id"); + + b.Property("StageKey") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("stage_key"); + + b.Property("StageVersion") + .HasColumnType("integer") + .HasColumnName("stage_version"); + + b.Property("StageVersionId") + .HasColumnType("uuid") + .HasColumnName("stage_version_id"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("status"); + + b.Property("StepKey") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("step_key"); + + b.HasKey("Id", "RunCreatedAt") + .HasName("pk_step_runs"); + + b.HasIndex("RunId", "RunCreatedAt", "Order") + .IsUnique() + .HasDatabaseName("ux_step_runs_run_id_run_created_at_order"); + + b.ToTable("step_runs", "psr", t => + { + t.HasCheckConstraint("ck_step_runs_status", "status IN ('Pending', 'Queued', 'Running', 'Succeeded', 'Failed', 'TimedOut', 'Cancelled', 'Skipped')"); + }); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.ArtifactEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.RunEntity", null) + .WithMany() + .HasForeignKey("RunId", "RunCreatedAt") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_artifacts_runs_run_id_run_created_at"); + + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.StepRunEntity", null) + .WithMany() + .HasForeignKey("StepRunId", "RunCreatedAt") + .OnDelete(DeleteBehavior.Cascade) + .HasConstraintName("fk_artifacts_step_runs_step_run_id_run_created_at"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.AttemptLogEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.StepAttemptEntity", null) + .WithMany() + .HasForeignKey("AttemptId", "RunCreatedAt") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_attempt_logs_step_attempts_attempt_id_run_created_at"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineStepEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineVersionEntity", null) + .WithMany("Steps") + .HasForeignKey("PipelineVersionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_pipeline_steps_pipeline_versions_pipeline_version_id"); + + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.StageVersionEntity", "StageVersion") + .WithMany() + .HasForeignKey("StageVersionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired() + .HasConstraintName("fk_pipeline_steps_stage_versions_stage_version_id"); + + b.Navigation("StageVersion"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineVersionEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineEntity", "Pipeline") + .WithMany("Versions") + .HasForeignKey("PipelineId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_pipeline_versions_pipelines_pipeline_id"); + + b.Navigation("Pipeline"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.ScheduleEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineEntity", null) + .WithMany() + .HasForeignKey("PipelineId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_schedules_pipelines_pipeline_id"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StageVersionEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.StageEntity", "Stage") + .WithMany("Versions") + .HasForeignKey("StageId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_stage_versions_stages_stage_id"); + + b.Navigation("Stage"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StepAttemptEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.StepRunEntity", null) + .WithMany("Attempts") + .HasForeignKey("StepRunId", "RunCreatedAt") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_step_attempts_step_runs_step_run_id_run_created_at"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StepRunEntity", b => + { + b.HasOne("PythonScriptRunner.EntityFrameworkCore.Entities.RunEntity", null) + .WithMany("Steps") + .HasForeignKey("RunId", "RunCreatedAt") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired() + .HasConstraintName("fk_step_runs_runs_run_id_run_created_at"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineEntity", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.PipelineVersionEntity", b => + { + b.Navigation("Steps"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.RunEntity", b => + { + b.Navigation("Steps"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StageEntity", b => + { + b.Navigation("Versions"); + }); + + modelBuilder.Entity("PythonScriptRunner.EntityFrameworkCore.Entities.StepRunEntity", b => + { + b.Navigation("Attempts"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/PsrDbContext.cs b/src/PythonScriptRunner.EntityFrameworkCore/PsrDbContext.cs new file mode 100644 index 0000000..3d01813 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/PsrDbContext.cs @@ -0,0 +1,373 @@ +using System.Text; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using PythonScriptRunner.EntityFrameworkCore.Entities; + +namespace PythonScriptRunner.EntityFrameworkCore; + +/// +/// Database of PythonScriptRunner (schema psr). Advanced API: the queue, worker and dashboard packages use it to +/// write in the same transaction as their own changes. +/// +/// Context options. +public sealed class PsrDbContext(DbContextOptions options) : DbContext(options) +{ + /// Dedicated schema of every table. + public const string Schema = "psr"; + + /// Table that records the applied migrations. + public const string MigrationsHistoryTable = "__ef_migrations_history"; + +#pragma warning disable CS1591 // One set per table of the documented schema. + public DbSet Stages => Set(); + + public DbSet StageVersions => Set(); + + public DbSet Pipelines => Set(); + + public DbSet PipelineVersions => Set(); + + public DbSet PipelineSteps => Set(); + + public DbSet Schedules => Set(); + + public DbSet Runs => Set(); + + public DbSet StepRuns => Set(); + + public DbSet StepAttempts => Set(); + + public DbSet AttemptLogs => Set(); + + public DbSet Artifacts => Set(); + + public DbSet ArtifactContents => Set(); + + public DbSet OutboxMessages => Set(); + + public DbSet IdempotencyKeys => Set(); + + public DbSet AuditEvents => Set(); +#pragma warning restore CS1591 + + /// + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.HasDefaultSchema(Schema); + ConfigureCatalog(modelBuilder); + ConfigureRuns(modelBuilder); + ConfigureInfrastructure(modelBuilder); + ApplySnakeCase(modelBuilder); + + // Identifiers are UUID v7 generated by the application: new rows reached through navigations are inserted. + foreach (var key in modelBuilder.Model.GetEntityTypes().SelectMany(e => e.GetKeys())) + { + foreach (var property in key.Properties.Where(p => p.ClrType == typeof(Guid))) + { + property.ValueGenerated = ValueGenerated.Never; + } + } + } + + private static void ConfigureCatalog(ModelBuilder model) + { + model.Entity(e => + { + e.ToTable("stages"); + Key(e.Property(x => x.Key)); + e.Property(x => x.Name).HasMaxLength(200); + Authorship(e); + RowVersion(e.Property(x => x.RowVersion)); + e.HasIndex(x => x.Key).IsUnique(); + e.HasMany(x => x.Versions).WithOne(x => x.Stage).HasForeignKey(x => x.StageId).OnDelete(DeleteBehavior.Cascade); + }); + + model.Entity(e => + { + e.ToTable("stage_versions", t => Check(t, "status", typeof(VersionStatus))); + Enum(e.Property(x => x.Status)).HasDefaultValue(VersionStatus.Draft); + e.Property(x => x.ScriptSha256).HasMaxLength(64).IsFixedLength(); + e.Property(x => x.PythonVersion).HasMaxLength(10); + e.Property(x => x.EnvironmentHash).HasMaxLength(64).IsFixedLength(); + Json(e.Property(x => x.InputFields)); + Json(e.Property(x => x.OutputFields)); + e.Property(x => x.RetryBackoffMultiplier).HasPrecision(4, 2); + e.Property(x => x.PublishedBy).HasMaxLength(200); + e.Property(x => x.ArchivedBy).HasMaxLength(200); + Authorship(e); + RowVersion(e.Property(x => x.RowVersion)); + e.HasIndex(x => new { x.StageId, x.Version }).IsUnique(); + e.HasIndex(x => x.StageId).IsUnique().HasFilter("status = 'Draft'").HasDatabaseName("ux_stage_versions_draft"); + }); + + model.Entity(e => + { + e.ToTable("pipelines"); + Key(e.Property(x => x.Key)); + e.Property(x => x.Name).HasMaxLength(200); + Authorship(e); + RowVersion(e.Property(x => x.RowVersion)); + e.HasIndex(x => x.Key).IsUnique(); + e.HasMany(x => x.Versions).WithOne(x => x.Pipeline).HasForeignKey(x => x.PipelineId).OnDelete(DeleteBehavior.Cascade); + }); + + model.Entity(e => + { + e.ToTable("pipeline_versions", t => Check(t, "status", typeof(VersionStatus))); + Enum(e.Property(x => x.Status)).HasDefaultValue(VersionStatus.Draft); + Json(e.Property(x => x.InputFields)); + Json(e.Property(x => x.Outputs)); + Json(e.Property(x => x.Layout)); + e.Property(x => x.PublishedBy).HasMaxLength(200); + e.Property(x => x.ArchivedBy).HasMaxLength(200); + Authorship(e); + RowVersion(e.Property(x => x.RowVersion)); + e.HasIndex(x => new { x.PipelineId, x.Version }).IsUnique(); + e.HasIndex(x => x.PipelineId).IsUnique().HasFilter("status = 'Draft'").HasDatabaseName("ux_pipeline_versions_draft"); + e.HasMany(x => x.Steps).WithOne().HasForeignKey(x => x.PipelineVersionId).OnDelete(DeleteBehavior.Cascade); + }); + + model.Entity(e => + { + e.ToTable("pipeline_steps"); + Key(e.Property(x => x.Key)); + e.Property(x => x.DisplayName).HasMaxLength(200); + Json(e.Property(x => x.Bindings)); + e.HasIndex(x => new { x.PipelineVersionId, x.Order }).IsUnique(); + e.HasIndex(x => new { x.PipelineVersionId, x.Key }).IsUnique(); + e.HasIndex(x => x.StageVersionId); + e.HasOne(x => x.StageVersion).WithMany().HasForeignKey(x => x.StageVersionId).OnDelete(DeleteBehavior.Restrict); + }); + + model.Entity(e => + { + e.ToTable("schedules"); + e.Property(x => x.Name).HasMaxLength(200); + e.Property(x => x.Cron).HasMaxLength(100); + e.Property(x => x.TimeZone).HasMaxLength(64).HasDefaultValue("America/Sao_Paulo"); + Json(e.Property(x => x.Inputs)).HasDefaultValueSql("'{}'::jsonb"); + e.Property(x => x.IsEnabled).HasDefaultValue(true); + Authorship(e); + RowVersion(e.Property(x => x.RowVersion)); + e.HasIndex(x => x.NextRunAt).HasFilter("is_enabled"); + e.HasOne().WithMany().HasForeignKey(x => x.PipelineId).OnDelete(DeleteBehavior.Cascade); + }); + } + + private static void ConfigureRuns(ModelBuilder model) + { + model.Entity(e => + { + e.ToTable("runs", t => + { + Check(t, "kind", typeof(RunKind)); + Check(t, "mode", typeof(ExecutionMode)); + Check(t, "trigger", typeof(RunTrigger)); + Check(t, "status", typeof(RunStatus)); + }); + e.HasKey(x => new { x.Id, x.CreatedAt }); + Enum(e.Property(x => x.Kind)); + Enum(e.Property(x => x.Mode), 8); + Enum(e.Property(x => x.Trigger)); + Enum(e.Property(x => x.Status)); + e.Property(x => x.PipelineKey).HasMaxLength(64); + e.Property(x => x.StageKey).HasMaxLength(64); + e.Property(x => x.TriggeredBy).HasMaxLength(200); + Json(e.Property(x => x.Inputs)); + Json(e.Property(x => x.Outputs)); + Json(e.Property(x => x.Error)); + e.Property(x => x.IdempotencyKey).HasMaxLength(200); + e.Property(x => x.CorrelationId).HasMaxLength(200); + e.Property(x => x.TraceParent).HasMaxLength(55); + e.Property(x => x.CancelRequestedBy).HasMaxLength(200); + e.HasIndex(x => new { x.CreatedAt, x.Id }).IsDescending(true, true); + e.HasIndex(x => new { x.PipelineId, x.CreatedAt, x.Id }).IsDescending(false, true, true); + e.HasIndex(x => new { x.StageKey, x.CreatedAt, x.Id }).IsDescending(false, true, true).HasFilter("stage_key IS NOT NULL"); + e.HasIndex(x => new { x.Status, x.CreatedAt }).HasFilter("status IN ('Pending', 'Queued', 'Running', 'Cancelling')"); + e.HasIndex(x => x.ExpiresAt).HasFilter("expires_at IS NOT NULL"); + e.HasMany(x => x.Steps).WithOne().HasForeignKey(x => new { x.RunId, x.RunCreatedAt }).OnDelete(DeleteBehavior.Cascade); + }); + + model.Entity(e => + { + e.ToTable("step_runs", t => Check(t, "status", typeof(StepStatus))); + e.HasKey(x => new { x.Id, x.RunCreatedAt }); + Key(e.Property(x => x.StepKey)); + Key(e.Property(x => x.StageKey)); + Enum(e.Property(x => x.Status)); + Json(e.Property(x => x.Inputs)); + Json(e.Property(x => x.Outputs)); + Json(e.Property(x => x.Error)); + e.HasIndex(x => new { x.RunId, x.RunCreatedAt, x.Order }).IsUnique(); + e.HasMany(x => x.Attempts).WithOne().HasForeignKey(x => new { x.StepRunId, x.RunCreatedAt }).OnDelete(DeleteBehavior.Cascade); + }); + + model.Entity(e => + { + e.ToTable("step_attempts", t => + { + Check(t, "cause", typeof(AttemptCause)); + Check(t, "status", typeof(AttemptStatus)); + }); + e.HasKey(x => new { x.Id, x.RunCreatedAt }); + Enum(e.Property(x => x.Cause)); + e.Property(x => x.TriggeredBy).HasMaxLength(200); + Enum(e.Property(x => x.Status)); + e.Property(x => x.WorkerId).HasMaxLength(200); + e.Property(x => x.EnvironmentHash).HasMaxLength(64).IsFixedLength(); + e.Property(x => x.ErrorKind).HasConversion().HasMaxLength(40); + Json(e.Property(x => x.Error)); + e.HasIndex(x => x.LeaseExpiresAt).HasFilter("status = 'Running'"); + }); + + model.Entity(e => + { + e.ToTable("attempt_logs", t => t.HasCheckConstraint("ck_attempt_logs_stream", "stream IN ('stdout', 'stderr')")); + e.HasKey(x => new { x.AttemptId, x.RunCreatedAt, x.Stream }); + e.Property(x => x.Stream).HasMaxLength(8).HasConversion(v => v == LogStream.Stdout ? "stdout" : "stderr", v => v == "stdout" ? LogStream.Stdout : LogStream.Stderr); + e.HasOne().WithMany().HasForeignKey(x => new { x.AttemptId, x.RunCreatedAt }).OnDelete(DeleteBehavior.Cascade); + }); + + model.Entity(e => + { + e.ToTable("artifacts", t => Check(t, "direction", typeof(ArtifactDirection))); + e.HasKey(x => new { x.Id, x.RunCreatedAt }); + Enum(e.Property(x => x.Direction), 8); + e.Property(x => x.FieldName).HasMaxLength(64); + e.Property(x => x.FileName).HasMaxLength(255); + e.Property(x => x.ContentType).HasMaxLength(255); + e.Property(x => x.Sha256).HasMaxLength(64).IsFixedLength(); + e.HasIndex(x => new { x.RunId, x.RunCreatedAt }); + e.HasIndex(x => x.ContentExpiresAt).HasFilter("content_deleted_at IS NULL"); + e.HasOne().WithMany().HasForeignKey(x => new { x.RunId, x.RunCreatedAt }).OnDelete(DeleteBehavior.Cascade); + e.HasOne().WithMany().HasForeignKey(x => new { x.StepRunId, x.RunCreatedAt }).OnDelete(DeleteBehavior.Cascade); + }); + + model.Entity(e => + { + e.ToTable("artifact_contents"); + e.HasKey(x => new { x.ArtifactId, x.CreatedAt }); + }); + } + + private static void ConfigureInfrastructure(ModelBuilder model) + { + model.Entity(e => + { + e.ToTable("outbox_messages"); + e.Property(x => x.MessageType).HasMaxLength(100); + e.Property(x => x.Exchange).HasMaxLength(200); + e.Property(x => x.RoutingKey).HasMaxLength(200); + Json(e.Property(x => x.Payload)); + Json(e.Property(x => x.Headers)); + e.HasIndex(x => x.AvailableAt).HasFilter("published_at IS NULL"); + }); + + model.Entity(e => + { + e.ToTable("idempotency_keys"); + e.HasKey(x => new { x.Scope, x.Key }); + e.Property(x => x.Scope).HasMaxLength(80); + e.Property(x => x.Key).HasMaxLength(200); + e.HasIndex(x => x.ExpiresAt); + }); + + model.Entity(e => + { + e.ToTable("audit_events"); + e.HasKey(x => new { x.Id, x.OccurredAt }); + e.Property(x => x.Actor).HasMaxLength(200); + e.Property(x => x.Action).HasMaxLength(60); + e.Property(x => x.EntityType).HasMaxLength(30); + e.Property(x => x.EntityKey).HasMaxLength(100); + Json(e.Property(x => x.Details)); + e.Property(x => x.TraceId).HasMaxLength(32); + e.HasIndex(x => new { x.OccurredAt, x.Id }).IsDescending(true, true); + }); + } + + private static PropertyBuilder Key(PropertyBuilder property) => property.HasMaxLength(64); + + private static PropertyBuilder Json(PropertyBuilder property) => property.HasColumnType("jsonb"); + + private static PropertyBuilder Enum(PropertyBuilder property, int maxLength = 16) => + property.HasConversion().HasMaxLength(maxLength); + + /// + /// Maps the PostgreSQL xmin system column (optimistic concurrency; never created by migrations). Not used on + /// partitioned tables: PostgreSQL cannot return system columns from them. + /// + private static void RowVersion(PropertyBuilder property) => property.IsRowVersion().HasColumnName("xmin"); + + private static void Authorship(EntityTypeBuilder entity) + where T : class + { + entity.Property("CreatedBy").HasMaxLength(200); + entity.Property("UpdatedBy").HasMaxLength(200); + } + + private static void Check(TableBuilder table, string column, Type enumType) + where T : class + { + var values = string.Join(", ", System.Enum.GetNames(enumType).Select(n => $"'{n}'")); + table.HasCheckConstraint($"ck_{table.Name}_{column}", $"{column} IN ({values})"); + } + + /// Tables, columns, keys and indexes in snake_case (explicit names are kept). + private static void ApplySnakeCase(ModelBuilder model) + { + foreach (var entity in model.Model.GetEntityTypes()) + { + var table = StoreObjectIdentifier.Table(entity.GetTableName()!, entity.GetSchema()); + foreach (var property in entity.GetProperties()) + { + if (property.GetColumnName(table) == property.Name) + { + property.SetColumnName(SnakeCase(property.Name)); + } + } + + foreach (var key in entity.GetKeys()) + { + key.SetName(key.IsPrimaryKey() ? $"pk_{entity.GetTableName()}" : $"ak_{entity.GetTableName()}_{Columns(key.Properties)}"); + } + + foreach (var foreignKey in entity.GetForeignKeys()) + { + foreignKey.SetConstraintName($"fk_{entity.GetTableName()}_{foreignKey.PrincipalEntityType.GetTableName()}_{Columns(foreignKey.Properties)}"); + } + + foreach (var index in entity.GetIndexes().Where(i => i.FindAnnotation(RelationalAnnotationNames.Name) is null)) + { + index.SetDatabaseName($"{(index.IsUnique ? "ux" : "ix")}_{entity.GetTableName()}_{Columns(index.Properties)}"); + } + } + } + + private static string Columns(IReadOnlyList properties) => string.Join("_", properties.Select(p => SnakeCase(p.Name))); + + private static string SnakeCase(string name) + { + var builder = new StringBuilder(name.Length + 8); + for (var i = 0; i < name.Length; i++) + { + var c = name[i]; + if (char.IsUpper(c)) + { + if (i > 0 && (char.IsLower(name[i - 1]) || (i + 1 < name.Length && char.IsLower(name[i + 1])))) + { + builder.Append('_'); + } + + builder.Append(char.ToLowerInvariant(c)); + } + else + { + builder.Append(c); + } + } + + return builder.ToString(); + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/PsrDbContextOptions.cs b/src/PythonScriptRunner.EntityFrameworkCore/PsrDbContextOptions.cs new file mode 100644 index 0000000..49a8083 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/PsrDbContextOptions.cs @@ -0,0 +1,10 @@ +using Npgsql.EntityFrameworkCore.PostgreSQL.Infrastructure; + +namespace PythonScriptRunner.EntityFrameworkCore; + +/// Npgsql settings shared by the runtime and the design-time context. +internal static class PsrDbContextOptions +{ + public static void ConfigureNpgsql(NpgsqlDbContextOptionsBuilder npgsql) => + npgsql.MigrationsHistoryTable(PsrDbContext.MigrationsHistoryTable, PsrDbContext.Schema); +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/PsrEntityFrameworkCoreExtensions.cs b/src/PythonScriptRunner.EntityFrameworkCore/PsrEntityFrameworkCoreExtensions.cs new file mode 100644 index 0000000..bf8d331 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/PsrEntityFrameworkCoreExtensions.cs @@ -0,0 +1,80 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; +using PythonScriptRunner; +using PythonScriptRunner.EntityFrameworkCore; +using PythonScriptRunner.EntityFrameworkCore.Catalog; +using PythonScriptRunner.EntityFrameworkCore.Infrastructure; +using PythonScriptRunner.EntityFrameworkCore.Maintenance; +using PythonScriptRunner.EntityFrameworkCore.Runs; + +namespace Microsoft.Extensions.DependencyInjection; + +/// Registers the PostgreSQL persistence of PythonScriptRunner. +public static class PsrEntityFrameworkCoreExtensions +{ + /// + /// Stores the catalog and every run in PostgreSQL: replaces the catalog (stages and pipelines then live in the database) + /// and the run recorder, and adds , , , + /// , , , and + /// the maintenance job. The schema is never migrated implicitly: call MigratePythonScriptRunnerAsync(). + /// + /// Builder returned by AddPythonScriptRunner. + /// Connection, retention and maintenance settings. + public static IPythonScriptRunnerBuilder AddEntityFrameworkCore(this IPythonScriptRunnerBuilder builder, Action configure) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(configure); + var options = new PsrEntityFrameworkOptions(); + configure(options); + options.Validate(); + + var services = builder.Services; + services.AddSingleton(options); + services.TryAddSingleton(); + services.AddPooledDbContextFactory((provider, db) => + db.UseNpgsql(provider.GetRequiredService().DataSource, PsrDbContextOptions.ConfigureNpgsql)); + + services.TryAddSingleton(); + services.Replace(ServiceDescriptor.Singleton(provider => provider.GetRequiredService())); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(provider => provider.GetRequiredService()); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.Replace(ServiceDescriptor.Singleton()); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(provider => new PsrMaintenance( + provider.GetRequiredService(), + options, + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService>())); + if (options.Maintenance.Enabled) + { + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + } + + return builder; + } + + /// + /// Health of the psr database: reachable, migrated and with partitions ahead. Reports + /// (default ) when the schema is pending or fewer than two periods of partitions remain. + /// + /// Health checks builder. + /// Check name. + /// Status reported on failure. + /// Tags (e.g. ready). + public static IHealthChecksBuilder AddPythonScriptRunnerDatabase( + this IHealthChecksBuilder builder, string name = "psr-database", HealthStatus? failureStatus = null, IEnumerable? tags = null) + { + ArgumentNullException.ThrowIfNull(builder); + builder.Services.TryAddSingleton(); + return builder.Add(new HealthCheckRegistration(name, provider => provider.GetRequiredService(), failureStatus, tags)); + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/PsrEntityFrameworkOptions.cs b/src/PythonScriptRunner.EntityFrameworkCore/PsrEntityFrameworkOptions.cs new file mode 100644 index 0000000..6ba5626 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/PsrEntityFrameworkOptions.cs @@ -0,0 +1,78 @@ +using Npgsql; + +namespace PythonScriptRunner.EntityFrameworkCore; + +/// Settings of the PostgreSQL persistence. +public sealed class PsrEntityFrameworkOptions +{ + internal string? ConnectionString { get; private set; } + + internal Action? ConfigureDataSource { get; private set; } + + /// Global retention (pipelines can only shorten it). + public PsrRetentionOptions Retention { get; } = new(); + + /// Maintenance job: partitions, retention and cleanups. + public PsrMaintenanceOptions Maintenance { get; } = new(); + + /// How long an idempotency key returns the same run. Default: 24 hours. + public TimeSpan IdempotencyWindow { get; set; } = TimeSpan.FromHours(24); + + /// How long "latest published version" lookups are cached in this process. Default: 30 seconds. + public TimeSpan LatestVersionCacheDuration { get; set; } = TimeSpan.FromSeconds(30); + + /// Connects to PostgreSQL (16 or newer). + /// Npgsql connection string. + /// Extra settings of the data source (e.g. logging, password provider). + public PsrEntityFrameworkOptions UseNpgsql(string connectionString, Action? configure = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionString); + ConnectionString = connectionString; + ConfigureDataSource = configure; + return this; + } + + internal void Validate() + { + if (string.IsNullOrWhiteSpace(ConnectionString)) + { + throw new PsrConfigurationException("AddEntityFrameworkCore requires a connection string: call o.UseNpgsql(...)."); + } + + if (Retention.Artifacts < TimeSpan.FromDays(1) || Retention.Runs < Retention.Artifacts || Retention.Audit < TimeSpan.FromDays(1)) + { + throw new PsrConfigurationException("Retention must keep files at least 1 day, runs at least as long as files, and audit at least 1 day."); + } + + if (Maintenance.Interval <= TimeSpan.Zero || Maintenance.BatchSize < 1) + { + throw new PsrConfigurationException("Maintenance interval and batch size must be positive."); + } + } +} + +/// Global retention. +public sealed class PsrRetentionOptions +{ + /// How long file contents are kept. Default: 7 days. + public TimeSpan Artifacts { get; set; } = TimeSpan.FromDays(7); + + /// How long runs, steps, attempts, logs and file metadata are kept. Default: 90 days. + public TimeSpan Runs { get; set; } = TimeSpan.FromDays(90); + + /// How long audit events are kept. Default: 365 days. + public TimeSpan Audit { get; set; } = TimeSpan.FromDays(365); +} + +/// Maintenance job settings. +public sealed class PsrMaintenanceOptions +{ + /// Runs the maintenance job in this process (only one instance runs each cycle). Default: true. + public bool Enabled { get; set; } = true; + + /// Time between cycles. Default: 1 hour. + public TimeSpan Interval { get; set; } = TimeSpan.FromHours(1); + + /// Rows deleted per statement when applying shortened retentions. Default: 1000. + public int BatchSize { get; set; } = 1000; +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/PsrMigrations.cs b/src/PythonScriptRunner.EntityFrameworkCore/PsrMigrations.cs new file mode 100644 index 0000000..f64070e --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/PsrMigrations.cs @@ -0,0 +1,41 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using PythonScriptRunner.EntityFrameworkCore; + +namespace Microsoft.Extensions.DependencyInjection; + +/// Applies or scripts the migrations of the psr schema. Migrations never run implicitly. +public static class PsrMigrations +{ + /// Creates or updates the psr schema (tables, partitions and functions). Safe to run again. + /// Application services. + /// Cancellation token. + public static async Task MigratePythonScriptRunnerAsync(this IServiceProvider services, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(services); + var factory = services.GetRequiredService>(); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + await db.Database.MigrateAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// Generates the SQL of every migration, for DBAs or deployment pipelines. + /// Application services. + /// Whether the script checks the history table and can run on any version of the schema. + /// Cancellation token. + public static async Task GeneratePythonScriptRunnerMigrationScriptAsync( + this IServiceProvider services, bool idempotent = true, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(services); + var factory = services.GetRequiredService>(); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var migrator = db.GetService(); + return migrator.GenerateScript(options: idempotent ? MigrationsSqlGenerationOptions.Idempotent : MigrationsSqlGenerationOptions.Default); + } + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/PythonScriptRunner.EntityFrameworkCore.csproj b/src/PythonScriptRunner.EntityFrameworkCore/PythonScriptRunner.EntityFrameworkCore.csproj new file mode 100644 index 0000000..e807626 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/PythonScriptRunner.EntityFrameworkCore.csproj @@ -0,0 +1,23 @@ + + + + PythonScriptRunner.EntityFrameworkCore + PostgreSQL persistence for PythonScriptRunner: catalog with versions and drafts, run history, files, idempotency, outbox, audit and retention. + + + + + + + + + + + + + + + + + + diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Runs/ArtifactRefs.cs b/src/PythonScriptRunner.EntityFrameworkCore/Runs/ArtifactRefs.cs new file mode 100644 index 0000000..ba536d5 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Runs/ArtifactRefs.cs @@ -0,0 +1,80 @@ +using System.Text.Json.Nodes; +using PythonScriptRunner.EntityFrameworkCore.Entities; + +namespace PythonScriptRunner.EntityFrameworkCore.Runs; + +/// +/// Converts file values between the wire format ({"$file": {name, contentType, ...}}) and the stored format +/// ({"$artifact": "<id>"}, data mapping §5.1); a list<file> is an array of either. +/// +internal static class ArtifactRefs +{ + public const string Marker = "$artifact"; + private const string FileMarker = "$file"; + + /// Replaces the file fields of with references to the stored files. + public static JsonObject ToStored(JsonObject wire, IReadOnlyDictionary>? files) + { + var stored = (JsonObject)wire.DeepClone(); + foreach (var (field, list) in files ?? new Dictionary>()) + { + if (list.Any(f => f.ArtifactId is null)) + { + continue; + } + + var references = list.Select(f => (JsonNode)new JsonObject { [Marker] = f.ArtifactId!.Value.ToString() }).ToList(); + stored[field] = stored[field] is JsonArray ? new JsonArray([.. references]) : references.FirstOrDefault(); + } + + return stored; + } + + /// Artifact identifiers referenced by a stored object. + public static IEnumerable Referenced(JsonObject? stored) => + stored is null + ? [] + : stored.Select(p => p.Value).SelectMany(v => v is JsonArray array ? array.AsEnumerable() : new[] { v }) + .Select(v => v is JsonObject reference && reference[Marker] is JsonValue id ? Guid.Parse(id.GetValue()) : (Guid?)null) + .Where(id => id is not null).Select(id => id!.Value); + + /// Turns a stored object back into the wire format and the files it references (streamed on demand). + public static (JsonObject Wire, Dictionary> Files) ToWire( + JsonObject? stored, IReadOnlyDictionary artifacts, Func> open) + { + var wire = new JsonObject(); + var files = new Dictionary>(StringComparer.Ordinal); + foreach (var (field, value) in stored ?? []) + { + var items = value is JsonArray array ? array.ToList() : [value]; + if (items.Count == 0 || items.Any(i => i is not JsonObject o || !o.ContainsKey(Marker))) + { + wire[field] = value?.DeepClone(); + continue; + } + + var list = new List(); + var metadata = new List(); + foreach (var item in items) + { + var id = Guid.Parse(item![Marker]!.GetValue()); + if (!artifacts.TryGetValue(id, out var row)) + { + continue; + } + + var artifactId = new ArtifactId(row.Id); + list.Add(PsrFile.FromArtifact(artifactId, row.FileName, row.ContentType, row.SizeBytes, row.Sha256, ct => open(artifactId, ct))); + metadata.Add(new JsonObject + { + [FileMarker] = new JsonObject { ["name"] = row.FileName, ["contentType"] = row.ContentType, ["size"] = row.SizeBytes, ["sha256"] = row.Sha256 }, + }); + } + + files[field] = list; + wire[field] = value is JsonArray ? new JsonArray([.. metadata]) : metadata.FirstOrDefault(); + } + + return (wire, files); + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Runs/ArtifactStore.cs b/src/PythonScriptRunner.EntityFrameworkCore/Runs/ArtifactStore.cs new file mode 100644 index 0000000..2d11c76 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Runs/ArtifactStore.cs @@ -0,0 +1,316 @@ +using System.Data; +using System.Security.Cryptography; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; +using Npgsql; +using NpgsqlTypes; +using PythonScriptRunner.EntityFrameworkCore.Entities; + +namespace PythonScriptRunner.EntityFrameworkCore.Runs; + +/// Npgsql data source of the persistence package (shared by EF and the streaming file store). +internal sealed class PsrDataSource : IDisposable +{ + public PsrDataSource(PsrEntityFrameworkOptions options) + { + var builder = new NpgsqlDataSourceBuilder(options.ConnectionString); + options.ConfigureDataSource?.Invoke(builder); + DataSource = builder.Build(); + } + + public NpgsqlDataSource DataSource { get; } + + public void Dispose() => DataSource.Dispose(); +} + +/// +/// Files in psr.artifacts (metadata) and psr.artifact_contents (bytea, partitioned by day). Contents are +/// streamed with Npgsql in both directions, outside the change tracker, so a 50 MB file never sits in memory. +/// +internal sealed class ArtifactStore( + IDbContextFactory factory, PsrDataSource dataSource, PsrEntityFrameworkOptions options, TimeProvider time) : IArtifactStore +{ + public async Task SaveAsync(NewArtifact artifact, Stream content, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(artifact); + ArgumentNullException.ThrowIfNull(content); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var transaction = await db.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + await using (transaction.ConfigureAwait(false)) + { + var row = await SaveAsync(db, artifact, content, cancellationToken).ConfigureAwait(false); + await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return ToInfo(row, null); + } + } + } + + /// + /// Streams the content inside the open transaction of and adds the metadata row to it (saved by the caller). + /// + public async Task SaveAsync(PsrDbContext db, NewArtifact artifact, Stream content, CancellationToken cancellationToken) + { + var transaction = db.Database.CurrentTransaction?.GetDbTransaction() as NpgsqlTransaction + ?? throw new InvalidOperationException("Artifacts are stored inside a transaction."); + var now = time.GetUtcNow(); + var id = artifact.Id ?? ArtifactId.New(); + Stream? spooled = null; + try + { + if (!content.CanSeek) + { + // Npgsql streams only seekable values (it needs the length up front); spool the rest to disk. + spooled = new FileStream(Path.GetTempFileName(), FileMode.Create, FileAccess.ReadWrite, FileShare.None, 81920, + FileOptions.Asynchronous | FileOptions.DeleteOnClose); + await content.CopyToAsync(spooled, cancellationToken).ConfigureAwait(false); + spooled.Position = 0; + } + + var hashing = new HashingStream(spooled ?? content); + await using (hashing.ConfigureAwait(false)) + { + var command = new NpgsqlCommand( + "INSERT INTO psr.artifact_contents (artifact_id, created_at, content) VALUES ($1, $2, $3)", + (NpgsqlConnection)db.Database.GetDbConnection(), + transaction); + await using (command.ConfigureAwait(false)) + { + command.Parameters.Add(new NpgsqlParameter { Value = id.Value }); + command.Parameters.Add(new NpgsqlParameter { Value = now }); + command.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Bytea, Value = hashing }); + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + var retention = artifact.Retention is { } shorter && shorter < options.Retention.Artifacts ? shorter : options.Retention.Artifacts; + var row = new ArtifactEntity + { + Id = id.Value, + RunId = artifact.RunId.Value, + RunCreatedAt = artifact.RunId.CreatedAt, + StepRunId = artifact.StepRunId?.Value, + Direction = artifact.Direction, + FieldName = artifact.FieldName, + ListIndex = artifact.ListIndex, + FileName = artifact.FileName, + ContentType = artifact.ContentType, + SizeBytes = hashing.BytesRead, + Sha256 = hashing.Hash, + ContentCreatedAt = now, + ContentExpiresAt = now + retention, + }; + db.Artifacts.Add(row); + return row; + } + } + finally + { + if (spooled is not null) + { + await spooled.DisposeAsync().ConfigureAwait(false); + } + } + } + + public async Task GetAsync(ArtifactId artifactId, CancellationToken cancellationToken = default) + { + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var row = await db.Artifacts.AsNoTracking().SingleOrDefaultAsync(a => a.Id == artifactId.Value, cancellationToken).ConfigureAwait(false); + if (row is null) + { + return null; + } + + var stepKey = row.StepRunId is { } stepRunId + ? await db.StepRuns.Where(s => s.Id == stepRunId && s.RunCreatedAt == row.RunCreatedAt).Select(s => s.StepKey) + .SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false) + : null; + return ToInfo(row, stepKey); + } + } + + public async Task OpenReadAsync(ArtifactId artifactId, CancellationToken cancellationToken = default) + { + var connection = await dataSource.DataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + try + { + var command = new NpgsqlCommand( + """ + SELECT a.content_expires_at, a.content_deleted_at, c.content + FROM psr.artifacts a + LEFT JOIN psr.artifact_contents c ON c.artifact_id = a.id AND c.created_at = a.content_created_at + WHERE a.id = $1 + """, + connection); + command.Parameters.Add(new NpgsqlParameter { Value = artifactId.Value }); + var reader = await command.ExecuteReaderAsync(CommandBehavior.SequentialAccess | CommandBehavior.SingleRow, cancellationToken) + .ConfigureAwait(false); + if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false)) + { + throw new ArtifactNotFoundException(artifactId); + } + + var expiresAt = reader.GetFieldValue(0); + var deleted = !await reader.IsDBNullAsync(1, cancellationToken).ConfigureAwait(false); + if (deleted || expiresAt <= time.GetUtcNow() || await reader.IsDBNullAsync(2, cancellationToken).ConfigureAwait(false)) + { + throw new PsrDomainException(PsrErrorCodes.ArtifactExpired, $"The content of artifact {artifactId} was deleted by the retention."); + } + + return new OwningStream(await reader.GetStreamAsync(2, cancellationToken).ConfigureAwait(false), reader, command, connection); + } + catch + { + await connection.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + public static ArtifactInfo ToInfo(ArtifactEntity row, string? stepKey) => new() + { + Id = new ArtifactId(row.Id), + RunId = new RunId(row.RunId), + StepKey = stepKey, + Direction = row.Direction, + FieldName = row.FieldName, + ListIndex = row.ListIndex, + FileName = row.FileName, + ContentType = row.ContentType, + SizeBytes = row.SizeBytes, + Sha256 = row.Sha256, + ContentExpiresAt = row.ContentExpiresAt, + ContentDeletedAt = row.ContentDeletedAt, + }; + + /// Reads the inner stream once while counting bytes and computing SHA-256; reports the inner length so Npgsql streams it. + private sealed class HashingStream(Stream inner) : Stream + { + private readonly IncrementalHash _hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + + public long BytesRead { get; private set; } + + public string Hash => Convert.ToHexStringLower(_hash.GetCurrentHash()); + + public override bool CanRead => true; + + public override bool CanSeek => inner.CanSeek; + + public override bool CanWrite => false; + + public override long Length => inner.Length; + + public override long Position + { + get => inner.Position; + set => throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + var read = inner.Read(buffer); + Track(buffer[..read]); + return read; + } + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + var read = await inner.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + Track(buffer.Span[..read]); + return read; + } + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override void Flush() + { + } + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _hash.Dispose(); + } + + base.Dispose(disposing); + } + + private void Track(ReadOnlySpan data) + { + BytesRead += data.Length; + _hash.AppendData(data); + } + } + + /// The content stream of a reader; disposing it releases the reader, the command and the connection. + private sealed class OwningStream(Stream inner, NpgsqlDataReader reader, NpgsqlCommand command, NpgsqlConnection connection) : Stream + { + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count); + + public override int Read(Span buffer) => inner.Read(buffer); + + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) => inner.ReadAsync(buffer, cancellationToken); + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + inner.ReadAsync(buffer, offset, count, cancellationToken); + + public override void Flush() + { + } + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async ValueTask DisposeAsync() + { + await inner.DisposeAsync().ConfigureAwait(false); + await reader.DisposeAsync().ConfigureAwait(false); + await command.DisposeAsync().ConfigureAwait(false); + await connection.DisposeAsync().ConfigureAwait(false); + await base.DisposeAsync().ConfigureAwait(false); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + inner.Dispose(); + reader.Dispose(); + command.Dispose(); + connection.Dispose(); + } + + base.Dispose(disposing); + } + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Runs/EfRunManager.cs b/src/PythonScriptRunner.EntityFrameworkCore/Runs/EfRunManager.cs new file mode 100644 index 0000000..21e0acf --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Runs/EfRunManager.cs @@ -0,0 +1,264 @@ +using System.Text.Json.Nodes; +using Microsoft.EntityFrameworkCore; +using PythonScriptRunner.EntityFrameworkCore.Catalog; +using PythonScriptRunner.EntityFrameworkCore.Entities; +using PythonScriptRunner.EntityFrameworkCore.Infrastructure; + +namespace PythonScriptRunner.EntityFrameworkCore.Runs; + +/// Queries, cancels and reprocesses persisted runs. +internal sealed class EfRunManager( + IDbContextFactory factory, + RunLoader loader, + IRunEngine engine, + TimeProvider time) : IRunManager +{ + public async Task GetAsync(RunId runId, CancellationToken cancellationToken = default) + { + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var loaded = await RunLoader.LoadAsync(db, runId, cancellationToken).ConfigureAwait(false); + return loaded is null ? null : RunLoader.ToSnapshot(loaded); + } + } + + public async Task> ListAsync(RunQuery query, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(query); + var (from, to) = TimeWindow.Resolve(query.From, query.To, time); + var limit = Math.Clamp(query.Limit, 1, 100); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var runs = db.Runs.AsNoTracking().Where(r => r.CreatedAt >= from && r.CreatedAt < to); + if (query.PipelineKey is { } pipelineKey) + { + var pipelineId = await db.Pipelines.Where(p => p.Key == pipelineKey).Select(p => (Guid?)p.Id) + .SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false); + runs = runs.Where(r => r.PipelineId == pipelineId); + } + + if (query.StageKey is { } stageKey) + { + runs = runs.Where(r => r.StageKey == stageKey); + } + + if (query.Kind is { } kind) + { + runs = runs.Where(r => r.Kind == kind); + } + + if (query.Status is { } status) + { + runs = runs.Where(r => r.Status == status); + } + + if (query.Trigger is { } trigger) + { + runs = runs.Where(r => r.Trigger == trigger); + } + + if (KeysetCursor.Decode(query.Cursor) is { } cursor) + { + runs = runs.Where(r => r.CreatedAt < cursor.At || (r.CreatedAt == cursor.At && r.Id < cursor.Id)); + } + + var rows = await runs.OrderByDescending(r => r.CreatedAt).ThenByDescending(r => r.Id).Take(limit + 1) + .Select(r => new + { + r.Id, + r.Kind, + r.Mode, + r.Status, + r.PipelineKey, + r.PipelineVersion, + r.StageKey, + r.StageVersion, + r.Trigger, + r.TriggeredBy, + r.CreatedAt, + r.StartedAt, + r.FinishedAt, + r.Error, + }) + .ToListAsync(cancellationToken).ConfigureAwait(false); + var items = rows.Take(limit).Select(r => new RunSummary( + new RunId(r.Id), r.Kind, r.Mode, r.Status, r.PipelineKey ?? r.StageKey ?? string.Empty, r.PipelineVersion ?? r.StageVersion ?? 0, + r.Trigger, r.TriggeredBy, r.CreatedAt, r.StartedAt, r.FinishedAt, StoredJson.Error(r.Error))).ToList(); + var next = rows.Count > limit ? KeysetCursor.Encode(rows[limit - 1].CreatedAt, rows[limit - 1].Id) : null; + return new KeysetPage(items, next); + } + } + + public async Task GetAttemptLogAsync(RunId runId, AttemptId attemptId, LogStream stream, CancellationToken cancellationToken = default) + { + var createdAt = runId.CreatedAt; + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + return await db.AttemptLogs.AsNoTracking() + .Where(l => l.AttemptId == attemptId.Value && l.RunCreatedAt == createdAt && l.Stream == stream) + .Select(l => new AttemptLog(l.Stream, l.Content, l.Truncated, l.OriginalBytes)) + .SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false); + } + } + + public async Task CancelAsync(RunId runId, Actor actor, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + var createdAt = runId.CreatedAt; + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var runs = db.Runs.Where(r => r.Id == runId.Value && r.CreatedAt == createdAt); + var steps = db.StepRuns.Where(s => s.RunId == runId.Value && s.RunCreatedAt == createdAt); + var now = time.GetUtcNow(); + var error = StoredJson.Error(new RunError(ErrorKind.Cancelled, $"Cancelled by {actor.Subject}.")); + + // Transitions are conditional on the current status, so a run that finishes meanwhile is never reopened. + var waiting = await runs.Where(r => r.Status == RunStatus.Pending || r.Status == RunStatus.Queued) + .ExecuteUpdateAsync(s => s + .SetProperty(r => r.Status, RunStatus.Cancelled) + .SetProperty(r => r.FinishedAt, now) + .SetProperty(r => r.Error, error) + .SetProperty(r => r.CancelRequestedAt, now) + .SetProperty(r => r.CancelRequestedBy, actor.Subject), cancellationToken).ConfigureAwait(false); + if (waiting > 0) + { + await steps.Where(s => s.Status == StepStatus.Queued) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.Status, StepStatus.Cancelled), cancellationToken).ConfigureAwait(false); + await steps.Where(s => s.Status == StepStatus.Pending) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.Status, StepStatus.Skipped), cancellationToken).ConfigureAwait(false); + } + else + { + var running = await runs.Where(r => r.Status == RunStatus.Running || r.Status == RunStatus.Cancelling) + .ExecuteUpdateAsync(s => s + .SetProperty(r => r.Status, RunStatus.Cancelling) + .SetProperty(r => r.CancelRequestedAt, r => r.CancelRequestedAt ?? now) + .SetProperty(r => r.CancelRequestedBy, r => r.CancelRequestedBy ?? actor.Subject), cancellationToken).ConfigureAwait(false); + if (running == 0) + { + var status = await runs.Select(r => (RunStatus?)r.Status).SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false) + ?? throw new RunNotFoundException(runId); + throw new PsrDomainException(PsrErrorCodes.RunNotCancellable, $"Run {runId} already finished ({status})."); + } + + // Runs executing in this process stop now; the queue package stops the others (control messages). + engine.RequestCancellation(runId); + } + + var target = await runs.Select(r => r.PipelineKey ?? r.StageKey).SingleAsync(cancellationToken).ConfigureAwait(false); + AuditLog.Add(db, new AuditEntry(actor, AuditActions.RunCancelled, "Run", runId.Value, target), now); + await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + } + + public async Task ReprocessAsync(RunId runId, Actor actor, ExecutionMode mode = ExecutionMode.Async, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(actor); + if (mode == ExecutionMode.Async) + { + throw new PsrConfigurationException( + "Asynchronous reprocessing needs a run queue (PythonScriptRunner.RabbitMQ). Use ExecutionMode.Sync to reprocess in this process."); + } + + var request = await PrepareResumeAsync(runId, actor, cancellationToken).ConfigureAwait(false); + try + { + var result = await engine.ResumeAsync(request, cancellationToken).ConfigureAwait(false); + await result.DisposeAsync().ConfigureAwait(false); + } + catch + { + await RestoreFailedAsync(runId).ConfigureAwait(false); + throw; + } + + return await GetAsync(runId, CancellationToken.None).ConfigureAwait(false) ?? throw new RunNotFoundException(runId); + } + + /// A reprocess that could not start leaves the run failed (it can be reprocessed again). + private async Task RestoreFailedAsync(RunId runId) + { + var createdAt = runId.CreatedAt; + var db = await factory.CreateDbContextAsync(CancellationToken.None).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + await db.Runs.Where(r => r.Id == runId.Value && r.CreatedAt == createdAt && r.Status == RunStatus.Running) + .ExecuteUpdateAsync(s => s.SetProperty(r => r.Status, RunStatus.Failed)).ConfigureAwait(false); + } + } + + /// Checks the run can be reprocessed, claims it (Failed → Running) and loads the state to resume. + private async Task PrepareResumeAsync(RunId runId, Actor actor, CancellationToken cancellationToken) + { + var createdAt = runId.CreatedAt; + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var loaded = await RunLoader.LoadAsync(db, runId, cancellationToken).ConfigureAwait(false) ?? throw new RunNotFoundException(runId); + var run = loaded.Run; + if (run.Status != RunStatus.Failed) + { + throw new PsrDomainException(PsrErrorCodes.RunNotFailed, $"Only failed runs can be reprocessed; run {runId} is {run.Status}."); + } + + var now = time.GetUtcNow(); + var needed = ArtifactRefs.Referenced(StoredJson.Object(run.Inputs)) + .Concat(run.Steps.Where(s => s.Status == StepStatus.Succeeded).SelectMany(s => ArtifactRefs.Referenced(StoredJson.Object(s.Outputs)))) + .ToHashSet(); + var expired = needed.Where(id => !loaded.Artifacts.TryGetValue(id, out var a) || a.ContentDeletedAt is not null || a.ContentExpiresAt <= now).ToList(); + if (expired.Count > 0) + { + throw new PsrDomainException(PsrErrorCodes.ArtifactsExpired, + $"Run {runId} cannot be reprocessed: {expired.Count} file(s) it needs are beyond the retention."); + } + + var (stage, pipeline) = await LoadDefinitionAsync(db, run, cancellationToken).ConfigureAwait(false); + var claimed = await db.Runs.Where(r => r.Id == run.Id && r.CreatedAt == createdAt && r.Status == RunStatus.Failed) + .ExecuteUpdateAsync(s => s.SetProperty(r => r.Status, RunStatus.Running), cancellationToken).ConfigureAwait(false); + if (claimed == 0) + { + throw new PsrDomainException(PsrErrorCodes.RunNotFailed, $"Run {runId} is already being reprocessed."); + } + + AuditLog.Add(db, new AuditEntry(actor, AuditActions.RunReprocessed, "Run", run.Id, RunLoader.TargetKey(run), + new JsonObject { ["mode"] = nameof(ExecutionMode.Sync) }), now); + await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + + var (inputs, inputFiles) = loader.Inputs(loaded); + return new ResumeRequest + { + RunId = runId, + Kind = run.Kind, + Stage = stage, + Pipeline = pipeline, + Inputs = inputs, + InputFiles = inputFiles, + PreviousSteps = loader.ToStepResults(loaded), + TriggeredBy = actor.Subject, + }; + } + } + + /// The exact version the run executed (any status: archived versions can still be reprocessed). + private static async Task<(StageDefinition? Stage, PipelineDefinition? Pipeline)> LoadDefinitionAsync( + PsrDbContext db, RunEntity run, CancellationToken cancellationToken) + { + if (run.Kind is RunKind.Pipeline or RunKind.PipelineTest) + { + var version = await db.PipelineVersions.AsNoTracking().Include(v => v.Pipeline) + .SingleOrDefaultAsync(v => v.Id == run.PipelineVersionId, cancellationToken).ConfigureAwait(false) + ?? throw new VersionNotFoundException(run.PipelineKey ?? string.Empty, run.PipelineVersion ?? 0); + var steps = await EfCatalog.LoadStepsAsync(db, version.Id, cancellationToken).ConfigureAwait(false); + return (null, CatalogMapping.ToPipeline(version.Pipeline, version, steps)); + } + + var stageVersion = await db.StageVersions.AsNoTracking().Include(v => v.Stage) + .SingleOrDefaultAsync(v => v.Id == run.StageVersionId, cancellationToken).ConfigureAwait(false) + ?? throw new VersionNotFoundException(run.StageKey ?? string.Empty, run.StageVersion ?? 0); + return (CatalogMapping.ToStage(stageVersion.Stage, stageVersion), null); + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Runs/EfRunRecorder.cs b/src/PythonScriptRunner.EntityFrameworkCore/Runs/EfRunRecorder.cs new file mode 100644 index 0000000..b853553 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Runs/EfRunRecorder.cs @@ -0,0 +1,347 @@ +using System.Text.Json.Nodes; +using Microsoft.EntityFrameworkCore; +using PythonScriptRunner.EntityFrameworkCore.Catalog; +using PythonScriptRunner.EntityFrameworkCore.Entities; + +namespace PythonScriptRunner.EntityFrameworkCore.Runs; + +/// +/// Persists every run executed by the engine in this process (synchronous runs, draft tests and reprocessing): run, +/// steps, attempts, log tails and files. Rows are addressed by run id + creation time (the partition) + step order. +/// +internal sealed class EfRunRecorder( + IDbContextFactory factory, + ArtifactStore artifacts, + IIdempotencyStore idempotency, + RunLoader loader, + PsrEntityFrameworkOptions options) : IRunRecorder +{ + private static readonly RunStatus[] Terminal = [RunStatus.Succeeded, RunStatus.Failed, RunStatus.Cancelled]; + + public async ValueTask RunStartingAsync(RunStarting run, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(run); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var transaction = await db.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + await using (transaction.ConfigureAwait(false)) + { + if (run.IdempotencyKey is { } key + && await idempotency.ReserveAsync(db, IdempotencyScopes.For(run.Kind, run.TargetKey), key, run.RunId, cancellationToken).ConfigureAwait(false) + is { } existingId) + { + var existing = await RunLoader.LoadAsync(db, existingId, cancellationToken).ConfigureAwait(false); + if (existing is not null) + { + return loader.ToResult(existing, deduplicated: true); + } + } + + var (row, retention) = await NewRunAsync(db, run, cancellationToken).ConfigureAwait(false); + db.Runs.Add(row); + await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + + await StoreFilesAsync(db, run.RunId, null, ArtifactDirection.Input, run.Inputs, run.InputFiles, retention, cancellationToken).ConfigureAwait(false); + row.Inputs = ArtifactRefs.ToStored(run.Inputs, run.InputFiles).ToJsonString(); + await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + return null; + } + } + } + + public async ValueTask RunResumingAsync(RunResuming run, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(run); + var createdAt = run.RunId.CreatedAt; + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + await db.Runs.Where(r => r.Id == run.RunId.Value && r.CreatedAt == createdAt) + .ExecuteUpdateAsync(s => s + .SetProperty(r => r.Status, RunStatus.Running) + .SetProperty(r => r.Outputs, (string?)null) + .SetProperty(r => r.Error, (string?)null) + .SetProperty(r => r.FinishedAt, (DateTimeOffset?)null), cancellationToken).ConfigureAwait(false); + await db.StepRuns.Where(s => s.RunId == run.RunId.Value && s.RunCreatedAt == createdAt && s.Status == StepStatus.Skipped) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.Status, StepStatus.Pending), cancellationToken).ConfigureAwait(false); + } + } + + public async ValueTask StepStartedAsync(StepStarted step, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(step); + var createdAt = step.RunId.CreatedAt; + var inputs = ArtifactRefs.ToStored(step.Inputs, step.InputFiles).ToJsonString(); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + await db.StepRuns.Where(s => s.RunId == step.RunId.Value && s.RunCreatedAt == createdAt && s.Order == step.Order) + .ExecuteUpdateAsync(s => s + .SetProperty(x => x.Status, StepStatus.Running) + .SetProperty(x => x.Inputs, inputs) + .SetProperty(x => x.StartedAt, step.StartedAt) + .SetProperty(x => x.FinishedAt, (DateTimeOffset?)null) + .SetProperty(x => x.Error, (string?)null), cancellationToken).ConfigureAwait(false); + } + } + + public async ValueTask AttemptStartedAsync(AttemptStarted attempt, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(attempt); + var createdAt = attempt.RunId.CreatedAt; + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var stepRunId = await db.StepRuns + .Where(s => s.RunId == attempt.RunId.Value && s.RunCreatedAt == createdAt && s.Order == attempt.Order) + .Select(s => s.Id).SingleAsync(cancellationToken).ConfigureAwait(false); + db.StepAttempts.Add(new StepAttemptEntity + { + Id = attempt.AttemptId.Value, + StepRunId = stepRunId, + RunId = attempt.RunId.Value, + RunCreatedAt = createdAt, + Attempt = attempt.Number, + Cause = attempt.Cause, + TriggeredBy = attempt.TriggeredBy, + Status = AttemptStatus.Running, + StartedAt = attempt.StartedAt, + }); + await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + await db.StepRuns.Where(s => s.Id == stepRunId && s.RunCreatedAt == createdAt) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.CurrentAttempt, attempt.Number), cancellationToken).ConfigureAwait(false); + } + } + + public async ValueTask AttemptFinishedAsync(AttemptFinished attempt, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(attempt); + var createdAt = attempt.RunId.CreatedAt; + var result = attempt.Attempt; + var error = StoredJson.Error(result.Error); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + await db.StepAttempts.Where(a => a.Id == result.Id.Value && a.RunCreatedAt == createdAt) + .ExecuteUpdateAsync(s => s + .SetProperty(a => a.Status, result.Status) + .SetProperty(a => a.ExitCode, result.ExitCode) + .SetProperty(a => a.ErrorKind, result.Error == null ? null : result.Error.Kind) + .SetProperty(a => a.Error, error) + .SetProperty(a => a.EnvironmentHash, result.EnvironmentHash) + .SetProperty(a => a.FinishedAt, result.StartedAt + result.Duration) + .SetProperty(a => a.DurationMs, (int)result.Duration.TotalMilliseconds) + .SetProperty(a => a.OverheadMs, (int)result.Overhead.TotalMilliseconds), cancellationToken).ConfigureAwait(false); + foreach (var (stream, tail) in (ReadOnlySpan<(LogStream, LogTail)>)[(LogStream.Stdout, result.Stdout), (LogStream.Stderr, result.Stderr)]) + { + if (tail.OriginalBytes > 0 || tail.Content.Length > 0) + { + db.AttemptLogs.Add(new AttemptLogEntity + { + AttemptId = result.Id.Value, + RunCreatedAt = createdAt, + Stream = stream, + Content = tail.Content, + Truncated = tail.Truncated, + OriginalBytes = tail.OriginalBytes, + }); + } + } + + await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + } + + public async ValueTask StepFinishedAsync(StepFinished step, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(step); + var result = step.Step; + var createdAt = step.RunId.CreatedAt; + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + var transaction = await db.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + await using (transaction.ConfigureAwait(false)) + { + var row = await db.StepRuns.SingleAsync(s => s.RunId == step.RunId.Value && s.RunCreatedAt == createdAt && s.Order == result.Order, cancellationToken) + .ConfigureAwait(false); + if (result.Status == StepStatus.Succeeded && result.Outputs is { } outputs) + { + var retention = await ArtifactRetentionAsync(db, step.RunId, cancellationToken).ConfigureAwait(false); + await StoreFilesAsync(db, step.RunId, new StepRunId(row.Id), ArtifactDirection.Output, outputs, result.Files, retention, cancellationToken) + .ConfigureAwait(false); + row.Outputs = ArtifactRefs.ToStored(outputs, result.Files).ToJsonString(); + } + + row.Status = result.Status; + row.Error = StoredJson.Error(result.Error); + row.FinishedAt = result.Status == StepStatus.Skipped ? null : step.FinishedAt; + row.CurrentAttempt = result.Attempts.Count == 0 ? row.CurrentAttempt : result.Attempts[^1].Number; + await db.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + } + } + + public async ValueTask RunFinishedAsync(RunFinished run, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(run); + var createdAt = run.RunId.CreatedAt; + var outputs = run.Outputs is null ? null : ArtifactRefs.ToStored(run.Outputs, run.OutputFiles).ToJsonString(); + var error = StoredJson.Error(run.Error); + var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using (db.ConfigureAwait(false)) + { + await db.Runs.Where(r => r.Id == run.RunId.Value && r.CreatedAt == createdAt) + .ExecuteUpdateAsync(s => s + .SetProperty(r => r.Status, run.Status) + .SetProperty(r => r.Outputs, outputs) + .SetProperty(r => r.Error, error) + .SetProperty(r => r.FinishedAt, run.FinishedAt), cancellationToken).ConfigureAwait(false); + if (Terminal.Contains(run.Status)) + { + // Steps interrupted by an unexpected error (or never reached) must not stay open. + var interrupted = run.Status == RunStatus.Cancelled ? StepStatus.Cancelled : StepStatus.Failed; + await db.StepRuns.Where(s => s.RunId == run.RunId.Value && s.RunCreatedAt == createdAt && (s.Status == StepStatus.Running || s.Status == StepStatus.Queued)) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.Status, interrupted).SetProperty(x => x.FinishedAt, run.FinishedAt), cancellationToken) + .ConfigureAwait(false); + await db.StepRuns.Where(s => s.RunId == run.RunId.Value && s.RunCreatedAt == createdAt && s.Status == StepStatus.Pending) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.Status, StepStatus.Skipped), cancellationToken).ConfigureAwait(false); + await db.StepAttempts.Where(a => a.RunId == run.RunId.Value && a.RunCreatedAt == createdAt && a.Status == AttemptStatus.Running) + .ExecuteUpdateAsync(s => s + .SetProperty(a => a.Status, run.Status == RunStatus.Cancelled ? AttemptStatus.Cancelled : AttemptStatus.Failed) + .SetProperty(a => a.FinishedAt, run.FinishedAt), cancellationToken).ConfigureAwait(false); + } + } + } + + /// Builds the run row and its pending steps; resolves catalog ids and the retention of the pipeline. + private async Task<(RunEntity Run, TimeSpan? ArtifactRetention)> NewRunAsync(PsrDbContext db, RunStarting run, CancellationToken cancellationToken) + { + var createdAt = run.RunId.CreatedAt; + var row = new RunEntity + { + Id = run.RunId.Value, + CreatedAt = createdAt, + Kind = run.Kind, + Mode = ExecutionMode.Sync, + Trigger = run.Trigger, + TriggeredBy = run.TriggeredBy, + Status = RunStatus.Running, + Inputs = run.Inputs.ToJsonString(), + IdempotencyKey = run.IdempotencyKey, + StartedAt = run.StartedAt, + }; + + TimeSpan? artifactRetention = null; + var pipelineSteps = new Dictionary(); + if (run.Kind is RunKind.Pipeline or RunKind.PipelineTest) + { + var pipeline = await db.PipelineVersions.AsNoTracking() + .Where(v => v.Pipeline.Key == run.TargetKey && v.Version == run.TargetVersion) + .Select(v => new { VersionId = v.Id, v.PipelineId, v.Pipeline.ArtifactRetentionDays, v.Pipeline.RunRetentionDays }) + .SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false); + row.PipelineKey = run.TargetKey; + row.PipelineVersion = run.TargetVersion; + if (pipeline is not null) + { + row.PipelineId = pipeline.PipelineId; + row.PipelineVersionId = pipeline.VersionId; + row.ExpiresAt = pipeline.RunRetentionDays is { } days ? createdAt.AddDays(days) : null; + artifactRetention = pipeline.ArtifactRetentionDays is { } artifactDays ? TimeSpan.FromDays(artifactDays) : null; + pipelineSteps = await db.PipelineSteps.AsNoTracking().Where(s => s.PipelineVersionId == pipeline.VersionId) + .ToDictionaryAsync(s => s.Order, s => s.Id, cancellationToken).ConfigureAwait(false); + } + } + else + { + row.StageKey = run.TargetKey; + row.StageVersion = run.TargetVersion; + } + + var stageKeys = run.Steps.Select(s => s.StageKey).Distinct().ToList(); + var stageVersions = await db.StageVersions.AsNoTracking() + .Where(v => stageKeys.Contains(v.Stage.Key)) + .Select(v => new { v.Id, v.Stage.Key, v.Version }) + .ToListAsync(cancellationToken).ConfigureAwait(false); + Guid? StageVersionId(string key, int version) => stageVersions.FirstOrDefault(v => v.Key == key && v.Version == version)?.Id; + + if (row.Kind is RunKind.Stage or RunKind.StageTest) + { + row.StageVersionId = StageVersionId(run.TargetKey, run.TargetVersion); + } + + foreach (var step in run.Steps) + { + row.Steps.Add(new StepRunEntity + { + Id = Guid.CreateVersion7(createdAt), + RunId = row.Id, + RunCreatedAt = createdAt, + PipelineStepId = pipelineSteps.TryGetValue(step.Order, out var pipelineStepId) ? pipelineStepId : null, + StepKey = step.StepKey, + Order = step.Order, + StageVersionId = StageVersionId(step.StageKey, step.StageVersion), + StageKey = step.StageKey, + StageVersion = step.StageVersion, + Status = StepStatus.Pending, + }); + } + + return (row, artifactRetention); + } + + private static async Task ArtifactRetentionAsync(PsrDbContext db, RunId runId, CancellationToken cancellationToken) + { + var createdAt = runId.CreatedAt; + var days = await db.Runs.AsNoTracking() + .Where(r => r.Id == runId.Value && r.CreatedAt == createdAt && r.PipelineId != null) + .Join(db.Pipelines, r => r.PipelineId, p => p.Id, (r, p) => p.ArtifactRetentionDays) + .SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return days is { } value ? TimeSpan.FromDays(value) : null; + } + + /// Stores the files that are not stored yet (a file passed between steps keeps its artifact). + private async Task StoreFilesAsync( + PsrDbContext db, + RunId runId, + StepRunId? stepRunId, + ArtifactDirection direction, + JsonObject wire, + IReadOnlyDictionary> files, + TimeSpan? retention, + CancellationToken cancellationToken) + { + foreach (var (field, list) in files) + { + var isList = wire[field] is JsonArray; + for (var i = 0; i < list.Count; i++) + { + var file = list[i]; + if (file.ArtifactId is not null) + { + continue; + } + + var content = await file.OpenReadAsync(cancellationToken).ConfigureAwait(false); + await using (content.ConfigureAwait(false)) + { + var stored = await artifacts.SaveAsync(db, new NewArtifact + { + RunId = runId, + StepRunId = stepRunId, + Direction = direction, + FieldName = field, + ListIndex = isList ? i : null, + FileName = file.FileName, + ContentType = file.ContentType, + Retention = retention ?? options.Retention.Artifacts, + }, content, cancellationToken).ConfigureAwait(false); + file.AttachArtifact(new ArtifactId(stored.Id)); + } + } + } + } +} diff --git a/src/PythonScriptRunner.EntityFrameworkCore/Runs/RunLoader.cs b/src/PythonScriptRunner.EntityFrameworkCore/Runs/RunLoader.cs new file mode 100644 index 0000000..88d1d10 --- /dev/null +++ b/src/PythonScriptRunner.EntityFrameworkCore/Runs/RunLoader.cs @@ -0,0 +1,147 @@ +using Microsoft.EntityFrameworkCore; +using PythonScriptRunner.EntityFrameworkCore.Catalog; +using PythonScriptRunner.EntityFrameworkCore.Entities; + +namespace PythonScriptRunner.EntityFrameworkCore.Runs; + +/// A run read from the database with its steps, attempts, log streams and files. +internal sealed record LoadedRun(RunEntity Run, IReadOnlyList<(Guid AttemptId, LogStream Stream)> Logs, IReadOnlyDictionary Artifacts); + +/// Reads persisted runs (the partition is found from the timestamp of the run identifier). +internal sealed class RunLoader(ArtifactStore artifactStore) +{ + public static async Task LoadAsync(PsrDbContext db, RunId runId, CancellationToken cancellationToken) + { + var createdAt = runId.CreatedAt; + var run = await db.Runs.AsNoTracking() + .Include(r => r.Steps).ThenInclude(s => s.Attempts) + .AsSplitQuery() + .SingleOrDefaultAsync(r => r.Id == runId.Value && r.CreatedAt == createdAt, cancellationToken).ConfigureAwait(false); + if (run is null) + { + return null; + } + + var attemptIds = run.Steps.SelectMany(s => s.Attempts).Select(a => a.Id).ToList(); + var logs = await db.AttemptLogs.AsNoTracking() + .Where(l => l.RunCreatedAt == createdAt && attemptIds.Contains(l.AttemptId)) + .Select(l => new { l.AttemptId, l.Stream }) + .ToListAsync(cancellationToken).ConfigureAwait(false); + var artifacts = await db.Artifacts.AsNoTracking() + .Where(a => a.RunId == runId.Value && a.RunCreatedAt == createdAt) + .ToDictionaryAsync(a => a.Id, cancellationToken).ConfigureAwait(false); + return new LoadedRun(run, [.. logs.Select(l => (l.AttemptId, l.Stream))], artifacts); + } + + public static RunSnapshot ToSnapshot(LoadedRun loaded) + { + var run = loaded.Run; + var stepKeys = run.Steps.ToDictionary(s => s.Id, s => s.StepKey); + return new RunSnapshot + { + Id = new RunId(run.Id), + Kind = run.Kind, + Mode = run.Mode, + Status = run.Status, + TargetKey = TargetKey(run), + TargetVersion = TargetVersion(run), + Trigger = run.Trigger, + TriggeredBy = run.TriggeredBy, + Inputs = StoredJson.Object(run.Inputs) ?? [], + Outputs = StoredJson.Object(run.Outputs), + Error = StoredJson.Error(run.Error), + IdempotencyKey = run.IdempotencyKey, + CreatedAt = run.CreatedAt, + QueuedAt = run.QueuedAt, + StartedAt = run.StartedAt, + FinishedAt = run.FinishedAt, + CancelRequestedAt = run.CancelRequestedAt, + CancelRequestedBy = run.CancelRequestedBy, + ExpiresAt = run.ExpiresAt, + Steps = [.. run.Steps.OrderBy(s => s.Order).Select(s => new StepRunSnapshot + { + Id = new StepRunId(s.Id), + StepKey = s.StepKey, + Order = s.Order, + StageKey = s.StageKey, + StageVersion = s.StageVersion, + Status = s.Status, + CurrentAttempt = s.CurrentAttempt, + Inputs = StoredJson.Object(s.Inputs), + Outputs = StoredJson.Object(s.Outputs), + Error = StoredJson.Error(s.Error), + StartedAt = s.StartedAt, + FinishedAt = s.FinishedAt, + Attempts = [.. s.Attempts.OrderBy(a => a.Attempt).Select(a => new AttemptSnapshot + { + Id = new AttemptId(a.Id), + Number = a.Attempt, + Cause = a.Cause, + TriggeredBy = a.TriggeredBy, + Status = a.Status, + WorkerId = a.WorkerId, + EnvironmentHash = a.EnvironmentHash, + ExitCode = a.ExitCode, + Error = StoredJson.Error(a.Error), + StartedAt = a.StartedAt, + FinishedAt = a.FinishedAt, + Duration = a.DurationMs is { } duration ? TimeSpan.FromMilliseconds(duration) : null, + Overhead = a.OverheadMs is { } overhead ? TimeSpan.FromMilliseconds(overhead) : null, + Logs = [.. loaded.Logs.Where(l => l.AttemptId == a.Id).Select(l => l.Stream).Order()], + })], + })], + Artifacts = [.. loaded.Artifacts.Values.OrderBy(a => a.Id) + .Select(a => ArtifactStore.ToInfo(a, a.StepRunId is { } stepRunId ? stepKeys.GetValueOrDefault(stepRunId) : null))], + }; + } + + /// Builds the result returned for a deduplicated request (files are streamed from the database on demand). + public RunResult ToResult(LoadedRun loaded, bool deduplicated) + { + var run = loaded.Run; + var (outputs, files) = ArtifactRefs.ToWire(StoredJson.Object(run.Outputs), loaded.Artifacts, artifactStore.OpenReadAsync); + var duration = run.StartedAt is { } started && run.FinishedAt is { } finished ? finished - started : TimeSpan.Zero; + return new RunResult(new RunId(run.Id), run.Kind, run.Status, duration, ToStepResults(loaded), outputs, files, StoredJson.Error(run.Error), + deduplicated: deduplicated); + } + + /// Steps as the engine sees them (outputs in wire format with their files). + public List ToStepResults(LoadedRun loaded) => + [.. loaded.Run.Steps.OrderBy(s => s.Order).Select(s => + { + var (outputs, files) = ArtifactRefs.ToWire(StoredJson.Object(s.Outputs), loaded.Artifacts, artifactStore.OpenReadAsync); + return new StepResult + { + StepKey = s.StepKey, + Order = s.Order, + StageKey = s.StageKey, + StageVersion = s.StageVersion, + Status = s.Status, + Attempts = [.. s.Attempts.OrderBy(a => a.Attempt).Select(ToAttempt)], + Outputs = s.Outputs is null ? null : outputs, + Files = files, + Error = StoredJson.Error(s.Error), + }; + })]; + + public (System.Text.Json.Nodes.JsonObject Wire, Dictionary> Files) Inputs(LoadedRun loaded) => + ArtifactRefs.ToWire(StoredJson.Object(loaded.Run.Inputs), loaded.Artifacts, artifactStore.OpenReadAsync); + + public static string TargetKey(RunEntity run) => run.PipelineKey ?? run.StageKey ?? string.Empty; + + public static int TargetVersion(RunEntity run) => run.PipelineVersion ?? run.StageVersion ?? 0; + + private static AttemptResult ToAttempt(StepAttemptEntity attempt) => new() + { + Id = new AttemptId(attempt.Id), + Number = attempt.Attempt, + Cause = attempt.Cause, + Status = attempt.Status, + StartedAt = attempt.StartedAt, + Duration = TimeSpan.FromMilliseconds(attempt.DurationMs ?? 0), + Overhead = TimeSpan.FromMilliseconds(attempt.OverheadMs ?? 0), + ExitCode = attempt.ExitCode, + Error = StoredJson.Error(attempt.Error), + EnvironmentHash = attempt.EnvironmentHash, + }; +} From 4a54b28e142841e3f9cc1332c768382d513cda4c Mon Sep 17 00:00:00 2001 From: net0well Date: Mon, 19 Oct 2026 19:29:51 -0300 Subject: [PATCH 3/4] =?UTF-8?q?test(ef):=20testes=20de=20persist=C3=AAncia?= =?UTF-8?q?=20com=20PostgreSQL=2016=20via=20Testcontainers=20(spec=20004,?= =?UTF-8?q?=20spec=20003)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Um teste por critério de aceite da spec 004 (AC-1 a AC-20) e da spec 003 (AC-10 reprocesso, AC-14 teste de rascunho). Co-Authored-By: Claude Opus 5.5 --- .../CatalogAdminTests.cs | 171 ++++++++ .../Infrastructure/PostgresFixture.cs | 27 ++ .../Infrastructure/PsrTestHost.cs | 202 ++++++++++ .../MaintenanceTests.cs | 133 +++++++ ...iptRunner.EntityFrameworkCore.Tests.csproj | 30 ++ .../RunHistoryTests.cs | 375 ++++++++++++++++++ .../SchemaTests.cs | 92 +++++ 7 files changed, 1030 insertions(+) create mode 100644 tests/PythonScriptRunner.EntityFrameworkCore.Tests/CatalogAdminTests.cs create mode 100644 tests/PythonScriptRunner.EntityFrameworkCore.Tests/Infrastructure/PostgresFixture.cs create mode 100644 tests/PythonScriptRunner.EntityFrameworkCore.Tests/Infrastructure/PsrTestHost.cs create mode 100644 tests/PythonScriptRunner.EntityFrameworkCore.Tests/MaintenanceTests.cs create mode 100644 tests/PythonScriptRunner.EntityFrameworkCore.Tests/PythonScriptRunner.EntityFrameworkCore.Tests.csproj create mode 100644 tests/PythonScriptRunner.EntityFrameworkCore.Tests/RunHistoryTests.cs create mode 100644 tests/PythonScriptRunner.EntityFrameworkCore.Tests/SchemaTests.cs diff --git a/tests/PythonScriptRunner.EntityFrameworkCore.Tests/CatalogAdminTests.cs b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/CatalogAdminTests.cs new file mode 100644 index 0000000..877e1d2 --- /dev/null +++ b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/CatalogAdminTests.cs @@ -0,0 +1,171 @@ +using PythonScriptRunner.EntityFrameworkCore.Tests.Infrastructure; +using static PythonScriptRunner.EntityFrameworkCore.Tests.Infrastructure.TestScripts; + +namespace PythonScriptRunner.EntityFrameworkCore.Tests; + +public sealed class CatalogAdminTests(PostgresFixture postgres, PythonFixture python) +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task Spec004_AC03_PublishLocksDependenciesAndAudits() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + + var published = await host.PublishStageAsync("eco", Echo, [Text("texto")], [Text("texto")], requirements: "six==1.16.0"); + + published.Status.ShouldBe(VersionStatus.Published); + published.LockedRequirements.ShouldNotBeNull().ShouldContain("six==1.16.0"); + published.LockedRequirements.ShouldContain("--hash=sha256:"); + published.EnvironmentHash.ShouldNotBeNull().Length.ShouldBe(64); + published.PublishedBy.ShouldBe(PsrTestHost.Author.Subject); + (await host.Admin.GetStageAsync("eco", Ct))!.LatestPublishedVersion.ShouldBe(1); + (await host.ScalarAsync("SELECT actor FROM psr.audit_events WHERE action = 'StagePublished' AND entity_key = 'eco@1'")) + .ShouldBe(PsrTestHost.Author.Subject); + } + + [Fact] + public async Task Spec004_AC04_SyntaxErrorReportsLineAndColumnAndKeepsTheDraft() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + var draft = await host.Admin.CreateStageAsync(new NewCatalogItem("quebrado", "Quebrado"), PsrTestHost.Author, Ct); + var saved = await host.Admin.SaveStageDraftAsync("quebrado", 1, draft.Data with { Script = "from psrunner import stage\n\ndef run(inputs:\n" }, + draft.RowVersion, PsrTestHost.Author, Ct); + + var error = await Should.ThrowAsync(() => host.Admin.PublishStageAsync("quebrado", 1, saved.RowVersion, PsrTestHost.Author, Ct)); + + error.Code.ShouldBe(PsrErrorCodes.ScriptSyntaxError); + error.Line.ShouldBe(3); + error.Column.ShouldNotBeNull(); + (await host.Admin.GetStageDraftAsync("quebrado", Ct)).ShouldNotBeNull().Version.ShouldBe(1); + } + + [Fact] + public async Task Spec004_AC05_UnknownDependencyFailsWithTheUvMessage() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + + var error = await Should.ThrowAsync(() => + host.PublishStageAsync("dependencia", Echo, [Text("texto")], [Text("texto")], requirements: "pacote-inexistente-psr-0000==1.0")); + + error.Code.ShouldBe(PsrErrorCodes.DependencyResolutionFailed); + error.Message.ShouldContain("pacote-inexistente-psr-0000"); + (await host.Admin.GetStageDraftAsync("dependencia", Ct)).ShouldNotBeNull(); + } + + [Fact] + public async Task Spec004_AC06_PublishedVersionsAreImmutableAndDraftsCopyThem() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + var v1 = await host.PublishStageAsync("eco", Echo, [Text("texto")], [Text("texto")]); + + var error = await Should.ThrowAsync(() => + host.Admin.SaveStageDraftAsync("eco", 1, v1.Data with { Script = "mudou" }, v1.RowVersion, PsrTestHost.Author, Ct)); + var draft = await host.Admin.CreateStageDraftFromAsync("eco", 1, PsrTestHost.Author, Ct); + + error.Code.ShouldBe(PsrErrorCodes.VersionImmutable); + draft.Version.ShouldBe(2); + draft.Data.Script.ShouldBe(Echo); + draft.Data.Inputs.Single().Name.ShouldBe("texto"); + (await Should.ThrowAsync(() => host.Admin.CreateStageDraftFromAsync("eco", 1, PsrTestHost.Author, Ct))) + .Code.ShouldBe(PsrErrorCodes.DraftAlreadyExists); + } + + [Fact] + public async Task Spec004_AC07_SecondSaveWithTheSameRowVersionConflicts() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + var draft = await host.Admin.CreateStageAsync(new NewCatalogItem("disputado", "Disputado"), PsrTestHost.Author, Ct); + + var first = await host.Admin.SaveStageDraftAsync("disputado", 1, draft.Data with { Script = "# autora 1" }, draft.RowVersion, PsrTestHost.Author, Ct); + var error = await Should.ThrowAsync(() => + host.Admin.SaveStageDraftAsync("disputado", 1, draft.Data with { Script = "# autora 2" }, draft.RowVersion, Actor.User("outra@teste"), Ct)); + + error.Code.ShouldBe(PsrErrorCodes.ConcurrencyConflict); + var current = error.Current.ShouldBeOfType(); + current.Data.Script.ShouldBe("# autora 1"); + current.RowVersion.ShouldBe(first.RowVersion); + } + + [Fact] + public async Task Spec004_AC08_ArchivingAStageUsedByAPublishedPipelineIsBlocked() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + await host.PublishStageAsync("eco", Echo, [Text("texto")], [Text("texto")]); + await host.PublishPipelineAsync("usa-eco", EchoPipeline()); + + var error = await Should.ThrowAsync(() => host.Admin.ArchiveStageVersionAsync("eco", 1, PsrTestHost.Author, Ct)); + + error.Code.ShouldBe(PsrErrorCodes.VersionReferenced); + error.Pipelines.ShouldBe(["usa-eco@1"]); + } + + [Fact] + public async Task Spec004_AC09_OnlyUnusedStagesCanBeDeleted() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + await host.PublishStageAsync("nunca-usado", Echo, [Text("texto")], [Text("texto")]); + await host.PublishStageAsync("eco", Echo, [Text("texto")], [Text("texto")]); + await host.PublishPipelineAsync("usa-eco", EchoPipeline()); + await host.PublishStageAsync("executado", Echo, [Text("texto")], [Text("texto")]); + await using (var result = await host.Runner.RunStageAsync("executado", new { texto = "x" }, cancellationToken: Ct)) + { + result.EnsureSuccess(); + } + + await host.Admin.DeleteStageAsync("nunca-usado", Actor.User("admin@teste"), Ct); + + (await host.Admin.GetStageAsync("nunca-usado", Ct)).ShouldBeNull(); + (await host.ScalarAsync("SELECT count(*) FROM psr.stage_versions v JOIN psr.stages s ON s.id = v.stage_id WHERE s.key = 'nunca-usado'")).ShouldBe(0); + (await Should.ThrowAsync(() => host.Admin.DeleteStageAsync("eco", PsrTestHost.Author, Ct))).Code.ShouldBe(PsrErrorCodes.DeleteNotAllowed); + (await Should.ThrowAsync(() => host.Admin.DeleteStageAsync("executado", PsrTestHost.Author, Ct))).Code.ShouldBe(PsrErrorCodes.DeleteNotAllowed); + } + + [Fact] + public async Task Spec004_AC16_PipelineRetentionCannotExceedTheGlobalOne() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + var draft = await host.Admin.CreatePipelineAsync(new NewCatalogItem("retida", "Retida"), PsrTestHost.Author, Ct); + var pipeline = (await host.Admin.GetPipelineAsync("retida", Ct))!; + + var error = await Should.ThrowAsync(() => + host.Admin.UpdatePipelineRetentionAsync("retida", new PipelineRetention(30, null), pipeline.RowVersion, PsrTestHost.Author, Ct)); + var shorter = await host.Admin.UpdatePipelineRetentionAsync("retida", new PipelineRetention(2, 30), pipeline.RowVersion, PsrTestHost.Author, Ct); + + error.Code.ShouldBe(PsrErrorCodes.RetentionExceedsGlobal); + shorter.Retention.ShouldBe(new PipelineRetention(2, 30)); + draft.Version.ShouldBe(1); + } + + [Fact] + public async Task ImportCreatesDraftsAndExistingKeysGetANewVersion() + { + await using var source = await PsrTestHost.CreateAsync(postgres, python); + await source.PublishStageAsync("eco", Echo, [Text("texto")], [Text("texto")]); + await source.PublishPipelineAsync("usa-eco", EchoPipeline()); + using var zip = new MemoryStream(); + await source.Admin.ExportPipelineVersionAsync("usa-eco", 1, zip, Ct); + + await using var target = await PsrTestHost.CreateAsync(postgres, python); + zip.Position = 0; + var fresh = await target.Admin.ImportAsync(zip, PsrTestHost.Author, Ct); + zip.Position = 0; + var again = await source.Admin.ImportAsync(zip, PsrTestHost.Author, Ct); + + fresh.Items.ShouldBe([new ImportedItem("Stage", "eco", 1, true), new ImportedItem("Pipeline", "usa-eco", 1, true)]); + again.Items.ShouldBe([new ImportedItem("Stage", "eco", 2, false), new ImportedItem("Pipeline", "usa-eco", 2, false)]); + var pipelineDraft = (await source.Admin.GetPipelineDraftAsync("usa-eco", Ct))!; + pipelineDraft.Data.Steps.Single().StageVersion.ShouldBe(2, "references follow the stage imported in the same archive"); + (await source.Admin.GetStageVersionAsync("eco", 1, Ct))!.Status.ShouldBe(VersionStatus.Published); + zip.Position = 0; + (await Should.ThrowAsync(() => source.Admin.ImportAsync(zip, PsrTestHost.Author, Ct))) + .Code.ShouldBe(PsrErrorCodes.ImportInvalid); + } + + private static PipelineDraftData EchoPipeline() => new() + { + Inputs = [Text("texto")], + Steps = [new PipelineStepDefinition("eco", "eco", 1)], + Outputs = [new PipelineOutputDefinition("texto", From.Step("eco", "texto"))], + }; +} diff --git a/tests/PythonScriptRunner.EntityFrameworkCore.Tests/Infrastructure/PostgresFixture.cs b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/Infrastructure/PostgresFixture.cs new file mode 100644 index 0000000..cfe76ea --- /dev/null +++ b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/Infrastructure/PostgresFixture.cs @@ -0,0 +1,27 @@ +using Npgsql; +using Testcontainers.PostgreSql; + +[assembly: AssemblyFixture(typeof(PythonScriptRunner.EntityFrameworkCore.Tests.Infrastructure.PostgresFixture))] + +namespace PythonScriptRunner.EntityFrameworkCore.Tests.Infrastructure; + +/// One PostgreSQL 16 container (the minimum supported version) for the whole test run; each test gets its own database. +public sealed class PostgresFixture : IAsyncLifetime +{ + private readonly PostgreSqlContainer _container = new PostgreSqlBuilder("postgres:16-alpine").Build(); + + public async ValueTask InitializeAsync() => await _container.StartAsync(); + + public async ValueTask DisposeAsync() => await _container.DisposeAsync(); + + /// Creates an empty database and returns its connection string. + public async Task CreateDatabaseAsync() + { + var name = $"psr_{Guid.NewGuid():N}"; + await using var connection = new NpgsqlConnection(_container.GetConnectionString()); + await connection.OpenAsync(); + await using var command = new NpgsqlCommand($"CREATE DATABASE {name}", connection); + await command.ExecuteNonQueryAsync(); + return new NpgsqlConnectionStringBuilder(_container.GetConnectionString()) { Database = name, Pooling = true }.ConnectionString; + } +} diff --git a/tests/PythonScriptRunner.EntityFrameworkCore.Tests/Infrastructure/PsrTestHost.cs b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/Infrastructure/PsrTestHost.cs new file mode 100644 index 0000000..8d8bed1 --- /dev/null +++ b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/Infrastructure/PsrTestHost.cs @@ -0,0 +1,202 @@ +using Microsoft.EntityFrameworkCore; +using Npgsql; + +namespace PythonScriptRunner.EntityFrameworkCore.Tests.Infrastructure; + +/// Runner + PostgreSQL persistence on a fresh database (maintenance job off; tests call it explicitly). +public sealed class PsrTestHost : IAsyncDisposable +{ + private PsrTestHost(ServiceProvider services, string connectionString, string workRoot) + { + Services = services; + ConnectionString = connectionString; + WorkRoot = workRoot; + } + + public static Actor Author { get; } = Actor.User("autora@teste"); + + public ServiceProvider Services { get; } + + public string ConnectionString { get; } + + public string WorkRoot { get; } + + public IPythonRunner Runner => Services.GetRequiredService(); + + public ICatalogAdmin Admin => Services.GetRequiredService(); + + public IRunManager Runs => Services.GetRequiredService(); + + public IArtifactStore Artifacts => Services.GetRequiredService(); + + public PsrMaintenance Maintenance => Services.GetRequiredService(); + + public static async Task CreateAsync( + PostgresFixture postgres, + PythonFixture python, + TimeProvider? time = null, + Action? configure = null, + bool migrate = true, + Action? services = null) + { + var connectionString = await postgres.CreateDatabaseAsync(); + var collection = new ServiceCollection(); + if (time is not null) + { + collection.AddSingleton(time); + } + + var workRoot = Path.Combine(Path.GetTempPath(), "psr-tests", "work", Guid.NewGuid().ToString("N")); + collection.AddPythonScriptRunner(o => + { + o.WorkRoot = workRoot; + o.EnvironmentCacheRoot = python.EnvironmentCacheRoot; + o.UvPath = python.UvPath; + o.PsrunnerRequirement = python.WheelPath; + }) + .AddEntityFrameworkCore(o => + { + o.UseNpgsql(connectionString); + o.Maintenance.Enabled = false; + o.LatestVersionCacheDuration = TimeSpan.Zero; + configure?.Invoke(o); + }); + services?.Invoke(collection); + var provider = collection.BuildServiceProvider(validateScopes: true); + if (migrate) + { + await provider.MigratePythonScriptRunnerAsync(TestContext.Current.CancellationToken); + } + + return new PsrTestHost(provider, connectionString, workRoot); + } + + public async Task DbAsync() => + await Services.GetRequiredService>().CreateDbContextAsync(TestContext.Current.CancellationToken); + + /// Runs a scalar query on the test database. + public async Task ScalarAsync(string sql) + { + await using var connection = new NpgsqlConnection(ConnectionString); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await using var command = new NpgsqlCommand(sql, connection); + return (T)(await command.ExecuteScalarAsync(TestContext.Current.CancellationToken))!; + } + + /// Runs a query on the test database and returns the first column of every row. + public async Task> ColumnAsync(string sql) + { + await using var connection = new NpgsqlConnection(ConnectionString); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await using var command = new NpgsqlCommand(sql, connection); + await using var reader = await command.ExecuteReaderAsync(TestContext.Current.CancellationToken); + var values = new List(); + while (await reader.ReadAsync(TestContext.Current.CancellationToken)) + { + values.Add(reader.GetFieldValue(0)); + } + + return values; + } + + /// Creates, saves and publishes a stage. + public async Task PublishStageAsync( + string key, string script, IReadOnlyList inputs, IReadOnlyList outputs, string? requirements = null, + RetryPolicy? retry = null) + { + var ct = TestContext.Current.CancellationToken; + var existing = await Admin.GetStageAsync(key, ct); + var draft = existing is null + ? await Admin.CreateStageAsync(new NewCatalogItem(key, key), Author, ct) + : await Admin.CreateStageDraftFromAsync(key, existing.Versions[0].Version, Author, ct); + var saved = await Admin.SaveStageDraftAsync(key, draft.Version, draft.Data with + { + Script = script, + Inputs = inputs, + Outputs = outputs, + Requirements = requirements, + Retry = retry ?? RetryPolicy.None, + }, draft.RowVersion, Author, ct); + return await Admin.PublishStageAsync(key, saved.Version, saved.RowVersion, Author, ct); + } + + /// Creates, saves and publishes a pipeline. + public async Task PublishPipelineAsync(string key, PipelineDraftData data) + { + var ct = TestContext.Current.CancellationToken; + var draft = await Admin.CreatePipelineAsync(new NewCatalogItem(key, key), Author, ct); + var saved = await Admin.SavePipelineDraftAsync(key, draft.Version, data, draft.RowVersion, Author, ct); + return await Admin.PublishPipelineAsync(key, saved.Version, saved.RowVersion, Author, ct); + } + + public async ValueTask DisposeAsync() + { + await Services.DisposeAsync(); + try + { + if (Directory.Exists(WorkRoot)) + { + Directory.Delete(WorkRoot, recursive: true); + } + } + catch (IOException) + { + // Best effort. + } + } +} + +/// Scripts shared by the persistence tests. +public static class TestScripts +{ + public const string Echo = """ + from psrunner import stage + + + @stage + def run(inputs, ctx): + print("eco:", inputs["texto"]) + return {"texto": inputs["texto"]} + """; + + public const string WriteFile = """ + from psrunner import stage + + + @stage + def run(inputs, ctx): + output = ctx.new_file("saida.txt", "text/plain") + output.path.write_text(inputs["texto"] * 3, encoding="utf-8") + print("arquivo gerado") + return {"arquivo": output, "tamanho": len(inputs["texto"]) * 3} + """; + + public const string ReadFile = """ + from psrunner import stage + + + @stage + def run(inputs, ctx): + return {"conteudo": inputs["arquivo"].read_text(encoding="utf-8")} + """; + + /// Fails the first time (creates the marker file) and succeeds afterwards. + public const string FailOnce = """ + import os + + from psrunner import StageError, stage + + + @stage + def run(inputs, ctx): + marker = inputs["marcador"] + if not os.path.exists(marker): + open(marker, "w").close() + raise StageError("primeira tentativa falha", code="PRIMEIRA") + return {"texto": inputs["texto"].upper()} + """; + + public static FieldDefinition Text(string name) => new(name, FieldType.String); + + public static FieldDefinition File(string name) => new(name, FieldType.File); +} diff --git a/tests/PythonScriptRunner.EntityFrameworkCore.Tests/MaintenanceTests.cs b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/MaintenanceTests.cs new file mode 100644 index 0000000..b338e1f --- /dev/null +++ b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/MaintenanceTests.cs @@ -0,0 +1,133 @@ +using Microsoft.Extensions.Time.Testing; +using Npgsql; +using PythonScriptRunner.EntityFrameworkCore.Tests.Infrastructure; +using static PythonScriptRunner.EntityFrameworkCore.Tests.Infrastructure.TestScripts; + +namespace PythonScriptRunner.EntityFrameworkCore.Tests; + +public sealed class MaintenanceTests(PostgresFixture postgres, PythonFixture python) +{ + private const string Partitions = """ + SELECT child.relname::text FROM pg_inherits inh + JOIN pg_class child ON child.oid = inh.inhrelid JOIN pg_class parent ON parent.oid = inh.inhparent + JOIN pg_namespace ns ON ns.oid = parent.relnamespace + WHERE ns.nspname = 'psr' AND parent.relkind = 'p' AND child.relkind = 'r' + """; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task Spec004_AC14_CreatesPartitionsAheadAndDropsThoseBeyondTheRetention() + { + var time = new FakeTimeProvider(new DateTimeOffset(2030, 1, 7, 12, 0, 0, TimeSpan.Zero)); + await using var host = await PsrTestHost.CreateAsync(postgres, python, time); + + var first = await host.Maintenance.RunOnceAsync(Ct); + var created = await host.ColumnAsync(Partitions); + time.Advance(TimeSpan.FromDays(100)); + var later = await host.Maintenance.RunOnceAsync(Ct); + var remaining = await host.ColumnAsync(Partitions); + + first.ShouldNotBeNull().PartitionsCreated.ShouldBe(5 * 5 + 8 + 4); + created.ShouldContain("runs_p20300107"); + created.ShouldContain("runs_p20300204"); + created.ShouldContain("artifacts_p20300204"); + created.ShouldContain("artifact_contents_p20300114"); + created.ShouldContain("audit_events_p203004"); + + later.ShouldNotBeNull(); + later.PartitionsDropped.ShouldContain("runs_p20300107"); + later.PartitionsDropped.ShouldContain("attempt_logs_p20300107"); + later.PartitionsDropped.ShouldContain("artifact_contents_p20300107"); + remaining.ShouldNotContain("runs_p20300107"); + remaining.ShouldContain("runs_p20300114", "partitions inside the 90-day retention are kept"); + remaining.ShouldContain("audit_events_p203001", "audit is kept for a year"); + remaining.ShouldContain("runs_p20300415"); + remaining.ShouldContain("runs_p20300513"); + remaining.ShouldContain("artifact_contents_p20300424"); + } + + [Fact] + public async Task Spec004_AC15_ShorterPipelineRetentionDeletesFileContentsButKeepsMetadata() + { + var time = new FakeTimeProvider(DateTimeOffset.UtcNow); + await using var host = await PsrTestHost.CreateAsync(postgres, python, time); + await host.PublishStageAsync("gera-arquivo", WriteFile, [Text("texto")], [File("arquivo"), new FieldDefinition("tamanho", FieldType.Integer)]); + await host.PublishPipelineAsync("curta", new PipelineDraftData + { + Inputs = [Text("texto")], + Steps = [new PipelineStepDefinition("gera", "gera-arquivo", 1)], + Outputs = [new PipelineOutputDefinition("arquivo", From.Step("gera", "arquivo"))], + }); + var pipeline = (await host.Admin.GetPipelineAsync("curta", Ct))!; + await host.Admin.UpdatePipelineRetentionAsync("curta", new PipelineRetention(2, null), pipeline.RowVersion, PsrTestHost.Author, Ct); + RunId runId; + await using (var result = await host.Runner.RunPipelineAsync("curta", new { texto = "z" }, cancellationToken: Ct)) + { + result.EnsureSuccess(); + runId = result.Id; + } + + var artifactId = (await host.Runs.GetAsync(runId, Ct))!.Artifacts.Single().Id; + time.Advance(TimeSpan.FromDays(1)); + await host.Maintenance.RunOnceAsync(Ct); + await using (var stillThere = await host.Artifacts.OpenReadAsync(artifactId, Ct)) + { + (await new StreamReader(stillThere).ReadToEndAsync(Ct)).ShouldBe("zzz"); + } + + time.Advance(TimeSpan.FromDays(1).Add(TimeSpan.FromMinutes(1))); + var report = await host.Maintenance.RunOnceAsync(Ct); + + report.ShouldNotBeNull().ContentsDeleted.ShouldBe(1); + var info = await host.Artifacts.GetAsync(artifactId, Ct); + info.ShouldNotBeNull().ContentDeletedAt.ShouldNotBeNull(); + (await host.ScalarAsync("SELECT count(*) FROM psr.artifact_contents")).ShouldBe(0); + (await Should.ThrowAsync(() => host.Artifacts.OpenReadAsync(artifactId, Ct))).Code.ShouldBe(PsrErrorCodes.ArtifactExpired); + (await host.Runs.GetAsync(runId, Ct)).ShouldNotBeNull("run metadata stays until the run retention"); + } + + [Fact] + public async Task Spec004_AC19_UnfinishedRunsEndExpiredBeforeTheirFilesAreDropped() + { + var time = new FakeTimeProvider(DateTimeOffset.UtcNow); + await using var host = await PsrTestHost.CreateAsync(postgres, python, time); + var old = await RunHistoryTests.InsertRunAsync(host, time.GetUtcNow(), RunStatus.Queued); + time.Advance(TimeSpan.FromDays(1)); + var recent = await RunHistoryTests.InsertRunAsync(host, time.GetUtcNow(), RunStatus.Queued); + + time.Advance(TimeSpan.FromDays(5)); + var report = await host.Maintenance.RunOnceAsync(Ct); + + report.ShouldNotBeNull().RunsExpired.ShouldBe(1); + var expired = (await host.Runs.GetAsync(old, Ct))!; + expired.Status.ShouldBe(RunStatus.Failed); + expired.Error!.Kind.ShouldBe(ErrorKind.Expired); + (await host.Runs.GetAsync(recent, Ct))!.Status.ShouldBe(RunStatus.Queued, "5 days old is still inside retention minus one day"); + } + + [Fact] + public async Task Spec004_AC20_OnlyOneInstanceRunsEachCycle() + { + await using var first = await PsrTestHost.CreateAsync(postgres, python); + await using var holder = new NpgsqlConnection(first.ConnectionString); + await holder.OpenAsync(Ct); + await using (var acquire = new NpgsqlCommand($"SELECT pg_advisory_lock({PsrMaintenance.LockKey})", holder)) + { + await acquire.ExecuteScalarAsync(Ct); + } + + var whileHeld = await first.Maintenance.RunOnceAsync(Ct); + await using (var release = new NpgsqlCommand($"SELECT pg_advisory_unlock({PsrMaintenance.LockKey})", holder)) + { + await release.ExecuteScalarAsync(Ct); + } + + var afterRelease = await first.Maintenance.RunOnceAsync(Ct); + var concurrent = await Task.WhenAll(Enumerable.Range(0, 4).Select(_ => first.Maintenance.RunOnceAsync(Ct))); + + whileHeld.ShouldBeNull(); + afterRelease.ShouldNotBeNull(); + concurrent.Count(r => r is not null).ShouldBeGreaterThanOrEqualTo(1); + } +} diff --git a/tests/PythonScriptRunner.EntityFrameworkCore.Tests/PythonScriptRunner.EntityFrameworkCore.Tests.csproj b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/PythonScriptRunner.EntityFrameworkCore.Tests.csproj new file mode 100644 index 0000000..ecd7e66 --- /dev/null +++ b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/PythonScriptRunner.EntityFrameworkCore.Tests.csproj @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/PythonScriptRunner.EntityFrameworkCore.Tests/RunHistoryTests.cs b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/RunHistoryTests.cs new file mode 100644 index 0000000..f707ca5 --- /dev/null +++ b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/RunHistoryTests.cs @@ -0,0 +1,375 @@ +using System.Text.Json.Nodes; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Time.Testing; +using PythonScriptRunner.EntityFrameworkCore.Entities; +using PythonScriptRunner.EntityFrameworkCore.Tests.Infrastructure; +using static PythonScriptRunner.EntityFrameworkCore.Tests.Infrastructure.TestScripts; + +namespace PythonScriptRunner.EntityFrameworkCore.Tests; + +public sealed class RunHistoryTests(PostgresFixture postgres, PythonFixture python) +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task Spec004_AC10_SyncRunIsPersistedWithStepsAttemptsLogsAndFiles() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + await host.PublishStageAsync("gera-arquivo", WriteFile, [Text("texto")], [File("arquivo"), new FieldDefinition("tamanho", FieldType.Integer)]); + + await using var result = await host.Runner.RunStageAsync("gera-arquivo", new { texto = "ab" }, cancellationToken: Ct); + result.EnsureSuccess(); + var run = await host.Runs.GetAsync(result.Id, Ct); + + run.ShouldNotBeNull(); + run.Status.ShouldBe(RunStatus.Succeeded); + run.Kind.ShouldBe(RunKind.Stage); + run.Mode.ShouldBe(ExecutionMode.Sync); + run.Trigger.ShouldBe(RunTrigger.Library); + run.TriggeredBy.ShouldBe("system:library"); + run.TargetKey.ShouldBe("gera-arquivo"); + run.Inputs["texto"]!.GetValue().ShouldBe("ab"); + run.FinishedAt.ShouldNotBeNull(); + var step = run.Steps.ShouldHaveSingleItem(); + step.Status.ShouldBe(StepStatus.Succeeded); + step.Outputs!["tamanho"]!.GetValue().ShouldBe(6); + var attempt = step.Attempts.ShouldHaveSingleItem(); + attempt.Status.ShouldBe(AttemptStatus.Succeeded); + attempt.Cause.ShouldBe(AttemptCause.Initial); + attempt.EnvironmentHash.ShouldNotBeNull(); + attempt.Logs.ShouldContain(LogStream.Stdout); + (await host.Runs.GetAttemptLogAsync(run.Id, attempt.Id, LogStream.Stdout, Ct))!.Content.ShouldContain("arquivo gerado"); + + var artifact = run.Artifacts.ShouldHaveSingleItem(); + artifact.Direction.ShouldBe(ArtifactDirection.Output); + artifact.StepKey.ShouldBe("gera-arquivo"); + artifact.FileName.ShouldBe("saida.txt"); + artifact.SizeBytes.ShouldBe(6); + artifact.Sha256.ShouldBe(result.GetFile("arquivo").Sha256); + run.Outputs!["arquivo"]!["$artifact"]!.GetValue().ShouldBe(artifact.Id.ToString()); + await using var content = await host.Artifacts.OpenReadAsync(artifact.Id, Ct); + (await new StreamReader(content).ReadToEndAsync(Ct)).ShouldBe("ababab"); + } + + [Fact] + public async Task FilesPassedBetweenStepsAreStoredOnce() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + await host.PublishStageAsync("gera-arquivo", WriteFile, [Text("texto")], [File("arquivo"), new FieldDefinition("tamanho", FieldType.Integer)]); + await host.PublishStageAsync("le-arquivo", ReadFile, [File("arquivo")], [Text("conteudo")]); + await host.PublishPipelineAsync("arquivo-entre-passos", new PipelineDraftData + { + Inputs = [Text("texto"), File("anexo")], + Steps = [new PipelineStepDefinition("gera", "gera-arquivo", 1), new PipelineStepDefinition("le", "le-arquivo", 1)], + Outputs = [new PipelineOutputDefinition("conteudo", From.Step("le", "conteudo")), new PipelineOutputDefinition("arquivo", From.Step("gera", "arquivo"))], + }); + + await using var result = await host.Runner.RunPipelineAsync("arquivo-entre-passos", + new { texto = "xy", anexo = PsrFile.FromText("entrada", "anexo.txt") }, cancellationToken: Ct); + result.EnsureSuccess(); + var run = (await host.Runs.GetAsync(result.Id, Ct))!; + + result.GetOutput("conteudo").ShouldBe("xyxyxy"); + run.Artifacts.Count.ShouldBe(2); + var input = run.Artifacts.Single(a => a.Direction == ArtifactDirection.Input); + input.FieldName.ShouldBe("anexo"); + run.Inputs["anexo"]!["$artifact"]!.GetValue().ShouldBe(input.Id.ToString()); + var produced = run.Artifacts.Single(a => a.Direction == ArtifactDirection.Output); + run.Steps[1].Inputs!["arquivo"]!["$artifact"]!.GetValue().ShouldBe(produced.Id.ToString()); + run.Outputs!["arquivo"]!["$artifact"]!.GetValue().ShouldBe(produced.Id.ToString()); + } + + [Fact] + public async Task Spec004_AC11_FiftyMegabyteFileIsStreamed() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + var runId = await InsertRunAsync(host, DateTimeOffset.UtcNow, RunStatus.Succeeded); + var path = Path.Combine(Path.GetTempPath(), $"psr-50mb-{Guid.NewGuid():N}.bin"); + await using (var file = new FileStream(path, FileMode.Create)) + { + var chunk = new byte[1024 * 1024]; + Random.Shared.NextBytes(chunk); + for (var i = 0; i < 50; i++) + { + await file.WriteAsync(chunk, Ct); + } + } + + try + { + await host.Artifacts.SaveAsync(new NewArtifact { RunId = runId, Direction = ArtifactDirection.Input, FieldName = "aquecimento", FileName = "a.bin", ContentType = "application/octet-stream" }, + new MemoryStream([1, 2, 3]), Ct); + var before = GC.GetTotalAllocatedBytes(precise: true); + ArtifactInfo info; + await using (var source = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 81920, useAsync: true)) + { + info = await host.Artifacts.SaveAsync(new NewArtifact + { + RunId = runId, + Direction = ArtifactDirection.Input, + FieldName = "grande", + FileName = "grande.bin", + ContentType = "application/octet-stream", + }, source, Ct); + } + + var afterWrite = GC.GetTotalAllocatedBytes(precise: true); + await using (var stored = await host.Artifacts.OpenReadAsync(info.Id, Ct)) + { + await stored.CopyToAsync(Stream.Null, Ct); + } + + var afterRead = GC.GetTotalAllocatedBytes(precise: true); + + info.SizeBytes.ShouldBe(50L * 1024 * 1024); + (afterWrite - before).ShouldBeLessThan(20L * 1024 * 1024, "writing must stream"); + (afterRead - afterWrite).ShouldBeLessThan(20L * 1024 * 1024, "reading must stream"); + } + finally + { + System.IO.File.Delete(path); + } + } + + [Fact] + public async Task Spec004_AC12_SameIdempotencyKeyReturnsTheSameRunFor24Hours() + { + var time = new FakeTimeProvider(DateTimeOffset.UtcNow); + await using var host = await PsrTestHost.CreateAsync(postgres, python, time); + await host.PublishStageAsync("eco", Echo, [Text("texto")], [Text("texto")]); + var options = new RunOptions { IdempotencyKey = "pedido-42" }; + + await using var first = await host.Runner.RunStageAsync("eco", new { texto = "um" }, options, Ct); + await using var second = await host.Runner.RunStageAsync("eco", new { texto = "dois" }, options, Ct); + time.Advance(TimeSpan.FromHours(25)); + await host.Maintenance.RunOnceAsync(Ct); + await using var third = await host.Runner.RunStageAsync("eco", new { texto = "tres" }, options, Ct); + + first.Deduplicated.ShouldBeFalse(); + second.Id.ShouldBe(first.Id); + second.Deduplicated.ShouldBeTrue(); + second.Status.ShouldBe(RunStatus.Succeeded); + second.GetOutput("texto").ShouldBe("um"); + third.Id.ShouldNotBe(first.Id); + third.Deduplicated.ShouldBeFalse(); + third.GetOutput("texto").ShouldBe("tres"); + (await host.ScalarAsync("SELECT count(*) FROM psr.runs")).ShouldBe(2); + } + + [Fact] + public async Task Spec004_AC13_KeysetPagesReturnEveryRunOnceInDescendingOrder() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + var start = DateTimeOffset.UtcNow.AddMinutes(-1); + var expected = new List(); + await using (var db = await host.DbAsync()) + { + for (var i = 0; i < 1000; i++) + { + var id = RunId.New(start.AddMilliseconds(i * 7)); + expected.Add(id); + db.Runs.Add(Run(id, RunStatus.Succeeded, stageKey: "lista")); + } + + await db.SaveChangesAsync(Ct); + } + + var seen = new List(); + string? cursor = null; + var page = 0; + do + { + var result = await host.Runs.ListAsync(new RunQuery { StageKey = "lista", From = start.AddMinutes(-1), To = start.AddHours(2), Cursor = cursor, Limit = 50 }, Ct); + seen.AddRange(result.Items); + cursor = result.NextCursor; + if (++page == 3) + { + await using var db = await host.DbAsync(); + for (var i = 0; i < 20; i++) + { + db.Runs.Add(Run(RunId.New(start.AddSeconds(30).AddMilliseconds(i)), RunStatus.Succeeded, stageKey: "lista")); + } + + await db.SaveChangesAsync(Ct); + } + } + while (cursor is not null); + + seen.Select(r => r.Id).ShouldBe(expected.AsEnumerable().Reverse(), "every run once, newest first; runs added meanwhile are newer than the cursor"); + page.ShouldBe(20); + } + + [Fact] + public async Task Spec004_AC17_RolledBackCreationLeavesNoOutboxNorIdempotencyKey() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + var idempotency = host.Services.GetRequiredService(); + var outbox = host.Services.GetRequiredService(); + var runId = RunId.New(); + + await using (var db = await host.DbAsync()) + { + await using var transaction = await db.Database.BeginTransactionAsync(Ct); + (await idempotency.ReserveAsync(db, IdempotencyScopes.Stage("eco"), "chave", runId, Ct)).ShouldBeNull(); + db.Runs.Add(Run(runId, RunStatus.Queued, stageKey: "eco")); + outbox.Enqueue(db, new OutboxMessage("psr.execute-step.v1", "psr.commands", "psr.step.execute", new JsonObject { ["runId"] = runId.ToString() })); + await db.SaveChangesAsync(Ct); + (await host.ScalarAsync("SELECT count(*) FROM psr.outbox_messages")).ShouldBe(0, "not visible before commit"); + await transaction.RollbackAsync(Ct); + } + + (await host.ScalarAsync("SELECT count(*) FROM psr.outbox_messages")).ShouldBe(0); + (await host.ScalarAsync("SELECT count(*) FROM psr.idempotency_keys")).ShouldBe(0); + (await host.ScalarAsync("SELECT count(*) FROM psr.runs")).ShouldBe(0); + } + + [Fact] + public async Task Spec004_AC18_OnlyHumanActionsAreAudited() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + var human = PsrTestHost.Author; + var created = await host.Admin.CreateStageAsync(new NewCatalogItem("auditado", "Antes"), human, Ct); + var stage = (await host.Admin.GetStageAsync("auditado", Ct))!; + await host.Admin.UpdateStageAsync("auditado", new CatalogItemChanges(Name: "Depois"), stage.RowVersion, human, Ct); + await host.Admin.CreateStageAsync(new NewCatalogItem("automatico", "Automatico"), Actor.System("queue"), Ct); + await host.PublishStageAsync("eco", Echo, [Text("texto")], [Text("texto")]); + var auditedBefore = await host.ScalarAsync("SELECT count(*) FROM psr.audit_events"); + await using (var result = await host.Runner.RunStageAsync("eco", new { texto = "lib" }, cancellationToken: Ct)) + { + result.EnsureSuccess(); + } + + var events = await host.Services.GetRequiredService().ListAsync(new AuditQuery(), Ct); + + created.Version.ShouldBe(1); + events.Items.ShouldAllBe(e => e.Actor == human.Subject); + events.Items.ShouldNotContain(e => e.EntityKey == "automatico"); + var update = events.Items.Single(e => e.Action == AuditActions.StageUpdated); + update.Details!["name"]!.AsArray().Select(n => n!.GetValue()).ShouldBe(["Antes", "Depois"]); + (await host.ScalarAsync("SELECT count(*) FROM psr.audit_events")).ShouldBe(auditedBefore, "library runs are recorded on the run, not audited"); + } + + [Fact] + public async Task Spec003_AC10_ReprocessReusesSucceededStepsAndRetriesTheFailedOne() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + var marker = Path.Combine(Path.GetTempPath(), $"psr-marcador-{Guid.NewGuid():N}"); + await host.PublishStageAsync("eco", Echo, [Text("texto")], [Text("texto")]); + await host.PublishStageAsync("falha-uma-vez", FailOnce, [Text("texto"), Text("marcador")], [Text("texto")]); + await host.PublishPipelineAsync("reprocesso", new PipelineDraftData + { + Inputs = [Text("texto")], + Steps = + [ + new PipelineStepDefinition("eco", "eco", 1), + new PipelineStepDefinition("falha", "falha-uma-vez", 1) { Bindings = [new InputBinding("marcador", From.Constant(marker))] }, + ], + Outputs = [new PipelineOutputDefinition("texto", From.Step("falha", "texto"))], + }); + + try + { + RunId runId; + await using (var failed = await host.Runner.RunPipelineAsync("reprocesso", new { texto = "abc" }, cancellationToken: Ct)) + { + failed.Status.ShouldBe(RunStatus.Failed); + failed.Error!.StepKey.ShouldBe("falha"); + runId = failed.Id; + } + + var reprocessed = await host.Runs.ReprocessAsync(runId, Actor.User("operador@teste"), ExecutionMode.Sync, Ct); + + reprocessed.Id.ShouldBe(runId); + reprocessed.Status.ShouldBe(RunStatus.Succeeded); + reprocessed.Outputs!["texto"]!.GetValue().ShouldBe("ABC"); + reprocessed.Steps[0].Attempts.ShouldHaveSingleItem("the succeeded step is not executed again"); + reprocessed.Steps[1].Attempts.Select(a => (a.Number, a.Cause)).ShouldBe([(1, AttemptCause.Initial), (2, AttemptCause.Reprocess)]); + reprocessed.Steps[1].Attempts[1].TriggeredBy.ShouldBe("operador@teste"); + (await Should.ThrowAsync(() => host.Runs.ReprocessAsync(runId, PsrTestHost.Author, ExecutionMode.Sync, Ct))) + .Code.ShouldBe(PsrErrorCodes.RunNotFailed); + (await host.ScalarAsync("SELECT count(*) FROM psr.audit_events WHERE action = 'RunReprocessed'")).ShouldBe(1); + } + finally + { + System.IO.File.Delete(marker); + } + } + + [Fact] + public async Task Spec003_AC14_PipelineDraftIsTestedWithoutPublishing() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + await host.PublishStageAsync("eco", Echo, [Text("texto")], [Text("texto")]); + var draft = await host.Admin.CreatePipelineAsync(new NewCatalogItem("rascunho", "Rascunho"), PsrTestHost.Author, Ct); + await host.Admin.SavePipelineDraftAsync("rascunho", 1, new PipelineDraftData + { + Inputs = [Text("texto")], + Steps = [new PipelineStepDefinition("eco", "eco", 1)], + Outputs = [new PipelineOutputDefinition("texto", From.Step("eco", "texto"))], + }, draft.RowVersion, PsrTestHost.Author, Ct); + var invalid = await host.Admin.CreatePipelineAsync(new NewCatalogItem("invalido", "Invalido"), PsrTestHost.Author, Ct); + await host.Admin.SavePipelineDraftAsync("invalido", 1, new PipelineDraftData { Steps = [new PipelineStepDefinition("eco", "eco", 1)] }, + invalid.RowVersion, PsrTestHost.Author, Ct); + + await using var result = await host.Admin.TestPipelineDraftAsync("rascunho", new { texto = "teste" }, PsrTestHost.Author, Ct); + var error = await Should.ThrowAsync(() => host.Admin.TestPipelineDraftAsync("invalido", new { }, PsrTestHost.Author, Ct)); + + result.Kind.ShouldBe(RunKind.PipelineTest); + result.GetOutput("texto").ShouldBe("teste"); + var run = (await host.Runs.GetAsync(result.Id, Ct))!; + run.Kind.ShouldBe(RunKind.PipelineTest); + run.TargetVersion.ShouldBe(1); + run.Trigger.ShouldBe(RunTrigger.Dashboard); + (await host.Admin.GetPipelineAsync("rascunho", Ct))!.LatestPublishedVersion.ShouldBeNull(); + (await host.Admin.GetPipelineDraftAsync("rascunho", Ct)).ShouldNotBeNull(); + await using (var db = await host.DbAsync()) + { + var row = await db.Runs.SingleAsync(r => r.Id == result.Id.Value, Ct); + var draftVersion = await db.PipelineVersions.SingleAsync(v => v.Pipeline.Key == "rascunho", Ct); + row.PipelineVersionId.ShouldBe(draftVersion.Id); + } + + error.Code.ShouldBe(PsrErrorCodes.PipelineInvalid); + error.Report.Errors.ShouldContain(e => e.Code == "PIPE_UNBOUND_REQUIRED" && e.Step == "eco" && e.Field == "texto"); + } + + [Fact] + public async Task CancellingAFinishedRunIsRejected() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python); + var finished = await InsertRunAsync(host, DateTimeOffset.UtcNow, RunStatus.Succeeded); + var queued = await InsertRunAsync(host, DateTimeOffset.UtcNow, RunStatus.Queued); + + var error = await Should.ThrowAsync(() => host.Runs.CancelAsync(finished, PsrTestHost.Author, Ct)); + await host.Runs.CancelAsync(queued, PsrTestHost.Author, Ct); + + error.Code.ShouldBe(PsrErrorCodes.RunNotCancellable); + var cancelled = (await host.Runs.GetAsync(queued, Ct))!; + cancelled.Status.ShouldBe(RunStatus.Cancelled); + cancelled.CancelRequestedBy.ShouldBe(PsrTestHost.Author.Subject); + } + + internal static RunEntity Run(RunId id, RunStatus status, string? stageKey = null, Guid? pipelineId = null) => new() + { + Id = id.Value, + CreatedAt = id.CreatedAt, + Kind = pipelineId is null ? RunKind.Stage : RunKind.Pipeline, + Mode = ExecutionMode.Async, + StageKey = stageKey, + StageVersion = stageKey is null ? null : 1, + PipelineId = pipelineId, + Trigger = RunTrigger.Library, + TriggeredBy = "system:library", + Status = status, + Inputs = "{}", + }; + + internal static async Task InsertRunAsync(PsrTestHost host, DateTimeOffset createdAt, RunStatus status, Guid? pipelineId = null) + { + var id = RunId.New(createdAt); + await using var db = await host.DbAsync(); + db.Runs.Add(Run(id, status, stageKey: pipelineId is null ? "manual" : null, pipelineId: pipelineId)); + await db.SaveChangesAsync(Ct); + return id; + } +} diff --git a/tests/PythonScriptRunner.EntityFrameworkCore.Tests/SchemaTests.cs b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/SchemaTests.cs new file mode 100644 index 0000000..62ffb7b --- /dev/null +++ b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/SchemaTests.cs @@ -0,0 +1,92 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; +using Npgsql; +using PythonScriptRunner.EntityFrameworkCore.Tests.Infrastructure; + +namespace PythonScriptRunner.EntityFrameworkCore.Tests; + +public sealed class SchemaTests(PostgresFixture postgres, PythonFixture python) +{ + private const string Signature = """ + SELECT string_agg(item, ',' ORDER BY item) FROM ( + SELECT c.relkind::text || ':' || c.relname AS item + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'psr' + UNION ALL + SELECT 'f:' || p.proname FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE n.nspname = 'psr' + ) objects + """; + + private static readonly string[] Tables = + [ + "stages", "stage_versions", "pipelines", "pipeline_versions", "pipeline_steps", "schedules", "runs", "step_runs", + "step_attempts", "attempt_logs", "artifacts", "artifact_contents", "outbox_messages", "idempotency_keys", "audit_events", + ]; + + [Fact] + public async Task Spec004_AC01_MigrateCreatesSchemaPartitionsAndIsRepeatable() + { + await using var migrated = await PsrTestHost.CreateAsync(postgres, python); + var tables = await migrated.ColumnAsync( + "SELECT c.relname::text FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'psr' AND c.relkind IN ('r', 'p') AND NOT c.relispartition"); + tables.ShouldBeSubsetOf([.. Tables, "__ef_migrations_history"]); + tables.Count.ShouldBe(Tables.Length + 1); + (await migrated.ScalarAsync("SELECT count(*) FROM pg_inherits i JOIN pg_class p ON p.oid = i.inhparent WHERE p.relname = 'runs'")).ShouldBe(5); + (await migrated.ScalarAsync("SELECT count(*) FROM pg_inherits i JOIN pg_class p ON p.oid = i.inhparent WHERE p.relname = 'artifact_contents'")).ShouldBe(8); + (await migrated.ScalarAsync("SELECT count(*) FROM pg_inherits i JOIN pg_class p ON p.oid = i.inhparent WHERE p.relname = 'audit_events'")).ShouldBe(4); + var signature = await migrated.ScalarAsync(Signature); + + await migrated.Services.MigratePythonScriptRunnerAsync(TestContext.Current.CancellationToken); + (await migrated.ScalarAsync(Signature)).ShouldBe(signature); + + var script = await migrated.Services.GeneratePythonScriptRunnerMigrationScriptAsync(idempotent: true, TestContext.Current.CancellationToken); + await using var scripted = await PsrTestHost.CreateAsync(postgres, python, migrate: false); + await ExecuteAsync(scripted.ConnectionString, script); + (await scripted.ScalarAsync(Signature)).ShouldBe(signature); + + await ExecuteAsync(scripted.ConnectionString, script); + (await scripted.ScalarAsync(Signature)).ShouldBe(signature); + (await scripted.ScalarAsync("SELECT count(*) FROM psr.__ef_migrations_history")).ShouldBe(1); + } + + [Fact] + public async Task Spec004_AC02_WithoutMigrationStartupDoesNotTouchTheDatabaseAndHealthReportsPendingSchema() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python, migrate: false, + configure: o => o.Maintenance.Enabled = true, + services: s => s.AddHealthChecks().AddPythonScriptRunnerDatabase()); + var hosted = host.Services.GetServices().ToList(); + foreach (var service in hosted) + { + await service.StartAsync(TestContext.Current.CancellationToken); + } + + var health = await host.Services.GetRequiredService().CheckHealthAsync(TestContext.Current.CancellationToken); + foreach (var service in hosted) + { + await service.StopAsync(TestContext.Current.CancellationToken); + } + + health.Status.ShouldBe(HealthStatus.Unhealthy); + health.Entries["psr-database"].Description.ShouldNotBeNull().ShouldContain("schema pending"); + (await host.ScalarAsync("SELECT count(*) FROM pg_namespace WHERE nspname = 'psr'")).ShouldBe(0); + } + + [Fact] + public async Task HealthIsHealthyAfterMigration() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python, services: s => s.AddHealthChecks().AddPythonScriptRunnerDatabase()); + + var health = await host.Services.GetRequiredService().CheckHealthAsync(TestContext.Current.CancellationToken); + + health.Status.ShouldBe(HealthStatus.Healthy, health.Entries["psr-database"].Description); + } + + private static async Task ExecuteAsync(string connectionString, string sql) + { + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(TestContext.Current.CancellationToken); + await using var command = new NpgsqlCommand(sql, connection); + await command.ExecuteNonQueryAsync(TestContext.Current.CancellationToken); + } +} From 541aa37454caf4fff2a158618f9eca265ee2fbff Mon Sep 17 00:00:00 2001 From: net0well Date: Mon, 19 Oct 2026 19:29:51 -0300 Subject: [PATCH 4/4] =?UTF-8?q?ci:=20testes=20de=20PostgreSQL=20s=C3=B3=20?= =?UTF-8?q?no=20Ubuntu;=20README=20com=20persist=C3=AAncia?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Os runners Windows do GitHub não executam containers Linux; no Windows rodam os testes do Core. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 7 +++++++ README.md | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4bdb406..3affef9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,9 +41,16 @@ jobs: - name: Build run: dotnet build --no-restore --configuration Release + # The persistence tests start PostgreSQL 16 with Testcontainers (Docker); GitHub's Windows runners + # cannot run Linux containers, so Windows runs the Core tests only. - name: Test + if: runner.os == 'Linux' run: dotnet test --no-build --configuration Release + - name: Test (Core) + if: runner.os == 'Windows' + run: dotnet test --no-build --configuration Release --project tests/PythonScriptRunner.Core.Tests + format: name: .NET format runs-on: ubuntu-latest diff --git a/README.md b/README.md index 1a9384d..aeea566 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ com contrato de entrada/saída, trilhas (**Pipelines**), execução assíncrona cron e painel Vue opcional. > Projeto em desenvolvimento. Esta versão entrega o **Core Runner** (execução síncrona de Stages e de -> trilhas) e o SDK Python **`psrunner`**. +> trilhas), o SDK Python **`psrunner`** e a **persistência em PostgreSQL** (catálogo versionado e histórico). ## Como funciona @@ -89,10 +89,42 @@ double[] termos = result.GetStepOutput("termos", "termos"); // depur A mesma trilha pode ser definida em C# com `AddPipeline("chave", p => p.Input(...).Step(...).Output(...))`. Se um passo falhar, os seguintes ficam como `Skipped` e a execução termina `Failed`. +## Persistência (PostgreSQL) + +Com o pacote `PythonScriptRunner.EntityFrameworkCore`, Stages e trilhas passam a viver no banco (schema `psr`) com +versões imutáveis (rascunho → publicado → arquivado), e **toda** execução fica registrada: passos, tentativas, cauda +de stdout/stderr e arquivos (em `bytea`, lidos e gravados em streaming). + +```csharp +builder.Services + .AddPythonScriptRunner() + .AddEntityFrameworkCore(o => + { + o.UseNpgsql(builder.Configuration.GetConnectionString("psr")!); // PostgreSQL 16+ + o.Retention.Artifacts = TimeSpan.FromDays(7); // arquivos + o.Retention.Runs = TimeSpan.FromDays(90); // histórico + }); +builder.Services.AddHealthChecks().AddPythonScriptRunnerDatabase(); + +var app = builder.Build(); +await app.Services.MigratePythonScriptRunnerAsync(); // explícito; ou GeneratePythonScriptRunnerMigrationScriptAsync() +``` + +- **Catálogo:** `ICatalogAdmin` cria rascunhos (concorrência otimista), publica (verifica a sintaxe com o Python + declarado e trava as dependências com `uv pip compile --generate-hashes`), arquiva, exclui, testa rascunhos e + importa/exporta zip no formato de pasta. +- **Execuções:** `IRunManager` consulta (paginação keyset), cancela e reprocessa uma execução que falhou — os passos + que deram certo reaproveitam as saídas e o que falhou ganha uma nova tentativa. `RunOptions.IdempotencyKey` devolve + a mesma execução por 24 h. +- **Retenção:** o histórico é particionado (semana para execuções, dia para conteúdo de arquivos, mês para auditoria); + um job de manutenção cria partições à frente e remove as antigas. Uma trilha pode encurtar a retenção. +- **Auditoria:** ações humanas (`Actor.User(...)`) ficam em `psr.audit_events`; disparos automáticos ficam na própria execução. + ## Requisitos - .NET 10 SDK - [uv](https://docs.astral.sh/uv/) no PATH (cria e cacheia um ambiente Python por conjunto de dependências) +- PostgreSQL 16+ apenas para o pacote de persistência (os testes usam Docker/Testcontainers) ## Estrutura @@ -100,6 +132,7 @@ Se um passo falhar, os seguintes ficam como `Skipped` e a execução termina `Fa |-------|----------| | `src/PythonScriptRunner.Abstractions` | contratos públicos (tipos de campo, `StageDefinition`, `RunResult`, `PsrFile`, interfaces) | | `src/PythonScriptRunner.Core` | catálogos (código e pasta), validação, ambientes uv, executor de processos, runner | +| `src/PythonScriptRunner.EntityFrameworkCore` | PostgreSQL: catálogo versionado, histórico, arquivos, idempotência, outbox, auditoria, retenção | | `python/psrunner` | SDK Python (`@stage`, `Context`, `File`, `StageError`, `psrunner.testing`, CLI) | | `schemas` | JSON Schemas do `stage.json` e do `*.pipeline.json` | | `tests` | testes .NET (unitários e de integração com Python real) e fixtures de trilha | @@ -109,6 +142,7 @@ Se um passo falhar, os seguintes ficam como `Skipped` e a execução termina `Fa ```bash dotnet build dotnet test --project tests/PythonScriptRunner.Core.Tests +dotnet test --project tests/PythonScriptRunner.EntityFrameworkCore.Tests # precisa de Docker cd python/psrunner uv sync