Skip to content

Commit be4beb5

Browse files
authored
Merge pull request #3 from net0well/fase-3/persistencia
Fase 3: persistência em PostgreSQL (spec 004)
2 parents c00401c + 541aa37 commit be4beb5

67 files changed

Lines changed: 11663 additions & 200 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,16 @@ jobs:
4141
- name: Build
4242
run: dotnet build --no-restore --configuration Release
4343

44+
# The persistence tests start PostgreSQL 16 with Testcontainers (Docker); GitHub's Windows runners
45+
# cannot run Linux containers, so Windows runs the Core tests only.
4446
- name: Test
47+
if: runner.os == 'Linux'
4548
run: dotnet test --no-build --configuration Release
4649

50+
- name: Test (Core)
51+
if: runner.os == 'Windows'
52+
run: dotnet test --no-build --configuration Release --project tests/PythonScriptRunner.Core.Tests
53+
4754
format:
4855
name: .NET format
4956
runs-on: ubuntu-latest

‎Directory.Packages.props‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,22 @@
44
</PropertyGroup>
55
<ItemGroup>
66
<PackageVersion Include="JsonSchema.Net" Version="9.4.1" />
7+
<PackageVersion Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.13" />
8+
<PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="10.0.13" />
79
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="10.0.13" />
810
<PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.13" />
911
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.13" />
1012
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.13" />
1113
<PackageVersion Include="Microsoft.Extensions.Diagnostics" Version="10.0.13" />
14+
<PackageVersion Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="10.0.13" />
1215
<PackageVersion Include="Microsoft.Extensions.Diagnostics.Testing" Version="10.11.0" />
1316
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.13" />
1417
<PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.13" />
1518
<PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.13" />
1619
<PackageVersion Include="Microsoft.Extensions.TimeProvider.Testing" Version="10.11.0" />
20+
<PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.3" />
1721
<PackageVersion Include="Shouldly" Version="4.3.0" />
22+
<PackageVersion Include="Testcontainers.PostgreSql" Version="4.16.0" />
1823
<PackageVersion Include="xunit.v3" Version="4.0.2" />
1924
</ItemGroup>
2025
</Project>

‎PythonScriptRunner.slnx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@
22
<Folder Name="/src/">
33
<Project Path="src/PythonScriptRunner.Abstractions/PythonScriptRunner.Abstractions.csproj" />
44
<Project Path="src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj" />
5+
<Project Path="src/PythonScriptRunner.EntityFrameworkCore/PythonScriptRunner.EntityFrameworkCore.csproj" />
56
</Folder>
67
<Folder Name="/tests/">
78
<Project Path="tests/PythonScriptRunner.Core.Tests/PythonScriptRunner.Core.Tests.csproj" />
9+
<Project Path="tests/PythonScriptRunner.EntityFrameworkCore.Tests/PythonScriptRunner.EntityFrameworkCore.Tests.csproj" />
810
</Folder>
911
</Solution>

‎README.md‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ com contrato de entrada/saída, trilhas (**Pipelines**), execução assíncrona
55
cron e painel Vue opcional.
66

77
> Projeto em desenvolvimento. Esta versão entrega o **Core Runner** (execução síncrona de Stages e de
8-
> trilhas) e o SDK Python **`psrunner`**.
8+
> trilhas), o SDK Python **`psrunner`** e a **persistência em PostgreSQL** (catálogo versionado e histórico).
99
1010
## Como funciona
1111

@@ -89,17 +89,50 @@ double[] termos = result.GetStepOutput<double[]>("termos", "termos"); // depur
8989
A mesma trilha pode ser definida em C# com `AddPipeline("chave", p => p.Input(...).Step(...).Output(...))`.
9090
Se um passo falhar, os seguintes ficam como `Skipped` e a execução termina `Failed`.
9191

92+
## Persistência (PostgreSQL)
93+
94+
Com o pacote `PythonScriptRunner.EntityFrameworkCore`, Stages e trilhas passam a viver no banco (schema `psr`) com
95+
versões imutáveis (rascunho → publicado → arquivado), e **toda** execução fica registrada: passos, tentativas, cauda
96+
de stdout/stderr e arquivos (em `bytea`, lidos e gravados em streaming).
97+
98+
```csharp
99+
builder.Services
100+
.AddPythonScriptRunner()
101+
.AddEntityFrameworkCore(o =>
102+
{
103+
o.UseNpgsql(builder.Configuration.GetConnectionString("psr")!); // PostgreSQL 16+
104+
o.Retention.Artifacts = TimeSpan.FromDays(7); // arquivos
105+
o.Retention.Runs = TimeSpan.FromDays(90); // histórico
106+
});
107+
builder.Services.AddHealthChecks().AddPythonScriptRunnerDatabase();
108+
109+
var app = builder.Build();
110+
await app.Services.MigratePythonScriptRunnerAsync(); // explícito; ou GeneratePythonScriptRunnerMigrationScriptAsync()
111+
```
112+
113+
- **Catálogo:** `ICatalogAdmin` cria rascunhos (concorrência otimista), publica (verifica a sintaxe com o Python
114+
declarado e trava as dependências com `uv pip compile --generate-hashes`), arquiva, exclui, testa rascunhos e
115+
importa/exporta zip no formato de pasta.
116+
- **Execuções:** `IRunManager` consulta (paginação keyset), cancela e reprocessa uma execução que falhou — os passos
117+
que deram certo reaproveitam as saídas e o que falhou ganha uma nova tentativa. `RunOptions.IdempotencyKey` devolve
118+
a mesma execução por 24 h.
119+
- **Retenção:** o histórico é particionado (semana para execuções, dia para conteúdo de arquivos, mês para auditoria);
120+
um job de manutenção cria partições à frente e remove as antigas. Uma trilha pode encurtar a retenção.
121+
- **Auditoria:** ações humanas (`Actor.User(...)`) ficam em `psr.audit_events`; disparos automáticos ficam na própria execução.
122+
92123
## Requisitos
93124

94125
- .NET 10 SDK
95126
- [uv](https://docs.astral.sh/uv/) no PATH (cria e cacheia um ambiente Python por conjunto de dependências)
127+
- PostgreSQL 16+ apenas para o pacote de persistência (os testes usam Docker/Testcontainers)
96128

97129
## Estrutura
98130

99131
| Pasta | Conteúdo |
100132
|-------|----------|
101133
| `src/PythonScriptRunner.Abstractions` | contratos públicos (tipos de campo, `StageDefinition`, `RunResult`, `PsrFile`, interfaces) |
102134
| `src/PythonScriptRunner.Core` | catálogos (código e pasta), validação, ambientes uv, executor de processos, runner |
135+
| `src/PythonScriptRunner.EntityFrameworkCore` | PostgreSQL: catálogo versionado, histórico, arquivos, idempotência, outbox, auditoria, retenção |
103136
| `python/psrunner` | SDK Python (`@stage`, `Context`, `File`, `StageError`, `psrunner.testing`, CLI) |
104137
| `schemas` | JSON Schemas do `stage.json` e do `*.pipeline.json` |
105138
| `tests` | testes .NET (unitários e de integração com Python real) e fixtures de trilha |
@@ -109,6 +142,7 @@ Se um passo falhar, os seguintes ficam como `Skipped` e a execução termina `Fa
109142
```bash
110143
dotnet build
111144
dotnet test --project tests/PythonScriptRunner.Core.Tests
145+
dotnet test --project tests/PythonScriptRunner.EntityFrameworkCore.Tests # precisa de Docker
112146

113147
cd python/psrunner
114148
uv sync

‎src/PythonScriptRunner.Abstractions/Exceptions.cs‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,61 @@ public VersionNotFoundException(string key, int version)
4747
public int Version { get; }
4848
}
4949

50+
/// <summary>The version exists but cannot run (it is a draft or was archived).</summary>
51+
public sealed class VersionNotRunnableException : PsrException
52+
{
53+
/// <summary>Creates the exception.</summary>
54+
/// <param name="key">Stage or pipeline key.</param>
55+
/// <param name="version">Requested version.</param>
56+
/// <param name="status">Current status of the version.</param>
57+
public VersionNotRunnableException(string key, int version, VersionStatus status)
58+
: base($"Version {version} of '{key}' is {status} and cannot run; only published versions run.")
59+
{
60+
Key = key;
61+
Version = version;
62+
Status = status;
63+
}
64+
65+
/// <summary>Stage or pipeline key.</summary>
66+
public string Key { get; }
67+
68+
/// <summary>Requested version.</summary>
69+
public int Version { get; }
70+
71+
/// <summary>Current status of the version.</summary>
72+
public VersionStatus Status { get; }
73+
}
74+
75+
/// <summary>The run does not exist (or is beyond the retention).</summary>
76+
public sealed class RunNotFoundException : PsrException
77+
{
78+
/// <summary>Creates the exception.</summary>
79+
/// <param name="runId">Requested run.</param>
80+
public RunNotFoundException(RunId runId)
81+
: base($"Run {runId} was not found.")
82+
{
83+
RunId = runId;
84+
}
85+
86+
/// <summary>Requested run.</summary>
87+
public RunId RunId { get; }
88+
}
89+
90+
/// <summary>The stored file does not exist (or is beyond the retention).</summary>
91+
public sealed class ArtifactNotFoundException : PsrException
92+
{
93+
/// <summary>Creates the exception.</summary>
94+
/// <param name="artifactId">Requested file.</param>
95+
public ArtifactNotFoundException(ArtifactId artifactId)
96+
: base($"Artifact {artifactId} was not found.")
97+
{
98+
ArtifactId = artifactId;
99+
}
100+
101+
/// <summary>Requested file.</summary>
102+
public ArtifactId ArtifactId { get; }
103+
}
104+
50105
/// <summary>One invalid input.</summary>
51106
/// <param name="Field">Field path (e.g. <c>coeficientes[2]</c>).</param>
52107
/// <param name="Code">Stable code (e.g. <c>MISSING_REQUIRED_INPUT</c>).</param>

‎src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs‎

Lines changed: 125 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,35 @@
22

33
namespace PythonScriptRunner;
44

5-
/// <summary>Source of stage definitions (memory, folder or database).</summary>
5+
/// <summary>Source of stage and pipeline definitions (memory, folder or database).</summary>
66
public interface ICatalog
77
{
88
/// <summary>Finds a stage version.</summary>
99
/// <param name="key">Stage key.</param>
1010
/// <param name="version">Exact version, or <c>null</c> for the latest published version.</param>
1111
/// <param name="cancellationToken">Cancellation token.</param>
1212
/// <returns>The definition, or <c>null</c> when not found.</returns>
13+
/// <exception cref="VersionNotRunnableException">The version exists but is a draft or archived.</exception>
1314
ValueTask<StageDefinition?> FindStageAsync(string key, int? version, CancellationToken cancellationToken = default);
1415

1516
/// <summary>Finds a pipeline version.</summary>
1617
/// <param name="key">Pipeline key.</param>
1718
/// <param name="version">Exact version, or <c>null</c> for the latest published version.</param>
1819
/// <param name="cancellationToken">Cancellation token.</param>
1920
/// <returns>The definition, or <c>null</c> when not found.</returns>
21+
/// <exception cref="VersionNotRunnableException">The version exists but is a draft or archived.</exception>
2022
ValueTask<PipelineDefinition?> FindPipelineAsync(string key, int? version, CancellationToken cancellationToken = default);
2123
}
2224

25+
/// <summary>Validates stage definitions (contract, limits and secret names).</summary>
26+
public interface IStageValidator
27+
{
28+
/// <summary>Validates a stage definition.</summary>
29+
/// <param name="stage">The definition.</param>
30+
/// <returns>Errors (code <c>STAGE_DEFINITION</c>) and warnings.</returns>
31+
ValidationReport Validate(StageDefinition stage);
32+
}
33+
2334
/// <summary>Resolves the secrets declared by stages.</summary>
2435
public interface ISecretProvider
2536
{
@@ -29,46 +40,145 @@ public interface ISecretProvider
2940
ValueTask<string?> GetSecretAsync(string name, CancellationToken cancellationToken = default);
3041
}
3142

32-
/// <summary>Receives run transitions so they can be persisted. The default implementation does nothing.</summary>
43+
/// <summary>
44+
/// Receives every transition of a run so it can be persisted. The default implementation does nothing; the
45+
/// EntityFrameworkCore package stores runs, steps, attempts, logs and files.
46+
/// </summary>
3347
public interface IRunRecorder
3448
{
35-
/// <summary>A run started.</summary>
49+
/// <summary>A run is about to start (inputs already validated). Reserves the idempotency key.</summary>
50+
/// <param name="run">Run data.</param>
51+
/// <param name="cancellationToken">Cancellation token.</param>
52+
/// <returns>The existing run when the idempotency key was already used (the new run must not start); otherwise <c>null</c>.</returns>
53+
ValueTask<RunResult?> RunStartingAsync(RunStarting run, CancellationToken cancellationToken);
54+
55+
/// <summary>A failed run is being reprocessed: it is running again and skipped steps are pending.</summary>
3656
/// <param name="run">Run data.</param>
3757
/// <param name="cancellationToken">Cancellation token.</param>
38-
ValueTask RunStartedAsync(RunStarted run, CancellationToken cancellationToken);
58+
ValueTask RunResumingAsync(RunResuming run, CancellationToken cancellationToken);
59+
60+
/// <summary>A step started with its resolved inputs.</summary>
61+
/// <param name="step">Step data.</param>
62+
/// <param name="cancellationToken">Cancellation token.</param>
63+
ValueTask StepStartedAsync(StepStarted step, CancellationToken cancellationToken);
3964

40-
/// <summary>An attempt finished.</summary>
65+
/// <summary>An attempt (one Python process) started.</summary>
66+
/// <param name="attempt">Attempt data.</param>
67+
/// <param name="cancellationToken">Cancellation token.</param>
68+
ValueTask AttemptStartedAsync(AttemptStarted attempt, CancellationToken cancellationToken);
69+
70+
/// <summary>An attempt finished (with its logs).</summary>
4171
/// <param name="attempt">Attempt data.</param>
4272
/// <param name="cancellationToken">Cancellation token.</param>
4373
ValueTask AttemptFinishedAsync(AttemptFinished attempt, CancellationToken cancellationToken);
4474

75+
/// <summary>A step finished (succeeded with outputs and files, failed, cancelled or skipped).</summary>
76+
/// <param name="step">Step data.</param>
77+
/// <param name="cancellationToken">Cancellation token.</param>
78+
ValueTask StepFinishedAsync(StepFinished step, CancellationToken cancellationToken);
79+
4580
/// <summary>A run finished.</summary>
4681
/// <param name="run">Final data.</param>
4782
/// <param name="cancellationToken">Cancellation token.</param>
4883
ValueTask RunFinishedAsync(RunFinished run, CancellationToken cancellationToken);
4984
}
5085

51-
/// <summary>A run started.</summary>
86+
/// <summary>A run is about to start.</summary>
87+
public sealed record RunStarting
88+
{
89+
/// <summary>Run identifier (its timestamp is the creation time).</summary>
90+
public required RunId RunId { get; init; }
91+
92+
/// <summary>What is executed.</summary>
93+
public required RunKind Kind { get; init; }
94+
95+
/// <summary>Stage or pipeline key.</summary>
96+
public required string TargetKey { get; init; }
97+
98+
/// <summary>Stage or pipeline version.</summary>
99+
public required int TargetVersion { get; init; }
100+
101+
/// <summary>How the run started.</summary>
102+
public required RunTrigger Trigger { get; init; }
103+
104+
/// <summary>Who requested the run.</summary>
105+
public required string TriggeredBy { get; init; }
106+
107+
/// <summary>Deduplication key given by the caller.</summary>
108+
public string? IdempotencyKey { get; init; }
109+
110+
/// <summary>Validated inputs in wire format (files carry metadata only).</summary>
111+
public required JsonObject Inputs { get; init; }
112+
113+
/// <summary>Input files by field name.</summary>
114+
public IReadOnlyDictionary<string, IReadOnlyList<PsrFile>> InputFiles { get; init; } = new Dictionary<string, IReadOnlyList<PsrFile>>();
115+
116+
/// <summary>Steps that will run, in order.</summary>
117+
public required IReadOnlyList<PlannedStep> Steps { get; init; }
118+
119+
/// <summary>Start time.</summary>
120+
public required DateTimeOffset StartedAt { get; init; }
121+
}
122+
123+
/// <summary>A step of a run that is about to start.</summary>
124+
/// <param name="StepKey">Step key (the stage key for single-stage runs).</param>
125+
/// <param name="Order">1-based position.</param>
126+
/// <param name="StageKey">Stage key.</param>
127+
/// <param name="StageVersion">Exact stage version.</param>
128+
public sealed record PlannedStep(string StepKey, int Order, string StageKey, int StageVersion);
129+
130+
/// <summary>A failed run is being reprocessed.</summary>
131+
/// <param name="RunId">Run identifier.</param>
132+
/// <param name="TriggeredBy">Who reprocesses.</param>
133+
/// <param name="ResumedAt">When.</param>
134+
public sealed record RunResuming(RunId RunId, string TriggeredBy, DateTimeOffset ResumedAt);
135+
136+
/// <summary>A step started.</summary>
52137
/// <param name="RunId">Run identifier.</param>
53-
/// <param name="Kind">What is executed.</param>
54-
/// <param name="TargetKey">Stage or pipeline key.</param>
55-
/// <param name="TargetVersion">Stage or pipeline version.</param>
56-
/// <param name="TriggeredBy">Who requested the run.</param>
138+
/// <param name="StepKey">Step key.</param>
139+
/// <param name="Order">1-based position.</param>
140+
/// <param name="Inputs">Resolved inputs in wire format.</param>
141+
/// <param name="InputFiles">Input files by field name.</param>
57142
/// <param name="StartedAt">Start time.</param>
58-
/// <param name="Inputs">Validated inputs in wire format.</param>
59-
public sealed record RunStarted(
60-
RunId RunId, RunKind Kind, string TargetKey, int TargetVersion, string TriggeredBy, DateTimeOffset StartedAt, JsonObject Inputs);
143+
public sealed record StepStarted(
144+
RunId RunId, string StepKey, int Order, JsonObject Inputs, IReadOnlyDictionary<string, IReadOnlyList<PsrFile>> InputFiles, DateTimeOffset StartedAt);
145+
146+
/// <summary>An attempt started.</summary>
147+
/// <param name="RunId">Run identifier.</param>
148+
/// <param name="StepKey">Step key.</param>
149+
/// <param name="Order">1-based position of the step.</param>
150+
/// <param name="AttemptId">Attempt identifier.</param>
151+
/// <param name="Number">1-based attempt number.</param>
152+
/// <param name="Cause">Why it started.</param>
153+
/// <param name="TriggeredBy">Who started it.</param>
154+
/// <param name="StartedAt">Start time.</param>
155+
public sealed record AttemptStarted(
156+
RunId RunId, string StepKey, int Order, AttemptId AttemptId, int Number, AttemptCause Cause, string TriggeredBy, DateTimeOffset StartedAt);
61157

62158
/// <summary>An attempt finished.</summary>
63159
/// <param name="RunId">Run identifier.</param>
64160
/// <param name="StepKey">Step key.</param>
161+
/// <param name="Order">1-based position of the step.</param>
65162
/// <param name="Attempt">Attempt outcome.</param>
66-
public sealed record AttemptFinished(RunId RunId, string StepKey, AttemptResult Attempt);
163+
public sealed record AttemptFinished(RunId RunId, string StepKey, int Order, AttemptResult Attempt);
164+
165+
/// <summary>A step finished.</summary>
166+
/// <param name="RunId">Run identifier.</param>
167+
/// <param name="Step">Step outcome (outputs and files when it succeeded).</param>
168+
/// <param name="FinishedAt">End time.</param>
169+
public sealed record StepFinished(RunId RunId, StepResult Step, DateTimeOffset FinishedAt);
67170

68171
/// <summary>A run finished.</summary>
69172
/// <param name="RunId">Run identifier.</param>
70173
/// <param name="Status">Final status.</param>
71174
/// <param name="FinishedAt">End time.</param>
72175
/// <param name="Outputs">Declared outputs in wire format.</param>
176+
/// <param name="OutputFiles">Declared file outputs by name.</param>
73177
/// <param name="Error">Failure, when the run did not succeed.</param>
74-
public sealed record RunFinished(RunId RunId, RunStatus Status, DateTimeOffset FinishedAt, JsonObject? Outputs, RunError? Error);
178+
public sealed record RunFinished(
179+
RunId RunId,
180+
RunStatus Status,
181+
DateTimeOffset FinishedAt,
182+
JsonObject? Outputs,
183+
IReadOnlyDictionary<string, IReadOnlyList<PsrFile>>? OutputFiles,
184+
RunError? Error);

0 commit comments

Comments
 (0)