Skip to content

Commit bdec525

Browse files
net0wellclaude
andcommitted
feat(ef): persistência em PostgreSQL (spec 004)
- PsrDbContext no schema psr com migration inicial: histórico particionado por semana, conteúdo de arquivos por dia e auditoria por mês. - Catálogo no banco: rascunhos com concorrência otimista, publicação com verificação de sintaxe e lock do uv, arquivamento, exclusão, teste de rascunho e importação/exportação em zip. - Histórico de toda execução síncrona, arquivos em bytea com streaming, idempotência de 24 h, outbox transacional e auditoria de ações humanas. - IRunManager: consulta, listagem keyset, cancelamento e reprocesso síncrono. - Manutenção com advisory lock (partições, retenção, limpezas) e health check. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 0b2f51a commit bdec525

33 files changed

Lines changed: 8217 additions & 0 deletions

‎Directory.Packages.props‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,22 @@
44
</PropertyGroup>
55
<ItemGroup>
66
<PackageVersion Include="JsonSchema.Net" Version="9.4.1" />
7+
<PackageVersion Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.13" />
8+
<PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="10.0.13" />
79
<PackageVersion Include="Microsoft.Extensions.Configuration" Version="10.0.13" />
810
<PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.13" />
911
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.13" />
1012
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.13" />
1113
<PackageVersion Include="Microsoft.Extensions.Diagnostics" Version="10.0.13" />
14+
<PackageVersion Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="10.0.13" />
1215
<PackageVersion Include="Microsoft.Extensions.Diagnostics.Testing" Version="10.11.0" />
1316
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.13" />
1417
<PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.13" />
1518
<PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.13" />
1619
<PackageVersion Include="Microsoft.Extensions.TimeProvider.Testing" Version="10.11.0" />
20+
<PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.3" />
1721
<PackageVersion Include="Shouldly" Version="4.3.0" />
22+
<PackageVersion Include="Testcontainers.PostgreSql" Version="4.16.0" />
1823
<PackageVersion Include="xunit.v3" Version="4.0.2" />
1924
</ItemGroup>
2025
</Project>

‎PythonScriptRunner.slnx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@
22
<Folder Name="/src/">
33
<Project Path="src/PythonScriptRunner.Abstractions/PythonScriptRunner.Abstractions.csproj" />
44
<Project Path="src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj" />
5+
<Project Path="src/PythonScriptRunner.EntityFrameworkCore/PythonScriptRunner.EntityFrameworkCore.csproj" />
56
</Folder>
67
<Folder Name="/tests/">
78
<Project Path="tests/PythonScriptRunner.Core.Tests/PythonScriptRunner.Core.Tests.csproj" />
9+
<Project Path="tests/PythonScriptRunner.EntityFrameworkCore.Tests/PythonScriptRunner.EntityFrameworkCore.Tests.csproj" />
810
</Folder>
911
</Solution>
Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
using System.Security.Cryptography;
2+
using System.Text;
3+
using PythonScriptRunner.EntityFrameworkCore.Entities;
4+
5+
namespace PythonScriptRunner.EntityFrameworkCore.Catalog;
6+
7+
/// <summary>A step row with the stage version it references.</summary>
8+
internal sealed record StepRow(PipelineStepEntity Step, string StageKey, int StageVersion);
9+
10+
/// <summary>Maps catalog rows to definitions and DTOs, and back.</summary>
11+
internal static class CatalogMapping
12+
{
13+
public static StageDefinition ToStage(StageEntity stage, StageVersionEntity version) => new()
14+
{
15+
Key = stage.Key,
16+
Version = version.Version,
17+
Name = stage.Name,
18+
Description = stage.Description,
19+
Script = ScriptSource.Inline(version.Script),
20+
PythonVersion = version.PythonVersion,
21+
Requirements = version.Requirements,
22+
LockedRequirements = version.LockedRequirements,
23+
Inputs = StoredJson.Fields(version.InputFields),
24+
Outputs = StoredJson.Fields(version.OutputFields),
25+
RequiredSecrets = version.RequiredSecrets,
26+
Timeout = TimeSpan.FromSeconds(version.TimeoutSeconds),
27+
MemoryLimitMb = version.MemoryLimitMb,
28+
Retry = new RetryPolicy(version.RetryMaxAttempts, TimeSpan.FromSeconds(version.RetryInitialDelaySeconds), (double)version.RetryBackoffMultiplier),
29+
Idempotent = version.Idempotent,
30+
};
31+
32+
public static StageDraftData ToStageData(StageVersionEntity version) => new()
33+
{
34+
Script = version.Script,
35+
PythonVersion = version.PythonVersion,
36+
Requirements = version.Requirements,
37+
Inputs = StoredJson.Fields(version.InputFields),
38+
Outputs = StoredJson.Fields(version.OutputFields),
39+
RequiredSecrets = version.RequiredSecrets,
40+
Timeout = TimeSpan.FromSeconds(version.TimeoutSeconds),
41+
MemoryLimitMb = version.MemoryLimitMb,
42+
Retry = new RetryPolicy(version.RetryMaxAttempts, TimeSpan.FromSeconds(version.RetryInitialDelaySeconds), (double)version.RetryBackoffMultiplier),
43+
Idempotent = version.Idempotent,
44+
};
45+
46+
/// <summary>Copies editable content into a version row.</summary>
47+
public static void Apply(StageVersionEntity version, StageDraftData data)
48+
{
49+
version.Script = data.Script;
50+
version.ScriptSha256 = Sha256(data.Script);
51+
version.PythonVersion = data.PythonVersion;
52+
version.Requirements = string.IsNullOrWhiteSpace(data.Requirements) ? null : data.Requirements;
53+
version.InputFields = StoredJson.Fields(data.Inputs);
54+
version.OutputFields = StoredJson.Fields(data.Outputs);
55+
version.RequiredSecrets = [.. data.RequiredSecrets];
56+
version.TimeoutSeconds = (int)Math.Ceiling(data.Timeout.TotalSeconds);
57+
version.MemoryLimitMb = data.MemoryLimitMb;
58+
version.RetryMaxAttempts = data.Retry.MaxAttempts;
59+
version.RetryInitialDelaySeconds = (int)data.Retry.InitialDelay.TotalSeconds;
60+
version.RetryBackoffMultiplier = (decimal)data.Retry.BackoffMultiplier;
61+
version.Idempotent = data.Idempotent;
62+
}
63+
64+
public static StageDraftData ToStageData(StageDefinition stage, string defaultPythonVersion) => new()
65+
{
66+
Script = stage.Script switch
67+
{
68+
InlineScriptSource inline => inline.Content,
69+
FileScriptSource file => File.ReadAllText(file.Path),
70+
_ => throw new InvalidOperationException("Unknown script source."),
71+
},
72+
PythonVersion = stage.PythonVersion ?? defaultPythonVersion,
73+
Requirements = stage.Requirements,
74+
Inputs = stage.Inputs,
75+
Outputs = stage.Outputs,
76+
RequiredSecrets = stage.RequiredSecrets,
77+
Timeout = stage.Timeout ?? TimeSpan.FromMinutes(5),
78+
MemoryLimitMb = stage.MemoryLimitMb,
79+
Retry = stage.Retry,
80+
Idempotent = stage.Idempotent,
81+
};
82+
83+
public static StageDraft ToStageDraft(StageEntity stage, StageVersionEntity version) =>
84+
new(stage.Key, version.Version, ToStageData(version), version.RowVersion, Authorship(version));
85+
86+
public static StageVersionDetail ToStageVersionDetail(StageEntity stage, StageVersionEntity version) => new()
87+
{
88+
Key = stage.Key,
89+
Version = version.Version,
90+
Status = version.Status,
91+
Data = ToStageData(version),
92+
ScriptSha256 = version.ScriptSha256,
93+
LockedRequirements = version.LockedRequirements,
94+
EnvironmentHash = version.EnvironmentHash,
95+
PublishedAt = version.PublishedAt,
96+
PublishedBy = version.PublishedBy,
97+
ArchivedAt = version.ArchivedAt,
98+
ArchivedBy = version.ArchivedBy,
99+
RowVersion = version.RowVersion,
100+
Authorship = Authorship(version),
101+
};
102+
103+
public static PipelineDefinition ToPipeline(PipelineEntity pipeline, PipelineVersionEntity version, IEnumerable<StepRow> steps) => new()
104+
{
105+
Key = pipeline.Key,
106+
Version = version.Version,
107+
Name = pipeline.Name,
108+
Description = pipeline.Description,
109+
Inputs = StoredJson.Fields(version.InputFields),
110+
Steps = [.. steps.OrderBy(s => s.Step.Order).Select(s => new PipelineStepDefinition(s.Step.Key, s.StageKey, s.StageVersion)
111+
{
112+
DisplayName = s.Step.DisplayName,
113+
Bindings = StoredJson.Bindings(s.Step.Bindings),
114+
})],
115+
Outputs = StoredJson.Outputs(version.Outputs),
116+
};
117+
118+
public static PipelineDraftData ToPipelineData(PipelineDefinition pipeline, string? layout) => new()
119+
{
120+
Inputs = pipeline.Inputs,
121+
Steps = pipeline.Steps,
122+
Outputs = pipeline.Outputs,
123+
Layout = StoredJson.Node(layout),
124+
};
125+
126+
public static PipelineVersionDetail ToPipelineVersionDetail(PipelineEntity pipeline, PipelineVersionEntity version, IEnumerable<StepRow> steps) => new()
127+
{
128+
Key = pipeline.Key,
129+
Version = version.Version,
130+
Status = version.Status,
131+
Data = ToPipelineData(ToPipeline(pipeline, version, steps), version.Layout),
132+
PublishedAt = version.PublishedAt,
133+
PublishedBy = version.PublishedBy,
134+
ArchivedAt = version.ArchivedAt,
135+
ArchivedBy = version.ArchivedBy,
136+
RowVersion = version.RowVersion,
137+
Authorship = Authorship(version),
138+
};
139+
140+
public static Authorship Authorship(StageEntity row) => new(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy);
141+
142+
public static Authorship Authorship(StageVersionEntity row) => new(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy);
143+
144+
public static Authorship Authorship(PipelineEntity row) => new(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy);
145+
146+
public static Authorship Authorship(PipelineVersionEntity row) => new(row.CreatedAt, row.CreatedBy, row.UpdatedAt, row.UpdatedBy);
147+
148+
public static string Sha256(string text) => Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text)));
149+
}
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
using System.Collections.Concurrent;
2+
using Microsoft.EntityFrameworkCore;
3+
4+
namespace PythonScriptRunner.EntityFrameworkCore.Catalog;
5+
6+
/// <summary>
7+
/// Catalog stored in PostgreSQL. Only published versions run; definitions and "latest published" lookups are cached for
8+
/// <see cref="PsrEntityFrameworkOptions.LatestVersionCacheDuration"/> and invalidated in this process on every change.
9+
/// </summary>
10+
internal sealed class EfCatalog(IDbContextFactory<PsrDbContext> factory, PsrEntityFrameworkOptions options, TimeProvider time) : ICatalog
11+
{
12+
private readonly ConcurrentDictionary<(string Key, int? Version), Cached<StageDefinition>> _stages = new();
13+
private readonly ConcurrentDictionary<(string Key, int? Version), Cached<PipelineDefinition>> _pipelines = new();
14+
15+
public async ValueTask<StageDefinition?> FindStageAsync(string key, int? version, CancellationToken cancellationToken = default)
16+
{
17+
ArgumentNullException.ThrowIfNull(key);
18+
if (_stages.TryGetValue((key, version), out var cached) && cached.ExpiresAt > time.GetUtcNow())
19+
{
20+
return cached.Value;
21+
}
22+
23+
var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
24+
await using (db.ConfigureAwait(false))
25+
{
26+
var query = db.StageVersions.AsNoTracking().Include(v => v.Stage).Where(v => v.Stage.Key == key);
27+
query = version is { } number
28+
? query.Where(v => v.Version == number)
29+
: query.Where(v => v.Version == v.Stage.LatestPublishedVersion);
30+
var row = await query.SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false);
31+
if (row is null)
32+
{
33+
return null;
34+
}
35+
36+
if (row.Status != VersionStatus.Published)
37+
{
38+
throw new VersionNotRunnableException(key, row.Version, row.Status);
39+
}
40+
41+
var stage = CatalogMapping.ToStage(row.Stage, row);
42+
_stages[(key, version)] = new(stage, time.GetUtcNow() + options.LatestVersionCacheDuration);
43+
return stage;
44+
}
45+
}
46+
47+
public async ValueTask<PipelineDefinition?> FindPipelineAsync(string key, int? version, CancellationToken cancellationToken = default)
48+
{
49+
ArgumentNullException.ThrowIfNull(key);
50+
if (_pipelines.TryGetValue((key, version), out var cached) && cached.ExpiresAt > time.GetUtcNow())
51+
{
52+
return cached.Value;
53+
}
54+
55+
var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
56+
await using (db.ConfigureAwait(false))
57+
{
58+
var query = db.PipelineVersions.AsNoTracking().Include(v => v.Pipeline).Where(v => v.Pipeline.Key == key);
59+
query = version is { } number
60+
? query.Where(v => v.Version == number)
61+
: query.Where(v => v.Version == v.Pipeline.LatestPublishedVersion);
62+
var row = await query.SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false);
63+
if (row is null)
64+
{
65+
return null;
66+
}
67+
68+
if (row.Status != VersionStatus.Published)
69+
{
70+
throw new VersionNotRunnableException(key, row.Version, row.Status);
71+
}
72+
73+
var steps = await LoadStepsAsync(db, row.Id, cancellationToken).ConfigureAwait(false);
74+
var pipeline = CatalogMapping.ToPipeline(row.Pipeline, row, steps);
75+
_pipelines[(key, version)] = new(pipeline, time.GetUtcNow() + options.LatestVersionCacheDuration);
76+
return pipeline;
77+
}
78+
}
79+
80+
/// <summary>Forgets cached definitions of a stage (called after every change in this process).</summary>
81+
public void InvalidateStage(string key)
82+
{
83+
foreach (var entry in _stages.Keys.Where(k => k.Key == key))
84+
{
85+
_stages.TryRemove(entry, out _);
86+
}
87+
}
88+
89+
/// <summary>Forgets cached definitions of a pipeline.</summary>
90+
public void InvalidatePipeline(string key)
91+
{
92+
foreach (var entry in _pipelines.Keys.Where(k => k.Key == key))
93+
{
94+
_pipelines.TryRemove(entry, out _);
95+
}
96+
}
97+
98+
public static async Task<List<StepRow>> LoadStepsAsync(PsrDbContext db, Guid pipelineVersionId, CancellationToken cancellationToken) =>
99+
await db.PipelineSteps.AsNoTracking()
100+
.Where(s => s.PipelineVersionId == pipelineVersionId)
101+
.OrderBy(s => s.Order)
102+
.Select(s => new StepRow(s, s.StageVersion.Stage.Key, s.StageVersion.Version))
103+
.ToListAsync(cancellationToken).ConfigureAwait(false);
104+
105+
private sealed record Cached<T>(T Value, DateTimeOffset ExpiresAt);
106+
}

0 commit comments

Comments
 (0)