Skip to content

Commit 045178f

Browse files
net0wellclaude
andcommitted
feat(hosts): WorkerHost com ServiceDefaults e migração protegida por advisory lock (spec 009)
- WorkerHost: worker standalone configurado por variáveis de ambiente, /health/live e /health/ready, PSR_AUTO_MIGRATE (D-107; Dockerfile fica para a Fase 7). - ServiceDefaults: Serilog (console JSON) e OpenTelemetry com exportação OTLP opcional (D-54). - MigratePythonScriptRunnerAsync serializa migrações concorrentes com pg_advisory_lock (AC-8). - Testes: WorkerHost de ponta a ponta (WebApplicationFactory) e migrações concorrentes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent b5f43ca commit 045178f

10 files changed

Lines changed: 249 additions & 5 deletions

File tree

‎PythonScriptRunner.slnx‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
<Solution>
2+
<Folder Name="/hosts/">
3+
<Project Path="hosts/PythonScriptRunner.ServiceDefaults/PythonScriptRunner.ServiceDefaults.csproj" />
4+
<Project Path="hosts/PythonScriptRunner.WorkerHost/PythonScriptRunner.WorkerHost.csproj" />
5+
</Folder>
26
<Folder Name="/src/">
37
<Project Path="src/PythonScriptRunner.Abstractions/PythonScriptRunner.Abstractions.csproj" />
48
<Project Path="src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj" />

‎hosts/Directory.Build.props‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
<Project>
2+
<Import Project="$([MSBuild]::GetPathOfFileAbove('Directory.Build.props', '$(MSBuildThisFileDirectory)../'))" />
3+
4+
<PropertyGroup>
5+
<!-- Hosts are deployed as containers, not published as packages. -->
6+
<IsPackable>false</IsPackable>
7+
<!-- Required by IDE0005 (unused usings) during build; host members are not documented. -->
8+
<GenerateDocumentationFile>true</GenerateDocumentationFile>
9+
<NoWarn>$(NoWarn);CS1591</NoWarn>
10+
</PropertyGroup>
11+
</Project>
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<RootNamespace>PythonScriptRunner.ServiceDefaults</RootNamespace>
5+
</PropertyGroup>
6+
7+
<ItemGroup>
8+
<FrameworkReference Include="Microsoft.AspNetCore.App" />
9+
</ItemGroup>
10+
11+
<ItemGroup>
12+
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
13+
<PackageReference Include="OpenTelemetry.Extensions.Hosting" />
14+
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" />
15+
<PackageReference Include="OpenTelemetry.Instrumentation.Runtime" />
16+
<PackageReference Include="Serilog.AspNetCore" />
17+
<PackageReference Include="Serilog.Formatting.Compact" />
18+
</ItemGroup>
19+
20+
</Project>
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
using Microsoft.AspNetCore.Builder;
2+
using Microsoft.AspNetCore.Diagnostics.HealthChecks;
3+
using Microsoft.Extensions.DependencyInjection;
4+
using Microsoft.Extensions.Diagnostics.HealthChecks;
5+
using OpenTelemetry;
6+
using OpenTelemetry.Metrics;
7+
using OpenTelemetry.Resources;
8+
using OpenTelemetry.Trace;
9+
using Serilog;
10+
using Serilog.Formatting.Compact;
11+
12+
namespace Microsoft.Extensions.Hosting;
13+
14+
/// <summary>
15+
/// Defaults of the standalone hosts (spec 009; D-54): Serilog writing JSON to the console, OpenTelemetry traces and metrics of
16+
/// the <c>PythonScriptRunner</c> source/meter (exported with OTLP when <c>OTEL_EXPORTER_OTLP_ENDPOINT</c> is set) and the
17+
/// <c>/health/live</c> and <c>/health/ready</c> endpoints.
18+
/// </summary>
19+
public static class ServiceDefaultsExtensions
20+
{
21+
/// <summary>Tag of the checks behind <c>/health/live</c>.</summary>
22+
public const string LiveTag = "live";
23+
24+
/// <summary>Tag of the checks behind <c>/health/ready</c>.</summary>
25+
public const string ReadyTag = "ready";
26+
27+
/// <summary>Adds logging, telemetry and the liveness check.</summary>
28+
/// <param name="builder">Host builder.</param>
29+
/// <param name="serviceName">Service name in logs and telemetry.</param>
30+
public static TBuilder AddServiceDefaults<TBuilder>(this TBuilder builder, string serviceName)
31+
where TBuilder : IHostApplicationBuilder
32+
{
33+
ArgumentNullException.ThrowIfNull(builder);
34+
builder.Services.AddSerilog((services, logger) => logger
35+
.ReadFrom.Configuration(builder.Configuration)
36+
.Enrich.FromLogContext()
37+
.Enrich.WithProperty("service", serviceName)
38+
.WriteTo.Console(new RenderedCompactJsonFormatter()));
39+
40+
var telemetry = builder.Services.AddOpenTelemetry()
41+
.ConfigureResource(resource => resource.AddService(serviceName))
42+
.WithTracing(tracing => tracing.AddSource("PythonScriptRunner").AddAspNetCoreInstrumentation())
43+
.WithMetrics(metrics => metrics.AddMeter("PythonScriptRunner").AddAspNetCoreInstrumentation().AddRuntimeInstrumentation());
44+
if (!string.IsNullOrWhiteSpace(builder.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"]))
45+
{
46+
telemetry.UseOtlpExporter();
47+
}
48+
49+
builder.Services.AddHealthChecks().AddCheck("self", () => HealthCheckResult.Healthy(), [LiveTag]);
50+
return builder;
51+
}
52+
53+
/// <summary>
54+
/// Maps <c>/health/live</c> (the process answers) and <c>/health/ready</c> (checks tagged <see cref="ReadyTag"/>: 200 when
55+
/// healthy or degraded, 503 when unhealthy).
56+
/// </summary>
57+
/// <param name="app">Application.</param>
58+
public static WebApplication MapDefaultEndpoints(this WebApplication app)
59+
{
60+
ArgumentNullException.ThrowIfNull(app);
61+
app.MapHealthChecks("/health/live", new HealthCheckOptions { Predicate = check => check.Tags.Contains(LiveTag) });
62+
app.MapHealthChecks("/health/ready", new HealthCheckOptions { Predicate = check => check.Tags.Contains(ReadyTag) });
63+
return app;
64+
}
65+
}
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Standalone worker of PythonScriptRunner (spec 009): runs queued steps, cron schedules and the maintenance job.
2+
// Configuration comes from environment variables, e.g. ConnectionStrings__psr, ConnectionStrings__rabbitmq,
3+
// PythonScriptRunner__Worker__MaxConcurrency, PythonScriptRunner__Secrets__<NAME> and PSR_AUTO_MIGRATE.
4+
var builder = WebApplication.CreateBuilder(args);
5+
builder.AddServiceDefaults("psr-worker");
6+
7+
var configuration = builder.Configuration;
8+
builder.Services
9+
.AddPythonScriptRunner(o => configuration.GetSection("PythonScriptRunner").Bind(o))
10+
.AddEntityFrameworkCore(o =>
11+
{
12+
configuration.GetSection("PythonScriptRunner:EntityFrameworkCore").Bind(o);
13+
o.UseNpgsql(configuration.GetConnectionString("psr")
14+
?? throw new InvalidOperationException("Set the PostgreSQL connection string ConnectionStrings__psr."));
15+
})
16+
.AddRabbitMq(o =>
17+
{
18+
configuration.GetSection("PythonScriptRunner:RabbitMQ").Bind(o);
19+
o.ConnectionString = configuration.GetConnectionString("rabbitmq")
20+
?? throw new InvalidOperationException("Set the RabbitMQ connection string ConnectionStrings__rabbitmq.");
21+
})
22+
.AddWorker(o => configuration.GetSection("PythonScriptRunner:Worker").Bind(o));
23+
builder.Services.AddHealthChecks().AddPythonScriptRunnerChecks();
24+
25+
var app = builder.Build();
26+
if (configuration.GetValue<bool>("PSR_AUTO_MIGRATE"))
27+
{
28+
// EF Core serializes concurrent migrations with a PostgreSQL advisory lock, so replicas can start together (spec 009, AC-8).
29+
await app.Services.MigratePythonScriptRunnerAsync();
30+
}
31+
32+
app.MapDefaultEndpoints();
33+
await app.RunAsync();
34+
35+
/// <summary>Entry point (public for the integration tests).</summary>
36+
public partial class Program;
Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
<Project Sdk="Microsoft.NET.Sdk.Web">
2+
3+
<PropertyGroup>
4+
<RootNamespace>PythonScriptRunner.WorkerHost</RootNamespace>
5+
</PropertyGroup>
6+
7+
<ItemGroup>
8+
<ProjectReference Include="..\..\src\PythonScriptRunner.Worker\PythonScriptRunner.Worker.csproj" />
9+
<ProjectReference Include="..\PythonScriptRunner.ServiceDefaults\PythonScriptRunner.ServiceDefaults.csproj" />
10+
</ItemGroup>
11+
12+
</Project>

‎src/PythonScriptRunner.EntityFrameworkCore/PsrMigrations.cs‎

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,52 @@
22
using Microsoft.EntityFrameworkCore.Infrastructure;
33
using Microsoft.EntityFrameworkCore.Migrations;
44
using PythonScriptRunner.EntityFrameworkCore;
5+
using PythonScriptRunner.EntityFrameworkCore.Runs;
56

67
namespace Microsoft.Extensions.DependencyInjection;
78

89
/// <summary>Applies or scripts the migrations of the <c>psr</c> schema. Migrations never run implicitly.</summary>
910
public static class PsrMigrations
1011
{
11-
/// <summary>Creates or updates the <c>psr</c> schema (tables, partitions and functions). Safe to run again.</summary>
12+
/// <summary>Advisory lock key that serializes migrations of the <c>psr</c> schema ("psrmigr" in ASCII).</summary>
13+
private const long MigrationLockKey = 0x707372_6D69_6772;
14+
15+
/// <summary>
16+
/// Creates or updates the <c>psr</c> schema (tables, partitions and functions). Safe to run again, and from several
17+
/// instances at once: a PostgreSQL advisory lock lets one migrate while the others wait (spec 009, AC-8).
18+
/// </summary>
1219
/// <param name="services">Application services.</param>
1320
/// <param name="cancellationToken">Cancellation token.</param>
1421
public static async Task MigratePythonScriptRunnerAsync(this IServiceProvider services, CancellationToken cancellationToken = default)
1522
{
1623
ArgumentNullException.ThrowIfNull(services);
17-
var factory = services.GetRequiredService<IDbContextFactory<PsrDbContext>>();
18-
var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
19-
await using (db.ConfigureAwait(false))
24+
var connection = await services.GetRequiredService<PsrDataSource>().DataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
25+
await using (connection.ConfigureAwait(false))
26+
{
27+
await LockAsync(connection, "SELECT pg_advisory_lock($1)", cancellationToken).ConfigureAwait(false);
28+
try
29+
{
30+
var factory = services.GetRequiredService<IDbContextFactory<PsrDbContext>>();
31+
var db = await factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false);
32+
await using (db.ConfigureAwait(false))
33+
{
34+
await db.Database.MigrateAsync(cancellationToken).ConfigureAwait(false);
35+
}
36+
}
37+
finally
38+
{
39+
await LockAsync(connection, "SELECT pg_advisory_unlock($1)", CancellationToken.None).ConfigureAwait(false);
40+
}
41+
}
42+
}
43+
44+
private static async Task LockAsync(Npgsql.NpgsqlConnection connection, string sql, CancellationToken cancellationToken)
45+
{
46+
var command = new Npgsql.NpgsqlCommand(sql, connection);
47+
await using (command.ConfigureAwait(false))
2048
{
21-
await db.Database.MigrateAsync(cancellationToken).ConfigureAwait(false);
49+
command.Parameters.Add(new Npgsql.NpgsqlParameter { Value = MigrationLockKey });
50+
await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
2251
}
2352
}
2453

‎tests/PythonScriptRunner.EntityFrameworkCore.Tests/SchemaTests.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,22 @@ public async Task Spec004_AC01_MigrateCreatesSchemaPartitionsAndIsRepeatable()
5151
(await scripted.ScalarAsync<long>("SELECT count(*) FROM psr.__ef_migrations_history")).ShouldBe(db.Database.GetMigrations().Count());
5252
}
5353

54+
[Fact]
55+
public async Task Spec009_AC08_ConcurrentMigrations_AreAppliedOnce()
56+
{
57+
await using var first = await PsrTestHost.CreateAsync(postgres, python, migrate: false);
58+
var services = new ServiceCollection();
59+
services.AddPythonScriptRunner().AddEntityFrameworkCore(o => o.UseNpgsql(first.ConnectionString));
60+
await using var second = services.BuildServiceProvider();
61+
62+
await Task.WhenAll(
63+
first.Services.MigratePythonScriptRunnerAsync(TestContext.Current.CancellationToken),
64+
second.MigratePythonScriptRunnerAsync(TestContext.Current.CancellationToken));
65+
66+
await using var db = await first.DbAsync();
67+
(await first.ScalarAsync<long>("SELECT count(*) FROM psr.__ef_migrations_history")).ShouldBe(db.Database.GetMigrations().Count());
68+
}
69+
5470
[Fact]
5571
public async Task Spec004_AC02_WithoutMigrationStartupDoesNotTouchTheDatabaseAndHealthReportsPendingSchema()
5672
{

‎tests/PythonScriptRunner.Worker.Tests/PythonScriptRunner.Worker.Tests.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
<ItemGroup>
44
<ProjectReference Include="..\..\src\PythonScriptRunner.Worker\PythonScriptRunner.Worker.csproj" />
5+
<ProjectReference Include="..\..\hosts\PythonScriptRunner.WorkerHost\PythonScriptRunner.WorkerHost.csproj" />
56
</ItemGroup>
67

78
<ItemGroup>
@@ -21,6 +22,7 @@
2122
</ItemGroup>
2223

2324
<ItemGroup>
25+
<PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" />
2426
<PackageReference Include="Microsoft.Extensions.Configuration" />
2527
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" />
2628
<PackageReference Include="Microsoft.Extensions.Diagnostics.Testing" />
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
using System.Net;
2+
using Microsoft.AspNetCore.Mvc.Testing;
3+
4+
namespace PythonScriptRunner.Worker.Tests;
5+
6+
/// <summary>Spec 009: the standalone WorkerHost (D-107).</summary>
7+
public sealed class WorkerHostTests(PostgresFixture postgres, BrokerFixture broker, PythonFixture python)
8+
{
9+
private static CancellationToken Ct => TestContext.Current.CancellationToken;
10+
11+
[Fact]
12+
public async Task WorkerHost_MigratesAnswersHealthAndRunsQueuedSteps()
13+
{
14+
var database = await postgres.CreateDatabaseAsync();
15+
var amqp = await broker.CreateVirtualHostAsync();
16+
var workRoot = Path.Combine(Path.GetTempPath(), "psr-tests", "work", Guid.NewGuid().ToString("N"));
17+
await using var factory = new WebApplicationFactory<Program>().WithWebHostBuilder(web => web
18+
.UseSetting("ConnectionStrings:psr", database)
19+
.UseSetting("ConnectionStrings:rabbitmq", amqp)
20+
.UseSetting("PSR_AUTO_MIGRATE", "true")
21+
.UseSetting("PythonScriptRunner:WorkRoot", workRoot)
22+
.UseSetting("PythonScriptRunner:UvPath", python.UvPath)
23+
.UseSetting("PythonScriptRunner:EnvironmentCacheRoot", python.EnvironmentCacheRoot)
24+
.UseSetting("PythonScriptRunner:PsrunnerRequirement", python.WheelPath)
25+
.UseSetting("PythonScriptRunner:Worker:MaxConcurrency", "2")
26+
.UseSetting("Serilog:MinimumLevel:Default", "Warning"));
27+
using var client = factory.CreateClient();
28+
29+
(await client.GetAsync("/health/live", Ct)).StatusCode.ShouldBe(HttpStatusCode.OK);
30+
await Eventually.WaitAsync(async () => (await client.GetStringAsync("/health/ready", Ct)) == "Healthy", TimeSpan.FromSeconds(30));
31+
32+
var admin = factory.Services.GetRequiredService<ICatalogAdmin>();
33+
var draft = await admin.CreateStageAsync(new NewCatalogItem("eco", "eco"), PsrEnvironment.Author, Ct);
34+
var saved = await admin.SaveStageDraftAsync("eco", draft.Version, draft.Data with
35+
{
36+
Script = Scripts.Echo,
37+
Inputs = [Scripts.Text("texto")],
38+
Outputs = [Scripts.Text("texto")],
39+
}, draft.RowVersion, PsrEnvironment.Author, Ct);
40+
await admin.PublishStageAsync("eco", saved.Version, saved.RowVersion, PsrEnvironment.Author, Ct);
41+
var handle = await factory.Services.GetRequiredService<IRunQueue>().EnqueueStageAsync("eco", new { texto = "host" }, cancellationToken: Ct);
42+
43+
var runs = factory.Services.GetRequiredService<IRunManager>();
44+
RunSnapshot? run = null;
45+
await Eventually.WaitAsync(async () => (run = await runs.GetAsync(handle.RunId, Ct))?.Status == RunStatus.Succeeded, TimeSpan.FromSeconds(60),
46+
() => $"run is {run?.Status}");
47+
run!.Steps.Single().Attempts.Single().WorkerId.ShouldNotBeNullOrEmpty();
48+
}
49+
}

0 commit comments

Comments
 (0)