diff --git a/README.md b/README.md index f524ee0..1a9384d 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ Biblioteca .NET 10 para executar Python de forma tipada, encadeável e observáv com contrato de entrada/saída, trilhas (**Pipelines**), execução assíncrona com RabbitMQ, agendamento cron e painel Vue opcional. -> Projeto em desenvolvimento. Esta versão entrega o **Core Runner** (execução síncrona de Stages) e o -> SDK Python **`psrunner`**. +> Projeto em desenvolvimento. Esta versão entrega o **Core Runner** (execução síncrona de Stages e de +> trilhas) e o SDK Python **`psrunner`**. ## Como funciona @@ -56,6 +56,39 @@ double[] termos = result.GetOutput("termos"); // [1, 2, 4, 8] Stages também podem ser definidos em C# (`AddStage("chave", s => s.Script(...).Input(...).Output(...))`) e testados localmente com pytest pelo `psrunner.testing.run_stage`, sem .NET. +## Trilhas (pipelines) + +Uma trilha encadeia Stages em ordem. Cada entrada de um passo recebe a entrada da trilha +(`$pipeline.campo`), a saída de qualquer passo anterior (`$steps.passo.campo`) ou um valor fixo +(`{"const": ...}`). Sem ligação explícita, vale o campo de mesmo nome do passo anterior. A trilha +declara as próprias saídas. Tipos e ligações são validados ao carregar o catálogo. + +```jsonc +// pipelines/calculo-progressao.pipeline.json +{ + "key": "calculo-progressao", "version": 1, "name": "Cálculo de progressão", + "inputs": [ { "name": "a1", "type": "number" }, { "name": "razao", "type": "number" }, + { "name": "n", "type": "integer" }, { "name": "coeficientes", "type": "list", "itemType": "number" } ], + "steps": [ + { "key": "termos", "stage": "progressao-termos@1" }, + { "key": "polinomio", "stage": "polinomio-avaliar@1", + "bindings": { "coeficientes": "$pipeline.coeficientes", "x": "$steps.termos.termos" } }, + { "key": "estatisticas", "stage": "estatisticas@1", "bindings": { "valores": "$steps.polinomio.valores" } } + ], + "outputs": [ { "name": "soma", "source": "$steps.estatisticas.soma" } ] +} +``` + +```csharp +await using var result = await runner.RunPipelineAsync("calculo-progressao", + new { a1 = 1, razao = 2, n = 4, coeficientes = new[] { 1.0, 0.0, 1.0 } }); +double soma = result.GetOutput("soma"); +double[] termos = result.GetStepOutput("termos", "termos"); // depuração +``` + +A mesma trilha pode ser definida em C# com `AddPipeline("chave", p => p.Input(...).Step(...).Output(...))`. +Se um passo falhar, os seguintes ficam como `Skipped` e a execução termina `Failed`. + ## Requisitos - .NET 10 SDK @@ -68,8 +101,8 @@ e testados localmente com pytest pelo `psrunner.testing.run_stage`, sem .NET. | `src/PythonScriptRunner.Abstractions` | contratos públicos (tipos de campo, `StageDefinition`, `RunResult`, `PsrFile`, interfaces) | | `src/PythonScriptRunner.Core` | catálogos (código e pasta), validação, ambientes uv, executor de processos, runner | | `python/psrunner` | SDK Python (`@stage`, `Context`, `File`, `StageError`, `psrunner.testing`, CLI) | -| `schemas` | JSON Schema do `stage.json` | -| `tests` | testes .NET (unitários e de integração com Python real) | +| `schemas` | JSON Schemas do `stage.json` e do `*.pipeline.json` | +| `tests` | testes .NET (unitários e de integração com Python real) e fixtures de trilha | ## Desenvolvimento diff --git a/schemas/pipeline.v1.json b/schemas/pipeline.v1.json new file mode 100644 index 0000000..0737cb5 --- /dev/null +++ b/schemas/pipeline.v1.json @@ -0,0 +1,354 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/net0well/PythonScriptRunner/main/schemas/pipeline.v1.json", + "title": "PythonScriptRunner pipeline manifest (v1)", + "description": "Declares one pipeline: inputs, ordered steps (stage@version + bindings) and declared outputs.", + "type": "object", + "required": [ + "key", + "version", + "name", + "steps" + ], + "additionalProperties": false, + "properties": { + "$schema": { + "type": "string" + }, + "key": { + "description": "Unique slug of the stage.", + "type": "string", + "minLength": 3, + "maxLength": 64, + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$" + }, + "version": { + "type": "integer", + "minimum": 1 + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "description": { + "type": "string" + }, + "inputs": { + "type": "array", + "items": { + "$ref": "#/$defs/field" + } + }, + "steps": { + "description": "Steps in execution order (linear).", + "type": "array", + "items": { + "type": "object", + "required": [ + "key", + "stage" + ], + "additionalProperties": false, + "properties": { + "key": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$", + "maxLength": 64 + }, + "stage": { + "description": "Stage reference key@version.", + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*@[1-9][0-9]*$" + }, + "displayName": { + "type": "string", + "maxLength": 200 + }, + "bindings": { + "description": "Input of the stage -> $pipeline., $steps.. or {\"const\": value}. Unbound inputs use the implicit binding by name.", + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/binding" + } + } + } + } + }, + "outputs": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "source" + ], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "pattern": "^[a-z_][a-z0-9_]{0,63}$" + }, + "description": { + "type": "string", + "maxLength": 500 + }, + "source": { + "type": "string", + "pattern": "^\\$steps\\.[a-z0-9]+(-[a-z0-9]+)*\\.[a-z_][a-z0-9_]{0,63}$" + } + } + } + } + }, + "$defs": { + "binding": { + "oneOf": [ + { + "type": "string", + "pattern": "^\\$pipeline\\.[a-z_][a-z0-9_]{0,63}$" + }, + { + "type": "string", + "pattern": "^\\$steps\\.[a-z0-9]+(-[a-z0-9]+)*\\.[a-z_][a-z0-9_]{0,63}$" + }, + { + "type": "object", + "required": [ + "const" + ], + "additionalProperties": false, + "properties": { + "const": {} + } + } + ] + }, + "scalarType": { + "enum": [ + "number", + "integer", + "decimal", + "string", + "bool", + "date", + "datetime" + ] + }, + "field": { + "type": "object", + "required": [ + "name", + "type" + ], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "pattern": "^[a-z_][a-z0-9_]{0,63}$" + }, + "type": { + "enum": [ + "number", + "integer", + "decimal", + "string", + "bool", + "date", + "datetime", + "enum", + "json", + "table", + "file", + "list" + ] + }, + "itemType": { + "enum": [ + "number", + "integer", + "decimal", + "string", + "bool", + "date", + "datetime", + "enum", + "file" + ] + }, + "required": { + "type": "boolean" + }, + "default": {}, + "description": { + "type": "string", + "maxLength": 500 + }, + "enumValues": { + "type": "array", + "minItems": 1, + "maxItems": 200, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + }, + "contentTypes": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + }, + "columns": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "name", + "type" + ], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "type": { + "$ref": "#/$defs/scalarType" + } + } + } + } + }, + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "list" + } + } + }, + "then": { + "required": [ + "itemType" + ] + }, + "else": { + "not": { + "required": [ + "itemType" + ] + } + } + }, + { + "if": { + "anyOf": [ + { + "properties": { + "type": { + "const": "enum" + } + } + }, + { + "properties": { + "type": { + "const": "list" + }, + "itemType": { + "const": "enum" + } + }, + "required": [ + "itemType" + ] + } + ] + }, + "then": { + "required": [ + "enumValues" + ] + }, + "else": { + "not": { + "required": [ + "enumValues" + ] + } + } + }, + { + "if": { + "anyOf": [ + { + "properties": { + "type": { + "const": "file" + } + } + }, + { + "properties": { + "type": { + "const": "list" + }, + "itemType": { + "const": "file" + } + }, + "required": [ + "itemType" + ] + } + ] + }, + "then": { + "not": { + "required": [ + "default" + ] + } + }, + "else": { + "not": { + "required": [ + "contentTypes" + ] + } + } + }, + { + "if": { + "properties": { + "type": { + "const": "table" + } + } + }, + "then": { + "not": { + "required": [ + "default" + ] + } + }, + "else": { + "not": { + "required": [ + "columns" + ] + } + } + } + ] + } + } +} diff --git a/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs b/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs index b7ed24f..9a57725 100644 --- a/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs +++ b/src/PythonScriptRunner.Abstractions/Extensibility/Interfaces.cs @@ -11,6 +11,13 @@ public interface ICatalog /// Cancellation token. /// The definition, or null when not found. ValueTask FindStageAsync(string key, int? version, CancellationToken cancellationToken = default); + + /// Finds a pipeline version. + /// Pipeline key. + /// Exact version, or null for the latest published version. + /// Cancellation token. + /// The definition, or null when not found. + ValueTask FindPipelineAsync(string key, int? version, CancellationToken cancellationToken = default); } /// Resolves the secrets declared by stages. diff --git a/src/PythonScriptRunner.Abstractions/Pipelines/PipelineDefinition.cs b/src/PythonScriptRunner.Abstractions/Pipelines/PipelineDefinition.cs new file mode 100644 index 0000000..15489ff --- /dev/null +++ b/src/PythonScriptRunner.Abstractions/Pipelines/PipelineDefinition.cs @@ -0,0 +1,105 @@ +using System.Text.Json.Nodes; + +namespace PythonScriptRunner; + +/// Immutable definition of one version of a pipeline: inputs, ordered steps and declared outputs. +public sealed record PipelineDefinition +{ + /// Unique slug of the pipeline (e.g. html-to-email). + public required string Key { get; init; } + + /// Version number (1, 2, ...). + public required int Version { get; init; } + + /// Display name. + public required string Name { get; init; } + + /// Optional description. + public string? Description { get; init; } + + /// Inputs received by the pipeline. + public IReadOnlyList Inputs { get; init; } = []; + + /// Steps, executed in this order (linear). + public required IReadOnlyList Steps { get; init; } + + /// Declared outputs of the pipeline (may be empty). + public IReadOnlyList Outputs { get; init; } = []; +} + +/// One step of a pipeline: a fixed stage version and the bindings of its inputs. +/// Step key (slug, unique in the pipeline). +/// Key of the executed stage. +/// Exact version of the executed stage. +public sealed record PipelineStepDefinition(string Key, string StageKey, int StageVersion) +{ + /// Label shown in the panel. + public string? DisplayName { get; init; } + + /// Explicit bindings; unbound inputs use the implicit binding by name. + public IReadOnlyList Bindings { get; init; } = []; +} + +/// Where one input of a step comes from. +/// Input name of the stage. +/// Origin of the value. +public sealed record InputBinding(string Input, BindingSource Source); + +/// A declared output of the pipeline, taken from an output of a step. +/// Output name. +/// Step output that provides the value (its type is inherited). +public sealed record PipelineOutputDefinition(string Name, StepOutputSource Source) +{ + /// Description shown in the panel. + public string? Description { get; init; } +} + +/// Origin of a bound value. +public abstract record BindingSource +{ + private protected BindingSource() + { + } +} + +/// An input of the pipeline ($pipeline.<field>). +/// Pipeline input name. +public sealed record PipelineInputSource(string Field) : BindingSource +{ + /// + public override string ToString() => $"$pipeline.{Field}"; +} + +/// An output of an earlier step ($steps.<step>.<field>). +/// Step key. +/// Output name of that step. +public sealed record StepOutputSource(string Step, string Field) : BindingSource +{ + /// + public override string ToString() => $"$steps.{Step}.{Field}"; +} + +/// A fixed value in wire format ({"const": ...}); files are not allowed. +/// The value. +public sealed record ConstantSource(JsonNode? Value) : BindingSource +{ + /// + public override string ToString() => $"const {Value?.ToJsonString() ?? "null"}"; +} + +/// Shortcuts to create binding sources. +public static class From +{ + /// An input of the pipeline. + /// Pipeline input name. + public static PipelineInputSource Pipeline(string field) => new(field); + + /// An output of an earlier step. + /// Step key. + /// Output name. + public static StepOutputSource Step(string step, string field) => new(step, field); + + /// A fixed value in wire format. + /// The value (e.g. 465 or "no-reply@example.com"). + public static ConstantSource Constant(JsonNode? value) => new(value); +} diff --git a/src/PythonScriptRunner.Abstractions/Pipelines/Validation.cs b/src/PythonScriptRunner.Abstractions/Pipelines/Validation.cs new file mode 100644 index 0000000..0a242f9 --- /dev/null +++ b/src/PythonScriptRunner.Abstractions/Pipelines/Validation.cs @@ -0,0 +1,54 @@ +namespace PythonScriptRunner; + +/// Validates pipeline definitions (used when loading catalogs, publishing and editing in the panel). +public interface IPipelineValidator +{ + /// Validates a pipeline against the stages of the catalog. + /// The definition. + /// Cancellation token. + /// Errors (block publication) and warnings. + ValueTask ValidateAsync(PipelineDefinition pipeline, CancellationToken cancellationToken = default); +} + +/// Result of a pipeline validation. +/// Problems that block publication. +/// Problems that do not block publication. +public sealed record ValidationReport(IReadOnlyList Errors, IReadOnlyList Warnings) +{ + /// Whether there are no errors. + public bool IsValid => Errors.Count == 0; + + /// + public override string ToString() => string.Join("; ", Errors.Concat(Warnings).Select(i => i.ToString())); +} + +/// One validation problem. +/// Stable code (e.g. PIPE_TYPE_MISMATCH). +/// Human-readable message. +public sealed record ValidationIssue(string Code, string Message) +{ + /// Step key, when the problem belongs to a step. + public string? Step { get; init; } + + /// Field name (step input, pipeline input or output). + public string? Field { get; init; } + + /// + public override string ToString() => + $"[{Code}]{(Step is null ? "" : $" step '{Step}'")}{(Field is null ? "" : $" field '{Field}'")}: {Message}"; +} + +/// The pipeline does not exist in the catalog. +public sealed class PipelineNotFoundException : PsrException +{ + /// Creates the exception. + /// Requested key. + public PipelineNotFoundException(string pipelineKey) + : base($"Pipeline '{pipelineKey}' was not found.") + { + PipelineKey = pipelineKey; + } + + /// Requested key. + public string PipelineKey { get; } +} diff --git a/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs b/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs index 53c5bba..5f649f7 100644 --- a/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs +++ b/src/PythonScriptRunner.Abstractions/Running/IPythonRunner.cs @@ -1,6 +1,6 @@ namespace PythonScriptRunner; -/// Runs stages synchronously in the current process (one Python process per attempt). +/// Runs stages and pipelines synchronously in the current process (one Python process per attempt). public interface IPythonRunner { /// Runs a published stage and waits for its result. @@ -14,6 +14,18 @@ public interface IPythonRunner /// The inputs do not match the contract (no run is created). Task RunStageAsync( string stageKey, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default); + + /// Runs a published pipeline step by step and waits for its result. + /// Pipeline key. + /// Pipeline inputs (same forms as ). + /// Version, idempotency key and caller identity. + /// Cancels the run (the running step is stopped; the next ones are skipped). + /// The result with the declared pipeline outputs; failures are reported in . + /// The pipeline does not exist. + /// The requested version does not exist. + /// The inputs do not match the pipeline contract (no run is created). + Task RunPipelineAsync( + string pipelineKey, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default); } /// Options of one run. diff --git a/src/PythonScriptRunner.Core/Catalog/CatalogSources.cs b/src/PythonScriptRunner.Core/Catalog/CatalogSources.cs new file mode 100644 index 0000000..b1f469c --- /dev/null +++ b/src/PythonScriptRunner.Core/Catalog/CatalogSources.cs @@ -0,0 +1,117 @@ +using System.Collections.Frozen; +using Microsoft.Extensions.Logging; +using PythonScriptRunner.Core.Diagnostics; +using PythonScriptRunner.Core.Pipelines; + +namespace PythonScriptRunner.Core.Catalog; + +/// Supplies stage and pipeline definitions to the . +internal interface ICatalogSource +{ + /// Human-readable origin used in error messages. + string Description { get; } + + IReadOnlyList LoadStages(); + + IReadOnlyList LoadPipelines(); +} + +/// A stage or a pipeline defined in code. +internal sealed class StaticCatalogSource(StageDefinition? stage = null, PipelineDefinition? pipeline = null) : ICatalogSource +{ + public string Description => $"code ({stage?.Key ?? pipeline?.Key})"; + + public IReadOnlyList LoadStages() => stage is null ? [] : [stage]; + + public IReadOnlyList LoadPipelines() => pipeline is null ? [] : [pipeline]; +} + +/// In-memory catalog built once from every registered ; pipelines are validated on load. +internal sealed class CompositeCatalog : ICatalog +{ + private readonly Lazy _snapshot; + + public CompositeCatalog(IEnumerable sources, ILogger logger) => + _snapshot = new(() => Load([.. sources], logger), LazyThreadSafetyMode.ExecutionAndPublication); + + /// Loads every source now (fail fast at startup). + public void EnsureLoaded() => _ = _snapshot.Value; + + public ValueTask FindStageAsync(string key, int? version, CancellationToken cancellationToken = default) => + ValueTask.FromResult(Find(_snapshot.Value.Stages, key, version)); + + public ValueTask FindPipelineAsync(string key, int? version, CancellationToken cancellationToken = default) => + ValueTask.FromResult(Find(_snapshot.Value.Pipelines, key, version)); + + private static T? Find(FrozenDictionary> items, string key, int? version) + where T : class + { + if (!items.TryGetValue(key, out var versions)) + { + return null; + } + + return version is null ? versions.Values[^1] : versions.GetValueOrDefault(version.Value); + } + + private static Snapshot Load(ICatalogSource[] sources, ILogger logger) + { + var stages = Collect(sources, s => s.LoadStages(), s => (s.Key, s.Version), "Stage"); + foreach (var (stage, origin) in stages.Values.SelectMany(v => v.Values)) + { + StageDefinitionValidator.EnsureValid(stage, origin); + } + + var pipelines = Collect(sources, s => s.LoadPipelines(), p => (p.Key, p.Version), "Pipeline"); + var stageMap = stages.ToFrozenDictionary(p => p.Key, p => new SortedList(p.Value.ToDictionary(v => v.Key, v => v.Value.Item)), StringComparer.Ordinal); + var problems = new List(); + foreach (var (pipeline, origin) in pipelines.Values.SelectMany(v => v.Values)) + { + var report = PipelineRules.Validate(pipeline, (key, version) => Find(stageMap, key, version)); + problems.AddRange(report.Errors.Select(e => $" - pipeline '{pipeline.Key}@{pipeline.Version}' ({origin}): {e}")); + foreach (var warning in report.Warnings) + { + Log.PipelineWarning(logger, pipeline.Key, pipeline.Version, warning.ToString()); + } + } + + if (problems.Count > 0) + { + throw new PsrConfigurationException("Invalid pipelines:" + Environment.NewLine + string.Join(Environment.NewLine, problems)); + } + + return new Snapshot( + stageMap, + pipelines.ToFrozenDictionary(p => p.Key, p => new SortedList(p.Value.ToDictionary(v => v.Key, v => v.Value.Item)), StringComparer.Ordinal)); + } + + private static Dictionary> Collect( + ICatalogSource[] sources, Func> load, Func identity, string kind) + { + var items = new Dictionary>(StringComparer.Ordinal); + foreach (var source in sources) + { + foreach (var item in load(source)) + { + var (key, version) = identity(item); + if (!items.TryGetValue(key, out var versions)) + { + items[key] = versions = []; + } + + if (versions.TryGetValue(version, out var existing)) + { + throw new PsrConfigurationException($"{kind} '{key}@{version}' is defined twice: {existing.Item2} and {source.Description}."); + } + + versions[version] = (item, source.Description); + } + } + + return items; + } + + private sealed record Snapshot( + FrozenDictionary> Stages, + FrozenDictionary> Pipelines); +} diff --git a/src/PythonScriptRunner.Core/Catalog/FileCatalogSource.cs b/src/PythonScriptRunner.Core/Catalog/FileCatalogSource.cs new file mode 100644 index 0000000..fe30434 --- /dev/null +++ b/src/PythonScriptRunner.Core/Catalog/FileCatalogSource.cs @@ -0,0 +1,178 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Json.Schema; +using PythonScriptRunner.Core.Pipelines; + +namespace PythonScriptRunner.Core.Catalog; + +/// +/// Loads stages from <root>/stages/<folder>/stage.json (+ script and requirements) +/// and pipelines from <root>/pipelines/*.pipeline.json. +/// +internal sealed class FileCatalogSource(string root) : ICatalogSource +{ + private const string LockFileName = "requirements.lock"; + + private static readonly Lazy StageSchema = new(() => LoadSchema("stage.v1.json"), LazyThreadSafetyMode.ExecutionAndPublication); + private static readonly Lazy PipelineSchema = new(() => LoadSchema("pipeline.v1.json"), LazyThreadSafetyMode.ExecutionAndPublication); + + public string Description => $"folder '{root}'"; + + public IReadOnlyList LoadStages() + { + var folder = Subfolder("stages"); + if (folder is null) + { + return []; + } + + var manifests = Directory.EnumerateDirectories(folder).Order(StringComparer.Ordinal).Select(d => Path.Combine(d, "stage.json")); + return LoadAll(manifests, "stage catalog", (manifest, errors) => + File.Exists(manifest) ? LoadStage(manifest, errors) : Missing(manifest, errors)); + } + + public IReadOnlyList LoadPipelines() + { + var folder = Subfolder("pipelines"); + if (folder is null) + { + return []; + } + + var manifests = Directory.EnumerateFiles(folder, "*.pipeline.json").Order(StringComparer.Ordinal); + return LoadAll(manifests, "pipeline catalog", LoadPipeline); + } + + private string? Subfolder(string name) + { + var fullRoot = Path.GetFullPath(root); + if (!Directory.Exists(fullRoot)) + { + throw new PsrConfigurationException($"Catalog folder not found: {fullRoot}"); + } + + var folder = Path.Combine(fullRoot, name); + return Directory.Exists(folder) ? folder : null; + } + + private static List LoadAll(IEnumerable manifests, string what, Func, T?> load) + where T : class + { + var errors = new List(); + var items = new List(); + foreach (var manifest in manifests) + { + try + { + if (load(manifest, errors) is { } item) + { + items.Add(item); + } + } + catch (Exception ex) when (ex is JsonException or IOException or ArgumentException or InvalidOperationException or PsrConfigurationException) + { + errors.Add($"{manifest}: {ex.Message}"); + } + } + + return errors.Count > 0 + ? throw new PsrConfigurationException($"Invalid {what}:" + Environment.NewLine + string.Join(Environment.NewLine, errors.Select(e => " - " + e))) + : items; + } + + private static StageDefinition? Missing(string manifest, List errors) + { + errors.Add($"{manifest}: stage.json not found"); + return null; + } + + private static StageDefinition? LoadStage(string manifest, List errors) + { + if (ParseAndValidate(manifest, StageSchema.Value, "schemas/stage.v1.json", errors) is not { } json) + { + return null; + } + + var folder = Path.GetDirectoryName(manifest)!; + var requirementsName = json["requirements"]?.GetValue(); + var requirementsPath = Path.Combine(folder, requirementsName ?? "requirements.txt"); + string? requirements = null; + if (File.Exists(requirementsPath)) + { + requirements = File.ReadAllText(requirementsPath); + } + else if (requirementsName is not null) + { + errors.Add($"{manifest}: requirements file not found: {requirementsPath}"); + return null; + } + + var lockPath = Path.Combine(folder, LockFileName); + var retry = json["retry"]?.AsObject(); + return new StageDefinition + { + Key = json["key"]!.GetValue(), + Version = json["version"]!.GetValue(), + Name = json["name"]!.GetValue(), + Description = json["description"]?.GetValue(), + Script = ScriptSource.FromFile(Path.Combine(folder, json["script"]?.GetValue() ?? "script.py")), + PythonVersion = json["python"]?.GetValue(), + Requirements = requirements, + LockedRequirements = File.Exists(lockPath) ? File.ReadAllText(lockPath) : null, + Inputs = [.. json["inputs"]!.AsArray().Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))], + Outputs = [.. json["outputs"]!.AsArray().Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))], + RequiredSecrets = json["secrets"] is JsonArray secrets ? [.. secrets.Select(s => s!.GetValue())] : [], + Timeout = json["timeoutSeconds"] is { } seconds ? TimeSpan.FromSeconds(seconds.GetValue()) : null, + MemoryLimitMb = json["memoryLimitMb"]?.GetValue(), + Retry = retry is null + ? RetryPolicy.None + : new RetryPolicy( + retry["maxAttempts"]?.GetValue() ?? 0, + TimeSpan.FromSeconds(retry["initialDelaySeconds"]?.GetValue() ?? 10), + retry["backoffMultiplier"]?.GetValue() ?? 2.0), + Idempotent = json["idempotent"]?.GetValue() ?? false, + }; + } + + private static PipelineDefinition? LoadPipeline(string manifest, List errors) => + ParseAndValidate(manifest, PipelineSchema.Value, "schemas/pipeline.v1.json", errors) is { } json + ? PipelineJson.FromJson(json) + : null; + + private static JsonObject? ParseAndValidate(string manifest, JsonSchema schema, string schemaName, List errors) + { + using var document = JsonDocument.Parse( + File.ReadAllText(manifest), + new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true }); + + var results = schema.Evaluate(document.RootElement, new EvaluationOptions { OutputFormat = OutputFormat.List }); + if (results.IsValid) + { + return JsonNode.Parse(document.RootElement.GetRawText())!.AsObject(); + } + + var before = errors.Count; + foreach (var detail in (IEnumerable)[results, .. results.Details ?? []]) + { + foreach (var message in detail.Errors?.Values ?? Enumerable.Empty()) + { + errors.Add($"{manifest}: {(detail.InstanceLocation.ToString() is { Length: > 0 } at ? at : "/")}: {message}"); + } + } + + if (errors.Count == before) + { + errors.Add($"{manifest}: does not match {schemaName}"); + } + + return null; + } + + private static JsonSchema LoadSchema(string name) + { + using var stream = typeof(FileCatalogSource).Assembly.GetManifestResourceStream($"PythonScriptRunner.Schemas.{name}") + ?? throw new InvalidOperationException($"Embedded schema {name} not found."); + using var reader = new StreamReader(stream); + return JsonSchema.FromText(reader.ReadToEnd()); + } +} diff --git a/src/PythonScriptRunner.Core/Catalog/FileStageSource.cs b/src/PythonScriptRunner.Core/Catalog/FileStageSource.cs deleted file mode 100644 index fa94fa5..0000000 --- a/src/PythonScriptRunner.Core/Catalog/FileStageSource.cs +++ /dev/null @@ -1,136 +0,0 @@ -using System.Text.Json; -using System.Text.Json.Nodes; -using Json.Schema; - -namespace PythonScriptRunner.Core.Catalog; - -/// Loads stages from <root>/stages/<folder>/stage.json (+ script and requirements). -internal sealed class FileStageSource(string root) : IStageSource -{ - private const string LockFileName = "requirements.lock"; - - private static readonly Lazy Schema = new(LoadSchema, LazyThreadSafetyMode.ExecutionAndPublication); - - public string Description => $"folder '{root}'"; - - public IReadOnlyList LoadStages() - { - var fullRoot = Path.GetFullPath(root); - if (!Directory.Exists(fullRoot)) - { - throw new PsrConfigurationException($"Catalog folder not found: {fullRoot}"); - } - - var stagesFolder = Path.Combine(fullRoot, "stages"); - if (!Directory.Exists(stagesFolder)) - { - return []; - } - - var errors = new List(); - var stages = new List(); - foreach (var folder in Directory.EnumerateDirectories(stagesFolder).Order(StringComparer.Ordinal)) - { - var manifest = Path.Combine(folder, "stage.json"); - if (!File.Exists(manifest)) - { - errors.Add($"{folder}: stage.json not found"); - continue; - } - - try - { - if (Load(manifest, errors) is { } stage) - { - stages.Add(stage); - } - } - catch (Exception ex) when (ex is JsonException or IOException or ArgumentException or InvalidOperationException) - { - errors.Add($"{manifest}: {ex.Message}"); - } - } - - return errors.Count > 0 - ? throw new PsrConfigurationException( - "Invalid stage catalog:" + Environment.NewLine + string.Join(Environment.NewLine, errors.Select(e => " - " + e))) - : stages; - } - - private static StageDefinition? Load(string manifest, List errors) - { - using var document = JsonDocument.Parse( - File.ReadAllText(manifest), - new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true }); - - var results = Schema.Value.Evaluate(document.RootElement, new EvaluationOptions { OutputFormat = OutputFormat.List }); - if (!results.IsValid) - { - var before = errors.Count; - foreach (var detail in (IEnumerable)[results, .. results.Details ?? []]) - { - foreach (var message in detail.Errors?.Values ?? Enumerable.Empty()) - { - errors.Add($"{manifest}: {(detail.InstanceLocation.ToString() is { Length: > 0 } at ? at : "/")}: {message}"); - } - } - - if (errors.Count == before) - { - errors.Add($"{manifest}: does not match schemas/stage.v1.json"); - } - - return null; - } - - var json = JsonNode.Parse(document.RootElement.GetRawText())!.AsObject(); - var folder = Path.GetDirectoryName(manifest)!; - - var requirementsName = json["requirements"]?.GetValue(); - var requirementsPath = Path.Combine(folder, requirementsName ?? "requirements.txt"); - string? requirements = null; - if (File.Exists(requirementsPath)) - { - requirements = File.ReadAllText(requirementsPath); - } - else if (requirementsName is not null) - { - errors.Add($"{manifest}: requirements file not found: {requirementsPath}"); - return null; - } - - var lockPath = Path.Combine(folder, LockFileName); - var retry = json["retry"]?.AsObject(); - return new StageDefinition - { - Key = json["key"]!.GetValue(), - Version = json["version"]!.GetValue(), - Name = json["name"]!.GetValue(), - Description = json["description"]?.GetValue(), - Script = ScriptSource.FromFile(Path.Combine(folder, json["script"]?.GetValue() ?? "script.py")), - PythonVersion = json["python"]?.GetValue(), - Requirements = requirements, - LockedRequirements = File.Exists(lockPath) ? File.ReadAllText(lockPath) : null, - Inputs = [.. json["inputs"]!.AsArray().Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))], - Outputs = [.. json["outputs"]!.AsArray().Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))], - RequiredSecrets = json["secrets"] is JsonArray secrets ? [.. secrets.Select(s => s!.GetValue())] : [], - Timeout = json["timeoutSeconds"] is { } seconds ? TimeSpan.FromSeconds(seconds.GetValue()) : null, - MemoryLimitMb = json["memoryLimitMb"]?.GetValue(), - Retry = retry is null - ? RetryPolicy.None - : new RetryPolicy( - retry["maxAttempts"]?.GetValue() ?? 0, - TimeSpan.FromSeconds(retry["initialDelaySeconds"]?.GetValue() ?? 10), - retry["backoffMultiplier"]?.GetValue() ?? 2.0), - Idempotent = json["idempotent"]?.GetValue() ?? false, - }; - } - - private static JsonSchema LoadSchema() - { - using var stream = typeof(FileStageSource).Assembly.GetManifestResourceStream("PythonScriptRunner.Schemas.stage.v1.json") - ?? throw new InvalidOperationException("Embedded stage schema not found."); - using var reader = new StreamReader(stream); - return JsonSchema.FromText(reader.ReadToEnd()); - } -} diff --git a/src/PythonScriptRunner.Core/Catalog/StageDefinitionValidator.cs b/src/PythonScriptRunner.Core/Catalog/StageDefinitionValidator.cs index fc10e34..60fcf09 100644 --- a/src/PythonScriptRunner.Core/Catalog/StageDefinitionValidator.cs +++ b/src/PythonScriptRunner.Core/Catalog/StageDefinitionValidator.cs @@ -7,10 +7,10 @@ namespace PythonScriptRunner.Core.Catalog; internal static partial class StageDefinitionValidator { [GeneratedRegex("^[a-z0-9]+(-[a-z0-9]+)*$")] - private static partial Regex KeyPattern(); + internal static partial Regex KeyPattern(); [GeneratedRegex("^[a-z_][a-z0-9_]{0,63}$")] - private static partial Regex FieldNamePattern(); + internal static partial Regex FieldNamePattern(); [GeneratedRegex("^[A-Z][A-Z0-9_]{0,63}$")] private static partial Regex SecretPattern(); @@ -96,7 +96,7 @@ public static void EnsureValid(StageDefinition stage, string source) } } - private static void ValidateFields(IReadOnlyList fields, string section, List errors) + internal static void ValidateFields(IReadOnlyList fields, string section, List errors) { var names = new HashSet(StringComparer.Ordinal); foreach (var field in fields) diff --git a/src/PythonScriptRunner.Core/Catalog/StageSources.cs b/src/PythonScriptRunner.Core/Catalog/StageSources.cs deleted file mode 100644 index 1835e37..0000000 --- a/src/PythonScriptRunner.Core/Catalog/StageSources.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System.Collections.Frozen; - -namespace PythonScriptRunner.Core.Catalog; - -/// Supplies stage definitions to the . -internal interface IStageSource -{ - /// Human-readable origin used in error messages. - string Description { get; } - - IReadOnlyList LoadStages(); -} - -/// Stages defined in code with . -internal sealed class StaticStageSource(StageDefinition stage) : IStageSource -{ - public string Description => $"code ({stage.Key})"; - - public IReadOnlyList LoadStages() => [stage]; -} - -/// In-memory catalog built once from every registered . -internal sealed class CompositeCatalog(IEnumerable sources) : ICatalog -{ - private readonly Lazy>> _stages = - new(() => Load(sources), LazyThreadSafetyMode.ExecutionAndPublication); - - /// Loads every source now (fail fast at startup). - public void EnsureLoaded() => _ = _stages.Value; - - public ValueTask FindStageAsync(string key, int? version, CancellationToken cancellationToken = default) - { - if (!_stages.Value.TryGetValue(key, out var versions)) - { - return ValueTask.FromResult(null); - } - - if (version is null) - { - return ValueTask.FromResult(versions.Values[^1]); - } - - return ValueTask.FromResult(versions.GetValueOrDefault(version.Value)); - } - - private static FrozenDictionary> Load(IEnumerable sources) - { - var stages = new Dictionary>(StringComparer.Ordinal); - var origins = new Dictionary<(string, int), string>(); - foreach (var source in sources) - { - foreach (var stage in source.LoadStages()) - { - StageDefinitionValidator.EnsureValid(stage, source.Description); - if (origins.TryGetValue((stage.Key, stage.Version), out var other)) - { - throw new PsrConfigurationException( - $"Stage '{stage.Key}@{stage.Version}' is defined twice: {other} and {source.Description}."); - } - - origins[(stage.Key, stage.Version)] = source.Description; - if (!stages.TryGetValue(stage.Key, out var versions)) - { - stages[stage.Key] = versions = []; - } - - versions[stage.Version] = stage; - } - } - - return stages.ToFrozenDictionary(StringComparer.Ordinal); - } -} diff --git a/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs b/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs index 697ea6a..bea51a8 100644 --- a/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs +++ b/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs @@ -7,6 +7,7 @@ using PythonScriptRunner.Core.Diagnostics; using PythonScriptRunner.Core.Environments; using PythonScriptRunner.Core.Execution; +using PythonScriptRunner.Core.Pipelines; using PythonScriptRunner.Core.Running; namespace Microsoft.Extensions.DependencyInjection; @@ -33,6 +34,7 @@ public static IPythonScriptRunnerBuilder AddPythonScriptRunner( services.TryAddEnumerable(ServiceDescriptor.Singleton, PythonScriptRunnerOptionsValidator>()); services.TryAddSingleton(TimeProvider.System); services.TryAddSingleton(); + services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); @@ -60,14 +62,14 @@ public interface IPythonScriptRunnerBuilder /// Catalog and provider registration. public static class PythonScriptRunnerBuilderExtensions { - /// Loads stages from <rootPath>/stages/<folder>/stage.json. + /// Loads stages from <rootPath>/stages/<folder>/stage.json and pipelines from <rootPath>/pipelines/*.pipeline.json. /// Builder. /// Catalog folder. public static IPythonScriptRunnerBuilder AddFileCatalog(this IPythonScriptRunnerBuilder builder, string rootPath) { ArgumentNullException.ThrowIfNull(builder); ArgumentException.ThrowIfNullOrWhiteSpace(rootPath); - builder.Services.AddSingleton(new FileStageSource(rootPath)); + builder.Services.AddSingleton(new FileCatalogSource(rootPath)); return builder; } @@ -84,7 +86,21 @@ public static IPythonScriptRunnerBuilder AddStage(this IPythonScriptRunnerBuilde configure(stageBuilder); var stage = stageBuilder.Build(); StageDefinitionValidator.EnsureValid(stage, "code"); - builder.Services.AddSingleton(new StaticStageSource(stage)); + builder.Services.AddSingleton(new StaticCatalogSource(stage: stage)); + return builder; + } + + /// Defines a pipeline in code. It is validated against the catalog stages when the catalog loads. + /// Builder. + /// Pipeline key (slug). + /// Fluent definition: inputs, steps (stage@version + bindings) and outputs. + public static IPythonScriptRunnerBuilder AddPipeline(this IPythonScriptRunnerBuilder builder, string key, Action configure) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(configure); + var pipelineBuilder = new PipelineBuilder(key); + configure(pipelineBuilder); + builder.Services.AddSingleton(new StaticCatalogSource(pipeline: pipelineBuilder.Build())); return builder; } diff --git a/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs b/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs index d0faec3..a29269c 100644 --- a/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs +++ b/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs @@ -70,4 +70,7 @@ internal static partial class Log [LoggerMessage(8, LogLevel.Information, "stderr of stage {StageKey} attempt {Attempt}:\n{Output}")] public static partial void ScriptStderr(ILogger logger, string stageKey, int attempt, string output); + + [LoggerMessage(9, LogLevel.Warning, "Pipeline {PipelineKey}@{PipelineVersion}: {Warning}")] + public static partial void PipelineWarning(ILogger logger, string pipelineKey, int pipelineVersion, string warning); } diff --git a/src/PythonScriptRunner.Core/Pipelines/Coercion.cs b/src/PythonScriptRunner.Core/Pipelines/Coercion.cs new file mode 100644 index 0000000..ff6065c --- /dev/null +++ b/src/PythonScriptRunner.Core/Pipelines/Coercion.cs @@ -0,0 +1,23 @@ +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace PythonScriptRunner.Core.Pipelines; + +/// Applies the safe widenings allowed by bindings (e.g. integer → decimal becomes a decimal string). +internal static class Coercion +{ + public static JsonNode Coerce(JsonNode value, FieldType target) + { + if (target.Kind == FieldKind.Decimal && value is JsonValue number && number.GetValueKind() == JsonValueKind.Number) + { + return JsonValue.Create(number.ToJsonString()); + } + + if (target.Kind == FieldKind.List && value is JsonArray items) + { + return new JsonArray([.. items.Select(item => item is null ? null : Coerce(item, target.ItemType!))]); + } + + return value.DeepClone(); + } +} diff --git a/src/PythonScriptRunner.Core/Pipelines/PipelineBuilder.cs b/src/PythonScriptRunner.Core/Pipelines/PipelineBuilder.cs new file mode 100644 index 0000000..94d11c2 --- /dev/null +++ b/src/PythonScriptRunner.Core/Pipelines/PipelineBuilder.cs @@ -0,0 +1,144 @@ +using System.Globalization; +using System.Text.Json.Nodes; + +namespace PythonScriptRunner; + +/// Fluent definition of a pipeline in C#. +public sealed class PipelineBuilder +{ + private readonly string _key; + private readonly List _inputs = []; + private readonly List _steps = []; + private readonly List _outputs = []; + private int _version = 1; + private string? _name; + private string? _description; + + internal PipelineBuilder(string key) => _key = key; + + /// Version number. Default: 1. + /// Version (≥ 1). + public PipelineBuilder Version(int version) + { + _version = version; + return this; + } + + /// Display name. Default: the key. + /// Name. + public PipelineBuilder Name(string name) + { + _name = name; + return this; + } + + /// Description. + /// Text. + public PipelineBuilder Description(string description) + { + _description = description; + return this; + } + + /// Declares a pipeline input. + /// Field name. + /// Field type. + /// Whether the input is required. + /// Default value (wire format). + /// Description. + public PipelineBuilder Input(string name, FieldType type, bool required = true, JsonNode? @default = null, string? description = null) => + Input(new FieldDefinition(name, type) { Required = required, Default = @default, Description = description }); + + /// Declares a pipeline input with every option. + /// Field declaration. + public PipelineBuilder Input(FieldDefinition field) + { + _inputs.Add(field); + return this; + } + + /// Adds a step that runs stage@version (e.g. "html-to-pdf@1"). + /// Step key (slug, unique in the pipeline). + /// Stage reference key@version. + /// Bindings and display name; unbound inputs use the implicit binding by name. + public PipelineBuilder Step(string key, string stage, Action? configure = null) + { + var (stageKey, stageVersion) = ParseStageReference(stage); + return Step(key, stageKey, stageVersion, configure); + } + + /// Adds a step that runs a stage version. + /// Step key (slug, unique in the pipeline). + /// Stage key. + /// Stage version. + /// Bindings and display name. + public PipelineBuilder Step(string key, string stageKey, int stageVersion, Action? configure = null) + { + var step = new PipelineStepBuilder(); + configure?.Invoke(step); + _steps.Add(new PipelineStepDefinition(key, stageKey, stageVersion) { DisplayName = step.DisplayNameValue, Bindings = [.. step.Bindings] }); + return this; + } + + /// Declares a pipeline output taken from a step output. + /// Output name. + /// Step output (e.g. From.Step("stats", "sum")). + /// Description. + public PipelineBuilder Output(string name, StepOutputSource source, string? description = null) + { + _outputs.Add(new PipelineOutputDefinition(name, source) { Description = description }); + return this; + } + + internal PipelineDefinition Build() => new() + { + Key = _key, + Version = _version, + Name = _name ?? _key, + Description = _description, + Inputs = [.. _inputs], + Steps = [.. _steps], + Outputs = [.. _outputs], + }; + + /// Parses key@version. + internal static (string Key, int Version) ParseStageReference(string reference) + { + var separator = reference.LastIndexOf('@'); + if (separator <= 0 || !int.TryParse(reference.AsSpan(separator + 1), NumberStyles.None, CultureInfo.InvariantCulture, out var version)) + { + throw new PsrConfigurationException($"Invalid stage reference '{reference}'; use key@version (e.g. html-to-pdf@1)."); + } + + return (reference[..separator], version); + } +} + +/// Bindings of one pipeline step. +public sealed class PipelineStepBuilder +{ + internal PipelineStepBuilder() + { + } + + internal List Bindings { get; } = []; + + internal string? DisplayNameValue { get; private set; } + + /// Binds an input of the stage to a source. + /// Stage input name. + /// Origin: , or . + public PipelineStepBuilder Bind(string input, BindingSource source) + { + Bindings.Add(new InputBinding(input, source)); + return this; + } + + /// Label shown in the panel. + /// Label. + public PipelineStepBuilder DisplayName(string displayName) + { + DisplayNameValue = displayName; + return this; + } +} diff --git a/src/PythonScriptRunner.Core/Pipelines/PipelineJson.cs b/src/PythonScriptRunner.Core/Pipelines/PipelineJson.cs new file mode 100644 index 0000000..06431c3 --- /dev/null +++ b/src/PythonScriptRunner.Core/Pipelines/PipelineJson.cs @@ -0,0 +1,65 @@ +using System.Text.Json.Nodes; +using PythonScriptRunner.Core.Catalog; + +namespace PythonScriptRunner.Core.Pipelines; + +/// Maps *.pipeline.json (compact binding syntax) to . +internal static class PipelineJson +{ + private const string PipelinePrefix = "$pipeline."; + private const string StepsPrefix = "$steps."; + + /// A binding or a stage reference is malformed. + public static PipelineDefinition FromJson(JsonObject json) => new() + { + Key = json["key"]!.GetValue(), + Version = json["version"]!.GetValue(), + Name = json["name"]!.GetValue(), + Description = json["description"]?.GetValue(), + Inputs = json["inputs"] is JsonArray inputs ? [.. inputs.Select(f => FieldDefinitionJson.FromJson(f!.AsObject()))] : [], + Steps = [.. json["steps"]!.AsArray().Select(s => ParseStep(s!.AsObject()))], + Outputs = json["outputs"] is JsonArray outputs + ? [.. outputs.Select(o => new PipelineOutputDefinition(o!["name"]!.GetValue(), ParseStepSource(o["source"]!.GetValue())) + { + Description = o["description"]?.GetValue(), + })] + : [], + }; + + private static PipelineStepDefinition ParseStep(JsonObject step) + { + var (stageKey, stageVersion) = PipelineBuilder.ParseStageReference(step["stage"]!.GetValue()); + var bindings = step["bindings"] is JsonObject map + ? map.Select(pair => new InputBinding(pair.Key, ParseSource(pair.Value))).ToList() + : []; + return new PipelineStepDefinition(step["key"]!.GetValue(), stageKey, stageVersion) + { + DisplayName = step["displayName"]?.GetValue(), + Bindings = bindings, + }; + } + + private static BindingSource ParseSource(JsonNode? node) + { + if (node is JsonObject constant && constant.TryGetPropertyValue("const", out var value)) + { + return new ConstantSource(value?.DeepClone()); + } + + var text = node?.GetValue() ?? throw new ArgumentException("Binding must be a string or {\"const\": value}."); + if (text.StartsWith(PipelinePrefix, StringComparison.Ordinal)) + { + return new PipelineInputSource(text[PipelinePrefix.Length..]); + } + + return ParseStepSource(text); + } + + private static StepOutputSource ParseStepSource(string text) + { + var parts = text.StartsWith(StepsPrefix, StringComparison.Ordinal) ? text[StepsPrefix.Length..].Split('.') : []; + return parts.Length == 2 + ? new StepOutputSource(parts[0], parts[1]) + : throw new ArgumentException($"Invalid binding '{text}'; use $pipeline., $steps.. or {{\"const\": value}}."); + } +} diff --git a/src/PythonScriptRunner.Core/Pipelines/PipelineRules.cs b/src/PythonScriptRunner.Core/Pipelines/PipelineRules.cs new file mode 100644 index 0000000..c5d1bfd --- /dev/null +++ b/src/PythonScriptRunner.Core/Pipelines/PipelineRules.cs @@ -0,0 +1,276 @@ +using PythonScriptRunner.Core.Catalog; +using PythonScriptRunner.Core.Values; + +namespace PythonScriptRunner.Core.Pipelines; + +/// Validation rules of a pipeline (spec 003): structure, bindings, types and outputs. +internal static class PipelineRules +{ + public const int MaxSteps = 50; + + /// Validates a pipeline given a way to find the stage version of each step. + public static ValidationReport Validate(PipelineDefinition pipeline, Func findStage) + { + var errors = new List(); + var warnings = new List(); + ValidateDefinition(pipeline, errors); + + if (pipeline.Steps.Count is 0 or > MaxSteps) + { + errors.Add(new("PIPE_NO_STEPS", $"a pipeline needs 1 to {MaxSteps} steps (has {pipeline.Steps.Count})")); + } + + var positions = new Dictionary(StringComparer.Ordinal); + for (var i = 0; i < pipeline.Steps.Count; i++) + { + var key = pipeline.Steps[i].Key; + if (key.Length is < 1 or > 64 || !StageDefinitionValidator.KeyPattern().IsMatch(key)) + { + errors.Add(new("PIPE_STEP_KEY", "step key must be a slug (a-z, 0-9, '-')") { Step = key }); + } + else if (!positions.TryAdd(key, i)) + { + errors.Add(new("PIPE_STEP_KEY", "step key is duplicated") { Step = key }); + } + } + + var stages = pipeline.Steps.Select(s => findStage(s.StageKey, s.StageVersion)).ToArray(); + for (var i = 0; i < pipeline.Steps.Count; i++) + { + if (stages[i] is null) + { + var step = pipeline.Steps[i]; + errors.Add(new("PIPE_STAGE_NOT_PUBLISHED", $"stage '{step.StageKey}@{step.StageVersion}' is not published in the catalog") { Step = step.Key }); + } + } + + var inputs = pipeline.Inputs.GroupBy(f => f.Name).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); + var usedInputs = new HashSet(StringComparer.Ordinal); + for (var i = 0; i < pipeline.Steps.Count; i++) + { + if (stages[i] is { } stage) + { + ValidateStep(pipeline, i, stage, stages, positions, inputs, usedInputs, errors); + } + } + + ValidateOutputs(pipeline, stages, positions, errors); + + foreach (var input in pipeline.Inputs) + { + if (!usedInputs.Contains(input.Name)) + { + warnings.Add(new("PIPE_UNUSED_INPUT", "pipeline input is not used by any step") { Field = input.Name }); + } + + if (input.Type.IsFile && input.Required) + { + warnings.Add(new("PIPE_FILE_INPUT", "pipelines with a required file input cannot be scheduled or started by the public queue command") { Field = input.Name }); + } + } + + return new ValidationReport(errors, warnings); + } + + private static void ValidateDefinition(PipelineDefinition pipeline, List errors) + { + var problems = new List(); + if (pipeline.Key.Length is < 3 or > 64 || !StageDefinitionValidator.KeyPattern().IsMatch(pipeline.Key)) + { + problems.Add("key must be a slug of 3-64 chars (a-z, 0-9, '-')"); + } + + if (pipeline.Version < 1) + { + problems.Add("version must be >= 1"); + } + + if (string.IsNullOrWhiteSpace(pipeline.Name) || pipeline.Name.Length > 200) + { + problems.Add("name must have 1-200 chars"); + } + + StageDefinitionValidator.ValidateFields(pipeline.Inputs, "inputs", problems); + errors.AddRange(problems.Select(p => new ValidationIssue("PIPE_DEFINITION", p))); + } + + private static void ValidateStep( + PipelineDefinition pipeline, + int index, + StageDefinition stage, + StageDefinition?[] stages, + Dictionary positions, + Dictionary inputs, + HashSet usedInputs, + List errors) + { + var step = pipeline.Steps[index]; + var targets = stage.Inputs.ToDictionary(f => f.Name, StringComparer.Ordinal); + foreach (var binding in step.Bindings) + { + if (!targets.ContainsKey(binding.Input)) + { + errors.Add(new("PIPE_UNKNOWN_INPUT", $"stage '{stage.Key}@{stage.Version}' has no input '{binding.Input}'") { Step = step.Key, Field = binding.Input }); + } + } + + foreach (var duplicate in step.Bindings.GroupBy(b => b.Input).Where(g => g.Count() > 1)) + { + errors.Add(new("PIPE_DEFINITION", "input is bound more than once") { Step = step.Key, Field = duplicate.Key }); + } + + foreach (var target in stage.Inputs) + { + var binding = step.Bindings.FirstOrDefault(b => b.Input == target.Name); + var issue = binding is null + ? CheckImplicit(pipeline, index, target, stages, inputs, usedInputs) + : CheckExplicit(binding.Source, index, target, stages, positions, inputs, usedInputs); + if (issue is not null) + { + errors.Add(issue with { Step = step.Key, Field = target.Name }); + } + } + } + + private static ValidationIssue? CheckExplicit( + BindingSource source, + int index, + FieldDefinition target, + StageDefinition?[] stages, + Dictionary positions, + Dictionary inputs, + HashSet usedInputs) + { + switch (source) + { + case PipelineInputSource pipelineInput: + if (!inputs.TryGetValue(pipelineInput.Field, out var field)) + { + return new("PIPE_UNKNOWN_SOURCE", $"pipeline has no input '{pipelineInput.Field}'"); + } + + usedInputs.Add(field.Name); + return CheckTypes(field, target, pipelineInput.ToString(), MayBeAbsent(field)); + case StepOutputSource stepOutput: + if (!positions.TryGetValue(stepOutput.Step, out var position)) + { + return new("PIPE_UNKNOWN_SOURCE", $"there is no step '{stepOutput.Step}'"); + } + + if (position >= index) + { + return new("PIPE_FORWARD_REF", $"'{stepOutput}' is not an earlier step; pipelines are linear"); + } + + if (stages[position] is not { } sourceStage) + { + return null; + } + + var output = sourceStage.Outputs.FirstOrDefault(o => o.Name == stepOutput.Field); + return output is null + ? new("PIPE_UNKNOWN_SOURCE", $"step '{stepOutput.Step}' has no output '{stepOutput.Field}'") + : CheckTypes(output, target, stepOutput.ToString(), !output.Required); + case ConstantSource constant: + if (target.Type.IsFile) + { + return new("PIPE_CONST_INVALID", "file inputs cannot receive constants"); + } + + if (constant.Value is null) + { + return target.Required && target.Default is null + ? new("PIPE_UNBOUND_REQUIRED", "required input is bound to a null constant") + : null; + } + + var wireErrors = new List(); + WireValidator.Validate(Coercion.Coerce(constant.Value, target.Type), target, target.Name, wireErrors); + return wireErrors.Count == 0 + ? null + : new("PIPE_CONST_INVALID", string.Join("; ", wireErrors.Select(e => e.Message))); + default: + return new("PIPE_DEFINITION", $"unsupported binding source {source.GetType().Name}"); + } + } + + private static ValidationIssue? CheckImplicit( + PipelineDefinition pipeline, + int index, + FieldDefinition target, + StageDefinition?[] stages, + Dictionary inputs, + HashSet usedInputs) + { + FieldDefinition? source; + bool optional; + string description; + if (index == 0) + { + source = inputs.GetValueOrDefault(target.Name); + optional = source is not null && MayBeAbsent(source); + description = $"$pipeline.{target.Name} (implicit)"; + if (source is not null) + { + usedInputs.Add(source.Name); + } + } + else + { + if (stages[index - 1] is not { } previous) + { + return null; + } + + source = previous.Outputs.FirstOrDefault(o => o.Name == target.Name); + optional = source is { Required: false }; + description = $"$steps.{pipeline.Steps[index - 1].Key}.{target.Name} (implicit)"; + } + + if (source is not null) + { + return CheckTypes(source, target, description, optional); + } + + return target.Required && target.Default is null + ? new("PIPE_UNBOUND_REQUIRED", "required input has no binding, no implicit source with the same name and no default") + : null; + } + + private static ValidationIssue? CheckTypes(FieldDefinition source, FieldDefinition target, string description, bool sourceMayBeAbsent) + { + if (!TypeCompatibility.IsAssignable(source, target)) + { + return new("PIPE_TYPE_MISMATCH", $"{description} is {source.Type} and cannot feed {target.Type}"); + } + + return sourceMayBeAbsent && target.Required && target.Default is null + ? new("PIPE_OPTIONAL_TO_REQUIRED", $"{description} is optional but the input is required and has no default") + : null; + } + + private static bool MayBeAbsent(FieldDefinition pipelineInput) => !pipelineInput.Required && pipelineInput.Default is null; + + private static void ValidateOutputs( + PipelineDefinition pipeline, StageDefinition?[] stages, Dictionary positions, List errors) + { + var names = new HashSet(StringComparer.Ordinal); + foreach (var output in pipeline.Outputs) + { + if (!StageDefinitionValidator.FieldNamePattern().IsMatch(output.Name) || !names.Add(output.Name)) + { + errors.Add(new("PIPE_OUTPUT_SOURCE", "output name must match ^[a-z_][a-z0-9_]{0,63}$ and be unique") { Field = output.Name }); + continue; + } + + if (!positions.TryGetValue(output.Source.Step, out var position)) + { + errors.Add(new("PIPE_OUTPUT_SOURCE", $"there is no step '{output.Source.Step}'") { Field = output.Name }); + } + else if (stages[position] is { } stage && stage.Outputs.All(o => o.Name != output.Source.Field)) + { + errors.Add(new("PIPE_OUTPUT_SOURCE", $"step '{output.Source.Step}' has no output '{output.Source.Field}'") { Field = output.Name, Step = output.Source.Step }); + } + } + } +} diff --git a/src/PythonScriptRunner.Core/Pipelines/PipelineValidator.cs b/src/PythonScriptRunner.Core/Pipelines/PipelineValidator.cs new file mode 100644 index 0000000..ed156a1 --- /dev/null +++ b/src/PythonScriptRunner.Core/Pipelines/PipelineValidator.cs @@ -0,0 +1,21 @@ +namespace PythonScriptRunner.Core.Pipelines; + +/// Validates pipelines against the stages available in the . +internal sealed class PipelineValidator(ICatalog catalog) : IPipelineValidator +{ + public async ValueTask ValidateAsync(PipelineDefinition pipeline, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(pipeline); + var stages = new Dictionary<(string, int), StageDefinition?>(); + foreach (var step in pipeline.Steps) + { + var reference = (step.StageKey, step.StageVersion); + if (!stages.ContainsKey(reference)) + { + stages[reference] = await catalog.FindStageAsync(step.StageKey, step.StageVersion, cancellationToken).ConfigureAwait(false); + } + } + + return PipelineRules.Validate(pipeline, (key, version) => stages[(key, version)]); + } +} diff --git a/src/PythonScriptRunner.Core/Pipelines/StepInputResolver.cs b/src/PythonScriptRunner.Core/Pipelines/StepInputResolver.cs new file mode 100644 index 0000000..8854dd9 --- /dev/null +++ b/src/PythonScriptRunner.Core/Pipelines/StepInputResolver.cs @@ -0,0 +1,95 @@ +using System.Text.Json.Nodes; +using PythonScriptRunner.Core.Execution; +using PythonScriptRunner.Core.Values; + +namespace PythonScriptRunner.Core.Pipelines; + +/// Inputs of one pipeline step after resolving its bindings. +internal sealed record StepInputs(JsonObject Wire, IReadOnlyList Files, IReadOnlyList Errors); + +/// Resolves the inputs of a step from pipeline inputs, earlier step outputs, constants and implicit bindings. +internal static class StepInputResolver +{ + public static StepInputs Resolve( + StageDefinition stage, + PipelineDefinition pipeline, + int index, + JsonObject pipelineInputs, + IReadOnlyDictionary> pipelineFiles, + IReadOnlyDictionary completed) + { + var step = pipeline.Steps[index]; + var wire = new JsonObject(); + var files = new List(); + var errors = new List(); + foreach (var target in stage.Inputs) + { + var source = step.Bindings.FirstOrDefault(b => b.Input == target.Name)?.Source ?? Implicit(target, index, pipeline); + var (value, sourceFiles) = Read(source, pipelineInputs, pipelineFiles, completed); + if (target.Type.IsFile) + { + if (sourceFiles is { Count: > 0 }) + { + files.AddRange(sourceFiles.Select((file, i) => + new InputFileCopy(target.Name, i, file.LocalPath!, file.FileName, file.ContentType, file.Length ?? 0, file.Sha256!))); + } + else if (target.Required) + { + errors.Add(Missing(target, source)); + } + + continue; + } + + if (value is null) + { + if (target.Default is not null) + { + wire[target.Name] = target.Default.DeepClone(); + } + else if (target.Required) + { + errors.Add(Missing(target, source)); + } + + continue; + } + + var coerced = Coercion.Coerce(value, target.Type); + var wireErrors = new List(); + WireValidator.Validate(coerced, target, target.Name, wireErrors); + errors.AddRange(wireErrors.Select(e => new InputError(e.Path, e.Code, e.Message))); + wire[target.Name] = coerced; + } + + return new StepInputs(wire, files, errors); + } + + private static BindingSource? Implicit(FieldDefinition target, int index, PipelineDefinition pipeline) + { + if (index > 0) + { + return new StepOutputSource(pipeline.Steps[index - 1].Key, target.Name); + } + + return pipeline.Inputs.Any(f => f.Name == target.Name) ? new PipelineInputSource(target.Name) : null; + } + + private static (JsonNode? Value, IReadOnlyList? Files) Read( + BindingSource? source, + JsonObject pipelineInputs, + IReadOnlyDictionary> pipelineFiles, + IReadOnlyDictionary completed) => source switch + { + PipelineInputSource input => (pipelineInputs[input.Field], pipelineFiles.GetValueOrDefault(input.Field)), + StepOutputSource output when completed.TryGetValue(output.Step, out var result) => + (result.Outputs?[output.Field], result.Files.GetValueOrDefault(output.Field)), + ConstantSource constant => (constant.Value, null), + _ => (null, null), + }; + + private static InputError Missing(FieldDefinition target, BindingSource? source) => + new(target.Name, "MISSING_REQUIRED_INPUT", source is null + ? "required input has no binding and no default" + : $"required input received no value from {source}"); +} diff --git a/src/PythonScriptRunner.Core/Pipelines/TypeCompatibility.cs b/src/PythonScriptRunner.Core/Pipelines/TypeCompatibility.cs new file mode 100644 index 0000000..bd8f471 --- /dev/null +++ b/src/PythonScriptRunner.Core/Pipelines/TypeCompatibility.cs @@ -0,0 +1,48 @@ +namespace PythonScriptRunner.Core.Pipelines; + +/// Which source types can feed which target types in a binding (spec 003, safe widenings only). +internal static class TypeCompatibility +{ + public static bool IsAssignable(FieldDefinition source, FieldDefinition target) => + IsAssignable(source.Type, target.Type, source, target); + + private static bool IsAssignable(FieldType source, FieldType target, FieldDefinition sourceField, FieldDefinition targetField) + { + if (target.Kind == FieldKind.Json) + { + return !source.IsFile; + } + + return (source.Kind, target.Kind) switch + { + (FieldKind.Integer, FieldKind.Number or FieldKind.Decimal) => true, + (FieldKind.Enum, FieldKind.String) => true, + (FieldKind.Enum, FieldKind.Enum) => source.EnumValues!.All(v => target.EnumValues!.Contains(v, StringComparer.Ordinal)), + (FieldKind.Table, FieldKind.Table) => ColumnsCompatible(sourceField.Columns, targetField.Columns), + (FieldKind.File, FieldKind.File) => ContentTypesCompatible(sourceField.ContentTypes, targetField.ContentTypes), + (FieldKind.List, FieldKind.List) => IsAssignable(source.ItemType!, target.ItemType!, sourceField, targetField), + var (s, t) when s == t => true, + _ => false, + }; + } + + private static bool ColumnsCompatible(IReadOnlyList? source, IReadOnlyList? target) + { + if (target is not { Count: > 0 }) + { + return true; + } + + if (source is not { Count: > 0 }) + { + return false; + } + + return target.All(t => source.Any(s => s.Name == t.Name + && IsAssignable(s.Type, t.Type, new FieldDefinition(s.Name, s.Type), new FieldDefinition(t.Name, t.Type)))); + } + + private static bool ContentTypesCompatible(IReadOnlyList? source, IReadOnlyList? target) => + source is not { Count: > 0 } || target is not { Count: > 0 } + || source.Intersect(target, StringComparer.OrdinalIgnoreCase).Any(); +} diff --git a/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj b/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj index 888109e..e5b04ba 100644 --- a/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj +++ b/src/PythonScriptRunner.Core/PythonScriptRunner.Core.csproj @@ -11,6 +11,7 @@ + diff --git a/src/PythonScriptRunner.Core/Running/PythonRunner.cs b/src/PythonScriptRunner.Core/Running/PythonRunner.cs index fd5ca23..a3b32b8 100644 --- a/src/PythonScriptRunner.Core/Running/PythonRunner.cs +++ b/src/PythonScriptRunner.Core/Running/PythonRunner.cs @@ -1,15 +1,20 @@ using System.Diagnostics; +using System.Globalization; using System.Security.Cryptography; using System.Text.Json.Nodes; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using PythonScriptRunner.Core.Diagnostics; using PythonScriptRunner.Core.Execution; +using PythonScriptRunner.Core.Pipelines; using PythonScriptRunner.Core.Values; namespace PythonScriptRunner.Core.Running; -/// Runs stages in the current process: validation, secrets, environment, attempts with retry. +/// +/// Runs stages and pipelines in the current process: validation, secrets, environment, +/// attempts with retry and, for pipelines, bindings between steps. +/// internal sealed class PythonRunner : IPythonRunner, IDisposable { private readonly ICatalog _catalog; @@ -76,49 +81,150 @@ public async Task RunStageAsync( var bound = InputBinder.Bind(stage.Inputs, inputs, settings.MaxFileBytes); cancellationToken.ThrowIfCancellationRequested(); + return await RunAsync(RunKind.Stage, stage.Key, stage.Version, bound, options, settings, async (context, inputFiles) => + { + var step = await ExecuteStepAsync(stage, stage.Key, 1, bound.Wire, ToCopies(inputFiles), context, cancellationToken).ConfigureAwait(false); + return new RunOutcome(StatusOf(step.Status), [step], step.Outputs, step.Files, step.Error); + }, cancellationToken).ConfigureAwait(false); + } + + public async Task RunPipelineAsync( + string pipelineKey, object? inputs, RunOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(pipelineKey); + var settings = Settings; + _janitor.EnsureCleaned(); + var pipeline = await ResolvePipelineAsync(pipelineKey, options?.Version, cancellationToken).ConfigureAwait(false); + var bound = InputBinder.Bind(pipeline.Inputs, inputs, settings.MaxFileBytes); + cancellationToken.ThrowIfCancellationRequested(); + + return await RunAsync(RunKind.Pipeline, pipeline.Key, pipeline.Version, bound, options, settings, + (context, inputFiles) => ExecutePipelineAsync(pipeline, bound.Wire, inputFiles, context, cancellationToken), + cancellationToken).ConfigureAwait(false); + } + + public void Dispose() + { + if (_concurrency.IsValueCreated) + { + _concurrency.Value.Dispose(); + } + } + + private async Task RunAsync( + RunKind kind, + string key, + int version, + BoundInputs bound, + RunOptions? options, + PythonScriptRunnerOptions settings, + Func>, Task> body, + CancellationToken cancellationToken) + { var runId = RunId.New(); var started = _time.GetTimestamp(); await _recorder.RunStartedAsync( - new RunStarted(runId, RunKind.Stage, stage.Key, stage.Version, options?.TriggeredBy ?? "system:library", _time.GetUtcNow(), (JsonObject)bound.Wire.DeepClone()), + new RunStarted(runId, kind, key, version, options?.TriggeredBy ?? "system:library", _time.GetUtcNow(), (JsonObject)bound.Wire.DeepClone()), cancellationToken).ConfigureAwait(false); - var workspace = new RunWorkspace(settings.WorkRoot, runId); + var context = new RunContext(new RunWorkspace(settings.WorkRoot, runId), settings); try { - var step = await ExecuteStepAsync(stage, stage.Key, 1, bound, workspace, runId, settings, cancellationToken).ConfigureAwait(false); - var status = step.Status switch + RunOutcome outcome; + try { - StepStatus.Succeeded => RunStatus.Succeeded, - StepStatus.Cancelled => RunStatus.Cancelled, - _ => RunStatus.Failed, - }; - await _recorder.RunFinishedAsync(new RunFinished(runId, status, _time.GetUtcNow(), step.Outputs, step.Error), CancellationToken.None) + var inputFiles = await MaterializeInputsAsync(bound.Files, context.Workspace, cancellationToken).ConfigureAwait(false); + context.UsedBytes = inputFiles.Values.SelectMany(files => files).Sum(file => file.Length ?? 0); + outcome = context.UsedBytes > settings.MaxRunFilesBytes + ? RunOutcome.Aborted(RunStatus.Failed, new RunError(ErrorKind.ArtifactLimitExceeded, "The input files exceed the run limit.") { Code = "RUN_FILES_TOO_LARGE" }) + : await body(context, inputFiles).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + outcome = RunOutcome.Aborted(RunStatus.Cancelled, new RunError(ErrorKind.Cancelled, "The run was cancelled.")); + } + + await _recorder.RunFinishedAsync(new RunFinished(runId, outcome.Status, _time.GetUtcNow(), outcome.Outputs, outcome.Error), CancellationToken.None) .ConfigureAwait(false); - return new RunResult(runId, RunKind.Stage, status, _time.GetElapsedTime(started), [step], step.Outputs, step.Files, step.Error, workspace.DeleteResultsAsync); + return new RunResult(runId, kind, outcome.Status, _time.GetElapsedTime(started), outcome.Steps, outcome.Outputs, outcome.Files, outcome.Error, + context.Workspace.DeleteResultsAsync); } catch { - await workspace.DeleteResultsAsync().ConfigureAwait(false); + await context.Workspace.DeleteResultsAsync().ConfigureAwait(false); throw; } finally { - workspace.DeleteRun(); + context.Workspace.DeleteRun(); } } - public void Dispose() + private async Task ExecutePipelineAsync( + PipelineDefinition pipeline, + JsonObject pipelineInputs, + IReadOnlyDictionary> pipelineFiles, + RunContext context, + CancellationToken cancellationToken) { - if (_concurrency.IsValueCreated) + var steps = new List(); + var completed = new Dictionary(StringComparer.Ordinal); + var status = RunStatus.Succeeded; + RunError? error = null; + for (var i = 0; i < pipeline.Steps.Count; i++) { - _concurrency.Value.Dispose(); + var definition = pipeline.Steps[i]; + if (status != RunStatus.Succeeded) + { + steps.Add(Skipped(definition, i + 1)); + continue; + } + + var stage = await _catalog.FindStageAsync(definition.StageKey, definition.StageVersion, cancellationToken).ConfigureAwait(false) + ?? throw new PsrConfigurationException($"Stage '{definition.StageKey}@{definition.StageVersion}' of pipeline '{pipeline.Key}' is not in the catalog."); + var inputs = StepInputResolver.Resolve(stage, pipeline, i, pipelineInputs, pipelineFiles, completed); + var result = inputs.Errors.Count > 0 + ? InvalidInputs(stage, definition.Key, i + 1, inputs.Errors) + : await ExecuteStepAsync(stage, definition.Key, i + 1, inputs.Wire, inputs.Files, context, cancellationToken).ConfigureAwait(false); + steps.Add(result); + if (result.Status == StepStatus.Succeeded) + { + completed[definition.Key] = result; + } + else + { + status = StatusOf(result.Status); + error = result.Error; + } + } + + if (status != RunStatus.Succeeded) + { + return new RunOutcome(status, steps, null, null, error); } + + var outputs = new JsonObject(); + var files = new Dictionary>(StringComparer.Ordinal); + foreach (var output in pipeline.Outputs) + { + var source = completed[output.Source.Step]; + if (source.Outputs?[output.Source.Field] is { } value) + { + outputs[output.Name] = value.DeepClone(); + } + + if (source.Files.TryGetValue(output.Source.Field, out var produced)) + { + files[output.Name] = produced; + } + } + + return new RunOutcome(RunStatus.Succeeded, steps, outputs, files, null); } private async Task ResolveStageAsync(string key, int? version, CancellationToken cancellationToken) { - var stage = await _catalog.FindStageAsync(key, version, cancellationToken).ConfigureAwait(false); - if (stage is not null) + if (await _catalog.FindStageAsync(key, version, cancellationToken).ConfigureAwait(false) is { } stage) { return stage; } @@ -131,14 +237,28 @@ private async Task ResolveStageAsync(string key, int? version, throw new StageNotFoundException(key); } + private async Task ResolvePipelineAsync(string key, int? version, CancellationToken cancellationToken) + { + if (await _catalog.FindPipelineAsync(key, version, cancellationToken).ConfigureAwait(false) is { } pipeline) + { + return pipeline; + } + + if (version is { } requested && await _catalog.FindPipelineAsync(key, null, cancellationToken).ConfigureAwait(false) is not null) + { + throw new VersionNotFoundException(key, requested); + } + + throw new PipelineNotFoundException(key); + } + private async Task ExecuteStepAsync( StageDefinition stage, string stepKey, int order, - BoundInputs bound, - RunWorkspace workspace, - RunId runId, - PythonScriptRunnerOptions settings, + JsonObject inputs, + IReadOnlyList files, + RunContext context, CancellationToken cancellationToken) { var attempts = new List(); @@ -155,33 +275,18 @@ private async Task ExecuteStepAsync( Error = error is null ? null : error with { StepKey = stepKey }, }; - IReadOnlyList files; - try - { - files = await MaterializeInputsAsync(bound.Files, workspace, cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - return Finish(StepStatus.Cancelled, new RunError(ErrorKind.Cancelled, "The run was cancelled.")); - } - - var inputBytes = files.Sum(f => f.Size); - if (inputBytes > settings.MaxRunFilesBytes) - { - return Finish(StepStatus.Failed, new RunError(ErrorKind.ArtifactLimitExceeded, "The input files exceed the run limit.") { Code = "RUN_FILES_TOO_LARGE" }); - } - for (var number = 1; ; number++) { var cause = number == 1 ? AttemptCause.Initial : AttemptCause.AutoRetry; - var outcome = await RunAttemptAsync(stage, stepKey, number, cause, bound, files, workspace, runId, settings, settings.MaxRunFilesBytes - inputBytes, cancellationToken) - .ConfigureAwait(false); + var outcome = await RunAttemptAsync(stage, stepKey, number, cause, inputs, files, context, cancellationToken).ConfigureAwait(false); attempts.Add(outcome.Attempt); - await _recorder.AttemptFinishedAsync(new AttemptFinished(runId, stepKey, outcome.Attempt), CancellationToken.None).ConfigureAwait(false); + await _recorder.AttemptFinishedAsync(new AttemptFinished(context.Workspace.RunId, stepKey, outcome.Attempt), CancellationToken.None) + .ConfigureAwait(false); switch (outcome.Attempt.Status) { case AttemptStatus.Succeeded: + context.UsedBytes += outcome.Files.Values.SelectMany(f => f).Sum(f => f.Length ?? 0); return Finish(StepStatus.Succeeded, null, outcome); case AttemptStatus.Cancelled: return Finish(StepStatus.Cancelled, outcome.Attempt.Error); @@ -212,24 +317,22 @@ private async Task RunAttemptAsync( string stepKey, int number, AttemptCause cause, - BoundInputs bound, + JsonObject inputs, IReadOnlyList files, - RunWorkspace workspace, - RunId runId, - PythonScriptRunnerOptions settings, - long fileBudget, + RunContext context, CancellationToken cancellationToken) { + var settings = context.Settings; var startedAt = _time.GetUtcNow(); var started = _time.GetTimestamp(); var queued = TimeSpan.Zero; string? environmentHash = null; - var workDirectory = workspace.AttemptDirectory(stepKey, number); + var workDirectory = context.Workspace.AttemptDirectory(stepKey, number); using var activity = PsrDiagnostics.Source.StartActivity("psr.stage.execute"); activity?.SetTag("psr.stage.key", stage.Key); activity?.SetTag("psr.stage.version", stage.Version); - activity?.SetTag("psr.run.id", runId.ToString()); + activity?.SetTag("psr.run.id", context.Workspace.RunId.ToString()); activity?.SetTag("psr.step.key", stepKey); activity?.SetTag("psr.attempt", number); @@ -255,17 +358,17 @@ private async Task RunAttemptAsync( { Stage = stage, Environment = environment, - RunId = runId, + RunId = context.Workspace.RunId, StepKey = stepKey, Attempt = number, WorkDirectory = workDirectory, - Inputs = bound.Wire, + Inputs = inputs, Files = files, Secrets = secrets, Timeout = stage.Timeout ?? settings.DefaultTimeout, MemoryLimitMb = stage.MemoryLimitMb ?? settings.DefaultMemoryLimitMb, MaxFileBytes = settings.MaxFileBytes, - FileBudgetBytes = fileBudget, + FileBudgetBytes = settings.MaxRunFilesBytes - context.UsedBytes, MaxLogBytes = settings.MaxLogBytes, }, cancellationToken).ConfigureAwait(false); @@ -288,7 +391,7 @@ private async Task RunAttemptAsync( try { var outputFiles = execution.Status == AttemptStatus.Succeeded - ? MoveOutputs(execution.Files, workspace, stepKey) + ? MoveOutputs(execution.Files, context.Workspace, stepKey) : new Dictionary>(); var duration = _time.GetElapsedTime(started); var overhead = duration - execution.ProcessDuration - queued; @@ -340,31 +443,40 @@ private async Task RunAttemptAsync( return (values, missing); } - private static async Task> MaterializeInputsAsync( + /// Copies the caller's files once to runs/<id>/inputs; every step and attempt reads them from there. + private static async Task>> MaterializeInputsAsync( IReadOnlyList files, RunWorkspace workspace, CancellationToken cancellationToken) { - var copies = new List(files.Count); - foreach (var bound in files) + var result = new Dictionary>(StringComparer.Ordinal); + foreach (var group in files.GroupBy(f => f.Field)) { - var name = RunWorkspace.SafeFileName(bound.File.FileName); - var target = Path.Combine(workspace.InputsDirectory, bound.Field, bound.Index.ToString(System.Globalization.CultureInfo.InvariantCulture), name); - Directory.CreateDirectory(Path.GetDirectoryName(target)!); - await bound.File.SaveAsAsync(target, cancellationToken).ConfigureAwait(false); - - string sha256; - long size; - await using (var stream = File.OpenRead(target)) + var list = new List(); + foreach (var bound in group.OrderBy(f => f.Index)) { - size = stream.Length; - sha256 = Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false)); + var name = RunWorkspace.SafeFileName(bound.File.FileName); + var target = Path.Combine(workspace.InputsDirectory, bound.Field, bound.Index.ToString(CultureInfo.InvariantCulture), name); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + await bound.File.SaveAsAsync(target, cancellationToken).ConfigureAwait(false); + + string sha256; + await using (var stream = File.OpenRead(target)) + { + sha256 = Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false)); + } + + list.Add(PsrFile.FromProducedFile(target, name, bound.File.ContentType, sha256)); } - copies.Add(new InputFileCopy(bound.Field, bound.Index, target, name, bound.File.ContentType, size, sha256)); + result[group.Key] = list; } - return copies; + return result; } + private static List ToCopies(IReadOnlyDictionary> files) => + [.. files.SelectMany(group => group.Value.Select((file, index) => + new InputFileCopy(group.Key, index, file.LocalPath!, file.FileName, file.ContentType, file.Length ?? 0, file.Sha256!)))]; + private static Dictionary> MoveOutputs(IReadOnlyList files, RunWorkspace workspace, string stepKey) { var result = new Dictionary>(StringComparer.Ordinal); @@ -374,7 +486,7 @@ private static Dictionary> MoveOutputs(IReadOnlyL foreach (var file in group.OrderBy(f => f.Index)) { var name = RunWorkspace.SafeFileName(file.FileName); - var target = Path.Combine(workspace.ResultsDirectory, stepKey, file.Field, file.Index.ToString(System.Globalization.CultureInfo.InvariantCulture), name); + var target = Path.Combine(workspace.ResultsDirectory, stepKey, file.Field, file.Index.ToString(CultureInfo.InvariantCulture), name); Directory.CreateDirectory(Path.GetDirectoryName(target)!); File.Move(file.FullPath, target, overwrite: true); list.Add(PsrFile.FromProducedFile(target, name, file.ContentType, file.Sha256)); @@ -386,6 +498,38 @@ private static Dictionary> MoveOutputs(IReadOnlyL return result; } + private static RunStatus StatusOf(StepStatus status) => status switch + { + StepStatus.Succeeded => RunStatus.Succeeded, + StepStatus.Cancelled => RunStatus.Cancelled, + _ => RunStatus.Failed, + }; + + private static StepResult Skipped(PipelineStepDefinition definition, int order) => new() + { + StepKey = definition.Key, + Order = order, + StageKey = definition.StageKey, + StageVersion = definition.StageVersion, + Status = StepStatus.Skipped, + Attempts = [], + }; + + private static StepResult InvalidInputs(StageDefinition stage, string stepKey, int order, IReadOnlyList errors) => new() + { + StepKey = stepKey, + Order = order, + StageKey = stage.Key, + StageVersion = stage.Version, + Status = StepStatus.Failed, + Attempts = [], + Error = new RunError(ErrorKind.InputValidation, string.Join("; ", errors.Select(e => $"'{e.Field}': {e.Message}"))) + { + Code = errors[0].Code, + StepKey = stepKey, + }, + }; + private void ForwardOutput(string stageKey, AttemptResult attempt) { if (attempt.Stdout.Content.Length > 0) @@ -400,4 +544,24 @@ private void ForwardOutput(string stageKey, AttemptResult attempt) } private sealed record AttemptOutcome(AttemptResult Attempt, JsonObject? Outputs, IReadOnlyDictionary> Files); + + private sealed record RunOutcome( + RunStatus Status, + IReadOnlyList Steps, + JsonObject? Outputs, + IReadOnlyDictionary>? Files, + RunError? Error) + { + public static RunOutcome Aborted(RunStatus status, RunError error) => new(status, [], null, null, error); + } + + /// State shared by the steps of one run (folders, settings and the file budget). + private sealed class RunContext(RunWorkspace workspace, PythonScriptRunnerOptions settings) + { + public RunWorkspace Workspace { get; } = workspace; + + public PythonScriptRunnerOptions Settings { get; } = settings; + + public long UsedBytes { get; set; } + } } diff --git a/src/PythonScriptRunner.Core/Running/RunWorkspace.cs b/src/PythonScriptRunner.Core/Running/RunWorkspace.cs index 23f27fd..d303ac5 100644 --- a/src/PythonScriptRunner.Core/Running/RunWorkspace.cs +++ b/src/PythonScriptRunner.Core/Running/RunWorkspace.cs @@ -8,6 +8,8 @@ namespace PythonScriptRunner.Core.Running; /// Folders of one run: runs/<id> (work, deleted at the end) and results/<id> (kept until the result is disposed). internal sealed class RunWorkspace(string workRoot, RunId runId) { + public RunId RunId { get; } = runId; + public string RunDirectory { get; } = Path.Combine(workRoot, "runs", runId.ToString()); public string ResultsDirectory { get; } = Path.Combine(workRoot, "results", runId.ToString()); diff --git a/tests/PythonScriptRunner.Core.Tests/Integration/PipelineRunTests.cs b/tests/PythonScriptRunner.Core.Tests/Integration/PipelineRunTests.cs new file mode 100644 index 0000000..753e14c --- /dev/null +++ b/tests/PythonScriptRunner.Core.Tests/Integration/PipelineRunTests.cs @@ -0,0 +1,329 @@ +using PythonScriptRunner.Core.Tests.Infrastructure; + +namespace PythonScriptRunner.Core.Tests.Integration; + +public sealed class PipelineRunTests(PythonFixture fixture) +{ + private const string Generator = """ + from psrunner import stage + + @stage + def run(inputs, ctx): + out = ctx.new_file(inputs["nome"], "text/plain") + out.path.write_text(inputs["texto"], encoding="utf-8") + return {"arquivo": out, "tamanho": len(inputs["texto"])} + """; + + private const string Reader = """ + from psrunner import stage + + @stage + def run(inputs, ctx): + conteudo = inputs["arquivo"].read_text() + return {"resumo": f"{inputs['prefixo']}: {conteudo}", "porta": inputs["porta"]} + """; + + private const string Ok = """ + from psrunner import stage + + @stage + def run(inputs, ctx): + return {"valor": inputs["valor"] + 1} + """; + + private string CalculoProgressao => Path.Combine(fixture.RepositoryRoot, "tests", "fixtures", "calculo-progressao"); + + [Fact] + public async Task Spec003_AC01_CalculoProgressao_ProducesTheExpectedNumbers() + { + // Arrange + await using var host = RunnerHost.Create(fixture, b => b.AddFileCatalog(CalculoProgressao)); + + // Act + await using var result = await host.Runner.RunPipelineAsync("calculo-progressao", new + { + a1 = 1, + razao = 2, + n = 4, + tipo = "geometrica", + coeficientes = new[] { 1.0, 0.0, 1.0 }, + }, cancellationToken: TestContext.Current.CancellationToken); + + // Assert + result.Status.ShouldBe(RunStatus.Succeeded, result.Error?.Message); + result.Kind.ShouldBe(RunKind.Pipeline); + result.GetOutput("soma").ShouldBe(89); + result.GetOutput("media").ShouldBe(22.25); + result.GetOutput("desvio_padrao").ShouldBe(25.3118, 1e-4); + var tabela = result.GetOutput>("tabela"); + tabela.ShouldBe([new(1, 1, 2), new(2, 2, 5), new(3, 4, 17), new(4, 8, 65)]); + result.Steps.Select(s => (s.StepKey, s.Status)).ShouldBe( + [ + ("termos", StepStatus.Succeeded), + ("polinomio", StepStatus.Succeeded), + ("estatisticas", StepStatus.Succeeded), + ("tabela", StepStatus.Succeeded), + ]); + } + + [Fact] + public async Task Spec003_AC02_StepCanReadAnyEarlierStep() + { + await using var host = RunnerHost.Create(fixture, b => b.AddFileCatalog(CalculoProgressao)); + + await using var result = await host.Runner.RunPipelineAsync("calculo-progressao", new + { + a1 = 3, + razao = 2, + n = 3, + coeficientes = new[] { 0.0, 1.0 }, + }, cancellationToken: TestContext.Current.CancellationToken); + + // 'tabela' (step 4) reads 'termos' from step 1, skipping steps 2 and 3; tipo uses its default (aritmetica). + result.GetStepOutput("termos", "termos").ShouldBe([3, 5, 7]); + result.GetStepOutput>("tabela", "tabela").Select(l => l.X).ShouldBe([3, 5, 7]); + } + + [Fact] + public async Task Spec003_AC03_AC04_AC09_ConstantsImplicitBindingsAndFilesBetweenSteps() + { + // Arrange + await using var host = RunnerHost.Create(fixture, b => b + .AddStage("gerar", s => s.Script(Generator) + .Input("texto", FieldType.String).Input("nome", FieldType.String) + .Output("arquivo", FieldType.File).Output("tamanho", FieldType.Integer)) + .AddStage("ler", s => s.Script(Reader) + .Input("arquivo", FieldType.File).Input("prefixo", FieldType.String).Input("porta", FieldType.Integer) + .Output("resumo", FieldType.String).Output("porta", FieldType.Integer)) + .AddPipeline("arquivo-entre-passos", p => p + .Input("texto", FieldType.String) + .Input("prefixo", FieldType.String) + .Step("gerar", "gerar@1", s => s.Bind("nome", From.Constant("dados.txt"))) + .Step("ler", "ler@1", s => s + .Bind("prefixo", From.Pipeline("prefixo")) + .Bind("porta", From.Constant(465))) + .Output("arquivo", From.Step("gerar", "arquivo")) + .Output("resumo", From.Step("ler", "resumo")) + .Output("porta", From.Step("ler", "porta")))); + + // Act + await using var result = await host.Runner.RunPipelineAsync("arquivo-entre-passos", new { texto = "olá", prefixo = "conteúdo" }, + cancellationToken: TestContext.Current.CancellationToken); + + // Assert + result.Status.ShouldBe(RunStatus.Succeeded, result.Error?.Message); + result.GetOutput("porta").ShouldBe(465); + result.GetOutput("resumo").ShouldBe("conteúdo: olá"); + var file = result.GetFile("arquivo"); + file.FileName.ShouldBe("dados.txt"); + File.ReadAllText(file.LocalPath!).ShouldBe("olá"); + Should.Throw(() => result.GetOutput("tamanho")); + } + + [Fact] + public async Task Spec003_AC07_FailedStep_SkipsTheRest() + { + await using var host = RunnerHost.Create(fixture, b => WithChain(b, failing: """ + from psrunner import StageError, stage + + @stage + def run(inputs, ctx): + raise StageError("divergiu", code="DIV") + """)); + + await using var result = await host.Runner.RunPipelineAsync("cadeia", new { valor = 1 }, cancellationToken: TestContext.Current.CancellationToken); + + result.Status.ShouldBe(RunStatus.Failed); + result.Steps.Select(s => s.Status).ShouldBe([StepStatus.Succeeded, StepStatus.Failed, StepStatus.Skipped, StepStatus.Skipped]); + result.Error!.StepKey.ShouldBe("polinomio"); + result.Error.Code.ShouldBe("DIV"); + result.Outputs.ShouldBeEmpty(); + result.GetStepOutput("termos", "valor").ShouldBe(2); + } + + [Fact] + public async Task Spec003_AC08_RetryInsideAStep_CompletesThePipeline() + { + var counter = Path.Combine(Path.GetTempPath(), "psr-tests", $"counter-{Guid.NewGuid():N}.txt"); + await using var host = RunnerHost.Create(fixture, b => WithChain(b, failing: $$""" + from pathlib import Path + from psrunner import StageError, stage + + @stage + def run(inputs, ctx): + counter = Path(r"{{counter}}") + count = int(counter.read_text()) + 1 if counter.exists() else 1 + counter.write_text(str(count)) + if count == 1: + raise StageError("falha transitória") + return {"valor": inputs["valor"] + 1} + """, retries: 1)); + + await using var result = await host.Runner.RunPipelineAsync("cadeia", new { valor = 1 }, cancellationToken: TestContext.Current.CancellationToken); + + result.Status.ShouldBe(RunStatus.Succeeded, result.Error?.Message); + result.Steps[1].Attempts.Count.ShouldBe(2); + result.GetOutput("valor").ShouldBe(5); + } + + [Fact] + public async Task Spec003_AC11_Cancellation_StopsTheStepAndSkipsTheRest() + { + await using var host = RunnerHost.Create(fixture, b => WithChain(b, failing: """ + import time + from psrunner import stage + + @stage + def run(inputs, ctx): + time.sleep(60) + return {"valor": 0} + """)); + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cancellation.CancelAfter(TimeSpan.FromSeconds(3)); + + await using var result = await host.Runner.RunPipelineAsync("cadeia", new { valor = 1 }, cancellationToken: cancellation.Token); + + result.Status.ShouldBe(RunStatus.Cancelled); + result.Steps.Select(s => s.Status).ShouldBe([StepStatus.Succeeded, StepStatus.Cancelled, StepStatus.Skipped, StepStatus.Skipped]); + } + + [Fact] + public async Task Spec003_AC12_FolderPipeline_WithMissingStageVersion_FailsToLoad() + { + // Arrange + using var catalog = new TempCatalog(); + catalog.AddStage("eco", Scripts.EchoManifest, Scripts.Echo); + var pipelines = Path.Combine(catalog.Root, "pipelines"); + Directory.CreateDirectory(pipelines); + await File.WriteAllTextAsync(Path.Combine(pipelines, "quebrada.pipeline.json"), """ + { + "key": "quebrada", "version": 1, "name": "Quebrada", + "inputs": [ { "name": "texto", "type": "string" } ], + "steps": [ { "key": "eco", "stage": "eco@2" } ] + } + """, TestContext.Current.CancellationToken); + await using var host = RunnerHost.Create(fixture, b => b.AddFileCatalog(catalog.Root)); + + // Act + var exception = await Should.ThrowAsync(() => + host.Runner.RunPipelineAsync("quebrada", new { texto = "x" }, cancellationToken: TestContext.Current.CancellationToken)); + + // Assert + exception.Message.ShouldContain("PIPE_STAGE_NOT_PUBLISHED"); + exception.Message.ShouldContain("eco@2"); + } + + [Fact] + public async Task Spec003_AC13_FluentAndFilePipelines_BehaveTheSame() + { + // Arrange: the same stages, with the pipeline from the JSON file or from C#. + using var stagesOnly = new TempCatalog(); + CopyDirectory(Path.Combine(CalculoProgressao, "stages"), Path.Combine(stagesOnly.Root, "stages")); + await using var fromFile = RunnerHost.Create(fixture, b => b.AddFileCatalog(CalculoProgressao)); + await using var fromCode = RunnerHost.Create(fixture, b => b + .AddFileCatalog(stagesOnly.Root) + .AddPipeline("calculo-progressao", p => p + .Name("Cálculo de progressão") + .Input("a1", FieldType.Number) + .Input("razao", FieldType.Number) + .Input("n", FieldType.Integer) + .Input("tipo", FieldType.Enum("aritmetica", "geometrica"), required: false, @default: "aritmetica") + .Input("coeficientes", FieldType.ListOf(FieldType.Number)) + .Step("termos", "progressao-termos@1") + .Step("polinomio", "polinomio-avaliar@1", s => s + .Bind("coeficientes", From.Pipeline("coeficientes")) + .Bind("x", From.Step("termos", "termos"))) + .Step("estatisticas", "estatisticas@1", s => s.Bind("valores", From.Step("polinomio", "valores"))) + .Step("tabela", "montar-tabela@1", s => s + .Bind("x", From.Step("termos", "termos")) + .Bind("y", From.Step("polinomio", "valores"))) + .Output("soma", From.Step("estatisticas", "soma")) + .Output("media", From.Step("estatisticas", "media")) + .Output("desvio_padrao", From.Step("estatisticas", "desvio_padrao")) + .Output("tabela", From.Step("tabela", "tabela")))); + var inputs = new { a1 = 2, razao = 3, n = 5, tipo = "aritmetica", coeficientes = new[] { 1.0, 1.0 } }; + + // Act + var fileReport = await Validate(fromFile); + var codeReport = await Validate(fromCode); + await using var fileResult = await fromFile.Runner.RunPipelineAsync("calculo-progressao", inputs, cancellationToken: TestContext.Current.CancellationToken); + await using var codeResult = await fromCode.Runner.RunPipelineAsync("calculo-progressao", inputs, cancellationToken: TestContext.Current.CancellationToken); + + // Assert + fileReport.Errors.ShouldBe(codeReport.Errors); + fileReport.Warnings.ShouldBe(codeReport.Warnings); + fileReport.IsValid.ShouldBeTrue(); + fileResult.Status.ShouldBe(RunStatus.Succeeded, fileResult.Error?.Message); + fileResult.Outputs.ToJsonString().ShouldBe(codeResult.Outputs.ToJsonString()); + } + + [Fact] + public async Task IntegerOutput_FeedsADecimalInput_AsAnExactDecimal() + { + await using var host = RunnerHost.Create(fixture, b => b + .AddStage("contar", s => s.Script(""" + from psrunner import stage + + @stage + def run(inputs, ctx): + return {"quantidade": 5} + """).Output("quantidade", FieldType.Integer)) + .AddStage("preco", s => s.Script(""" + from decimal import Decimal + from psrunner import stage + + @stage + def run(inputs, ctx): + assert isinstance(inputs["quantidade"], Decimal) + return {"total": inputs["quantidade"] * Decimal("0.10")} + """).Input("quantidade", FieldType.Decimal).Output("total", FieldType.Decimal)) + .AddPipeline("venda", p => p.Step("contar", "contar@1").Step("preco", "preco@1").Output("total", From.Step("preco", "total")))); + + await using var result = await host.Runner.RunPipelineAsync("venda", null, cancellationToken: TestContext.Current.CancellationToken); + + result.Status.ShouldBe(RunStatus.Succeeded, result.Error?.Message); + result.GetOutput("total").ShouldBe(0.50m); + } + + [Fact] + public async Task UnknownPipeline_AndInvalidInputs_AreReportedBeforeRunning() + { + await using var host = RunnerHost.Create(fixture, b => b.AddFileCatalog(CalculoProgressao)); + + await Should.ThrowAsync(() => host.Runner.RunPipelineAsync("inexistente", null, cancellationToken: TestContext.Current.CancellationToken)); + var invalid = await Should.ThrowAsync(() => + host.Runner.RunPipelineAsync("calculo-progressao", new { a1 = 1 }, cancellationToken: TestContext.Current.CancellationToken)); + invalid.Errors.Select(e => e.Field).ShouldBe(["razao", "n", "coeficientes"]); + } + + private static IPythonScriptRunnerBuilder WithChain(IPythonScriptRunnerBuilder builder, string failing, int retries = 0) => builder + .AddStage("incremento", s => s.Script(Ok).Input("valor", FieldType.Integer).Output("valor", FieldType.Integer)) + .AddStage("instavel", s => s.Script(failing).Input("valor", FieldType.Integer).Output("valor", FieldType.Integer) + .Retry(retries, TimeSpan.FromMilliseconds(50))) + .AddPipeline("cadeia", p => p + .Input("valor", FieldType.Integer) + .Step("termos", "incremento@1") + .Step("polinomio", "instavel@1") + .Step("estatisticas", "incremento@1") + .Step("tabela", "incremento@1") + .Output("valor", From.Step("tabela", "valor"))); + + private static async Task Validate(RunnerHost host) + { + var catalog = host.Services.GetRequiredService(); + var pipeline = await catalog.FindPipelineAsync("calculo-progressao", null, TestContext.Current.CancellationToken); + return await host.Services.GetRequiredService().ValidateAsync(pipeline!, TestContext.Current.CancellationToken); + } + + private static void CopyDirectory(string source, string target) + { + foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories)) + { + var destination = Path.Combine(target, Path.GetRelativePath(source, file)); + Directory.CreateDirectory(Path.GetDirectoryName(destination)!); + File.Copy(file, destination); + } + } + + public sealed record Linha(long I, double X, double Y); +} diff --git a/tests/PythonScriptRunner.Core.Tests/Unit/PipelineRulesTests.cs b/tests/PythonScriptRunner.Core.Tests/Unit/PipelineRulesTests.cs new file mode 100644 index 0000000..3c72833 --- /dev/null +++ b/tests/PythonScriptRunner.Core.Tests/Unit/PipelineRulesTests.cs @@ -0,0 +1,129 @@ +using PythonScriptRunner.Core.Pipelines; + +namespace PythonScriptRunner.Core.Tests.Unit; + +public sealed class PipelineRulesTests +{ + private static readonly StageDefinition Producer = Stage("produtor", + inputs: [new("n", FieldType.Integer)], + outputs: [new("valor", FieldType.Integer), new("talvez", FieldType.Number) { Required = false }, new("rotulo", FieldType.Enum("a", "b"))]); + + private static readonly StageDefinition Consumer = Stage("consumidor", + inputs: [new("valor", FieldType.Number), new("porta", FieldType.Integer) { Required = false, Default = 465 }]); + + [Fact] + public void ValidPipeline_HasNoErrors() + { + var report = Validate(Pipeline( + [new("n", FieldType.Integer)], + [Step("p", "produtor"), Step("c", "consumidor")], + [new PipelineOutputDefinition("valor", From.Step("p", "valor"))])); + + report.IsValid.ShouldBeTrue(report.ToString()); + report.Warnings.ShouldBeEmpty(); + } + + [Theory] + [MemberData(nameof(InvalidPipelines))] + public void Spec003_AC05_EachRuleReportsItsCodeStepAndField(string code, PipelineDefinition pipeline, string? step, string? field) + { + var report = Validate(pipeline); + + var issue = report.Errors.ShouldHaveSingleItem(report.ToString()); + issue.Code.ShouldBe(code); + issue.Step.ShouldBe(step); + issue.Field.ShouldBe(field); + } + + public static TheoryData InvalidPipelines() => new() + { + { "PIPE_DEFINITION", Pipeline([new("N", FieldType.Integer)], [Step("p", "produtor", ("n", From.Constant(1)))], []), null, null }, + { "PIPE_NO_STEPS", Pipeline([], [], []), null, null }, + { "PIPE_STEP_KEY", Pipeline([new("n", FieldType.Integer)], [Step("p", "produtor"), Step("p", "produtor", ("n", From.Pipeline("n")))], []), "p", null }, + { "PIPE_STAGE_NOT_PUBLISHED", Pipeline([new("n", FieldType.Integer)], [Step("p", "produtor"), new PipelineStepDefinition("x", "produtor", 9)], []), "x", null }, + { "PIPE_FORWARD_REF", Pipeline([new("n", FieldType.Integer)], [Step("c", "consumidor", ("valor", From.Step("p", "valor"))), Step("p", "produtor", ("n", From.Pipeline("n")))], []), "c", "valor" }, + { "PIPE_UNKNOWN_SOURCE", Pipeline([new("n", FieldType.Integer)], [Step("p", "produtor"), Step("c", "consumidor", ("valor", From.Step("p", "inexistente")))], []), "c", "valor" }, + { "PIPE_UNKNOWN_INPUT", Pipeline([new("n", FieldType.Integer)], [Step("p", "produtor", ("nao_existe", From.Constant(1)))], []), "p", "nao_existe" }, + { "PIPE_UNBOUND_REQUIRED", Pipeline([], [Step("p", "produtor")], []), "p", "n" }, + { "PIPE_TYPE_MISMATCH", Pipeline([new("n", FieldType.Number)], [Step("p", "produtor")], []), "p", "n" }, + { "PIPE_OPTIONAL_TO_REQUIRED", Pipeline([new("n", FieldType.Integer)], [Step("p", "produtor"), Step("c", "consumidor", ("valor", From.Step("p", "talvez")))], []), "c", "valor" }, + { "PIPE_CONST_INVALID", Pipeline([], [Step("p", "produtor", ("n", From.Constant("dez")))], []), "p", "n" }, + { "PIPE_OUTPUT_SOURCE", Pipeline([new("n", FieldType.Integer)], [Step("p", "produtor")], [new PipelineOutputDefinition("saida", From.Step("p", "inexistente"))]), "p", "saida" }, + }; + + [Fact] + public void Spec003_AC05_WarningsDoNotBlock() + { + var report = Validate(Pipeline( + [new("n", FieldType.Integer), new("sobra", FieldType.String), new("anexo", FieldType.File)], + [Step("p", "produtor")], + [])); + + report.IsValid.ShouldBeTrue(); + report.Warnings.Select(w => (w.Code, w.Field)).ShouldBe([("PIPE_UNUSED_INPUT", "sobra"), ("PIPE_UNUSED_INPUT", "anexo"), ("PIPE_FILE_INPUT", "anexo")]); + } + + [Theory] + [MemberData(nameof(Compatibility))] + public void Spec003_AC06_TypeCompatibility(FieldType source, FieldType target, bool accepted) + { + TypeCompatibility.IsAssignable(new FieldDefinition("s", source), new FieldDefinition("t", target)).ShouldBe(accepted); + } + + public static TheoryData Compatibility() => new() + { + { FieldType.Integer, FieldType.Number, true }, + { FieldType.Integer, FieldType.Decimal, true }, + { FieldType.Enum("a", "b"), FieldType.Enum("a", "b", "c"), true }, + { FieldType.Enum("a", "b"), FieldType.String, true }, + { FieldType.ListOf(FieldType.Integer), FieldType.ListOf(FieldType.Number), true }, + { FieldType.Table, FieldType.Json, true }, + { FieldType.Number, FieldType.Integer, false }, + { FieldType.String, FieldType.Enum("a"), false }, + { FieldType.Enum("a", "z"), FieldType.Enum("a", "b"), false }, + { FieldType.File, FieldType.Json, false }, + { FieldType.File, FieldType.ListOf(FieldType.File), false }, + }; + + [Fact] + public void TableColumns_MustExistInTheSource() + { + var source = new FieldDefinition("s", FieldType.Table) { Columns = [new TableColumn("i", FieldType.Integer), new TableColumn("x", FieldType.Number)] }; + var target = new FieldDefinition("t", FieldType.Table) { Columns = [new TableColumn("x", FieldType.Number)] }; + var missing = new FieldDefinition("t", FieldType.Table) { Columns = [new TableColumn("y", FieldType.Number)] }; + + TypeCompatibility.IsAssignable(source, target).ShouldBeTrue(); + TypeCompatibility.IsAssignable(source, missing).ShouldBeFalse(); + } + + private static ValidationReport Validate(PipelineDefinition pipeline) => + PipelineRules.Validate(pipeline, (key, version) => (key, version) switch + { + ("produtor", 1) => Producer, + ("consumidor", 1) => Consumer, + _ => null, + }); + + private static StageDefinition Stage(string key, FieldDefinition[] inputs, FieldDefinition[]? outputs = null) => new() + { + Key = key, + Version = 1, + Name = key, + Script = ScriptSource.Inline("x"), + Inputs = inputs, + Outputs = outputs ?? [], + }; + + private static PipelineStepDefinition Step(string key, string stage, params (string Input, BindingSource Source)[] bindings) => + new(key, stage, 1) { Bindings = [.. bindings.Select(b => new InputBinding(b.Input, b.Source))] }; + + private static PipelineDefinition Pipeline(FieldDefinition[] inputs, PipelineStepDefinition[] steps, PipelineOutputDefinition[] outputs) => new() + { + Key = "trilha", + Version = 1, + Name = "Trilha", + Inputs = inputs, + Steps = steps, + Outputs = outputs, + }; +} diff --git a/tests/fixtures/calculo-progressao/pipelines/calculo-progressao.pipeline.json b/tests/fixtures/calculo-progressao/pipelines/calculo-progressao.pipeline.json new file mode 100644 index 0000000..22e3588 --- /dev/null +++ b/tests/fixtures/calculo-progressao/pipelines/calculo-progressao.pipeline.json @@ -0,0 +1,88 @@ +{ + "$schema": "../../../../schemas/pipeline.v1.json", + "key": "calculo-progressao", + "version": 1, + "name": "Cálculo de progressão", + "inputs": [ + { + "name": "a1", + "type": "number" + }, + { + "name": "razao", + "type": "number" + }, + { + "name": "n", + "type": "integer" + }, + { + "name": "tipo", + "type": "enum", + "enumValues": [ + "aritmetica", + "geometrica" + ], + "required": false, + "default": "aritmetica" + }, + { + "name": "coeficientes", + "type": "list", + "itemType": "number" + } + ], + "steps": [ + { + "key": "termos", + "stage": "progressao-termos@1", + "bindings": { + "a1": "$pipeline.a1", + "razao": "$pipeline.razao", + "n": "$pipeline.n", + "tipo": "$pipeline.tipo" + } + }, + { + "key": "polinomio", + "stage": "polinomio-avaliar@1", + "bindings": { + "coeficientes": "$pipeline.coeficientes", + "x": "$steps.termos.termos" + } + }, + { + "key": "estatisticas", + "stage": "estatisticas@1", + "bindings": { + "valores": "$steps.polinomio.valores" + } + }, + { + "key": "tabela", + "stage": "montar-tabela@1", + "bindings": { + "x": "$steps.termos.termos", + "y": "$steps.polinomio.valores" + } + } + ], + "outputs": [ + { + "name": "soma", + "source": "$steps.estatisticas.soma" + }, + { + "name": "media", + "source": "$steps.estatisticas.media" + }, + { + "name": "desvio_padrao", + "source": "$steps.estatisticas.desvio_padrao" + }, + { + "name": "tabela", + "source": "$steps.tabela.tabela" + } + ] +} diff --git a/tests/fixtures/calculo-progressao/stages/estatisticas/requirements.txt b/tests/fixtures/calculo-progressao/stages/estatisticas/requirements.txt new file mode 100644 index 0000000..24ce15a --- /dev/null +++ b/tests/fixtures/calculo-progressao/stages/estatisticas/requirements.txt @@ -0,0 +1 @@ +numpy diff --git a/tests/fixtures/calculo-progressao/stages/estatisticas/script.py b/tests/fixtures/calculo-progressao/stages/estatisticas/script.py new file mode 100644 index 0000000..70d93a4 --- /dev/null +++ b/tests/fixtures/calculo-progressao/stages/estatisticas/script.py @@ -0,0 +1,12 @@ +import numpy as np +from psrunner import stage + + +@stage +def run(inputs, ctx): + valores = np.array(inputs["valores"], dtype=float) + return { + "soma": float(valores.sum()), + "media": float(valores.mean()), + "desvio_padrao": float(valores.std()), + } diff --git a/tests/fixtures/calculo-progressao/stages/estatisticas/stage.json b/tests/fixtures/calculo-progressao/stages/estatisticas/stage.json new file mode 100644 index 0000000..fceafb7 --- /dev/null +++ b/tests/fixtures/calculo-progressao/stages/estatisticas/stage.json @@ -0,0 +1,29 @@ +{ + "$schema": "../../../../../schemas/stage.v1.json", + "key": "estatisticas", + "version": 1, + "name": "Estatísticas", + "idempotent": true, + "inputs": [ + { + "name": "valores", + "type": "list", + "itemType": "number" + } + ], + "outputs": [ + { + "name": "soma", + "type": "number" + }, + { + "name": "media", + "type": "number" + }, + { + "name": "desvio_padrao", + "type": "number", + "description": "Desvio padrão populacional" + } + ] +} diff --git a/tests/fixtures/calculo-progressao/stages/montar-tabela/requirements.txt b/tests/fixtures/calculo-progressao/stages/montar-tabela/requirements.txt new file mode 100644 index 0000000..fb6c7ed --- /dev/null +++ b/tests/fixtures/calculo-progressao/stages/montar-tabela/requirements.txt @@ -0,0 +1 @@ +pandas diff --git a/tests/fixtures/calculo-progressao/stages/montar-tabela/script.py b/tests/fixtures/calculo-progressao/stages/montar-tabela/script.py new file mode 100644 index 0000000..024da02 --- /dev/null +++ b/tests/fixtures/calculo-progressao/stages/montar-tabela/script.py @@ -0,0 +1,9 @@ +import pandas as pd +from psrunner import stage + + +@stage +def run(inputs, ctx): + x, y = inputs["x"], inputs["y"] + tabela = pd.DataFrame({"i": range(1, len(x) + 1), "x": x, "y": y}) + return {"tabela": tabela} diff --git a/tests/fixtures/calculo-progressao/stages/montar-tabela/stage.json b/tests/fixtures/calculo-progressao/stages/montar-tabela/stage.json new file mode 100644 index 0000000..6fd62c6 --- /dev/null +++ b/tests/fixtures/calculo-progressao/stages/montar-tabela/stage.json @@ -0,0 +1,39 @@ +{ + "$schema": "../../../../../schemas/stage.v1.json", + "key": "montar-tabela", + "version": 1, + "name": "Montar tabela", + "idempotent": true, + "inputs": [ + { + "name": "x", + "type": "list", + "itemType": "number" + }, + { + "name": "y", + "type": "list", + "itemType": "number" + } + ], + "outputs": [ + { + "name": "tabela", + "type": "table", + "columns": [ + { + "name": "i", + "type": "integer" + }, + { + "name": "x", + "type": "number" + }, + { + "name": "y", + "type": "number" + } + ] + } + ] +} diff --git a/tests/fixtures/calculo-progressao/stages/polinomio-avaliar/requirements.txt b/tests/fixtures/calculo-progressao/stages/polinomio-avaliar/requirements.txt new file mode 100644 index 0000000..24ce15a --- /dev/null +++ b/tests/fixtures/calculo-progressao/stages/polinomio-avaliar/requirements.txt @@ -0,0 +1 @@ +numpy diff --git a/tests/fixtures/calculo-progressao/stages/polinomio-avaliar/script.py b/tests/fixtures/calculo-progressao/stages/polinomio-avaliar/script.py new file mode 100644 index 0000000..211221c --- /dev/null +++ b/tests/fixtures/calculo-progressao/stages/polinomio-avaliar/script.py @@ -0,0 +1,10 @@ +import numpy as np +from psrunner import stage + + +@stage +def run(inputs, ctx): + coeficientes = np.array(inputs["coeficientes"], dtype=float) + x = np.array(inputs["x"], dtype=float) + valores = np.polynomial.polynomial.polyval(x, coeficientes) + return {"valores": valores.tolist()} diff --git a/tests/fixtures/calculo-progressao/stages/polinomio-avaliar/stage.json b/tests/fixtures/calculo-progressao/stages/polinomio-avaliar/stage.json new file mode 100644 index 0000000..a0e87f6 --- /dev/null +++ b/tests/fixtures/calculo-progressao/stages/polinomio-avaliar/stage.json @@ -0,0 +1,27 @@ +{ + "$schema": "../../../../../schemas/stage.v1.json", + "key": "polinomio-avaliar", + "version": 1, + "name": "Avaliar polinômio", + "idempotent": true, + "inputs": [ + { + "name": "coeficientes", + "type": "list", + "itemType": "number", + "description": "a, b, c, d... de a + b·x + c·x² + ..." + }, + { + "name": "x", + "type": "list", + "itemType": "number" + } + ], + "outputs": [ + { + "name": "valores", + "type": "list", + "itemType": "number" + } + ] +} diff --git a/tests/fixtures/calculo-progressao/stages/progressao-termos/script.py b/tests/fixtures/calculo-progressao/stages/progressao-termos/script.py new file mode 100644 index 0000000..c639ef4 --- /dev/null +++ b/tests/fixtures/calculo-progressao/stages/progressao-termos/script.py @@ -0,0 +1,12 @@ +from psrunner import stage + + +@stage +def run(inputs, ctx): + a1, razao, n = inputs["a1"], inputs["razao"], inputs["n"] + if inputs["tipo"] == "geometrica": + termos = [a1 * razao**i for i in range(n)] + else: + termos = [a1 + razao * i for i in range(n)] + ctx.log.info("gerados %d termos", n) + return {"termos": termos} diff --git a/tests/fixtures/calculo-progressao/stages/progressao-termos/stage.json b/tests/fixtures/calculo-progressao/stages/progressao-termos/stage.json new file mode 100644 index 0000000..8cca782 --- /dev/null +++ b/tests/fixtures/calculo-progressao/stages/progressao-termos/stage.json @@ -0,0 +1,40 @@ +{ + "$schema": "../../../../../schemas/stage.v1.json", + "key": "progressao-termos", + "version": 1, + "name": "Termos de progressão", + "idempotent": true, + "inputs": [ + { + "name": "a1", + "type": "number", + "description": "Primeiro termo" + }, + { + "name": "razao", + "type": "number" + }, + { + "name": "n", + "type": "integer", + "description": "Quantidade de termos" + }, + { + "name": "tipo", + "type": "enum", + "enumValues": [ + "aritmetica", + "geometrica" + ], + "required": false, + "default": "aritmetica" + } + ], + "outputs": [ + { + "name": "termos", + "type": "list", + "itemType": "number" + } + ] +}