diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d0c7c94 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +**/bin/ +**/obj/ +**/node_modules/ +**/.venv/ +**/__pycache__/ +**/TestResults/ +panel/dist/ +.git/ +.github/ +.vs/ +.vscode/ +.idea/ +docs/ +tests/ +artifacts/ +*.user diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..dea491e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,53 @@ +# Weekly dependency updates (spec 009, D-184): one grouped pull request per ecosystem for minor and patch versions; +# majors arrive one by one so they can be reviewed against the decisions that pinned them. +version: 2 +updates: + - package-ecosystem: nuget + directory: / + schedule: + interval: weekly + day: monday + open-pull-requests-limit: 5 + groups: + aspire: + patterns: ["Aspire.*"] + update-types: [minor, patch] + dotnet: + patterns: ["Microsoft.*", "System.*", "Npgsql*"] + update-types: [minor, patch] + testing: + patterns: ["xunit*", "Shouldly", "Testcontainers*", "Microsoft.Playwright", "Deque.*"] + update-types: [minor, patch] + other: + patterns: ["*"] + update-types: [minor, patch] + + - package-ecosystem: npm + directory: /panel + schedule: + interval: weekly + day: monday + open-pull-requests-limit: 5 + groups: + panel: + patterns: ["*"] + update-types: [minor, patch] + + - package-ecosystem: uv + directory: /python/psrunner + schedule: + interval: weekly + day: monday + groups: + psrunner-dev: + patterns: ["*"] + update-types: [minor, patch] + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + day: monday + groups: + actions: + patterns: ["*"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4978e7..10df6f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,8 @@ on: branches: [main] pull_request: branches: [main] + # The release workflow runs the same checks on the tagged commit before publishing anything (spec 009). + workflow_call: permissions: contents: read @@ -26,7 +28,10 @@ jobs: os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} steps: + # Full history: MinVer computes the package version from the tags (D-179). - uses: actions/checkout@v7 + with: + fetch-depth: 0 - uses: actions/setup-dotnet@v6 with: @@ -48,6 +53,18 @@ jobs: - name: Build run: dotnet build --no-restore --configuration Release + # AC-5 in dry-run: the 7 packages (6 libraries + the psr tool) with one version, symbols included. + - name: Pack (dry run) + if: runner.os == 'Linux' + run: | + dotnet pack PythonScriptRunner.slnx --no-build --configuration Release --output artifacts/packages + ls artifacts/packages + count=$(ls artifacts/packages/*.nupkg | wc -l) + versions=$(ls artifacts/packages/*.nupkg | sed -E 's#.*/PythonScriptRunner(\.[A-Za-z]+)+\.([0-9].*)\.nupkg#\2#' | sort -u | wc -l) + test "$count" -eq 7 || { echo "Expected 7 packages, got $count"; exit 1; } + test "$versions" -eq 1 || { echo "The packages do not share one version"; exit 1; } + test "$(ls artifacts/packages/*.snupkg | wc -l)" -eq 7 || { echo "Missing symbol packages"; exit 1; } + # weasyprint (CS-2 fixture, html-para-pdf) renders with pango. - name: Install pango if: runner.os == 'Linux' @@ -82,19 +99,120 @@ jobs: if: runner.os == 'Windows' run: dotnet test --no-build --configuration Release --project tests/PythonScriptRunner.Core.Tests - format: - name: .NET format + quality: + name: Quality runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 + with: + fetch-depth: 0 - uses: actions/setup-dotnet@v6 with: global-json-file: global.json + - uses: astral-sh/setup-uv@v10.3.0 + + - uses: actions/setup-node@v7 + with: + node-version-file: panel/.nvmrc + cache: npm + cache-dependency-path: panel/package-lock.json + - name: Verify formatting run: dotnet format --verify-no-changes + # Dependency audit (spec 009): known vulnerabilities in NuGet packages (transitive included), in the panel's + # runtime npm packages (high and critical) and in the Python SDK's dependencies. + - name: Audit NuGet packages + run: | + dotnet list PythonScriptRunner.slnx package --vulnerable --include-transitive | tee vulnerable.txt + if grep -q "has the following vulnerable packages" vulnerable.txt; then exit 1; fi + + - name: Audit npm packages + working-directory: panel + run: npm audit --omit=dev --audit-level=high + + - name: Audit Python packages + working-directory: python/psrunner + run: | + uv export --frozen --no-dev --no-emit-project --no-hashes --format requirements-txt > requirements-audit.txt + uvx pip-audit --requirement requirements-audit.txt --no-deps --disable-pip + + # Spec 009, AC-3/AC-7/AC-8 and spec 001, AC-30: the host images as they run in production, and the docker-compose + # generated by `aspire publish`. + containers: + name: Containers + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + + - uses: docker/setup-buildx-action@v4 + + - name: Build the worker image + uses: docker/build-push-action@v7 + with: + context: . + file: hosts/PythonScriptRunner.WorkerHost/Dockerfile + tags: psr-worker:ci + load: true + cache-from: type=gha,scope=psr-worker + cache-to: type=gha,scope=psr-worker,mode=max + + - name: Build the dashboard image + uses: docker/build-push-action@v7 + with: + context: . + file: hosts/PythonScriptRunner.DashboardHost/Dockerfile + tags: psr-dashboard:ci + load: true + cache-from: type=gha,scope=psr-dashboard + cache-to: type=gha,scope=psr-dashboard,mode=max + + - name: Install the Aspire CLI + run: dotnet tool install --global Aspire.Cli --version 13.6.2 + + - name: Test + run: dotnet test --project tests/PythonScriptRunner.Containers.Tests + env: + PSR_TEST_WORKER_IMAGE: psr-worker:ci + PSR_TEST_DASHBOARD_IMAGE: psr-dashboard:ci + + # Spec 009, AC-1/AC-2: the Aspire AppHost brings everything up and the author runs CS-1 and CS-2. + apphost: + name: Aspire AppHost + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + + - uses: astral-sh/setup-uv@v10.3.0 + + # The panel runs as a Vite app inside the AppHost. + - uses: actions/setup-node@v7 + with: + node-version-file: panel/.nvmrc + cache: npm + cache-dependency-path: panel/package-lock.json + + # weasyprint (CS-2) renders with pango. + - name: Install pango + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz0b fonts-dejavu-core + + - name: Test + run: dotnet test --project tests/PythonScriptRunner.AppHost.Tests + panel: name: Panel (Vue) runs-on: ubuntu-latest diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml new file mode 100644 index 0000000..3d8f2d4 --- /dev/null +++ b/.github/workflows/release-python.yml @@ -0,0 +1,45 @@ +name: Release (psrunner) + +# A tag python/vX.Y.Z publishes the psrunner SDK to PyPI with trusted publishing (OIDC, no stored token; spec 009, +# AC-6). The SDK has its own version, compatible with protocol 1 while it is 1.x; the CI must be green on the commit. +on: + push: + tags: ['python/v*'] + +permissions: + contents: read + +jobs: + ci: + name: CI + uses: ./.github/workflows/ci.yml + + publish: + name: PyPI + needs: ci + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/psrunner/ + permissions: + id-token: write + defaults: + run: + working-directory: python/psrunner + steps: + - uses: actions/checkout@v7 + + - uses: astral-sh/setup-uv@v10.3.0 + + - name: Check the version + run: | + expected="${GITHUB_REF_NAME#python/v}" + actual=$(uv run --no-project python -c "exec(open('src/psrunner/_version.py').read()); print(__version__)") + test "$actual" = "$expected" || { echo "src/psrunner/_version.py has $actual, the tag says $expected"; exit 1; } + + - name: Build + run: uv build --out-dir dist + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: python/psrunner/dist diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b264187 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,166 @@ +name: Release + +# A tag vX.Y.Z publishes the 7 NuGet packages, the two container images and the GitHub Release (spec 009, AC-5). +# Nothing is published unless the CI is green on the tagged commit, and the `release` environment asks a maintainer +# to approve the publication. +on: + push: + tags: ['v*'] + +permissions: + contents: read + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +env: + DOTNET_NOLOGO: true + DOTNET_CLI_TELEMETRY_OPTOUT: true + +jobs: + ci: + name: CI + uses: ./.github/workflows/ci.yml + + packages: + name: NuGet packages + needs: ci + runs-on: ubuntu-latest + environment: release + permissions: + contents: read + id-token: write # NuGet trusted publishing (D-170) + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + + # Packing in Release embeds the Vue panel in PythonScriptRunner.Dashboard (D-155). + - uses: actions/setup-node@v7 + with: + node-version-file: panel/.nvmrc + cache: npm + cache-dependency-path: panel/package-lock.json + + - name: Pack + run: dotnet pack PythonScriptRunner.slnx --configuration Release --output artifacts/packages + + - name: Check the version + run: | + expected="${GITHUB_REF_NAME#v}" + for package in artifacts/packages/*.nupkg; do + case "$package" in + *".$expected.nupkg") ;; + *) echo "$package does not have version $expected"; exit 1 ;; + esac + done + test "$(ls artifacts/packages/*.nupkg | wc -l)" -eq 7 + + - name: Log in to NuGet + id: nuget + uses: NuGet/login@v1 + with: + user: ${{ vars.NUGET_USER }} + + # The symbol packages (.snupkg) next to each .nupkg are pushed with it. + - name: Push + run: >- + dotnet nuget push "artifacts/packages/*.nupkg" + --api-key "${{ steps.nuget.outputs.NUGET_API_KEY }}" + --source https://api.nuget.org/v3/index.json + --skip-duplicate + + - uses: actions/upload-artifact@v7 + with: + name: packages + path: artifacts/packages/ + + images: + name: Container images (${{ matrix.image }}) + needs: ci + runs-on: ubuntu-latest + environment: release + permissions: + contents: read + packages: write + strategy: + matrix: + include: + - image: psr-worker + dockerfile: hosts/PythonScriptRunner.WorkerHost/Dockerfile + - image: psr-dashboard + dockerfile: hosts/PythonScriptRunner.DashboardHost/Dockerfile + steps: + - uses: actions/checkout@v7 + + # The build stages compile on the runner's platform; only the final stage runs for arm64 (emulated). + - uses: docker/setup-qemu-action@v4 + + - uses: docker/setup-buildx-action@v4 + + - uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: meta + uses: docker/metadata-action@v6 + with: + images: ghcr.io/scriptrunnerkit/${{ matrix.image }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}},enable=${{ !contains(github.ref_name, '-') }} + type=raw,value=latest,enable=${{ !contains(github.ref_name, '-') }} + + - uses: docker/build-push-action@v7 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: VERSION=${{ steps.meta.outputs.version }} + sbom: true + provenance: mode=max + cache-from: type=gha,scope=release-${{ matrix.image }} + cache-to: type=gha,scope=release-${{ matrix.image }},mode=max + + github-release: + name: GitHub Release + needs: [packages, images] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + + - uses: actions/download-artifact@v8 + with: + name: packages + path: artifacts/packages + + # The notes are the CHANGELOG section of the version, which also carries the compatibility matrix (D-178). + - name: Release notes + run: | + version="${GITHUB_REF_NAME#v}" + awk -v v="$version" ' + index($0, "## [" v "]") == 1 { found = 1; next } + found && index($0, "## [") == 1 { exit } + found { print } + ' CHANGELOG.md > notes.md + test -s notes.md || { echo "CHANGELOG.md has no section for $version"; exit 1; } + + - name: Create the release + env: + GH_TOKEN: ${{ github.token }} + run: | + prerelease=$([[ "$GITHUB_REF_NAME" == *-* ]] && echo --prerelease || true) + gh release create "$GITHUB_REF_NAME" artifacts/packages/*.nupkg \ + --title "PythonScriptRunner ${GITHUB_REF_NAME#v}" --notes-file notes.md --verify-tag $prerelease diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..436a04f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,61 @@ +# Changelog + +All notable changes to PythonScriptRunner are documented in this file. The format is based on +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project follows +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). The NuGet packages and the container images share one +version; the `psrunner` SDK has its own version on PyPI. + +## [0.1.0] - Unreleased + +The first release. + +### Added + +- **Core runner** (`PythonScriptRunner.Core`, `PythonScriptRunner.Abstractions`): + - stages with typed inputs and outputs (12 field types, including `table`, `file` and `list`), defined in + `stage.json` or in C#; + - each attempt runs in an isolated process: a clean environment, its own work folder, a timeout and a memory limit + (Job Object on Windows; cgroup v2 when it can be delegated on Linux, otherwise `RLIMIT_AS` with + `ThreadsPerProcess`); + - secrets reach only the stages that declare them, and are masked in the captured output; + - Python environments created and cached by uv, one per set of dependencies. +- **Pipelines:** + - steps bound to the pipeline's inputs, to earlier steps' outputs or to constants; + - types and bindings validated when the catalog loads; + - defined in `*.pipeline.json` or in C#. +- **`psrunner` 0.1.0**, the Python SDK: + - `@stage`, `Context`, `File` and `StageError`; + - `psrunner.testing.run_stage`, to test stages with pytest without .NET. +- **Persistence** (`PythonScriptRunner.EntityFrameworkCore`, PostgreSQL 16+): + - a versioned catalog (draft, published, archived), with dependencies locked by hash on publish and zip import and + export; + - the full history of runs, attempts, logs and files; + - idempotency keys, an audit log and partitioned retention. +- **Queue** (`PythonScriptRunner.RabbitMQ`, RabbitMQ 3.13+): + - queued runs through a transactional outbox; + - the public `StartPipelineRun` command, `run.*` and `step.*` events, and subscriptions for event handlers. +- **Worker** (`PythonScriptRunner.Worker`): + - leases, retries with exponential backoff, recovery of lost workers and cancellation; + - concurrency limits per worker and per pipeline; + - cron schedules with IANA time zones and catch-up runs. +- **Dashboard** (`PythonScriptRunner.Dashboard`): + - an embeddable dashboard with an internal API and real-time updates (SignalR); + - a Vue UI to edit, test and publish stages and pipelines, follow runs live, and manage schedules, the audit log and + administration; + - four cumulative roles, and access by fixed login, a custom filter, host authentication or local requests. +- **`psr` tool** (`PythonScriptRunner.Tool`): `psr hash-password` generates the password hashes for fixed login. +- **Hosts and images:** + - `WorkerHost` and `DashboardHost`, configured by environment variables, with health checks, JSON logs, OpenTelemetry + and optional migrations at startup (`PSR_AUTO_MIGRATE`). The DashboardHost also supports OpenID Connect; + - multi-arch images `ghcr.io/scriptrunnerkit/psr-worker` and `ghcr.io/scriptrunnerkit/psr-dashboard`: non-root, + read-only root file system, with an SBOM and provenance. +- **Local environment:** a .NET Aspire AppHost with PostgreSQL, RabbitMQ, Mailpit, both hosts and the UI in development + mode. `aspire publish` generates a Docker Compose file from it. + +### Compatibility + +| PythonScriptRunner (NuGet and images) | .NET | Protocol | `psrunner` | Python | +|---------------------------------------|------|----------|------------|--------| +| 0.1.0 | 10 | 1 | 0.1.x | 3.10–3.13 | + +[0.1.0]: https://github.com/ScriptRunnerKit/PythonScriptRunner/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4d4843c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,83 @@ +# Contributing to PythonScriptRunner + +Thanks for helping! The [ScriptRunnerKit contributing guide](https://github.com/ScriptRunnerKit/.github/blob/main/CONTRIBUTING.md) +and the [Code of Conduct](https://github.com/ScriptRunnerKit/.github/blob/main/CODE_OF_CONDUCT.md) apply here. This file +adds what is specific to this repository: the tools, the checks the CI runs and how releases work. + +Before anything bigger than a small fix, open an issue to agree on the behavior and its acceptance criteria. Report +security problems privately, as described in [SECURITY.md](SECURITY.md), never in a public issue. + +## Prerequisites + +- .NET 10 SDK (the version in `global.json`). +- [uv](https://docs.astral.sh/uv/): the integration tests create real Python environments. +- Node.js 24 (`panel/.nvmrc`): the dashboard UI, and Release builds that embed it. +- Docker: the tests for PostgreSQL, RabbitMQ, the worker, the dashboard, the containers and the Aspire AppHost use + Testcontainers or Aspire. + +## Build and test + +```bash +dotnet build +dotnet test --solution tests/PythonScriptRunner.SemE2E.slnf # unit, integration and dashboard tests +dotnet test --project tests/PythonScriptRunner.Containers.Tests # host images and compose +dotnet test --project tests/PythonScriptRunner.AppHost.Tests # Aspire AppHost +dotnet build tests/PythonScriptRunner.Panel.E2E.Tests -p:PsrBuildPanel=true +dotnet test --project tests/PythonScriptRunner.Panel.E2E.Tests --no-build # run it on its own + +cd python/psrunner && uv sync && uv run pytest +cd panel && npm ci && npm test +``` + +The [README](README.md#development) explains each test project, the environment variables the tests read, and how to run +the local environment with .NET Aspire. + +## What the CI checks + +A pull request is merged only when every job of `.github/workflows/ci.yml` is green. Run the same checks locally: + +| Area | Checks | +|------|--------| +| .NET | builds with warnings as errors, the latest analyzers and code style enforced in the build; `dotnet format --verify-no-changes`; tests on Linux, and the Core tests on Windows | +| Packages | `dotnet pack` produces the 7 packages with one version and their symbol packages | +| Containers | both Dockerfiles build; the images run read-only as a non-root user; the memory limit and the compose from `aspire publish` work | +| Python SDK | `uv run ruff check src tests`, `uv run ruff format --check src tests`, `uv run mypy` (strict) and `uv run pytest`, on Python 3.10 to 3.13 | +| Panel | `npm run lint`, `npm run typecheck`, `npm run api:check`, `npm test` and `npm run build` (initial bundle ≤ 500 KB gzip) | +| Dependencies | no known vulnerable NuGet packages, no high or critical npm advisories in runtime packages, `pip-audit` clean | + +## Conventions + +- **Tests:** every acceptance criterion and every fixed bug has a test. Integration tests run against real Python (uv), + PostgreSQL and RabbitMQ (Testcontainers). +- **Public API:** public types and members have XML documentation (the build generates it and fails on warnings). +- **Packages:** versions live in `Directory.Packages.props` (Central Package Management). Dependabot proposes updates + every Monday. +- **Database:** a change to the model needs a migration. The design-time factory opens no connection: + + ```bash + dotnet ef migrations add --project src/PythonScriptRunner.EntityFrameworkCore + ``` + +- **Dashboard API:** the OpenAPI document is a snapshot. After an intended change, accept it with + `PSR_ACCEPT_SNAPSHOTS=1` and regenerate the UI types with `npm run api` in `panel/`. +- **Contracts:** the JSON Schemas in `schemas/` and the protocol shared with `psrunner` are public contracts. Changing + them in an incompatible way needs a new schema or protocol version. +- **Changelog:** user-visible changes get a line in [CHANGELOG.md](CHANGELOG.md), under the version being prepared. +- **Commits:** [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `docs:`, `test:`, + `build:`, `ci:`, `chore:`). + +## Releases + +Maintainers release from tags on `main`; each tag runs the full CI before anything is published. + +- **.NET packages and images:** + 1. Date the version's section in `CHANGELOG.md`. + 2. Push the tag `vX.Y.Z`. + 3. `release.yml` waits for approval in the `release` environment. It then publishes the 7 NuGet packages, the two + multi-arch images to GHCR and the GitHub Release, with the CHANGELOG section as its notes. + + MinVer takes the version from the tag. A tag with a suffix (`v0.2.0-rc.1`) is published as a prerelease. +- **psrunner:** + 1. Set the version in `python/psrunner/src/psrunner/_version.py`. + 2. Push the tag `python/vX.Y.Z`. + 3. `release-python.yml` publishes it to PyPI with trusted publishing. diff --git a/Directory.Build.props b/Directory.Build.props index 015da48..35fa55a 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -11,13 +11,22 @@ true + + + v + 0.1 + + + + + + - 0.1.0 - net0well - Copyright (c) 2026 net0well + ScriptRunnerKit + Copyright (c) 2026 ScriptRunnerKit contributors MIT - https://github.com/net0well/PythonScriptRunner - https://github.com/net0well/PythonScriptRunner + https://github.com/ScriptRunnerKit/PythonScriptRunner + https://github.com/ScriptRunnerKit/PythonScriptRunner git true true diff --git a/Directory.Packages.props b/Directory.Packages.props index e592cf5..38cf71e 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,6 +3,11 @@ true + + + + + @@ -27,6 +32,7 @@ + diff --git a/LICENSE b/LICENSE index 45d5eff..dd32e48 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2026 net0well +Copyright (c) 2026 ScriptRunnerKit contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/PythonScriptRunner.slnx b/PythonScriptRunner.slnx index 9b7c6a2..84b777c 100644 --- a/PythonScriptRunner.slnx +++ b/PythonScriptRunner.slnx @@ -1,5 +1,6 @@ + @@ -14,6 +15,8 @@ + + diff --git a/README.md b/README.md index a596653..55a8c33 100644 --- a/README.md +++ b/README.md @@ -1,101 +1,106 @@ # PythonScriptRunner -Biblioteca .NET 10 para executar Python de forma tipada, encadeável e observável: etapas (**Stages**) -com contrato de entrada/saída, trilhas (**Pipelines**), execução assíncrona com RabbitMQ, agendamento -cron e painel Vue opcional. +A .NET 10 library for running Python in a typed, composable and observable way. **Stages** declare an input/output +contract, **pipelines** chain them, and RabbitMQ runs them asynchronously. Cron scheduling and an optional Vue +dashboard are included. -> Projeto em desenvolvimento. Esta versão entrega o **Core Runner** (execução síncrona de Stages e de -> trilhas), o SDK Python **`psrunner`**, a **persistência em PostgreSQL** (catálogo versionado e histórico), a -> **execução assíncrona** com RabbitMQ, worker e agendamento cron e o **painel embutível** (interface Vue, API, tempo -> real e acesso configurável). +> The project is under active development, heading for its first release (0.1.0). This version ships the **Core Runner** +> (synchronous stages and pipelines), the **`psrunner`** Python SDK, **PostgreSQL persistence** (versioned catalog and +> history), **asynchronous execution** with RabbitMQ, a worker and cron scheduling, the **embeddable dashboard** (Vue UI, +> API, real time and configurable access), ready-made **hosts and container images** and a local environment on +> **.NET Aspire**. -## Como funciona +## How it works -Cada Stage declara suas entradas e saídas (12 tipos: `number`, `integer`, `decimal`, `string`, `bool`, -`date`, `datetime`, `enum`, `json`, `table`, `file`, `list`) e tem exatamente um script Python. -O runner valida as entradas, executa o script num processo isolado (ambiente limpo, timeout, limite -de memória, segredos injetados só nele), valida as saídas e devolve o resultado tipado. +Each stage declares its inputs and outputs and has exactly one Python script. There are 12 field types: `number`, +`integer`, `decimal`, `string`, `bool`, `date`, `datetime`, `enum`, `json`, `table`, `file` and `list`. The runner +validates the inputs and runs the script in an isolated process: a clean environment, a timeout, a memory limit, and +secrets injected into that process only. It then validates the outputs and returns the typed result. ```python -# stages/progressao-termos/script.py +# stages/progression-terms/script.py from psrunner import stage @stage def run(inputs, ctx): - a1, razao, n = inputs["a1"], inputs["razao"], inputs["n"] - termos = [a1 * razao**i for i in range(n)] - ctx.log.info("gerados %d termos", n) - return {"termos": termos} + a1, ratio, n = inputs["a1"], inputs["ratio"], inputs["n"] + terms = [a1 * ratio**i for i in range(n)] + ctx.log.info("generated %d terms", n) + return {"terms": terms} ``` ```jsonc -// stages/progressao-termos/stage.json +// stages/progression-terms/stage.json { - "$schema": "https://raw.githubusercontent.com/net0well/PythonScriptRunner/main/schemas/stage.v1.json", - "key": "progressao-termos", + "$schema": "https://raw.githubusercontent.com/ScriptRunnerKit/PythonScriptRunner/main/schemas/stage.v1.json", + "key": "progression-terms", "version": 1, - "name": "Termos de progressão", + "name": "Progression terms", "inputs": [ { "name": "a1", "type": "number" }, - { "name": "razao", "type": "number" }, + { "name": "ratio", "type": "number" }, { "name": "n", "type": "integer" } ], - "outputs": [ { "name": "termos", "type": "list", "itemType": "number" } ] + "outputs": [ { "name": "terms", "type": "list", "itemType": "number" } ] } ``` ```csharp builder.Services .AddPythonScriptRunner() - .AddFileCatalog("./psr"); // pasta com stages//stage.json + .AddFileCatalog("./psr"); // folder with stages//stage.json // ... -await using var result = await runner.RunStageAsync("progressao-termos", new { a1 = 1, razao = 2, n = 4 }); +await using var result = await runner.RunStageAsync("progression-terms", new { a1 = 1, ratio = 2, n = 4 }); result.EnsureSuccess(); -double[] termos = result.GetOutput("termos"); // [1, 2, 4, 8] +double[] terms = result.GetOutput("terms"); // [1, 2, 4, 8] ``` -Stages também podem ser definidos em C# (`AddStage("chave", s => s.Script(...).Input(...).Output(...))`) -e testados localmente com pytest pelo `psrunner.testing.run_stage`, sem .NET. +Stages can also be defined in C# (`AddStage("key", s => s.Script(...).Input(...).Output(...))`) and tested locally with +pytest through `psrunner.testing.run_stage`, without .NET. -## Trilhas (pipelines) +## Pipelines -Uma trilha encadeia Stages em ordem. Cada entrada de um passo recebe a entrada da trilha -(`$pipeline.campo`), a saída de qualquer passo anterior (`$steps.passo.campo`) ou um valor fixo -(`{"const": ...}`). Sem ligação explícita, vale o campo de mesmo nome do passo anterior. A trilha -declara as próprias saídas. Tipos e ligações são validados ao carregar o catálogo. +A pipeline chains stages in order. Each input of a step takes one of these: + +- an input of the pipeline (`$pipeline.field`); +- an output of any earlier step (`$steps.step.field`); +- a fixed value (`{"const": ...}`). + +Without an explicit binding, the field with the same name from the previous step is used. The pipeline declares its own +outputs. Types and bindings are validated when the catalog loads. ```jsonc -// pipelines/calculo-progressao.pipeline.json +// pipelines/progression-calc.pipeline.json { - "key": "calculo-progressao", "version": 1, "name": "Cálculo de progressão", - "inputs": [ { "name": "a1", "type": "number" }, { "name": "razao", "type": "number" }, - { "name": "n", "type": "integer" }, { "name": "coeficientes", "type": "list", "itemType": "number" } ], + "key": "progression-calc", "version": 1, "name": "Progression calculation", + "inputs": [ { "name": "a1", "type": "number" }, { "name": "ratio", "type": "number" }, + { "name": "n", "type": "integer" }, { "name": "coefficients", "type": "list", "itemType": "number" } ], "steps": [ - { "key": "termos", "stage": "progressao-termos@1" }, - { "key": "polinomio", "stage": "polinomio-avaliar@1", - "bindings": { "coeficientes": "$pipeline.coeficientes", "x": "$steps.termos.termos" } }, - { "key": "estatisticas", "stage": "estatisticas@1", "bindings": { "valores": "$steps.polinomio.valores" } } + { "key": "terms", "stage": "progression-terms@1" }, + { "key": "polynomial", "stage": "polynomial-eval@1", + "bindings": { "coefficients": "$pipeline.coefficients", "x": "$steps.terms.terms" } }, + { "key": "statistics", "stage": "statistics@1", "bindings": { "values": "$steps.polynomial.values" } } ], - "outputs": [ { "name": "soma", "source": "$steps.estatisticas.soma" } ] + "outputs": [ { "name": "sum", "source": "$steps.statistics.sum" } ] } ``` ```csharp -await using var result = await runner.RunPipelineAsync("calculo-progressao", - new { a1 = 1, razao = 2, n = 4, coeficientes = new[] { 1.0, 0.0, 1.0 } }); -double soma = result.GetOutput("soma"); -double[] termos = result.GetStepOutput("termos", "termos"); // depuração +await using var result = await runner.RunPipelineAsync("progression-calc", + new { a1 = 1, ratio = 2, n = 4, coefficients = new[] { 1.0, 0.0, 1.0 } }); +double sum = result.GetOutput("sum"); +double[] terms = result.GetStepOutput("terms", "terms"); // debugging ``` -A mesma trilha pode ser definida em C# com `AddPipeline("chave", p => p.Input(...).Step(...).Output(...))`. -Se um passo falhar, os seguintes ficam como `Skipped` e a execução termina `Failed`. +The same pipeline can be defined in C# with `AddPipeline("key", p => p.Input(...).Step(...).Output(...))`. If a step +fails, the following steps end as `Skipped` and the run ends as `Failed`. -## Persistência (PostgreSQL) +## Persistence (PostgreSQL) -Com o pacote `PythonScriptRunner.EntityFrameworkCore`, Stages e trilhas passam a viver no banco (schema `psr`) com -versões imutáveis (rascunho → publicado → arquivado), e **toda** execução fica registrada: passos, tentativas, cauda -de stdout/stderr e arquivos (em `bytea`, lidos e gravados em streaming). +With the `PythonScriptRunner.EntityFrameworkCore` package, stages and pipelines live in the database (schema `psr`). +Their versions are immutable and move from draft to published to archived. **Every** run is recorded: steps, attempts, +the tail of stdout/stderr and files (stored in `bytea`, read and written as streams). ```csharp builder.Services @@ -103,89 +108,117 @@ builder.Services .AddEntityFrameworkCore(o => { o.UseNpgsql(builder.Configuration.GetConnectionString("psr")!); // PostgreSQL 16+ - o.Retention.Artifacts = TimeSpan.FromDays(7); // arquivos - o.Retention.Runs = TimeSpan.FromDays(90); // histórico + o.Retention.Artifacts = TimeSpan.FromDays(7); // files + o.Retention.Runs = TimeSpan.FromDays(90); // history }); builder.Services.AddHealthChecks().AddPythonScriptRunnerDatabase(); var app = builder.Build(); -await app.Services.MigratePythonScriptRunnerAsync(); // explícito; ou GeneratePythonScriptRunnerMigrationScriptAsync() +await app.Services.MigratePythonScriptRunnerAsync(); // explicit; or GeneratePythonScriptRunnerMigrationScriptAsync() ``` -- **Catálogo:** `ICatalogAdmin` cria rascunhos (concorrência otimista), publica (verifica a sintaxe com o Python - declarado e trava as dependências com `uv pip compile --generate-hashes`), arquiva, exclui, testa rascunhos e - importa/exporta zip no formato de pasta. -- **Execuções:** `IRunManager` consulta (paginação keyset), cancela e reprocessa uma execução que falhou — os passos - que deram certo reaproveitam as saídas e o que falhou ganha uma nova tentativa. `RunOptions.IdempotencyKey` devolve - a mesma execução por 24 h. -- **Retenção:** o histórico é particionado (semana para execuções, dia para conteúdo de arquivos, mês para auditoria); - um job de manutenção cria partições à frente e remove as antigas. Uma trilha pode encurtar a retenção. -- **Auditoria:** ações humanas (`Actor.User(...)`) ficam em `psr.audit_events`; disparos automáticos ficam na própria execução. +- **Catalog:** `ICatalogAdmin` handles the whole life of a version: + - creates drafts, with optimistic concurrency; + - publishes them, after checking the syntax with the declared Python and locking the dependencies with + `uv pip compile --generate-hashes`; + - archives, deletes and tests drafts; + - imports and exports zip files in the folder format. +- **Runs:** `IRunManager` queries runs (keyset pagination), cancels them and retries a failed run. Retried steps that + succeeded reuse their outputs; the step that failed gets a new attempt. `RunOptions.IdempotencyKey` returns the same + run for 24 h. +- **Retention:** the history is partitioned (by week for runs, by day for file contents, by month for the audit log). + A maintenance job creates partitions ahead of time and drops old ones. A pipeline can shorten its retention. +- **Audit:** human actions (`Actor.User(...)`) go to `psr.audit_events`; automatic triggers are recorded on the run + itself. + +## Queue, worker and scheduling (RabbitMQ) -## Fila, worker e agendamento (RabbitMQ) +With `PythonScriptRunner.RabbitMQ` and `PythonScriptRunner.Worker`, runs can be queued. Each step is a message: -Com `PythonScriptRunner.RabbitMQ` e `PythonScriptRunner.Worker`, execuções podem ser enfileiradas. Cada passo é uma -mensagem: o worker assume o passo com um *lease*, executa o processo Python e grava o resultado e a mensagem do próximo -passo na mesma transação (outbox no PostgreSQL). Enfileirar funciona mesmo com o broker fora do ar. +1. A worker claims the step with a *lease* and runs the Python process. +2. It writes the result, and the message for the next step, in the same transaction (an outbox in PostgreSQL). + +Enqueuing works even while the broker is down. ```csharp builder.Services .AddPythonScriptRunner() .AddEntityFrameworkCore(o => o.UseNpgsql(builder.Configuration.GetConnectionString("psr")!)) .AddRabbitMq(o => o.ConnectionString = builder.Configuration.GetConnectionString("rabbitmq")) // RabbitMQ 3.13+ - .AddWorker(o => o.MaxConcurrency = 8) // só nos processos que executam passos - .AddRunEventHandler(s => s.Subscription("erp").On("run.succeeded.calculo-progressao", "run.failed.*")); -builder.Services.AddHealthChecks().AddPythonScriptRunnerChecks(); // banco, broker e uv + .AddWorker(o => o.MaxConcurrency = 8) // only in the processes that run steps + .AddRunEventHandler(s => s.Subscription("erp").On("run.succeeded.progression-calc", "run.failed.*")); +builder.Services.AddHealthChecks().AddPythonScriptRunnerChecks(); // database, broker and uv // ... -RunHandle run = await queue.EnqueuePipelineAsync("calculo-progressao", new { a1 = 1, razao = 2, n = 4, coeficientes = new[] { 1, 0, 1 } }); +RunHandle run = await queue.EnqueuePipelineAsync("progression-calc", new { a1 = 1, ratio = 2, n = 4, coefficients = new[] { 1, 0, 1 } }); ``` -- **Confiabilidade:** retry automático por Stage com atraso exponencial; worker perdido é detectado pelo *lease* e o - passo é retomado (Stage idempotente ou com retry restante) ou falha com `WorkerLost` para reprocesso manual; - mensagens duplicadas não executam o passo duas vezes. -- **Concorrência:** `MaxConcurrency` por worker e `max_concurrent_runs` por trilha (as demais esperam como `Pending`, em ordem). -- **Cancelamento:** `IRunManager.CancelAsync` encerra a árvore de processos no worker que executa o passo. -- **Agendamento:** `ICatalogAdmin.CreateScheduleAsync` com expressão cron e fuso IANA (padrão `America/Sao_Paulo`); - ocorrências perdidas viram uma execução de recuperação; vários workers nunca disparam a mesma ocorrência. -- **Sistemas externos:** o comando público `StartPipelineRun` (exchange `psr.public`) inicia trilhas e os eventos - `run.*`/`step.*` (exchange `psr.events`) informam o andamento e as saídas. Contratos em `schemas/messages`. -- **Observabilidade:** `ActivitySource` e `Meter` `PythonScriptRunner`; o `traceparent` acompanha a execução até o worker. - -O host `hosts/PythonScriptRunner.WorkerHost` é um worker pronto, configurado por variáveis de ambiente -(`ConnectionStrings__psr`, `ConnectionStrings__rabbitmq`, `PythonScriptRunner__Worker__MaxConcurrency`, -`PythonScriptRunner__Secrets__`, `PSR_AUTO_MIGRATE=true`), com `/health/live`, `/health/ready`, logs JSON e OTLP. - -## Painel (Dashboard) - -Com `PythonScriptRunner.Dashboard`, o painel é embutido na sua aplicação, no estilo do Hangfire: API interna, atualizações -em tempo real (SignalR) e a interface Vue servida no prefixo escolhido. Sem `MapPythonScriptRunnerDashboard` não existe painel. - -A interface (em pt-BR, tema claro/escuro) cobre a visão geral, Etapas (lista filtrável por trilha, editor Monaco com -construtor de campos, testar pela fila e publicar), Trilhas (editor visual com *handles* tipados, ou lista editável, testar e publicar), execuções ao -vivo (fluxo por status, logs, arquivos, linha do tempo, cancelar e reprocessar), agendamentos com prévia do cron, -auditoria e administração. Ela vai dentro do pacote: não há nada para hospedar à parte. +- **Reliability:** + - automatic retry per stage, with exponential backoff; + - a lost worker is detected through its *lease*. The step resumes if the stage is idempotent or has retries left; + otherwise it fails with `WorkerLost` for a manual retry; + - duplicate messages never run a step twice. +- **Concurrency:** `MaxConcurrency` per worker and `max_concurrent_runs` per pipeline (the other runs wait as `Pending`, + in order). +- **Cancellation:** `IRunManager.CancelAsync` stops the process tree on the worker that runs the step. +- **Scheduling:** `ICatalogAdmin.CreateScheduleAsync` takes a cron expression and an IANA time zone (default + `America/Sao_Paulo`). A missed occurrence becomes one catch-up run, and several workers never fire the same occurrence. +- **External systems:** the public `StartPipelineRun` command (exchange `psr.public`) starts pipelines. The `run.*` and + `step.*` events (exchange `psr.events`) report progress and outputs. The contracts are in `schemas/messages`. +- **Observability:** `ActivitySource` and `Meter` named `PythonScriptRunner`; the `traceparent` follows the run all the + way to the worker. The relay polls the outbox every 250 ms, so the library logs its SQL commands at `Debug` + (`Microsoft.EntityFrameworkCore.Database.Command`). + +`hosts/PythonScriptRunner.WorkerHost` is a ready-made worker. It provides `/health/live`, `/health/ready`, JSON logs and +OTLP, and is configured with environment variables: + +- `ConnectionStrings__psr` and `ConnectionStrings__rabbitmq`; +- `PythonScriptRunner__Worker__MaxConcurrency`; +- `PythonScriptRunner__Secrets__`; +- `PSR_AUTO_MIGRATE=true`. + +## Dashboard + +With `PythonScriptRunner.Dashboard`, the dashboard is embedded in your application, in the style of Hangfire. It has an +internal API, real-time updates (SignalR) and the Vue UI, served under the prefix you choose. Without +`MapPythonScriptRunnerDashboard` there is no dashboard. + +The UI is in Brazilian Portuguese, with light and dark themes. It covers: + +- an overview; +- **Stages**: a list you can filter by pipeline, a Monaco editor with a field builder, test runs through the queue, and + publishing; +- **Pipelines**: a visual editor with typed *handles* or an editable list, test runs, and publishing; +- live runs: the flow by status, logs, files, a timeline, cancel and retry; +- schedules with a cron preview; +- the audit log and administration. + +The UI ships inside the package: there is nothing to host separately. ```csharp builder.Services .AddPythonScriptRunner() .AddEntityFrameworkCore(o => o.UseNpgsql(builder.Configuration.GetConnectionString("psr")!)) .AddRabbitMq(o => o.ConnectionString = builder.Configuration.GetConnectionString("rabbitmq")) - .AddDashboard(o => o.UseFixedLogin(login => login.BindConfiguration())); // seção PythonScriptRunner:Dashboard:Login + .AddDashboard(o => o.UseFixedLogin(login => login.BindConfiguration())); // section PythonScriptRunner:Dashboard:Login var app = builder.Build(); -app.MapPythonScriptRunnerDashboard("/psr"); // API em /psr/api, tempo real em /psr/hubs/runs, painel em /psr/ +app.MapPythonScriptRunnerDashboard("/psr"); // API at /psr/api, real time at /psr/hubs/runs, UI at /psr/ ``` -Escolha **um** modo de acesso; os papéis são cumulativos: `Reader` (vê tudo) → `Executor` (dispara, cancela, -reprocessa) → `Author` (edita, testa, publica, importa, agenda) → `Admin` (exclui, retenção, segredos, auditoria). +Choose **one** access mode. Roles are cumulative: + +1. `Reader` sees everything. +2. `Executor` also triggers, cancels and retries runs. +3. `Author` also edits, tests, publishes, imports and schedules. +4. `Admin` also deletes and manages retention, secrets and the audit log. -| Modo | Configuração | -|------|--------------| -| Login fixo | `o.UseFixedLogin(l => l.BindConfiguration())` com usuários na configuração (abaixo) ou `l.Users` em código | -| Filtro próprio | `o.UseAuthorizationFilter()`: `IDashboardAuthorizationFilter` devolve `DashboardUser(nome, papel)` ou `null` | -| Autenticação do host | `o.UseHostAuthentication(r => { r.Reader = p => p.RequireRole("psr-leitor"); /* Executor, Author, Admin */ r.LoginUrl = "/conta/entrar?returnUrl={returnUrl}"; })` | -| Somente local | `o.AllowLocalRequests = true` — só em Development (fora dele a inicialização falha); soma-se a um modo principal | +| Mode | Configuration | +|------|---------------| +| Fixed login | `o.UseFixedLogin(l => l.BindConfiguration())` with users in the configuration (below) or `l.Users` in code | +| Custom filter | `o.UseAuthorizationFilter()`: an `IDashboardAuthorizationFilter` returns `DashboardUser(name, role)` or `null` | +| Host authentication | `o.UseHostAuthentication(r => { r.Reader = p => p.RequireRole("psr-reader"); /* Executor, Author, Admin */ r.LoginUrl = "/account/login?returnUrl={returnUrl}"; })` | +| Local only | `o.AllowLocalRequests = true`: Development only (outside it, startup fails); it adds to a main mode | ```json "PythonScriptRunner": { @@ -200,101 +233,353 @@ reprocessa) → `Author` (edita, testa, publica, importa, agenda) → `Admin` (e } ``` -A senha nunca vai para a configuração, só o hash (PBKDF2, formato do ASP.NET Identity), gerado pela ferramenta `psr`: +The password never goes into the configuration, only its hash (PBKDF2, in the ASP.NET Identity format). The `psr` tool +generates it: ```bash dotnet tool install -g PythonScriptRunner.Tool -psr hash-password # digita a senha duas vezes, sem eco -echo "senha" | psr hash-password --stdin # em scripts +psr hash-password # type the password twice, without echo +echo "password" | psr hash-password --stdin # in scripts +``` + +How fixed login behaves: + +- **Session:** a `psr.session` cookie (HttpOnly, `SameSite=Lax`), sliding for 8 h, 24 h at most. +- **Lockout:** 5 failures in a row lock the user for 15 min; one IP can try 10 times per minute. +- **Configuration changes** apply without a restart. A removed user or a changed password ends the session; a new role + applies immediately. +- **Cookie keys** live in `psr.dashboard_keys`, so a session works on every instance and survives restarts. + +In the API, requests that change something require the `X-PSR-Request: 1` header. Edits use `ETag`/`If-Match`, and +errors follow `ProblemDetails` with a `code`. In Development, the (internal) OpenAPI document is at +`/psr/api/openapi/v1.json`. + +`hosts/PythonScriptRunner.DashboardHost` is a ready-made dashboard. It authenticates in one of two ways: + +- **Fixed login**, with users from environment variables (`PythonScriptRunner__Dashboard__Login__Users__0__Name`, + `__Role`, `__PasswordHash`). +- **OpenID Connect**, with `Oidc__Authority`, `Oidc__ClientId` and `Oidc__ClientSecret`. Roles come from the + `Dashboard__RoleClaim` claim (default `roles`), mapped with `Dashboard__Roles__Reader|Executor|Author|Admin`. + +It also accepts these variables: + +- `Dashboard__PathPrefix` (default `/psr`); +- `ASPNETCORE_FORWARDEDHEADERS_ENABLED=true` behind a proxy; +- `PSR_AUTO_MIGRATE=true`. + +## Packages + +| Package | Contents | +|---------|----------| +| `PythonScriptRunner.Abstractions` | public contracts: field types, `StageDefinition`, `RunResult`, `PsrFile`, interfaces | +| `PythonScriptRunner.Core` | the runner: catalogs (code and folder), validation, uv environments, process executor | +| `PythonScriptRunner.EntityFrameworkCore` | PostgreSQL persistence: versioned catalog, history, files, audit, retention | +| `PythonScriptRunner.RabbitMQ` | queue, outbox relay, public command and events | +| `PythonScriptRunner.Worker` | runs queued steps; cron scheduling | +| `PythonScriptRunner.Dashboard` | embeddable dashboard (API, SignalR, Vue UI) | +| `PythonScriptRunner.Tool` | the `psr` dotnet tool (`psr hash-password`) | + +```bash +dotnet add package PythonScriptRunner.Core +dotnet add package PythonScriptRunner.EntityFrameworkCore # optional: PostgreSQL +dotnet add package PythonScriptRunner.RabbitMQ # optional: queue +dotnet add package PythonScriptRunner.Worker # optional: worker and cron +dotnet add package PythonScriptRunner.Dashboard # optional: dashboard +``` + +All 7 packages share one version and ship symbols with SourceLink. The runner installs the `psrunner` SDK from PyPI in +every Python environment it creates (`PythonScriptRunner:PsrunnerRequirement`, default `psrunner==0.1.0`). To use a +private index, set `PythonScriptRunner:PackageIndexUrl` and `PythonScriptRunner:ExtraIndexUrls`. + +## Containers + +The hosts are published as multi-arch images (amd64 and arm64), with the same version as the NuGet packages, plus +`latest`: + +| Image | Host | +|-------|------| +| `ghcr.io/scriptrunnerkit/psr-worker` | `WorkerHost` | +| `ghcr.io/scriptrunnerkit/psr-dashboard` | `DashboardHost` (the Vue UI embedded) | + +Both images share these properties: + +- They are built on `mcr.microsoft.com/dotnet/aspnet:10.0` with `uv`. +- They run as the non-root user `psr` (uid 10001) and listen on port 8080, with `/health/live` and `/health/ready`. +- They include the `psrunner` wheel of the same commit: the Python environments do not download the SDK. The stage + dependencies still come from PyPI or from your index. +- They carry an SBOM and provenance attestations. + +```bash +docker run -d --read-only \ + --tmpfs /tmp/psr:uid=10001,gid=10001,exec \ + -v psr-envs:/var/lib/psr/envs -v psr-python:/var/lib/psr/python \ + -e ConnectionStrings__psr="Host=pg;Database=psr;Username=psr_app;Password=..." \ + -e ConnectionStrings__rabbitmq="amqp://psr:...@rabbit:5672" \ + ghcr.io/scriptrunnerkit/psr-worker:0.1.0 +``` + +| Variable | Host | Example | +|----------|------|---------| +| `ConnectionStrings__psr` | both | `Host=pg;Database=psr;Username=psr_app;Password=…` | +| `ConnectionStrings__rabbitmq` | both | `amqps://psr:…@rabbit:5671/psr` | +| `PSR_AUTO_MIGRATE` | both | `true` applies the migrations at startup (default `false`); concurrent hosts wait on an advisory lock | +| `PythonScriptRunner__Worker__MaxConcurrency` | worker | `8` | +| `PythonScriptRunner__Secrets__` | worker | from the platform's secret store | +| `PythonScriptRunner__ThreadsPerProcess` | both | threads for the numeric libraries (see [Memory limits](#memory-limits)) | +| `PythonScriptRunner__PackageIndexUrl` | both | private package index | +| `PythonScriptRunner__Dashboard__Login__Users__N__Name` / `__Role` / `__PasswordHash` | dashboard | fixed-login users | +| `Oidc__Authority`, `Oidc__ClientId`, `Oidc__ClientSecret` | dashboard | optional: turns on OpenID Connect | +| `Dashboard__PathPrefix` | dashboard | `/psr` | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | both | `http://otel-collector:4317` | +| `Serilog__MinimumLevel__Override__` | both | `Information` for `Microsoft.AspNetCore` brings back the request logs (default `Warning`) | + +### Volumes and read-only file system + +The images run with a read-only root file system. Python writes only to these volumes: + +| Path | Contents | If it is not persistent | +|------|----------|-------------------------| +| `/var/lib/psr/envs` | cached virtual environments (one per set of dependencies) and the uv cache | every start recreates the environments, so the first run of each stage is slower | +| `/var/lib/psr/python` | Python interpreters installed by uv | the interpreter is downloaded again | +| `/tmp/psr` | work folders of the runs and `TMPDIR` | expected: it only holds temporary files | + +A tmpfs works for any of them, but it must allow execution (`exec`), because the volumes run Python; Docker mounts tmpfs +`noexec` by default. Without access to PyPI, the environments fail with `EnvironmentSetupFailed`. In that case, point +`PythonScriptRunner__PackageIndexUrl` to a private index, or build the image with an interpreter already installed: + +```bash +docker build --build-arg PSR_PREINSTALL_PYTHON=3.12 -f hosts/PythonScriptRunner.WorkerHost/Dockerfile -t my-psr-worker . +``` + +### Memory limits + +Every Python process gets the memory limit of its stage (default 1024 MB). How it is enforced depends on the platform, +and the host logs the mode it uses at startup: + +- **Windows:** a Job Object per process. +- **Linux with a delegable cgroup v2:** each process runs in its own cgroup (`memory.max`, no swap). When the process + exceeds it, the kernel kills it and the attempt ends with `MemoryLimitExceeded` (code `OUT_OF_MEMORY`). The host needs + write access to its own cgroup: a systemd service with `Delegate=yes`, or a container running as root with a writable + cgroup file system. Log: `Memory limits of Python processes use cgroup v2 under …`. +- **Other Linux setups, including standard containers** (they mount the cgroup file system read-only): an address-space + limit (`RLIMIT_AS`). Numeric libraries (OpenBLAS, MKL, OpenMP, NumExpr, vecLib) reserve address space per thread, so in + this mode they get 1 thread per process. Log: `Memory limits of Python processes use RLIMIT_AS …`. + +`PythonScriptRunner:ThreadsPerProcess` sets the thread count in every mode. Raise it for parallel numeric work, with a +memory limit that has room for it. + +### Kubernetes + +There is no official Helm chart yet. This example of a worker `Deployment` shows the settings that matter: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: psr-worker +spec: + replicas: 2 + selector: + matchLabels: { app: psr-worker } + template: + metadata: + labels: { app: psr-worker } + spec: + # Longer than PythonScriptRunner__Worker__ShutdownGracePeriod (default 5 min), so running steps can finish. + terminationGracePeriodSeconds: 330 + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + containers: + - name: worker + image: ghcr.io/scriptrunnerkit/psr-worker:0.1.0 + env: + - name: ConnectionStrings__psr + valueFrom: { secretKeyRef: { name: psr, key: postgres } } + - name: ConnectionStrings__rabbitmq + valueFrom: { secretKeyRef: { name: psr, key: rabbitmq } } + ports: [ { containerPort: 8080 } ] + livenessProbe: { httpGet: { path: /health/live, port: 8080 } } + readinessProbe: { httpGet: { path: /health/ready, port: 8080 } } + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + resources: + limits: { memory: 4Gi } # room for MaxConcurrency processes at their stage limit + volumeMounts: + - { name: envs, mountPath: /var/lib/psr/envs } + - { name: python, mountPath: /var/lib/psr/python } + - { name: work, mountPath: /tmp/psr } + volumes: + - { name: envs, emptyDir: {} } # a PersistentVolumeClaim keeps the environments across restarts + - { name: python, emptyDir: {} } + - { name: work, emptyDir: {} } ``` -No login fixo a sessão é um cookie `psr.session` (HttpOnly, `SameSite=Lax`) de 8 h deslizantes e no máximo 24 h; 5 -falhas seguidas bloqueiam o usuário por 15 min e um IP pode tentar 10 vezes por minuto. Alterações na configuração valem -sem reiniciar (usuário removido ou senha trocada perde a sessão; papel novo vale na hora). As chaves do cookie ficam em -`psr.dashboard_keys`, então a sessão vale em todas as instâncias e sobrevive a reinícios. - -Na API, requisições que alteram algo exigem o cabeçalho `X-PSR-Request: 1`, edições usam `ETag`/`If-Match` e erros -seguem `ProblemDetails` com `code`. Em Development, o documento OpenAPI (interno) fica em `/psr/api/openapi/v1.json`. - -O host `hosts/PythonScriptRunner.DashboardHost` é um painel pronto: login fixo por variáveis de ambiente -(`PythonScriptRunner__Dashboard__Login__Users__0__Name`, `__Role`, `__PasswordHash`) ou OpenID Connect com -`Oidc__Authority`, `Oidc__ClientId` e `Oidc__ClientSecret` (papéis pela claim `Dashboard__RoleClaim`, padrão `roles`, e -`Dashboard__Roles__Reader|Executor|Author|Admin`). Também aceita `Dashboard__PathPrefix` (padrão `/psr`), -`ASPNETCORE_FORWARDEDHEADERS_ENABLED=true` atrás de proxy e `PSR_AUTO_MIGRATE=true`. - -## Requisitos - -- .NET 10 SDK -- [uv](https://docs.astral.sh/uv/) no PATH (cria e cacheia um ambiente Python por conjunto de dependências) -- PostgreSQL 16+ apenas para o pacote de persistência e RabbitMQ 3.13+ para a fila (os testes usam Docker/Testcontainers) -- Node.js 24 só para compilar em Release ou empacotar o `PythonScriptRunner.Dashboard` (o build embute o painel de `panel/`) - -## Estrutura - -| Pasta | Conteúdo | -|-------|----------| -| `src/PythonScriptRunner.Abstractions` | contratos públicos (tipos de campo, `StageDefinition`, `RunResult`, `PsrFile`, interfaces) | -| `src/PythonScriptRunner.Core` | catálogos (código e pasta), validação, ambientes uv, executor de processos, runner | -| `src/PythonScriptRunner.EntityFrameworkCore` | PostgreSQL: catálogo versionado, histórico, arquivos, idempotência, outbox, auditoria, retenção, agendamentos e ciclo das execuções enfileiradas | -| `src/PythonScriptRunner.RabbitMQ` | topologia, relay do outbox, `IRunQueue`, comando público, eventos e assinaturas | -| `src/PythonScriptRunner.Worker` | execução dos passos enfileirados, *lease*, retry, cancelamento, cron | -| `src/PythonScriptRunner.Dashboard` | painel embutível: API interna, hub SignalR, modos de acesso (login fixo, filtro, host, local) | -| `panel` | interface do painel: Vue 3, Vuetify 4, Vue Flow e Monaco (TypeScript, Vite, Vitest) | -| `src/PythonScriptRunner.Tool` | dotnet tool `psr` (`psr hash-password`) | -| `hosts` | `WorkerHost`, `DashboardHost` e `ServiceDefaults` (Serilog, OpenTelemetry, health checks) | -| `python/psrunner` | SDK Python (`@stage`, `Context`, `File`, `StageError`, `psrunner.testing`, CLI) | -| `schemas` | JSON Schemas do `stage.json`, do `*.pipeline.json` e das mensagens públicas | -| `tests` | testes .NET (unitários, de integração com Python real e E2E do painel com Playwright) e fixtures de trilha | - -## Desenvolvimento +If `terminationGracePeriodSeconds` is shorter than `ShutdownGracePeriod`, the kubelet kills attempts that are still +running. They then follow the lease rules: they resume on another worker or fail with `WorkerLost`. + +## Local environment (.NET Aspire) + +With Docker, the .NET 10 SDK, Node.js 24 and uv installed: + +```bash +dotnet run --project hosts/PythonScriptRunner.AppHost +``` + +The AppHost starts PostgreSQL 16, RabbitMQ 3.13 (with the management UI), Mailpit (a test SMTP server), the WorkerHost, +the DashboardHost and the Vue UI in development mode (Vite, with hot reload). The Aspire dashboard +(`http://localhost:15000`) shows the resources, logs and traces of every run. The link **Painel** on the `dashboard` +resource opens the UI. + +The DashboardHost has one development user per role, all with the password `psr-dev`: + +| User | Role | +|------|------| +| `leitor` | Reader | +| `executor` | Executor | +| `autor` | Author | +| `admin` | Admin | + +The sample pipelines are in `tests/fixtures`. To import one: + +1. Zip the contents of its folder, with `stages/` and `pipelines/` at the root of the zip. +2. Import it in the UI as `autor`. +3. Publish the stages, then the pipeline. + +`html-para-email` sends its e-mail to Mailpit. + +## Docker Compose + +`aspire publish` turns the AppHost into a `docker-compose.yaml` that uses the published images: + +```bash +dotnet tool install --global Aspire.Cli --version 13.6.2 +aspire publish --apphost hosts/PythonScriptRunner.AppHost/PythonScriptRunner.AppHost.csproj -o deploy +``` + +Fill in `deploy/.env`, then start it with `docker compose up -d` in `deploy`: + +| Variable | Meaning | +|----------|---------| +| `POSTGRES_PASSWORD` | password of the `postgres` user | +| `RABBITMQ_PASSWORD` | password of the RabbitMQ `guest` user | +| `PSR_VERSION` | image version (default `latest`) | +| `PSR_DASHBOARD_PORT` | port of the dashboard on the host (default `8080`); the UI is at `/psr/` | + +The compose includes an Aspire dashboard for the traces. It is a starting point, and it keeps the development setup: + +- the four users with the password `psr-dev`; +- Mailpit; +- the test SMTP secrets. + +Before you expose it, replace the `PasswordHash` values with hashes from `psr hash-password` (or configure OpenID +Connect), and remove what you do not use. + +## Compatibility + +| PythonScriptRunner (NuGet and images) | .NET | Protocol | `psrunner` | Python | +|---------------------------------------|------|----------|------------|--------| +| 0.1.x | 10 | 1 | 0.1.x | 3.10–3.13 | + +The `psrunner` SDK has its own version on PyPI. It stays compatible with protocol 1 for as long as protocol 1 is in use. +The infrastructure needs PostgreSQL 16+ and RabbitMQ 3.13+. + +## Requirements + +- .NET 10 SDK. +- [uv](https://docs.astral.sh/uv/) on the PATH. It creates and caches one Python environment per set of dependencies. +- Docker, only for the tests and the local environment: the tests use Testcontainers. +- PostgreSQL 16+, only for the persistence package. +- RabbitMQ 3.13+, only for the queue. +- Node.js 24, only to build in Release or to pack `PythonScriptRunner.Dashboard`: the build embeds the UI from `panel/`. + +## Repository layout + +| Folder | Contents | +|--------|----------| +| `src/PythonScriptRunner.Abstractions` | public contracts (field types, `StageDefinition`, `RunResult`, `PsrFile`, interfaces) | +| `src/PythonScriptRunner.Core` | catalogs (code and folder), validation, uv environments, process executor, runner | +| `src/PythonScriptRunner.EntityFrameworkCore` | PostgreSQL: versioned catalog, history, files, idempotency, outbox, audit, retention, schedules and the lifecycle of queued runs | +| `src/PythonScriptRunner.RabbitMQ` | topology, outbox relay, `IRunQueue`, public command, events and subscriptions | +| `src/PythonScriptRunner.Worker` | runs queued steps: *lease*, retry, cancellation, cron | +| `src/PythonScriptRunner.Dashboard` | embeddable dashboard: internal API, SignalR hub, access modes (fixed login, filter, host, local) | +| `src/PythonScriptRunner.Tool` | the `psr` dotnet tool (`psr hash-password`) | +| `panel` | the dashboard UI: Vue 3, Vuetify 4, Vue Flow and Monaco (TypeScript, Vite, Vitest) | +| `hosts` | `WorkerHost` and `DashboardHost` (with their Dockerfiles), `ServiceDefaults` (Serilog, OpenTelemetry, health checks) and the Aspire `AppHost` | +| `python/psrunner` | the Python SDK (`@stage`, `Context`, `File`, `StageError`, `psrunner.testing`, CLI) | +| `schemas` | JSON Schemas for `stage.json`, `*.pipeline.json` and the public messages | +| `tests` | .NET tests (unit, integration with real Python, containers, Aspire and dashboard E2E with Playwright) and sample pipelines | + +## Development ```bash dotnet build dotnet test --project tests/PythonScriptRunner.Core.Tests -dotnet test --project tests/PythonScriptRunner.EntityFrameworkCore.Tests # precisa de Docker -dotnet test --project tests/PythonScriptRunner.RabbitMQ.Tests # precisa de Docker -dotnet test --project tests/PythonScriptRunner.Worker.Tests # precisa de Docker -dotnet test --project tests/PythonScriptRunner.Dashboard.Tests # precisa de Docker +dotnet test --project tests/PythonScriptRunner.EntityFrameworkCore.Tests # needs Docker +dotnet test --project tests/PythonScriptRunner.RabbitMQ.Tests # needs Docker +dotnet test --project tests/PythonScriptRunner.Worker.Tests # needs Docker +dotnet test --project tests/PythonScriptRunner.Dashboard.Tests # needs Docker +dotnet test --project tests/PythonScriptRunner.AppHost.Tests # needs Docker, Node.js and uv cd python/psrunner uv sync uv run pytest ``` -O painel fica em `panel/` (Node 24). Os builds em Debug embutem uma página provisória; em Release (ou com -`-p:PsrBuildPanel=true`) o MSBuild roda `npm ci` e `npm run build` e embute `panel/dist`. +The UI lives in `panel/` (Node 24). Debug builds embed a placeholder page. In Release (or with `-p:PsrBuildPanel=true`), +MSBuild runs `npm ci` and `npm run build` and embeds `panel/dist`. ```bash cd panel npm ci -npm run dev # Vite em http://localhost:5173/; /api e /hubs vão para o DashboardHost em PSR_DASHBOARD_URL (padrão http://localhost:5080, prefixo /psr) +npm run dev # Vite at http://localhost:5173/; /api and /hubs go to the DashboardHost at PSR_DASHBOARD_URL (default http://localhost:5080, prefix /psr) npm test # Vitest npm run lint # ESLint + Prettier -npm run build # checagem de tipos, build e orçamento do bundle inicial (≤ 500 KB gzip) -npm run api # regera src/api/schema.d.ts a partir do snapshot OpenAPI +npm run build # type check, build and the budget of the initial bundle (≤ 500 KB gzip) +npm run api # regenerates src/api/schema.d.ts from the OpenAPI snapshot ``` -Os testes E2E do painel sobem o `DashboardHost` com 4 usuários de login fixo, um worker, PostgreSQL e RabbitMQ e dirigem -o Chromium com Playwright (precisam de Docker e do painel embutido): +The dashboard E2E tests need Docker and the embedded UI. They start the `DashboardHost` with 4 fixed-login users, a +worker, PostgreSQL and RabbitMQ, and drive Chromium with Playwright: ```bash dotnet build tests/PythonScriptRunner.Panel.E2E.Tests -p:PsrBuildPanel=true -dotnet test --project tests/PythonScriptRunner.Panel.E2E.Tests --no-build # PSR_E2E_HEADED=1 mostra o navegador +dotnet test --project tests/PythonScriptRunner.Panel.E2E.Tests --no-build # PSR_E2E_HEADED=1 shows the browser ``` -Quando um teste falha, o *trace* do Playwright fica em `TestResults/traces` ao lado do assembly de testes -(`npx playwright show-trace .zip`). Rode o E2E sozinho: contêineres subindo ou parando em outros projetos de -teste mudam as interfaces de rede e o Chromium aborta requisições com `ERR_NETWORK_CHANGED`. Para todo o resto de uma vez, -use o filtro de solução sem o E2E (é o que o CI faz): +When a test fails, the Playwright trace is saved to `TestResults/traces` next to the test assembly +(`npx playwright show-trace .zip`). + +Run the E2E tests on their own. Containers starting or stopping in other test projects change the network interfaces, +and Chromium then aborts requests with `ERR_NETWORK_CHANGED`. The solution filter runs the unit, integration and +dashboard tests at once (the CI does the same; the container and AppHost tests run in their own jobs): ```bash dotnet test --solution tests/PythonScriptRunner.SemE2E.slnf ``` -O teste de desempenho do painel usa ~200 mil execuções; `PSR_PERF_RUNS_PER_DAY=50000` roda o volume médio completo (90 dias). -O documento OpenAPI do painel é comparado com `tests/PythonScriptRunner.Dashboard.Tests/Snapshots/openapi-v1.verified.json`; -depois de revisar uma mudança intencional, `PSR_ACCEPT_SNAPSHOTS=1` atualiza o snapshot. +The container tests build both images and run them the way production does: read-only, with tmpfs volumes. They cover +the memory limit through cgroup v2 and the compose from `aspire publish`; the compose test needs the Aspire CLI +(`PSR_ASPIRE_CLI` points to it if it is not on the PATH). To test images you built yourself, set +`PSR_TEST_WORKER_IMAGE` and `PSR_TEST_DASHBOARD_IMAGE`: + +```bash +dotnet test --project tests/PythonScriptRunner.Containers.Tests # needs Docker +``` + +The dashboard performance test uses about 200,000 runs; `PSR_PERF_RUNS_PER_DAY=50000` runs the full average volume (90 +days). The dashboard's OpenAPI document is compared with +`tests/PythonScriptRunner.Dashboard.Tests/Snapshots/openapi-v1.verified.json`. After you review an intended change, +`PSR_ACCEPT_SNAPSHOTS=1` updates the snapshot. + +## Contributing and security + +- [CONTRIBUTING.md](https://github.com/ScriptRunnerKit/PythonScriptRunner/blob/main/CONTRIBUTING.md): how to build, test and send changes. +- [CHANGELOG.md](https://github.com/ScriptRunnerKit/PythonScriptRunner/blob/main/CHANGELOG.md): the history of each version. +- [SECURITY.md](https://github.com/ScriptRunnerKit/PythonScriptRunner/blob/main/SECURITY.md): how to report a vulnerability privately. -## Licença +## License -[MIT](LICENSE) +[MIT](https://github.com/ScriptRunnerKit/PythonScriptRunner/blob/main/LICENSE) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..b7c7935 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,65 @@ +# Security Policy + +## Reporting a vulnerability + +**Please do not report security vulnerabilities through public issues, discussions or pull requests.** + +Report them privately with GitHub's private vulnerability reporting: + +1. Open the **Security** tab of this repository. +2. Choose **Report a vulnerability**. + +Only the maintainers can see the report. Please include as much as you can of: + +- the affected package or image and its version (or the commit); +- a description of the issue and its impact; +- steps to reproduce, a proof of concept or the affected code; +- any known workaround. + +We will acknowledge the report as soon as possible, usually within a week. We will keep you informed and agree on a +disclosure date with you. Fixes ship as a new patch version, with a GitHub Security Advisory that credits you, unless you +prefer to remain anonymous. + +## Supported versions + +| Version | Supported | +|---------|-----------| +| 0.1.x | ✅ | + +Security fixes go to the latest minor version: the NuGet packages, the `psr-worker` and `psr-dashboard` images, and the +matching `psrunner` release. + +## Security model + +PythonScriptRunner runs Python code written by **trusted authors**. A stage is arbitrary code running with the +permissions of the worker's user, and it is **not a sandbox** for untrusted code. Anyone with the `Author` role can +publish code that runs on the workers, so give that role only to people you would let deploy code. + +What the runner guarantees: + +- Each attempt runs in its own process. That process gets a clean environment (no variables from the host), its own + work folder, a timeout and a memory limit. +- A secret reaches only the processes of the stages that declare it. Secrets are never written to `input.json`, logs, + metrics or exceptions, and their values are masked (`***`) in the captured stdout and stderr. +- Published dependencies are locked with hashes (`uv pip compile --generate-hashes`) and installed with + `--require-hashes`. +- In the dashboard, every action checks the user's role. Requests that change something require the `X-PSR-Request` + header, and fixed login stores only PBKDF2 hashes, with lockout and rate limiting. + +Examples of vulnerabilities we want to hear about: + +- getting past the dashboard's authentication or roles; +- a secret leaking to another stage, to the logs or to the database; +- breaking the guarantees above, such as a process that outlives its timeout or exceeds its memory limit; +- an issue in the container images. + +A stage author running code on the workers is not a vulnerability: it is how the product works. + +## Hardening + +- Run workers as an unprivileged user with access only to what the stages need. The images run as the non-root user + `psr` with a read-only root file system. +- Keep secrets in the platform's secret store, and pass them as `PythonScriptRunner__Secrets__`. +- Serve the dashboard over HTTPS. Behind a proxy, set `ASPNETCORE_FORWARDEDHEADERS_ENABLED=true`. +- Do not expose the development setup of the AppHost or of the generated compose: its users share the password + `psr-dev`, and it includes Mailpit and test SMTP secrets. diff --git a/hosts/PythonScriptRunner.AppHost/AppHost.cs b/hosts/PythonScriptRunner.AppHost/AppHost.cs new file mode 100644 index 0000000..9457552 --- /dev/null +++ b/hosts/PythonScriptRunner.AppHost/AppHost.cs @@ -0,0 +1,83 @@ +// Ambiente local completo (spec 009; ADR-0019): PostgreSQL, RabbitMQ, SMTP de teste (Mailpit), worker, painel e o +// painel Vue em modo dev. `aspire publish` gera o docker-compose com os Dockerfiles dos hosts (as mesmas imagens do GHCR). +var builder = DistributedApplication.CreateBuilder(args); + +builder.AddDockerComposeEnvironment("compose"); + +var postgres = builder.AddPostgres("postgres").WithImageTag("16-alpine").WithDataVolume("psr-postgres"); +var database = postgres.AddDatabase("psr"); +var rabbitmq = builder.AddRabbitMQ("rabbitmq").WithImageTag("3.13-management-alpine").WithManagementPlugin(); + +// SMTP de teste do CS-2: aceita qualquer usuário, sem TLS (D-172). +builder.AddContainer("mailpit", "axllent/mailpit", "v1.27.0") + .WithEnvironment("MP_SMTP_AUTH_ACCEPT_ANY", "1") + .WithEnvironment("MP_SMTP_AUTH_ALLOW_INSECURE", "1") + .WithHttpEndpoint(targetPort: 8025, name: "http") + .WithEndpoint(targetPort: 1025, name: "smtp", scheme: "tcp"); + +// Em desenvolvimento o psrunner vem do próprio repositório; nas imagens, do wheel embutido (D-169). +var psrunner = Path.GetFullPath(Path.Combine(builder.AppHostDirectory, "..", "..", "python", "psrunner")); + +var worker = builder.AddProject("worker") + .WithHttpEndpoint(name: "http") + .WithHttpHealthCheck("/health/ready") + .WithReference(database).WaitFor(database) + .WithReference(rabbitmq).WaitFor(rabbitmq) + .WithEnvironment("PSR_AUTO_MIGRATE", "true") + .WithEnvironment("PythonScriptRunner__Secrets__SMTP_USUARIO", "psr") + .WithEnvironment("PythonScriptRunner__Secrets__SMTP_SENHA", "psr-dev") + .PublishAsDockerFile(c => c.WithDockerfile("../..", "hosts/PythonScriptRunner.WorkerHost/Dockerfile")); + +// Login fixo com 4 usuários de desenvolvimento, um por papel, todos com a senha psr-dev (D-133, D-177). +var passwordHash = builder.Configuration["PythonScriptRunner:DevPasswordHash"] + ?? throw new InvalidOperationException("PythonScriptRunner:DevPasswordHash is missing from the AppHost configuration."); +(string Name, string Role)[] users = [("leitor", "Reader"), ("executor", "Executor"), ("autor", "Author"), ("admin", "Admin")]; +var dashboard = builder.AddProject("dashboard") + .WithHttpEndpoint(name: "http") + .WithExternalHttpEndpoints() + .WithUrlForEndpoint("http", url => + { + url.Url = "/psr/"; + url.DisplayText = "Painel"; + }) + .WithHttpHealthCheck("/health/ready") + .WithReference(database).WaitFor(database) + .WithReference(rabbitmq).WaitFor(rabbitmq) + .WithEnvironment("PSR_AUTO_MIGRATE", "true") + .PublishAsDockerFile(c => c.WithDockerfile("../..", "hosts/PythonScriptRunner.DashboardHost/Dockerfile")); +for (var i = 0; i < users.Length; i++) +{ + dashboard + .WithEnvironment($"PythonScriptRunner__Dashboard__Login__Users__{i}__Name", users[i].Name) + .WithEnvironment($"PythonScriptRunner__Dashboard__Login__Users__{i}__Role", users[i].Role) + .WithEnvironment($"PythonScriptRunner__Dashboard__Login__Users__{i}__PasswordHash", passwordHash); +} + +// No docker-compose (`aspire publish`): o próprio PostgreSQL cria o banco psr (em dev o Aspire o cria), imagens do GHCR na +// versão PSR_VERSION, reinício até o banco aceitar conexões e o painel numa porta fixa (PSR_DASHBOARD_PORT, padrão 8080). +postgres.PublishAsDockerComposeService((_, service) => service.Environment["POSTGRES_DB"] = "psr"); +worker.PublishAsDockerComposeService((_, service) => +{ + service.Image = "ghcr.io/scriptrunnerkit/psr-worker:${PSR_VERSION:-latest}"; + service.Restart = "unless-stopped"; +}); +dashboard.PublishAsDockerComposeService((_, service) => +{ + service.Image = "ghcr.io/scriptrunnerkit/psr-dashboard:${PSR_VERSION:-latest}"; + service.Restart = "unless-stopped"; + service.Ports = ["${PSR_DASHBOARD_PORT:-8080}:8080"]; +}); + +if (builder.ExecutionContext.IsRunMode) +{ + worker.WithEnvironment("PythonScriptRunner__PsrunnerRequirement", psrunner); + dashboard.WithEnvironment("PythonScriptRunner__PsrunnerRequirement", psrunner); + + // Painel Vue com hot reload, com proxy da API para o DashboardHost (no compose o painel vem embutido na imagem). + builder.AddViteApp("panel", "../../panel") + .WithNpm() + .WithEnvironment("PSR_DASHBOARD_URL", dashboard.GetEndpoint("http")) + .WaitFor(dashboard); +} + +builder.Build().Run(); diff --git a/hosts/PythonScriptRunner.AppHost/Properties/launchSettings.json b/hosts/PythonScriptRunner.AppHost/Properties/launchSettings.json new file mode 100644 index 0000000..dbce516 --- /dev/null +++ b/hosts/PythonScriptRunner.AppHost/Properties/launchSettings.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:15000", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "ASPIRE_DASHBOARD_OTLP_HTTP_ENDPOINT_URL": "http://localhost:19000", + "ASPIRE_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20000", + "ASPIRE_ALLOW_UNSECURED_TRANSPORT": "true" + } + } + } +} diff --git a/hosts/PythonScriptRunner.AppHost/PythonScriptRunner.AppHost.csproj b/hosts/PythonScriptRunner.AppHost/PythonScriptRunner.AppHost.csproj new file mode 100644 index 0000000..f8cf536 --- /dev/null +++ b/hosts/PythonScriptRunner.AppHost/PythonScriptRunner.AppHost.csproj @@ -0,0 +1,21 @@ + + + + Exe + true + psr-apphost-6f1c2a4e + + + + + + + + + + + + + + + diff --git a/hosts/PythonScriptRunner.AppHost/appsettings.json b/hosts/PythonScriptRunner.AppHost/appsettings.json new file mode 100644 index 0000000..ce0b4e4 --- /dev/null +++ b/hosts/PythonScriptRunner.AppHost/appsettings.json @@ -0,0 +1,12 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning", + "Aspire.Hosting.Dcp": "Warning" + } + }, + "PythonScriptRunner": { + "DevPasswordHash": "AQAAAAIAAYagAAAAEOzP04xls5e0roG3bww80/+u+X1d75u5LMZkVxmb/UCA9st4tKCh8ccvJgNTM4os5A==" + } +} diff --git a/hosts/PythonScriptRunner.AppHost/aspire.config.json b/hosts/PythonScriptRunner.AppHost/aspire.config.json new file mode 100644 index 0000000..c32ab61 --- /dev/null +++ b/hosts/PythonScriptRunner.AppHost/aspire.config.json @@ -0,0 +1,5 @@ +{ + "appHost": { + "path": "PythonScriptRunner.AppHost.csproj" + } +} diff --git a/hosts/PythonScriptRunner.DashboardHost/Dockerfile b/hosts/PythonScriptRunner.DashboardHost/Dockerfile new file mode 100644 index 0000000..f14b757 --- /dev/null +++ b/hosts/PythonScriptRunner.DashboardHost/Dockerfile @@ -0,0 +1,66 @@ +# syntax=docker/dockerfile:1 +# Imagem do DashboardHost (spec 009): API, painel Vue embutido e relay do outbox; inclui o uv porque publicar uma +# Etapa resolve as dependências. Runtime ASP.NET 10, usuário não-root psr. +# Build a partir da raiz do repositório: docker build -f hosts/PythonScriptRunner.DashboardHost/Dockerfile . +ARG UV_VERSION=0.13.4 +# Mesma faixa do global.json (10.0.4xx); a tag 10.0 ainda aponta para 10.0.3xx. +ARG DOTNET_SDK=10.0.402 + +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv + +# psrunner do mesmo commit (D-169): a imagem não depende do PyPI para o SDK. +FROM --platform=$BUILDPLATFORM ghcr.io/astral-sh/uv:${UV_VERSION}-python3.12-trixie-slim AS psrunner +WORKDIR /src +COPY python/psrunner/ . +RUN uv build --wheel --out-dir /wheel + +# Painel Vue (spec 008; D-155, D-158). +FROM --platform=$BUILDPLATFORM node:24-bookworm-slim AS panel +WORKDIR /panel +COPY panel/package.json panel/package-lock.json ./ +RUN npm ci --no-audit --no-fund +COPY panel/ . +RUN npm run build + +# Compila na arquitetura da máquina e publica para a de destino (multi-arch sem emulação); o painel já vem pronto. +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:${DOTNET_SDK} AS build +ARG TARGETARCH +ARG VERSION=0.0.0-docker +WORKDIR /src +COPY global.json Directory.Build.props Directory.Packages.props .editorconfig ./ +COPY src/ src/ +COPY schemas/ schemas/ +COPY --from=panel /panel/dist/ panel/dist/ +COPY hosts/Directory.Build.props hosts/ +COPY hosts/PythonScriptRunner.ServiceDefaults/ hosts/PythonScriptRunner.ServiceDefaults/ +COPY hosts/PythonScriptRunner.DashboardHost/ hosts/PythonScriptRunner.DashboardHost/ +RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo arm64 || echo x64) \ + && dotnet publish hosts/PythonScriptRunner.DashboardHost -c Release -a "$arch" --self-contained false -o /app \ + -p:MinVerVersionOverride="$VERSION" -p:PsrPanelPrebuilt=true + +FROM mcr.microsoft.com/dotnet/aspnet:10.0 +LABEL org.opencontainers.image.title="psr-dashboard" \ + org.opencontainers.image.description="PythonScriptRunner dashboard: Vue panel, API and real-time updates." \ + org.opencontainers.image.source="https://github.com/ScriptRunnerKit/PythonScriptRunner" \ + org.opencontainers.image.licenses="MIT" +COPY --from=uv /uv /uvx /usr/local/bin/ +RUN groupadd --gid 10001 psr \ + && useradd --uid 10001 --gid psr --home-dir /var/lib/psr --no-create-home --shell /usr/sbin/nologin psr \ + && mkdir -p /var/lib/psr/envs /var/lib/psr/python /tmp/psr /opt/psr \ + && chown -R psr:psr /var/lib/psr /tmp/psr +COPY --from=psrunner /wheel/ /opt/psr/ +COPY --chmod=755 hosts/docker/entrypoint.sh /opt/psr/entrypoint.sh +ENV HOME=/var/lib/psr \ + TMPDIR=/tmp/psr/tmp \ + ASPNETCORE_HTTP_PORTS=8080 \ + UV_PYTHON_INSTALL_DIR=/var/lib/psr/python \ + UV_CACHE_DIR=/var/lib/psr/envs/.uv-cache \ + PythonScriptRunner__UvPath=/usr/local/bin/uv \ + PythonScriptRunner__WorkRoot=/tmp/psr \ + PythonScriptRunner__EnvironmentCacheRoot=/var/lib/psr/envs +USER psr +WORKDIR /app +COPY --from=build /app . +EXPOSE 8080 +VOLUME ["/var/lib/psr/envs", "/var/lib/psr/python", "/tmp/psr"] +ENTRYPOINT ["/opt/psr/entrypoint.sh", "dotnet", "PythonScriptRunner.DashboardHost.dll"] diff --git a/hosts/PythonScriptRunner.ServiceDefaults/ServiceDefaultsExtensions.cs b/hosts/PythonScriptRunner.ServiceDefaults/ServiceDefaultsExtensions.cs index 6cf5c7a..91877e4 100644 --- a/hosts/PythonScriptRunner.ServiceDefaults/ServiceDefaultsExtensions.cs +++ b/hosts/PythonScriptRunner.ServiceDefaults/ServiceDefaultsExtensions.cs @@ -7,6 +7,7 @@ using OpenTelemetry.Resources; using OpenTelemetry.Trace; using Serilog; +using Serilog.Events; using Serilog.Formatting.Compact; namespace Microsoft.Extensions.Hosting; @@ -31,7 +32,10 @@ public static TBuilder AddServiceDefaults(this TBuilder builder, strin where TBuilder : IHostApplicationBuilder { ArgumentNullException.ThrowIfNull(builder); + // ASP.NET Core logs every request at Information; the traces already cover them (D-187). The configuration + // (Serilog:MinimumLevel:Override:Microsoft.AspNetCore) can bring them back. builder.Services.AddSerilog((services, logger) => logger + .MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning) .ReadFrom.Configuration(builder.Configuration) .Enrich.FromLogContext() .Enrich.WithProperty("service", serviceName) diff --git a/hosts/PythonScriptRunner.WorkerHost/Dockerfile b/hosts/PythonScriptRunner.WorkerHost/Dockerfile new file mode 100644 index 0000000..bf2e1c8 --- /dev/null +++ b/hosts/PythonScriptRunner.WorkerHost/Dockerfile @@ -0,0 +1,59 @@ +# syntax=docker/dockerfile:1 +# Imagem do WorkerHost (spec 009): runtime ASP.NET 10 + uv + psrunner do mesmo commit, usuário não-root psr. +# Build a partir da raiz do repositório: docker build -f hosts/PythonScriptRunner.WorkerHost/Dockerfile . +ARG UV_VERSION=0.13.4 +# Mesma faixa do global.json (10.0.4xx); a tag 10.0 ainda aponta para 10.0.3xx. +ARG DOTNET_SDK=10.0.402 + +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv + +# psrunner do mesmo commit (D-169): a imagem não depende do PyPI para o SDK. +FROM --platform=$BUILDPLATFORM ghcr.io/astral-sh/uv:${UV_VERSION}-python3.12-trixie-slim AS psrunner +WORKDIR /src +COPY python/psrunner/ . +RUN uv build --wheel --out-dir /wheel + +# Compila na arquitetura da máquina e publica para a de destino (multi-arch sem emulação). +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:${DOTNET_SDK} AS build +ARG TARGETARCH +ARG VERSION=0.0.0-docker +WORKDIR /src +COPY global.json Directory.Build.props Directory.Packages.props .editorconfig ./ +COPY src/ src/ +COPY schemas/ schemas/ +COPY hosts/Directory.Build.props hosts/ +COPY hosts/PythonScriptRunner.ServiceDefaults/ hosts/PythonScriptRunner.ServiceDefaults/ +COPY hosts/PythonScriptRunner.WorkerHost/ hosts/PythonScriptRunner.WorkerHost/ +RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo arm64 || echo x64) \ + && dotnet publish hosts/PythonScriptRunner.WorkerHost -c Release -a "$arch" --self-contained false -o /app \ + -p:MinVerVersionOverride="$VERSION" + +FROM mcr.microsoft.com/dotnet/aspnet:10.0 +LABEL org.opencontainers.image.title="psr-worker" \ + org.opencontainers.image.description="PythonScriptRunner worker: runs queued Python stages and cron schedules." \ + org.opencontainers.image.source="https://github.com/ScriptRunnerKit/PythonScriptRunner" \ + org.opencontainers.image.licenses="MIT" +# Pré-aquecimento opcional: --build-arg PSR_PREINSTALL_PYTHON=3.12 instala o intérprete na imagem. +ARG PSR_PREINSTALL_PYTHON= +COPY --from=uv /uv /uvx /usr/local/bin/ +RUN groupadd --gid 10001 psr \ + && useradd --uid 10001 --gid psr --home-dir /var/lib/psr --no-create-home --shell /usr/sbin/nologin psr \ + && mkdir -p /var/lib/psr/envs /var/lib/psr/python /tmp/psr /opt/psr \ + && chown -R psr:psr /var/lib/psr /tmp/psr +COPY --from=psrunner /wheel/ /opt/psr/ +COPY --chmod=755 hosts/docker/entrypoint.sh /opt/psr/entrypoint.sh +ENV HOME=/var/lib/psr \ + TMPDIR=/tmp/psr/tmp \ + ASPNETCORE_HTTP_PORTS=8080 \ + UV_PYTHON_INSTALL_DIR=/var/lib/psr/python \ + UV_CACHE_DIR=/var/lib/psr/envs/.uv-cache \ + PythonScriptRunner__UvPath=/usr/local/bin/uv \ + PythonScriptRunner__WorkRoot=/tmp/psr \ + PythonScriptRunner__EnvironmentCacheRoot=/var/lib/psr/envs +USER psr +RUN if [ -n "$PSR_PREINSTALL_PYTHON" ]; then uv python install "$PSR_PREINSTALL_PYTHON"; fi +WORKDIR /app +COPY --from=build /app . +EXPOSE 8080 +VOLUME ["/var/lib/psr/envs", "/var/lib/psr/python", "/tmp/psr"] +ENTRYPOINT ["/opt/psr/entrypoint.sh", "dotnet", "PythonScriptRunner.WorkerHost.dll"] diff --git a/hosts/docker/entrypoint.sh b/hosts/docker/entrypoint.sh new file mode 100755 index 0000000..c187079 --- /dev/null +++ b/hosts/docker/entrypoint.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# Entrada das imagens dos hosts (spec 009): usa o psrunner embutido na imagem (D-169), a menos que +# PythonScriptRunner__PsrunnerRequirement já venha definido (por exemplo, um índice privado). +set -e +# Arquivos temporários (buffer de requisições do ASP.NET Core, spool de arquivos) vão para o volume de trabalho: a raiz +# pode ser somente leitura. +mkdir -p "${TMPDIR:-/tmp}" +if [ -z "${PythonScriptRunner__PsrunnerRequirement:-}" ]; then + for wheel in /opt/psr/psrunner-*.whl; do + PythonScriptRunner__PsrunnerRequirement="$wheel" + done + export PythonScriptRunner__PsrunnerRequirement +fi + +exec "$@" diff --git a/panel/package-lock.json b/panel/package-lock.json index c2d7b52..8da6009 100644 --- a/panel/package-lock.json +++ b/panel/package-lock.json @@ -2308,9 +2308,9 @@ } }, "node_modules/dompurify": { - "version": "3.4.15", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.15.tgz", - "integrity": "sha512-EUBjM+B+lkDE41iE82DDSCfkoPGfXx8IxFxPMjNzm/Uk4xDet77rTN9wqlxlVg71kK7XGuUMv6wUxJUwwv+Xyw==", + "version": "3.4.16", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.16.tgz", + "integrity": "sha512-sqo+pNp3qRhCIpbgRi1y8Tgk27Bo2Ry7w0dC1NBeNTdZChWjz9Xb/KOoZbRP/R6pQZ80Qw8YhXw13hWWBbMRnQ==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" diff --git a/panel/package.json b/panel/package.json index 4cf1d40..69eac73 100644 --- a/panel/package.json +++ b/panel/package.json @@ -55,5 +55,8 @@ "vite-plugin-vuetify": "2.1.3", "vitest": "5.0.5", "vue-tsc": "3.3.13" + }, + "overrides": { + "dompurify": "3.4.16" } } diff --git a/python/psrunner/README.md b/python/psrunner/README.md index ea5dca5..4223301 100644 --- a/python/psrunner/README.md +++ b/python/psrunner/README.md @@ -1,6 +1,6 @@ # psrunner -Python SDK for [PythonScriptRunner](https://github.com/net0well/PythonScriptRunner) stages. +Python SDK for [PythonScriptRunner](https://github.com/ScriptRunnerKit/PythonScriptRunner) stages. A stage is one Python script with typed inputs and outputs. The .NET runner validates the inputs, runs the script in an isolated process and validates the outputs. `psrunner` handles the protocol, diff --git a/python/psrunner/pyproject.toml b/python/psrunner/pyproject.toml index cdadafa..3899f2f 100644 --- a/python/psrunner/pyproject.toml +++ b/python/psrunner/pyproject.toml @@ -9,7 +9,7 @@ description = "Python SDK for PythonScriptRunner stages: typed inputs and output readme = "README.md" requires-python = ">=3.10" license = "MIT" -authors = [{ name = "net0well" }] +authors = [{ name = "ScriptRunnerKit contributors" }] keywords = ["pythonscriptrunner", "dotnet", "pipeline", "stage"] classifiers = [ "Development Status :: 3 - Alpha", @@ -29,7 +29,7 @@ dependencies = [] pandas = ["pandas>=2.0"] [project.urls] -Repository = "https://github.com/net0well/PythonScriptRunner" +Repository = "https://github.com/ScriptRunnerKit/PythonScriptRunner" [project.scripts] psrunner = "psrunner.cli:main" diff --git a/python/psrunner/src/psrunner/host.py b/python/psrunner/src/psrunner/host.py index 41a76d8..ed115ad 100644 --- a/python/psrunner/src/psrunner/host.py +++ b/python/psrunner/src/psrunner/host.py @@ -17,8 +17,10 @@ from psrunner.context import Context from psrunner.convert import inputs_to_python, outputs_to_wire from psrunner.fields import specs_from_dicts -from psrunner.limits import apply_memory_limit +from psrunner.limits import apply_limits from psrunner.protocol import ( + ENV_CGROUP, + ENV_FALLBACK_THREADS, ENV_INPUT, ENV_MEMORY_LIMIT_MB, ENV_OUTPUT, @@ -66,7 +68,11 @@ def _run() -> dict[str, Any]: f"this psrunner speaks {PROTOCOL_VERSION}." ) - apply_memory_limit(int(os.environ.get(ENV_MEMORY_LIMIT_MB, "0") or 0)) + apply_limits( + int(os.environ.get(ENV_MEMORY_LIMIT_MB, "0") or 0), + os.environ.get(ENV_CGROUP, ""), + int(os.environ.get(ENV_FALLBACK_THREADS, "0") or 0), + ) logger = _configure_logging() try: diff --git a/python/psrunner/src/psrunner/limits.py b/python/psrunner/src/psrunner/limits.py index fa7ecfc..101be35 100644 --- a/python/psrunner/src/psrunner/limits.py +++ b/python/psrunner/src/psrunner/limits.py @@ -7,12 +7,55 @@ from __future__ import annotations +import os import sys +from pathlib import Path from typing import Any _MB = 1024 * 1024 _keep_alive: list[Any] = [] +# Thread pools of the numeric libraries (OpenMP, OpenBLAS, MKL, numexpr, Accelerate). +THREAD_VARIABLES = ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + "VECLIB_MAXIMUM_THREADS", +) + + +def apply_limits(limit_mb: int, cgroup: str, fallback_threads: int) -> str: + """Applies the memory limit of the attempt; returns how: ``cgroup``, ``process`` or ``none``. + + On Linux the runner may create a cgroup v2 with ``memory.max`` for the attempt (spec 001, + D-180): joining it limits the memory really used. Without it, the limit falls back to + ``RLIMIT_AS``, which counts reserved address space, so the numeric libraries are limited to + ``fallback_threads`` threads: OpenBLAS reserves memory per thread and, on a machine with many + cores, ``import numpy`` alone would exceed the limit. Both happen before the script is imported. + """ + if cgroup and join_cgroup(cgroup): + return "cgroup" + if fallback_threads > 0: + limit_threads(fallback_threads) + return "process" if apply_memory_limit(limit_mb) else "none" + + +def join_cgroup(path: str) -> bool: + """Moves this process into the cgroup at ``path``. Returns ``True`` when it joined.""" + try: + Path(path, "cgroup.procs").write_text(str(os.getpid()), encoding="ascii") + except OSError as exc: + print(f"psrunner: could not join the cgroup {path}: {exc}", file=sys.stderr) + return False + return True + + +def limit_threads(count: int) -> None: + """Limits the thread pools of the numeric libraries, keeping values set by the runner.""" + for name in THREAD_VARIABLES: + os.environ.setdefault(name, str(count)) + def apply_memory_limit(limit_mb: int) -> bool: """Limits the memory of this process. Returns ``True`` when a limit was applied.""" diff --git a/python/psrunner/src/psrunner/protocol.py b/python/psrunner/src/psrunner/protocol.py index 6712d0a..1cd083d 100644 --- a/python/psrunner/src/psrunner/protocol.py +++ b/python/psrunner/src/psrunner/protocol.py @@ -10,6 +10,10 @@ ENV_WORK_DIR = "PSR_WORK_DIR" ENV_SCRIPT = "PSR_SCRIPT" ENV_MEMORY_LIMIT_MB = "PSR_MEMORY_LIMIT_MB" +# Linux: cgroup v2 of the attempt (memory.max) and, if it cannot be joined, the threads of the +# numeric libraries. +ENV_CGROUP = "PSR_CGROUP" +ENV_FALLBACK_THREADS = "PSR_FALLBACK_THREADS" ENV_SECRET_PREFIX = "PSR_SECRET_" FILE_MARKER = "$file" diff --git a/python/psrunner/tests/conftest.py b/python/psrunner/tests/conftest.py index bbe880a..d66e942 100644 --- a/python/psrunner/tests/conftest.py +++ b/python/psrunner/tests/conftest.py @@ -51,6 +51,7 @@ def _run( secrets: dict[str, str] | None = None, protocol: str = "1", memory_limit_mb: int = 0, + extra_env: dict[str, str] | None = None, ) -> HostRun: work = tmp_path / "work" for sub in ("in", "out", "tmp"): @@ -110,6 +111,7 @@ def _run( ) for name, value in (secrets or {}).items(): env[f"PSR_SECRET_{name}"] = value + env.update(extra_env or {}) completed = subprocess.run( [sys.executable, "-I", "-X", "utf8", "-m", "psrunner.host"], diff --git a/python/psrunner/tests/test_host.py b/python/psrunner/tests/test_host.py index fedefb9..8aaf2b3 100644 --- a/python/psrunner/tests/test_host.py +++ b/python/psrunner/tests/test_host.py @@ -2,6 +2,8 @@ from __future__ import annotations +from pathlib import Path + import pytest from conftest import RunHost @@ -267,6 +269,87 @@ def run(inputs, ctx): assert run.error["kind"] == "MemoryLimitExceeded" +def test_spec001_ac30_joins_the_cgroup_of_the_attempt_instead_of_limiting_itself( + run_host: RunHost, tmp_path: Path +) -> None: + # A stand-in for the cgroup the runner created: the real memory.max lives in the kernel. + cgroup = tmp_path / "psr-runs" / "tentativa" + cgroup.mkdir(parents=True) + (cgroup / "cgroup.procs").write_text("", encoding="ascii") + script = """ + import os + from psrunner import stage + + @stage + def run(inputs, ctx): + data = bytearray(256 * 1024 * 1024) + return {"pid": os.getpid(), "threads": os.environ.get("OPENBLAS_NUM_THREADS", "")} + """ + + run = run_host( + script, + output_fields=[{"name": "pid", "type": "integer"}, {"name": "threads", "type": "string"}], + memory_limit_mb=128, + extra_env={"PSR_CGROUP": str(cgroup), "PSR_FALLBACK_THREADS": "1"}, + ) + + assert (cgroup / "cgroup.procs").read_text(encoding="ascii") == str(run.outputs["pid"]) + assert run.outputs["threads"] == "", "with a cgroup the numeric libraries keep their threads" + + +def test_spec001_ac29_without_the_cgroup_limits_the_threads_and_keeps_the_memory_limit( + run_host: RunHost, tmp_path: Path +) -> None: + script = """ + import os + from psrunner import stage + + @stage + def run(inputs, ctx): + names = [ + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + "NUMEXPR_NUM_THREADS", + ] + threads = ",".join(os.environ.get(name, "") for name in names) + data = bytearray(1024 * 1024 * 1024) + return {"threads": threads, "tamanho": len(data)} + """ + + run = run_host( + script, + output_fields=[ + {"name": "threads", "type": "string"}, + {"name": "tamanho", "type": "integer"}, + ], + memory_limit_mb=128, + extra_env={"PSR_CGROUP": str(tmp_path / "nao-existe"), "PSR_FALLBACK_THREADS": "3"}, + ) + + assert run.error["kind"] == "MemoryLimitExceeded" + assert "could not join the cgroup" in run.stderr + + +def test_spec001_ac29_fallback_threads_reach_the_script(run_host: RunHost, tmp_path: Path) -> None: + script = """ + import os + from psrunner import stage + + @stage + def run(inputs, ctx): + return {"threads": os.environ.get("OPENBLAS_NUM_THREADS", "")} + """ + + run = run_host( + script, + output_fields=[{"name": "threads", "type": "string"}], + extra_env={"PSR_CGROUP": str(tmp_path / "nao-existe"), "PSR_FALLBACK_THREADS": "3"}, + ) + + assert run.outputs["threads"] == "3" + + def test_spec002_ac11_protocol_mismatch(run_host: RunHost) -> None: run = run_host("from psrunner import stage\n", protocol="2") diff --git a/schemas/messages/run-event.v1.json b/schemas/messages/run-event.v1.json index 8655b82..131fa87 100644 --- a/schemas/messages/run-event.v1.json +++ b/schemas/messages/run-event.v1.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/net0well/PythonScriptRunner/main/schemas/messages/run-event.v1.json", + "$id": "https://raw.githubusercontent.com/ScriptRunnerKit/PythonScriptRunner/main/schemas/messages/run-event.v1.json", "title": "PythonScriptRunner run event (psr.run-event.v1)", "description": "Published on the topic exchange psr.events with the routing key . or .stage: (AMQP type psr.run-event.v1). Fields are only added within v1; consumers must ignore unknown fields.", "type": "object", diff --git a/schemas/messages/start-pipeline-run.v1.json b/schemas/messages/start-pipeline-run.v1.json index 3d4acf3..f136108 100644 --- a/schemas/messages/start-pipeline-run.v1.json +++ b/schemas/messages/start-pipeline-run.v1.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/net0well/PythonScriptRunner/main/schemas/messages/start-pipeline-run.v1.json", + "$id": "https://raw.githubusercontent.com/ScriptRunnerKit/PythonScriptRunner/main/schemas/messages/start-pipeline-run.v1.json", "title": "PythonScriptRunner StartPipelineRun command (psr.start-pipeline-run.v1)", "description": "Publish to the exchange psr.public with the routing key pipeline.start (AMQP type psr.start-pipeline-run.v1, content type application/json). The run is announced with run.queued, or refused with run.rejected; both carry the correlationId.", "type": "object", diff --git a/schemas/pipeline.v1.json b/schemas/pipeline.v1.json index 0737cb5..f3588b1 100644 --- a/schemas/pipeline.v1.json +++ b/schemas/pipeline.v1.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/net0well/PythonScriptRunner/main/schemas/pipeline.v1.json", + "$id": "https://raw.githubusercontent.com/ScriptRunnerKit/PythonScriptRunner/main/schemas/pipeline.v1.json", "title": "PythonScriptRunner pipeline manifest (v1)", "description": "Declares one pipeline: inputs, ordered steps (stage@version + bindings) and declared outputs.", "type": "object", diff --git a/schemas/stage.v1.json b/schemas/stage.v1.json index e4a10da..bb27f42 100644 --- a/schemas/stage.v1.json +++ b/schemas/stage.v1.json @@ -1,6 +1,6 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/net0well/PythonScriptRunner/main/schemas/stage.v1.json", + "$id": "https://raw.githubusercontent.com/ScriptRunnerKit/PythonScriptRunner/main/schemas/stage.v1.json", "title": "PythonScriptRunner stage manifest (v1)", "description": "Declares one Stage: typed inputs, exactly one Python script and typed outputs.", "type": "object", diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 38cecf1..a4cbe1d 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -4,5 +4,10 @@ true true + README.md + + + + diff --git a/src/PythonScriptRunner.Core/Catalog/CatalogManifests.cs b/src/PythonScriptRunner.Core/Catalog/CatalogManifests.cs index f653d03..a2b0ed9 100644 --- a/src/PythonScriptRunner.Core/Catalog/CatalogManifests.cs +++ b/src/PythonScriptRunner.Core/Catalog/CatalogManifests.cs @@ -28,7 +28,7 @@ public static class CatalogManifests /// Suffix of pipeline manifests. public const string PipelineSuffix = ".pipeline.json"; - private const string SchemaBase = "https://raw.githubusercontent.com/net0well/PythonScriptRunner/main/schemas/"; + private const string SchemaBase = "https://raw.githubusercontent.com/ScriptRunnerKit/PythonScriptRunner/main/schemas/"; private static readonly Lazy StageSchema = new(() => LoadSchema("stage.v1.json"), LazyThreadSafetyMode.ExecutionAndPublication); private static readonly Lazy PipelineSchema = new(() => LoadSchema("pipeline.v1.json"), LazyThreadSafetyMode.ExecutionAndPublication); diff --git a/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerOptionsValidator.cs b/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerOptionsValidator.cs index 97984f3..02c6a04 100644 --- a/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerOptionsValidator.cs +++ b/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerOptionsValidator.cs @@ -50,6 +50,11 @@ public ValidateOptionsResult Validate(string? name, PythonScriptRunnerOptions op errors.Add("MaxConcurrentProcesses must be at least 1."); } + if (options.ThreadsPerProcess is < 1) + { + errors.Add("ThreadsPerProcess must be at least 1 (or null)."); + } + if (string.IsNullOrWhiteSpace(options.PsrunnerRequirement)) { errors.Add("PsrunnerRequirement is required."); diff --git a/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs b/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs index f9d248d..94f17fe 100644 --- a/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs +++ b/src/PythonScriptRunner.Core/DependencyInjection/PythonScriptRunnerServiceCollectionExtensions.cs @@ -39,6 +39,7 @@ public static IPythonScriptRunnerBuilder AddPythonScriptRunner( services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); + services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); diff --git a/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs b/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs index 1c9fce8..3c7e245 100644 --- a/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs +++ b/src/PythonScriptRunner.Core/Diagnostics/PsrDiagnostics.cs @@ -76,4 +76,10 @@ internal static partial class Log [LoggerMessage(10, LogLevel.Error, "Could not record the failure of run {RunId}")] public static partial void RecordingFailed(ILogger logger, RunId runId, Exception exception); + + [LoggerMessage(11, LogLevel.Information, "Memory limits of Python processes use cgroup v2 under {Path}")] + public static partial void CgroupMemoryLimits(ILogger logger, string path); + + [LoggerMessage(12, LogLevel.Information, "Memory limits of Python processes use RLIMIT_AS ({Reason}); numeric libraries get {Threads} thread(s) per process")] + public static partial void ProcessMemoryLimits(ILogger logger, string reason, int threads); } diff --git a/src/PythonScriptRunner.Core/Environments/UvEnvironmentManager.cs b/src/PythonScriptRunner.Core/Environments/UvEnvironmentManager.cs index 701abeb..97aee35 100644 --- a/src/PythonScriptRunner.Core/Environments/UvEnvironmentManager.cs +++ b/src/PythonScriptRunner.Core/Environments/UvEnvironmentManager.cs @@ -42,7 +42,8 @@ with open(sys.argv[1], encoding="utf-8") as source_file: public async Task CheckSyntaxAsync(string? pythonVersion, string script, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(script); - var folder = Path.Combine(Path.GetTempPath(), "psr-syntax", Guid.NewGuid().ToString("N")); + // Inside WorkRoot: a container with a read-only root file system only writes its volumes. + var folder = Path.Combine(options.Value.WorkRoot, "syntax", Guid.NewGuid().ToString("N")); Directory.CreateDirectory(folder); try { diff --git a/src/PythonScriptRunner.Core/Execution/CgroupMemory.cs b/src/PythonScriptRunner.Core/Execution/CgroupMemory.cs new file mode 100644 index 0000000..0252fae --- /dev/null +++ b/src/PythonScriptRunner.Core/Execution/CgroupMemory.cs @@ -0,0 +1,233 @@ +using System.Globalization; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using PythonScriptRunner.Core.Diagnostics; + +namespace PythonScriptRunner.Core.Execution; + +/// +/// Memory limits of Python processes with cgroup v2 on Linux (spec 001, D-180). +/// +/// +/// On first use the host prepares, under its own cgroup: psr-host, where it moves its own processes (cgroup v2 +/// forbids processes in a cgroup that hands controllers to its children), the memory controller turned on, and +/// psr-runs. Each attempt then gets psr-runs/<id> with memory.max, which the SDK joins before +/// importing the script. Where the cgroup cannot be written (standard containers mount it read-only), +/// returns null and the limit falls back to RLIMIT_AS with the numeric libraries limited to +/// threads. +/// +internal sealed class CgroupMemory +{ + private const string HostGroup = "psr-host"; + private const string RunsGroup = "psr-runs"; + + private readonly ILogger _logger; + private readonly IOptions _options; + private readonly string _root; + private readonly string _procSelfCgroup; + private readonly bool _linux; + private readonly Lazy _runs; + + public CgroupMemory(ILogger logger, IOptions options) + : this(logger, options, "/sys/fs/cgroup", "/proc/self/cgroup", OperatingSystem.IsLinux()) + { + } + + /// Tests: a fake cgroup file system under . + internal CgroupMemory(ILogger? logger, IOptions options, string root, string procSelfCgroup, bool linux) + { + _logger = logger ?? NullLogger.Instance; + _options = options; + _root = root; + _procSelfCgroup = procSelfCgroup; + _linux = linux; + _runs = new Lazy(Prepare, LazyThreadSafetyMode.ExecutionAndPublication); + } + + /// The cgroup holding the attempt cgroups, or null when they are not available. + public string? RunsPath => _runs.Value; + + /// Creates the cgroup of one attempt with of memory, or returns null. + public CgroupLease? Create(long limitBytes) + { + if (_runs.Value is not { } runs) + { + return null; + } + + var path = Path.Combine(runs, Guid.NewGuid().ToString("N")); + try + { + Directory.CreateDirectory(path); + Write(Path.Combine(path, "memory.max"), limitBytes.ToString(CultureInfo.InvariantCulture)); + + // Going over the limit ends in the OOM killer instead of swapping (absent without swap accounting). + TryWrite(Path.Combine(path, "memory.swap.max"), "0"); + return new CgroupLease(path); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + TryRemove(path); + return null; + } + } + + internal static void Write(string path, string value) + { + // One write(2) of the whole value: cgroup interface files take each write as a command. + using var stream = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite, bufferSize: 0); + stream.Write(Encoding.ASCII.GetBytes(value)); + } + + internal static bool TryWrite(string path, string value) + { + try + { + Write(path, value); + return true; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; + } + } + + internal static void TryRemove(string path) + { + try + { + // rmdir: a cgroup without processes goes away with its interface files. + Directory.Delete(path); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // Still has processes, or already gone. + } + } + + private string? Prepare() + { + if (!_linux) + { + return null; + } + + var threads = _options.Value.ThreadsPerProcess ?? 1; + try + { + if (OwnCgroup() is not { } own) + { + Log.ProcessMemoryLimits(_logger, "no cgroup v2", threads); + return null; + } + + if (!Words(Path.Combine(own, "cgroup.controllers")).Contains("memory")) + { + Log.ProcessMemoryLimits(_logger, "the memory controller of the cgroup is not available", threads); + return null; + } + + if (!Words(Path.Combine(own, "cgroup.subtree_control")).Contains("memory")) + { + var host = Path.Combine(own, HostGroup); + Directory.CreateDirectory(host); + foreach (var pid in File.ReadAllLines(Path.Combine(own, "cgroup.procs")).Where(l => l.Length > 0)) + { + // A process may exit meanwhile; one that cannot move makes enabling the controller fail below. + TryWrite(Path.Combine(host, "cgroup.procs"), pid); + } + + Write(Path.Combine(own, "cgroup.subtree_control"), "+memory"); + } + + var runs = Path.Combine(own, RunsGroup); + Directory.CreateDirectory(runs); + if (!Words(Path.Combine(runs, "cgroup.subtree_control")).Contains("memory")) + { + Write(Path.Combine(runs, "cgroup.subtree_control"), "+memory"); + } + + // Attempt cgroups left by a host that stopped abruptly (empty once their processes are gone). + foreach (var leftover in Directory.EnumerateDirectories(runs)) + { + TryRemove(leftover); + } + + Log.CgroupMemoryLimits(_logger, runs); + return runs; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + Log.ProcessMemoryLimits(_logger, ex.Message, threads); + return null; + } + } + + /// The cgroup of this process: cgroup v2 lists one line 0::/path. + private string? OwnCgroup() + { + var line = File.Exists(_procSelfCgroup) ? File.ReadLines(_procSelfCgroup).FirstOrDefault(l => l.StartsWith("0::", StringComparison.Ordinal)) : null; + if (line is null) + { + return null; + } + + var own = Path.Combine(_root, line[3..].TrimStart('/')); + return File.Exists(Path.Combine(own, "cgroup.controllers")) ? own : null; + } + + private static HashSet Words(string path) => + File.Exists(path) ? [.. File.ReadAllText(path).Split((char[])[' ', '\n'], StringSplitOptions.RemoveEmptyEntries)] : []; +} + +/// The cgroup v2 of one attempt: out-of-memory check, kill and removal. +internal sealed class CgroupLease(string path) : IDisposable +{ + private static readonly TimeSpan EmptyWait = TimeSpan.FromSeconds(2); + + public string Path { get; } = path; + + /// Whether the kernel killed a process of the attempt for going over memory.max. + public bool OutOfMemory() + { + try + { + return File.ReadLines(System.IO.Path.Combine(Path, "memory.events")) + .Select(l => l.Split(' ')) + .Any(p => p.Length == 2 && p[0] == "oom_kill" && long.TryParse(p[1], CultureInfo.InvariantCulture, out var count) && count > 0); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; + } + } + + /// Kills every process left in the cgroup (cgroup.kill, Linux 5.14+). + public void Kill() => CgroupMemory.TryWrite(System.IO.Path.Combine(Path, "cgroup.kill"), "1"); + + public void Dispose() + { + Kill(); + var deadline = DateTime.UtcNow + EmptyWait; + while (Populated() && DateTime.UtcNow < deadline) + { + Thread.Sleep(20); + } + + CgroupMemory.TryRemove(Path); + } + + private bool Populated() + { + try + { + return File.ReadLines(System.IO.Path.Combine(Path, "cgroup.events")).Any(l => l == "populated 1"); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return false; + } + } +} diff --git a/src/PythonScriptRunner.Core/Execution/ProcessPythonExecutor.cs b/src/PythonScriptRunner.Core/Execution/ProcessPythonExecutor.cs index de48e1c..066d914 100644 --- a/src/PythonScriptRunner.Core/Execution/ProcessPythonExecutor.cs +++ b/src/PythonScriptRunner.Core/Execution/ProcessPythonExecutor.cs @@ -2,6 +2,7 @@ using System.Diagnostics; using System.Globalization; using System.Text.Json.Nodes; +using Microsoft.Extensions.Options; using PythonScriptRunner.Core.Values; namespace PythonScriptRunner.Core.Execution; @@ -10,9 +11,14 @@ namespace PythonScriptRunner.Core.Execution; /// Runs one attempt as a separate python -I -X utf8 -m psrunner.host process with a clean environment, /// a timeout, cancellation, a memory limit and captured output (protocol version 1). /// -internal sealed class ProcessPythonExecutor(TimeProvider time) : IPythonExecutor +internal sealed class ProcessPythonExecutor(TimeProvider time, CgroupMemory cgroups, IOptions options) + : IPythonExecutor { private const int ProtocolVersion = 1; + + /// Thread pools of the numeric libraries (OpenMP, OpenBLAS, MKL, numexpr, Accelerate). + internal static readonly string[] ThreadVariables = + ["OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "NUMEXPR_NUM_THREADS", "VECLIB_MAXIMUM_THREADS"]; private static readonly TimeSpan KillWait = TimeSpan.FromSeconds(10); private static readonly TimeSpan PumpWait = TimeSpan.FromSeconds(5); @@ -32,7 +38,12 @@ await File.WriteAllTextAsync(Path.Combine(work, "input.json"), BuildInputDocumen var masker = new SecretMasker(request.Secrets.Values); var stdout = new TailBuffer(request.MaxLogBytes); var stderr = new TailBuffer(request.MaxLogBytes); - using var process = new Process { StartInfo = BuildStartInfo(request, scriptPath) }; + // Linux: a cgroup v2 with the real memory limit when the host can create one; otherwise RLIMIT_AS, which counts + // reserved memory, so the numeric libraries get one thread each by default (OpenBLAS reserves memory per thread). + using var cgroup = cgroups.Create(request.MemoryLimitMb * 1024L * 1024L); + var configured = options.Value.ThreadsPerProcess; + var threads = configured ?? (cgroup is null && OperatingSystem.IsLinux() ? 1 : null); + using var process = new Process { StartInfo = BuildStartInfo(request, scriptPath, cgroup, threads, configured ?? 1) }; var job = OperatingSystem.IsWindows() ? new WindowsJobObject(request.MemoryLimitMb * 1024L * 1024L) : null; try { @@ -67,6 +78,7 @@ await File.WriteAllTextAsync(Path.Combine(work, "input.json"), BuildInputDocumen catch (OperationCanceledException) { stoppedAs = cancellationToken.IsCancellationRequested ? AttemptStatus.Cancelled : AttemptStatus.TimedOut; + cgroup?.Kill(); await KillAsync(process, job).ConfigureAwait(false); } } @@ -90,13 +102,22 @@ await File.WriteAllTextAsync(Path.Combine(work, "input.json"), BuildInputDocumen var stdoutTail = stdout.ToLogTail(masker); var stderrTail = stderr.ToLogTail(masker); var exitCode = process.HasExited ? process.ExitCode : -1; - return stoppedAs switch + var result = stoppedAs switch { AttemptStatus.Cancelled => Stopped(ErrorKind.Cancelled, "The run was cancelled.", AttemptStatus.Cancelled), AttemptStatus.TimedOut => Stopped(ErrorKind.Timeout, $"The attempt exceeded its timeout of {request.Timeout}.", AttemptStatus.TimedOut), _ => OutputReader.Read(request, exitCode, processDuration, stdoutTail, stderrTail), }; + // With a cgroup the kernel kills the process over memory.max (no MemoryError reaches the SDK). + return stoppedAs is null && result.Status == AttemptStatus.Failed && cgroup?.OutOfMemory() == true + ? result with + { + Error = ExecutionResult.Failure( + ErrorKind.MemoryLimitExceeded, $"The attempt exceeded its memory limit of {request.MemoryLimitMb} MB.", "OUT_OF_MEMORY").Error, + } + : result; + ExecutionResult Stopped(ErrorKind kind, string message, AttemptStatus status) => ExecutionResult.Failure(kind, message, status: status) with { @@ -218,7 +239,7 @@ private static async Task ResolveScriptAsync(ExecutionRequest request, C } } - private static ProcessStartInfo BuildStartInfo(ExecutionRequest request, string scriptPath) + private static ProcessStartInfo BuildStartInfo(ExecutionRequest request, string scriptPath, CgroupLease? cgroup, int? threads, int fallbackThreads) { var work = request.WorkDirectory; var temp = Path.Combine(work, "tmp"); @@ -259,6 +280,20 @@ private static ProcessStartInfo BuildStartInfo(ExecutionRequest request, string environment["PSR_WORK_DIR"] = work; environment["PSR_SCRIPT"] = scriptPath; environment["PSR_MEMORY_LIMIT_MB"] = request.MemoryLimitMb.ToString(CultureInfo.InvariantCulture); + if (cgroup is not null) + { + // The SDK joins it before importing the script; if it cannot, it falls back like a host without cgroups. + environment["PSR_CGROUP"] = cgroup.Path; + environment["PSR_FALLBACK_THREADS"] = fallbackThreads.ToString(CultureInfo.InvariantCulture); + } + + if (threads is { } count) + { + foreach (var name in ThreadVariables) + { + environment[name] = count.ToString(CultureInfo.InvariantCulture); + } + } foreach (var (name, value) in request.Secrets) { environment[$"PSR_SECRET_{name}"] = value; diff --git a/src/PythonScriptRunner.Core/PythonScriptRunnerOptions.cs b/src/PythonScriptRunner.Core/PythonScriptRunnerOptions.cs index 6ac3086..cfbd011 100644 --- a/src/PythonScriptRunner.Core/PythonScriptRunnerOptions.cs +++ b/src/PythonScriptRunner.Core/PythonScriptRunnerOptions.cs @@ -7,8 +7,10 @@ public sealed class PythonScriptRunnerOptions public string WorkRoot { get; set; } = Path.Combine(Path.GetTempPath(), "psr"); /// Folder where Python environments (venvs) and lock files are cached. + /// The default only names the folder: it is created when the first environment needs it (a container with a + /// read-only file system sets another root and must not fail on the default). public string EnvironmentCacheRoot { get; set; } = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData, Environment.SpecialFolderOption.Create), "psr"); + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData, Environment.SpecialFolderOption.DoNotVerify), "psr"); /// Path or name of the uv executable. Default: uv (searched in PATH). public string UvPath { get; set; } = "uv"; @@ -34,6 +36,13 @@ public sealed class PythonScriptRunnerOptions /// Maximum number of Python processes running at the same time. Default: processor count. public int MaxConcurrentProcesses { get; set; } = Environment.ProcessorCount; + /// + /// Threads of the numeric libraries (OpenMP, OpenBLAS, MKL, numexpr) in each Python process. Default: null — + /// 1 on Linux when the memory limit falls back to RLIMIT_AS (no delegable cgroup v2, as in a standard container: + /// OpenBLAS reserves memory per thread), no limit otherwise. A value applies to every process (spec 001, D-180). + /// + public int? ThreadsPerProcess { get; set; } + /// Package index (pip --index-url). Default: PyPI. public string? PackageIndexUrl { get; set; } diff --git a/src/PythonScriptRunner.Dashboard/PythonScriptRunner.Dashboard.csproj b/src/PythonScriptRunner.Dashboard/PythonScriptRunner.Dashboard.csproj index 7b28a6c..b142205 100644 --- a/src/PythonScriptRunner.Dashboard/PythonScriptRunner.Dashboard.csproj +++ b/src/PythonScriptRunner.Dashboard/PythonScriptRunner.Dashboard.csproj @@ -26,7 +26,8 @@ $([MSBuild]::NormalizeDirectory('$(MSBuildThisFileDirectory)', '..', '..', 'panel')) @@ -38,7 +39,7 @@ - + diff --git a/src/PythonScriptRunner.EntityFrameworkCore/PsrDbContextOptions.cs b/src/PythonScriptRunner.EntityFrameworkCore/PsrDbContextOptions.cs index 49a8083..ce483bc 100644 --- a/src/PythonScriptRunner.EntityFrameworkCore/PsrDbContextOptions.cs +++ b/src/PythonScriptRunner.EntityFrameworkCore/PsrDbContextOptions.cs @@ -1,3 +1,5 @@ +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.Extensions.Logging; using Npgsql.EntityFrameworkCore.PostgreSQL.Infrastructure; namespace PythonScriptRunner.EntityFrameworkCore; @@ -7,4 +9,12 @@ internal static class PsrDbContextOptions { public static void ConfigureNpgsql(NpgsqlDbContextOptionsBuilder npgsql) => npgsql.MigrationsHistoryTable(PsrDbContext.MigrationsHistoryTable, PsrDbContext.Schema); + + /// + /// The outbox relay polls every 250 ms and the worker every few seconds (D-58), so each executed command is logged at + /// Debug instead of Information: otherwise the host's log fills with the same queries (D-187). Failed + /// commands are still logged as errors. + /// + public static void ConfigureWarnings(WarningsConfigurationBuilder warnings) => + warnings.Log((RelationalEventId.CommandExecuted, LogLevel.Debug)); } diff --git a/src/PythonScriptRunner.EntityFrameworkCore/PsrEntityFrameworkCoreExtensions.cs b/src/PythonScriptRunner.EntityFrameworkCore/PsrEntityFrameworkCoreExtensions.cs index 5f07ac7..069d553 100644 --- a/src/PythonScriptRunner.EntityFrameworkCore/PsrEntityFrameworkCoreExtensions.cs +++ b/src/PythonScriptRunner.EntityFrameworkCore/PsrEntityFrameworkCoreExtensions.cs @@ -33,8 +33,9 @@ public static IPythonScriptRunnerBuilder AddEntityFrameworkCore(this IPythonScri var services = builder.Services; services.AddSingleton(options); services.TryAddSingleton(); - services.AddPooledDbContextFactory((provider, db) => - db.UseNpgsql(provider.GetRequiredService().DataSource, PsrDbContextOptions.ConfigureNpgsql)); + services.AddPooledDbContextFactory((provider, db) => db + .UseNpgsql(provider.GetRequiredService().DataSource, PsrDbContextOptions.ConfigureNpgsql) + .ConfigureWarnings(PsrDbContextOptions.ConfigureWarnings)); services.TryAddSingleton(); services.Replace(ServiceDescriptor.Singleton(provider => provider.GetRequiredService())); diff --git a/tests/PythonScriptRunner.AppHost.Tests/AppHostTests.cs b/tests/PythonScriptRunner.AppHost.Tests/AppHostTests.cs new file mode 100644 index 0000000..a90cf2c --- /dev/null +++ b/tests/PythonScriptRunner.AppHost.Tests/AppHostTests.cs @@ -0,0 +1,88 @@ +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json.Nodes; +using PythonScriptRunner.Tests.Infrastructure; + +namespace PythonScriptRunner.AppHost.Tests; + +/// +/// Spec 009, AC-1 and AC-2: the Aspire AppHost brings the whole environment up, and the fixed user autor imports the +/// fixtures and runs CS-1 and CS-2 through the dashboard API, with the e-mail arriving in Mailpit. +/// +public sealed class AppHostTests +{ + private static readonly TimeSpan StartTimeout = TimeSpan.FromMinutes(5); + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task Spec009_AC01_AC02_TheAppHostBringsEverythingUp_AndTheAuthorRunsCs1AndCs2() + { + var appHost = await DistributedApplicationTestingBuilder.CreateAsync( + [], (options, _) => options.DisableDashboard = false, Ct); + + // A fresh database each run: the persistent volume is for developers, not for tests. + foreach (var container in appHost.Resources.OfType()) + { + foreach (var volume in container.Annotations.OfType().Where(m => m.Type == ContainerMountType.Volume).ToList()) + { + container.Annotations.Remove(volume); + } + } + + await using var app = await appHost.BuildAsync(Ct); + await app.StartAsync(Ct); + + // AC-1: every resource is up, and the hosts send their telemetry to the Aspire dashboard. + using (var start = CancellationTokenSource.CreateLinkedTokenSource(Ct)) + { + start.CancelAfter(StartTimeout); + await Task.WhenAll( + app.ResourceNotifications.WaitForResourceHealthyAsync("worker", start.Token), + app.ResourceNotifications.WaitForResourceHealthyAsync("dashboard", start.Token), + app.ResourceNotifications.WaitForResourceAsync("mailpit", KnownResourceStates.Running, start.Token), + app.ResourceNotifications.WaitForResourceAsync("panel", KnownResourceStates.Running, start.Token)); + } + + foreach (var name in new[] { "worker", "dashboard" }) + { + var running = await app.ResourceNotifications.WaitForResourceAsync(name, e => !e.Snapshot.EnvironmentVariables.IsDefaultOrEmpty, Ct); + running.Snapshot.EnvironmentVariables.Select(v => v.Name).ShouldContain("OTEL_EXPORTER_OTLP_ENDPOINT", $"{name} sends telemetry to the dashboard"); + } + + // AC-2: the fixed user autor imports the fixtures, publishes them and runs CS-1 and CS-2. + using var http = await PanelApi.LoginAsync(app.GetEndpoint("dashboard", "http")); + var smtp = app.GetEndpoint("mailpit", "smtp"); + await PanelApi.ImportAndPublishAsync(http, "calculo-progressao"); + var runsCs2 = !OperatingSystem.IsWindows(); + if (runsCs2) + { + await PanelApi.ImportAndPublishAsync(http, "html-para-email", (path, content) => path.EndsWith(".pipeline.json", StringComparison.Ordinal) + ? content.Replace("smtp.exemplo.com", smtp.Host, StringComparison.Ordinal) + .Replace("\"const\": 465", $"\"const\": {smtp.Port}", StringComparison.Ordinal) + : content); + } + + var cs1 = await PanelApi.RunAsync(http, "calculo-progressao", JsonContent.Create(new + { + inputs = new { a1 = 1, razao = 2, n = 3, coeficientes = new[] { 0, 1 } }, + })); + (await PanelApi.WaitForSuccessAsync(http, cs1))["outputs"]!["soma"]!.GetValue().ShouldBe(9); + + // CS-2 needs weasyprint (GTK/pango), which runs on Linux only (D-106). + if (runsCs2) + { + using var form = new MultipartFormDataContent(); + form.Add(new StringContent("""{"destinatarios": ["financeiro@exemplo.com"], "assunto": "Relatório do Aspire"}"""), "inputs"); + var html = new ByteArrayContent(Encoding.UTF8.GetBytes("

Relatório

")); + html.Headers.ContentType = new MediaTypeHeaderValue("text/html"); + form.Add(html, "html", "pagina.html"); + await PanelApi.WaitForSuccessAsync(http, await PanelApi.RunAsync(http, "html-para-email", form)); + + using var mailpit = app.CreateHttpClient("mailpit", "http"); + var messages = (await mailpit.GetFromJsonAsync("/api/v1/messages", Ct))!["messages"]!.AsArray(); + messages.Select(m => m!["Subject"]!.GetValue()).ShouldContain("Relatório do Aspire"); + } + } +} diff --git a/tests/PythonScriptRunner.AppHost.Tests/Infrastructure/PanelApi.cs b/tests/PythonScriptRunner.AppHost.Tests/Infrastructure/PanelApi.cs new file mode 100644 index 0000000..ed079ed --- /dev/null +++ b/tests/PythonScriptRunner.AppHost.Tests/Infrastructure/PanelApi.cs @@ -0,0 +1,128 @@ +using System.IO.Compression; +using System.Net; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json.Nodes; + +namespace PythonScriptRunner.Tests.Infrastructure; + +/// +/// The dashboard API as an author uses it (spec 007): fixed login, import of a test fixture, publication and runs. Shared +/// by the Aspire AppHost tests and the docker-compose test (spec 009). +/// +public static class PanelApi +{ + public const string DevPassword = "psr-dev"; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + /// A client of the dashboard at , signed in with the fixed login. + public static async Task LoginAsync(Uri baseAddress, string name = "autor", string password = DevPassword) + { + var http = new HttpClient(new HttpClientHandler { CookieContainer = new CookieContainer() }) { BaseAddress = baseAddress }; + http.DefaultRequestHeaders.Add("X-PSR-Request", "1"); + var login = await http.PostAsJsonAsync("/psr/api/auth/login", new { name, password }, Ct); + login.StatusCode.ShouldBe(HttpStatusCode.NoContent, await login.Content.ReadAsStringAsync(Ct)); + return http; + } + + /// Imports a fixture of tests/fixtures as a zip (drafts), then publishes its stages and pipelines. + public static async Task ImportAndPublishAsync(HttpClient http, string fixture, Func? transform = null) + { + var root = Path.Combine(RepositoryRoot(), "tests", "fixtures", fixture); + var files = new Dictionary(); + foreach (var file in Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories).Where(f => !f.Contains("__pycache__", StringComparison.Ordinal))) + { + var relative = Path.GetRelativePath(root, file).Replace('\\', '/'); + var content = await File.ReadAllTextAsync(file, Ct); + files[relative] = transform is null ? content : transform(relative, content); + } + + await ImportAndPublishAsync(http, fixture, files); + } + + /// Imports files in the folder format (stages/<key>/…, pipelines/…) and publishes them. + public static async Task ImportAndPublishAsync(HttpClient http, string name, IReadOnlyDictionary files) + { + using var zip = new MemoryStream(); + using (var archive = new ZipArchive(zip, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var (path, content) in files) + { + await using var writer = new StreamWriter(archive.CreateEntry(path).Open()); + await writer.WriteAsync(content); + } + } + + using var form = new MultipartFormDataContent(); + var archiveContent = new ByteArrayContent(zip.ToArray()); + archiveContent.Headers.ContentType = new MediaTypeHeaderValue("application/zip"); + form.Add(archiveContent, "archive", $"{name}.zip"); + var import = await http.PostAsync("/psr/api/import", form, Ct); + import.StatusCode.ShouldBe(HttpStatusCode.OK, await import.Content.ReadAsStringAsync(Ct)); + var items = (await import.Content.ReadFromJsonAsync(Ct))!["items"]!.AsArray(); + + // Stages first: the pipeline references their published versions. + foreach (var item in items.OrderBy(i => i!["kind"]!.GetValue() == "Stage" ? 0 : 1)) + { + var kind = item!["kind"]!.GetValue() == "Stage" ? "stages" : "pipelines"; + var key = item["key"]!.GetValue(); + var draft = await http.GetAsync($"/psr/api/{kind}/{key}/draft", Ct); + using var publish = new HttpRequestMessage(HttpMethod.Post, $"/psr/api/{kind}/{key}/draft/publish"); + publish.Headers.IfMatch.Add(draft.Headers.ETag!); + var published = await http.SendAsync(publish, Ct); + published.StatusCode.ShouldBe(HttpStatusCode.OK, await published.Content.ReadAsStringAsync(Ct)); + } + } + + /// Starts a pipeline run (JSON or multipart body) and returns its id. + public static Task RunAsync(HttpClient http, string pipeline, HttpContent body) => StartAsync(http, $"pipelines/{pipeline}", body); + + /// Starts a run of a single stage and returns its id. + public static Task RunStageAsync(HttpClient http, string stage, HttpContent body) => StartAsync(http, $"stages/{stage}", body); + + /// Waits until the run ends (up to , default 5 min) and returns it, whatever the status. + public static async Task WaitForEndAsync(HttpClient http, string runId, TimeSpan? timeout = null) + { + using var limit = CancellationTokenSource.CreateLinkedTokenSource(Ct); + limit.CancelAfter(timeout ?? TimeSpan.FromMinutes(5)); + while (true) + { + var run = (await http.GetFromJsonAsync($"/psr/api/runs/{runId}", limit.Token))!; + if (run["status"]!.GetValue() is "Succeeded" or "Failed" or "Cancelled" or "TimedOut") + { + return run; + } + + await Task.Delay(TimeSpan.FromSeconds(1), limit.Token); + } + } + + private static async Task StartAsync(HttpClient http, string target, HttpContent body) + { + var response = await http.PostAsync($"/psr/api/{target}/runs", body, Ct); + response.IsSuccessStatusCode.ShouldBeTrue(await response.Content.ReadAsStringAsync(Ct)); + return (await response.Content.ReadFromJsonAsync(Ct))!["runId"]!.GetValue(); + } + + /// Waits until the run ends (up to , default 5 min) and checks it succeeded. + public static async Task WaitForSuccessAsync(HttpClient http, string runId, TimeSpan? timeout = null) + { + var run = await WaitForEndAsync(http, runId, timeout); + run["status"]!.GetValue().ShouldBe("Succeeded", run["error"]?.ToJsonString()); + return run; + } + + public static string RepositoryRoot() + { + for (var directory = new DirectoryInfo(AppContext.BaseDirectory); directory is not null; directory = directory.Parent) + { + if (File.Exists(Path.Combine(directory.FullName, "PythonScriptRunner.slnx"))) + { + return directory.FullName; + } + } + + throw new InvalidOperationException("The repository root (PythonScriptRunner.slnx) was not found."); + } +} diff --git a/tests/PythonScriptRunner.AppHost.Tests/PythonScriptRunner.AppHost.Tests.csproj b/tests/PythonScriptRunner.AppHost.Tests/PythonScriptRunner.AppHost.Tests.csproj new file mode 100644 index 0000000..9a09815 --- /dev/null +++ b/tests/PythonScriptRunner.AppHost.Tests/PythonScriptRunner.AppHost.Tests.csproj @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/PythonScriptRunner.Containers.Tests/CgroupTests.cs b/tests/PythonScriptRunner.Containers.Tests/CgroupTests.cs new file mode 100644 index 0000000..ae1b423 --- /dev/null +++ b/tests/PythonScriptRunner.Containers.Tests/CgroupTests.cs @@ -0,0 +1,74 @@ +using System.Net.Http.Json; +using PythonScriptRunner.Containers.Tests.Infrastructure; +using PythonScriptRunner.Tests.Infrastructure; + +namespace PythonScriptRunner.Containers.Tests; + +/// +/// Spec 001, AC-30 (D-180): with a writable cgroup v2 (a privileged container) the memory limit of an attempt is the +/// memory really used, enforced by the kernel, and the numeric libraries keep their threads. +/// +public sealed class CgroupTests(HostStack stack) +{ + private static readonly Dictionary Stages = new() + { + ["stages/consome-memoria/stage.json"] = """ + {"key": "consome-memoria", "version": 1, "name": "Consome memória", "memoryLimitMb": 128, + "inputs": [], "outputs": [{"name": "tamanho", "type": "integer"}]} + """, + ["stages/consome-memoria/script.py"] = """ + from psrunner import stage + + + @stage + def run(inputs, ctx): + data = bytearray(512 * 1024 * 1024) + data[::4096] = b"\x01" * (len(data) // 4096) + return {"tamanho": len(data)} + """, + ["stages/threads-numpy/stage.json"] = """ + {"key": "threads-numpy", "version": 1, "name": "Threads do numpy", + "inputs": [], "outputs": [{"name": "threads", "type": "string"}]} + """, + ["stages/threads-numpy/requirements.txt"] = "numpy\n", + ["stages/threads-numpy/script.py"] = """ + import os + + import numpy + from psrunner import stage + + + @stage + def run(inputs, ctx): + numpy.ones(3) + return {"threads": os.environ.get("OPENBLAS_NUM_THREADS", "")} + """, + }; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task Spec001_AC30_InAPrivilegedContainer_TheCgroupEnforcesTheMemoryLimit_AndNumpyKeepsItsThreads() + { + var database = await stack.CreateDatabaseAsync("ac30"); + await using var dashboard = stack.Host(stack.DashboardImage, database, HostStack.DashboardUsers()); + await dashboard.StartAsync(Ct); + await using var worker = stack.Host(stack.WorkerImage, database, privileged: true); + await worker.StartAsync(Ct); + using var http = await PanelApi.LoginAsync( + new Uri($"http://{dashboard.Hostname}:{dashboard.GetMappedPublicPort(HostStack.HttpPort)}"), "admin", HostStack.Password); + await PanelApi.ImportAndPublishAsync(http, "ac30", Stages); + var noInputs = JsonContent.Create(new { inputs = new { } }); + + var greedy = await PanelApi.WaitForEndAsync(http, await PanelApi.RunStageAsync(http, "consome-memoria", noInputs), TimeSpan.FromMinutes(8)); + var numpy = await PanelApi.WaitForSuccessAsync(http, await PanelApi.RunStageAsync(http, "threads-numpy", noInputs), TimeSpan.FromMinutes(8)); + + var (stdout, _) = await worker.GetLogsAsync(ct: Ct); + var memoryLimits = stdout.Split('\n').Where(l => l.Contains("Memory limits", StringComparison.Ordinal)); + stdout.ShouldContain("cgroup v2", customMessage: string.Join(Environment.NewLine, memoryLimits)); + greedy["status"]!.GetValue().ShouldBe("Failed"); + greedy["error"]!["kind"]!.GetValue().ShouldBe("MemoryLimitExceeded"); + greedy["error"]!["code"]!.GetValue().ShouldBe("OUT_OF_MEMORY", "the kernel enforced memory.max of the attempt cgroup"); + numpy["outputs"]!["threads"]!.GetValue().ShouldBe(string.Empty, "with a cgroup the numeric libraries keep their threads"); + } +} diff --git a/tests/PythonScriptRunner.Containers.Tests/ComposeTests.cs b/tests/PythonScriptRunner.Containers.Tests/ComposeTests.cs new file mode 100644 index 0000000..5eec07d --- /dev/null +++ b/tests/PythonScriptRunner.Containers.Tests/ComposeTests.cs @@ -0,0 +1,125 @@ +using System.Diagnostics; +using System.Net; +using System.Net.Http.Json; +using System.Net.Sockets; +using PythonScriptRunner.Containers.Tests.Infrastructure; +using PythonScriptRunner.Tests.Infrastructure; + +namespace PythonScriptRunner.Containers.Tests; + +/// Spec 009, AC-3: the docker-compose generated by aspire publish runs CS-1 with the host images. +public sealed class ComposeTests(HostStack stack) +{ + private const string Version = "compose-test"; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task Spec009_AC03_TheComposeFromAspirePublish_RunsCs1() + { + var aspire = Environment.GetEnvironmentVariable("PSR_ASPIRE_CLI") ?? "aspire"; + Assert.SkipUnless(await ProbeAsync(aspire), $"Aspire CLI not found ('{aspire}'); install Aspire.Cli or set PSR_ASPIRE_CLI."); + var root = PanelApi.RepositoryRoot(); + var output = Directory.CreateTempSubdirectory("psr-compose-").FullName; + var (published, publishLog) = await RunAsync(aspire, root, + "publish", "--apphost", Path.Combine(root, "hosts", "PythonScriptRunner.AppHost", "PythonScriptRunner.AppHost.csproj"), + "-o", output, "--non-interactive", "--nologo"); + published.ShouldBe(0, publishLog); + + // The compose uses ghcr.io/scriptrunnerkit/psr-*:${PSR_VERSION}; the images under test take that name locally. + foreach (var (image, name) in new[] { (stack.WorkerImage, "psr-worker"), (stack.DashboardImage, "psr-dashboard") }) + { + (await HostImages.DockerAsync(["tag", image, $"ghcr.io/scriptrunnerkit/{name}:{Version}"], Ct)).Code.ShouldBe(0); + } + + var port = FreePort(); + await File.WriteAllTextAsync(Path.Combine(output, ".env"), $""" + POSTGRES_PASSWORD={Guid.NewGuid():N} + RABBITMQ_PASSWORD={Guid.NewGuid():N} + PSR_VERSION={Version} + PSR_DASHBOARD_PORT={port} + """, Ct); + var project = $"psr-ac03-{Guid.NewGuid():N}"[..20]; + try + { + var (up, upLog) = await RunAsync("docker", output, "compose", "-p", project, "up", "-d"); + up.ShouldBe(0, upLog); + var dashboard = new Uri($"http://localhost:{port}"); + await WaitUntilReadyAsync(dashboard); + + using var http = await PanelApi.LoginAsync(dashboard); + await PanelApi.ImportAndPublishAsync(http, "calculo-progressao"); + var run = await PanelApi.RunAsync(http, "calculo-progressao", JsonContent.Create(new + { + inputs = new { a1 = 1, razao = 2, n = 3, coeficientes = new[] { 0, 1 } }, + })); + (await PanelApi.WaitForSuccessAsync(http, run, TimeSpan.FromMinutes(8)))["outputs"]!["soma"]!.GetValue().ShouldBe(9); + } + finally + { + await RunAsync("docker", output, "compose", "-p", project, "down", "-v"); + } + } + + private static async Task WaitUntilReadyAsync(Uri dashboard) + { + using var http = new HttpClient { BaseAddress = dashboard, Timeout = TimeSpan.FromSeconds(5) }; + var deadline = Stopwatch.StartNew(); + while (deadline.Elapsed < TimeSpan.FromMinutes(3)) + { + try + { + if ((await http.GetAsync("/health/ready", Ct)).StatusCode == HttpStatusCode.OK) + { + return; + } + } + catch (HttpRequestException) + { + // Still starting (or restarting until PostgreSQL accepts connections). + } + catch (TaskCanceledException) when (!Ct.IsCancellationRequested) + { + // Request timeout while starting. + } + + await Task.Delay(TimeSpan.FromSeconds(2), Ct); + } + + throw new TimeoutException("The dashboard of the compose did not become ready in 3 minutes."); + } + + private static async Task ProbeAsync(string aspire) + { + try + { + return (await RunAsync(aspire, Environment.CurrentDirectory, "--version")).Code == 0; + } + catch (System.ComponentModel.Win32Exception) + { + return false; + } + } + + private static async Task<(int Code, string Output)> RunAsync(string file, string directory, params string[] arguments) + { + var start = new ProcessStartInfo(file) { WorkingDirectory = directory, RedirectStandardOutput = true, RedirectStandardError = true }; + foreach (var argument in arguments) + { + start.ArgumentList.Add(argument); + } + + using var process = Process.Start(start)!; + var stdout = process.StandardOutput.ReadToEndAsync(Ct); + var stderr = process.StandardError.ReadToEndAsync(Ct); + await process.WaitForExitAsync(Ct); + return (process.ExitCode, await stdout + await stderr); + } + + private static int FreePort() + { + using var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + return ((IPEndPoint)listener.LocalEndpoint).Port; + } +} diff --git a/tests/PythonScriptRunner.Containers.Tests/HostImageTests.cs b/tests/PythonScriptRunner.Containers.Tests/HostImageTests.cs new file mode 100644 index 0000000..f7de94a --- /dev/null +++ b/tests/PythonScriptRunner.Containers.Tests/HostImageTests.cs @@ -0,0 +1,74 @@ +using System.Net; +using System.Reflection; +using Microsoft.EntityFrameworkCore.Migrations; +using PythonScriptRunner.Containers.Tests.Infrastructure; +using PythonScriptRunner.EntityFrameworkCore; + +namespace PythonScriptRunner.Containers.Tests; + +/// Spec 009: the worker and dashboard images as they run in production. +public sealed class HostImageTests(HostStack stack) +{ + private const long MaxWorkerImageBytes = 400L * 1024 * 1024; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task Spec009_AC07_BothImages_RunAsNonRootOnAReadOnlyFileSystem_AndAnswerTheHealthChecks() + { + var database = await stack.CreateDatabaseAsync("ac07"); + await using var worker = stack.Host(stack.WorkerImage, database); + await using var dashboard = stack.Host(stack.DashboardImage, database, HostStack.DashboardUsers()); + + await worker.StartAsync(Ct); + await dashboard.StartAsync(Ct); + + foreach (var host in new[] { worker, dashboard }) + { + using var http = Client(host); + (await http.GetAsync("/health/live", Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + (await http.GetAsync("/health/ready", Ct)).StatusCode.ShouldBe(HttpStatusCode.OK); + (await host.ExecAsync(["id", "-u"], Ct)).Stdout.Trim().ShouldBe("10001"); + (await host.ExecAsync(["sh", "-c", "touch /app/escrita"], Ct)).ExitCode.ShouldNotBe(0, "the root file system is read-only"); + + // The psrunner wheel of the same commit (D-169) is what the host installs in each environment. + var environment = (await host.ExecAsync(["sh", "-c", "tr '\\0' '\\n' < /proc/1/environ"], Ct)).Stdout; + environment.ShouldContain("PythonScriptRunner__PsrunnerRequirement=/opt/psr/psrunner-"); + } + + using var panel = Client(dashboard); + (await panel.GetStringAsync("/psr/", Ct)).ShouldContain("id=\"app\"", customMessage: "the image embeds the built panel"); + } + + [Fact] + public async Task Spec009_AC08_TwoWorkersWithAutoMigrateStartingTogether_ApplyTheMigrationsOnce() + { + var database = await stack.CreateDatabaseAsync("ac08"); + await using var first = stack.Host(stack.WorkerImage, database); + await using var second = stack.Host(stack.WorkerImage, database); + + await Task.WhenAll(first.StartAsync(Ct), second.StartAsync(Ct)); + + var migrations = typeof(PsrDbContext).Assembly.GetTypes().Count(t => t.GetCustomAttribute() is not null); + migrations.ShouldBeGreaterThan(0); + var applied = await stack.ScalarAsync(database, $"SELECT count(*) FROM {PsrDbContext.Schema}.{PsrDbContext.MigrationsHistoryTable}"); + applied.ShouldBe(migrations.ToString(System.Globalization.CultureInfo.InvariantCulture)); + foreach (var worker in new[] { first, second }) + { + var (stdout, stderr) = await worker.GetLogsAsync(ct: Ct); + (stdout + stderr).ShouldNotContain("Hosting failed to start"); + } + } + + [Fact] + public async Task Spec009_WorkerImage_IsAtMost400MBWithoutPythonInterpreters() + { + var (code, output) = await HostImages.DockerAsync(["image", "inspect", "--format", "{{.Size}}", stack.WorkerImage], Ct); + + code.ShouldBe(0, output); + long.Parse(output.Trim(), System.Globalization.CultureInfo.InvariantCulture).ShouldBeLessThanOrEqualTo(MaxWorkerImageBytes); + } + + private static HttpClient Client(IContainer host) => + new() { BaseAddress = new Uri($"http://{host.Hostname}:{host.GetMappedPublicPort(HostStack.HttpPort)}") }; +} diff --git a/tests/PythonScriptRunner.Containers.Tests/Infrastructure/HostImages.cs b/tests/PythonScriptRunner.Containers.Tests/Infrastructure/HostImages.cs new file mode 100644 index 0000000..7a73cfb --- /dev/null +++ b/tests/PythonScriptRunner.Containers.Tests/Infrastructure/HostImages.cs @@ -0,0 +1,60 @@ +using System.Diagnostics; + +namespace PythonScriptRunner.Containers.Tests.Infrastructure; + +/// +/// The host images under test (spec 009): PSR_TEST_WORKER_IMAGE / PSR_TEST_DASHBOARD_IMAGE when the CI already +/// built them, otherwise built here from the Dockerfiles with docker build (BuildKit). +/// +public static class HostImages +{ + public const string Worker = "worker"; + + public const string Dashboard = "dashboard"; + + private static readonly string Root = FindRoot(); + + public static async Task ResolveAsync(string host, CancellationToken cancellationToken) + { + var variable = host == Worker ? "PSR_TEST_WORKER_IMAGE" : "PSR_TEST_DASHBOARD_IMAGE"; + if (Environment.GetEnvironmentVariable(variable) is { Length: > 0 } image) + { + return image; + } + + var tag = $"psr-{host}:test"; + var project = host == Worker ? "PythonScriptRunner.WorkerHost" : "PythonScriptRunner.DashboardHost"; + var (code, output) = await DockerAsync(["build", "-f", Path.Combine("hosts", project, "Dockerfile"), "-t", tag, "."], cancellationToken); + code.ShouldBe(0, $"docker build of {tag} failed:\n{output}"); + return tag; + } + + /// Runs the docker CLI in the repository root and returns the exit code with stdout and stderr. + public static async Task<(int Code, string Output)> DockerAsync(IReadOnlyList arguments, CancellationToken cancellationToken) + { + var start = new ProcessStartInfo("docker") { WorkingDirectory = Root, RedirectStandardOutput = true, RedirectStandardError = true }; + foreach (var argument in arguments) + { + start.ArgumentList.Add(argument); + } + + using var process = Process.Start(start)!; + var stdout = process.StandardOutput.ReadToEndAsync(cancellationToken); + var stderr = process.StandardError.ReadToEndAsync(cancellationToken); + await process.WaitForExitAsync(cancellationToken); + return (process.ExitCode, await stdout + await stderr); + } + + private static string FindRoot() + { + for (var directory = new DirectoryInfo(AppContext.BaseDirectory); directory is not null; directory = directory.Parent) + { + if (File.Exists(Path.Combine(directory.FullName, "PythonScriptRunner.slnx"))) + { + return directory.FullName; + } + } + + throw new InvalidOperationException("The repository root (PythonScriptRunner.slnx) was not found."); + } +} diff --git a/tests/PythonScriptRunner.Containers.Tests/Infrastructure/HostStack.cs b/tests/PythonScriptRunner.Containers.Tests/Infrastructure/HostStack.cs new file mode 100644 index 0000000..2769a6f --- /dev/null +++ b/tests/PythonScriptRunner.Containers.Tests/Infrastructure/HostStack.cs @@ -0,0 +1,123 @@ +using DotNet.Testcontainers.Networks; +using PythonScriptRunner.Dashboard; +using Testcontainers.PostgreSql; +using Testcontainers.RabbitMq; + +[assembly: AssemblyFixture(typeof(PythonScriptRunner.Containers.Tests.Infrastructure.HostStack))] + +namespace PythonScriptRunner.Containers.Tests.Infrastructure; + +/// +/// PostgreSQL and RabbitMQ on a private network (aliases pg and rabbit) and the two host images; each test +/// starts its hosts against its own database, the way they run in production (read-only file system, volumes as tmpfs). +/// +public sealed class HostStack : IAsyncLifetime +{ + public const int HttpPort = 8080; + + public const string Password = "senha-de-teste"; + + private static readonly string[] Volumes = ["/var/lib/psr/envs", "/var/lib/psr/python", "/tmp/psr"]; + + private readonly INetwork _network = new NetworkBuilder().Build(); + private PostgreSqlContainer? _postgres; + private RabbitMqContainer? _rabbit; + + public string WorkerImage { get; private set; } = string.Empty; + + public string DashboardImage { get; private set; } = string.Empty; + + public PostgreSqlContainer Postgres => _postgres!; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + public async ValueTask InitializeAsync() + { + await _network.CreateAsync(Ct); + _postgres = new PostgreSqlBuilder("postgres:16-alpine") + .WithNetwork(_network).WithNetworkAliases("pg") + .WithUsername("psr").WithPassword("psr").WithDatabase("psr") + .Build(); + _rabbit = new RabbitMqBuilder("rabbitmq:3.13-alpine") + .WithNetwork(_network).WithNetworkAliases("rabbit") + .WithUsername("psr").WithPassword("psr") + .Build(); + var images = Task.WhenAll(HostImages.ResolveAsync(HostImages.Worker, Ct), HostImages.ResolveAsync(HostImages.Dashboard, Ct)); + await Task.WhenAll(_postgres.StartAsync(Ct), _rabbit.StartAsync(Ct), images); + (WorkerImage, DashboardImage) = (images.Result[0], images.Result[1]); + } + + /// Creates an empty database for one test. + public async Task CreateDatabaseAsync(string name) + { + var result = await Postgres.ExecAsync(["psql", "-U", "psr", "-d", "psr", "-c", $"CREATE DATABASE {name}"], Ct); + result.ExitCode.ShouldBe(0, result.Stderr); + return name; + } + + /// Runs a query in a database and returns the single value. + public async Task ScalarAsync(string database, string sql) + { + var result = await Postgres.ExecAsync(["psql", "-U", "psr", "-d", database, "-tAc", sql], Ct); + result.ExitCode.ShouldBe(0, result.Stderr); + return result.Stdout.Trim(); + } + + /// + /// A host container (not started) with migrations on, waiting until /health/ready answers 200; + /// runs it privileged and as root, so it owns a writable /sys/fs/cgroup: a + /// delegable cgroup v2 (D-180), like a systemd service with Delegate=yes. + /// + public IContainer Host(string image, string database, IReadOnlyDictionary? environment = null, bool privileged = false) + { + var builder = new ContainerBuilder(image) + .WithNetwork(_network) + .WithPrivileged(privileged) + .WithEnvironment("ConnectionStrings__psr", $"Host=pg;Database={database};Username=psr;Password=psr") + .WithEnvironment("ConnectionStrings__rabbitmq", "amqp://psr:psr@rabbit:5672/") + .WithEnvironment("PSR_AUTO_MIGRATE", "true") + .WithPortBinding(HttpPort, assignRandomHostPort: true) + .WithCreateParameterModifier(parameters => + { + // Testcontainers fills HostConfig (port bindings, network) before the modifiers run. The volumes run + // Python (interpreters, venvs), so the tmpfs needs exec (Docker mounts tmpfs noexec by default). + parameters.HostConfig!.ReadonlyRootfs = true; + if (privileged) + { + parameters.User = "0:0"; + } + + parameters.HostConfig.Tmpfs = Volumes.ToDictionary(v => v, _ => "uid=10001,gid=10001,exec"); + }) + .WithWaitStrategy(Wait.ForUnixContainer().UntilHttpRequestIsSucceeded(r => r.ForPort(HttpPort).ForPath("/health/ready"))); + foreach (var (name, value) in environment ?? new Dictionary()) + { + builder = builder.WithEnvironment(name, value); + } + + return builder.Build(); + } + + /// Fixed login users of the dashboard image (one admin). + public static IReadOnlyDictionary DashboardUsers() => new Dictionary + { + ["PythonScriptRunner__Dashboard__Login__Users__0__Name"] = "admin", + ["PythonScriptRunner__Dashboard__Login__Users__0__Role"] = "Admin", + ["PythonScriptRunner__Dashboard__Login__Users__0__PasswordHash"] = DashboardPasswords.Hash(Password), + }; + + public async ValueTask DisposeAsync() + { + if (_postgres is not null) + { + await _postgres.DisposeAsync(); + } + + if (_rabbit is not null) + { + await _rabbit.DisposeAsync(); + } + + await _network.DisposeAsync(); + } +} diff --git a/tests/PythonScriptRunner.Containers.Tests/PythonScriptRunner.Containers.Tests.csproj b/tests/PythonScriptRunner.Containers.Tests/PythonScriptRunner.Containers.Tests.csproj new file mode 100644 index 0000000..43b41c2 --- /dev/null +++ b/tests/PythonScriptRunner.Containers.Tests/PythonScriptRunner.Containers.Tests.csproj @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/PythonScriptRunner.Core.Tests/Integration/IsolationTests.cs b/tests/PythonScriptRunner.Core.Tests/Integration/IsolationTests.cs index 388a480..0152342 100644 --- a/tests/PythonScriptRunner.Core.Tests/Integration/IsolationTests.cs +++ b/tests/PythonScriptRunner.Core.Tests/Integration/IsolationTests.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Text.Json.Nodes; +using PythonScriptRunner.Core.Execution; using PythonScriptRunner.Core.Tests.Infrastructure; namespace PythonScriptRunner.Core.Tests.Integration; @@ -86,6 +87,30 @@ def run(inputs, ctx): result.Error!.Kind.ShouldBe(ErrorKind.MemoryLimitExceeded, result.Error.Message); } + [Fact] + public async Task Spec001_AC29_NumericLibraries_GetOneThreadWhenTheLimitIsRlimitAs_OrTheConfiguredCount() + { + const string script = """ + import os + from psrunner import stage + + @stage + def run(inputs, ctx): + return {"threads": os.environ.get("OPENBLAS_NUM_THREADS", "") + "/" + os.environ.get("OMP_NUM_THREADS", "")} + """; + await using var standard = RunnerHost.Create(fixture, b => b.AddStage("threads", s => s.Script(script).Output("threads", FieldType.String))); + await using var configured = RunnerHost.Create( + fixture, b => b.AddStage("threads", s => s.Script(script).Output("threads", FieldType.String)), o => o.ThreadsPerProcess = 4); + + await using var byDefault = await standard.Runner.RunStageAsync("threads", null, cancellationToken: TestContext.Current.CancellationToken); + await using var explicitly = await configured.Runner.RunStageAsync("threads", null, cancellationToken: TestContext.Current.CancellationToken); + + // The CI and developer machines have no delegable cgroup: Linux falls back to RLIMIT_AS (one thread each); Windows + // limits committed memory with a Job Object and keeps the libraries' own default. + byDefault.GetOutput("threads").ShouldBe(OperatingSystem.IsLinux() ? "1/1" : "/"); + explicitly.GetOutput("threads").ShouldBe("4/4"); + } + [Fact] public async Task Spec001_AC10_Secrets_ReachTheScriptButNotLogsOrInputs() { @@ -142,8 +167,9 @@ def run(inputs, ctx): var names = result.GetOutput("nomes").Select(n => n!.GetValue().ToUpperInvariant()).ToList(); names.ShouldNotContain("PSR_TEST_HOST_ONLY"); - // LC_CTYPE is set by the interpreter itself on Linux (PEP 538 locale coercion), not inherited from the host. - string[] allowed = ["PATH", "SYSTEMROOT", "TEMP", "TMP", "TMPDIR", "HOME", "LC_CTYPE"]; + // LC_CTYPE is set by the interpreter itself on Linux (PEP 538 locale coercion), not inherited from the host. The + // thread variables come from the runner when the memory limit falls back to RLIMIT_AS (Linux without cgroup; D-180). + string[] allowed = ["PATH", "SYSTEMROOT", "TEMP", "TMP", "TMPDIR", "HOME", "LC_CTYPE", .. ProcessPythonExecutor.ThreadVariables]; names.Where(n => !n.StartsWith("PSR_", StringComparison.Ordinal) && !allowed.Contains(n)).ShouldBeEmpty(); } diff --git a/tests/PythonScriptRunner.Core.Tests/Unit/CgroupMemoryTests.cs b/tests/PythonScriptRunner.Core.Tests/Unit/CgroupMemoryTests.cs new file mode 100644 index 0000000..7eef6ae --- /dev/null +++ b/tests/PythonScriptRunner.Core.Tests/Unit/CgroupMemoryTests.cs @@ -0,0 +1,72 @@ +using Microsoft.Extensions.Options; +using PythonScriptRunner.Core.Execution; + +namespace PythonScriptRunner.Core.Tests.Unit; + +/// Spec 001 (D-180): preparing cgroup v2 memory limits, over a fake cgroup file system. +public sealed class CgroupMemoryTests : IDisposable +{ + private readonly string _root = Directory.CreateTempSubdirectory("psr-cgroup-").FullName; + + [Fact] + public void Spec001_AC30_WithAWritableCgroup_MovesTheHostAside_AndEachAttemptGetsItsMemoryMax() + { + var own = Cgroup("servico", controllers: "cpu io memory pids", processes: "100\n"); + + using var lease = Memory().Create(128L * 1024 * 1024); + + lease.ShouldNotBeNull(); + File.ReadAllText(Path.Combine(own, "psr-host", "cgroup.procs")).ShouldBe("100"); + File.ReadAllText(Path.Combine(own, "cgroup.subtree_control")).ShouldBe("+memory"); + File.ReadAllText(Path.Combine(own, "psr-runs", "cgroup.subtree_control")).ShouldBe("+memory"); + Path.GetDirectoryName(lease.Path).ShouldBe(Path.Combine(own, "psr-runs")); + File.ReadAllText(Path.Combine(lease.Path, "memory.max")).ShouldBe("134217728"); + File.ReadAllText(Path.Combine(lease.Path, "memory.swap.max")).ShouldBe("0"); + } + + [Fact] + public void Spec001_AC30_TheKernelOomKill_IsSeenInMemoryEvents() + { + Cgroup("servico", controllers: "memory", processes: "1\n"); + using var lease = Memory().Create(64L * 1024 * 1024)!; + lease.OutOfMemory().ShouldBeFalse(); + + File.WriteAllText(Path.Combine(lease.Path, "memory.events"), "low 0\nhigh 0\nmax 3\noom 1\noom_kill 1\n"); + + lease.OutOfMemory().ShouldBeTrue(); + } + + [Fact] + public void Spec001_AC29_WithoutTheMemoryController_OrOutsideLinux_ThereIsNoCgroup() + { + Cgroup("servico", controllers: "cpu pids", processes: "1\n"); + + Memory().Create(64L * 1024 * 1024).ShouldBeNull(); + Memory(linux: false).Create(64L * 1024 * 1024).ShouldBeNull(); + } + + [Fact] + public void Spec001_AC29_WithACgroupV1Host_ThereIsNoCgroup() + { + File.WriteAllText(Path.Combine(_root, "self-cgroup"), "12:memory:/servico\n1:name=systemd:/servico\n"); + + Memory().Create(64L * 1024 * 1024).ShouldBeNull(); + } + + public void Dispose() => Directory.Delete(_root, recursive: true); + + private CgroupMemory Memory(bool linux = true) => + new(null, Options.Create(new PythonScriptRunnerOptions()), Path.Combine(_root, "fs"), Path.Combine(_root, "self-cgroup"), linux); + + /// The cgroup of the host (as /proc/self/cgroup names it) with its interface files. + private string Cgroup(string name, string controllers, string processes) + { + var own = Path.Combine(_root, "fs", name); + Directory.CreateDirectory(own); + File.WriteAllText(Path.Combine(own, "cgroup.controllers"), controllers + "\n"); + File.WriteAllText(Path.Combine(own, "cgroup.subtree_control"), "\n"); + File.WriteAllText(Path.Combine(own, "cgroup.procs"), processes); + File.WriteAllText(Path.Combine(_root, "self-cgroup"), $"0::/{name}\n"); + return own; + } +} diff --git a/tests/PythonScriptRunner.EntityFrameworkCore.Tests/LoggingTests.cs b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/LoggingTests.cs new file mode 100644 index 0000000..3f47e6a --- /dev/null +++ b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/LoggingTests.cs @@ -0,0 +1,29 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.Extensions.Logging; +using PythonScriptRunner.EntityFrameworkCore.Tests.Infrastructure; + +namespace PythonScriptRunner.EntityFrameworkCore.Tests; + +public sealed class LoggingTests(PostgresFixture postgres, PythonFixture python) +{ + /// + /// The relay and the worker poll the database several times per second (D-58); with the executed commands at + /// Information, the hosts logged ~4 lines per second of the same query (D-187). + /// + [Fact] + public async Task ExecutedSqlCommands_AreLoggedAtDebug() + { + await using var host = await PsrTestHost.CreateAsync(postgres, python, services: s => s + .AddLogging(logging => logging.SetMinimumLevel(LogLevel.Debug)) + .AddFakeLogging()); + + (await host.Admin.GetStageAsync("inexistente", TestContext.Current.CancellationToken)).ShouldBeNull(); + + var executed = host.Services.GetFakeLogCollector().GetSnapshot() + .Where(r => r.Category == DbLoggerCategory.Database.Command.Name && r.Id.Id == RelationalEventId.CommandExecuted.Id) + .ToList(); + executed.ShouldNotBeEmpty(); + executed.ShouldAllBe(r => r.Level == LogLevel.Debug); + } +} diff --git a/tests/PythonScriptRunner.EntityFrameworkCore.Tests/PythonScriptRunner.EntityFrameworkCore.Tests.csproj b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/PythonScriptRunner.EntityFrameworkCore.Tests.csproj index ecd7e66..fd061f0 100644 --- a/tests/PythonScriptRunner.EntityFrameworkCore.Tests/PythonScriptRunner.EntityFrameworkCore.Tests.csproj +++ b/tests/PythonScriptRunner.EntityFrameworkCore.Tests/PythonScriptRunner.EntityFrameworkCore.Tests.csproj @@ -20,6 +20,7 @@ +