Skip to content

Commit 642d2ca

Browse files
net0wellclaude
andcommitted
feat(sdk): SDK Python psrunner e schema do stage.json (spec 002)
- @stage, Context (log, segredos, new_file), File e StageError - entrypoint psrunner.host (protocolo 1): conversão e validação dos 12 tipos, table como DataFrame quando há pandas, limite de memória no Linux, traceback do usuário - psrunner.testing.run_stage (pytest sem .NET) e CLI validate/run - schemas/stage.v1.json - pytest (54 testes), mypy strict e ruff Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent cc64324 commit 642d2ca

25 files changed

Lines changed: 3759 additions & 0 deletions

‎python/psrunner/README.md‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# psrunner
2+
3+
Python SDK for [PythonScriptRunner](https://github.com/net0well/PythonScriptRunner) stages.
4+
5+
A stage is one Python script with typed inputs and outputs. The .NET runner validates the inputs,
6+
runs the script in an isolated process and validates the outputs. `psrunner` handles the protocol,
7+
so the script only contains the logic:
8+
9+
```python
10+
from psrunner import stage
11+
12+
@stage
13+
def run(inputs, ctx):
14+
a1, ratio, n = inputs["a1"], inputs["ratio"], inputs["n"]
15+
terms = [a1 * ratio**i for i in range(n)]
16+
ctx.log.info("generated %d terms", n)
17+
return {"terms": terms}
18+
```
19+
20+
Test it locally with pytest, without .NET:
21+
22+
```python
23+
from psrunner.testing import run_stage
24+
25+
def test_terms():
26+
result = run_stage("stages/terms/script.py", {"a1": 1, "ratio": 2, "n": 4},
27+
stage_json="stages/terms/stage.json")
28+
assert result.outputs == {"terms": [1, 2, 4, 8]}
29+
```
30+
31+
Install the `pandas` extra to receive `table` inputs as `pandas.DataFrame`: `pip install psrunner[pandas]`.

‎python/psrunner/pyproject.toml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
[build-system]
2+
requires = ["hatchling>=1.27"]
3+
build-backend = "hatchling.build"
4+
5+
[project]
6+
name = "psrunner"
7+
dynamic = ["version"]
8+
description = "Python SDK for PythonScriptRunner stages: typed inputs and outputs, files, secrets and logging."
9+
readme = "README.md"
10+
requires-python = ">=3.10"
11+
license = "MIT"
12+
authors = [{ name = "net0well" }]
13+
keywords = ["pythonscriptrunner", "dotnet", "pipeline", "stage"]
14+
classifiers = [
15+
"Development Status :: 3 - Alpha",
16+
"Intended Audience :: Developers",
17+
"Operating System :: Microsoft :: Windows",
18+
"Operating System :: POSIX :: Linux",
19+
"Programming Language :: Python :: 3",
20+
"Programming Language :: Python :: 3.10",
21+
"Programming Language :: Python :: 3.11",
22+
"Programming Language :: Python :: 3.12",
23+
"Programming Language :: Python :: 3.13",
24+
"Typing :: Typed",
25+
]
26+
dependencies = []
27+
28+
[project.optional-dependencies]
29+
pandas = ["pandas>=2.0"]
30+
31+
[project.urls]
32+
Repository = "https://github.com/net0well/PythonScriptRunner"
33+
34+
[project.scripts]
35+
psrunner = "psrunner.cli:main"
36+
37+
[dependency-groups]
38+
dev = [
39+
"pytest>=8.3",
40+
"mypy>=1.13",
41+
"ruff>=0.8",
42+
"pandas>=2.0",
43+
]
44+
45+
[tool.hatch.version]
46+
path = "src/psrunner/_version.py"
47+
48+
[tool.hatch.build.targets.wheel]
49+
packages = ["src/psrunner"]
50+
51+
[tool.pytest.ini_options]
52+
testpaths = ["tests"]
53+
addopts = "-ra"
54+
55+
[tool.mypy]
56+
strict = true
57+
python_version = "3.10"
58+
files = ["src"]
59+
60+
[[tool.mypy.overrides]]
61+
module = ["pandas", "pandas.*", "numpy", "numpy.*"]
62+
ignore_missing_imports = true
63+
64+
[tool.ruff]
65+
line-length = 100
66+
target-version = "py310"
67+
68+
[tool.ruff.lint]
69+
select = ["E", "F", "W", "I", "UP", "B", "SIM", "RUF"]
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
"""Python SDK for PythonScriptRunner stages."""
2+
3+
from psrunner._version import __version__
4+
from psrunner.context import Context
5+
from psrunner.errors import StageError
6+
from psrunner.files import File
7+
from psrunner.protocol import PROTOCOL_VERSION
8+
from psrunner.stage import stage
9+
10+
__all__ = ["PROTOCOL_VERSION", "Context", "File", "StageError", "__version__", "stage"]
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
from psrunner.cli import main
2+
3+
raise SystemExit(main())
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
"""Shared helpers to call the stage function and classify failures."""
2+
3+
from __future__ import annotations
4+
5+
import asyncio
6+
import inspect
7+
import traceback
8+
from collections.abc import Callable
9+
from pathlib import Path
10+
from typing import Any
11+
12+
from psrunner.context import Context
13+
from psrunner.errors import ContractError, ScriptLoadError, StageError
14+
from psrunner.protocol import KIND_MEMORY_LIMIT, KIND_SCRIPT_ERROR
15+
16+
_PACKAGE_DIR = str(Path(__file__).resolve().parent)
17+
18+
19+
def call_stage(fn: Callable[..., Any], inputs: dict[str, Any], ctx: Context) -> Any:
20+
"""Calls ``fn(inputs, ctx)``; coroutines (``async def run``) are awaited."""
21+
result = fn(inputs, ctx)
22+
if inspect.iscoroutine(result):
23+
result = asyncio.run(result)
24+
return result
25+
26+
27+
def _is_internal(filename: str) -> bool:
28+
if filename.startswith("<frozen importlib"):
29+
return True
30+
try:
31+
return str(Path(filename).resolve()).startswith(_PACKAGE_DIR)
32+
except OSError:
33+
return False
34+
35+
36+
def user_traceback(exc: BaseException) -> str:
37+
"""Traceback without the SDK's own frames, so it points at the user script."""
38+
frames = [f for f in traceback.extract_tb(exc.__traceback__) if not _is_internal(f.filename)]
39+
lines = ["Traceback (most recent call last):\n"]
40+
lines += traceback.format_list(frames)
41+
lines += traceback.format_exception_only(type(exc), exc)
42+
return "".join(lines)
43+
44+
45+
def classify_exception(exc: BaseException) -> dict[str, Any]:
46+
"""Maps an exception to the protocol error object."""
47+
if isinstance(exc, StageError):
48+
error: dict[str, Any] = {
49+
"kind": KIND_SCRIPT_ERROR,
50+
"code": exc.code,
51+
"message": exc.message,
52+
"traceback": user_traceback(exc),
53+
}
54+
if exc.details is not None:
55+
error["details"] = exc.details
56+
return error
57+
if isinstance(exc, ContractError):
58+
return {"kind": exc.kind, "code": exc.code, "message": exc.message}
59+
if isinstance(exc, ScriptLoadError):
60+
error = {"kind": KIND_SCRIPT_ERROR, "code": exc.code, "message": exc.message}
61+
if exc.__cause__ is not None and not isinstance(exc.__cause__, SyntaxError):
62+
error["traceback"] = user_traceback(exc.__cause__)
63+
return error
64+
if isinstance(exc, MemoryError):
65+
return {
66+
"kind": KIND_MEMORY_LIMIT,
67+
"code": "MEMORY_LIMIT_EXCEEDED",
68+
"message": "The script exceeded its memory limit.",
69+
}
70+
if isinstance(exc, SystemExit):
71+
return {
72+
"kind": KIND_SCRIPT_ERROR,
73+
"code": "SCRIPT_EXITED",
74+
"message": (
75+
f"The script called sys.exit({exc.code!r}); return a dict or raise StageError."
76+
),
77+
}
78+
if isinstance(exc, KeyboardInterrupt):
79+
return {
80+
"kind": KIND_SCRIPT_ERROR,
81+
"code": "INTERRUPTED",
82+
"message": "The script was interrupted.",
83+
}
84+
return {
85+
"kind": KIND_SCRIPT_ERROR,
86+
"code": None,
87+
"message": f"{type(exc).__name__}: {exc}",
88+
"traceback": user_traceback(exc),
89+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
__version__ = "0.1.0"
Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
"""Command line helpers for stage authors.
2+
3+
``python -m psrunner validate <stage-folder>``
4+
Validates ``stage.json`` and checks that the script declares exactly one ``@stage``.
5+
6+
``python -m psrunner run <stage-folder> --input sample.json [--secret NAME=VALUE ...]``
7+
Runs the stage locally with the real protocol (separate process, temporary work folder).
8+
File inputs in the sample are written as ``{"$path": "relative/to/sample.json"}``.
9+
"""
10+
11+
from __future__ import annotations
12+
13+
import argparse
14+
import json
15+
import os
16+
import shutil
17+
import subprocess
18+
import sys
19+
import tempfile
20+
import uuid
21+
from collections.abc import Sequence
22+
from pathlib import Path
23+
from typing import Any
24+
25+
from psrunner.errors import ScriptLoadError
26+
from psrunner.manifest import Manifest, ManifestError, load_manifest
27+
from psrunner.protocol import (
28+
ENV_INPUT,
29+
ENV_MEMORY_LIMIT_MB,
30+
ENV_OUTPUT,
31+
ENV_PROTOCOL,
32+
ENV_SCRIPT,
33+
ENV_SECRET_PREFIX,
34+
ENV_WORK_DIR,
35+
PROTOCOL_VERSION,
36+
)
37+
from psrunner.stage import load_stage_function
38+
from psrunner.testing import _copy_input_files
39+
40+
41+
def _validate(folder: Path) -> int:
42+
try:
43+
manifest = load_manifest(folder)
44+
load_stage_function(manifest.script_path)
45+
except ManifestError as exc:
46+
print(f"Invalid manifest {exc.path}:", file=sys.stderr)
47+
for error in exc.errors:
48+
print(f" - {error}", file=sys.stderr)
49+
return 1
50+
except ScriptLoadError as exc:
51+
print(f"Invalid script: [{exc.code}] {exc.message}", file=sys.stderr)
52+
return 1
53+
print(
54+
f"OK: {manifest.key}@{manifest.version} "
55+
f"({len(manifest.inputs)} inputs, {len(manifest.outputs)} outputs)"
56+
)
57+
return 0
58+
59+
60+
def _resolve_sample(sample: dict[str, Any], sample_dir: Path, work_dir: Path) -> dict[str, Any]:
61+
resolved: dict[str, Any] = {}
62+
for name, value in sample.items():
63+
if isinstance(value, dict) and "$path" in value:
64+
resolved[name] = _copy_input_files(name, sample_dir / value["$path"], work_dir)
65+
elif (
66+
isinstance(value, list)
67+
and value
68+
and all(isinstance(v, dict) and "$path" in v for v in value)
69+
):
70+
resolved[name] = _copy_input_files(
71+
name, [sample_dir / v["$path"] for v in value], work_dir
72+
)
73+
else:
74+
resolved[name] = value
75+
return resolved
76+
77+
78+
def _run(folder: Path, input_file: Path | None, secrets: Sequence[str], keep: bool) -> int:
79+
try:
80+
manifest: Manifest = load_manifest(folder)
81+
except ManifestError as exc:
82+
print(f"Invalid manifest {exc.path}: {'; '.join(exc.errors)}", file=sys.stderr)
83+
return 1
84+
85+
work_dir = Path(tempfile.mkdtemp(prefix="psrunner-run-"))
86+
for sub in ("in", "out", "tmp"):
87+
(work_dir / sub).mkdir()
88+
89+
sample: dict[str, Any] = {}
90+
if input_file is not None:
91+
sample = json.loads(input_file.read_text(encoding="utf-8"))
92+
inputs = _resolve_sample(
93+
sample, (input_file or folder).parent if input_file else folder, work_dir
94+
)
95+
96+
document = {
97+
"protocol": PROTOCOL_VERSION,
98+
"context": {
99+
"runId": uuid.uuid4().hex,
100+
"stepKey": manifest.key,
101+
"attempt": 1,
102+
"stage": {"key": manifest.key, "version": manifest.version},
103+
"isTest": True,
104+
"inputFields": manifest.raw["inputs"],
105+
"outputFields": manifest.raw["outputs"],
106+
},
107+
"inputs": inputs,
108+
}
109+
(work_dir / "input.json").write_text(json.dumps(document, ensure_ascii=False), encoding="utf-8")
110+
111+
env = {key: value for key, value in os.environ.items() if key in {"PATH", "SYSTEMROOT", "LANG"}}
112+
env.update(
113+
{
114+
"TEMP": str(work_dir / "tmp"),
115+
"TMP": str(work_dir / "tmp"),
116+
ENV_PROTOCOL: str(PROTOCOL_VERSION),
117+
ENV_INPUT: str(work_dir / "input.json"),
118+
ENV_OUTPUT: str(work_dir / "output.json"),
119+
ENV_WORK_DIR: str(work_dir),
120+
ENV_SCRIPT: str(manifest.script_path.resolve()),
121+
ENV_MEMORY_LIMIT_MB: str(manifest.raw.get("memoryLimitMb", 0)),
122+
}
123+
)
124+
for item in secrets:
125+
name, _, value = item.partition("=")
126+
env[f"{ENV_SECRET_PREFIX}{name}"] = value
127+
128+
completed = subprocess.run(
129+
[sys.executable, "-I", "-X", "utf8", "-m", "psrunner.host"],
130+
cwd=work_dir,
131+
env=env,
132+
capture_output=True,
133+
text=True,
134+
encoding="utf-8",
135+
errors="replace",
136+
check=False,
137+
)
138+
if completed.stdout:
139+
print("--- stdout ---")
140+
print(completed.stdout.rstrip())
141+
if completed.stderr:
142+
print("--- stderr ---")
143+
print(completed.stderr.rstrip())
144+
145+
output_path = work_dir / "output.json"
146+
if not output_path.exists():
147+
print(
148+
f"Process exited with code {completed.returncode} without output.json.", file=sys.stderr
149+
)
150+
return 1
151+
output = json.loads(output_path.read_text(encoding="utf-8"))
152+
print("--- output.json ---")
153+
print(json.dumps(output, indent=2, ensure_ascii=False))
154+
if keep:
155+
print(f"Work folder: {work_dir}")
156+
else:
157+
shutil.rmtree(work_dir, ignore_errors=True)
158+
return 0 if output.get("status") == "ok" else 1
159+
160+
161+
def main(argv: Sequence[str] | None = None) -> int:
162+
parser = argparse.ArgumentParser(prog="psrunner", description="PythonScriptRunner stage tools")
163+
commands = parser.add_subparsers(dest="command", required=True)
164+
165+
validate = commands.add_parser("validate", help="validate stage.json and the script")
166+
validate.add_argument("folder", type=Path)
167+
168+
run = commands.add_parser("run", help="run a stage locally with the real protocol")
169+
run.add_argument("folder", type=Path)
170+
run.add_argument("--input", type=Path, help="JSON file with the inputs")
171+
run.add_argument("--secret", action="append", default=[], help="NAME=VALUE (repeatable)")
172+
run.add_argument("--keep", action="store_true", help="print the work folder path")
173+
174+
args = parser.parse_args(argv)
175+
if args.command == "validate":
176+
return _validate(args.folder)
177+
return _run(args.folder, args.input, args.secret, args.keep)

0 commit comments

Comments
 (0)