diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ceb5948..f44b1db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,15 @@ on: description: Run the reviewed published SDK/native timer-redrive tuple type: boolean default: false + activity_retry_cap: + description: Run the reviewed installed scalar local retry-cap case + type: boolean + default: false + retry_cap_artifact_source: + description: Select the approved build candidate or its identical registry wheel + type: choice + options: [candidate, registry] + default: candidate permissions: contents: read @@ -36,6 +45,58 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + activity-retry-cap: + if: github.event_name == 'workflow_dispatch' && inputs.activity_retry_cap + needs: [action-policy, lint] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + actions: read + env: + RETRY_CAP_ARTIFACT_SOURCE: ${{ inputs.retry_cap_artifact_source }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + fetch-depth: 0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12.16" + - name: Resolve the exact approved build candidate + if: inputs.retry_cap_artifact_source == 'candidate' + id: retry-candidate + env: + GH_TOKEN: ${{ github.token }} + run: python .github/scripts/promote_release.py resolve --version 2.18.0 + - name: Download the approved candidate handoff + if: inputs.retry_cap_artifact_source == 'candidate' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ steps.retry-candidate.outputs.artifact-id }} + run-id: ${{ steps.retry-candidate.outputs.run-id }} + github-token: ${{ github.token }} + repository: ${{ github.repository }} + path: .release-candidate + archive: false + - name: Verify candidate archive and distribution bytes + if: inputs.retry_cap_artifact_source == 'candidate' + run: python .github/scripts/promote_release.py extract --version 2.18.0 + - name: Bootstrap patched pip trust + uses: durable-workflow/.github/.github/actions/bootstrap-patched-pip@75587fc33ff40c34f32ef777293dad56bb6f48c7 # clean-trust + with: + clients: pip,python + - name: Run the approved scalar local retry case + run: >- + python scripts/ci/published-activity-acceptance.py + --scenario retry-cap --artifact-source "$RETRY_CAP_ARTIFACT_SOURCE" + - name: Retain exact package and retry evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: activity-retry-cap-${{ github.sha }} + path: published-activity-results/ + retention-days: 7 published-timer-redrive: if: github.event_name == 'workflow_dispatch' && inputs.published_timer_redrive needs: [action-policy, lint] diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ca988f..3f796e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.18.0] - 2026-11-14 + ### Added - `ActivityRetryPolicy.exponential(...)` creates generated activity retry schedules with a finite 60-second default maximum. It rejects `None` or nonfinite maxima and retains the existing wire shape and interval rounding. The legacy constructor, explicit backoff lists and policies already recorded in history keep their original values. (durable-workflow/.github#219) diff --git a/pyproject.toml b/pyproject.toml index 388f210..7f927f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "durable-workflow" -version = "2.17.2" +version = "2.18.0" description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server" readme = "README.md" requires-python = ">=3.10" @@ -79,8 +79,8 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main" durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main" [tool.durable-workflow] -product-train = "2.17.2" -registry-version = "2.17.2" +product-train = "2.18.0" +registry-version = "2.18.0" supported-server-versions = "2.8.0" supported-server-release-baselines = { "2" = "2.8.0", "3" = "3.0.0" } worker-protocol-version = "1.20" diff --git a/scripts/ci/activity-retry-cap.md b/scripts/ci/activity-retry-cap.md new file mode 100644 index 0000000..b1aa87f --- /dev/null +++ b/scripts/ci/activity-retry-cap.md @@ -0,0 +1,46 @@ +The retry-cap case qualifies the exact reviewed 2.18.0 wheel on the published +Server/Python tuple in `tests/fixtures/published-retry-cap.json`. It uses the +existing internal two-container profile, complete hash lock, captured no-site +entry, before/after installed inventory and independent ID/label cleanup. +It adds no host mount, port, capability or alternate local-activity carrier. + +The release approval record supplies the SDK hash. Candidate mode reuses the +protected publication resolver and archive extractor to bind the successful +same-repository build, source/tree, handoff artifact and both distributions. +The controller captures that record from Git, reads the wheel through a held +descriptor and checks its approved hash again. Only the 14 already locked +dependencies are downloaded from PyPI. Registry mode downloads all 15 locked +wheels and requires the identical approved SDK hash. The final private installer +always receives the complete 15-package hash lock and has no network fallback. + +After source, artifact, runner and accepted-main gates are complete, select one +bounded candidate invocation through the ordinary workflow: + +```bash +gh workflow run ci.yml --ref main -f activity_retry_cap=true -f retry_cap_artifact_source=candidate +``` + +After protected publication, select `retry_cap_artifact_source=registry` for the +normal-registry repeat. Neither invocation runs in ordinary push/PR CI. + +Three real scalar local executions exercise the factory's default cap, a short +explicit list and an empty list. The generated policy has two total attempts, +an initial interval of 61 seconds and the factory's omitted maximum, so its +single retry must wait at least 60 seconds. Five attempts with `[1, 2]` select +`1, 2, 2, 2`; the empty list selects zero waits. Monotonic callback observations +independently reject shortened nonzero waits, with a 10-millisecond clock +tolerance. No upper worker-start guarantee is inferred from scheduling time. + +The case binds complete ordered attempt/callback counts, reported backoffs, +accepted command kinds and actual retained public activity snapshots/retry +events to the selected policies. Each committed history replays with a callback +that refuses execution. Public exports bind the requested workflow/run and +unredacted retained state. Every source history is re-read unchanged. + +Each Worker has a 100-second or 30-second SDK budget. The complete selected +pytest case is bounded at 180 seconds within the existing 240-second consumer +exchange and six-minute controller workload. Independent cleanup retains its +per-call bounds. The seven-day artifact contains the wheel set, complete +effective lock, source/tuple hashes, command/history/timing observations, +installed inventory and full-ID/label absence. Failures remain NONPASS; source +or virtual-sleep controls do not qualify an installed tuple. diff --git a/scripts/ci/activity_acceptance_consumer.py b/scripts/ci/activity_acceptance_consumer.py index f51b756..1b51e06 100644 --- a/scripts/ci/activity_acceptance_consumer.py +++ b/scripts/ci/activity_acceptance_consumer.py @@ -27,6 +27,7 @@ CASE_FILES = { "activity-identity": {"test_activity_identity.py"}, "timer-redrive": {"test_native_timer_redrive.py", "redrive_timer_clock_workflow.py"}, + "retry-cap": {"test_activity_retry_cap.py"}, } @@ -247,7 +248,10 @@ def entry_source(mode, case_name, case_sha, installed, worker_entry=""): def execute_case(verifier, python, packet, before, environment, admission, installed): redrive = packet["scenario"] == "timer-redrive" - test_name = "test_native_timer_redrive.py" if redrive else "test_activity_identity.py" + test_name = { + "timer-redrive": "test_native_timer_redrive.py", "retry-cap": "test_activity_retry_cap.py", + "activity-identity": "test_activity_identity.py", + }[packet["scenario"]] environment.update( { "DURABLE_WORKFLOW_SERVER_URL": admission["server_url"], @@ -258,6 +262,8 @@ def execute_case(verifier, python, packet, before, environment, admission, insta ) if redrive: environment["DURABLE_WORKFLOW_REDRIVE_CELL"] = "1" + if packet["scenario"] == "retry-cap": + environment["DURABLE_WORKFLOW_RETRY_CAP_CELL"] = "1" worker_entry = entry_source("worker", test_name, packet["files"][test_name]["sha256"], installed) parent_entry = entry_source("pytest", test_name, packet["files"][test_name]["sha256"], installed, worker_entry) result = subprocess.run( @@ -273,7 +279,7 @@ def execute_case(verifier, python, packet, before, environment, admission, insta input=json.dumps({"inventory": installed}), capture_output=True, text=True, - timeout=180 if redrive else 90, + timeout=90 if packet["scenario"] == "activity-identity" else 180, ) after = inventory(verifier, python, packet) if after != before: diff --git a/scripts/ci/activity_acceptance_release.py b/scripts/ci/activity_acceptance_release.py new file mode 100644 index 0000000..ef8e489 --- /dev/null +++ b/scripts/ci/activity_acceptance_release.py @@ -0,0 +1,34 @@ +"""Bind the retry case to the independently reviewed SDK release bytes.""" + +from __future__ import annotations + +import hashlib +import re +from pathlib import Path + +from scripts.ci.activity_acceptance_entry import captured_file + +VERSION = "2.18.0" +WHEEL = f"durable_workflow-{VERSION}-py3-none-any.whl" +SDIST = f"durable_workflow-{VERSION}.tar.gz" + + +def lock(dependencies: bytes, approval: dict) -> tuple[bytes, str]: + if approval["version"] != VERSION or set(approval["files"]) != {WHEEL, SDIST}: + raise ValueError("retry case requires its exact reviewed release") + digest = approval["files"][WHEEL] + if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None: + raise ValueError("retry case requires the approved wheel digest") + if re.search(rb"(?im)^[ \t]*durable[-_]workflow\b", dependencies): + raise ValueError("retry dependency lock already contains an SDK selection") + result = dependencies.rstrip() + f"\ndurable-workflow=={VERSION} --hash=sha256:{digest}\n".encode() + return result, digest + + +def candidate_wheel(root: Path, digest: str) -> bytes: + # The protected download/extract steps validate the same-repository build + # and complete handoff archive. This read binds the consumed bytes again. + data = captured_file(root / "dist" / WHEEL) + if hashlib.sha256(data).hexdigest() != digest: + raise ValueError("candidate wheel differs from the approved release") + return data diff --git a/scripts/ci/published-activity-acceptance.py b/scripts/ci/published-activity-acceptance.py index 01b9fe6..63f19df 100644 --- a/scripts/ci/published-activity-acceptance.py +++ b/scripts/ci/published-activity-acceptance.py @@ -1,4 +1,4 @@ -"""Run one fixed published SDK acceptance case inside an owned internal network.""" +"""Run one fixed approved SDK acceptance case inside an owned internal network.""" from __future__ import annotations @@ -19,6 +19,7 @@ try: from scripts.ci import activity_acceptance_output as output_files + from scripts.ci import activity_acceptance_release as release from scripts.ci import activity_acceptance_runtime as runtime from scripts.ci.activity_acceptance_buildx import BuildxState from scripts.ci.activity_acceptance_session import Session @@ -27,6 +28,7 @@ raise sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from scripts.ci import activity_acceptance_output as output_files + from scripts.ci import activity_acceptance_release as release from scripts.ci import activity_acceptance_runtime as runtime from scripts.ci.activity_acceptance_buildx import BuildxState from scripts.ci.activity_acceptance_session import Session @@ -40,6 +42,7 @@ ID_FORMAT = "{{.ID}}" DIAGNOSTIC_LIMIT = 64 * 1024 SCENARIO = "activity-identity" +ARTIFACT_SOURCE = "registry" SCENARIOS = { "activity-identity": ( "test_activity_identity.py", "test_retry_preserves_received_execution_id_and_changes_attempt_id", @@ -47,6 +50,9 @@ "timer-redrive": ( "test_native_timer_redrive.py", "test_cold_workers_preserve_fractional_timer_clock_across_repeated_redrive", ), + "retry-cap": ( + "test_activity_retry_cap.py", "test_installed_local_retries_preserve_cap_tail_and_committed_replay", + ), } # No inherited context, registry credentials, proxy, SSH or remote Docker endpoint. # This kernel directory is private to each CLI process and contains no config.json. @@ -154,18 +160,25 @@ def capture_source(report): if re.fullmatch(r"[0-9a-f]{40}", commit) is None or os.environ.get("GITHUB_SHA", commit) != commit: raise ValueError("checkout differs from the selected Actions commit") paths = { - REQUIREMENTS: "scripts/ci/published-activity-requirements.txt", + REQUIREMENTS: "scripts/ci/" + ( + "retry-cap-dependencies.txt" if SCENARIO == "retry-cap" else "published-activity-requirements.txt" + ), VERIFIER_FILENAME: "scripts/ci/verify_activity_install.py", SCENARIOS[SCENARIO][0]: "tests/integration/" + SCENARIOS[SCENARIO][0], "conftest.py": "tests/integration/conftest.py", "bootstrap.py": "scripts/ci/activity_acceptance_consumer.py", "entry.py": "scripts/ci/activity_acceptance_entry.py", "tuple.json": "tests/fixtures/" + ( - "published-timer-redrive.json" if SCENARIO == "timer-redrive" else "published-activity-acceptance.json" + {"timer-redrive": "published-timer-redrive.json", "retry-cap": "published-retry-cap.json"}.get( + SCENARIO, "published-activity-acceptance.json" + ) ), } if SCENARIO == "timer-redrive": paths["redrive_timer_clock_workflow.py"] = "tests/integration/redrive_timer_clock_workflow.py" + if SCENARIO == "retry-cap": + paths["release.json"] = f".github/release-candidates/{release.VERSION}.json" + paths["release-loader.py"] = "scripts/ci/activity_acceptance_release.py" captured = { name: call("git", "show", commit + ":" + path, cwd=ROOT).stdout.encode() for name, path in paths.items() } @@ -175,10 +188,21 @@ def capture_source(report): def download_wheels(selected, captured, files): + if ARTIFACT_SOURCE == "candidate" and SCENARIO != "retry-cap": + raise ValueError("candidate artifacts require the fixed retry-cap case") approval_path = ".github/release-candidates/" + selected["sdk_version"] + ".json" approval = json.loads(call("git", "show", "HEAD:" + approval_path, cwd=ROOT).stdout) wheel_name = next(name for name in approval["files"] if name.endswith(".whl")) + dependency_lock = captured[REQUIREMENTS] + if SCENARIO == "retry-cap": + if selected["sdk_version"] != release.VERSION or approval != json.loads(captured["release.json"]): + raise ValueError("retry release selection differs from captured authority") + captured[REQUIREMENTS], _digest = release.lock(dependency_lock, approval) files.write(REQUIREMENTS, captured[REQUIREMENTS]) + download_requirements = REQUIREMENTS + if ARTIFACT_SOURCE == "candidate": + download_requirements = "retry-dependencies.txt" + files.write(download_requirements, dependency_lock) files.check() wheelhouse = OUTPUT / "wheels" wheelhouse.mkdir(mode=0o700) @@ -201,11 +225,16 @@ def download_wheels(selected, captured, files): "--dest", str(wheelhouse), "-r", - str(OUTPUT / REQUIREMENTS), + str(OUTPUT / download_requirements), env=environment, ) files.write("download.log", result.stdout + result.stderr) files.check() + if ARTIFACT_SOURCE == "candidate": + data = release.candidate_wheel(ROOT, approval["files"][wheel_name]) + descriptor = os.open(wheelhouse / wheel_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "wb") as output: + output.write(data) wheels = {} for path in sorted(wheelhouse.iterdir()): if path.is_symlink() or not path.is_file() or path.suffix != ".whl": @@ -489,7 +518,7 @@ def run_case(owned, selected, images, packet, bootstrap, report): if ready["sdk_inventory"] != result["sdk_inventory"]: raise ValueError("installed SDK bytes changed during the case") owned.files.write("sdk-inventory.json", json.dumps(result["sdk_inventory"], indent=2)) - name = "timer-redrive" if SCENARIO == "timer-redrive" else "identity" + name = "identity" if SCENARIO == "activity-identity" else SCENARIO owned.files.write(name + ".log", result["case_output"]) owned.files.write(name + ".xml", result["junit"]) report["consumer_ready"] = {key: value for key, value in ready.items() if key != "sdk_inventory"} @@ -504,7 +533,8 @@ def run_case(owned, selected, images, packet, bootstrap, report): def main(): files = output_files.Output(OUTPUT) - report = {"complete": False, "scenario": SCENARIO, "started_at": datetime.now(timezone.utc).isoformat()} + report = {"complete": False, "scenario": SCENARIO, "artifact_source": ARTIFACT_SOURCE, + "started_at": datetime.now(timezone.utc).isoformat()} owned = OwnedDocker(uuid4().hex, files, report) signal.signal(signal.SIGALRM, deadline) signal.alarm(360) @@ -514,6 +544,7 @@ def main(): report["tuple"] = selected report["engine"] = engine() packet = download_wheels(selected, captured, files) + report["requirements_sha256"] = packet["files"][REQUIREMENTS]["sha256"] report["wheel_sha256"] = packet["wheel_sha256"] report["wheel_set"] = {name: item["sha256"] for name, item in packet["wheels"].items()} images = {role: verify_published_image(selected, role, files) for role in ("runtime", "consumer")} @@ -537,5 +568,9 @@ def main(): if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--scenario", choices=tuple(SCENARIOS), default=SCENARIO) - SCENARIO = parser.parse_args().scenario + parser.add_argument("--artifact-source", choices=("registry", "candidate"), default=ARTIFACT_SOURCE) + arguments = parser.parse_args() + SCENARIO, ARTIFACT_SOURCE = arguments.scenario, arguments.artifact_source + if ARTIFACT_SOURCE == "candidate" and SCENARIO != "retry-cap": + parser.error("candidate artifacts are accepted only by the fixed retry-cap case") main() diff --git a/scripts/ci/retry-cap-dependencies.txt b/scripts/ci/retry-cap-dependencies.txt new file mode 100644 index 0000000..c277b54 --- /dev/null +++ b/scripts/ci/retry-cap-dependencies.txt @@ -0,0 +1,80 @@ +anyio==4.16.0 \ + --hash=sha256:59bb03251907983d1603d27c3087ddb1e93299ea2163adcdba37899a4fb12ffb \ + --hash=sha256:c66ae634e0b8228396b801741beb3294326d4b6102f7bba4041c0dfb58c29026 +certifi==2026.10.23 \ + --hash=sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd \ + --hash=sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1 +fastavro==1.13.1 \ + --hash=sha256:01810229c86dcec75da8cc08f18f509e7a1883681c5c83c69f85589998440624 \ + --hash=sha256:045af8ab8fec214e3ff6241fed32c5124582888d5dce1da3ef3fa48629bd25b2 \ + --hash=sha256:0723398cd2b246a47bb6f44cb8230f158391c59e998f79687ba256cfa37127d7 \ + --hash=sha256:0994c545a4e2038b6d0b3ca54214d9573024e659fc5e618c4577329c89b9e016 \ + --hash=sha256:142e97f126358d910fc1d54742f8129f7c8ddee5d6c6c2da4ac8440483d03964 \ + --hash=sha256:28305b4e0764f362cffe5bb6993021d584c050d49256f153d1f46ee4fb188ba8 \ + --hash=sha256:2c44e98f32f59478ff0636b0415859327775a62433c2a184541595fb806ef33c \ + --hash=sha256:2f56a127d71e45083306d2650efff827cad0f4b0744dd42cb69c631d77943b1d \ + --hash=sha256:300a3c13dfa4ae7940224021dd5d41ea9fbad0a7bfa446e3f4176a969d18e596 \ + --hash=sha256:3fbe18a47dc1ea35bcdf01c16b7c9fe0dbeb22aa0e57e75d8c4dcd7b57395ea6 \ + --hash=sha256:3fd052bf63c097a34da732eba9f4eea179ae1104664e58c2404b48768b3d550f \ + --hash=sha256:46ff9c48be24798e1926eaa3733f80967439cd7f1c7514e32c64714cb6c405d9 \ + --hash=sha256:47ddd4d831eced3765b0f98d597bea8e07973b62be5aefce75ff7fc12fdb0f9e \ + --hash=sha256:5678573fd7a01d7b91099e9aa5ceb4a12f94979b421a710ae079c07c6470c864 \ + --hash=sha256:59a3ade141eb59cf723bede90a7cce0b1f9d49c642fe19d34737b421ac385495 \ + --hash=sha256:6bc39e1b87893307df49c6117cb2525e216af02da6b292d78685396366a41205 \ + --hash=sha256:6f05aa2539bf7a19e9eb3bdaf6580c4d0f082a8230f641eaf9c84e4bcf0e6bc4 \ + --hash=sha256:73fc8234e0dd162b69374bb66bbfb37dd6eac48d4e43c4c8609d2ffafb92797f \ + --hash=sha256:754a483d1f161545da76b3d6a3155b7e37477f1e149f00ccfff740d9ec5c143e \ + --hash=sha256:78251e44f96079b1d884b1977eeadee5a18b32098a42aa950a6914e5b6ec6e16 \ + --hash=sha256:783d3fa1a0b1cf785893788b276e674f69824d104498f7aee2d80f5fb73f619e \ + --hash=sha256:7db91731ae8f77e638525245a5b74c673c6ef1b1d3b1e64b91a5232cb4e34f6e \ + --hash=sha256:8ceecd6896adbc57c9e59ee3295c8016ae372f17df9787c4d1ba5a73209d723a \ + --hash=sha256:8f12f7f8154fbae11bad499ad93fbff08764c390acd43461ca4f7dc7807925b8 \ + --hash=sha256:90049246bc000da01715194e038da1121a24288c702a8482cc660069a41aacba \ + --hash=sha256:950f2e260f65c7e6135288c142b078d06d2f1c90fc52f91a14c08e5f8811bf06 \ + --hash=sha256:9be0b06f90784f5e04bfb29a467c698ab1f88409c0db4821bbc4d86d583bc82a \ + --hash=sha256:9f53c6e3179ef6c35724e5193c69bda85d001d987bbfb487a171fa04f526bd7c \ + --hash=sha256:a06d21d9ef55a9ab56eb869713ee88371b05da9fd9600a44170649eab71c6310 \ + --hash=sha256:a1b96aceb181a699dcadd1b0dad7026047ee62f606d1df36ca5a52acd4fe9dc3 \ + --hash=sha256:aef0ba9b7b9c0b6febeb4c14da9f13957dc02bc522ca4ab01d226c4d0dcde08a \ + --hash=sha256:bf36a4391f62b3c8292ff8461def7192738eb9311edd26c6d730788e92ee2560 \ + --hash=sha256:d596200f71c5706e931708ab4cb6f39decbdebe660453c54707a36e7a66b4aba \ + --hash=sha256:db65955d681266091392756ea80728b7f002e038b0c45f88873897b95c7963a0 \ + --hash=sha256:deab9d233ca9e3b03021c5b87a7807a1986a0375ef64975cbee9ad104e7eb3ea \ + --hash=sha256:e3d7e0850230a9af977184dd0677e2bc6341659835d55a73a2fa76c7d2d2d65e \ + --hash=sha256:f4126ba2e1097e42e5f911f16efca9df62ec54d40c27e18ff304c017c32a8af9 \ + --hash=sha256:f59980a60ecc1bce5a9a0f95116bd05928936514f199e127770b7afc7d423842 \ + --hash=sha256:ffa147df1278b8a849586da1f2b520e856e78ea797edc4c974c8bb1e6b4bfd66 \ + --hash=sha256:ffa4b0b942e3aa7e66cc97a1862a2da6a3fce3dbcbd17a9b4be6ff1c33c93976 +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c +iniconfig==2.3.1 \ + --hash=sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960 \ + --hash=sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7 +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c +pluggy==1.7.0 \ + --hash=sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec \ + --hash=sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8 +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c +pytest==9.2.0 \ + --hash=sha256:07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588 \ + --hash=sha256:97c4f3b1f3c7414aa36465e09b8220f1d03627993bd51b1282e193753c80557b +pytest-asyncio==1.5.0 \ + --hash=sha256:557f255ee03cc84f84a8d5f438bc9c6e0f714bf86970ae743c7ecb5a38d7e07d \ + --hash=sha256:92c8fa83ca419fd0d4bd6a34df6520dae435841d3733c319343249492adf37f7 +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 diff --git a/tests/fixtures/published-retry-cap.json b/tests/fixtures/published-retry-cap.json new file mode 100644 index 0000000..a1997ca --- /dev/null +++ b/tests/fixtures/published-retry-cap.json @@ -0,0 +1,11 @@ +{ + "schema": "sdk-python.published-activity-acceptance/v1", + "sdk_version": "2.18.0", + "server_image": "ghcr.io/durable-workflow/server@sha256:8ffcdb2a36af3129a1987849aafe8715fe0d3bf9a9648467f0b5cb8eeef43ba9", + "server_source": "5614274d2629d03f92932a07dfee0fb10349ddd9", + "server_amd64_manifest": "sha256:813c9337c44be7d497efd29e1b463546519e488aa796b5ecf9dfc6f4f10d2128", + "server_qualification": "https://github.com/durable-workflow/server/pull/1209#issuecomment-6858753154", + "python_image": "docker.io/library/python@sha256:935c3c5b12357a7e10caa9c1e6ca5692344d97cf229832ef584be2cec9fc03df", + "python_amd64_manifest": "sha256:0f086103b026048cc50cb5da25ec435c023c5241a83fc44daea593377abc61c4", + "python_version": "3.12.16" +} diff --git a/tests/integration/test_activity_retry_cap.py b/tests/integration/test_activity_retry_cap.py new file mode 100644 index 0000000..3692292 --- /dev/null +++ b/tests/integration/test_activity_retry_cap.py @@ -0,0 +1,147 @@ +"""Actual scalar local retry timing; admitted only by the reviewed package cell.""" + +from __future__ import annotations + +import json +import os +import time +from collections import Counter +from copy import deepcopy +from typing import Any +from uuid import uuid4 + +import pytest + +from durable_workflow import ActivityRetryPolicy, Client, Worker, activity, workflow +from durable_workflow.workflow import CompleteWorkflow, replay + +CASES = { + "generated-default-cap": [60], + "short-list": [1, 2, 2, 2], + "empty-list": [0, 0], +} +ACTIVITY = "tests.python-retry-cap" +WORKFLOW = "tests.python-local-retry-cap" + + +def selected_policy(mode: str) -> dict[str, Any]: + if mode == "generated-default-cap": + return ActivityRetryPolicy.exponential(max_attempts=2, initial_interval_seconds=61).to_dict() + if mode == "short-list": + return {"max_attempts": 5, "backoff_seconds": [1, 2]} + assert mode == "empty-list" + return {"max_attempts": 3, "backoff_seconds": []} + + +@workflow.defn(name=WORKFLOW) +class LocalRetryCapWorkflow: + def run(self, ctx: Any, mode: str) -> Any: + result = yield ctx.local_activity(ACTIVITY, [mode], retry_policy=selected_policy(mode)) + return {"mode": mode, "result": result} + + +def verify_retry( + mode: str, callbacks: list[dict[str, Any]], commands: list[dict[str, Any]], history: list[dict], +) -> None: + backoff = CASES[mode] + count = len(backoff) + 1 + assert all(type(item["attempt"]) is int for item in callbacks) + assert [item["attempt"] for item in callbacks] == list(range(1, count + 1)) + assert all(type(item["at_ns"]) is int and item["at_ns"] > 0 for item in callbacks) + gaps = [later["at_ns"] - earlier["at_ns"] for earlier, later in zip(callbacks[:-1], callbacks[1:], strict=True)] + assert all(gap >= 0 for gap in gaps) + # A monotonic clock observes actual callbacks, separately from reported + # durations. Scheduling may add time; it must not erase a declared wait. + assert all(gap >= delay * 1_000_000_000 - 10_000_000 for gap, delay in zip(gaps, backoff, strict=True)) + assert [command["type"] for command in commands] == ["record_local_activity", "complete_workflow"] + recorded = commands[0] + assert recorded["activity_type"] == ACTIVITY + assert recorded["execution_mode"] == "local" + assert recorded["retry_policy"] == selected_policy(mode) + assert recorded["outcome"] == "completed" + assert all(type(item["attempt_number"]) is int for item in recorded["attempts"]) + assert [item["attempt_number"] for item in recorded["attempts"]] == list(range(1, count + 1)) + assert [item["outcome"] for item in recorded["attempts"]] == [*(["failed"] * len(backoff)), "completed"] + assert [item.get("backoff_seconds") for item in recorded["attempts"]] == [*backoff, None] + events = [event for event in history if event["event_type"].startswith("Activity")] + assert Counter(event["event_type"] for event in events) == { + "ActivityScheduled": 1, "ActivityStarted": count, + "ActivityRetryScheduled": len(backoff), "ActivityCompleted": 1, + } + assert all(event["payload"]["activity_type"] == ACTIVITY for event in events) + assert all(event["payload"]["execution_mode"] == "local" for event in events) + assert all(event["payload"]["activity"]["retry_policy"] == selected_policy(mode) for event in events) + retries = [event["payload"] for event in events if event["event_type"] == "ActivityRetryScheduled"] + assert [item["retry_after_attempt"] for item in retries] == list(range(1, count)) + assert [item["retry_backoff_seconds"] for item in retries] == backoff + + +async def observe_retry(client: Client, server_url: str, server_token: str, mode: str) -> dict[str, Any]: + callbacks = [] + commands = [] + + class ObservedClient(Client): + async def complete_workflow_task(self, **kwargs: Any) -> Any: + commands.extend(deepcopy(kwargs["commands"])) + return await super().complete_workflow_task(**kwargs) + + @activity.defn(name=ACTIVITY) + async def attempt(selected: str) -> str: + assert selected == mode + callbacks.append({"attempt": activity.context().info.attempt_number, "at_ns": time.monotonic_ns()}) + if len(callbacks) <= len(CASES[mode]): + raise RuntimeError("fixed ordinary local retry") + return "retained" + + queue = "python-retry-cap-" + uuid4().hex + async with ObservedClient(server_url, token=server_token, namespace="default") as worker_client: + worker = Worker(worker_client, task_queue=queue, workflows=[LocalRetryCapWorkflow], activities=[attempt]) + handle = await client.start_workflow( + workflow_type=WORKFLOW, workflow_id=queue, task_queue=queue, input=[mode], + ) + try: + status = await worker.run_until(workflow_id=queue, timeout=100 if mode == "generated-default-cap" else 30) + assert status.status == "completed" + finally: + await worker.stop() + assert handle.run_id + result = await handle.result(timeout=10) + assert result == {"mode": mode, "result": "retained"} + history = [event async for event in client.iter_history(queue, handle.run_id)] + exported = await client.export_history(queue, handle.run_id) + assert exported["workflow"]["instance_id"] == queue + assert exported["workflow"]["run_id"] == handle.run_id + assert exported["workflow"]["details_state"] == "retained" + assert exported["history_complete"] is True + assert exported["redaction"]["applied"] is False + verify_retry(mode, callbacks, commands, history) + retained = deepcopy(history) + + def refuse_callback(_command): + pytest.fail("committed local retry must replay without callback execution") + + outcome = replay( + LocalRetryCapWorkflow, history, [mode], workflow_id=queue, run_id=handle.run_id, + local_activity_executor=refuse_callback, + ) + try: + assert len(outcome.commands) == 1 + assert isinstance(outcome.commands[0], CompleteWorkflow) + assert outcome.commands[0].result == result + assert history == retained + finally: + outcome.close() + assert [event async for event in client.iter_history(queue, handle.run_id)] == retained + return {"mode": mode, "workflow_id": queue, "run_id": handle.run_id, + "callbacks": callbacks, "commands": commands, "history": history, "export": exported} + + +@pytest.mark.skipif(os.environ.get("DURABLE_WORKFLOW_RETRY_CAP_CELL") != "1", reason="reviewed package cell only") +async def test_installed_local_retries_preserve_cap_tail_and_committed_replay( + server_url: str, server_token: str, +) -> None: + observations = [] + async with Client(server_url, token=server_token, namespace="default") as client: + for mode in CASES: + observations.append(await observe_retry(client, server_url, server_token, mode)) + print(json.dumps({"scope": "ordinary scalar local retries", "observations": observations}, sort_keys=True)) diff --git a/tests/test_activity_retry_cap_cell.py b/tests/test_activity_retry_cap_cell.py new file mode 100644 index 0000000..0fe2418 --- /dev/null +++ b/tests/test_activity_retry_cap_cell.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from copy import deepcopy + +import pytest +from scripts.ci import activity_acceptance_release as release + +from tests.integration import test_activity_retry_cap as cell + + +def observations(mode): + delays = cell.CASES[mode] + count = len(delays) + 1 + clock = 1_000_000_000 + callbacks = [{"attempt": 1, "at_ns": clock}] + for attempt, delay in enumerate(delays, start=2): + clock += delay * 1_000_000_000 + callbacks.append({"attempt": attempt, "at_ns": clock}) + policy = cell.selected_policy(mode) + commands = [{ + "type": "record_local_activity", "activity_type": cell.ACTIVITY, "execution_mode": "local", + "retry_policy": policy, "outcome": "completed", "attempts": [ + {"attempt_number": attempt, "outcome": "failed", "backoff_seconds": delay} + for attempt, delay in enumerate(delays, start=1) + ] + [{"attempt_number": count, "outcome": "completed"}], + }, {"type": "complete_workflow"}] + payload = {"activity_type": cell.ACTIVITY, "execution_mode": "local", "activity": {"retry_policy": policy}} + history = [{"event_type": "ActivityScheduled", "payload": deepcopy(payload)}] + for attempt in range(1, count + 1): + history.append({"event_type": "ActivityStarted", "payload": deepcopy(payload)}) + if attempt < count: + history.append({"event_type": "ActivityRetryScheduled", "payload": { + **deepcopy(payload), "retry_after_attempt": attempt, "retry_backoff_seconds": delays[attempt - 1], + }}) + history.append({"event_type": "ActivityCompleted", "payload": deepcopy(payload)}) + return callbacks, commands, history + + +@pytest.mark.parametrize("mode", cell.CASES) +def test_exact_policy_attempt_history_and_monotonic_observations_pass_without_mutation(mode): + values = observations(mode) + original = deepcopy(values) + cell.verify_retry(mode, *values) + assert values == original + + +@pytest.mark.parametrize("fault", [ + "actual_wait", "tail_report", "attempt_order", "extra_command", "policy", "history", "callback_bool", "report_bool", +]) +def test_incomplete_or_invented_retry_evidence_refuses(fault): + callbacks, commands, history = observations("short-list") + if fault == "actual_wait": + callbacks[-1]["at_ns"] = callbacks[-2]["at_ns"] + elif fault == "tail_report": + commands[0]["attempts"][-2]["backoff_seconds"] = 0 + elif fault == "attempt_order": + callbacks[-1]["attempt"] = 1 + elif fault == "callback_bool": + callbacks[0]["attempt"] = True + elif fault == "report_bool": + commands[0]["attempts"][0]["attempt_number"] = True + elif fault == "extra_command": + commands.append({"type": "start_timer"}) + elif fault == "policy": + commands[0]["retry_policy"] = {"max_attempts": 5, "backoff_seconds": [1, 2, 0, 0]} + else: + next(event for event in history if event["event_type"] == "ActivityRetryScheduled")["payload"][ + "retry_backoff_seconds" + ] = 0 + with pytest.raises(AssertionError): + cell.verify_retry("short-list", callbacks, commands, history) + + +def test_empty_backoff_does_not_admit_a_backwards_monotonic_clock(): + callbacks, commands, history = observations("empty-list") + callbacks[-1]["at_ns"] -= 1 + with pytest.raises(AssertionError): + cell.verify_retry("empty-list", callbacks, commands, history) + + +def approval(digest="a" * 64): + return {"version": release.VERSION, "files": {release.WHEEL: digest, release.SDIST: "b" * 64}} + + +def test_effective_lock_adds_only_the_approved_sdk_to_the_fixed_dependencies(): + result, digest = release.lock(b"pytest==1 --hash=sha256:" + b"c" * 64 + b"\n", approval()) + assert result.endswith(f"durable-workflow==2.18.0 --hash=sha256:{digest}\n".encode()) + assert result.count(b"durable-workflow") == 1 + + +@pytest.mark.parametrize("fault", ["version", "digest", "member", "sdk_in_dependencies"]) +def test_unapproved_release_or_dependency_override_refuses(fault): + record = approval() + dependencies = b"pytest==1 --hash=sha256:" + b"c" * 64 + if fault == "version": + record["version"] = "2.17.2" + elif fault == "digest": + record["files"][release.WHEEL] = "wrong" + elif fault == "member": + record["files"]["foreign.whl"] = "d" * 64 + else: + dependencies += b"\ndurable-workflow==2.17.2 --hash=sha256:" + b"d" * 64 + with pytest.raises(ValueError): + release.lock(dependencies, record) + + +def test_candidate_consumption_binds_bytes_and_refuses_replacement_or_symlink(tmp_path): + import hashlib + + dist = tmp_path / "dist" + dist.mkdir() + wheel = dist / release.WHEEL + data = b"fixed inert candidate bytes" + wheel.write_bytes(data) + digest = hashlib.sha256(data).hexdigest() + assert release.candidate_wheel(tmp_path, digest) == data + wheel.write_bytes(b"replacement") + with pytest.raises(ValueError, match="differs from the approved"): + release.candidate_wheel(tmp_path, digest) + wheel.unlink() + foreign = tmp_path / "foreign" + foreign.write_bytes(data) + wheel.symlink_to(foreign) + with pytest.raises(OSError): + release.candidate_wheel(tmp_path, digest) diff --git a/tests/test_retry_cap_download.py b/tests/test_retry_cap_download.py new file mode 100644 index 0000000..4612514 --- /dev/null +++ b/tests/test_retry_cap_download.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +from types import SimpleNamespace + +import pytest +from scripts.ci.activity_acceptance_output import Output + +from tests.test_published_activity_acceptance import runner + + +@pytest.mark.parametrize("artifact_source", ["candidate", "registry"]) +def test_closed_retry_download_uses_the_same_approved_bytes_and_complete_final_lock( + tmp_path, monkeypatch, artifact_source, +): + release = runner.release + data = b"inert candidate marker; never installed" + digest = hashlib.sha256(data).hexdigest() + approval = {"version": release.VERSION, "files": {release.WHEEL: digest, release.SDIST: "b" * 64}} + dependencies = b"anyio==1 --hash=sha256:" + b"c" * 64 + b"\n" + captured = {runner.REQUIREMENTS: dependencies, "release.json": json.dumps(approval).encode()} + for filename in (runner.VERIFIER_FILENAME, runner.SCENARIOS["retry-cap"][0], "conftest.py"): + captured[filename] = b"fixed inert source" + dist = tmp_path / "dist" + dist.mkdir() + (dist / release.WHEEL).write_bytes(data) + monkeypatch.setattr(runner, "ROOT", tmp_path) + monkeypatch.setattr(runner, "OUTPUT", tmp_path / "output") + monkeypatch.setattr(runner, "SCENARIO", "retry-cap") + monkeypatch.setattr(runner, "ARTIFACT_SOURCE", artifact_source) + requests = [] + + def call(*arguments, **kwargs): + if arguments[0] == "git": + return SimpleNamespace(stdout=json.dumps(approval)) + assert arguments[1:5] == ("-I", "-m", "pip", "download") + assert "--require-hashes" in arguments + assert "--only-binary=:all:" in arguments + assert kwargs["env"]["PIP_INDEX_URL"] == "https://pypi.org/simple" + selected_lock = (runner.OUTPUT / arguments[-1]).read_bytes() + requests.append(selected_lock) + wheels = runner.OUTPUT / "wheels" + (wheels / "dependency.whl").write_bytes(b"inert dependency marker") + if artifact_source == "registry": + (wheels / release.WHEEL).write_bytes(data) + return SimpleNamespace(stdout="fixed download result", stderr="") + + monkeypatch.setattr(runner, "call", call) + files = Output(runner.OUTPUT) + try: + result = runner.download_wheels({"sdk_version": release.VERSION}, captured, files) + assert len(requests) == 1 + assert (b"durable-workflow" in requests[0]) is (artifact_source == "registry") + installed_lock = base64.b64decode(result["files"][runner.REQUIREMENTS]["base64"]) + assert installed_lock == release.lock(dependencies, approval)[0] + assert result["wheel_sha256"] == digest + assert base64.b64decode(result["wheels"][release.WHEEL]["base64"]) == data + assert (runner.OUTPUT / "wheels" / release.WHEEL).read_bytes() == data + finally: + files.close() + + +@pytest.mark.parametrize("scenario", ["activity-identity", "timer-redrive"]) +def test_candidate_mode_cannot_expand_an_existing_published_case(monkeypatch, scenario): + monkeypatch.setattr(runner, "SCENARIO", scenario) + monkeypatch.setattr(runner, "ARTIFACT_SOURCE", "candidate") + monkeypatch.setattr(runner, "call", lambda *a, **k: pytest.fail("refuse before Git, download or Docker")) + with pytest.raises(ValueError, match="fixed retry-cap case"): + runner.download_wheels({}, {}, None) diff --git a/tests/test_retry_cap_workflow.py b/tests/test_retry_cap_workflow.py new file mode 100644 index 0000000..656e1e0 --- /dev/null +++ b/tests/test_retry_cap_workflow.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from tests.test_activity_acceptance_consumer import consumer +from tests.test_published_activity_acceptance import runner + +ROOT = Path(__file__).parents[1] + + +def test_retry_effect_is_disabled_in_ordinary_ci_and_candidate_access_is_read_only(): + configuration = yaml.load((ROOT / ".github/workflows/ci.yml").read_text(), Loader=yaml.BaseLoader) + inputs = configuration["on"]["workflow_dispatch"]["inputs"] + assert inputs["activity_retry_cap"]["default"] == "false" + assert inputs["retry_cap_artifact_source"]["options"] == ["candidate", "registry"] + job = configuration["jobs"]["activity-retry-cap"] + assert job["if"] == "github.event_name == 'workflow_dispatch' && inputs.activity_retry_cap" + assert job["permissions"] == {"contents": "read", "actions": "read"} + resolver = next(step for step in job["steps"] if step.get("id") == "retry-candidate") + assert resolver["if"] == "inputs.retry_cap_artifact_source == 'candidate'" + assert resolver["env"] == {"GH_TOKEN": "${{ github.token }}"} + assert resolver["run"].endswith(" --version " + runner.release.VERSION) + selected = json.loads((ROOT / "tests/fixtures/published-retry-cap.json").read_text()) + assert selected["sdk_version"] == runner.release.VERSION + + +def test_retry_case_has_closed_packet_membership_and_cannot_borrow_identity_source(): + packet = { + "scenario": "retry-cap", "sdk_version": "2.18.0", "wheel_name": "sdk.whl", "wheel_sha256": "a" * 64, + "wheels": {"sdk.whl": {"sha256": "a" * 64}}, + "files": dict.fromkeys(( + "requirements.txt", "verify_activity_install.py", "conftest.py", "test_activity_retry_cap.py", + ), {}), + } + consumer.validate_setup(packet) + packet["files"]["test_activity_identity.py"] = packet["files"].pop("test_activity_retry_cap.py") + with pytest.raises(ValueError, match="membership differs"): + consumer.validate_setup(packet) + + +def test_retry_result_cannot_borrow_another_cases_passing_junit(monkeypatch): + monkeypatch.setattr(runner, "SCENARIO", "retry-cap") + accepted = runner.SCENARIOS["retry-cap"][1] + runner.verify_case_result({"complete": True, "junit": f''}) + other = runner.SCENARIOS["activity-identity"][1] + with pytest.raises(ValueError, match="inventory differs"): + runner.verify_case_result({"complete": True, "junit": f''})