diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a70514..41f11e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -208,3 +208,24 @@ jobs: dist/SHA256SUMS.txt if-no-files-found: error retention-days: 7 + + ci-required: + if: always() + needs: + - test + - lifecycle-subprocess + - package + - real-hermes + - audit-docker-isolation + - macos-process-lifecycle + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + - name: Require successful CI dependencies + env: + ZEUS_CI_NEEDS_JSON: ${{ toJSON(needs) }} + run: python scripts/check_ci_required.py diff --git a/README.md b/README.md index fe6bd97..655a747 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,90 @@ The matching authenticated routes are `GET /reconcile/runs`, `GET /reconcile/runs/`, and `GET /fleet`. See [API](docs/API.md) and [reconciliation](docs/RECONCILE.md) for filters and freshness semantics. +## Quick Start + +### 1. Credential-free offline demo + +The fastest first success needs no Hermes installation, Docker, or provider +credentials. From a checkout: + +```bash +python3 -m venv .venv +. .venv/bin/activate +python -m pip install -e . + +zeus demo up +zeus demo status +zeus demo down +``` + +The demo uses Zeus' packaged fake-Hermes executable and stores its disposable +runtime under `ZEUS_STATE_DIR` (the workspace-local `.zeus/` directory by +default). It exercises real profile rendering and process lifecycle behavior +without contacting a provider. + +### 2. Real Hermes setup + +Check the installed Hermes version, then prepare a private workspace secret +file: + +```bash +hermes --version +cp .env.example .env +chmod 0600 .env +``` + +`.env.example` contains empty placeholders and is not ready to import. Stop here +until `.env` contains a real, non-empty provider key required by the selected +template, such as `OPENROUTER_API_KEY` for `coding-bot`. As an alternative, +provide the same named secret through a secure process-environment mechanism. + +Then validate Zeus and render the real Hermes profile: + +```bash +zeus doctor +zeus template list +zeus bot create coder --template coding-bot --env-from OPENROUTER_API_KEY +zeus bot doctor coder +``` + +`--env-from NAME` imports a named value from the process environment first and +then the trusted workspace `./.env`; the value never enters the Zeus argument +list or command output. A present but empty process value is an error and does +not fall back to `.env`. Keep the workspace `.env` private with `chmod 0600 .env`. +The legacy `--env NAME=VALUE` form remains available +for non-secret compatibility values, but is unsafe for secrets because command +arguments can be retained in shell history and exposed in process listings. + +Safety model: Zeus is a local process orchestrator, not a sandbox. Use Docker or +another Hermes terminal backend for untrusted tasks. Do not expose the API +directly to a network; keep it on loopback or behind a separately hardened +access layer. Logs and audit events may contain sensitive operational data, so +protect and rotate `$ZEUS_STATE_DIR`. + +Start the local API with an explicit key: + +```bash +ZEUS_API_KEY=change-me sh scripts/start.sh +``` + +## 60-Second Demo + +The pre-recorded asciinema cast in [docs/assets/demo.cast](docs/assets/demo.cast) +illustrates the local operator flow. It is not evidence that the current Zeus +checkout is compatible with whichever Hermes version is installed today; use +the live verification steps below for that evidence. + +```bash +zeus doctor +zeus template list +zeus bot create coder --template coding-bot +zeus bot start coder +zeus bot status coder +zeus bot logs coder +zeus bot stop coder +``` + ## Repository Audit `zeus audit` is a report-only, host-local review of the exact committed `HEAD`. @@ -205,90 +289,6 @@ do not count as security coverage. Commands carrying `control_ids` run with the committed snapshot read-only; configure those tools to place caches and build output under `/tmp`. -## Quick Start - -### 1. Credential-free offline demo - -The fastest first success needs neither Hermes nor provider credentials. From a -checkout: - -```bash -python3 -m venv .venv -. .venv/bin/activate -python -m pip install -e . - -zeus demo up -zeus demo status -zeus demo down -``` - -The demo uses Zeus' packaged fake-Hermes executable and stores its disposable -runtime under `ZEUS_STATE_DIR` (the workspace-local `.zeus/` directory by -default). It exercises real profile rendering and process lifecycle behavior -without contacting a provider. - -### 2. Real Hermes setup - -Check the installed Hermes version, then prepare a private workspace secret -file: - -```bash -hermes version -cp .env.example .env -chmod 0600 .env -``` - -`.env.example` contains empty placeholders and is not ready to import. Stop here -until `.env` contains a real, non-empty provider key required by the selected -template, such as `OPENROUTER_API_KEY` for `coding-bot`. As an alternative, -provide the same named secret through a secure process-environment mechanism. - -Then validate Zeus and render the real Hermes profile: - -```bash -zeus doctor -zeus template list -zeus bot create coder --template coding-bot --env-from OPENROUTER_API_KEY -zeus bot doctor coder -``` - -`--env-from NAME` imports a named value from the process environment first and -then the trusted workspace `./.env`; the value never enters the Zeus argument -list or command output. A present but empty process value is an error and does -not fall back to `.env`. Keep the workspace `.env` private with `chmod 0600 .env`. -The legacy `--env NAME=VALUE` form remains available -for non-secret compatibility values, but is unsafe for secrets because command -arguments can be retained in shell history and exposed in process listings. - -Safety model: Zeus is a local process orchestrator, not a sandbox. Use Docker or -another Hermes terminal backend for untrusted tasks. Do not expose the API -directly to a network; keep it on loopback or behind a separately hardened -access layer. Logs and audit events may contain sensitive operational data, so -protect and rotate `$ZEUS_STATE_DIR`. - -Start the local API with an explicit key: - -```bash -ZEUS_API_KEY=change-me sh scripts/start.sh -``` - -## 60-Second Demo - -The pre-recorded asciinema cast in [docs/assets/demo.cast](docs/assets/demo.cast) -illustrates the local operator flow. It is not evidence that the current Zeus -checkout is compatible with whichever Hermes version is installed today; use -the live verification steps below for that evidence. - -```bash -zeus doctor -zeus template list -zeus bot create coder --template coding-bot -zeus bot start coder -zeus bot status coder -zeus bot logs coder -zeus bot stop coder -``` - ## Documentation - [Architecture](docs/ARCHITECTURE.md) diff --git a/docs/API.md b/docs/API.md index 24fdac4..f8b32ed 100644 --- a/docs/API.md +++ b/docs/API.md @@ -243,7 +243,7 @@ Returns: Authenticated state-store readiness check, also available as `GET /v1/ready`. It opens the existing SQLite database in read-only mode, requires schema version -6, and executes `SELECT 1`; it does not inspect or start bots. A stopped bot does +10, and executes `SELECT 1`; it does not inspect or start bots. A stopped bot does not make Zeus unready. The route uses the normal read-endpoint authentication policy. It requires @@ -254,7 +254,7 @@ probe. Success returns: ```json -{"schema_version":9,"status":"ready"} +{"schema_version":10,"status":"ready"} ``` An unavailable, missing, malformed, older, or newer state database returns diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 2965a97..545451d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -33,7 +33,9 @@ Set `ZEUS_STATE_DIR` to use a different runtime root. - `zeus.templates`: Bundled plus local TOML template discovery with duplicate ID checks. - `zeus.renderer`: Hermes profile rendering. - `zeus.sqlite_db`: Shared SQLite connection factory and per-connection durability policy. -- `zeus.schema`: Schema-v6 initialization, compatibility guards, and forward migrations. +- `zeus.schema`: Schema-v10 initialization, compatibility guards, and forward migrations. +- `zeus.message_store`: Durable job receipts, read-only capacity observations, + and bounded logical archival that retains replay identities. - `zeus.idempotency_store`: Durable API mutation claims and replay responses. - `zeus.reconcile_store`: Persisted fleet reconciliation runs and ordered results. - `zeus.bot_lifecycle_store`: Bot projection, intent, lifecycle ledger, history, and audit mirror. @@ -55,9 +57,15 @@ Set `ZEUS_STATE_DIR` to use a different runtime root. - `zeus.gateway_runtime`: Public process-effects facade; launch, marker, ownership, stop, and low-level process helpers are isolated behind it. - `zeus.intent_recovery`: Store-free pending-intent recovery decisions through a structural host. -- `zeus.supervisor`: Public lifecycle compatibility facade. Focused internal - modules own core coordination, runtime compatibility, start, stop/restart, - reconciliation/recovery, status/inspection, and registry/profile operations. +- `zeus.supervisor`: Public lifecycle facade with explicit delegates to five + stateless operation services for registry, status, start, stop, and reconcile. + Each service receives the current supervisor through a narrow typed host + interface; callbacks are resolved when called so supported overrides remain live. +- `zeus.supervisor_runtime`: The single concrete supervisor core owns construction, + locks, event coordination, and runtime compatibility properties. The old + `supervisor_core` import path and `_SupervisorRuntime` name remain compatibility + aliases. `GatewayRuntime` retains process effects; `ProfileManager` retains + profile transactions; `PendingIntentRecovery` retains bounded recovery decisions. - `zeus.api`: Local HTTP routes and compatibility facade. - `zeus.cli`: Operator CLI. - `zeus.audit_*`: Native, report-only audit components for committed `HEAD` @@ -242,13 +250,20 @@ The v2-to-v3 migration is also one transaction. It creates a the projection/event invariant, and advances the schema version only after all steps succeed. Additive v3-to-v4 and v4-to-v5 upgrades add durable idempotency and desired/pending intent in forward-only transactions. Databases newer than -schema v9 are rejected rather than downgraded. +schema v10 are rejected rather than downgraded. Schema v8 adds operator-message receipts without rewriting bot projections or events. A FULL-synchronous transaction reserves a stable upstream idempotency key before an HTTP request; no database lock spans network I/O. Attempt leases and compare-and-swap versions prevent overlapping retries or stale acknowledgements. Schema v9 adds a nullable release timestamp without rewriting existing receipts. +Schema v10 adds logical archival to the same table. Only rejected or accepted +terminal receipts can be archived; unresolved and released nonterminal work +remains ineligible. Archival frees admission slots while retaining complete +identity, historical replay lookup, and outcomes. It advances the concurrency +version without refreshing gateway observation timestamps and does not reclaim +disk space. Capacity reporting and archive preview read existing state without +migrations, workflow construction, or gateway calls. One unreleased unresolved/nonterminal receipt is admitted per bot incarnation. The captured process generation and launch-bound messaging policy are checked around network operations. See [operator messaging](MESSAGING.md) for recovery diff --git a/docs/COMPATIBILITY.md b/docs/COMPATIBILITY.md index ff69f17..84cca8b 100644 --- a/docs/COMPATIBILITY.md +++ b/docs/COMPATIBILITY.md @@ -29,7 +29,11 @@ platform guarantee. Python 3.14 is a provisional Zeus-only lane with `continue-on-error` behavior. It does not promote Python 3.14 to required Hermes compatibility: the repository pins Hermes Agent 0.21.0, whose package metadata requires Python 3.11 through -3.13, and runs that compatibility gate only on Python 3.11. +3.13, and runs that compatibility gate only on Python 3.11. The `ci-required` +merge aggregate therefore covers the supported Python 3.11 through 3.13 matrix +and the five focused required jobs, without the provisional Python 3.14 lane. +Tagged release promotion applies a stricter evidence policy and separately +requires the Python 3.14 job to have succeeded for the exact release commit. The package metadata declares `requires-python = ">=3.11"`, while committed CI currently tests the versions listed above. A version absent from that matrix is @@ -52,10 +56,24 @@ setting, to persist dispatch intent before submitting a job. The synchronous policy itself does not change database structure. Zeus v0.6 adds an independent forward-only migration from schema v6 to schema v7 for operator-query indexes. Schema v8 then adds durable operator-message receipts. -Schema v9 adds explicit local release timestamps to those receipts. -Existing v6/v7/v8 databases upgrade during normal startup; +Schema v9 adds explicit local release timestamps to those receipts. Schema v10 +adds nullable logical archival timestamps and an unarchived selection index, +preserving existing rows, unique keys, and the active-target uniqueness rule. +Zeus `0.6.1.dev0` now reports readiness schema 10. Existing v6/v7/v8/v9 databases +upgrade transactionally during normal startup; read-only history/fleet commands require the current schema. Keep all writers -on the same Zeus version and retain a quiesced backup for rollback. +on the same Zeus build and retain a quiesced backup for rollback. Stop every +writer before backup and upgrade. An interrupted migration rolls back both schema +and data. Older schema-9 binaries reject schema 10; rollback restores the complete +pre-upgrade backup with its matching binary, rather than downgrading metadata. + +Olymp must explicitly accept the Zeus `0.6.1.dev0` / readiness-schema-10 pairing +before this build is deployed with it. The same development package version may +exist with schema 9, so a matching version string alone is insufficient evidence. +No compatibility with an unchanged schema-9-only Olymp client is claimed. +Message capacity/archive commands require current state and never migrate it. +Archival restores only the unarchived admission allowance; all receipt identity +and deduplication history remain retained and disk space is not reclaimed. ## Manual clean-host evidence @@ -112,7 +130,7 @@ rendered profile, environment, logs, or process arguments. The manual [`scripts/verify_real_hermes.sh`](../scripts/verify_real_hermes.sh) check still uses whichever `hermes` executable is installed on `PATH` unless -`ZEUS_VERIFY_EXPECTED_HERMES_VERSION` is set. Record `hermes version` with manual +`ZEUS_VERIFY_EXPECTED_HERMES_VERSION` is set. Record `hermes --version` with manual evidence. Passing the pinned baseline does not establish compatibility with every Hermes release or optional integration. diff --git a/docs/MESSAGING.md b/docs/MESSAGING.md index 3f3f3d2..4b90c87 100644 --- a/docs/MESSAGING.md +++ b/docs/MESSAGING.md @@ -55,6 +55,7 @@ zeus message send coder --file request.txt --request-key incident-104 --json zeus message list --bot-id coder --limit 20 --json zeus message status --json zeus message cancel --json +zeus message capacity --json ``` Input must be a regular UTF-8 file containing nonblank text, at most 16,000 @@ -64,6 +65,15 @@ not accepted as a shell argument. Submission returns a receipt immediately; characters. Use a returned `next_before` cursor with `list --before ` to inspect older receipts. +`message capacity` observes the existing receipt database without initializing, +migrating, checkpointing, or reconciling state and makes no gateway request. It +reports the fixed 10,000-unarchived-receipt limit, unarchived `used` receipts, +remaining capacity, retained `total`, `archived`, active per-incarnation admission +blockers, and database, WAL, and filesystem-free byte observations. Capacity status changes to `warning` at 80%, `critical` at 95%, +and `full` at 100%. A full but valid database is a successful observation and +exits zero. Missing, incompatible, or malformed state fails closed with a nonzero +exit. Unavailable filesystem size observations appear as `null` in JSON. + `accepted` means Hermes acknowledged the run. It does not mean the job completed successfully. Status distinguishes queued, running, waiting for approval, stopping, completed, failed, cancelled and interrupted runs. Zeus reports @@ -86,8 +96,8 @@ persisted run status stale; it is not automatically treated as completed. Zeus commits a receipt with SQLite `synchronous=FULL` before submitting the job. The receipt stores hashes and bounded routing/run metadata, never the input, output or API key. These writes use FULL even when ordinary Zeus state uses -NORMAL. Receipts survive bot deletion; capacity is 10,000 records, with no -automatic pruning. Back up the database together with private profiles. +NORMAL. Receipts survive bot deletion; admission capacity is 10,000 unarchived +records, with no automatic archival or pruning. Back up the database together with private profiles. A lost response or changed gateway generation leaves an `unknown` receipt. An interrupted submit can retain `prepared` if it stops before recording its outcome. @@ -142,7 +152,7 @@ A failed stop request does not refresh `last_checked_at` or the cached run statu These checks avoid automatic duplicate dispatch, but cannot guarantee exactly-once external tool effects after a crash or an upstream persistence failure. No message -commands initialize/migrate Zeus state or start/reconcile bots. Schema 9 must +commands initialize/migrate Zeus state or start/reconcile bots. Schema 10 must already have been initialized by ordinary startup. There are no new Zeus HTTP messaging routes in this version. @@ -150,3 +160,41 @@ Before dispatch, Zeus rechecks receipt ownership and leaves the complete two-sec HTTP budget inside its lease and retry window. A local process can still be suspended between that check and sending bytes; the trusted-host boundary and Hermes's finite idempotency retention remain part of the recovery limits. + + +### Logical receipt archival + +Preview a bounded batch, then explicitly apply archival when appropriate: + +```sh +zeus message archive --json +zeus message archive --before 2026-01-01T00:00:00+00:00 --limit 100 --apply --json +``` + +The default cutoff is 30 days ago. `--before` must be a timezone-aware ISO timestamp +no later than the current time; eligibility uses `updated_at` strictly before that +cutoff, so the exact boundary is excluded. `--limit` accepts 1–500, default 100. +Selection is deterministic by `updated_at`, then `message_id`. The JSON response +contains `message_ids`, `count`, `before`, and `applied`. Preview is read-only and +reserves nothing; apply reselects and strictly validates the complete batch inside +one FULL-durable transaction. A corrupt selected receipt or storage failure rolls +back the entire batch. + +Only rejected dispatches and accepted runs last observed as completed, failed, +cancelled, or interrupted qualify. Prepared/unknown dispatches, nonterminal runs, +and released nonterminal runs never qualify. Age, bot deletion, an unavailable +gateway, an expired upstream result, or release alone do not prove completion. + +Archival stamps `archived_at` and advances the receipt's version without changing +its original observations, run identity, outcome, or deduplication keys. It restores +admission capacity while retaining the complete history for `list`, `status`, and +request-key lookup. Compatible request replay and accepted retry still return the +original receipt without resubmission, even when capacity is full; changed input +or target conflicts. Stale writers fail version checks. Clock checks include the +archive time. A later observation of the same terminal status may advance +`updated_at` while preserving `archived_at`; terminal outcomes cannot regress. + +Logical archival does not delete rows, compact/checkpoint SQLite, or free disk +space. Monitor the reported database/WAL sizes and available filesystem bytes +separately. Neither preview nor apply constructs a messaging workflow, contacts +Hermes, or initializes/migrates state. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 846be7e..3ec3245 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -46,7 +46,7 @@ when `ZEUS_ALLOW_UNAUTH_READS=1` is explicitly enabled. It opens the existing database read-only, requires the current schema version, and runs `SELECT 1`. It never creates or migrates a database and does not require bots to be running. -A ready service returns `{"schema_version":9,"status":"ready"}`. State-store +A ready service returns `{"schema_version":10,"status":"ready"}`. State-store failures return `503` with `error.code=not_ready`. If state initialization fails before the API binds, the process exits instead of serving `/ready`. @@ -536,6 +536,45 @@ cannot supply those environment keys, and stored profile assignments are rejected before launch. This prevents Hermes's dotenv loader from undoing Zeus's process ownership policy. +### Message receipt capacity + +Run `zeus message capacity --json` to observe local receipt usage before it +reaches the fixed 10,000-unarchived-record admission limit. The report separates +unarchived `used`, `archived`, retained `total`, and active admission blockers and includes database, WAL, and +available-filesystem byte observations. Status is `ok` below 80%, `warning` from +80%, `critical` from 95%, and `full` from 100%. A valid `full` report exits zero; +state errors exit nonzero. This command is read-only and does not initialize, +migrate, checkpoint, reconcile, or contact Hermes. A missing WAL is reported as +zero when its absence can be confirmed; unavailable size observations are null. + +Preview eligible old receipts with `zeus message archive --json`. Apply a fresh +selection with `zeus message archive --limit 100 --apply --json`; repeat bounded +batches when more eligible history remains. The default cutoff is 30 days ago; +`--before` accepts a timezone-aware ISO timestamp no later than now, and `--limit` +is 1–500. Eligibility requires `updated_at` strictly before the cutoff plus a +rejected dispatch or an accepted terminal outcome. Released nonterminal and +uncertain receipts remain in use. Preview is not a reservation. Apply validates +and commits the entire batch with FULL durability, preserving deduplication keys, +receipt identity, and observations while incrementing versions for stale-writer +protection. Later terminal observations preserve the archive timestamp. + +Archival only restores admission slots. It does not remove historical rows, +vacuum, checkpoint, compact, or recover disk space. A full filesystem can still +prevent a durable dispatch reservation or archive transaction. Storage failures +fail closed; no new submission proceeds without durable intent. Preserve enough +free storage for normal database/WAL growth and backups. + +Zeus `0.6.1.dev0` requires schema 10. Before upgrading schema 9, quiesce every +writer and keep a consistent backup of the complete state tree and private +profiles using the backup procedure above. The normal initialization/upgrade +path performs the additive transactional migration; message commands never +migrate. Resume only matching Zeus binaries and an Olymp build explicitly +compatible with Zeus `0.6.1.dev0` and readiness schema 10. Schema-9-only clients +must not be treated as compatible because the package version is unchanged. +Older Zeus binaries reject schema 10. Rollback requires restoring the quiesced +pre-upgrade backup with its matching binary, not editing the schema version or +deleting the new column. See [compatibility policy](COMPATIBILITY.md). + ### Live gateway diagnostics Use `zeus bot diagnostics --json` or the authenticated diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 41befb2..0f851db 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -125,10 +125,34 @@ requests, and manual runs do not qualify. Its current attempt must contain successful Python 3.11/3.12/3.13 matrix jobs, Python 3.14, subprocess lifecycle, Docker isolation, real Hermes, macOS lifecycle, and package jobs. Python 3.14 remains provisional in ordinary CI but is required for release promotion. +The release verifier also requires the successful `ci-required` aggregate job. +That job runs with `always()` after the supported-version matrix, subprocess +lifecycle, package, real-Hermes, Docker-isolation, and macOS-lifecycle jobs. It +fails unless every one of those six dependency results is explicitly +`success`, so a skipped, cancelled, missing, failed, or malformed result cannot +turn into a green aggregate check. If a partial rerun omits required jobs, rerun all jobs before retrying the release build. An older successful run cannot override a newer failed or unfinished run for the same commit. +Adding the workflow job does not change repository rules. Repository +administration is a separate action and requires separate authorization. Only +after a real `.github/workflows/ci.yml` run completes successfully, inspect the +check runs it produced and resolve the producer `app.id` for each of these exact +contexts: `real-hermes`, `audit-docker-isolation`, +`macos-process-lifecycle`, and `ci-required`. Verify that each observed producer +is GitHub Actions before using that numeric ID as the required-check `app_id`. + +Roll out those four required checks additively as `(context, app_id)` pairs. +Preserve every existing required check, strictness setting, review requirement, +ruleset condition, bypass rule, and other protection setting; this rollout must +only add the four verified pairs. After the separately authorized update, read +the repository rule back and confirm all previous settings are unchanged and +each of the four entries has both the exact context and its matching verified +GitHub Actions `app_id`. A matching context name without the verified producer +binding is insufficient. No repository-settings mutation is part of the release +workflow or this documented verification procedure. + The build job alone receives `actions: read`; its token is passed only through the environment. API requests reject redirects and use bounded response sizes, pagination, and socket timeouts; each verification step has a three-minute diff --git a/docs/openapi.json b/docs/openapi.json index 7567207..f94d423 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -105,7 +105,7 @@ "properties": { "schema_version": { "type": "integer", - "const": 9 + "const": 10 }, "status": { "type": "string", diff --git a/scripts/check_ci_required.py b/scripts/check_ci_required.py new file mode 100644 index 0000000..9f147ac --- /dev/null +++ b/scripts/check_ci_required.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Fail closed unless every required CI dependency completed successfully.""" + +from __future__ import annotations + +import json +import os +import sys +from collections.abc import Mapping +from typing import Any, TextIO + +ENVIRONMENT_VARIABLE = "ZEUS_CI_NEEDS_JSON" +MAX_INPUT_CHARACTERS = 64 * 1024 +REQUIRED_DEPENDENCIES = frozenset( + { + "test", + "lifecycle-subprocess", + "package", + "real-hermes", + "audit-docker-isolation", + "macos-process-lifecycle", + } +) + + +class CIRequiredError(RuntimeError): + """A required CI result is absent, malformed, or unsuccessful.""" + + +def _reject_constant(_value: str) -> None: + raise ValueError("non-standard JSON constant") + + +def _strict_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ValueError("duplicate JSON field") + value[key] = item + return value + + +def evaluate_needs(raw: str) -> None: + if not raw or len(raw) > MAX_INPUT_CHARACTERS: + raise CIRequiredError("invalid_needs") + try: + needs = json.loads( + raw, + object_pairs_hook=_strict_object, + parse_constant=_reject_constant, + ) + except (ValueError, RecursionError): + raise CIRequiredError("invalid_needs") from None + if not isinstance(needs, dict) or set(needs) != REQUIRED_DEPENDENCIES: + raise CIRequiredError("invalid_needs") + for dependency in REQUIRED_DEPENDENCIES: + result = needs[dependency] + if not isinstance(result, dict) or result.get("result") != "success": + raise CIRequiredError("required_job_not_successful") + + +def main( + *, + environ: Mapping[str, str] | None = None, + stdin: TextIO = sys.stdin, + stdout: TextIO = sys.stdout, + stderr: TextIO = sys.stderr, +) -> int: + active_environment = os.environ if environ is None else environ + raw = active_environment.get(ENVIRONMENT_VARIABLE) + if raw is None: + raw = stdin.read(MAX_INPUT_CHARACTERS + 1) + try: + evaluate_needs(raw) + except CIRequiredError as error: + print(f"required CI verification failed: {error}", file=stderr) + return 1 + print("Verified all required CI dependencies succeeded.", file=stdout) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_release_ci.py b/scripts/check_release_ci.py index 1823795..da0e9b2 100644 --- a/scripts/check_release_ci.py +++ b/scripts/check_release_ci.py @@ -34,6 +34,7 @@ "real-hermes", "macos-process-lifecycle", "package", + "ci-required", } ) ERROR_CODES = frozenset( diff --git a/scripts/fresh_vps_verify.sh b/scripts/fresh_vps_verify.sh index 7551aad..89ad172 100755 --- a/scripts/fresh_vps_verify.sh +++ b/scripts/fresh_vps_verify.sh @@ -369,7 +369,7 @@ run sh scripts/test.sh run sh scripts/repo_check.sh section "Hermes Diagnostics" -run hermes version +run hermes --version run hermes doctor section "Real Hermes Compatibility" diff --git a/scripts/wheel_smoke.sh b/scripts/wheel_smoke.sh index b135e05..1df2c2d 100755 --- a/scripts/wheel_smoke.sh +++ b/scripts/wheel_smoke.sh @@ -101,12 +101,43 @@ for template_id in \ done grep '"checks"' doctor.json >/dev/null + +# The offline lifecycle path must work when real audit tools cannot execute. +unavailable_tools="$tmp_dir/unavailable-tools" +mkdir -p "$unavailable_tools" +export ZEUS_WHEEL_RUNTIME_CALLS="$tmp_dir/unexpected-runtime-calls" +for tool in hermes docker; do + cat >"$unavailable_tools/$tool" <<'SH' +#!/bin/sh +set -eu +printf '%s\n' 'unexpected runtime invocation' >> "$ZEUS_WHEEL_RUNTIME_CALLS" +exit 97 +SH + chmod 0700 "$unavailable_tools/$tool" +done +export PATH="$tmp_dir/venv/bin:$unavailable_tools:$PATH" +export ZEUS_HERMES_BIN="$unavailable_tools/hermes" demo_started=1 "$venv_zeus" demo up --json >demo-up.json "$venv_zeus" demo status --json >demo-status.json "$venv_zeus" demo down --json >demo-down.json demo_started=0 +"$venv_zeus" message capacity --json >message-capacity.json +"$venv_zeus" message archive --json >message-archive-preview.json +"$venv_python" - <<'PY' +import json +from pathlib import Path + +capacity = json.loads(Path("message-capacity.json").read_text()) +assert capacity["used"] == 0 and capacity["total"] == 0 +assert capacity["status"] == "ok" +preview = json.loads(Path("message-archive-preview.json").read_text()) +assert preview["applied"] is False and preview["count"] == 0 +assert preview["message_ids"] == [] +PY +[ ! -e "$ZEUS_WHEEL_RUNTIME_CALLS" ] || fail "offline commands invoked Hermes or Docker" + grep '"fake_hermes_bin"' demo-up.json >/dev/null grep -F "\"fake_hermes_bin\": \"$venv_fake_hermes\"" demo-up.json >/dev/null grep '"status": "running"' demo-up.json >/dev/null diff --git a/tests/test_ci_required.py b/tests/test_ci_required.py new file mode 100644 index 0000000..da15e49 --- /dev/null +++ b/tests/test_ci_required.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import io +import json +import runpy +import unittest + +SCRIPT = runpy.run_path("scripts/check_ci_required.py") +CIRequiredError = SCRIPT["CIRequiredError"] +ENVIRONMENT_VARIABLE = SCRIPT["ENVIRONMENT_VARIABLE"] +REQUIRED_DEPENDENCIES = SCRIPT["REQUIRED_DEPENDENCIES"] +evaluate_needs = SCRIPT["evaluate_needs"] +main = SCRIPT["main"] + + +def _needs() -> dict[str, object]: + return { + dependency: {"result": "success", "outputs": {}} for dependency in REQUIRED_DEPENDENCIES + } + + +class CIRequiredTests(unittest.TestCase): + def test_accepts_exact_successful_dependency_set(self) -> None: + evaluate_needs(json.dumps(_needs())) + + def test_every_missing_or_unsuccessful_dependency_fails(self) -> None: + for dependency in REQUIRED_DEPENDENCIES: + with self.subTest(dependency=dependency, state="missing"): + needs = _needs() + del needs[dependency] + with self.assertRaisesRegex(CIRequiredError, "invalid_needs"): + evaluate_needs(json.dumps(needs)) + for result in ("failure", "cancelled", "skipped", "neutral", None): + with self.subTest(dependency=dependency, result=result): + needs = _needs() + needs[dependency] = {"result": result} + with self.assertRaisesRegex(CIRequiredError, "required_job_not_successful"): + evaluate_needs(json.dumps(needs)) + + def test_malformed_ambiguous_and_extra_data_fails_closed(self) -> None: + malformed = ( + "", + "[]", + "null", + "{", + '{"test":{"result":"success"},"test":{"result":"success"}}', + '{"test":{"result":NaN}}', + "x" * (SCRIPT["MAX_INPUT_CHARACTERS"] + 1), + ) + for raw in malformed: + with ( + self.subTest(raw=raw[:40]), + self.assertRaisesRegex(CIRequiredError, "invalid_needs"), + ): + evaluate_needs(raw) + needs = _needs() + needs["unexpected"] = {"result": "success"} + with self.assertRaisesRegex(CIRequiredError, "invalid_needs"): + evaluate_needs(json.dumps(needs)) + + def test_dependency_records_and_result_values_are_typed_strictly(self) -> None: + dependency = next(iter(REQUIRED_DEPENDENCIES)) + for record in ("success", [], None, {"result": True}, {"result": 1}, {}): + with self.subTest(record=record): + needs = _needs() + needs[dependency] = record + with self.assertRaisesRegex(CIRequiredError, "required_job_not_successful"): + evaluate_needs(json.dumps(needs)) + + def test_main_prefers_environment_and_falls_back_to_bounded_stdin(self) -> None: + successful = json.dumps(_needs()) + for environ, stdin in ( + ({ENVIRONMENT_VARIABLE: successful}, io.StringIO("invalid")), + ({}, io.StringIO(successful)), + ): + with self.subTest(environ=environ): + stdout, stderr = io.StringIO(), io.StringIO() + self.assertEqual( + 0, + main(environ=environ, stdin=stdin, stdout=stdout, stderr=stderr), + ) + self.assertEqual("", stderr.getvalue()) + self.assertIn("Verified all required CI", stdout.getvalue()) + + def test_main_reports_only_fixed_failure_codes(self) -> None: + stdout, stderr = io.StringIO(), io.StringIO() + self.assertEqual( + 1, + main( + environ={ENVIRONMENT_VARIABLE: "untrusted-sentinel"}, + stdout=stdout, + stderr=stderr, + ), + ) + self.assertEqual("", stdout.getvalue()) + self.assertEqual( + "required CI verification failed: invalid_needs\n", + stderr.getvalue(), + ) diff --git a/tests/test_fresh_vps_script.py b/tests/test_fresh_vps_script.py index 47db09c..a2c26dd 100644 --- a/tests/test_fresh_vps_script.py +++ b/tests/test_fresh_vps_script.py @@ -47,7 +47,7 @@ def setUp(self) -> None: 'mkdir -p "$HOME/.local/bin"', "cat > \"$HOME/.local/bin/hermes\" <<'HERMES'", "#!/usr/bin/env bash", - "exit 0", + 'case "${1:-}" in --version|doctor|-p) exit 0 ;; *) exit 64 ;; esac', "HERMES", 'chmod 700 "$HOME/.local/bin/hermes"', "printf 'executed\\n' > \"$INSTALLER_EXECUTED\"", diff --git a/tests/test_message_archive.py b/tests/test_message_archive.py new file mode 100644 index 0000000..c7cfef2 --- /dev/null +++ b/tests/test_message_archive.py @@ -0,0 +1,553 @@ +from __future__ import annotations + +import io +import json +import os +import sqlite3 +import tempfile +import threading +import unittest +from concurrent.futures import ThreadPoolExecutor +from contextlib import closing, contextmanager, redirect_stdout +from datetime import UTC, datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import patch + +from tests import test_bot_messaging as fixtures +from zeus.bot_messaging import MessagingError +from zeus.cli import main +from zeus.message_store import MessageStore, MessageStoreError +from zeus.schema import SchemaManager, _assert_schema_current +from zeus.sqlite_db import SQLiteDatabase +from zeus.state import StateStore + + +class MessageArchiveTests(unittest.TestCase): + def setUp(self): + self.root = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.path = self.root / "zeus.db" + StateStore(self.path).init() + self.store = MessageStore(self.path) + self.old = datetime(2026, 1, 1, tzinfo=UTC) + self.now = self.old + timedelta(days=31) + self.sequence = 0 + + def args(self, **changes): + args = dict( + bot_id="coder", + incarnation=self.old - timedelta(days=1), + target_fingerprint="a" * 64, + endpoint="http://127.0.0.1:8765/health", + credential_fingerprint="b" * 64, + input_fingerprint="c" * 64, + request_key_fingerprint=None, + now=self.old, + retry_before=self.old + timedelta(minutes=10), + ) + args.update(changes) + return args + + def receipt(self, state="rejected", status=None, released=False, error=None): + self.sequence += 1 + receipt, _ = self.store.prepare(**self.args(bot_id=f"bot-{self.sequence}")) + if state != "prepared": + receipt = self.store.finish_attempt( + receipt.message_id, + expected_version=1, + dispatch_state=state, + run_id=f"run-{self.sequence}" if state == "accepted" else None, + run_status=status, + error_code=error, + now=self.old, + ) + if released: + receipt = self.store.release( + receipt.message_id, expected_version=receipt.version, now=self.old + ) + return receipt + + def dump(self): + with closing(sqlite3.connect(self.path)) as conn: + return list(conn.iterdump()) + + def test_eligibility_matrix_and_preview_identity(self): + eligible = [self.receipt(error="invalid_request")] + eligible += [ + self.receipt("accepted", status) + for status in ("completed", "failed", "cancelled", "interrupted") + ] + for state in ("prepared", "unknown"): + self.receipt(state) + for status in ("queued", "running", "waiting_for_approval", "stopping"): + self.receipt("accepted", status) + self.receipt("accepted", status, released=True) + released = self.receipt("accepted", "running", released=True) + eligible.append( + self.store.update_run( + released.message_id, + expected_version=released.version, + run_status="completed", + now=self.old, + ) + ) + before = self.dump() + preview = self.store.archive(now=self.now) + self.assertEqual(before, self.dump()) + self.assertFalse(preview["applied"]) + self.assertEqual(sorted(r.message_id for r in eligible), preview["message_ids"]) + applied = self.store.archive(now=self.now, apply=True) + self.assertEqual(preview["message_ids"], applied["message_ids"]) + self.assertEqual(len(eligible), self.store.capacity()["archived"]) + for original in eligible: + current = self.store.get(original.message_id) + self.assertEqual(self.now, current.archived_at) + self.assertEqual(original.version + 1, current.version) + self.assertEqual(original.updated_at, current.updated_at) + self.assertEqual(original.upstream_key, current.upstream_key) + self.assertEqual(original.released_at, current.released_at) + self.assertEqual(0, self.store.archive(now=self.now, apply=True)["count"]) + + def test_cutoff_timezone_limits_and_preview_not_reservation(self): + receipt = self.receipt("accepted", "completed") + self.assertEqual(0, self.store.archive(before=self.old, now=self.now)["count"]) + cutoff = (self.old + timedelta(microseconds=1)).astimezone(timezone(timedelta(hours=2))) + self.assertEqual(1, self.store.archive(before=cutoff, now=self.now)["count"]) + for limit in (0, 501, True, 1.0): + with self.assertRaises(ValueError): + self.store.archive(limit=limit, now=self.now) + for cutoff in (self.now + timedelta(seconds=1), self.now.replace(tzinfo=None)): + with self.assertRaises(ValueError): + self.store.archive(before=cutoff, now=self.now) + self.store.update_run( + receipt.message_id, + expected_version=receipt.version, + run_status="completed", + now=self.now, + ) + self.assertEqual(0, self.store.archive(now=self.now, apply=True)["count"]) + self.assertIsNone(self.store.get(receipt.message_id).archived_at) + + def test_archive_invalidates_writers_and_terminal_observation_keeps_identity(self): + receipt = self.receipt("accepted", "completed") + self.store.archive(apply=True, now=self.now) + for method, kwargs in ( + (self.store.update_run, {"run_status": "completed"}), + (self.store.record_cancel_intent, {}), + (self.store.release, {}), + ): + with self.assertRaisesRegex(MessageStoreError, "receipt_changed"): + method(receipt.message_id, expected_version=receipt.version, now=self.now, **kwargs) + current = self.store.get(receipt.message_id) + with self.assertRaisesRegex(MessageStoreError, "clock_rollback"): + self.store.update_run( + current.message_id, + expected_version=current.version, + run_status="completed", + now=self.now - timedelta(seconds=1), + ) + later = self.store.update_run( + current.message_id, + expected_version=current.version, + run_status="completed", + now=self.now + timedelta(days=1), + ) + self.assertEqual(current.archived_at, later.archived_at) + self.assertEqual(current.run_id, later.run_id) + with self.assertRaisesRegex(MessageStoreError, "invalid_transition"): + self.store.update_run( + later.message_id, + expected_version=later.version, + run_status="running", + now=self.now + timedelta(days=2), + ) + + def test_replay_precedes_capacity_and_target_blocker_and_rejects_changes(self): + original, _ = self.store.prepare(**self.args(request_key_fingerprint="d" * 64)) + self.store.finish_attempt( + original.message_id, expected_version=1, dispatch_state="rejected", now=self.old + ) + self.store.archive(apply=True, now=self.now) + current = self.store.get(original.message_id) + with patch("zeus.message_store.MAX_MESSAGE_RECEIPTS", 1): + self.store.prepare( + **self.args(now=self.now, retry_before=self.now + timedelta(minutes=10)) + ) + replay, created = self.store.prepare( + **self.args( + request_key_fingerprint="d" * 64, + now=self.now, + retry_before=self.now + timedelta(minutes=10), + ) + ) + self.assertFalse(created) + self.assertEqual(current, replay) + for changes in ({"input_fingerprint": "e" * 64}, {"bot_id": "different"}): + with self.assertRaisesRegex(MessageStoreError, "request_conflict"): + self.store.prepare(**self.args(request_key_fingerprint="d" * 64, **changes)) + with self.assertRaisesRegex(MessageStoreError, "capacity_exceeded"): + self.store.prepare(**self.args(bot_id="another")) + with self.assertRaisesRegex(MessageStoreError, "clock_rollback"): + self.store.prepare(**self.args(request_key_fingerprint="d" * 64)) + + def test_overlapping_archivers_select_disjoint_batches(self): + expected = {self.receipt().message_id for _ in range(8)} + barrier = threading.Barrier(2) + + def archive(_): + barrier.wait(timeout=3) + return MessageStore(self.path).archive(limit=4, apply=True, now=self.now) + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(archive, range(2))) + self.assertEqual(expected, set(results[0]["message_ids"]) | set(results[1]["message_ids"])) + self.assertFalse(set(results[0]["message_ids"]) & set(results[1]["message_ids"])) + + def test_corrupt_selected_row_and_failed_transaction_roll_back_whole_batch(self): + self.receipt() + corrupt = self.receipt() + with closing(sqlite3.connect(self.path)) as conn: + conn.execute( + "UPDATE message_receipts SET target_bot_id = '!!' WHERE message_id = ?", + (corrupt.message_id,), + ) + conn.commit() + before = self.dump() + with self.assertRaisesRegex(MessageStoreError, "invalid_receipt"): + self.store.archive(now=self.now, apply=True) + self.assertEqual(before, self.dump()) + with closing(sqlite3.connect(self.path)) as conn: + conn.execute("UPDATE message_receipts SET target_bot_id = 'fixed'") + conn.execute( + "CREATE TRIGGER archive_full BEFORE UPDATE OF archived_at ON message_receipts " + "WHEN NEW.message_id = (SELECT max(message_id) FROM message_receipts) " + "BEGIN SELECT RAISE(ABORT, 'database or disk is full'); END" + ) + conn.commit() + before = self.dump() + with self.assertRaisesRegex(MessageStoreError, "state_unavailable"): + self.store.archive(now=self.now, apply=True) + self.assertEqual(before, self.dump()) + + def test_full_durability_and_interrupted_transaction(self): + self.receipt() + original = self.dump() + with self.assertRaises(KeyboardInterrupt), self.store._write() as conn: + self.assertEqual(2, conn.execute("PRAGMA synchronous").fetchone()[0]) + conn.execute( + "UPDATE message_receipts SET archived_at = ?, version = version + 1", + (self.now.isoformat(),), + ) + raise KeyboardInterrupt() + self.assertEqual(original, self.dump()) + with self.store._read() as conn: + self.assertIn( + "message_receipts_unarchived_idx", + str( + conn.execute( + "EXPLAIN QUERY PLAN SELECT count(*) FROM message_receipts " + "WHERE archived_at IS NULL" + ).fetchall()[0][3] + ), + ) + + def test_cli_has_no_workflow_and_missing_or_old_state_is_not_initialized(self): + self.receipt() + + def cli(*args): + output = io.StringIO() + with ( + patch.dict(os.environ, {"ZEUS_STATE_DIR": str(self.root)}, clear=True), + redirect_stdout(output), + ): + result = main(["message", "archive", "--json", *args]) + return result, json.loads(output.getvalue()) + + before = self.dump() + forbidden = AssertionError("archive crossed local storage boundary") + with ( + patch("zeus.messaging_cli.BotMessaging", side_effect=forbidden), + patch("zeus.messaging_cli.Supervisor", side_effect=forbidden), + patch("zeus.messaging_cli.StateStore", side_effect=forbidden), + patch("subprocess.Popen", side_effect=forbidden), + patch("subprocess.run", side_effect=forbidden), + patch("os.kill", side_effect=forbidden), + patch("os.killpg", side_effect=forbidden), + patch("zeus.hermes_runs_client.request_json", side_effect=forbidden), + patch("zeus.gateway_http.request_json", side_effect=forbidden), + patch("zeus.gateway_http.socket.socket", side_effect=forbidden), + ): + code, result = cli("--before", self.now.isoformat()) + self.assertEqual(0, code) + self.assertEqual(1, result["count"]) + self.assertEqual(before, self.dump()) + code, result = cli("--before", self.now.isoformat(), "--apply") + self.assertEqual(0, code) + self.assertTrue(result["applied"]) + for cutoff in ("not-a-date", "2026-01-01", "9999-01-01T00:00:00+00:00"): + self.assertEqual(1, cli("--before", cutoff)[0]) + with closing(sqlite3.connect(self.path)) as conn: + conn.execute("UPDATE schema_version SET version = 9") + conn.commit() + before = self.dump() + self.assertEqual(1, cli("--apply")[0]) + self.assertEqual(before, self.dump()) + self.path.unlink() + self.assertEqual(1, cli()[0]) + self.assertFalse(self.path.exists()) + + def test_storage_drill_25001_retained_and_11001_actual_admissions(self): + # Test-only persistent connection avoids 22000 fsyncs. Other tests verify FULL. + # All 11001 admissions call prepare and finish_attempt; 14000 older rows are seeded. + with closing(sqlite3.connect(self.path)) as conn: + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA synchronous=OFF") + + @contextmanager + def fast_write(): + with conn: + conn.execute("BEGIN IMMEDIATE") + _assert_schema_current(conn) + yield conn + + with patch.object(self.store, "_write", fast_write): + first = self.receipt() + self.store.archive(now=self.now, apply=True) + columns = [r[1] for r in conn.execute("PRAGMA table_info(message_receipts)")] + row = dict(conn.execute("SELECT * FROM message_receipts").fetchone()) + values = [] + for index in range(14000): + seeded = dict( + row, message_id=f"{index:032x}", upstream_key=f"{index + 14000:032x}" + ) + values.append(tuple(seeded[c] for c in columns)) + conn.executemany( + "INSERT INTO message_receipts VALUES (" + ",".join("?" for _ in columns) + ")", + values, + ) + conn.commit() + admitted = 1 + for index in range(11000): + receipt, created = self.store.prepare(**self.args()) + self.assertTrue(created) + self.store.finish_attempt( + receipt.message_id, + expected_version=1, + dispatch_state="rejected", + now=self.old, + ) + admitted += 1 + if index % 500 == 499: + self.assertEqual( + 500, self.store.archive(limit=500, now=self.now, apply=True)["count"] + ) + self.assertEqual(11001, admitted) + capacity = self.store.capacity() + self.assertEqual(25001, capacity["total"]) + self.assertEqual(25001, capacity["archived"]) + self.assertEqual(0, capacity["used"]) + self.assertEqual(first.message_id, self.store.get(first.message_id).message_id) + self.assertEqual(100, len(self.store.list(limit=100)["items"])) + + +class ArchiveWorkflowTests(unittest.TestCase): + def setUp(self): + self.fixture = fixtures.BotMessagingTests() + self.addCleanup(self.fixture.doCleanups) + self.fixture.setUp() + + def test_fake_submit_counter_archived_retry_and_replay_do_not_resubmit(self): + fixture = self.fixture + workflow = fixture.messaging + receipt = workflow.send("coder", "operator message", request_key="original") + fixture.remote_status.return_value = {"run_id": fixture.run_id, "status": "completed"} + workflow.status(receipt["message_id"]) + fixture.now += timedelta(days=31) + workflow.store.archive(now=fixture.now, apply=True) + self.assertEqual(1, fixture.submit.call_count) + replay = workflow.send("coder", "operator message", request_key="original") + self.assertIsNotNone(replay["archived_at"]) + self.assertEqual(replay, workflow.retry(receipt["message_id"], "operator message")) + self.assertEqual(1, fixture.submit.call_count) + + def test_archived_workflow_clock_rollback_precedes_target_and_http_work(self): + fixture = self.fixture + workflow = fixture.messaging + receipt = workflow.send("coder", "operator message") + fixture.remote_status.return_value = {"run_id": fixture.run_id, "status": "completed"} + workflow.status(receipt["message_id"]) + fixture.now += timedelta(days=31) + archived_at = fixture.now + workflow.store.archive(now=archived_at, apply=True) + stored = workflow.store.get(receipt["message_id"]) + fixture.now -= timedelta(seconds=1) + self.assertGreater(fixture.now, stored.updated_at) + for mock in (fixture.submit, fixture.remote_status, fixture.stop, fixture.health): + mock.reset_mock() + with patch.object(workflow, "_receipt_target", side_effect=AssertionError("target read")): + for action in ( + lambda: workflow.retry(receipt["message_id"], "operator message"), + lambda: workflow.status(receipt["message_id"]), + lambda: workflow.cancel(receipt["message_id"]), + ): + with self.assertRaisesRegex(MessagingError, "clock_rollback"): + action() + for now in (archived_at, archived_at + timedelta(seconds=1)): + fixture.now = now + replay = workflow.retry(receipt["message_id"], "operator message") + self.assertEqual(receipt["message_id"], replay["message_id"]) + self.assertEqual(archived_at.isoformat(), replay["archived_at"]) + self.assertEqual("completed", replay["run_status"]) + for mock in (fixture.submit, fixture.remote_status, fixture.stop, fixture.health): + mock.assert_not_called() + + def test_status_and_cancel_racing_archive_fail_cas(self): + fixture = self.fixture + workflow = fixture.messaging + receipt = workflow.send("coder", "operator message") + fixture.remote_status.return_value = {"run_id": fixture.run_id, "status": "completed"} + workflow.status(receipt["message_id"]) + fixture.now += timedelta(days=31) + + def response(*args, **kwargs): + workflow.store.archive(now=fixture.now, apply=True) + return {"run_id": fixture.run_id, "status": "completed"} + + fixture.remote_status.side_effect = response + with self.assertRaisesRegex(MessageStoreError, "receipt_changed"): + workflow.status(receipt["message_id"]) + self.assertIsNotNone(workflow.store.get(receipt["message_id"]).archived_at) + # A cancel that captured the old version before archival cannot dispatch. + fixture.remote_status.side_effect = None + second = workflow.send("coder", "next message") + workflow.status(second["message_id"]) + fixture.now += timedelta(days=31) + get_receipt = workflow._receipt + + def capture_then_archive(message_id): + captured = get_receipt(message_id) + workflow.store.archive(now=fixture.now, apply=True) + return captured + + with ( + patch.object(workflow, "_receipt", side_effect=capture_then_archive), + self.assertRaisesRegex(MessageStoreError, "receipt_changed"), + ): + workflow.cancel(second["message_id"]) + fixture.stop.assert_not_called() + + def test_archive_during_submit_skips_uncertain_receipt_and_failed_prepare_never_dispatches( + self, + ): + fixture = self.fixture + workflow = fixture.messaging + + def submit(*args, **kwargs): + self.assertEqual( + 0, workflow.store.archive(before=fixture.now, now=fixture.now, apply=True)["count"] + ) + return {"run_id": fixture.run_id, "status": "running"} + + fixture.submit.side_effect = submit + receipt = workflow.send("coder", "operator message") + self.assertIsNone(receipt["archived_at"]) + fixture.submit.reset_mock() + fixture.remote_status.return_value = {"run_id": fixture.run_id, "status": "completed"} + workflow.status(receipt["message_id"]) + with closing(sqlite3.connect(workflow.store.database_path)) as conn: + conn.execute( + "CREATE TRIGGER disk_full BEFORE INSERT ON message_receipts " + "BEGIN SELECT RAISE(ABORT, 'database or disk is full'); END" + ) + conn.commit() + with self.assertRaisesRegex(MessageStoreError, "state_unavailable"): + workflow.send("coder", "next message") + fixture.submit.assert_not_called() + + +class MessageArchiveSchemaTests(unittest.TestCase): + def setUp(self): + self.root = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.path = self.root / "zeus.db" + self.database = SQLiteDatabase(self.path) + with patch("zeus.schema.SCHEMA_VERSION", 9): + SchemaManager(self.database).init() + self.old = "2026-01-01T00:00:00+00:00" + with closing(sqlite3.connect(self.path)) as conn: + conn.execute( + "INSERT INTO message_receipts (message_id, request_key_hash, target_bot_id, " + "target_created_at, target_fingerprint, endpoint, credential_fingerprint, " + "request_hash, upstream_key, dispatch_state, created_at, updated_at, " + "retry_before, version) VALUES (?, ?, 'coder', ?, ?, ?, ?, ?, ?, " + "'rejected', ?, ?, ?, 1)", + ( + "a" * 32, + "b" * 64, + self.old, + "c" * 64, + "http://127.0.0.1:8765/health", + "d" * 64, + "e" * 64, + "f" * 32, + self.old, + self.old, + "2026-01-01T00:10:00+00:00", + ), + ) + conn.commit() + + def snapshot(self): + with closing(sqlite3.connect(self.path)) as conn: + return list(conn.iterdump()) + + def test_additive_upgrade_preserves_identity_indexes_and_old_readers_reject(self): + with closing(sqlite3.connect(self.path)) as conn: + before = conn.execute("SELECT rowid, * FROM message_receipts").fetchall() + indexes = dict(conn.execute("SELECT name, sql FROM sqlite_master WHERE type='index'")) + SchemaManager(self.database).migrate() + with closing(sqlite3.connect(self.path)) as conn: + after = conn.execute("SELECT rowid, * FROM message_receipts").fetchall() + self.assertEqual(before, [row[:-1] for row in after]) + self.assertIsNone(after[0][-1]) + current = dict(conn.execute("SELECT name, sql FROM sqlite_master WHERE type='index'")) + self.assertTrue(all(current[name] == sql for name, sql in indexes.items())) + self.assertEqual(10, conn.execute("SELECT version FROM schema_version").fetchone()[0]) + migrated = self.snapshot() + SchemaManager(self.database).migrate() + self.assertEqual(migrated, self.snapshot()) + with patch("zeus.schema.SCHEMA_VERSION", 9): + with self.assertRaisesRegex(RuntimeError, "newer than supported"): + StateStore(self.path).connect() + with self.assertRaisesRegex(MessageStoreError, "state_unavailable"): + MessageStore(self.path).archive(apply=True) + self.assertEqual(migrated, self.snapshot()) + + def test_failed_migration_preserves_schema_nine_and_data(self): + class FailingSchema(SchemaManager): + def _migrate_v9_to_v10(self, conn): + super()._migrate_v9_to_v10(conn) + raise sqlite3.OperationalError("injected interruption") + + original = self.snapshot() + with self.assertRaisesRegex(sqlite3.OperationalError, "injected interruption"): + FailingSchema(self.database).migrate() + self.assertEqual(original, self.snapshot()) + + def test_invalid_archival_is_rejected_by_sql_and_reader(self): + SchemaManager(self.database).migrate() + with closing(sqlite3.connect(self.path)) as conn: + for timestamp in ( + "not-time", + "2026-01-01T00:00:00Z", + "2025-01-01T00:00:00+00:00", + "2026-01-01T00:00:00.000000+00:00", + ): + with self.assertRaises(sqlite3.IntegrityError): + conn.execute("UPDATE message_receipts SET archived_at = ?", (timestamp,)) + conn.execute("UPDATE message_receipts SET dispatch_state = 'unknown'") + with self.assertRaises(sqlite3.IntegrityError): + conn.execute("UPDATE message_receipts SET archived_at = ?", (self.old,)) + conn.execute("PRAGMA ignore_check_constraints=ON") + conn.execute("UPDATE message_receipts SET archived_at = ?", (self.old,)) + conn.commit() + with self.assertRaisesRegex(MessageStoreError, "invalid_receipt"): + MessageStore(self.path).get("a" * 32) diff --git a/tests/test_message_capacity.py b/tests/test_message_capacity.py new file mode 100644 index 0000000..1d42f33 --- /dev/null +++ b/tests/test_message_capacity.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +import io +import json +import os +import sqlite3 +import tempfile +import unittest +from contextlib import closing, redirect_stderr, redirect_stdout +from datetime import UTC, datetime, timedelta +from pathlib import Path +from unittest.mock import patch + +from zeus.cli import main +from zeus.message_store import MessageStore, MessageStoreError +from zeus.state import StateStore + + +class MessageCapacityTests(unittest.TestCase): + def setUp(self) -> None: + self.root = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.state_dir = self.root / "state" + self.path = self.state_dir / "zeus.db" + StateStore(self.path).init() + self.store = MessageStore(self.path) + now = datetime(2026, 1, 1, tzinfo=UTC) + receipt, _ = self.store.prepare( + bot_id="coder", + incarnation=now - timedelta(days=1), + target_fingerprint="a" * 64, + endpoint="http://127.0.0.1:8765/health", + credential_fingerprint="b" * 64, + input_fingerprint="c" * 64, + request_key_fingerprint=None, + now=now, + retry_before=now + timedelta(minutes=10), + ) + self.store.finish_attempt( + receipt.message_id, expected_version=1, dispatch_state="rejected", now=now + ) + self.receipt_id = receipt.message_id + + def _resize(self, total: int) -> None: + with closing(sqlite3.connect(self.path)) as conn: + conn.execute("DELETE FROM message_receipts WHERE message_id != ?", (self.receipt_id,)) + conn.executemany( + "INSERT INTO message_receipts SELECT ?, NULL, target_bot_id, target_created_at, " + "target_fingerprint, endpoint, credential_fingerprint, request_hash, ?, " + "dispatch_state, run_id, run_status, created_at, updated_at, retry_before, " + "last_checked_at, cancel_requested_at, lease_until, error_code, version, " + "released_at, archived_at " + "FROM message_receipts WHERE message_id = ?", + ( + (f"{index:032x}", f"{index + 20000:032x}", self.receipt_id) + for index in range(1, total) + ), + ) + conn.commit() + + def _cli(self, *args: str) -> tuple[int, str, str]: + stdout, stderr = io.StringIO(), io.StringIO() + with ( + patch.dict(os.environ, {"ZEUS_STATE_DIR": str(self.state_dir)}, clear=True), + redirect_stdout(stdout), + redirect_stderr(stderr), + ): + code = main(["message", "capacity", *args]) + return code, stdout.getvalue(), stderr.getvalue() + + def test_exact_capacity_thresholds(self) -> None: + for total, expected in ( + (7999, "ok"), + (8000, "warning"), + (8001, "warning"), + (9499, "warning"), + (9500, "critical"), + (9501, "critical"), + (9999, "critical"), + (10000, "full"), + (10001, "full"), + ): + with self.subTest(total=total): + self._resize(total) + result = self.store.capacity() + self.assertEqual(expected, result["status"]) + self.assertEqual(total, result["used"]) + self.assertEqual(total, result["total"]) + self.assertEqual(max(0, 10000 - total), result["remaining"]) + self.assertEqual(0, result["archived"]) + + def test_blockers_are_distinct_from_retained_receipts(self) -> None: + now = datetime(2026, 1, 2, tzinfo=UTC) + active, _ = self.store.prepare( + bot_id="active", + incarnation=now - timedelta(days=1), + target_fingerprint="d" * 64, + endpoint="http://127.0.0.1:8766/health", + credential_fingerprint="e" * 64, + input_fingerprint="f" * 64, + request_key_fingerprint=None, + now=now, + retry_before=now + timedelta(minutes=10), + ) + result = self.store.capacity() + self.assertEqual(2, result["used"]) + self.assertEqual(1, result["blocking"]) + self.store.finish_attempt( + active.message_id, + expected_version=1, + dispatch_state="accepted", + run_id="run-1", + run_status="running", + now=now, + ) + self.assertEqual(1, self.store.capacity()["blocking"]) + self.store.release(active.message_id, expected_version=2, now=now) + self.assertEqual(0, self.store.capacity()["blocking"]) + + def test_invalid_receipt_fails_closed(self) -> None: + with closing(sqlite3.connect(self.path)) as conn: + conn.execute( + "UPDATE message_receipts SET target_bot_id = '!!' WHERE message_id = ?", + (self.receipt_id,), + ) + conn.commit() + with self.assertRaisesRegex(MessageStoreError, "invalid_receipt"): + self.store.capacity() + + def test_cli_is_read_only_one_document_and_full_exits_zero(self) -> None: + self._resize(10000) + forbidden = AssertionError("capacity command crossed its read-only boundary") + with ( + patch("zeus.messaging_cli.BotMessaging", side_effect=forbidden), + patch("zeus.messaging_cli.Supervisor", side_effect=forbidden), + patch("zeus.messaging_cli.StateStore", side_effect=forbidden), + patch("subprocess.Popen", side_effect=forbidden), + patch("subprocess.run", side_effect=forbidden), + patch("os.kill", side_effect=forbidden), + patch("os.killpg", side_effect=forbidden), + patch("zeus.hermes_runs_client.request_json", side_effect=forbidden), + patch("zeus.gateway_http.request_json", side_effect=forbidden), + patch("zeus.gateway_http.socket.socket", side_effect=forbidden), + ): + code, output, error = self._cli("--json") + self.assertEqual(0, code) + self.assertEqual("", error) + payload = json.loads(output) + self.assertEqual("full", payload["status"]) + self.assertEqual(10000, payload["limit"]) + self.assertIsInstance(payload["database_bytes"], int) + self.assertIsInstance(payload["filesystem_free_bytes"], int) + + def test_missing_state_and_unavailable_observations_are_safe(self) -> None: + self.path.unlink() + code, output, _error = self._cli("--json") + self.assertEqual(1, code) + self.assertIn(json.loads(output)["error"]["code"], {"state_unavailable", "not_ready"}) + self.assertFalse(self.path.exists()) + + StateStore(self.path).init() + with ( + patch("zeus.message_store._file_size", return_value=None), + patch("zeus.message_store._wal_size", return_value=None), + patch("zeus.message_store._filesystem_free", return_value=None), + ): + result = MessageStore(self.path).capacity() + self.assertIsNone(result["database_bytes"]) + self.assertIsNone(result["wal_bytes"]) + self.assertIsNone(result["filesystem_free_bytes"]) + + def test_real_incompatible_schemas_are_rejected_without_writes(self) -> None: + for version in (9, 11): + with self.subTest(version=version): + with closing(sqlite3.connect(self.path)) as conn: + conn.execute("UPDATE schema_version SET version = ?", (version,)) + conn.commit() + before = self.path.read_bytes() + code, output, error = self._cli("--json") + self.assertEqual(1, code) + self.assertEqual("", error) + self.assertEqual("state_unavailable", json.loads(output)["error"]["code"]) + self.assertEqual(before, self.path.read_bytes()) + + def test_incompatible_and_malformed_databases_are_not_modified(self) -> None: + for content in (b"not sqlite", b""): + with self.subTest(content=content): + self.path.write_bytes(content) + before = self.path.read_bytes() + code, output, _error = self._cli("--json") + self.assertEqual(1, code) + self.assertEqual("state_unavailable", json.loads(output)["error"]["code"]) + self.assertEqual(before, self.path.read_bytes()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_message_release_schema.py b/tests/test_message_release_schema.py index c3a9c2d..15258b3 100644 --- a/tests/test_message_release_schema.py +++ b/tests/test_message_release_schema.py @@ -5,8 +5,9 @@ import unittest from contextlib import closing from pathlib import Path +from unittest.mock import patch -from zeus.schema import SCHEMA_VERSION, SchemaManager +from zeus.schema import SchemaManager from zeus.sqlite_db import SQLiteDatabase _CREATED = "2026-01-01T00:00:00+00:00" @@ -21,6 +22,7 @@ def _migrate_v8_to_v9(self, conn: sqlite3.Connection) -> None: class MessageReleaseSchemaTests(unittest.TestCase): def setUp(self) -> None: + self.enterContext(patch("zeus.schema.SCHEMA_VERSION", 9)) root = Path(self.enterContext(tempfile.TemporaryDirectory())) self.path = root / "zeus.db" self.database = SQLiteDatabase(self.path) @@ -89,7 +91,6 @@ def test_upgrade_preserves_v8_data_rowids_and_other_indexes(self) -> None: SchemaManager(self.database).migrate() with closing(sqlite3.connect(self.path)) as conn: - self.assertEqual(9, SCHEMA_VERSION) self.assertEqual(9, conn.execute("SELECT version FROM schema_version").fetchone()[0]) columns = [row[1] for row in conn.execute("PRAGMA table_info(message_receipts)")] self.assertEqual([*old_columns, "released_at"], columns) diff --git a/tests/test_message_store.py b/tests/test_message_store.py index 4dc1f57..ccb8b59 100644 --- a/tests/test_message_store.py +++ b/tests/test_message_store.py @@ -356,7 +356,7 @@ def test_capacity_is_global_and_never_prunes_receipts(self) -> None: "target_fingerprint, endpoint, credential_fingerprint, request_hash, ?, " "dispatch_state, run_id, run_status, created_at, updated_at, retry_before, " "last_checked_at, cancel_requested_at, lease_until, error_code, version, " - "released_at " + "released_at, archived_at " "FROM message_receipts WHERE message_id = ?", [ (f"{index:032x}", f"{index + 10000:032x}", receipt.message_id) @@ -546,7 +546,8 @@ def _migrate_v7_to_v8(self, conn): def _migrate_v8_to_v9(self, conn): pass - PriorSchemaManager(SQLiteDatabase(path)).init() + with patch("zeus.schema.SCHEMA_VERSION", 9): + PriorSchemaManager(SQLiteDatabase(path)).init() with closing(sqlite3.connect(path)) as conn: conn.execute("UPDATE schema_version SET version = 7") conn.commit() @@ -571,7 +572,7 @@ def test_v7_migration_preserves_existing_data_and_is_idempotent(self) -> None: ) self.assertTrue(all(after_schema[name] == sql for name, sql in before_schema)) self.assertEqual( - 9, conn.execute("SELECT version FROM schema_version").fetchone()[0] + 10, conn.execute("SELECT version FROM schema_version").fetchone()[0] ) self.assertEqual( [], conn.execute("PRAGMA foreign_key_list(message_receipts)").fetchall() diff --git a/tests/test_release_ci.py b/tests/test_release_ci.py index ff1cbdb..4b24f39 100644 --- a/tests/test_release_ci.py +++ b/tests/test_release_ci.py @@ -34,6 +34,7 @@ "real-hermes", "macos-process-lifecycle", "package", + "ci-required", ) @@ -269,7 +270,7 @@ def test_job_pagination_checks_additional_jobs_instead_of_stopping_after_require data = _responses() path = f"{PREFIX}/runs/1010/attempts/1/jobs?per_page=100&page=1" jobs = _jobs_page(data)["jobs"] - for number in range(9, 101): + for number in range(len(JOB_NAMES), 101): jobs.append({**jobs[0], "id": 2000 + number, "name": f"extra-{number}"}) data[path] = {"total_count": 101, "jobs": jobs[:100]} data[path.replace("&page=1", "&page=2")] = {"total_count": 101, "jobs": jobs[100:]} diff --git a/tests/test_repo_contracts.py b/tests/test_repo_contracts.py index 8fe3724..724fd04 100644 --- a/tests/test_repo_contracts.py +++ b/tests/test_repo_contracts.py @@ -229,6 +229,7 @@ def test_publishable_repository_files_exist(self) -> None: "scripts/install_pinned_hermes.sh", "scripts/verify_pinned_hermes_runs.py", "scripts/check_verified_release_ref.py", + "scripts/check_ci_required.py", "scripts/wheel_smoke.sh", "scripts/fresh_vps_verify.sh", "zeus/bundled_skills/__init__.py", @@ -312,6 +313,7 @@ def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: "macos-process-lifecycle": "macos-26", "real-hermes": "ubuntu-24.04", "package": "ubuntu-24.04", + "ci-required": "ubuntu-24.04", } expected_python_versions = { "test": ("3.11", "3.12", "3.13"), @@ -321,6 +323,7 @@ def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: "macos-process-lifecycle": ("3.13",), "real-hermes": ("3.11",), "package": ("3.11",), + "ci-required": ("3.11",), } expected_setup_python = { "test": "${{ matrix.python-version }}", @@ -330,6 +333,7 @@ def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: "macos-process-lifecycle": '"3.13"', "real-hermes": '"3.11"', "package": '"3.11"', + "ci-required": '"3.11"', } expected_commands = { "test": ( @@ -403,6 +407,7 @@ def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: 'sh scripts/verify_service_recovery.sh "$(command -v python)"', "sh scripts/generate_checksums.sh dist", ), + "ci-required": ("python scripts/check_ci_required.py",), } self.assertEqual(set(expected_runners), set(jobs)) @@ -425,6 +430,22 @@ def test_ci_runs_project_test_script_on_supported_python_versions(self) -> None: job_level_continue_on_error[job_name] = values[0] self.assertEqual({"python-3-14": "true"}, job_level_continue_on_error) + ci_required = jobs["ci-required"] + self.assertEqual("always()", _job_level_scalar(ci_required, "if")) + self.assertEqual("5", _job_level_scalar(ci_required, "timeout-minutes")) + self.assertEqual( + { + "test", + "lifecycle-subprocess", + "package", + "real-hermes", + "audit-docker-isolation", + "macos-process-lifecycle", + }, + set(re.findall(r"(?m)^ - ([a-z0-9-]+)$", ci_required)), + ) + self.assertIn("ZEUS_CI_NEEDS_JSON: ${{ toJSON(needs) }}", ci_required) + real_hermes = jobs["real-hermes"] self.assertEqual("15", _job_level_scalar(real_hermes, "timeout-minutes")) self.assertNotIn("secrets.", real_hermes) @@ -1001,7 +1022,7 @@ def test_onboarding_compatibility_and_roadmap_match_current_evidence(self) -> No ): self.assertIn(command, offline_path) for command in ( - "hermes version", + "hermes --version", "cp .env.example .env", "chmod 0600 .env", "zeus doctor", diff --git a/tests/test_sqlite_schema.py b/tests/test_sqlite_schema.py index 7e7af27..abfd3c7 100644 --- a/tests/test_sqlite_schema.py +++ b/tests/test_sqlite_schema.py @@ -1096,7 +1096,7 @@ def test_state_store_passes_one_configured_database_to_every_delegate(self) -> N child_type.assert_called_once_with(database) def test_durability_configuration_does_not_change_schema_version(self) -> None: - self.assertEqual(9, SCHEMA_VERSION) + self.assertEqual(10, SCHEMA_VERSION) for mode in (SQLiteSynchronous.NORMAL, SQLiteSynchronous.FULL): with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp: database_path = Path(tmp) / "zeus.db" diff --git a/tests/test_subprocess_lifecycle.py b/tests/test_subprocess_lifecycle.py index 9b2515f..33f4643 100644 --- a/tests/test_subprocess_lifecycle.py +++ b/tests/test_subprocess_lifecycle.py @@ -114,7 +114,7 @@ def test_cli_stop_completes_while_another_supervisor_holds_exited_gateway_child( "try:\n" " result = supervisor.start(\n" " 'coder', source='api', request_id=uuid.uuid4().hex)\n" - " print(json.dumps({'pid': result.pid}), flush=True)\n" + " print(json.dumps(result.to_dict()), flush=True)\n" " sys.stdin.read()\n" "finally:\n" " for child in supervisor._runtime._processes.values():\n" @@ -142,7 +142,7 @@ def test_cli_stop_completes_while_another_supervisor_holds_exited_gateway_child( _stdout, stderr = parent.communicate("", timeout=5) self.fail(f"gateway parent exited during launch: {stderr}") child_pid = json.loads(launched)["pid"] - self.assertIsInstance(child_pid, int) + self.assertIsInstance(child_pid, int, launched) self.assertIs(PidState.alive, pid_state(child_pid)) stopped = self._run_cli(env, "bot", "stop", "coder", timeout=10) diff --git a/tests/test_subsystem_boundaries.py b/tests/test_subsystem_boundaries.py new file mode 100644 index 0000000..5ba3741 --- /dev/null +++ b/tests/test_subsystem_boundaries.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import ast +import io +import json +import sqlite3 +import subprocess +import tempfile +import unittest +from contextlib import ExitStack, redirect_stdout +from dataclasses import replace +from pathlib import Path +from unittest.mock import patch + +from tests.test_audit_store import _report +from zeus.audit import AuditService +from zeus.audit_store import AuditStore +from zeus.cli import main + +ROOT = Path(__file__).resolve().parents[1] + + +def _imports(path: Path) -> set[str]: + names = set() + for node in ast.walk(ast.parse(path.read_text())): + if isinstance(node, ast.Import): + names.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + module = node.module or "" + names.add(module) + names.update(f"{module}.{alias.name}" for alias in node.names) + return names + + +class SubsystemBoundaryTests(unittest.TestCase): + def test_runtime_and_audit_have_no_lifecycle_storage_dependencies(self) -> None: + forbidden = {"sqlite3", "zeus.sqlite_db", "zeus.schema", "zeus.state"} + paths = [*ROOT.glob("zeus/gateway_runtime*.py"), *ROOT.glob("zeus/audit*.py")] + self.assertTrue(paths) + for path in paths: + with self.subTest(module=path.name): + imports = _imports(path) + self.assertFalse(imports & forbidden, imports & forbidden) + if path.name.startswith("audit"): + self.assertNotIn("zeus.supervisor", imports) + + def test_profile_transactions_do_not_signal_processes(self) -> None: + path = ROOT / "zeus/profile_manager.py" + self.assertNotIn("subprocess", _imports(path)) + forbidden = {"kill", "killpg", "pidfd_send_signal", "Popen"} + calls = { + node.func.attr + for node in ast.walk(ast.parse(path.read_text())) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + } + self.assertFalse(calls & forbidden, calls & forbidden) + + def test_stored_audit_cli_works_without_runtime_or_lifecycle_database(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + state = Path(temporary).resolve() / "state" + service = AuditService.from_cwd(cwd=ROOT, env={"ZEUS_STATE_DIR": str(state)}) + report = _report("a" * 32) + report = replace( + report, + repository_id=service.location.repository_id, + metadata=replace(report.metadata, target_commit=service.location.head), + ) + artifacts = AuditStore(state).install(report) + database = state / "zeus.db" + database.write_bytes(b"unavailable lifecycle database") + before = { + p: p.read_bytes() for p in (artifacts.json_path, artifacts.markdown_path, database) + } + real_popen = subprocess.Popen + + def only_git(command, *args, **kwargs): + self.assertIsInstance(command, (list, tuple)) + self.assertEqual("git", Path(command[0]).name) + return real_popen(command, *args, **kwargs) + + with ExitStack() as stack: + for target in ( + "zeus.cli._services", + "zeus.state.StateStore.init", + "zeus.state.StateStore.migrate", + "zeus.audit.AuditContainerRuntime.__init__", + "zeus.audit.AuditRunner.__init__", + "zeus.audit.run_audit_doctor", + ): + stack.enter_context(patch(target, side_effect=AssertionError(target))) + stack.enter_context( + patch.object(sqlite3, "connect", side_effect=AssertionError("SQL")) + ) + stack.enter_context(patch("subprocess.Popen", side_effect=only_git)) + stack.enter_context(patch.object(AuditService, "from_cwd", return_value=service)) + for action in ("list", "show", "gate"): + with self.subTest(action=action), redirect_stdout(io.StringIO()) as output: + args = ["audit", action] + if action != "list": + args.append(report.run_id) + result = main([*args, "--json"]) + payload = json.loads(output.getvalue()) + if action == "gate": + self.assertEqual(1, result) + self.assertFalse(payload["passed"]) + else: + self.assertEqual(0, result) + item = payload[0] if action == "list" else payload + self.assertEqual(report.run_id, item["run_id"]) + self.assertEqual(before, {p: p.read_bytes() for p in before}) + self.assertFalse((state / "audit").exists()) diff --git a/tests/test_supervisor_composition.py b/tests/test_supervisor_composition.py new file mode 100644 index 0000000..5c175c9 --- /dev/null +++ b/tests/test_supervisor_composition.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import inspect +import tempfile +import unittest +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path +from unittest.mock import Mock, patch + +from zeus.gateway_runtime import OwnershipCheck +from zeus.models import BotRecord, BotStatus, DesiredState +from zeus.state import StateStore +from zeus.supervisor import Supervisor + +EXPECTED_SIGNATURES = { + "archive_bot": ( + "(self, bot_id: 'str', *, stop_if_running: 'bool' = False, source: 'str' = 'cli'," + " request_id: 'str | None' = None) -> 'dict[str, object]'" + ), + "bot_lock": ("(self, bot_id: 'str') -> 'threading.RLock'"), + "create_bot": ( + "(self, request: 'BotCreateRequest', template: 'HermesTemplate', *, replace_exist" + "ing: 'bool' = False, stop_if_running: 'bool' = False, source: 'str' = 'cli', req" + "uest_id: 'str | None' = None) -> 'BotRecord'" + ), + "delete_bot": ( + "(self, bot_id: 'str', *, stop_if_running: 'bool' = False, remove_profile: 'bool'" + " = False, source: 'str' = 'cli', request_id: 'str | None' = None) -> 'BotStatusR" + "esponse'" + ), + "inspect": ("(self, bot_id: 'str', max_log_bytes: 'int' = 20000) -> 'dict[str, object]'"), + "log_path": ("(self, profile_path: 'str') -> 'Path'"), + "logs": ("(self, bot_id: 'str', max_bytes: 'int' = 20000) -> 'str'"), + "pid_marker_path": ("(self, profile_path: 'str') -> 'Path'"), + "reconcile": ( + "(self, bot_id: 'str | None' = None, *, now: 'datetime | None' = None, force: 'bo" + "ol' = False, reset_restart: 'bool' = False, source: 'str' = 'reconcile', request" + "_id: 'str | None' = None, bot_snapshot: 'Sequence[tuple[str, str]] | None' = Non" + "e) -> 'list[BotStatusResponse]'" + ), + "reconcile_execution": ( + "(self, bot_id: 'str | None' = None, *, now: 'datetime | None' = None, force: 'bo" + "ol' = False, reset_restart: 'bool' = False, source: 'str' = 'reconcile', request" + "_id: 'str | None' = None, bot_snapshot: 'Sequence[tuple[str, str]] | None' = Non" + "e) -> 'ReconcileExecution'" + ), + "reconcile_one": ( + "(self, bot_id: 'str', *, now: 'datetime | None' = None, force: 'bool' = False, r" + "eset_restart: 'bool' = False, source: 'str' = 'reconcile', request_id: 'str | No" + "ne' = None, expected_profile_path: 'str | None' = None) -> 'BotReconcileResult'" + ), + "reconcile_one_execution": ( + "(self, bot_id: 'str', *, now: 'datetime | None' = None, force: 'bool' = False, r" + "eset_restart: 'bool' = False, source: 'str' = 'reconcile', request_id: 'str | No" + "ne' = None, expected_profile_path: 'str | None' = None) -> 'tuple[BotReconcileRe" + "sult, BotStatusResponse]'" + ), + "reconcile_summary": ( + "(self, bot_id: 'str | None' = None, *, now: 'datetime | None' = None, force: 'bo" + "ol' = False, reset_restart: 'bool' = False, source: 'str' = 'reconcile', request" + "_id: 'str | None' = None, bot_snapshot: 'Sequence[tuple[str, str]] | None' = Non" + "e) -> 'ReconcileRunSummary'" + ), + "restart": ( + "(self, bot_id: 'str', *, wait: 'bool' = False, timeout_seconds: 'float | None' =" + " None, source: 'str' = 'cli', request_id: 'str | None' = None) -> 'BotStatusResp" + "onse'" + ), + "start": ( + "(self, bot_id: 'str', *, wait: 'bool' = False, timeout_seconds: 'float | None' =" + " None, source: 'str' = 'cli', request_id: 'str | None' = None) -> 'BotStatusResp" + "onse'" + ), + "status": ( + "(self, bot_id: 'str', *, source: 'str' = 'cli', request_id: 'str | None' = None)" + " -> 'BotStatusResponse'" + ), + "stop": ( + "(self, bot_id: 'str', *, kill_after_timeout: 'bool | None' = None, source: 'str'" + " = 'cli', request_id: 'str | None' = None) -> 'BotStatusResponse'" + ), + "validate_reconcile_request": ("(self, source: 'str', request_id: 'str | None') -> 'None'"), + "validate_reconcile_target": ( + "(self, bot_id: 'str', *, expected_profile_path: 'str | None' = None) -> 'str'" + ), + "__init__": ( + "(self, store: 'StateStore', hermes_bin: 'str', hermes_root: 'Path | str', popen_" + "factory: 'PopenFactory' = , kill_fn: 'KillFn' = , pid_alive_fn: 'PidAliveFn | None' = None, cmdline_reader: 'C" + "mdlineReader | None' = None, startup_grace_seconds: 'float' = 0.25, stop_grace_s" + "econds: 'float' = 60.0, kill_after_timeout: 'bool' = False, lock_timeout_seconds" + ": 'float' = 30.0, readiness_timeout_seconds: 'float' = 30.0, readiness_interval_" + "seconds: 'float' = 0.5, allow_legacy_pid_markers: 'bool' = True, restart_backoff" + "_cap_seconds: 'float' = 3600.0, proc_start_fingerprint_reader: 'ProcStartFingerp" + "rintReader | None' = None, restart_stability_seconds: 'float' = 30.0) -> 'None'" + ), +} + + +class SupervisorCompositionCharacterizationTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve() + self.store = StateStore(self.root / "state.db") + self.store.init() + self.launch = Mock(side_effect=AssertionError("unexpected process launch")) + self.signal = Mock(side_effect=AssertionError("unexpected process signal")) + self.alive = Mock(return_value=False) + self.cmdline = Mock(return_value=None) + self.fingerprint = Mock(return_value=None) + self.supervisor = Supervisor( + self.store, + "hermes", + self.root / "hermes", + popen_factory=self.launch, + kill_fn=self.signal, + pid_alive_fn=self.alive, + cmdline_reader=self.cmdline, + proc_start_fingerprint_reader=self.fingerprint, + ) + + def test_complete_public_signatures(self) -> None: + signatures = { + name: str(inspect.signature(getattr(Supervisor, name))) + for name in dir(Supervisor) + if not name.startswith("_") and callable(getattr(Supervisor, name)) + } + signatures["__init__"] = str(inspect.signature(Supervisor.__init__)) + self.assertEqual(EXPECTED_SIGNATURES, signatures) + + def test_constructor_callbacks_and_all_proxy_nested_exception_restoration(self) -> None: + callbacks = { + "popen_factory": self.launch, + "kill_fn": self.signal, + "pid_alive_fn": self.alive, + "cmdline_reader": self.cmdline, + "proc_start_fingerprint_reader": self.fingerprint, + } + for name, value in callbacks.items(): + self.assertIs(value, getattr(self.supervisor._runtime, name)) + proxies = ( + *callbacks, + "stop_grace_seconds", + "kill_after_timeout", + "lock_timeout_seconds", + "_processes", + ) + for name in proxies: + with self.subTest(name=name): + original = getattr(self.supervisor, name) + first = Mock() if name in callbacks else 0.5 + second = Mock() if name in callbacks else 0.75 + if name == "_processes": + first, second = {}, {} + if name == "kill_after_timeout": + first, second = True, False + with patch.object(self.supervisor, name, first): + self.assertIs(first, getattr(self.supervisor._runtime, name)) + with ( + self.assertRaisesRegex(RuntimeError, "exit"), + patch.object(self.supervisor, name, second), + ): + self.assertIs(second, getattr(self.supervisor._runtime, name)) + raise RuntimeError("exit") + self.assertIs(first, getattr(self.supervisor._runtime, name)) + self.assertIs(original, getattr(self.supervisor._runtime, name)) + + def test_hooks_resolve_public_globals_after_construction(self) -> None: + hooks = { + "os.pipe": "pipe", + "os.close": "close", + "_read_bounded_file": "read_bounded_file", + "_remove_marker_if_owned_locked": "remove_marker_if_owned_locked", + "probe_once": "probe_once", + } + provider = self.supervisor._runtime._hooks_provider + for target, field in hooks.items(): + with self.subTest(target=target), patch(f"zeus.supervisor.{target}") as replacement: + self.assertIs(replacement, getattr(provider(), field)) + # Constructor captures the provider; each invocation resolves its globals. + with patch.object(self.supervisor, "_runtime_hooks", Mock()): + self.assertIs(provider, self.supervisor._runtime._hooks_provider) + + def test_late_method_replacement_used_by_public_operations(self) -> None: + replacement = Mock(return_value="replacement status") + self.store.upsert_bot( + BotRecord("bot", "test", "Bot", str(self.root / "hermes/profiles/bot")) + ) + with patch.object(self.supervisor, "_status_locked", replacement): + self.assertEqual("replacement status", self.supervisor.status("bot")) + replacement.assert_called_once() + runtime_replacement = Mock(return_value=OwnershipCheck(True, "owned")) + with patch.object( + self.supervisor._runtime, "verify_gateway_pid_ownership", runtime_replacement + ): + self.assertTrue(self.supervisor._pid_owned("profile", 123, "bot")) + runtime_replacement.assert_called_once() + + def test_status_and_inspect_never_launch_or_signal(self) -> None: + record = BotRecord( + "bot", + "test", + "Bot", + str(self.root / "hermes/profiles/bot"), + desired_state=DesiredState.running, + ) + self.store.upsert_bot(record) + response = self.supervisor.status("bot") + self.assertEqual(BotStatus.failed, response.status) + before = self.store.get_bot("bot") + self.supervisor.inspect("bot") + self.assertEqual(before, self.store.get_bot("bot")) + self.launch.assert_not_called() + self.signal.assert_not_called() + + def test_status_pending_intents_never_launch_or_signal(self) -> None: + record = BotRecord( + "bot", + "test", + "Bot", + str(self.root / "hermes/profiles/bot"), + desired_state=DesiredState.running, + pending_operation_id="a" * 32, + pending_since=datetime.now(UTC), + ) + for action in ("start", "stop", "restart"): + with self.subTest(action=action): + self.store.upsert_bot(replace(record, pending_action=action)) + self.supervisor.status("bot") + self.launch.assert_not_called() + self.signal.assert_not_called() diff --git a/zeus/bot_messaging.py b/zeus/bot_messaging.py index 02499b2..c1e16e9 100644 --- a/zeus/bot_messaging.py +++ b/zeus/bot_messaging.py @@ -101,6 +101,7 @@ def _public(receipt: MessageReceipt) -> dict[str, object]: receipt.cancel_requested_at.isoformat() if receipt.cancel_requested_at else None ), "released_at": receipt.released_at.isoformat() if receipt.released_at else None, + "archived_at": receipt.archived_at.isoformat() if receipt.archived_at else None, "error_code": receipt.error_code, } @@ -223,7 +224,7 @@ def _receipt(self, message_id: str) -> MessageReceipt: receipt = self.store.get(message_id) if receipt is None: raise MessagingError("unknown_message") - if self.clock() < receipt.updated_at: + if self.clock() < max(receipt.updated_at, receipt.archived_at or receipt.updated_at): raise MessagingError("clock_rollback") return receipt diff --git a/zeus/message_store.py b/zeus/message_store.py index d098340..9bf789b 100644 --- a/zeus/message_store.py +++ b/zeus/message_store.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import re import sqlite3 import uuid @@ -94,6 +95,7 @@ class MessageReceipt: lease_until: datetime | None error_code: str | None version: int + archived_at: datetime | None def _identifier(value: str, *, hashed: bool = False) -> str: @@ -166,7 +168,13 @@ def _receipt(row: sqlite3.Row, *, observe: bool = False) -> MessageReceipt: } optional = { name: _stored_time(row[name]) if row[name] is not None else None - for name in ("last_checked_at", "cancel_requested_at", "released_at", "lease_until") + for name in ( + "last_checked_at", + "cancel_requested_at", + "released_at", + "lease_until", + "archived_at", + ) } if not times["target_created_at"] <= times["created_at"] <= times[ "updated_at" @@ -178,6 +186,15 @@ def _receipt(row: sqlite3.Row, *, observe: bool = False) -> MessageReceipt: raise ValueError("message receipt observation time is inconsistent") if optional["released_at"] is not None and row["dispatch_state"] != "accepted": raise ValueError("only acknowledged message receipts can be released") + archived = optional["archived_at"] + if archived is not None and ( + archived < times["created_at"] + or not ( + row["dispatch_state"] == "rejected" + or (row["dispatch_state"] == "accepted" and row["run_status"] in _TERMINAL) + ) + ): + raise ValueError("message receipt archival is inconsistent") lease = optional["lease_until"] if row["dispatch_state"] == "prepared": if lease is None or lease != times["updated_at"] + timedelta( @@ -208,6 +225,7 @@ def _receipt(row: sqlite3.Row, *, observe: bool = False) -> MessageReceipt: lease_until=lease, error_code=row["error_code"], version=_version(row["version"]), + archived_at=archived, ) return ( replace(receipt, dispatch_state="unknown") @@ -218,6 +236,31 @@ def _receipt(row: sqlite3.Row, *, observe: bool = False) -> MessageReceipt: raise MessageStoreError("invalid_receipt") from None +def _file_size(path: Path) -> int | None: + try: + return path.stat().st_size + except OSError: + return None + + +def _wal_size(database_path: Path) -> int | None: + wal_path = database_path.with_name(f"{database_path.name}-wal") + try: + return wal_path.stat().st_size + except FileNotFoundError: + return 0 + except OSError: + return None + + +def _filesystem_free(path: Path) -> int | None: + try: + observation = os.statvfs(path) + return observation.f_bavail * observation.f_frsize + except OSError: + return None + + class MessageStore: def __init__(self, database_path: Path | str) -> None: self.database_path = Path(database_path) @@ -265,6 +308,92 @@ def get(self, message_id: str) -> MessageReceipt | None: ).fetchone() return _receipt(row, observe=True) if row is not None else None + def capacity(self) -> dict[str, object]: + """Observe receipt capacity and storage without changing SQLite state.""" + total = 0 + blocking = 0 + archived = 0 + with self._read() as conn: + cursor = conn.execute("SELECT * FROM message_receipts") + while rows := cursor.fetchmany(256): + for row in rows: + receipt = _receipt(row, observe=True) + total += 1 + archived += receipt.archived_at is not None + if receipt.released_at is None and ( + receipt.dispatch_state in {"prepared", "unknown"} + or ( + receipt.dispatch_state == "accepted" + and receipt.run_status not in _TERMINAL + ) + ): + blocking += 1 + + used = total - archived + remaining = max(0, MAX_MESSAGE_RECEIPTS - used) + if used >= MAX_MESSAGE_RECEIPTS: + status = "full" + elif used * 100 >= MAX_MESSAGE_RECEIPTS * 95: + status = "critical" + elif used * 100 >= MAX_MESSAGE_RECEIPTS * 80: + status = "warning" + else: + status = "ok" + return { + "limit": MAX_MESSAGE_RECEIPTS, + "used": used, + "remaining": remaining, + "total": total, + "archived": archived, + "blocking": blocking, + "status": status, + "database_bytes": _file_size(self.database_path), + "wal_bytes": _wal_size(self.database_path), + "filesystem_free_bytes": _filesystem_free(self.database_path.parent), + } + + def archive( + self, + *, + before: datetime | None = None, + limit: int = 100, + apply: bool = False, + now: datetime | None = None, + ) -> dict[str, object]: + """Preview or durably archive a bounded batch; preview reserves nothing.""" + now = _time(now if now is not None else datetime.now(UTC)) + before = _time(before) if before is not None else now - timedelta(days=30) + if before > now: + raise ValueError("message archive cutoff cannot be in the future") + if type(limit) is not int or not 1 <= limit <= 500 or type(apply) is not bool: + raise ValueError("message archive limit must be between 1 and 500") + with self._write() if apply else self._read() as conn: + rows = conn.execute( + "SELECT * FROM message_receipts WHERE archived_at IS NULL " + "AND updated_at < ? AND (dispatch_state = 'rejected' OR " + "(dispatch_state = 'accepted' AND run_status IN " + "('completed', 'failed', 'cancelled', 'interrupted'))) " + "ORDER BY updated_at, message_id LIMIT ?", + (before.isoformat(), limit), + ).fetchall() + receipts = [_receipt(row) for row in rows] + for receipt in receipts: + self._check_clock(receipt, now) + if receipt.version == 2**63 - 1: + raise MessageStoreError("invalid_receipt") + if apply: + conn.executemany( + "UPDATE message_receipts SET archived_at = ?, version = version + 1 " + "WHERE message_id = ?", + ((now.isoformat(), receipt.message_id) for receipt in receipts), + ) + return { + "message_ids": [receipt.message_id for receipt in receipts], + "count": len(receipts), + "applied": apply, + "before": before.isoformat(), + } + def list( self, *, bot_id: str | None = None, limit: int = 50, before: str | None = None ) -> dict[str, object]: @@ -358,7 +487,9 @@ def prepare( _receipt(blocker) raise MessageStoreError("bot_busy") if ( - conn.execute("SELECT count(*) FROM message_receipts").fetchone()[0] + conn.execute( + "SELECT count(*) FROM message_receipts WHERE archived_at IS NULL" + ).fetchone()[0] >= MAX_MESSAGE_RECEIPTS ): raise MessageStoreError("capacity_exceeded") @@ -398,7 +529,7 @@ def _load(conn: sqlite3.Connection, message_id: str) -> MessageReceipt: @staticmethod def _check_clock(receipt: MessageReceipt, now: datetime) -> None: - if now < receipt.updated_at: + if now < max(receipt.updated_at, receipt.archived_at or receipt.updated_at): raise MessageStoreError("clock_rollback") def _current( diff --git a/zeus/messaging_cli.py b/zeus/messaging_cli.py index cab99a3..6bbffb4 100644 --- a/zeus/messaging_cli.py +++ b/zeus/messaging_cli.py @@ -7,12 +7,13 @@ import os import stat import sys +from datetime import datetime from typing import Any from zeus.bot_messaging import MAX_INPUT_BYTES, BotMessaging, MessagingError from zeus.config import Settings from zeus.hermes_runs_client import HermesRunsClientError -from zeus.message_store import MessageStoreError +from zeus.message_store import MessageStore, MessageStoreError from zeus.state import StateReadinessError, StateStore from zeus.supervisor import Supervisor @@ -20,12 +21,12 @@ def add_messaging_parsers(sub: Any) -> None: messages = sub.add_parser("message", help="submit and inspect explicit operator jobs") actions = messages.add_subparsers(dest="action", required=True) - for action in ("send", "retry", "status", "cancel", "release", "list"): + for action in ("send", "retry", "status", "cancel", "release", "list", "capacity", "archive"): parser = actions.add_parser(action) if action == "send": parser.add_argument("bot_id") parser.add_argument("--request-key", help="optional stable key for this submission") - elif action != "list": + elif action not in {"list", "capacity", "archive"}: parser.add_argument("message_id") if action in {"send", "retry"}: parser.add_argument("--file", required=True, help="UTF-8 input file, or - for stdin") @@ -35,6 +36,12 @@ def add_messaging_parsers(sub: Any) -> None: action="store_true", help="release the local busy blocker; the job may still run and is not cancelled", ) + if action == "archive": + parser.add_argument( + "--before", help="exclusive timezone-aware ISO cutoff (default: 30 days ago)" + ) + parser.add_argument("--limit", type=int, default=100, help="maximum receipts (1-500)") + parser.add_argument("--apply", action="store_true", help="apply logical archival") if action == "list": parser.add_argument("--bot-id") parser.add_argument("--before", help="cursor returned by the previous page") @@ -65,33 +72,62 @@ def _read_input(path: str) -> str: def run_messaging_command(args: argparse.Namespace, settings: Settings) -> int: # Existing schema is required; commands do not initialize state, migrate, # reconcile, or start gateways as a side effect of sending a message. - workflow = BotMessaging( - Supervisor(StateStore(settings.database_path), settings.hermes_bin, settings.hermes_root) - ) try: - if args.action == "send": - payload = workflow.send( - args.bot_id, _read_input(args.file), request_key=args.request_key - ) - elif args.action == "retry": - payload = workflow.retry(args.message_id, _read_input(args.file)) - elif args.action == "status": - payload = workflow.status(args.message_id) - elif args.action == "cancel": - payload = workflow.cancel(args.message_id) - elif args.action == "release": - payload = workflow.release( - args.message_id, acknowledge_unknown_outcome=args.acknowledge_unknown_outcome + if args.action == "capacity": + payload = MessageStore(settings.database_path).capacity() + elif args.action == "archive": + payload = MessageStore(settings.database_path).archive( + before=datetime.fromisoformat(args.before) if args.before is not None else None, + limit=args.limit, + apply=args.apply, ) else: - payload = workflow.list(bot_id=args.bot_id, limit=args.limit, before=args.before) + workflow = BotMessaging( + Supervisor( + StateStore(settings.database_path), settings.hermes_bin, settings.hermes_root + ) + ) + if args.action == "send": + payload = workflow.send( + args.bot_id, _read_input(args.file), request_key=args.request_key + ) + elif args.action == "retry": + payload = workflow.retry(args.message_id, _read_input(args.file)) + elif args.action == "status": + payload = workflow.status(args.message_id) + elif args.action == "cancel": + payload = workflow.cancel(args.message_id) + elif args.action == "release": + payload = workflow.release( + args.message_id, + acknowledge_unknown_outcome=args.acknowledge_unknown_outcome, + ) + else: + payload = workflow.list(bot_id=args.bot_id, limit=args.limit, before=args.before) except StateReadinessError: return _error("not_ready", args.as_json) except (MessagingError, MessageStoreError, HermesRunsClientError) as exc: return _error(exc.code, args.as_json) except (OSError, RuntimeError, ValueError): return _error("message_state_unavailable", args.as_json) - print(json.dumps(payload, sort_keys=True, indent=None if args.as_json else 2)) + if args.action == "capacity" and not args.as_json: + for name in ( + "status", + "used", + "limit", + "remaining", + "total", + "archived", + "blocking", + "database_bytes", + "wal_bytes", + "filesystem_free_bytes", + ): + print(f"{name}: {payload[name]}") + else: + print(json.dumps(payload, sort_keys=True, indent=None if args.as_json else 2)) + if args.action == "capacity": + return 0 return 1 if payload.get("dispatch_state") in {"unknown", "rejected", "prepared"} else 0 diff --git a/zeus/schema.py b/zeus/schema.py index a3c0d4a..7a10723 100644 --- a/zeus/schema.py +++ b/zeus/schema.py @@ -7,7 +7,7 @@ from zeus.lifecycle import serialize_lifecycle_details -SCHEMA_VERSION = 9 +SCHEMA_VERSION = 10 class _SchemaDatabase(Protocol): @@ -168,6 +168,10 @@ def _migrate(self, conn: sqlite3.Connection) -> None: if current_version < 9: self._migrate_v8_to_v9(conn) conn.execute("UPDATE schema_version SET version = ?", (9,)) + current_version = 9 + if current_version < 10 <= SCHEMA_VERSION: + self._migrate_v9_to_v10(conn) + conn.execute("UPDATE schema_version SET version = ?", (10,)) def _ensure_restart_schema(self, conn: sqlite3.Connection) -> None: columns = {row["name"] for row in conn.execute("PRAGMA table_info(bots)").fetchall()} @@ -649,3 +653,35 @@ def _migrate_v8_to_v9(self, conn: sqlite3.Connection) -> None: "OR (dispatch_state = 'accepted' " "AND run_status NOT IN ('completed', 'failed', 'cancelled', 'interrupted')))" ) + + def _migrate_v9_to_v10(self, conn: sqlite3.Connection) -> None: + # Logical archival retains the complete receipt and all deduplication keys. + # Later terminal observations may advance updated_at past archived_at. + conn.execute( + """ + ALTER TABLE message_receipts ADD COLUMN archived_at TEXT CHECK ( + archived_at IS NULL OR ( + typeof(archived_at) = 'text' + AND archived_at >= created_at + AND (dispatch_state = 'rejected' OR ( + dispatch_state = 'accepted' + AND run_status IN ('completed', 'failed', 'cancelled', 'interrupted') + )) + AND length(archived_at) IN (25, 32) + AND substr(archived_at, -6) = '+00:00' + AND datetime(substr(archived_at, 1, 19)) IS NOT NULL + AND strftime('%Y-%m-%dT%H:%M:%S', substr(archived_at, 1, 19)) + = substr(archived_at, 1, 19) + AND (length(archived_at) = 25 OR ( + substr(archived_at, 20, 1) = '.' + AND substr(archived_at, 21, 6) NOT GLOB '*[^0-9]*' + AND substr(archived_at, 21, 6) != '000000' + )) + ) + ) + """ + ) + conn.execute( + "CREATE INDEX message_receipts_unarchived_idx " + "ON message_receipts (updated_at, message_id) WHERE archived_at IS NULL" + ) diff --git a/zeus/supervisor.py b/zeus/supervisor.py index 21e5e77..0d1ba10 100644 --- a/zeus/supervisor.py +++ b/zeus/supervisor.py @@ -3,6 +3,9 @@ import os import platform import subprocess # nosec B404 +from collections.abc import Sequence +from datetime import datetime +from pathlib import Path from zeus import process_identity as _process_identity from zeus.gateway_launcher import ( @@ -22,9 +25,35 @@ PopenLike, RuntimeHooks, SignalResult, + StopEffect, +) +from zeus.lifecycle import LifecycleEvent +from zeus.models import ( + BotCreateRequest, + BotRecord, + BotStatus, + BotStatusResponse, + HermesTemplate, ) from zeus.readiness import ReadinessProbe, ReadinessResult, probe_once -from zeus.supervisor_registry import _SupervisorRegistry +from zeus.reconciliation import ( + BotReconcileResult, + ReconcileExecution, + ReconcileOutcome, + ReconcileRunSummary, +) +from zeus.supervisor_contracts import ( + _READINESS_PROBE_UNSET, + _LifecycleContext, + _ReadinessProbeUnset, + _ReconcileLaunch, +) +from zeus.supervisor_reconcile import ReconcileOperations +from zeus.supervisor_registry import RegistryOperations +from zeus.supervisor_runtime import _SupervisorCore +from zeus.supervisor_start import StartOperations +from zeus.supervisor_status import StatusOperations +from zeus.supervisor_stop import StopOperations PidAliveFn = _process_identity.PidAliveFn CmdlineReader = _process_identity.CmdlineReader @@ -77,7 +106,20 @@ ] -class Supervisor(_SupervisorRegistry): +_REGISTRY = RegistryOperations() +_STATUS = StatusOperations() +_START = StartOperations() +_STOP = StopOperations() +_RECONCILE = ReconcileOperations() + + +class Supervisor(_SupervisorCore): + """Public lifecycle facade over one state-owning core and stateless operations. + + Delegates pass the current host so operation-to-operation calls resolve live + methods, including supported instance patches and subclass overrides. + """ + @staticmethod def _default_cmdline_reader(pid: int) -> list[str] | None: return _read_process_cmdline(pid) @@ -113,6 +155,649 @@ def _pid_state(self, pid: int) -> _PidState: def _process_start_fingerprint_required() -> bool: return platform.system() in {"Linux", "Darwin"} + def create_bot( + self, + request: BotCreateRequest, + template: HermesTemplate, + *, + replace_existing: bool = False, + stop_if_running: bool = False, + source: str = "cli", + request_id: str | None = None, + ) -> BotRecord: + return _REGISTRY.create_bot( + self, + request, + template, + replace_existing=replace_existing, + stop_if_running=stop_if_running, + source=source, + request_id=request_id, + ) + + def delete_bot( + self, + bot_id: str, + *, + stop_if_running: bool = False, + remove_profile: bool = False, + source: str = "cli", + request_id: str | None = None, + ) -> BotStatusResponse: + return _REGISTRY.delete_bot( + self, + bot_id, + stop_if_running=stop_if_running, + remove_profile=remove_profile, + source=source, + request_id=request_id, + ) + + def archive_bot( + self, + bot_id: str, + *, + stop_if_running: bool = False, + source: str = "cli", + request_id: str | None = None, + ) -> dict[str, object]: + return _REGISTRY.archive_bot( + self, bot_id, stop_if_running=stop_if_running, source=source, request_id=request_id + ) + + def _record_may_be_active(self, record: BotRecord) -> bool: + return _REGISTRY._record_may_be_active(self, record) + + def _recover_previously_active_bot( + self, + record: BotRecord, + operation: str, + *, + context: _LifecycleContext, + ) -> None: + return _REGISTRY._recover_previously_active_bot(self, record, operation, context=context) + + def _assert_unregistered_profile_inactive( + self, + bot_id: str, + profile_path: Path, + ) -> None: + return _REGISTRY._assert_unregistered_profile_inactive(self, bot_id, profile_path) + + def _safe_profile_path(self, bot_id: str, profile_path: str) -> Path: + return _REGISTRY._safe_profile_path(self, bot_id, profile_path) + + def _stage_profile_deletion(self, bot_id: str, profile_path: str) -> Path | None: + return _REGISTRY._stage_profile_deletion(self, bot_id, profile_path) + + def _restore_tombstoned_profile( + self, + bot_id: str, + profile_path: str, + tombstone: Path, + ) -> None: + return _REGISTRY._restore_tombstoned_profile(self, bot_id, profile_path, tombstone) + + def _restore_archived_profile( + self, + bot_id: str, + profile_path: str, + archive_path: Path, + ) -> None: + return _REGISTRY._restore_archived_profile(self, bot_id, profile_path, archive_path) + + def status( + self, + bot_id: str, + *, + source: str = "cli", + request_id: str | None = None, + ) -> BotStatusResponse: + return _STATUS.status(self, bot_id, source=source, request_id=request_id) + + def _status_locked(self, bot_id: str, *, context: _LifecycleContext) -> BotStatusResponse: + return _STATUS._status_locked(self, bot_id, context=context) + + def _status_dead_record_locked( + self, + record: BotRecord, + *, + context: _LifecycleContext, + ) -> BotStatusResponse: + return _STATUS._status_dead_record_locked(self, record, context=context) + + def logs(self, bot_id: str, max_bytes: int = 20_000) -> str: + return _STATUS.logs(self, bot_id, max_bytes) + + def inspect(self, bot_id: str, max_log_bytes: int = 20_000) -> dict[str, object]: + return _STATUS.inspect(self, bot_id, max_log_bytes) + + def start( + self, + bot_id: str, + *, + wait: bool = False, + timeout_seconds: float | None = None, + source: str = "cli", + request_id: str | None = None, + ) -> BotStatusResponse: + return _START.start( + self, + bot_id, + wait=wait, + timeout_seconds=timeout_seconds, + source=source, + request_id=request_id, + ) + + def _start_locked( + self, + bot_id: str, + *, + wait: bool = False, + timeout_seconds: float | None = None, + context: _LifecycleContext, + ) -> BotStatusResponse: + return _START._start_locked( + self, bot_id, wait=wait, timeout_seconds=timeout_seconds, context=context + ) + + def _start_record( + self, + record: BotRecord, + *, + reset_restart: bool, + message: str, + wait: bool = False, + timeout_seconds: float | None = None, + context: _LifecycleContext, + probe: ReadinessProbe | _ReadinessProbeUnset | None = _READINESS_PROBE_UNSET, + ) -> BotStatusResponse: + return _START._start_record( + self, + record, + reset_restart=reset_restart, + message=message, + wait=wait, + timeout_seconds=timeout_seconds, + context=context, + probe=probe, + ) + + def _preflight_start( + self, record: BotRecord, *, timeout_seconds: float | None + ) -> ReadinessProbe | None: + return _START._preflight_start(self, record, timeout_seconds=timeout_seconds) + + def _write_pipe_payload(self, fd: int, payload: bytes) -> None: + return _START._write_pipe_payload(self, fd, payload) + + def _read_launcher_ack(self, fd: int) -> bytes: + return _START._read_launcher_ack(self, fd) + + def _complete_started_intent( + self, + record: BotRecord, + *, + context: _LifecycleContext, + status: BotStatus, + pid: int, + reason: str, + ready_at: datetime | None = None, + last_error: str | None = None, + reset_restart: bool = False, + ) -> BotRecord: + return _START._complete_started_intent( + self, + record, + context=context, + status=status, + pid=pid, + reason=reason, + ready_at=ready_at, + last_error=last_error, + reset_restart=reset_restart, + ) + + def _complete_failed_intent( + self, + record: BotRecord, + *, + context: _LifecycleContext, + pid: int | None, + message: str, + reason: str, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + ) -> BotRecord: + return _START._complete_failed_intent( + self, + record, + context=context, + pid=pid, + message=message, + reason=reason, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + ) + + def _cleanup_interrupted_intent_launch( + self, + record: BotRecord, + process: PopenLike, + *, + expected_fingerprint: str, + ) -> bool: + return _START._cleanup_interrupted_intent_launch( + self, record, process, expected_fingerprint=expected_fingerprint + ) + + def _launch_completion_failure_response( + self, + record: BotRecord, + generation: _GatewayGeneration, + ) -> BotStatusResponse: + return _START._launch_completion_failure_response(self, record, generation) + + def stop( + self, + bot_id: str, + *, + kill_after_timeout: bool | None = None, + source: str = "cli", + request_id: str | None = None, + ) -> BotStatusResponse: + return _STOP.stop( + self, + bot_id, + kill_after_timeout=kill_after_timeout, + source=source, + request_id=request_id, + ) + + def _stop_locked( + self, + bot_id: str, + *, + kill_after_timeout: bool | None = None, + context: _LifecycleContext, + ) -> BotStatusResponse: + return _STOP._stop_locked( + self, bot_id, kill_after_timeout=kill_after_timeout, context=context + ) + + def _stop_record_effect( + self, + record: BotRecord, + *, + kill_after_timeout: bool | None = None, + context: _LifecycleContext, + complete_stop: bool, + ) -> BotStatusResponse: + return _STOP._stop_record_effect( + self, + record, + kill_after_timeout=kill_after_timeout, + context=context, + complete_stop=complete_stop, + ) + + def _stop_record_effect_locked( + self, + record: BotRecord, + *, + kill_after_timeout: bool | None, + context: _LifecycleContext, + complete_stop: bool, + ) -> BotStatusResponse: + return _STOP._stop_record_effect_locked( + self, + record, + kill_after_timeout=kill_after_timeout, + context=context, + complete_stop=complete_stop, + ) + + def _complete_stopped_intent( + self, + record: BotRecord, + *, + context: _LifecycleContext, + reason: str, + ) -> BotRecord: + return _STOP._complete_stopped_intent(self, record, context=context, reason=reason) + + def _remove_owned_launch_marker_locked( + self, + record: BotRecord, + *, + observed: _MarkerObservation | None = None, + ) -> bool: + return _STOP._remove_owned_launch_marker_locked(self, record, observed=observed) + + def restart( + self, + bot_id: str, + *, + wait: bool = False, + timeout_seconds: float | None = None, + source: str = "cli", + request_id: str | None = None, + ) -> BotStatusResponse: + return _STOP.restart( + self, + bot_id, + wait=wait, + timeout_seconds=timeout_seconds, + source=source, + request_id=request_id, + ) + + def reconcile( + self, + bot_id: str | None = None, + *, + now: datetime | None = None, + force: bool = False, + reset_restart: bool = False, + source: str = "reconcile", + request_id: str | None = None, + bot_snapshot: Sequence[tuple[str, str]] | None = None, + ) -> list[BotStatusResponse]: + return _RECONCILE.reconcile( + self, + bot_id, + now=now, + force=force, + reset_restart=reset_restart, + source=source, + request_id=request_id, + bot_snapshot=bot_snapshot, + ) + + def reconcile_summary( + self, + bot_id: str | None = None, + *, + now: datetime | None = None, + force: bool = False, + reset_restart: bool = False, + source: str = "reconcile", + request_id: str | None = None, + bot_snapshot: Sequence[tuple[str, str]] | None = None, + ) -> ReconcileRunSummary: + return _RECONCILE.reconcile_summary( + self, + bot_id, + now=now, + force=force, + reset_restart=reset_restart, + source=source, + request_id=request_id, + bot_snapshot=bot_snapshot, + ) + + def reconcile_execution( + self, + bot_id: str | None = None, + *, + now: datetime | None = None, + force: bool = False, + reset_restart: bool = False, + source: str = "reconcile", + request_id: str | None = None, + bot_snapshot: Sequence[tuple[str, str]] | None = None, + ) -> ReconcileExecution: + return _RECONCILE.reconcile_execution( + self, + bot_id, + now=now, + force=force, + reset_restart=reset_restart, + source=source, + request_id=request_id, + bot_snapshot=bot_snapshot, + ) + + def validate_reconcile_request(self, source: str, request_id: str | None) -> None: + return _RECONCILE.validate_reconcile_request(self, source, request_id) + + def validate_reconcile_target( + self, + bot_id: str, + *, + expected_profile_path: str | None = None, + ) -> str: + return _RECONCILE.validate_reconcile_target( + self, bot_id, expected_profile_path=expected_profile_path + ) + + def reconcile_one( + self, + bot_id: str, + *, + now: datetime | None = None, + force: bool = False, + reset_restart: bool = False, + source: str = "reconcile", + request_id: str | None = None, + expected_profile_path: str | None = None, + ) -> BotReconcileResult: + return _RECONCILE.reconcile_one( + self, + bot_id, + now=now, + force=force, + reset_restart=reset_restart, + source=source, + request_id=request_id, + expected_profile_path=expected_profile_path, + ) + + def reconcile_one_execution( + self, + bot_id: str, + *, + now: datetime | None = None, + force: bool = False, + reset_restart: bool = False, + source: str = "reconcile", + request_id: str | None = None, + expected_profile_path: str | None = None, + ) -> tuple[BotReconcileResult, BotStatusResponse]: + return _RECONCILE.reconcile_one_execution( + self, + bot_id, + now=now, + force=force, + reset_restart=reset_restart, + source=source, + request_id=request_id, + expected_profile_path=expected_profile_path, + ) + + def _latest_reconcile_event( + self, + bot_id: str, + prior_event_id: int | None, + ) -> LifecycleEvent | None: + return _RECONCILE._latest_reconcile_event(self, bot_id, prior_event_id) + + def _reconcile_result_from_response( + self, + before: BotRecord, + after: BotRecord, + response: BotStatusResponse, + *, + current_event: LifecycleEvent | None, + started_at: datetime, + ) -> BotReconcileResult: + return _RECONCILE._reconcile_result_from_response( + self, before, after, response, current_event=current_event, started_at=started_at + ) + + @staticmethod + def _reconcile_outcome( + before: BotRecord, + after: BotRecord, + response: BotStatusResponse, + *, + current_event_action: str | None, + ) -> ReconcileOutcome: + return _RECONCILE._reconcile_outcome( + before, after, response, current_event_action=current_event_action + ) + + def _reconcile_record( + self, + record: BotRecord, + now: datetime, + *, + force: bool, + reset_restart: bool, + context: _LifecycleContext, + ) -> BotStatusResponse: + return _RECONCILE._reconcile_record( + self, record, now, force=force, reset_restart=reset_restart, context=context + ) + + def _prepare_reconcile_dead_record_locked( + self, + record: BotRecord, + now: datetime, + *, + force: bool, + context: _LifecycleContext, + ) -> BotStatusResponse | _ReconcileLaunch: + return _RECONCILE._prepare_reconcile_dead_record_locked( + self, record, now, force=force, context=context + ) + + @staticmethod + def _is_compat_runtime_marker(payload: dict[str, object]) -> bool: + return _RECONCILE._is_compat_runtime_marker(payload) + + def _recover_pending_intent( + self, + record: BotRecord, + *, + context: _LifecycleContext, + allow_launch: bool, + ) -> BotStatusResponse: + return _RECONCILE._recover_pending_intent( + self, record, context=context, allow_launch=allow_launch + ) + + @staticmethod + def _recovery_lifecycle_context( + operation_id: str, + context: _LifecycleContext, + ) -> _LifecycleContext: + return _RECONCILE._recovery_lifecycle_context(operation_id, context) + + def _pending_launch_preflight( + self, + record: BotRecord, + operation_id: str, + ) -> tuple[ReadinessProbe | None, str]: + return _RECONCILE._pending_launch_preflight(self, record, operation_id) + + def _recover_pending_stop_intent( + self, + record: BotRecord, + *, + context: _LifecycleContext, + allow_stop: bool, + ) -> BotStatusResponse: + return _RECONCILE._recover_pending_stop_intent( + self, record, context=context, allow_stop=allow_stop + ) + + def _recover_pending_launch( + self, + record: BotRecord, + *, + context: _LifecycleContext, + probe: ReadinessProbe | None, + fingerprint: str, + action: str, + allow_launch: bool, + ) -> BotStatusResponse | None: + return _RECONCILE._recover_pending_launch( + self, + record, + context=context, + probe=probe, + fingerprint=fingerprint, + action=action, + allow_launch=allow_launch, + ) + + def _recover_pending_stop_intent_locked( + self, + record: BotRecord, + *, + context: _LifecycleContext, + allow_stop: bool, + ) -> BotStatusResponse: + return _RECONCILE._recover_pending_stop_intent_locked( + self, record, context=context, allow_stop=allow_stop + ) + + def _pending_restart_old_marker( + self, + record: BotRecord, + observed: _MarkerObservation | None = None, + ) -> _MarkerObservation | None: + return _RECONCILE._pending_restart_old_marker(self, record, observed) + + def _recover_pending_restart_predecessor( + self, + record: BotRecord, + *, + context: _LifecycleContext, + allow_stop: bool, + ) -> BotStatusResponse | None: + return _RECONCILE._recover_pending_restart_predecessor( + self, record, context=context, allow_stop=allow_stop + ) + + def _recover_pending_restart_old_gateway( + self, + record: BotRecord, + marker: _MarkerObservation, + *, + context: _LifecycleContext, + allow_stop: bool, + ) -> BotStatusResponse: + return _RECONCILE._recover_pending_restart_old_gateway( + self, record, marker, context=context, allow_stop=allow_stop + ) + + def _stop_pending_restart_old_gateway( + self, + record: BotRecord, + generation: _GatewayGeneration, + *, + context: _LifecycleContext, + ) -> BotStatusResponse: + return _RECONCILE._stop_pending_restart_old_gateway( + self, record, generation, context=context + ) + + def _stop_gateway_generation_locked( + self, + record: BotRecord, + generation: _GatewayGeneration, + ) -> StopEffect: + return _RECONCILE._stop_gateway_generation_locked(self, record, generation) + + def _append_recovery_audit_event(self, action: str, **values: object) -> None: + return _RECONCILE._append_recovery_audit_event(self, action, **values) + + def _restart_delay(self, record: BotRecord) -> float: + return _RECONCILE._restart_delay(self, record) + def _read_process_cmdline(pid: int) -> list[str] | None: return _process_identity.read_process_cmdline( diff --git a/zeus/supervisor_contracts.py b/zeus/supervisor_contracts.py new file mode 100644 index 0000000..796755f --- /dev/null +++ b/zeus/supervisor_contracts.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from zeus.gateway_marker import GatewayGeneration +from zeus.gateway_runtime import MarkerObservation, SignalResult +from zeus.models import BotRecord +from zeus.readiness import ReadinessProbe + + +class _ReadinessProbeUnset: + pass + + +_READINESS_PROBE_UNSET = _ReadinessProbeUnset() +_MarkerObservation = MarkerObservation +_GatewayGeneration = GatewayGeneration +_SignalResult = SignalResult + + +@dataclass(frozen=True) +class _LifecycleContext: + operation_id: str + source: str + request_id: str | None + + +@dataclass(frozen=True) +class _ReconcileLaunch: + record: BotRecord + probe: ReadinessProbe | None + attempt: int + restart_max_attempts: int diff --git a/zeus/supervisor_core.py b/zeus/supervisor_core.py index fae7f9a..28dd8f8 100644 --- a/zeus/supervisor_core.py +++ b/zeus/supervisor_core.py @@ -1,524 +1,24 @@ -from __future__ import annotations +"""Compatibility imports for the consolidated supervisor core.""" -import contextlib -import os -import platform -import re -import subprocess # nosec B404 -import threading -import uuid -from dataclasses import dataclass -from datetime import datetime -from pathlib import Path - -from zeus import process_identity as _process_identity -from zeus.gateway_launcher import ( - _read_bounded_file, - _remove_marker_if_owned_locked, +from zeus.supervisor_contracts import ( + _READINESS_PROBE_UNSET as _READINESS_PROBE_UNSET, ) -from zeus.gateway_marker import ( - GatewayGeneration, - readiness_probe_from_payload, - readiness_probe_to_payload, +from zeus.supervisor_contracts import ( + _GatewayGeneration as _GatewayGeneration, ) -from zeus.gateway_runtime import ( - GatewayRuntime, - KillFn, - MarkerObservation, - PopenFactory, - PopenLike, - RuntimeHooks, - SignalResult, - gateway_process_launch_kwargs, +from zeus.supervisor_contracts import ( + _LifecycleContext as _LifecycleContext, ) -from zeus.hermes_adapter import HermesAdapter -from zeus.intent_recovery import PendingIntentRecovery -from zeus.lifecycle import LifecycleEventInput -from zeus.models import ( - BotRecord, - BotStatus, - validate_id, +from zeus.supervisor_contracts import ( + _MarkerObservation as _MarkerObservation, ) -from zeus.private_io import nofollow_absolute_path -from zeus.process_lock import BotProcessLock -from zeus.profile_manager import ProfileManager -from zeus.readiness import ReadinessProbe, ReadinessResult, probe_once -from zeus.state import StateStore - -PidAliveFn = _process_identity.PidAliveFn -CmdlineReader = _process_identity.CmdlineReader -ProcStartFingerprintReader = _process_identity.ProcStartFingerprintReader - -_CommandCheck = _process_identity.CommandCheck -_PidState = _process_identity.PidState -_looks_like_python_interpreter = _process_identity.looks_like_python_interpreter -_read_linux_cmdline = _process_identity.read_linux_cmdline -_read_linux_process_start_fingerprint = _process_identity.read_linux_process_start_fingerprint -_resolve_executable = _process_identity.resolve_executable -_resolve_launcher_exec_target = _process_identity.resolve_launcher_exec_target -_safe_command_shape = _process_identity.safe_command_shape -_trusted_hermes_paths = _process_identity.trusted_hermes_paths -_verify_gateway_command = _process_identity.verify_gateway_command - - -_SignalResult = SignalResult - - -class _ReadinessProbeUnset: - pass - - -_REQUEST_ID_RE = re.compile(r"^[0-9a-f]{32}$") -_LIFECYCLE_SOURCES = frozenset({"api", "cli", "reconcile", "recovery", "system"}) -_READINESS_PROBE_UNSET = _ReadinessProbeUnset() - - -@dataclass(frozen=True) -class _LifecycleContext: - operation_id: str - source: str - request_id: str | None - - -_MarkerObservation = MarkerObservation - - -_GatewayGeneration = GatewayGeneration - - -@dataclass(frozen=True) -class _ReconcileLaunch: - record: BotRecord - probe: ReadinessProbe | None - attempt: int - restart_max_attempts: int - - -def _gateway_process_launch_kwargs() -> dict[str, object]: - return gateway_process_launch_kwargs() - - -def _nofollow_absolute_path(path: Path) -> Path: - return nofollow_absolute_path(path) - - -def _same_identity(first: os.stat_result, second: os.stat_result) -> bool: - return first.st_dev == second.st_dev and first.st_ino == second.st_ino - - -def _caused_by_missing_path(exc: BaseException) -> bool: - current: BaseException | None = exc - while current is not None: - if isinstance(current, FileNotFoundError): - return True - current = current.__cause__ - return False - - -class _SupervisorCore: - @staticmethod - def _default_cmdline_reader(pid: int) -> list[str] | None: - return _read_process_cmdline(pid) - - @staticmethod - def _default_process_start_fingerprint_reader(pid: int) -> str | None: - return _read_process_start_fingerprint(pid) - - @staticmethod - def _probe_once(probe: ReadinessProbe) -> ReadinessResult: - return probe_once( - probe.url, - timeout_seconds=min(1.0, max(0.2, probe.interval_seconds)), - expected_status=probe.expected_status, - expected_platform=probe.expected_platform, - ) - - def __init__( - self, - store: StateStore, - hermes_bin: str, - hermes_root: Path | str, - popen_factory: PopenFactory = subprocess.Popen, - kill_fn: KillFn = os.kill, - pid_alive_fn: PidAliveFn | None = None, - cmdline_reader: CmdlineReader | None = None, - startup_grace_seconds: float = 0.25, - stop_grace_seconds: float = 60.0, - kill_after_timeout: bool = False, - lock_timeout_seconds: float = 30.0, - readiness_timeout_seconds: float = 30.0, - readiness_interval_seconds: float = 0.5, - allow_legacy_pid_markers: bool = True, - restart_backoff_cap_seconds: float = 3600.0, - proc_start_fingerprint_reader: ProcStartFingerprintReader | None = None, - restart_stability_seconds: float = 30.0, - ) -> None: - if not 0.0 <= restart_stability_seconds <= 86_400.0: - raise ValueError("restart_stability_seconds must be between 0 and 86400") - self.store = store - configured_hermes_root = _nofollow_absolute_path(Path(hermes_root)) - self.adapter = HermesAdapter( - hermes_bin=hermes_bin, - hermes_root=configured_hermes_root.resolve(), - ) - self._profile_manager = ProfileManager( - self.adapter.hermes_root, - self.store.database_path.parent / "archive", - ) - self._marker_profiles_root = configured_hermes_root / "profiles" - self.startup_grace_seconds = startup_grace_seconds - self.lock_dir = self.store.database_path.parent / "locks" / "bots" - self.readiness_timeout_seconds = readiness_timeout_seconds - self.readiness_interval_seconds = readiness_interval_seconds - self.allow_legacy_pid_markers = allow_legacy_pid_markers - self.restart_backoff_cap_seconds = restart_backoff_cap_seconds - self.restart_stability_seconds = restart_stability_seconds - self._cleanup_process_group = os.name == "posix" and popen_factory is subprocess.Popen - self._runtime = GatewayRuntime( - self.adapter, - self._profile_manager, - self._marker_profiles_root, - popen_factory=popen_factory, - kill_fn=kill_fn, - pid_alive_fn=pid_alive_fn, - cmdline_reader=cmdline_reader or self._default_cmdline_reader, - proc_start_fingerprint_reader=( - proc_start_fingerprint_reader or self._default_process_start_fingerprint_reader - ), - startup_grace_seconds=startup_grace_seconds, - stop_grace_seconds=stop_grace_seconds, - kill_after_timeout=kill_after_timeout, - lock_timeout_seconds=lock_timeout_seconds, - readiness_timeout_seconds=readiness_timeout_seconds, - readiness_interval_seconds=readiness_interval_seconds, - allow_legacy_pid_markers=allow_legacy_pid_markers, - cleanup_process_group=self._cleanup_process_group, - hooks_provider=self._runtime_hooks, - ) - self._intent_recovery = PendingIntentRecovery() - self._locks_guard = threading.Lock() - self._bot_locks: dict[str, threading.RLock] = {} - - def _runtime_hooks(self) -> RuntimeHooks: - return RuntimeHooks( - pipe=os.pipe, - close=os.close, - read_bounded_file=_read_bounded_file, - remove_marker_if_owned_locked=_remove_marker_if_owned_locked, - probe_once=probe_once, - ) - - def _get_runtime_proxy(self, name: str) -> object: - runtime = self.__dict__.get("_runtime") - if runtime is not None: - return getattr(runtime, name) - return self.__dict__.get(f"_runtime_proxy_{name}") - - def _set_runtime_proxy(self, name: str, value: object) -> None: - runtime = self.__dict__.get("_runtime") - history = self.__dict__.setdefault(f"_runtime_proxy_history_{name}", []) - if isinstance(history, list): - if len(history) >= 32: - del history[0] - history.append( - getattr(runtime, name) - if runtime is not None - else self.__dict__.get(f"_runtime_proxy_{name}") - ) - if runtime is not None: - setattr(runtime, name, value) - else: - self.__dict__[f"_runtime_proxy_{name}"] = value - - def _delete_runtime_proxy(self, name: str) -> None: - history = self.__dict__.get(f"_runtime_proxy_history_{name}") - if not isinstance(history, list) or not history: - self.__dict__.pop(f"_runtime_proxy_{name}", None) - return - previous = history.pop() - runtime = self.__dict__.get("_runtime") - if runtime is not None: - setattr(runtime, name, previous) - else: - self.__dict__[f"_runtime_proxy_{name}"] = previous - - @property - def popen_factory(self) -> PopenFactory: - return self._get_runtime_proxy("popen_factory") # type: ignore[return-value] - - @popen_factory.setter - def popen_factory(self, value: PopenFactory) -> None: - self._set_runtime_proxy("popen_factory", value) - - @popen_factory.deleter - def popen_factory(self) -> None: - self._delete_runtime_proxy("popen_factory") - - @property - def kill_fn(self) -> KillFn: - return self._get_runtime_proxy("kill_fn") # type: ignore[return-value] - - @kill_fn.setter - def kill_fn(self, value: KillFn) -> None: - self._set_runtime_proxy("kill_fn", value) - - @kill_fn.deleter - def kill_fn(self) -> None: - self._delete_runtime_proxy("kill_fn") - - @property - def pid_alive_fn(self) -> PidAliveFn | None: - return self._get_runtime_proxy("pid_alive_fn") # type: ignore[return-value] - - @pid_alive_fn.setter - def pid_alive_fn(self, value: PidAliveFn | None) -> None: - self._set_runtime_proxy("pid_alive_fn", value) - - @pid_alive_fn.deleter - def pid_alive_fn(self) -> None: - self._delete_runtime_proxy("pid_alive_fn") - - @property - def cmdline_reader(self) -> CmdlineReader: - return self._get_runtime_proxy("cmdline_reader") # type: ignore[return-value] - - @cmdline_reader.setter - def cmdline_reader(self, value: CmdlineReader) -> None: - self._set_runtime_proxy("cmdline_reader", value) - - @cmdline_reader.deleter - def cmdline_reader(self) -> None: - self._delete_runtime_proxy("cmdline_reader") - - @property - def proc_start_fingerprint_reader(self) -> ProcStartFingerprintReader: - return self._get_runtime_proxy("proc_start_fingerprint_reader") # type: ignore[return-value] - - @proc_start_fingerprint_reader.setter - def proc_start_fingerprint_reader(self, value: ProcStartFingerprintReader) -> None: - self._set_runtime_proxy("proc_start_fingerprint_reader", value) - - @proc_start_fingerprint_reader.deleter - def proc_start_fingerprint_reader(self) -> None: - self._delete_runtime_proxy("proc_start_fingerprint_reader") - - @property - def _processes(self) -> dict[str, PopenLike]: - return self._get_runtime_proxy("_processes") # type: ignore[return-value] - - @_processes.setter - def _processes(self, value: dict[str, PopenLike]) -> None: - self._set_runtime_proxy("_processes", value) - - @_processes.deleter - def _processes(self) -> None: - self._delete_runtime_proxy("_processes") - - @property - def stop_grace_seconds(self) -> float: - return self._get_runtime_proxy("stop_grace_seconds") # type: ignore[return-value] - - @stop_grace_seconds.setter - def stop_grace_seconds(self, value: float) -> None: - self._set_runtime_proxy("stop_grace_seconds", value) - - @stop_grace_seconds.deleter - def stop_grace_seconds(self) -> None: - self._delete_runtime_proxy("stop_grace_seconds") - - @property - def kill_after_timeout(self) -> bool: - return self._get_runtime_proxy("kill_after_timeout") # type: ignore[return-value] - - @kill_after_timeout.setter - def kill_after_timeout(self, value: bool) -> None: - self._set_runtime_proxy("kill_after_timeout", value) - - @kill_after_timeout.deleter - def kill_after_timeout(self) -> None: - self._delete_runtime_proxy("kill_after_timeout") - - @property - def lock_timeout_seconds(self) -> float: - return self._get_runtime_proxy("lock_timeout_seconds") # type: ignore[return-value] - - @lock_timeout_seconds.setter - def lock_timeout_seconds(self, value: float) -> None: - self._set_runtime_proxy("lock_timeout_seconds", value) - - @lock_timeout_seconds.deleter - def lock_timeout_seconds(self) -> None: - self._delete_runtime_proxy("lock_timeout_seconds") - - def _lifecycle_context(self, source: str, request_id: str | None) -> _LifecycleContext: - if source not in _LIFECYCLE_SOURCES: - raise ValueError("invalid lifecycle event source") - if source == "api": - if request_id is None or _REQUEST_ID_RE.fullmatch(request_id) is None: - raise ValueError("API lifecycle operations require a generated request ID") - elif request_id is not None: - raise ValueError("only API lifecycle operations may carry a request ID") - return _LifecycleContext(uuid.uuid4().hex, source, request_id) - - def _event( - self, - context: _LifecycleContext, - bot_id: str, - *, - action: str, - outcome: str = "success", - reason: str = "", - error_code: str | None = None, - error_message: str | None = None, - details: dict[str, object] | None = None, - ) -> LifecycleEventInput: - return LifecycleEventInput( - bot_id=bot_id, - operation_id=context.operation_id, - request_id=context.request_id, - source=context.source, - action=action, - outcome=outcome, - reason=reason, - error_code=error_code, - error_message=error_message, - details=details or {}, - ) - - def _update_lifecycle( - self, - context: _LifecycleContext, - bot_id: str, - status: BotStatus, - pid: int | None = None, - *, - action: str | None = None, - started_at: datetime | None = None, - ready_at: datetime | None = None, - stopped_at: datetime | None = None, - last_exit_code: int | None = None, - last_error: str | None = None, - last_transition_reason: str | None = None, - reset_restart: bool = False, - clear_ready_at: bool = False, - clear_stopped_at: bool = False, - details: dict[str, object] | None = None, - ) -> None: - reason = last_transition_reason or "" - failed = status in {BotStatus.failed, BotStatus.unknown} - self.store.update_lifecycle_with_event( - bot_id, - status, - pid, - event=self._event( - context, - bot_id, - action=action or f"bot.{status.value}", - outcome="failure" if failed else "success", - reason=reason, - error_code=f"bot_{status.value}" if failed else None, - error_message=last_error, - details=details, - ), - started_at=started_at, - ready_at=ready_at, - stopped_at=stopped_at, - last_exit_code=last_exit_code, - last_error=last_error, - last_transition_reason=last_transition_reason, - reset_restart=reset_restart, - clear_ready_at=clear_ready_at, - clear_stopped_at=clear_stopped_at, - ) - - def _update_restart( - self, - context: _LifecycleContext, - bot_id: str, - *, - status: BotStatus, - pid: int | None, - restart_attempts: int, - next_restart_at: datetime | None, - action: str, - reason: str, - outcome: str = "success", - error_code: str | None = None, - ) -> None: - self.store.update_restart_with_event( - bot_id, - status=status, - pid=pid, - restart_attempts=restart_attempts, - next_restart_at=next_restart_at, - event=self._event( - context, - bot_id, - action=action, - outcome=outcome, - reason=reason, - error_code=error_code, - details={ - "restart_attempts": restart_attempts, - "next_restart_at": ( - next_restart_at.isoformat() if next_restart_at is not None else None - ), - }, - ), - ) - - def bot_lock(self, bot_id: str) -> threading.RLock: - with self._locks_guard: - lock = self._bot_locks.get(bot_id) - if lock is None: - lock = threading.RLock() - self._bot_locks[bot_id] = lock - return lock - - def _bot_process_lock(self, bot_id: str) -> BotProcessLock: - safe_bot_id = validate_id(bot_id, "bot_id") - return BotProcessLock( - self.lock_dir / f"{safe_bot_id}.lock", - timeout_seconds=self.lock_timeout_seconds, - ) - - def _marker_publication_lock( - self, - record: BotRecord, - ) -> contextlib.AbstractContextManager[object]: - return self._runtime.marker_publication_lock(record) - - -def _read_process_cmdline(pid: int) -> list[str] | None: - return _process_identity.read_process_cmdline( - pid, - system=platform.system(), - run_process=subprocess.run, - ) - - -def _readiness_probe_marker_payload(probe: ReadinessProbe | None) -> dict[str, object] | None: - return readiness_probe_to_payload(probe) - - -def _readiness_probe_from_marker(value: object) -> ReadinessProbe | None: - return readiness_probe_from_payload(value) - - -def _read_darwin_cmdline(pid: int) -> list[str] | None: - return _process_identity.read_darwin_cmdline(pid, run_process=subprocess.run) - - -def _read_process_start_fingerprint(pid: int) -> str | None: - return _process_identity.read_process_start_fingerprint( - pid, - system=platform.system(), - run_process=subprocess.run, - ) - - -def _read_darwin_process_start_fingerprint(pid: int) -> str | None: - return _process_identity.read_darwin_process_start_fingerprint( - pid, - run_process=subprocess.run, - ) +from zeus.supervisor_contracts import ( + _ReadinessProbeUnset as _ReadinessProbeUnset, +) +from zeus.supervisor_contracts import ( + _ReconcileLaunch as _ReconcileLaunch, +) +from zeus.supervisor_contracts import ( + _SignalResult as _SignalResult, +) +from zeus.supervisor_runtime import _SupervisorCore as _SupervisorCore diff --git a/zeus/supervisor_reconcile.py b/zeus/supervisor_reconcile.py index 5fb71d8..4af7b84 100644 --- a/zeus/supervisor_reconcile.py +++ b/zeus/supervisor_reconcile.py @@ -36,33 +36,23 @@ ReconcileRunSummary, ReconcileSnapshotDriftError, ) -from zeus.supervisor_core import ( +from zeus.supervisor_contracts import ( _GatewayGeneration, _LifecycleContext, _MarkerObservation, _ReconcileLaunch, ) -from zeus.supervisor_stop import _SupervisorStop +from zeus.supervisor_reconcile_host import ReconcileHost -PidAliveFn = _process_identity.PidAliveFn -CmdlineReader = _process_identity.CmdlineReader -ProcStartFingerprintReader = _process_identity.ProcStartFingerprintReader - -_CommandCheck = _process_identity.CommandCheck _PidState = _process_identity.PidState -_looks_like_python_interpreter = _process_identity.looks_like_python_interpreter -_read_linux_cmdline = _process_identity.read_linux_cmdline -_read_linux_process_start_fingerprint = _process_identity.read_linux_process_start_fingerprint -_resolve_executable = _process_identity.resolve_executable -_resolve_launcher_exec_target = _process_identity.resolve_launcher_exec_target -_safe_command_shape = _process_identity.safe_command_shape -_trusted_hermes_paths = _process_identity.trusted_hermes_paths -_verify_gateway_command = _process_identity.verify_gateway_command -class _SupervisorReconcile(_SupervisorStop): +class ReconcileOperations: + """Stateless reconcile operations; callbacks are resolved from the current host.""" + + @staticmethod def reconcile( - self, + host: ReconcileHost, bot_id: str | None = None, *, now: datetime | None = None, @@ -73,7 +63,7 @@ def reconcile( bot_snapshot: Sequence[tuple[str, str]] | None = None, ) -> list[BotStatusResponse]: try: - execution = self.reconcile_execution( + execution = host.reconcile_execution( bot_id, now=now, force=force, @@ -86,8 +76,9 @@ def reconcile( raise LockTimeoutError(error.lock_path, error.timeout_seconds) from error return list(execution.legacy_responses) + @staticmethod def reconcile_summary( - self, + host: ReconcileHost, bot_id: str | None = None, *, now: datetime | None = None, @@ -97,7 +88,7 @@ def reconcile_summary( request_id: str | None = None, bot_snapshot: Sequence[tuple[str, str]] | None = None, ) -> ReconcileRunSummary: - return self.reconcile_execution( + return host.reconcile_execution( bot_id, now=now, force=force, @@ -107,8 +98,9 @@ def reconcile_summary( bot_snapshot=bot_snapshot, ).summary + @staticmethod def reconcile_execution( - self, + host: ReconcileHost, bot_id: str | None = None, *, now: datetime | None = None, @@ -118,7 +110,7 @@ def reconcile_execution( request_id: str | None = None, bot_snapshot: Sequence[tuple[str, str]] | None = None, ) -> ReconcileExecution: - return FleetReconciler(self.store, self).execute( + return FleetReconciler(host.store, host).execute( bot_id, now=now, force=force, @@ -128,25 +120,30 @@ def reconcile_execution( bot_snapshot=bot_snapshot, ) - def validate_reconcile_request(self, source: str, request_id: str | None) -> None: - self._lifecycle_context(source, request_id) + @staticmethod + def validate_reconcile_request( + host: ReconcileHost, source: str, request_id: str | None + ) -> None: + host._lifecycle_context(source, request_id) + @staticmethod def validate_reconcile_target( - self, + host: ReconcileHost, bot_id: str, *, expected_profile_path: str | None = None, ) -> str: - with self.bot_lock(bot_id), self._bot_process_lock(bot_id): - record = self.store.get_bot(bot_id) + with host.bot_lock(bot_id), host._bot_process_lock(bot_id): + record = host.store.get_bot(bot_id) if record is None: raise KeyError(f"unknown bot: {bot_id}") if expected_profile_path is not None and record.profile_path != expected_profile_path: raise ReconcileSnapshotDriftError(bot_id) return record.profile_path + @staticmethod def reconcile_one( - self, + host: ReconcileHost, bot_id: str, *, now: datetime | None = None, @@ -156,7 +153,7 @@ def reconcile_one( request_id: str | None = None, expected_profile_path: str | None = None, ) -> BotReconcileResult: - result, _response = self.reconcile_one_execution( + result, _response = host.reconcile_one_execution( bot_id, now=now, force=force, @@ -167,8 +164,9 @@ def reconcile_one( ) return result + @staticmethod def reconcile_one_execution( - self, + host: ReconcileHost, bot_id: str, *, now: datetime | None = None, @@ -178,21 +176,21 @@ def reconcile_one_execution( request_id: str | None = None, expected_profile_path: str | None = None, ) -> tuple[BotReconcileResult, BotStatusResponse]: - context = self._lifecycle_context(source, request_id) + context = host._lifecycle_context(source, request_id) current_time = now or datetime.now(UTC) started_at = datetime.now(UTC) - with self.bot_lock(bot_id), self._bot_process_lock(bot_id): - before = self.store.get_bot(bot_id) + with host.bot_lock(bot_id), host._bot_process_lock(bot_id): + before = host.store.get_bot(bot_id) if before is None: if expected_profile_path is not None: raise ReconcileSnapshotDriftError(bot_id) raise KeyError(f"unknown bot: {bot_id}") if expected_profile_path is not None and before.profile_path != expected_profile_path: raise ReconcileSnapshotDriftError(bot_id) - prior_events = self.store.list_lifecycle_events(bot_id, limit=1, before=None) + prior_events = host.store.list_lifecycle_events(bot_id, limit=1, before=None) prior_event_id = prior_events[0].event_id if prior_events else None try: - response = self._reconcile_record( + response = host._reconcile_record( before, current_time, force=force, @@ -202,11 +200,11 @@ def reconcile_one_execution( except ReconcileSnapshotDriftError: raise except Exception as error: - loaded_after_error = self.store.get_bot(bot_id) + loaded_after_error = host.store.get_bot(bot_id) if loaded_after_error is None and expected_profile_path is not None: raise ReconcileSnapshotDriftError(bot_id) from error after = loaded_after_error or before - current_event = self._latest_reconcile_event(bot_id, prior_event_id) + current_event = host._latest_reconcile_event(bot_id, prior_event_id) lock_timeout = isinstance(error, LockTimeoutError) message = ( "bot reconciliation lock timed out" @@ -233,14 +231,14 @@ def reconcile_one_execution( profile_path=before.profile_path, message=message, ) - loaded_after = self.store.get_bot(bot_id) + loaded_after = host.store.get_bot(bot_id) if loaded_after is None: if expected_profile_path is not None: raise ReconcileSnapshotDriftError(bot_id) raise KeyError(f"unknown bot: {bot_id}") - current_event = self._latest_reconcile_event(bot_id, prior_event_id) + current_event = host._latest_reconcile_event(bot_id, prior_event_id) return ( - self._reconcile_result_from_response( + host._reconcile_result_from_response( before, loaded_after, response, @@ -250,18 +248,20 @@ def reconcile_one_execution( response, ) + @staticmethod def _latest_reconcile_event( - self, + host: ReconcileHost, bot_id: str, prior_event_id: int | None, ) -> LifecycleEvent | None: - current_events = self.store.list_lifecycle_events(bot_id, limit=1, before=None) + current_events = host.store.list_lifecycle_events(bot_id, limit=1, before=None) if not current_events or current_events[0].event_id == prior_event_id: return None return current_events[0] + @staticmethod def _reconcile_result_from_response( - self, + host: ReconcileHost, before: BotRecord, after: BotRecord, response: BotStatusResponse, @@ -269,7 +269,7 @@ def _reconcile_result_from_response( current_event: LifecycleEvent | None, started_at: datetime, ) -> BotReconcileResult: - outcome = self._reconcile_outcome( + outcome = host._reconcile_outcome( before, after, response, @@ -347,8 +347,9 @@ def _reconcile_outcome( return ReconcileOutcome.changed return ReconcileOutcome.healthy + @staticmethod def _reconcile_record( - self, + host: ReconcileHost, record: BotRecord, now: datetime, *, @@ -357,9 +358,9 @@ def _reconcile_record( context: _LifecycleContext, ) -> BotStatusResponse: if record.pending_operation_id is not None: - return self._recover_pending_intent(record, context=context, allow_launch=True) + return host._recover_pending_intent(record, context=context, allow_launch=True) if reset_restart: - self._update_restart( + host._update_restart( context, record.bot_id, status=record.status, @@ -371,12 +372,12 @@ def _reconcile_record( ) record = replace(record, restart_attempts=0, next_restart_at=None) - pid_state = self._pid_state(record.pid) if record.pid else _PidState.dead + pid_state = host._pid_state(record.pid) if record.pid else _PidState.dead if record.pid and pid_state == _PidState.unknown: - return self._unknown_pid_response(record, "reconcile the gateway", context=context) + return host._unknown_pid_response(record, "reconcile the gateway", context=context) if record.pid and pid_state == _PidState.alive: - if not self._pid_owned(record.profile_path, record.pid, record.bot_id): - self._update_lifecycle( + if not host._pid_owned(record.profile_path, record.pid, record.bot_id): + host._update_lifecycle( context, record.bot_id, BotStatus.failed, @@ -391,7 +392,7 @@ def _reconcile_record( profile_path=record.profile_path, message="recorded gateway PID is alive but ownership could not be verified", ) - response = self._status_for_live_record(record, context=context) + response = host._status_for_live_record(record, context=context) return BotStatusResponse( bot_id=record.bot_id, status=response.status, @@ -401,18 +402,18 @@ def _reconcile_record( ) try: - with self._marker_publication_lock(record): - prepared = self._prepare_reconcile_dead_record_locked( + with host._marker_publication_lock(record): + prepared = host._prepare_reconcile_dead_record_locked( record, now, force=force, context=context, ) except (BotDeleteError, LaunchPayloadError) as exc: - return self._pending_action_required(record, str(exc)) + return host._pending_action_required(record, str(exc)) if isinstance(prepared, BotStatusResponse): return prepared - result = self._start_record( + result = host._start_record( prepared.record, reset_restart=False, message=( @@ -423,7 +424,7 @@ def _reconcile_record( probe=prepared.probe, ) if result.status == BotStatus.running: - self.store.append_audit_event( + host.store.append_audit_event( "bot.reconcile.restart_started", bot_id=record.bot_id, pid=result.pid, @@ -431,32 +432,33 @@ def _reconcile_record( ) return result + @staticmethod def _prepare_reconcile_dead_record_locked( - self, + host: ReconcileHost, record: BotRecord, now: datetime, *, force: bool, context: _LifecycleContext, ) -> BotStatusResponse | _ReconcileLaunch: - marker = self._classify_existing_runtime_marker(record, expected_pid=record.pid) + marker = host._classify_existing_runtime_marker(record, expected_pid=record.pid) if marker.kind == "dead": - generation = self._gateway_generation(marker) - if generation is None or not self._remove_gateway_generation_marker_locked( + generation = host._gateway_generation(marker) + if generation is None or not host._remove_gateway_generation_marker_locked( record, generation ): - return self._pending_action_required( + return host._pending_action_required( record, "dead gateway marker cleanup could not be verified" ) elif marker.kind != "missing": - return self._pending_action_required( + return host._pending_action_required( record, marker.reason or "recorded gateway marker ownership is unresolved", ) if record.desired_state is DesiredState.stopped: if record.status is not BotStatus.stopped or record.pid is not None: - self._update_lifecycle( + host._update_lifecycle( context, record.bot_id, BotStatus.stopped, @@ -473,7 +475,7 @@ def _prepare_reconcile_dead_record_locked( ) if record.restart_policy != RestartPolicy.on_failure: - self._update_lifecycle( + host._update_lifecycle( context, record.bot_id, BotStatus.failed, @@ -495,7 +497,7 @@ def _prepare_reconcile_dead_record_locked( # Scheduling already counted the pending attempt; it must still run. completed_attempts = max(0, completed_attempts - 1) if completed_attempts >= record.restart_max_attempts: - self._update_restart( + host._update_restart( context, record.bot_id, status=BotStatus.failed, @@ -519,10 +521,10 @@ def _prepare_reconcile_dead_record_locked( ) if record.next_restart_at is None and not force: - delay = self._restart_delay(record) + delay = host._restart_delay(record) next_restart_at = now + timedelta(seconds=delay) attempt = record.restart_attempts + 1 - self._update_restart( + host._update_restart( context, record.bot_id, status=BotStatus.failed, @@ -532,7 +534,7 @@ def _prepare_reconcile_dead_record_locked( action="bot.restart.schedule", reason="restart scheduled by reconcile", ) - self.store.append_audit_event( + host.store.append_audit_event( "bot.reconcile.restart_scheduled", bot_id=record.bot_id, attempt=attempt, @@ -565,7 +567,7 @@ def _prepare_reconcile_dead_record_locked( attempt = record.restart_attempts if record.next_restart_at is None or attempt == 0: attempt += 1 - self._update_restart( + host._update_restart( context, record.bot_id, status=BotStatus.failed, @@ -575,9 +577,9 @@ def _prepare_reconcile_dead_record_locked( action="bot.restart.attempt", reason="restart attempt started by reconcile", ) - refreshed = self._require_bot(record.bot_id) - probe = self._preflight_start(refreshed, timeout_seconds=None) - refreshed = self.store.begin_lifecycle_intent( + refreshed = host._require_bot(record.bot_id) + probe = host._preflight_start(refreshed, timeout_seconds=None) + refreshed = host.store.begin_lifecycle_intent( record.bot_id, action="start", operation_id=context.operation_id, @@ -596,15 +598,16 @@ def _prepare_reconcile_dead_record_locked( def _is_compat_runtime_marker(payload: dict[str, object]) -> bool: return is_compat_runtime_marker(payload) + @staticmethod def _recover_pending_intent( - self, + host: ReconcileHost, record: BotRecord, *, context: _LifecycleContext, allow_launch: bool, ) -> BotStatusResponse: - return self._intent_recovery.recover( - self, + return host._intent_recovery.recover( + host, record, context=context, allow_launch=allow_launch, @@ -617,13 +620,14 @@ def _recovery_lifecycle_context( ) -> _LifecycleContext: return _LifecycleContext(operation_id, context.source, context.request_id) + @staticmethod def _pending_launch_preflight( - self, + host: ReconcileHost, record: BotRecord, operation_id: str, ) -> tuple[ReadinessProbe | None, str]: - probe = self._preflight_start(record, timeout_seconds=None) - expected = self.adapter.launcher_payload( + probe = host._preflight_start(record, timeout_seconds=None) + expected = host.adapter.launcher_payload( record.bot_id, operation_id=operation_id, desired_revision=record.desired_revision, @@ -634,25 +638,27 @@ def _pending_launch_preflight( raise ValueError("invalid expected marker") return probe, str(marker_template["command_fingerprint"]) + @staticmethod def _recover_pending_stop_intent( - self, + host: ReconcileHost, record: BotRecord, *, context: _LifecycleContext, allow_stop: bool, ) -> BotStatusResponse: try: - with self._marker_publication_lock(record): - return self._recover_pending_stop_intent_locked( + with host._marker_publication_lock(record): + return host._recover_pending_stop_intent_locked( record, context=context, allow_stop=allow_stop, ) except (BotDeleteError, LaunchPayloadError) as exc: - return self._pending_action_required(record, str(exc)) + return host._pending_action_required(record, str(exc)) + @staticmethod def _recover_pending_launch( - self, + host: ReconcileHost, record: BotRecord, *, context: _LifecycleContext, @@ -662,9 +668,9 @@ def _recover_pending_launch( allow_launch: bool, ) -> BotStatusResponse | None: try: - with self._marker_publication_lock(record): - return self._intent_recovery.recover_pending_launch_locked( - self, + with host._marker_publication_lock(record): + return host._intent_recovery.recover_pending_launch_locked( + host, record, context=context, probe=probe, @@ -673,97 +679,105 @@ def _recover_pending_launch( allow_launch=allow_launch, ) except (BotDeleteError, LaunchPayloadError) as exc: - return self._pending_action_required(record, str(exc)) + return host._pending_action_required(record, str(exc)) + @staticmethod def _recover_pending_stop_intent_locked( - self, + host: ReconcileHost, record: BotRecord, *, context: _LifecycleContext, allow_stop: bool, ) -> BotStatusResponse: - return self._intent_recovery.recover_pending_stop_intent_locked( - self, + return host._intent_recovery.recover_pending_stop_intent_locked( + host, record, context=context, allow_stop=allow_stop, ) + @staticmethod def _pending_restart_old_marker( - self, + host: ReconcileHost, record: BotRecord, observed: _MarkerObservation | None = None, ) -> _MarkerObservation | None: - return self._intent_recovery.pending_restart_old_marker( - self, + return host._intent_recovery.pending_restart_old_marker( + host, record, observed, ) + @staticmethod def _recover_pending_restart_predecessor( - self, + host: ReconcileHost, record: BotRecord, *, context: _LifecycleContext, allow_stop: bool, ) -> BotStatusResponse | None: try: - with self._marker_publication_lock(record): - return self._intent_recovery.recover_pending_restart_predecessor_locked( - self, + with host._marker_publication_lock(record): + return host._intent_recovery.recover_pending_restart_predecessor_locked( + host, record, context=context, allow_stop=allow_stop, ) except (BotDeleteError, LaunchPayloadError) as exc: - return self._pending_action_required(record, str(exc)) + return host._pending_action_required(record, str(exc)) + @staticmethod def _recover_pending_restart_old_gateway( - self, + host: ReconcileHost, record: BotRecord, marker: _MarkerObservation, *, context: _LifecycleContext, allow_stop: bool, ) -> BotStatusResponse: - return self._intent_recovery.recover_pending_restart_old_gateway( - self, + return host._intent_recovery.recover_pending_restart_old_gateway( + host, record, marker, context=context, allow_stop=allow_stop, ) + @staticmethod def _stop_pending_restart_old_gateway( - self, + host: ReconcileHost, record: BotRecord, generation: _GatewayGeneration, *, context: _LifecycleContext, ) -> BotStatusResponse: - return self._intent_recovery.stop_pending_restart_old_gateway( - self, + return host._intent_recovery.stop_pending_restart_old_gateway( + host, record, generation, context=context, ) + @staticmethod def _stop_gateway_generation_locked( - self, + host: ReconcileHost, record: BotRecord, generation: _GatewayGeneration, ) -> StopEffect: - return self._runtime.stop_generation_locked( + return host._runtime.stop_generation_locked( record, generation, kill_after_timeout=None, - classify_exact=self._classify_exact_gateway_generation, - remove_generation=self._remove_gateway_generation_marker_locked, + classify_exact=host._classify_exact_gateway_generation, + remove_generation=host._remove_gateway_generation_marker_locked, ) - def _append_recovery_audit_event(self, action: str, **values: object) -> None: - self.store.append_audit_event(action, **values) + @staticmethod + def _append_recovery_audit_event(host: ReconcileHost, action: str, **values: object) -> None: + host.store.append_audit_event(action, **values) - def _restart_delay(self, record: BotRecord) -> float: + @staticmethod + def _restart_delay(host: ReconcileHost, record: BotRecord) -> float: delay = record.restart_backoff_seconds * (2**record.restart_attempts) - return float(min(delay, self.restart_backoff_cap_seconds)) + return float(min(delay, host.restart_backoff_cap_seconds)) diff --git a/zeus/supervisor_reconcile_host.py b/zeus/supervisor_reconcile_host.py new file mode 100644 index 0000000..45c0da1 --- /dev/null +++ b/zeus/supervisor_reconcile_host.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import contextlib +import threading +from collections.abc import Sequence +from datetime import datetime +from typing import Protocol + +from zeus import process_identity as _process_identity +from zeus.gateway_runtime import ( + GatewayRuntime, +) +from zeus.hermes_adapter import HermesAdapter +from zeus.intent_recovery import PendingIntentRecovery, _RecoveryHost +from zeus.lifecycle import LifecycleEvent +from zeus.models import ( + BotRecord, + BotStatus, + BotStatusResponse, +) +from zeus.process_lock import BotProcessLock +from zeus.readiness import ReadinessProbe +from zeus.reconciliation import ( + BotReconcileResult, + ReconcileExecution, + ReconcileOutcome, + _ReconciliationSupervisor, +) +from zeus.state import StateStore +from zeus.supervisor_contracts import ( + _READINESS_PROBE_UNSET, + _GatewayGeneration, + _LifecycleContext, + _MarkerObservation, + _ReadinessProbeUnset, + _ReconcileLaunch, +) + +_PidState = _process_identity.PidState + + +class ReconcileHost(_RecoveryHost[_LifecycleContext], _ReconciliationSupervisor, Protocol): + """Only the host capabilities required by reconcile operations.""" + + def _bot_process_lock(self, bot_id: str) -> BotProcessLock: ... + + def _classify_exact_gateway_generation( + self, record: BotRecord, generation: _GatewayGeneration + ) -> _MarkerObservation: ... + + def _classify_existing_runtime_marker( + self, record: BotRecord, *, expected_pid: int | None = None + ) -> _MarkerObservation: ... + + def _gateway_generation(self, marker: _MarkerObservation) -> _GatewayGeneration | None: ... + + _intent_recovery: PendingIntentRecovery + + def _latest_reconcile_event( + self, bot_id: str, prior_event_id: int | None + ) -> LifecycleEvent | None: ... + + def _lifecycle_context(self, source: str, request_id: str | None) -> _LifecycleContext: ... + + def _marker_publication_lock( + self, record: BotRecord + ) -> contextlib.AbstractContextManager[object]: ... + + def _pending_action_required(self, record: BotRecord, reason: str) -> BotStatusResponse: ... + + def _pid_owned(self, profile_path: str, pid: int, bot_id: str) -> bool: ... + + def _pid_state(self, pid: int) -> _PidState: ... + + def _preflight_start( + self, record: BotRecord, *, timeout_seconds: float | None + ) -> ReadinessProbe | None: ... + + def _prepare_reconcile_dead_record_locked( + self, record: BotRecord, now: datetime, *, force: bool, context: _LifecycleContext + ) -> BotStatusResponse | _ReconcileLaunch: ... + + @staticmethod + def _reconcile_outcome( + before: BotRecord, + after: BotRecord, + response: BotStatusResponse, + *, + current_event_action: str | None, + ) -> ReconcileOutcome: ... + + def _reconcile_record( + self, + record: BotRecord, + now: datetime, + *, + force: bool, + reset_restart: bool, + context: _LifecycleContext, + ) -> BotStatusResponse: ... + + def _reconcile_result_from_response( + self, + before: BotRecord, + after: BotRecord, + response: BotStatusResponse, + *, + current_event: LifecycleEvent | None, + started_at: datetime, + ) -> BotReconcileResult: ... + + def _recover_pending_intent( + self, record: BotRecord, *, context: _LifecycleContext, allow_launch: bool + ) -> BotStatusResponse: ... + + def _recover_pending_stop_intent_locked( + self, record: BotRecord, *, context: _LifecycleContext, allow_stop: bool + ) -> BotStatusResponse: ... + + def _remove_gateway_generation_marker_locked( + self, record: BotRecord, generation: _GatewayGeneration + ) -> bool: ... + + def _require_bot(self, bot_id: str) -> BotRecord: ... + + def _restart_delay(self, record: BotRecord) -> float: ... + + _runtime: GatewayRuntime + + def _start_record( + self, + record: BotRecord, + *, + reset_restart: bool, + message: str, + wait: bool = False, + timeout_seconds: float | None = None, + context: _LifecycleContext, + probe: ReadinessProbe | _ReadinessProbeUnset | None = _READINESS_PROBE_UNSET, + ) -> BotStatusResponse: ... + + def _status_for_live_record( + self, record: BotRecord, *, context: _LifecycleContext + ) -> BotStatusResponse: ... + + def _unknown_pid_response( + self, record: BotRecord, operation: str, *, context: _LifecycleContext + ) -> BotStatusResponse: ... + + def _update_lifecycle( + self, + context: _LifecycleContext, + bot_id: str, + status: BotStatus, + pid: int | None = None, + *, + action: str | None = None, + started_at: datetime | None = None, + ready_at: datetime | None = None, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + last_error: str | None = None, + last_transition_reason: str | None = None, + reset_restart: bool = False, + clear_ready_at: bool = False, + clear_stopped_at: bool = False, + details: dict[str, object] | None = None, + ) -> None: ... + + def _update_restart( + self, + context: _LifecycleContext, + bot_id: str, + *, + status: BotStatus, + pid: int | None, + restart_attempts: int, + next_restart_at: datetime | None, + action: str, + reason: str, + outcome: str = "success", + error_code: str | None = None, + ) -> None: ... + + adapter: HermesAdapter + + def bot_lock(self, bot_id: str) -> threading.RLock: ... + + def reconcile_execution( + self, + bot_id: str | None = None, + *, + now: datetime | None = None, + force: bool = False, + reset_restart: bool = False, + source: str = "reconcile", + request_id: str | None = None, + bot_snapshot: Sequence[tuple[str, str]] | None = None, + ) -> ReconcileExecution: ... + + def reconcile_one_execution( + self, + bot_id: str, + *, + now: datetime | None = None, + force: bool = False, + reset_restart: bool = False, + source: str = "reconcile", + request_id: str | None = None, + expected_profile_path: str | None = None, + ) -> tuple[BotReconcileResult, BotStatusResponse]: ... + + restart_backoff_cap_seconds: float + + store: StateStore diff --git a/zeus/supervisor_registry.py b/zeus/supervisor_registry.py index a3e38ad..7c91516 100644 --- a/zeus/supervisor_registry.py +++ b/zeus/supervisor_registry.py @@ -22,30 +22,20 @@ validate_id, ) from zeus.profile_manager import ProfileArchive, ProfileDeletion -from zeus.supervisor_core import ( +from zeus.supervisor_contracts import ( _LifecycleContext, ) -from zeus.supervisor_status import _SupervisorStatus +from zeus.supervisor_registry_host import RegistryHost -PidAliveFn = _process_identity.PidAliveFn -CmdlineReader = _process_identity.CmdlineReader -ProcStartFingerprintReader = _process_identity.ProcStartFingerprintReader - -_CommandCheck = _process_identity.CommandCheck _PidState = _process_identity.PidState -_looks_like_python_interpreter = _process_identity.looks_like_python_interpreter -_read_linux_cmdline = _process_identity.read_linux_cmdline -_read_linux_process_start_fingerprint = _process_identity.read_linux_process_start_fingerprint -_resolve_executable = _process_identity.resolve_executable -_resolve_launcher_exec_target = _process_identity.resolve_launcher_exec_target -_safe_command_shape = _process_identity.safe_command_shape -_trusted_hermes_paths = _process_identity.trusted_hermes_paths -_verify_gateway_command = _process_identity.verify_gateway_command -class _SupervisorRegistry(_SupervisorStatus): +class RegistryOperations: + """Stateless registry operations; callbacks are resolved from the current host.""" + + @staticmethod def create_bot( - self, + host: RegistryHost, request: BotCreateRequest, template: HermesTemplate, *, @@ -54,18 +44,18 @@ def create_bot( source: str = "cli", request_id: str | None = None, ) -> BotRecord: - context = self._lifecycle_context(source, request_id) + context = host._lifecycle_context(source, request_id) bot_id = validate_id(request.bot_id, "bot_id") with ( - self.store.defer_lifecycle_audit(), - self.bot_lock(bot_id), - self._bot_process_lock(bot_id), + host.store.defer_lifecycle_audit(), + host.bot_lock(bot_id), + host._bot_process_lock(bot_id), ): - existing = self.store.get_bot(bot_id) - profile_path = Path(self.adapter.hermes_root) / "profiles" / bot_id + existing = host.store.get_bot(bot_id) + profile_path = Path(host.adapter.hermes_root) / "profiles" / bot_id profile_exists = os.path.lexists(profile_path) if existing is not None: - active = self._record_may_be_active(existing) + active = host._record_may_be_active(existing) if active and (not replace_existing or not stop_if_running): raise BotRunningError( "bot is running or starting; use --replace --stop to replace it" @@ -73,7 +63,7 @@ def create_bot( if not active and not replace_existing: raise BotExistsError("bot already exists; use --replace to replace it") try: - self._safe_profile_path(bot_id, existing.profile_path) + host._safe_profile_path(bot_id, existing.profile_path) except BotDeleteError as exc: raise BotReplaceError(str(exc)) from exc elif profile_exists: @@ -83,24 +73,24 @@ def create_bot( raise BotExistsError( "bot profile path is not a safe directory; resolve it manually" ) - self._assert_unregistered_profile_inactive(bot_id, profile_path) + host._assert_unregistered_profile_inactive(bot_id, profile_path) - self._profile_manager.preflight(request, template) + host._profile_manager.preflight(request, template) stopped_record: BotRecord | None = None if existing is not None: - active = self._record_may_be_active(existing) + active = host._record_may_be_active(existing) if active: - stopped = self._stop_locked(bot_id, context=context) + stopped = host._stop_locked(bot_id, context=context) if stopped.status != BotStatus.stopped: raise BotReplaceError(f"could not stop existing bot: {stopped.message}") stopped_record = existing try: - with self._profile_manager.install_transaction(request, template) as record: - self._remove_pid_marker(record.profile_path) - self.store.upsert_bot_with_event( + with host._profile_manager.install_transaction(request, template) as record: + host._remove_pid_marker(record.profile_path) + host.store.upsert_bot_with_event( record, - event=self._event( + event=host._event( context, record.bot_id, action="bot.replace" if existing else "bot.create", @@ -111,7 +101,7 @@ def create_bot( except BaseException: if stopped_record is not None: try: - self._recover_previously_active_bot( + host._recover_previously_active_bot( stopped_record, "replacement", context=context ) except Exception as recovery_error: @@ -119,15 +109,16 @@ def create_bot( "bot replacement failed and the previous bot could not be restarted" ) from recovery_error raise - self.store.append_audit_event( + host.store.append_audit_event( "bot.replace" if existing else "bot.create", bot_id=record.bot_id, template_id=record.template_id, ) return record + @staticmethod def delete_bot( - self, + host: RegistryHost, bot_id: str, *, stop_if_running: bool = False, @@ -135,34 +126,34 @@ def delete_bot( source: str = "cli", request_id: str | None = None, ) -> BotStatusResponse: - context = self._lifecycle_context(source, request_id) + context = host._lifecycle_context(source, request_id) safe_bot_id = validate_id(bot_id, "bot_id") with ( - self.store.defer_lifecycle_audit(), - self.bot_lock(safe_bot_id), - self._bot_process_lock(safe_bot_id), + host.store.defer_lifecycle_audit(), + host.bot_lock(safe_bot_id), + host._bot_process_lock(safe_bot_id), ): - record = self._require_bot(safe_bot_id) + record = host._require_bot(safe_bot_id) if remove_profile: - self._safe_profile_path(safe_bot_id, record.profile_path) - was_active = self._record_may_be_active(record) + host._safe_profile_path(safe_bot_id, record.profile_path) + was_active = host._record_may_be_active(record) if was_active: if not stop_if_running: raise BotRunningError("bot is running or starting; use --stop before delete") - stopped = self._stop_locked(safe_bot_id, context=context) + stopped = host._stop_locked(safe_bot_id, context=context) if stopped.status != BotStatus.stopped: raise BotDeleteError(f"could not stop bot before delete: {stopped.message}") profile_deletion: ProfileDeletion | None = None try: if remove_profile: - profile_deletion = self._profile_manager.stage_delete( + profile_deletion = host._profile_manager.stage_delete( safe_bot_id, record.profile_path ) else: - self._remove_pid_marker(record.profile_path) - deleted = self.store.delete_bot_with_event( + host._remove_pid_marker(record.profile_path) + deleted = host.store.delete_bot_with_event( safe_bot_id, - event=self._event( + event=host._event( context, safe_bot_id, action="bot.delete", @@ -175,12 +166,12 @@ def delete_bot( except BaseException as operation_error: if profile_deletion is not None: try: - self._profile_manager.rollback_delete(profile_deletion) + host._profile_manager.rollback_delete(profile_deletion) except BotDeleteError as rollback_error: raise rollback_error from operation_error if was_active: try: - self._recover_previously_active_bot(record, "deletion", context=context) + host._recover_previously_active_bot(record, "deletion", context=context) except Exception as recovery_error: raise BotDeleteError( "bot deletion failed and the previous bot could not be restarted" @@ -188,15 +179,15 @@ def delete_bot( raise cleanup_pending = False if profile_deletion is not None: - cleanup_error = self._profile_manager.finish_delete(profile_deletion) + cleanup_error = host._profile_manager.finish_delete(profile_deletion) if cleanup_error is not None: cleanup_pending = True - self.store.append_audit_event( + host.store.append_audit_event( "bot.delete_cleanup_pending", bot_id=safe_bot_id, error=type(cleanup_error).__name__, ) - self.store.append_audit_event( + host.store.append_audit_event( "bot.delete", bot_id=safe_bot_id, profile_removed=remove_profile, @@ -210,40 +201,41 @@ def delete_bot( message=("deleted; profile cleanup is pending" if cleanup_pending else "deleted"), ) + @staticmethod def archive_bot( - self, + host: RegistryHost, bot_id: str, *, stop_if_running: bool = False, source: str = "cli", request_id: str | None = None, ) -> dict[str, object]: - context = self._lifecycle_context(source, request_id) + context = host._lifecycle_context(source, request_id) safe_bot_id = validate_id(bot_id, "bot_id") with ( - self.store.defer_lifecycle_audit(), - self.bot_lock(safe_bot_id), - self._bot_process_lock(safe_bot_id), + host.store.defer_lifecycle_audit(), + host.bot_lock(safe_bot_id), + host._bot_process_lock(safe_bot_id), ): - record = self._require_bot(safe_bot_id) + record = host._require_bot(safe_bot_id) try: - profile_path = self._safe_profile_path(safe_bot_id, record.profile_path) + profile_path = host._safe_profile_path(safe_bot_id, record.profile_path) except BotDeleteError as exc: raise BotArchiveError(str(exc)) from exc - was_active = self._record_may_be_active(record) + was_active = host._record_may_be_active(record) if was_active: if not stop_if_running: raise BotRunningError("bot is running or starting; use --stop before archive") - stopped = self._stop_locked(safe_bot_id, context=context) + stopped = host._stop_locked(safe_bot_id, context=context) if stopped.status != BotStatus.stopped: raise BotArchiveError(f"could not stop bot before archive: {stopped.message}") profile_archive: ProfileArchive | None = None try: - profile_archive = self._profile_manager.stage_archive(safe_bot_id, profile_path) - deleted = self.store.delete_bot_with_event( + profile_archive = host._profile_manager.stage_archive(safe_bot_id, profile_path) + deleted = host.store.delete_bot_with_event( safe_bot_id, - event=self._event( + event=host._event( context, safe_bot_id, action="bot.archive", @@ -255,19 +247,19 @@ def archive_bot( except BaseException as operation_error: if profile_archive is not None: try: - self._profile_manager.rollback_archive(profile_archive) + host._profile_manager.rollback_archive(profile_archive) except BotArchiveError as rollback_error: raise rollback_error from operation_error if was_active: try: - self._recover_previously_active_bot(record, "archive", context=context) + host._recover_previously_active_bot(record, "archive", context=context) except Exception as recovery_error: raise BotArchiveError( "bot archive failed and the previous bot could not be restarted" ) from recovery_error raise archive_path = profile_archive.archive_path if profile_archive is not None else None - self.store.append_audit_event( + host.store.append_audit_event( "bot.archive", bot_id=safe_bot_id, archive_path=str(archive_path) if archive_path else None, @@ -281,15 +273,17 @@ def archive_bot( "message": "archived", } - def _record_may_be_active(self, record: BotRecord) -> bool: + @staticmethod + def _record_may_be_active(host: RegistryHost, record: BotRecord) -> bool: if record.pending_operation_id is not None: return True - if record.pid and self._pid_state(record.pid) != _PidState.dead: + if record.pid and host._pid_state(record.pid) != _PidState.dead: return True return record.status in {BotStatus.starting, BotStatus.running} + @staticmethod def _recover_previously_active_bot( - self, + host: RegistryHost, record: BotRecord, operation: str, *, @@ -306,24 +300,24 @@ def _recover_previously_active_bot( last_error=None, last_transition_reason=f"recovering after failed {operation}", ) - self.store.upsert_bot_with_event( + host.store.upsert_bot_with_event( recoverable, - event=self._event( + event=host._event( recovery_context, record.bot_id, action="bot.recovery.prepare", reason=f"recovering after failed {operation}", ), ) - probe = self._preflight_start(recoverable, timeout_seconds=None) - recoverable = self.store.begin_lifecycle_intent( + probe = host._preflight_start(recoverable, timeout_seconds=None) + recoverable = host.store.begin_lifecycle_intent( record.bot_id, action="start", operation_id=context.operation_id, source="recovery", reason=f"recovering after failed {operation}", ) - result = self._start_record( + result = host._start_record( recoverable, reset_restart=False, message=f"restored after failed {operation}", @@ -333,44 +327,49 @@ def _recover_previously_active_bot( if result.status not in {BotStatus.starting, BotStatus.running}: raise RuntimeError(f"previous bot restart failed after {operation}: {result.message}") + @staticmethod def _assert_unregistered_profile_inactive( - self, + host: RegistryHost, bot_id: str, profile_path: Path, ) -> None: - self._runtime.assert_unregistered_profile_inactive(bot_id, profile_path) + host._runtime.assert_unregistered_profile_inactive(bot_id, profile_path) - def _safe_profile_path(self, bot_id: str, profile_path: str) -> Path: - return self._profile_manager.validate_profile_path(bot_id, profile_path) + @staticmethod + def _safe_profile_path(host: RegistryHost, bot_id: str, profile_path: str) -> Path: + return host._profile_manager.validate_profile_path(bot_id, profile_path) - def _stage_profile_deletion(self, bot_id: str, profile_path: str) -> Path | None: - deletion = self._profile_manager.stage_delete(bot_id, profile_path) + @staticmethod + def _stage_profile_deletion(host: RegistryHost, bot_id: str, profile_path: str) -> Path | None: + deletion = host._profile_manager.stage_delete(bot_id, profile_path) if deletion is None: return None return deletion.tombstone_path + @staticmethod def _restore_tombstoned_profile( - self, + host: RegistryHost, bot_id: str, profile_path: str, tombstone: Path, ) -> None: - profile = self._profile_manager._pin_profile_path(bot_id, profile_path) - self._profile_manager.rollback_delete( + profile = host._profile_manager._pin_profile_path(bot_id, profile_path) + host._profile_manager.rollback_delete( ProfileDeletion( profile_path=profile, tombstone_path=tombstone, ) ) + @staticmethod def _restore_archived_profile( - self, + host: RegistryHost, bot_id: str, profile_path: str, archive_path: Path, ) -> None: - profile = self._profile_manager._pin_profile_path(bot_id, profile_path) - self._profile_manager.rollback_archive( + profile = host._profile_manager._pin_profile_path(bot_id, profile_path) + host._profile_manager.rollback_archive( ProfileArchive( profile_path=profile, archive_path=archive_path, diff --git a/zeus/supervisor_registry_host.py b/zeus/supervisor_registry_host.py new file mode 100644 index 0000000..6ac54a8 --- /dev/null +++ b/zeus/supervisor_registry_host.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import threading +from pathlib import Path +from typing import Protocol + +from zeus import process_identity as _process_identity +from zeus.gateway_runtime import ( + GatewayRuntime, +) +from zeus.hermes_adapter import HermesAdapter +from zeus.lifecycle import LifecycleEventInput +from zeus.models import ( + BotRecord, + BotStatusResponse, +) +from zeus.process_lock import BotProcessLock +from zeus.profile_manager import ProfileManager +from zeus.readiness import ReadinessProbe +from zeus.state import StateStore +from zeus.supervisor_contracts import ( + _READINESS_PROBE_UNSET, + _LifecycleContext, + _ReadinessProbeUnset, +) + +_PidState = _process_identity.PidState + + +class RegistryHost(Protocol): + """Only the host capabilities required by registry operations.""" + + def _assert_unregistered_profile_inactive(self, bot_id: str, profile_path: Path) -> None: ... + + def _bot_process_lock(self, bot_id: str) -> BotProcessLock: ... + + def _event( + self, + context: _LifecycleContext, + bot_id: str, + *, + action: str, + outcome: str = "success", + reason: str = "", + error_code: str | None = None, + error_message: str | None = None, + details: dict[str, object] | None = None, + ) -> LifecycleEventInput: ... + + def _lifecycle_context(self, source: str, request_id: str | None) -> _LifecycleContext: ... + + def _pid_state(self, pid: int) -> _PidState: ... + + def _preflight_start( + self, record: BotRecord, *, timeout_seconds: float | None + ) -> ReadinessProbe | None: ... + + _profile_manager: ProfileManager + + def _record_may_be_active(self, record: BotRecord) -> bool: ... + + def _recover_previously_active_bot( + self, record: BotRecord, operation: str, *, context: _LifecycleContext + ) -> None: ... + + def _remove_pid_marker(self, profile_path: str) -> None: ... + + def _require_bot(self, bot_id: str) -> BotRecord: ... + + _runtime: GatewayRuntime + + def _safe_profile_path(self, bot_id: str, profile_path: str) -> Path: ... + + def _start_record( + self, + record: BotRecord, + *, + reset_restart: bool, + message: str, + wait: bool = False, + timeout_seconds: float | None = None, + context: _LifecycleContext, + probe: ReadinessProbe | _ReadinessProbeUnset | None = _READINESS_PROBE_UNSET, + ) -> BotStatusResponse: ... + + def _stop_locked( + self, bot_id: str, *, kill_after_timeout: bool | None = None, context: _LifecycleContext + ) -> BotStatusResponse: ... + + adapter: HermesAdapter + + def bot_lock(self, bot_id: str) -> threading.RLock: ... + + store: StateStore diff --git a/zeus/supervisor_runtime.py b/zeus/supervisor_runtime.py index cc1ba28..f989c58 100644 --- a/zeus/supervisor_runtime.py +++ b/zeus/supervisor_runtime.py @@ -1,49 +1,467 @@ from __future__ import annotations +import contextlib +import os import platform +import re import signal +import subprocess # nosec B404 +import threading +import uuid from dataclasses import replace from datetime import UTC, datetime from pathlib import Path from zeus import process_identity as _process_identity +from zeus.gateway_launcher import ( + _read_bounded_file, + _remove_marker_if_owned_locked, +) +from zeus.gateway_marker import ( + readiness_probe_from_payload, + readiness_probe_to_payload, +) from zeus.gateway_runtime import ( + GatewayRuntime, + KillFn, OwnershipCheck, + PopenFactory, PopenLike, + RuntimeHooks, + gateway_process_launch_kwargs, ) +from zeus.hermes_adapter import HermesAdapter +from zeus.intent_recovery import PendingIntentRecovery +from zeus.lifecycle import LifecycleEventInput from zeus.models import ( BotRecord, BotStatus, BotStatusResponse, + validate_id, ) -from zeus.readiness import ReadinessProbe, ReadinessResult -from zeus.supervisor_core import ( +from zeus.private_io import nofollow_absolute_path +from zeus.process_lock import BotProcessLock +from zeus.profile_manager import ProfileManager +from zeus.readiness import ReadinessProbe, ReadinessResult, probe_once +from zeus.state import StateStore +from zeus.supervisor_contracts import ( _READINESS_PROBE_UNSET, _GatewayGeneration, _LifecycleContext, _MarkerObservation, _ReadinessProbeUnset, _SignalResult, - _SupervisorCore, ) PidAliveFn = _process_identity.PidAliveFn CmdlineReader = _process_identity.CmdlineReader ProcStartFingerprintReader = _process_identity.ProcStartFingerprintReader -_CommandCheck = _process_identity.CommandCheck _PidState = _process_identity.PidState -_looks_like_python_interpreter = _process_identity.looks_like_python_interpreter -_read_linux_cmdline = _process_identity.read_linux_cmdline -_read_linux_process_start_fingerprint = _process_identity.read_linux_process_start_fingerprint _resolve_executable = _process_identity.resolve_executable -_resolve_launcher_exec_target = _process_identity.resolve_launcher_exec_target -_safe_command_shape = _process_identity.safe_command_shape _trusted_hermes_paths = _process_identity.trusted_hermes_paths -_verify_gateway_command = _process_identity.verify_gateway_command -class _SupervisorRuntime(_SupervisorCore): +_REQUEST_ID_RE = re.compile(r"^[0-9a-f]{32}$") +_LIFECYCLE_SOURCES = frozenset({"api", "cli", "reconcile", "recovery", "system"}) + + +def _gateway_process_launch_kwargs() -> dict[str, object]: + return gateway_process_launch_kwargs() + + +def _nofollow_absolute_path(path: Path) -> Path: + return nofollow_absolute_path(path) + + +def _same_identity(first: os.stat_result, second: os.stat_result) -> bool: + return first.st_dev == second.st_dev and first.st_ino == second.st_ino + + +def _caused_by_missing_path(exc: BaseException) -> bool: + current: BaseException | None = exc + while current is not None: + if isinstance(current, FileNotFoundError): + return True + current = current.__cause__ + return False + + +class _SupervisorCore: + """Own shared state, locks, persistence coordination and live runtime hooks.""" + + @staticmethod + def _default_cmdline_reader(pid: int) -> list[str] | None: + return _read_process_cmdline(pid) + + @staticmethod + def _default_process_start_fingerprint_reader(pid: int) -> str | None: + return _read_process_start_fingerprint(pid) + + @staticmethod + def _probe_once(probe: ReadinessProbe) -> ReadinessResult: + return probe_once( + probe.url, + timeout_seconds=min(1.0, max(0.2, probe.interval_seconds)), + expected_status=probe.expected_status, + expected_platform=probe.expected_platform, + ) + + def __init__( + self, + store: StateStore, + hermes_bin: str, + hermes_root: Path | str, + popen_factory: PopenFactory = subprocess.Popen, + kill_fn: KillFn = os.kill, + pid_alive_fn: PidAliveFn | None = None, + cmdline_reader: CmdlineReader | None = None, + startup_grace_seconds: float = 0.25, + stop_grace_seconds: float = 60.0, + kill_after_timeout: bool = False, + lock_timeout_seconds: float = 30.0, + readiness_timeout_seconds: float = 30.0, + readiness_interval_seconds: float = 0.5, + allow_legacy_pid_markers: bool = True, + restart_backoff_cap_seconds: float = 3600.0, + proc_start_fingerprint_reader: ProcStartFingerprintReader | None = None, + restart_stability_seconds: float = 30.0, + ) -> None: + if not 0.0 <= restart_stability_seconds <= 86_400.0: + raise ValueError("restart_stability_seconds must be between 0 and 86400") + self.store = store + configured_hermes_root = _nofollow_absolute_path(Path(hermes_root)) + self.adapter = HermesAdapter( + hermes_bin=hermes_bin, + hermes_root=configured_hermes_root.resolve(), + ) + self._profile_manager = ProfileManager( + self.adapter.hermes_root, + self.store.database_path.parent / "archive", + ) + self._marker_profiles_root = configured_hermes_root / "profiles" + self.startup_grace_seconds = startup_grace_seconds + self.lock_dir = self.store.database_path.parent / "locks" / "bots" + self.readiness_timeout_seconds = readiness_timeout_seconds + self.readiness_interval_seconds = readiness_interval_seconds + self.allow_legacy_pid_markers = allow_legacy_pid_markers + self.restart_backoff_cap_seconds = restart_backoff_cap_seconds + self.restart_stability_seconds = restart_stability_seconds + self._cleanup_process_group = os.name == "posix" and popen_factory is subprocess.Popen + self._runtime = GatewayRuntime( + self.adapter, + self._profile_manager, + self._marker_profiles_root, + popen_factory=popen_factory, + kill_fn=kill_fn, + pid_alive_fn=pid_alive_fn, + cmdline_reader=cmdline_reader or self._default_cmdline_reader, + proc_start_fingerprint_reader=( + proc_start_fingerprint_reader or self._default_process_start_fingerprint_reader + ), + startup_grace_seconds=startup_grace_seconds, + stop_grace_seconds=stop_grace_seconds, + kill_after_timeout=kill_after_timeout, + lock_timeout_seconds=lock_timeout_seconds, + readiness_timeout_seconds=readiness_timeout_seconds, + readiness_interval_seconds=readiness_interval_seconds, + allow_legacy_pid_markers=allow_legacy_pid_markers, + cleanup_process_group=self._cleanup_process_group, + hooks_provider=self._runtime_hooks, + ) + self._intent_recovery = PendingIntentRecovery() + self._locks_guard = threading.Lock() + self._bot_locks: dict[str, threading.RLock] = {} + + def _runtime_hooks(self) -> RuntimeHooks: + return RuntimeHooks( + pipe=os.pipe, + close=os.close, + read_bounded_file=_read_bounded_file, + remove_marker_if_owned_locked=_remove_marker_if_owned_locked, + probe_once=probe_once, + ) + + def _get_runtime_proxy(self, name: str) -> object: + runtime = self.__dict__.get("_runtime") + if runtime is not None: + return getattr(runtime, name) + return self.__dict__.get(f"_runtime_proxy_{name}") + + def _set_runtime_proxy(self, name: str, value: object) -> None: + runtime = self.__dict__.get("_runtime") + history = self.__dict__.setdefault(f"_runtime_proxy_history_{name}", []) + if isinstance(history, list): + if len(history) >= 32: + del history[0] + history.append( + getattr(runtime, name) + if runtime is not None + else self.__dict__.get(f"_runtime_proxy_{name}") + ) + if runtime is not None: + setattr(runtime, name, value) + else: + self.__dict__[f"_runtime_proxy_{name}"] = value + + def _delete_runtime_proxy(self, name: str) -> None: + history = self.__dict__.get(f"_runtime_proxy_history_{name}") + if not isinstance(history, list) or not history: + self.__dict__.pop(f"_runtime_proxy_{name}", None) + return + previous = history.pop() + runtime = self.__dict__.get("_runtime") + if runtime is not None: + setattr(runtime, name, previous) + else: + self.__dict__[f"_runtime_proxy_{name}"] = previous + + @property + def popen_factory(self) -> PopenFactory: + return self._get_runtime_proxy("popen_factory") # type: ignore[return-value] + + @popen_factory.setter + def popen_factory(self, value: PopenFactory) -> None: + self._set_runtime_proxy("popen_factory", value) + + @popen_factory.deleter + def popen_factory(self) -> None: + self._delete_runtime_proxy("popen_factory") + + @property + def kill_fn(self) -> KillFn: + return self._get_runtime_proxy("kill_fn") # type: ignore[return-value] + + @kill_fn.setter + def kill_fn(self, value: KillFn) -> None: + self._set_runtime_proxy("kill_fn", value) + + @kill_fn.deleter + def kill_fn(self) -> None: + self._delete_runtime_proxy("kill_fn") + + @property + def pid_alive_fn(self) -> PidAliveFn | None: + return self._get_runtime_proxy("pid_alive_fn") # type: ignore[return-value] + + @pid_alive_fn.setter + def pid_alive_fn(self, value: PidAliveFn | None) -> None: + self._set_runtime_proxy("pid_alive_fn", value) + + @pid_alive_fn.deleter + def pid_alive_fn(self) -> None: + self._delete_runtime_proxy("pid_alive_fn") + + @property + def cmdline_reader(self) -> CmdlineReader: + return self._get_runtime_proxy("cmdline_reader") # type: ignore[return-value] + + @cmdline_reader.setter + def cmdline_reader(self, value: CmdlineReader) -> None: + self._set_runtime_proxy("cmdline_reader", value) + + @cmdline_reader.deleter + def cmdline_reader(self) -> None: + self._delete_runtime_proxy("cmdline_reader") + + @property + def proc_start_fingerprint_reader(self) -> ProcStartFingerprintReader: + return self._get_runtime_proxy("proc_start_fingerprint_reader") # type: ignore[return-value] + + @proc_start_fingerprint_reader.setter + def proc_start_fingerprint_reader(self, value: ProcStartFingerprintReader) -> None: + self._set_runtime_proxy("proc_start_fingerprint_reader", value) + + @proc_start_fingerprint_reader.deleter + def proc_start_fingerprint_reader(self) -> None: + self._delete_runtime_proxy("proc_start_fingerprint_reader") + + @property + def _processes(self) -> dict[str, PopenLike]: + return self._get_runtime_proxy("_processes") # type: ignore[return-value] + + @_processes.setter + def _processes(self, value: dict[str, PopenLike]) -> None: + self._set_runtime_proxy("_processes", value) + + @_processes.deleter + def _processes(self) -> None: + self._delete_runtime_proxy("_processes") + + @property + def stop_grace_seconds(self) -> float: + return self._get_runtime_proxy("stop_grace_seconds") # type: ignore[return-value] + + @stop_grace_seconds.setter + def stop_grace_seconds(self, value: float) -> None: + self._set_runtime_proxy("stop_grace_seconds", value) + + @stop_grace_seconds.deleter + def stop_grace_seconds(self) -> None: + self._delete_runtime_proxy("stop_grace_seconds") + + @property + def kill_after_timeout(self) -> bool: + return self._get_runtime_proxy("kill_after_timeout") # type: ignore[return-value] + + @kill_after_timeout.setter + def kill_after_timeout(self, value: bool) -> None: + self._set_runtime_proxy("kill_after_timeout", value) + + @kill_after_timeout.deleter + def kill_after_timeout(self) -> None: + self._delete_runtime_proxy("kill_after_timeout") + + @property + def lock_timeout_seconds(self) -> float: + return self._get_runtime_proxy("lock_timeout_seconds") # type: ignore[return-value] + + @lock_timeout_seconds.setter + def lock_timeout_seconds(self, value: float) -> None: + self._set_runtime_proxy("lock_timeout_seconds", value) + + @lock_timeout_seconds.deleter + def lock_timeout_seconds(self) -> None: + self._delete_runtime_proxy("lock_timeout_seconds") + + def _lifecycle_context(self, source: str, request_id: str | None) -> _LifecycleContext: + if source not in _LIFECYCLE_SOURCES: + raise ValueError("invalid lifecycle event source") + if source == "api": + if request_id is None or _REQUEST_ID_RE.fullmatch(request_id) is None: + raise ValueError("API lifecycle operations require a generated request ID") + elif request_id is not None: + raise ValueError("only API lifecycle operations may carry a request ID") + return _LifecycleContext(uuid.uuid4().hex, source, request_id) + + def _event( + self, + context: _LifecycleContext, + bot_id: str, + *, + action: str, + outcome: str = "success", + reason: str = "", + error_code: str | None = None, + error_message: str | None = None, + details: dict[str, object] | None = None, + ) -> LifecycleEventInput: + return LifecycleEventInput( + bot_id=bot_id, + operation_id=context.operation_id, + request_id=context.request_id, + source=context.source, + action=action, + outcome=outcome, + reason=reason, + error_code=error_code, + error_message=error_message, + details=details or {}, + ) + + def _update_lifecycle( + self, + context: _LifecycleContext, + bot_id: str, + status: BotStatus, + pid: int | None = None, + *, + action: str | None = None, + started_at: datetime | None = None, + ready_at: datetime | None = None, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + last_error: str | None = None, + last_transition_reason: str | None = None, + reset_restart: bool = False, + clear_ready_at: bool = False, + clear_stopped_at: bool = False, + details: dict[str, object] | None = None, + ) -> None: + reason = last_transition_reason or "" + failed = status in {BotStatus.failed, BotStatus.unknown} + self.store.update_lifecycle_with_event( + bot_id, + status, + pid, + event=self._event( + context, + bot_id, + action=action or f"bot.{status.value}", + outcome="failure" if failed else "success", + reason=reason, + error_code=f"bot_{status.value}" if failed else None, + error_message=last_error, + details=details, + ), + started_at=started_at, + ready_at=ready_at, + stopped_at=stopped_at, + last_exit_code=last_exit_code, + last_error=last_error, + last_transition_reason=last_transition_reason, + reset_restart=reset_restart, + clear_ready_at=clear_ready_at, + clear_stopped_at=clear_stopped_at, + ) + + def _update_restart( + self, + context: _LifecycleContext, + bot_id: str, + *, + status: BotStatus, + pid: int | None, + restart_attempts: int, + next_restart_at: datetime | None, + action: str, + reason: str, + outcome: str = "success", + error_code: str | None = None, + ) -> None: + self.store.update_restart_with_event( + bot_id, + status=status, + pid=pid, + restart_attempts=restart_attempts, + next_restart_at=next_restart_at, + event=self._event( + context, + bot_id, + action=action, + outcome=outcome, + reason=reason, + error_code=error_code, + details={ + "restart_attempts": restart_attempts, + "next_restart_at": ( + next_restart_at.isoformat() if next_restart_at is not None else None + ), + }, + ), + ) + + def bot_lock(self, bot_id: str) -> threading.RLock: + with self._locks_guard: + lock = self._bot_locks.get(bot_id) + if lock is None: + lock = threading.RLock() + self._bot_locks[bot_id] = lock + return lock + + def _bot_process_lock(self, bot_id: str) -> BotProcessLock: + safe_bot_id = validate_id(bot_id, "bot_id") + return BotProcessLock( + self.lock_dir / f"{safe_bot_id}.lock", + timeout_seconds=self.lock_timeout_seconds, + ) + + def _marker_publication_lock( + self, + record: BotRecord, + ) -> contextlib.AbstractContextManager[object]: + return self._runtime.marker_publication_lock(record) + def _read_strict_runtime_marker( self, bot_id: str, registered_profile_path: str ) -> _MarkerObservation: @@ -525,3 +943,42 @@ def _wait_for_exit(self, bot_id: str, pid: int) -> bool: def _poll_startup(self, process: PopenLike) -> int | None: return self._runtime.poll_startup(process) + + +def _read_process_cmdline(pid: int) -> list[str] | None: + return _process_identity.read_process_cmdline( + pid, + system=platform.system(), + run_process=subprocess.run, + ) + + +def _readiness_probe_marker_payload(probe: ReadinessProbe | None) -> dict[str, object] | None: + return readiness_probe_to_payload(probe) + + +def _readiness_probe_from_marker(value: object) -> ReadinessProbe | None: + return readiness_probe_from_payload(value) + + +def _read_darwin_cmdline(pid: int) -> list[str] | None: + return _process_identity.read_darwin_cmdline(pid, run_process=subprocess.run) + + +def _read_process_start_fingerprint(pid: int) -> str | None: + return _process_identity.read_process_start_fingerprint( + pid, + system=platform.system(), + run_process=subprocess.run, + ) + + +def _read_darwin_process_start_fingerprint(pid: int) -> str | None: + return _process_identity.read_darwin_process_start_fingerprint( + pid, + run_process=subprocess.run, + ) + + +# Historical internal import; this is the same concrete class, not another layer. +_SupervisorRuntime = _SupervisorCore diff --git a/zeus/supervisor_start.py b/zeus/supervisor_start.py index cbb6283..3c338c1 100644 --- a/zeus/supervisor_start.py +++ b/zeus/supervisor_start.py @@ -14,33 +14,23 @@ StoredProfilePreflightError, ) from zeus.readiness import ReadinessProbe -from zeus.supervisor_core import ( +from zeus.supervisor_contracts import ( _READINESS_PROBE_UNSET, _GatewayGeneration, _LifecycleContext, _ReadinessProbeUnset, ) -from zeus.supervisor_runtime import _SupervisorRuntime +from zeus.supervisor_start_host import StartHost -PidAliveFn = _process_identity.PidAliveFn -CmdlineReader = _process_identity.CmdlineReader -ProcStartFingerprintReader = _process_identity.ProcStartFingerprintReader - -_CommandCheck = _process_identity.CommandCheck _PidState = _process_identity.PidState -_looks_like_python_interpreter = _process_identity.looks_like_python_interpreter -_read_linux_cmdline = _process_identity.read_linux_cmdline -_read_linux_process_start_fingerprint = _process_identity.read_linux_process_start_fingerprint -_resolve_executable = _process_identity.resolve_executable -_resolve_launcher_exec_target = _process_identity.resolve_launcher_exec_target -_safe_command_shape = _process_identity.safe_command_shape -_trusted_hermes_paths = _process_identity.trusted_hermes_paths -_verify_gateway_command = _process_identity.verify_gateway_command -class _SupervisorStart(_SupervisorRuntime): +class StartOperations: + """Stateless start operations; callbacks are resolved from the current host.""" + + @staticmethod def start( - self, + host: StartHost, bot_id: str, *, wait: bool = False, @@ -48,32 +38,33 @@ def start( source: str = "cli", request_id: str | None = None, ) -> BotStatusResponse: - context = self._lifecycle_context(source, request_id) - with self.bot_lock(bot_id), self._bot_process_lock(bot_id): - return self._start_locked( + context = host._lifecycle_context(source, request_id) + with host.bot_lock(bot_id), host._bot_process_lock(bot_id): + return host._start_locked( bot_id, wait=wait, timeout_seconds=timeout_seconds, context=context, ) + @staticmethod def _start_locked( - self, + host: StartHost, bot_id: str, *, wait: bool = False, timeout_seconds: float | None = None, context: _LifecycleContext, ) -> BotStatusResponse: - record = self._require_bot(bot_id) + record = host._require_bot(bot_id) if record.pending_operation_id is not None: - return self._pending_action_required(record, "lifecycle intent is already pending") - pid_state = self._pid_state(record.pid) if record.pid else _PidState.dead + return host._pending_action_required(record, "lifecycle intent is already pending") + pid_state = host._pid_state(record.pid) if record.pid else _PidState.dead if record.pid and pid_state == _PidState.unknown: - return self._unknown_pid_response(record, "start another gateway", context=context) + return host._unknown_pid_response(record, "start another gateway", context=context) if record.pid and pid_state == _PidState.alive: - if not self._pid_owned(record.profile_path, record.pid, bot_id): - self._update_lifecycle( + if not host._pid_owned(record.profile_path, record.pid, bot_id): + host._update_lifecycle( context, bot_id, BotStatus.failed, @@ -88,7 +79,7 @@ def _start_locked( profile_path=record.profile_path, message="recorded gateway PID is alive but ownership could not be verified", ) - response = self._status_for_live_record(record, context=context) + response = host._status_for_live_record(record, context=context) return BotStatusResponse( bot_id=bot_id, status=response.status, @@ -98,22 +89,22 @@ def _start_locked( "already running" if response.status == BotStatus.running else response.message ), ) - marker = self._classify_existing_runtime_marker(record) + marker = host._classify_existing_runtime_marker(record) if marker.kind == "dead": - if not self._remove_exact_schema3_marker(record, marker): - return self._pending_action_required( + if not host._remove_exact_schema3_marker(record, marker): + return host._pending_action_required( record, "stale gateway marker cleanup could not be verified" ) elif marker.kind != "missing": - return self._pending_action_required( + return host._pending_action_required( record, marker.reason or "existing gateway marker ownership is unresolved", ) try: - probe = self._preflight_start(record, timeout_seconds=timeout_seconds) + probe = host._preflight_start(record, timeout_seconds=timeout_seconds) except (OSError, StoredProfilePreflightError) as exc: message = f"failed to start gateway: {exc}" - self._update_lifecycle( + host._update_lifecycle( context, bot_id, BotStatus.failed, @@ -122,7 +113,7 @@ def _start_locked( last_error=message, last_transition_reason="gateway launch preflight failed", ) - self.store.append_audit_event( + host.store.append_audit_event( "bot.start_failed", bot_id=bot_id, error=type(exc).__name__, @@ -135,7 +126,7 @@ def _start_locked( record.profile_path, message, ) - record = self.store.begin_lifecycle_intent( + record = host.store.begin_lifecycle_intent( bot_id, action="start", operation_id=context.operation_id, @@ -143,7 +134,7 @@ def _start_locked( request_id=context.request_id, reason="gateway start requested", ) - return self._start_record( + return host._start_record( record, reset_restart=True, message="started", @@ -153,8 +144,9 @@ def _start_locked( probe=probe, ) + @staticmethod def _start_record( - self, + host: StartHost, record: BotRecord, *, reset_restart: bool, @@ -171,7 +163,7 @@ def _start_record( raise RuntimeError("gateway launch requires a pending start or restart intent") if isinstance(probe, _ReadinessProbeUnset): try: - probe = self._preflight_start(record, timeout_seconds=timeout_seconds) + probe = host._preflight_start(record, timeout_seconds=timeout_seconds) except (OSError, StoredProfilePreflightError) as exc: return BotStatusResponse( bot_id, @@ -180,19 +172,19 @@ def _start_record( record.profile_path, f"restart aborted: launch preflight failed: {exc}", ) - effect = self._runtime.launch( + effect = host._runtime.launch( record, probe=probe, wait=wait, - marker_lock=self._marker_publication_lock, - marker_matcher=self._matching_runtime_marker, - ack_reader=self._read_launcher_ack, - pipe_writer=self._write_pipe_payload, + marker_lock=host._marker_publication_lock, + marker_matcher=host._matching_runtime_marker, + ack_reader=host._read_launcher_ack, + pipe_writer=host._write_pipe_payload, ) if effect.outcome == "launch_failed": failure_message = f"failed to start gateway: {effect.reason}" try: - self._complete_failed_intent( + host._complete_failed_intent( record, context=context, pid=None, @@ -200,10 +192,10 @@ def _start_record( reason="gateway process launch failed", ) except Exception: - return self._pending_action_required( + return host._pending_action_required( record, "launch failure could not be persisted" ) - self.store.append_audit_event( + host.store.append_audit_event( "bot.start_failed", bot_id=bot_id, error=effect.error_type, @@ -246,7 +238,7 @@ def _start_record( ) terminal = replace(record, pid=pid) try: - self._complete_failed_intent( + host._complete_failed_intent( terminal, context=context, pid=None, @@ -256,8 +248,8 @@ def _start_record( reason="gateway exited during startup grace period", ) except Exception: - return self._launch_completion_failure_response(record, generation) - self.store.append_audit_event( + return host._launch_completion_failure_response(record, generation) + host.store.append_audit_event( "bot.start_failed", bot_id=bot_id, pid=pid, @@ -272,7 +264,7 @@ def _start_record( ) if effect.outcome == "readiness_exited": try: - self._complete_failed_intent( + host._complete_failed_intent( record, context=context, pid=None, @@ -282,8 +274,8 @@ def _start_record( reason="readiness process exited", ) except Exception: - return self._launch_completion_failure_response(record, generation) - self.store.append_audit_event( + return host._launch_completion_failure_response(record, generation) + host.store.append_audit_event( "bot.start_failed", bot_id=bot_id, pid=pid, @@ -299,7 +291,7 @@ def _start_record( ) if effect.outcome == "ready": try: - self._complete_started_intent( + host._complete_started_intent( record, context=context, status=BotStatus.running, @@ -309,8 +301,8 @@ def _start_record( reason="gateway readiness probe passed", ) except Exception: - return self._launch_completion_failure_response(record, generation) - self.store.append_audit_event("bot.start", bot_id=bot_id, pid=pid) + return host._launch_completion_failure_response(record, generation) + host.store.append_audit_event("bot.start", bot_id=bot_id, pid=pid) return BotStatusResponse( bot_id=bot_id, status=BotStatus.running, @@ -320,7 +312,7 @@ def _start_record( ) if effect.outcome == "readiness_timeout": try: - self._complete_started_intent( + host._complete_started_intent( record, context=context, status=BotStatus.starting, @@ -330,8 +322,8 @@ def _start_record( reason="readiness probe timed out", ) except Exception: - return self._launch_completion_failure_response(record, generation) - self.store.append_audit_event( + return host._launch_completion_failure_response(record, generation) + host.store.append_audit_event( "bot.start_readiness_pending", bot_id=bot_id, pid=pid, @@ -347,7 +339,7 @@ def _start_record( ) if effect.outcome == "readiness_pending": try: - self._complete_started_intent( + host._complete_started_intent( record, context=context, status=BotStatus.starting, @@ -356,8 +348,8 @@ def _start_record( reason="gateway process started; readiness probe pending", ) except Exception: - return self._launch_completion_failure_response(record, generation) - self.store.append_audit_event( + return host._launch_completion_failure_response(record, generation) + host.store.append_audit_event( "bot.start_readiness_pending", bot_id=bot_id, pid=pid, @@ -373,7 +365,7 @@ def _start_record( if effect.outcome != "running": raise RuntimeError(f"unknown gateway launch outcome: {effect.outcome}") try: - self._complete_started_intent( + host._complete_started_intent( record, context=context, status=BotStatus.running, @@ -383,8 +375,8 @@ def _start_record( reason="gateway process started without readiness probe", ) except Exception: - return self._launch_completion_failure_response(record, generation) - self.store.append_audit_event("bot.start", bot_id=bot_id, pid=pid) + return host._launch_completion_failure_response(record, generation) + host.store.append_audit_event("bot.start", bot_id=bot_id, pid=pid) return BotStatusResponse( bot_id=bot_id, status=BotStatus.running, @@ -393,19 +385,23 @@ def _start_record( message=message, ) + @staticmethod def _preflight_start( - self, record: BotRecord, *, timeout_seconds: float | None + host: StartHost, record: BotRecord, *, timeout_seconds: float | None ) -> ReadinessProbe | None: - return self._runtime.preflight_start(record, timeout_seconds=timeout_seconds) + return host._runtime.preflight_start(record, timeout_seconds=timeout_seconds) - def _write_pipe_payload(self, fd: int, payload: bytes) -> None: - self._runtime.write_pipe_payload(fd, payload) + @staticmethod + def _write_pipe_payload(host: StartHost, fd: int, payload: bytes) -> None: + host._runtime.write_pipe_payload(fd, payload) - def _read_launcher_ack(self, fd: int) -> bytes: - return self._runtime.read_launcher_ack(fd) + @staticmethod + def _read_launcher_ack(host: StartHost, fd: int) -> bytes: + return host._runtime.read_launcher_ack(fd) + @staticmethod def _complete_started_intent( - self, + host: StartHost, record: BotRecord, *, context: _LifecycleContext, @@ -420,7 +416,7 @@ def _complete_started_intent( operation_id = record.pending_operation_id if action not in {"start", "restart"} or operation_id is None: raise RuntimeError("pending launch intent is unavailable") - return self.store.complete_lifecycle_intent( + return host.store.complete_lifecycle_intent( record.bot_id, action=action, operation_id=operation_id, @@ -439,8 +435,9 @@ def _complete_started_intent( clear_stopped_at=True, ) + @staticmethod def _complete_failed_intent( - self, + host: StartHost, record: BotRecord, *, context: _LifecycleContext, @@ -454,7 +451,7 @@ def _complete_failed_intent( operation_id = record.pending_operation_id if action not in {"start", "restart"} or operation_id is None: raise RuntimeError("pending launch intent is unavailable") - return self.store.complete_lifecycle_intent( + return host.store.complete_lifecycle_intent( record.bot_id, action=action, operation_id=operation_id, @@ -474,25 +471,27 @@ def _complete_failed_intent( clear_ready_at=True, ) + @staticmethod def _cleanup_interrupted_intent_launch( - self, + host: StartHost, record: BotRecord, process: PopenLike, *, expected_fingerprint: str, ) -> bool: - return self._runtime.cleanup_interrupted_launch( + return host._runtime.cleanup_interrupted_launch( record, process, expected_fingerprint=expected_fingerprint, ) + @staticmethod def _launch_completion_failure_response( - self, + host: StartHost, record: BotRecord, generation: _GatewayGeneration, ) -> BotStatusResponse: - cleaned = self._runtime.cleanup_registered_launch(record, generation) + cleaned = host._runtime.cleanup_registered_launch(record, generation) if cleaned: return BotStatusResponse( record.bot_id, diff --git a/zeus/supervisor_start_host.py b/zeus/supervisor_start_host.py new file mode 100644 index 0000000..e8e838f --- /dev/null +++ b/zeus/supervisor_start_host.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import contextlib +import threading +from datetime import datetime +from typing import Protocol + +from zeus import process_identity as _process_identity +from zeus.gateway_runtime import ( + GatewayRuntime, +) +from zeus.models import ( + BotRecord, + BotStatus, + BotStatusResponse, +) +from zeus.process_lock import BotProcessLock +from zeus.readiness import ReadinessProbe +from zeus.state import StateStore +from zeus.supervisor_contracts import ( + _READINESS_PROBE_UNSET, + _GatewayGeneration, + _LifecycleContext, + _MarkerObservation, + _ReadinessProbeUnset, +) + +_PidState = _process_identity.PidState + + +class StartHost(Protocol): + """Only the host capabilities required by start operations.""" + + def _bot_process_lock(self, bot_id: str) -> BotProcessLock: ... + + def _classify_existing_runtime_marker( + self, record: BotRecord, *, expected_pid: int | None = None + ) -> _MarkerObservation: ... + + def _complete_failed_intent( + self, + record: BotRecord, + *, + context: _LifecycleContext, + pid: int | None, + message: str, + reason: str, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + ) -> BotRecord: ... + + def _complete_started_intent( + self, + record: BotRecord, + *, + context: _LifecycleContext, + status: BotStatus, + pid: int, + reason: str, + ready_at: datetime | None = None, + last_error: str | None = None, + reset_restart: bool = False, + ) -> BotRecord: ... + + def _launch_completion_failure_response( + self, record: BotRecord, generation: _GatewayGeneration + ) -> BotStatusResponse: ... + + def _lifecycle_context(self, source: str, request_id: str | None) -> _LifecycleContext: ... + + def _marker_publication_lock( + self, record: BotRecord + ) -> contextlib.AbstractContextManager[object]: ... + + def _matching_runtime_marker( + self, + record: BotRecord, + *, + expected_fingerprint: str, + expected_pid: int | None = None, + require_live_command: bool, + ) -> _MarkerObservation: ... + + def _pending_action_required(self, record: BotRecord, reason: str) -> BotStatusResponse: ... + + def _pid_owned(self, profile_path: str, pid: int, bot_id: str) -> bool: ... + + def _pid_state(self, pid: int) -> _PidState: ... + + def _preflight_start( + self, record: BotRecord, *, timeout_seconds: float | None + ) -> ReadinessProbe | None: ... + + def _read_launcher_ack(self, fd: int) -> bytes: ... + + def _remove_exact_schema3_marker( + self, record: BotRecord, marker: _MarkerObservation + ) -> bool: ... + + def _require_bot(self, bot_id: str) -> BotRecord: ... + + _runtime: GatewayRuntime + + def _start_locked( + self, + bot_id: str, + *, + wait: bool = False, + timeout_seconds: float | None = None, + context: _LifecycleContext, + ) -> BotStatusResponse: ... + + def _start_record( + self, + record: BotRecord, + *, + reset_restart: bool, + message: str, + wait: bool = False, + timeout_seconds: float | None = None, + context: _LifecycleContext, + probe: ReadinessProbe | _ReadinessProbeUnset | None = _READINESS_PROBE_UNSET, + ) -> BotStatusResponse: ... + + def _status_for_live_record( + self, record: BotRecord, *, context: _LifecycleContext + ) -> BotStatusResponse: ... + + def _unknown_pid_response( + self, record: BotRecord, operation: str, *, context: _LifecycleContext + ) -> BotStatusResponse: ... + + def _update_lifecycle( + self, + context: _LifecycleContext, + bot_id: str, + status: BotStatus, + pid: int | None = None, + *, + action: str | None = None, + started_at: datetime | None = None, + ready_at: datetime | None = None, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + last_error: str | None = None, + last_transition_reason: str | None = None, + reset_restart: bool = False, + clear_ready_at: bool = False, + clear_stopped_at: bool = False, + details: dict[str, object] | None = None, + ) -> None: ... + + def _write_pipe_payload(self, fd: int, payload: bytes) -> None: ... + + def bot_lock(self, bot_id: str) -> threading.RLock: ... + + store: StateStore diff --git a/zeus/supervisor_status.py b/zeus/supervisor_status.py index b132460..68ea9c3 100644 --- a/zeus/supervisor_status.py +++ b/zeus/supervisor_status.py @@ -21,54 +21,47 @@ DesiredState, validate_id, ) -from zeus.supervisor_core import ( +from zeus.supervisor_contracts import ( _LifecycleContext, ) -from zeus.supervisor_reconcile import _SupervisorReconcile +from zeus.supervisor_status_host import StatusHost -PidAliveFn = _process_identity.PidAliveFn -CmdlineReader = _process_identity.CmdlineReader -ProcStartFingerprintReader = _process_identity.ProcStartFingerprintReader - -_CommandCheck = _process_identity.CommandCheck _PidState = _process_identity.PidState -_looks_like_python_interpreter = _process_identity.looks_like_python_interpreter -_read_linux_cmdline = _process_identity.read_linux_cmdline -_read_linux_process_start_fingerprint = _process_identity.read_linux_process_start_fingerprint -_resolve_executable = _process_identity.resolve_executable -_resolve_launcher_exec_target = _process_identity.resolve_launcher_exec_target -_safe_command_shape = _process_identity.safe_command_shape -_trusted_hermes_paths = _process_identity.trusted_hermes_paths -_verify_gateway_command = _process_identity.verify_gateway_command -class _SupervisorStatus(_SupervisorReconcile): +class StatusOperations: + """Stateless status operations; callbacks are resolved from the current host.""" + + @staticmethod def status( - self, + host: StatusHost, bot_id: str, *, source: str = "cli", request_id: str | None = None, ) -> BotStatusResponse: - context = self._lifecycle_context(source, request_id) + context = host._lifecycle_context(source, request_id) # Reject unknown bots before allocating any lock state so that requests # for nonexistent ids cannot grow the in-memory lock table or the # on-disk lock directory. safe_bot_id = validate_id(bot_id, "bot_id") - self._require_bot(safe_bot_id) - with self.bot_lock(safe_bot_id), self._bot_process_lock(safe_bot_id): - return self._status_locked(safe_bot_id, context=context) + host._require_bot(safe_bot_id) + with host.bot_lock(safe_bot_id), host._bot_process_lock(safe_bot_id): + return host._status_locked(safe_bot_id, context=context) - def _status_locked(self, bot_id: str, *, context: _LifecycleContext) -> BotStatusResponse: - record = self._require_bot(bot_id) + @staticmethod + def _status_locked( + host: StatusHost, bot_id: str, *, context: _LifecycleContext + ) -> BotStatusResponse: + record = host._require_bot(bot_id) if record.pending_operation_id is not None: - return self._recover_pending_intent(record, context=context, allow_launch=False) - pid_state = self._pid_state(record.pid) if record.pid else _PidState.dead + return host._recover_pending_intent(record, context=context, allow_launch=False) + pid_state = host._pid_state(record.pid) if record.pid else _PidState.dead if record.pid and pid_state == _PidState.unknown: - return self._unknown_pid_response(record, "determine gateway status", context=context) + return host._unknown_pid_response(record, "determine gateway status", context=context) alive = bool(record.pid and pid_state == _PidState.alive) - if alive and record.pid and not self._pid_owned(record.profile_path, record.pid, bot_id): - self._update_lifecycle( + if alive and record.pid and not host._pid_owned(record.profile_path, record.pid, bot_id): + host._update_lifecycle( context, bot_id, BotStatus.failed, @@ -84,44 +77,45 @@ def _status_locked(self, bot_id: str, *, context: _LifecycleContext) -> BotStatu message="recorded gateway PID is alive but ownership could not be verified", ) if alive: - return self._status_for_live_record(record, context=context) + return host._status_for_live_record(record, context=context) try: - with self._marker_publication_lock(record): - return self._status_dead_record_locked(record, context=context) + with host._marker_publication_lock(record): + return host._status_dead_record_locked(record, context=context) except (BotDeleteError, LaunchPayloadError) as exc: - return self._pending_action_required(record, str(exc)) + return host._pending_action_required(record, str(exc)) + @staticmethod def _status_dead_record_locked( - self, + host: StatusHost, record: BotRecord, *, context: _LifecycleContext, ) -> BotStatusResponse: - observed = self._read_strict_runtime_marker(record.bot_id, record.profile_path) + observed = host._read_strict_runtime_marker(record.bot_id, record.profile_path) if observed.kind == "present" and observed.payload is not None: if record.pid is None: - return self._pending_action_required( + return host._pending_action_required( record, "stale gateway marker PID is not recorded" ) - marker = self._classify_schema3_runtime_marker( + marker = host._classify_schema3_runtime_marker( record, observed.payload, expected_pid=record.pid, expected_revision=record.desired_revision, require_live_command=True, ) - generation = self._gateway_generation(marker) + generation = host._gateway_generation(marker) if ( marker.kind != "dead" or generation is None - or not self._remove_gateway_generation_marker_locked(record, generation) + or not host._remove_gateway_generation_marker_locked(record, generation) ): - return self._pending_action_required( + return host._pending_action_required( record, marker.reason or "stale gateway marker ownership could not be verified", ) elif observed.kind != "missing": - return self._pending_action_required( + return host._pending_action_required( record, observed.reason or "stale gateway marker ownership could not be verified", ) @@ -132,7 +126,7 @@ def _status_dead_record_locked( if record.status in {BotStatus.starting, BotStatus.running}: last_error = "gateway process is not running" if record.status in {BotStatus.starting, BotStatus.running}: - self._update_lifecycle( + host._update_lifecycle( context, record.bot_id, status, @@ -142,7 +136,7 @@ def _status_dead_record_locked( last_transition_reason="gateway process was not running", ) elif record.pid is not None: - self._update_lifecycle( + host._update_lifecycle( context, record.bot_id, status, @@ -154,7 +148,7 @@ def _status_dead_record_locked( if record.desired_state is DesiredState.running: status = BotStatus.failed last_error = "desired running gateway is missing; action required: run reconcile" - self._update_lifecycle( + host._update_lifecycle( context, record.bot_id, status, @@ -177,22 +171,24 @@ def _status_dead_record_locked( message=message, ) - def logs(self, bot_id: str, max_bytes: int = 20_000) -> str: - self._require_bot(bot_id) - with self.bot_lock(bot_id): - record = self._require_bot(bot_id) - return tail_file(self.log_path(record.profile_path), max_bytes=max_bytes) + @staticmethod + def logs(host: StatusHost, bot_id: str, max_bytes: int = 20_000) -> str: + host._require_bot(bot_id) + with host.bot_lock(bot_id): + record = host._require_bot(bot_id) + return tail_file(host.log_path(record.profile_path), max_bytes=max_bytes) - def inspect(self, bot_id: str, max_log_bytes: int = 20_000) -> dict[str, object]: - self._require_bot(bot_id) - with self.bot_lock(bot_id): - record = self._require_bot(bot_id) + @staticmethod + def inspect(host: StatusHost, bot_id: str, max_log_bytes: int = 20_000) -> dict[str, object]: + host._require_bot(bot_id) + with host.bot_lock(bot_id): + record = host._require_bot(bot_id) profile_path = Path(record.profile_path) - marker = self._read_pid_marker(record.profile_path) + marker = host._read_pid_marker(record.profile_path) ownership = OwnershipCheck(False, "not-running") - pid_state = self._pid_state(record.pid) if record.pid else _PidState.dead + pid_state = host._pid_state(record.pid) if record.pid else _PidState.dead if record.pid and pid_state == _PidState.alive: - ownership = self._verify_gateway_pid_ownership( + ownership = host._verify_gateway_pid_ownership( record.profile_path, record.pid, bot_id ) elif record.pid and pid_state == _PidState.unknown: @@ -228,6 +224,6 @@ def inspect(self, bot_id: str, max_log_bytes: int = 20_000) -> dict[str, object] }, }, "recent_logs": tail_file( - self.log_path(record.profile_path), max_bytes=max_log_bytes + host.log_path(record.profile_path), max_bytes=max_log_bytes ), } diff --git a/zeus/supervisor_status_host.py b/zeus/supervisor_status_host.py new file mode 100644 index 0000000..9d3edd0 --- /dev/null +++ b/zeus/supervisor_status_host.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import contextlib +import threading +from datetime import datetime +from pathlib import Path +from typing import Protocol + +from zeus import process_identity as _process_identity +from zeus.gateway_runtime import ( + OwnershipCheck, +) +from zeus.models import ( + BotRecord, + BotStatus, + BotStatusResponse, +) +from zeus.process_lock import BotProcessLock +from zeus.supervisor_contracts import ( + _GatewayGeneration, + _LifecycleContext, + _MarkerObservation, +) + +_PidState = _process_identity.PidState + + +class StatusHost(Protocol): + """Only the host capabilities required by status operations.""" + + def _bot_process_lock(self, bot_id: str) -> BotProcessLock: ... + + def _classify_schema3_runtime_marker( + self, + record: BotRecord, + payload: dict[str, object], + *, + expected_pid: int | None = None, + expected_operation_id: str | None = None, + expected_revision: int | None = None, + expected_fingerprint: str | None = None, + require_live_command: bool, + ) -> _MarkerObservation: ... + + def _gateway_generation(self, marker: _MarkerObservation) -> _GatewayGeneration | None: ... + + def _lifecycle_context(self, source: str, request_id: str | None) -> _LifecycleContext: ... + + def _marker_publication_lock( + self, record: BotRecord + ) -> contextlib.AbstractContextManager[object]: ... + + def _pending_action_required(self, record: BotRecord, reason: str) -> BotStatusResponse: ... + + def _pid_owned(self, profile_path: str, pid: int, bot_id: str) -> bool: ... + + def _pid_state(self, pid: int) -> _PidState: ... + + def _read_pid_marker(self, profile_path: str) -> dict[str, object]: ... + + def _read_strict_runtime_marker( + self, bot_id: str, registered_profile_path: str + ) -> _MarkerObservation: ... + + def _recover_pending_intent( + self, record: BotRecord, *, context: _LifecycleContext, allow_launch: bool + ) -> BotStatusResponse: ... + + def _remove_gateway_generation_marker_locked( + self, record: BotRecord, generation: _GatewayGeneration + ) -> bool: ... + + def _require_bot(self, bot_id: str) -> BotRecord: ... + + def _status_dead_record_locked( + self, record: BotRecord, *, context: _LifecycleContext + ) -> BotStatusResponse: ... + + def _status_for_live_record( + self, record: BotRecord, *, context: _LifecycleContext + ) -> BotStatusResponse: ... + + def _status_locked(self, bot_id: str, *, context: _LifecycleContext) -> BotStatusResponse: ... + + def _unknown_pid_response( + self, record: BotRecord, operation: str, *, context: _LifecycleContext + ) -> BotStatusResponse: ... + + def _update_lifecycle( + self, + context: _LifecycleContext, + bot_id: str, + status: BotStatus, + pid: int | None = None, + *, + action: str | None = None, + started_at: datetime | None = None, + ready_at: datetime | None = None, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + last_error: str | None = None, + last_transition_reason: str | None = None, + reset_restart: bool = False, + clear_ready_at: bool = False, + clear_stopped_at: bool = False, + details: dict[str, object] | None = None, + ) -> None: ... + + def _verify_gateway_pid_ownership( + self, profile_path: str, pid: int, bot_id: str + ) -> OwnershipCheck: ... + + def bot_lock(self, bot_id: str) -> threading.RLock: ... + + def log_path(self, profile_path: str) -> Path: ... diff --git a/zeus/supervisor_stop.py b/zeus/supervisor_stop.py index 1b3c9b0..b4de49b 100644 --- a/zeus/supervisor_stop.py +++ b/zeus/supervisor_stop.py @@ -2,7 +2,6 @@ from datetime import UTC, datetime -from zeus import process_identity as _process_identity from zeus.errors import ( BotDeleteError, ) @@ -14,56 +13,45 @@ BotStatus, BotStatusResponse, ) -from zeus.supervisor_core import ( +from zeus.supervisor_contracts import ( _LifecycleContext, _MarkerObservation, ) -from zeus.supervisor_start import _SupervisorStart +from zeus.supervisor_stop_host import StopHost -PidAliveFn = _process_identity.PidAliveFn -CmdlineReader = _process_identity.CmdlineReader -ProcStartFingerprintReader = _process_identity.ProcStartFingerprintReader -_CommandCheck = _process_identity.CommandCheck -_PidState = _process_identity.PidState -_looks_like_python_interpreter = _process_identity.looks_like_python_interpreter -_read_linux_cmdline = _process_identity.read_linux_cmdline -_read_linux_process_start_fingerprint = _process_identity.read_linux_process_start_fingerprint -_resolve_executable = _process_identity.resolve_executable -_resolve_launcher_exec_target = _process_identity.resolve_launcher_exec_target -_safe_command_shape = _process_identity.safe_command_shape -_trusted_hermes_paths = _process_identity.trusted_hermes_paths -_verify_gateway_command = _process_identity.verify_gateway_command +class StopOperations: + """Stateless stop operations; callbacks are resolved from the current host.""" - -class _SupervisorStop(_SupervisorStart): + @staticmethod def stop( - self, + host: StopHost, bot_id: str, *, kill_after_timeout: bool | None = None, source: str = "cli", request_id: str | None = None, ) -> BotStatusResponse: - context = self._lifecycle_context(source, request_id) - with self.bot_lock(bot_id), self._bot_process_lock(bot_id): - return self._stop_locked( + context = host._lifecycle_context(source, request_id) + with host.bot_lock(bot_id), host._bot_process_lock(bot_id): + return host._stop_locked( bot_id, kill_after_timeout=kill_after_timeout, context=context, ) + @staticmethod def _stop_locked( - self, + host: StopHost, bot_id: str, *, kill_after_timeout: bool | None = None, context: _LifecycleContext, ) -> BotStatusResponse: - record = self._require_bot(bot_id) + record = host._require_bot(bot_id) if record.pending_operation_id is not None: - return self._pending_action_required(record, "lifecycle intent is already pending") - record = self.store.begin_lifecycle_intent( + return host._pending_action_required(record, "lifecycle intent is already pending") + record = host.store.begin_lifecycle_intent( bot_id, action="stop", operation_id=context.operation_id, @@ -71,15 +59,16 @@ def _stop_locked( request_id=context.request_id, reason="gateway stop requested", ) - return self._stop_record_effect( + return host._stop_record_effect( record, kill_after_timeout=kill_after_timeout, context=context, complete_stop=True, ) + @staticmethod def _stop_record_effect( - self, + host: StopHost, record: BotRecord, *, kill_after_timeout: bool | None = None, @@ -87,36 +76,37 @@ def _stop_record_effect( complete_stop: bool, ) -> BotStatusResponse: try: - with self._marker_publication_lock(record): - return self._stop_record_effect_locked( + with host._marker_publication_lock(record): + return host._stop_record_effect_locked( record, kill_after_timeout=kill_after_timeout, context=context, complete_stop=complete_stop, ) except (BotDeleteError, LaunchPayloadError) as exc: - return self._pending_action_required(record, str(exc)) + return host._pending_action_required(record, str(exc)) + @staticmethod def _stop_record_effect_locked( - self, + host: StopHost, record: BotRecord, *, kill_after_timeout: bool | None, context: _LifecycleContext, complete_stop: bool, ) -> BotStatusResponse: - effect = self._runtime.stop_locked( + effect = host._runtime.stop_locked( record, kill_after_timeout=kill_after_timeout, - read_marker=self._read_strict_runtime_marker, - classify_existing=self._classify_existing_runtime_marker, - classify_exact=self._classify_exact_gateway_generation, - remove_owned=self._remove_owned_launch_marker_locked, - remove_generation=self._remove_gateway_generation_marker_locked, + read_marker=host._read_strict_runtime_marker, + classify_existing=host._classify_existing_runtime_marker, + classify_exact=host._classify_exact_gateway_generation, + remove_owned=host._remove_owned_launch_marker_locked, + remove_generation=host._remove_gateway_generation_marker_locked, ) if effect.outcome not in {"not_running", "stopped"}: if effect.kill_result is not None: - self.store.append_audit_event( + host.store.append_audit_event( "bot.stop_kill", bot_id=record.bot_id, pid=effect.pid, @@ -129,20 +119,20 @@ def _stop_record_effect_locked( ) else: reason = effect.reason - return self._pending_action_required(record, reason) + return host._pending_action_required(record, reason) if effect.outcome == "not_running": if complete_stop: try: - self._complete_stopped_intent( + host._complete_stopped_intent( record, context=context, reason="gateway process was not running", ) except Exception: - return self._pending_action_required( + return host._pending_action_required( record, "stopped state could not be persisted" ) - self.store.append_audit_event("bot.stop", bot_id=record.bot_id, pid=record.pid) + host.store.append_audit_event("bot.stop", bot_id=record.bot_id, pid=record.pid) return BotStatusResponse( bot_id=record.bot_id, status=BotStatus.stopped, @@ -151,7 +141,7 @@ def _stop_record_effect_locked( message="not running", ) if effect.kill_result is not None: - self.store.append_audit_event( + host.store.append_audit_event( "bot.stop_kill", bot_id=record.bot_id, pid=effect.pid, @@ -159,7 +149,7 @@ def _stop_record_effect_locked( ) if not complete_stop: try: - self._update_lifecycle( + host._update_lifecycle( context, record.bot_id, BotStatus.stopped, @@ -170,19 +160,19 @@ def _stop_record_effect_locked( clear_ready_at=True, ) except Exception: - return self._pending_action_required( + return host._pending_action_required( record, "previous gateway stop could not be persisted" ) if complete_stop: try: - self._complete_stopped_intent( + host._complete_stopped_intent( record, context=context, reason="gateway shutdown completed", ) except Exception: - return self._pending_action_required(record, "stopped state could not be persisted") - self.store.append_audit_event("bot.stop", bot_id=record.bot_id, pid=record.pid) + return host._pending_action_required(record, "stopped state could not be persisted") + host.store.append_audit_event("bot.stop", bot_id=record.bot_id, pid=record.pid) return BotStatusResponse( bot_id=record.bot_id, status=BotStatus.stopped, @@ -191,8 +181,9 @@ def _stop_record_effect_locked( message="gateway shutdown completed", ) + @staticmethod def _complete_stopped_intent( - self, + host: StopHost, record: BotRecord, *, context: _LifecycleContext, @@ -201,7 +192,7 @@ def _complete_stopped_intent( operation_id = record.pending_operation_id if record.pending_action != "stop" or operation_id is None: raise RuntimeError("pending stop intent is unavailable") - return self.store.complete_lifecycle_intent( + return host.store.complete_lifecycle_intent( record.bot_id, action="stop", operation_id=operation_id, @@ -217,16 +208,18 @@ def _complete_stopped_intent( clear_ready_at=True, ) + @staticmethod def _remove_owned_launch_marker_locked( - self, + host: StopHost, record: BotRecord, *, observed: _MarkerObservation | None = None, ) -> bool: - return self._runtime.remove_owned_launch_marker_locked(record, observed=observed) + return host._runtime.remove_owned_launch_marker_locked(record, observed=observed) + @staticmethod def restart( - self, + host: StopHost, bot_id: str, *, wait: bool = False, @@ -234,13 +227,13 @@ def restart( source: str = "cli", request_id: str | None = None, ) -> BotStatusResponse: - context = self._lifecycle_context(source, request_id) - with self.bot_lock(bot_id), self._bot_process_lock(bot_id): - record = self._require_bot(bot_id) + context = host._lifecycle_context(source, request_id) + with host.bot_lock(bot_id), host._bot_process_lock(bot_id): + record = host._require_bot(bot_id) if record.pending_operation_id is not None: - return self._pending_action_required(record, "lifecycle intent is already pending") - probe = self._preflight_start(record, timeout_seconds=timeout_seconds) - record = self.store.begin_lifecycle_intent( + return host._pending_action_required(record, "lifecycle intent is already pending") + probe = host._preflight_start(record, timeout_seconds=timeout_seconds) + record = host.store.begin_lifecycle_intent( bot_id, action="restart", operation_id=context.operation_id, @@ -248,7 +241,7 @@ def restart( request_id=context.request_id, reason="gateway restart requested", ) - stopped = self._stop_record_effect( + stopped = host._stop_record_effect( record, context=context, complete_stop=False, @@ -262,8 +255,8 @@ def restart( message="restart aborted: " + stopped.message, ) - refreshed = self._require_bot(bot_id) - started = self._start_record( + refreshed = host._require_bot(bot_id) + started = host._start_record( refreshed, reset_restart=True, message="restarted", diff --git a/zeus/supervisor_stop_host.py b/zeus/supervisor_stop_host.py new file mode 100644 index 0000000..58ab387 --- /dev/null +++ b/zeus/supervisor_stop_host.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import contextlib +import threading +from datetime import datetime +from typing import Protocol + +from zeus.gateway_runtime import ( + GatewayRuntime, +) +from zeus.models import ( + BotRecord, + BotStatus, + BotStatusResponse, +) +from zeus.process_lock import BotProcessLock +from zeus.readiness import ReadinessProbe +from zeus.state import StateStore +from zeus.supervisor_contracts import ( + _READINESS_PROBE_UNSET, + _GatewayGeneration, + _LifecycleContext, + _MarkerObservation, + _ReadinessProbeUnset, +) + + +class StopHost(Protocol): + """Only the host capabilities required by stop operations.""" + + def _bot_process_lock(self, bot_id: str) -> BotProcessLock: ... + + def _classify_exact_gateway_generation( + self, record: BotRecord, generation: _GatewayGeneration + ) -> _MarkerObservation: ... + + def _classify_existing_runtime_marker( + self, record: BotRecord, *, expected_pid: int | None = None + ) -> _MarkerObservation: ... + + def _complete_stopped_intent( + self, record: BotRecord, *, context: _LifecycleContext, reason: str + ) -> BotRecord: ... + + def _lifecycle_context(self, source: str, request_id: str | None) -> _LifecycleContext: ... + + def _marker_publication_lock( + self, record: BotRecord + ) -> contextlib.AbstractContextManager[object]: ... + + def _pending_action_required(self, record: BotRecord, reason: str) -> BotStatusResponse: ... + + def _preflight_start( + self, record: BotRecord, *, timeout_seconds: float | None + ) -> ReadinessProbe | None: ... + + def _read_strict_runtime_marker( + self, bot_id: str, registered_profile_path: str + ) -> _MarkerObservation: ... + + def _remove_gateway_generation_marker_locked( + self, record: BotRecord, generation: _GatewayGeneration + ) -> bool: ... + + def _remove_owned_launch_marker_locked( + self, record: BotRecord, *, observed: _MarkerObservation | None = None + ) -> bool: ... + + def _require_bot(self, bot_id: str) -> BotRecord: ... + + _runtime: GatewayRuntime + + def _start_record( + self, + record: BotRecord, + *, + reset_restart: bool, + message: str, + wait: bool = False, + timeout_seconds: float | None = None, + context: _LifecycleContext, + probe: ReadinessProbe | _ReadinessProbeUnset | None = _READINESS_PROBE_UNSET, + ) -> BotStatusResponse: ... + + def _stop_locked( + self, bot_id: str, *, kill_after_timeout: bool | None = None, context: _LifecycleContext + ) -> BotStatusResponse: ... + + def _stop_record_effect( + self, + record: BotRecord, + *, + kill_after_timeout: bool | None = None, + context: _LifecycleContext, + complete_stop: bool, + ) -> BotStatusResponse: ... + + def _stop_record_effect_locked( + self, + record: BotRecord, + *, + kill_after_timeout: bool | None, + context: _LifecycleContext, + complete_stop: bool, + ) -> BotStatusResponse: ... + + def _update_lifecycle( + self, + context: _LifecycleContext, + bot_id: str, + status: BotStatus, + pid: int | None = None, + *, + action: str | None = None, + started_at: datetime | None = None, + ready_at: datetime | None = None, + stopped_at: datetime | None = None, + last_exit_code: int | None = None, + last_error: str | None = None, + last_transition_reason: str | None = None, + reset_restart: bool = False, + clear_ready_at: bool = False, + clear_stopped_at: bool = False, + details: dict[str, object] | None = None, + ) -> None: ... + + def bot_lock(self, bot_id: str) -> threading.RLock: ... + + store: StateStore