From 56712d67fba7d2f2c3b47aff8771661c90217de6 Mon Sep 17 00:00:00 2001 From: kirocrew-agent Date: Tue, 6 Oct 2026 21:00:24 +0000 Subject: [PATCH] fix(aws-control): claim the nightly backup through the Job SDK The nightly loop ran the backup directly on a thread, so a scheduled run claimed no Job SDK run: it shared no (kind, dedupe_key) with a manual click (both could upload, both paid) and the Backup row read idle while a nightly ran. _run_once now fires start_async(kind, dedupe_key=account, params={caller: scheduled}) per due kind -- the same call the manual route makes -- so the two callers dedupe and the row sees the run. The one registered runner reads the caller from params and, for a scheduled run, records a failure into the backoff due_for_nightly reads (minus teardown), so a deterministic nightly fault still is not retried every half hour. Two refinements keep the shared-caller path honest: Adopted-run attribution. Because both callers share one (kind, dedupe_key), an owner's "Back up now" click during a nightly adopts the in-flight scheduled run -- the SDK returns the first caller's run and params do not merge, so the adopted run stays CALLER_SCHEDULED and runs the scheduled-only re-checks. A scheduled-only precondition the owner controls (the unattended grant withdrawn, or scheduled transcripts blocked because redaction is on) now raises ScheduledUploadWithheld; the runner audits it withheld and does NOT feed the nightly backoff. Without this an owner who clicked during a nightly and surfaced one of those gates would see a failure they did not cause and watch the backoff grow. An unregistered-kind fail-closed stays a fault that backs off. Stale profile fallback. resolve_account_profile_cached serves only from the warm snapshot and returns None once it is past _PROBE_TTL_SECS, and start_async is fire-and-forget, so the runner resolves later than the loop's live probe; a gap past the TTL audited a just-verified nightly as failed. The loop now names the (profile, region) it freshly probed under JOB_PARAM_PROFILE / JOB_PARAM_REGION, and a scheduled run falls back to that pair when the cache has expired. The live sts:GetCallerIdentity re-check still guards the account, and the bucket is still re-discovered per run. Fixes #7783 - Restore the per-kind "cancelled" SEL audit on a setup (drive-lookup) cancel (dropped in the rewrite): teardown still leaves a trail it was interrupted, without backing off. The loop-side failure and cancel audits route through backup._nightly_audit / _nightly_subject (one audit helper, not a second inline copy). Pinned by test_a_cancel_during_the_lookup_audits_cancelled_but_does_not_back_off. --- docs/system-specs/modules/aws-control.md | 76 +- .../builtins/aws_control/backend/backup.py | 250 ++++++- .../backend/backup_parts/uploads.py | 74 ++ .../apps/builtins/aws_control/hooks.py | 365 ++++------ test/test_aws_control_app.py | 61 +- ...aws_control_backup_composition_contract.py | 4 + test/test_aws_control_backup_job.py | 432 +++++++++++- test/test_aws_control_hooks.py | 649 ++++++------------ test/test_aws_control_nightly_sessions.py | 302 ++------ 9 files changed, 1237 insertions(+), 976 deletions(-) diff --git a/docs/system-specs/modules/aws-control.md b/docs/system-specs/modules/aws-control.md index e3785ca7d32e..e854962e6437 100644 --- a/docs/system-specs/modules/aws-control.md +++ b/docs/system-specs/modules/aws-control.md @@ -548,9 +548,9 @@ case no other term can see, including the coarse-clock collision reproduced by h two sequence type guards cover each other on a record with no sequence, so they are pinned as a pair. If the slot is absent or moved while the archive is being built, the skip is refused, the refusal leaves the newer record in place, and the -run logs the reason before uploading the archive. `hooks._run_once` reads `uploaded` and -reports and audits a skip as `unchanged` rather than as a push; a record without the field -reads as a push. Neither the label publish nor the retention sweep runs on a successful +run logs the reason before uploading the archive. The scheduled run's +`backup.make_job_runner` runner reads `uploaded` and audits a skip as `unchanged` +rather than as a push; a record without the field reads as a push. Neither the label publish nor the retention sweep runs on a successful skip. Labels follow a push, so a local rename reaches the drive on the next real upload rather than on a skip. An unchanged night cannot consume a keep slot or prune the archive the next skip depends on. @@ -1391,15 +1391,57 @@ the gateway stopped. The panel copy says exactly that, because "share memory between my laptop and my cloud desktop" is the request that leads people here. The nightly toggle records whether an account is eligible for a scheduled -snapshot. `aws_control.hooks._run_once` resolves an account and drive, checks -S3 consent, runs only due backups, and SEL-audits invocation, success, and -failure. It skips unavailable accounts or absent drives rather than creating -resources itself. When `hooks._note_shared_drive` sees another install's prefix it -logs and SEL-audits the observation and then PROCEEDS. It does not take ownership -of the schedule: with the keys namespaced there is nothing left to collide, and a +snapshot. `aws_control.hooks._run_once` resolves the account, checks S3 consent, and +checks for the drive; it skips an unavailable account or an absent drive rather than +creating resources, so when there is nowhere to push it CLAIMS no run at all (as the +loop did before it went through the SDK — a claimed-but-empty `done` run would clear +an owner's last failure on the Backup row). Otherwise, for each due kind it claims a +Job SDK run through the same `start_async(kind, dedupe_key=account, …)` call the +owner's manual backup uses — so a scheduled and a manual backup of one account dedupe +on one key and the run shows on the Backup row. The run's `backup.make_job_runner` +runner SEL-audits invocation, success, the `unchanged` skip, and failure, and feeds a +failed scheduled run into the backoff `due_for_nightly` reads (minus a +`ScheduledUploadWithheld` policy refusal, which is audited `withheld` and recorded +nowhere — see the adopted-caller rule above). A drive lookup that +RAISES (not one that returns no drive) is a fault that happens in the loop before any +run is claimed, so the runner never sees it; `hooks._nightly_setup_failure` records and +SEL-audits it per due kind there, the same two writes the runner makes after a claim, +so a setup fault backs off exactly like a push fault. When the runner sees +another install's prefix (checked once, on the snapshot kind only) it logs and +SEL-audits the observation and then PROCEEDS. It does not take ownership of the +schedule: with the keys namespaced there is nothing left to collide, and a single-owner schedule would leave one machine silently un-backed-up, which is discovered at restore time and is worse than the state it replaced. +Because a scheduled and a manual backup of one account share one `(kind, +dedupe_key)`, an owner's "Back up now" click DURING a nightly adopts the in-flight +scheduled run: the SDK returns the first caller's run and `start_async` does not +merge params, so the adopted run keeps `CALLER_SCHEDULED` and still runs the +scheduled-only re-checks in `backup_parts/uploads.py`. The caller an adopted run +keeps is therefore the SCHEDULED one, by construction — and the rule that keeps that +honest for the owner is the backoff split below: a scheduled-only precondition the +OWNER controls — the per-kind unattended grant withdrawn, or scheduled transcripts no +longer allowed because outbound redaction was turned on mid-build — raises +`uploads.ScheduledUploadWithheld` rather than a bare failure, and the runner audits it +`withheld` and does NOT call `record_nightly_failure`. Without that split, an owner +who clicked during a nightly and surfaced one of those gates would see a failure they +did not cause and watch the backoff grow. The unregistered-kind fail-closed (no grant +DEFINED for a kind) stays a fault that backs off: it is a configuration error, not an +owner declining the upload. + +The scheduled runner re-resolves the drive per run, but it reads `(profile, region)` +differently from the owner path. `accounts.resolve_account_profile_cached` serves only +from the warm snapshot and returns `None` once it is past `accounts._PROBE_TTL_SECS`, +and `start_async` is fire-and-forget, so the runner thread resolves LATER than the +loop's live `probe_identity`. A gap longer than the TTL would resolve a nightly the +loop JUST verified to `None` and audit it as failed. So the loop names the +`(profile, region)` it freshly probed (`use_cache=False`) under `JOB_PARAM_PROFILE` / +`JOB_PARAM_REGION`, and a scheduled run falls back to that pair when the cache has +expired. This cannot operate on the wrong account: `_authorize_upload` still re-checks +the LIVE account with `sts:GetCallerIdentity` before any byte leaves. The owner path +carries neither param and keeps the cache-only resolve, because its route pre-flight +has just warmed the snapshot. The bucket is always re-discovered either way. + ### A failed unattended attempt is recorded, so the loop can back off The half-hourly wake is a due-CHECK interval and never a retry interval. If only @@ -1410,10 +1452,13 @@ would take its never-ran branch on every wake, re-stage the whole data home into temporary directory, and repeat the same traceback roughly every half hour for as long as the fault lasted. -`hooks._failed_attempt` closes that. It is the single place a failed unattended -attempt is both SEL-audited and recorded, reached from the shared setup's handler -and from each kind's own push, so the backoff cannot depend on which way the run -broke. +A failed unattended attempt is both SEL-audited and recorded in two honest places — +`backup.make_job_runner`'s post-claim handler for a fault during the push, and +`hooks._nightly_setup_failure` for a drive lookup that raises before any run is +claimed — so the backoff cannot depend on which way the run broke. Both make the +same pair of writes (the `failed` SEL audit under `backup/` and +`record_nightly_failure`), filed under one subject spelling per kind so a setup +failure and a push failure for one kind never land apart. It closes that for a fault that leaves the state file writable, which is not every fault. A full disk fails the failure-write as well, so nothing is recorded and every @@ -1424,7 +1469,10 @@ leaves the loop retrying as it does today. So a full disk is still a repeated-tr case; what this closes is the larger class where the disk is fine and the backup is not. Cancellation does not reach it: a cancelled attempt is teardown, and -counting it would let a clean shutdown push the next night out. +counting it would let a clean shutdown push the next night out. Neither does a +`ScheduledUploadWithheld` refusal: the owner withdrawing an unattended grant, or +turning redaction on, is a policy decision not to upload, not a fault, so counting it +would let the owner's own setting grow the schedule's backoff. The record lives under `backup.NIGHTLY_FAILURE_STATE_KEY`, per account then per kind, as `{"at": iso8601, "since": iso8601, "consecutive": int, "error": str}`. It is a SEPARATE key diff --git a/src/kiro_crew/apps/builtins/aws_control/backend/backup.py b/src/kiro_crew/apps/builtins/aws_control/backend/backup.py index b9050e0ec477..9e192d9d78b5 100644 --- a/src/kiro_crew/apps/builtins/aws_control/backend/backup.py +++ b/src/kiro_crew/apps/builtins/aws_control/backend/backup.py @@ -167,6 +167,9 @@ from kiro_crew import snapshot from kiro_crew.apps.builtins.aws_control.backend import accounts as accounts_mod from kiro_crew.apps.builtins.aws_control.backend import storage +from kiro_crew.apps.builtins.aws_control.backend.backup_parts.catalog import ( + other_install_ids, +) from kiro_crew.apps.builtins.aws_control.backend.backup_parts.egress_text import ( _redact_egress, ) @@ -191,6 +194,7 @@ from kiro_crew.apps.builtins.aws_control.backend.backup_parts.layer_b import ( _audit_layer_b_decision, layer_b_grant_covers_conversations, + sel, sessions_layer_b_enabled, ) from kiro_crew.apps.builtins.aws_control.backend.backup_parts.ledger import ( @@ -199,6 +203,10 @@ uploaded_objects, uploaded_versions, ) +from kiro_crew.apps.builtins.aws_control.backend.backup_parts.nightly import ( + nightly_run_witness, + record_nightly_failure, +) from kiro_crew.apps.builtins.aws_control.backend.backup_parts.retention import ( _audit_retention, _prune_remote_archives, @@ -218,7 +226,13 @@ _add_pinned, ) from kiro_crew.apps.builtins.aws_control.backend.backup_parts.uploads import ( + _STOP, CALLER_OWNER, + CALLER_SCHEDULED, + JOB_PARAM_CALLER, + JOB_PARAM_PROFILE, + JOB_PARAM_REGION, + ScheduledUploadWithheld, _authorize_recovery_read, _authorize_upload, _refuse_upload, @@ -1784,6 +1798,34 @@ def run_sessions_backup( JOB_KINDS = (KIND_SNAPSHOT, KIND_SESSIONS) +def _nightly_subject(kind: str) -> str: + """The SEL subject for one scheduled backup kind. One spelling, so a run's + records all file under the same subject for the kind.""" + return f"backup/{KIND_SUBPATHS[kind]}" + + +def _nightly_audit(operation: str, resources: str, outcome: str, *, error: str = "") -> None: + """SEL record for an UNATTENDED backup step. + + The HTTP handlers get their audit from the dashboard layer; a scheduled run + has no request, so without this the one S3 mutation this app makes with nobody + present would leave no trail of having run, succeeded, found nothing to do, or + failed. Best-effort by the same rule the handlers use: an audit failure must + never abort the backup. + """ + try: + sel().log_api_access( + caller="aws-control-nightly", + operation=f"aws_control.{operation}", + outcome=outcome, + source=APP_NAME, + resources=resources[:200], + error=error[:200], + ) + except Exception: + logger.debug("aws-control nightly SEL audit failed", exc_info=True) + + def make_job_runner(sdk: Any, kind: str) -> Any: """Build the Job SDK runner for ``kind``. Registered once, at app startup. @@ -1795,8 +1837,8 @@ def make_job_runner(sdk: Any, kind: str) -> Any: property is the app's to keep. That constraint is what shapes the resolution below. The SDK gives a runner - its handle and nothing else -- there is no ``params`` channel in P1 -- so the - run's target is read back out of its own record, where ``start`` put it: + its handle and nothing else, so the run's target is read back out of its own + record, where ``start`` put it: * The ACCOUNT comes from ``dedupe_key``. It is the right carrier on its own merits, because the account is exactly this run's concurrency identity -- @@ -1805,9 +1847,34 @@ def make_job_runner(sdk: Any, kind: str) -> Any: same account still run independently. It is also the only field a runner can read without a private attribute (``get`` is public; the key is withheld from the HTTP view and never logged by the SDK). + * The CALLER role comes from ``handle.params`` under ``JOB_PARAM_CALLER``. + This is what lets ONE runner serve both the owner click and the nightly + loop: the loop states ``CALLER_SCHEDULED`` when it starts the run, and this + reads it back so the run applies the scheduled-only gates. An absent or + unrecognised value is the owner path, the safe default. Because both + callers share one ``(kind, dedupe_key)``, an owner's "Back up now" during a + nightly ADOPTS the in-flight scheduled run -- the SDK returns the first + caller's run and params do not merge, so the adopted run keeps + ``CALLER_SCHEDULED`` and still runs the scheduled-only gates. The rule that + keeps this honest for the owner is in the failure handler below: a + scheduled-only precondition the owner controls (``ScheduledUploadWithheld``: + the unattended grant is withdrawn, or scheduled transcripts are blocked here) + is audited ``withheld`` and NOT fed into the nightly backoff, so an owner + whose adopting click surfaces one of those gates never sees the schedule's + backoff grow for a run they asked for. * profile/region/bucket are RE-RESOLVED here rather than carried, which is the rule this app already documents for the nightly loop: the drive is - tag-discovered per run rather than trusted from memory. + tag-discovered per run rather than trusted from memory. The one exception + is the SCHEDULED path's profile/region: ``resolve_account_profile_cached`` + serves only from the warm snapshot and returns ``None`` once it is past + ``accounts._PROBE_TTL_SECS``, and ``start_async`` is fire-and-forget, so + this runner resolves LATER than the loop's live probe -- a gap longer than + the TTL would audit a just-verified nightly as failed. The loop names the + ``(profile, region)`` it freshly probed under ``JOB_PARAM_PROFILE`` / + ``JOB_PARAM_REGION``, and a scheduled run falls back to it when the cache + has expired. Wrong-account risk stays closed: the live + ``sts:GetCallerIdentity`` below re-verifies the account before any byte + leaves. The bucket is still always re-discovered. Every resolution step is therefore sync. ``accounts.resolve_account_profile`` and ``aws_consent.authorize`` are coroutines and are NOT reachable from a @@ -1834,6 +1901,19 @@ def make_job_runner(sdk: Any, kind: str) -> Any: def _run(handle: Any) -> None: run = sdk.get(handle.run_id) account = run.dedupe_key if run is not None else "" + # The caller role rides in `params` under JOB_PARAM_CALLER; the account + # already rides in `dedupe_key`. This is the one extra fact a start has to + # carry so a single registered runner serves BOTH the owner click and the + # nightly loop: the loop states CALLER_SCHEDULED, and reading it back here + # is what makes `_authorize_upload` and `work` apply the unattended grant + # and sessions-redaction re-reads a scheduled payload must honour. Absent, + # empty or unrecognised means the owner path -- the safe default, since + # only the scheduled loop sets it and the owner path is the fuller one at + # the drive gate. + params = handle.params if hasattr(handle, "params") else {} + caller = ( + CALLER_SCHEDULED if params.get(JOB_PARAM_CALLER) == CALLER_SCHEDULED else CALLER_OWNER + ) # An empty key reaches here from `POST /_jobs/{kind}/start` with no body: # the generic surface defaults `dedupe_key` to "". There is no account to # act on, and picking one would be acting on an account nobody named. @@ -1843,34 +1923,136 @@ def _run(handle: Any) -> None: raise RuntimeError( "this backup run does not name an account id; nothing was sent to AWS" ) - resolved = accounts_mod.resolve_account_profile_cached(account) - if resolved is None: - raise RuntimeError( - "no working connection for this account — reconnect it, then run the backup again" - ) - profile, region = resolved - # Authorize BEFORE discovery, not just before the upload. `find_drive` - # reaches AWS to resolve the bucket by tags, so with consent withdrawn or - # the app disabled the old order sent tagging-API requests on the owner's - # credentials before any gate had run -- unauthorized calls made in the - # course of refusing the work. The gate needs no bucket, so nothing forces - # it to wait for discovery. - # - # This does NOT replace the pre-upload re-check inside `work`: an archive - # build takes minutes, and consent can be withdrawn during it. This one - # decides whether we may touch AWS at all; that one decides whether the - # bytes may leave. Both are needed, and both audit through the same helper. - _authorize_upload(account, profile, region, caller=CALLER_OWNER, payload_kind=kind) - bucket = storage.find_drive(profile, region, account=account) - if not bucket: - raise RuntimeError("this account has no drive yet; nothing was sent to AWS") - # Resolved by NAME at call time, not captured at registration: the module - # attribute stays the single definition of what a snapshot backup is. - work = run_snapshot_backup if kind == KIND_SNAPSHOT else run_sessions_backup - # A job exists because an owner asked for one through the app's route or - # the `_jobs` surface, both owner-gated. The nightly loop does not come - # through here and states `CALLER_SCHEDULED` for itself. - work(account, profile, region, bucket, caller=CALLER_OWNER) + # The run slot's identity, read BEFORE the attempt and only for a scheduled + # run: `record_nightly_failure` refuses to write when the slot has moved + # since, so the witness has to be the slot as it stood before this attempt + # could change it. An owner run never records a backoff (someone is present + # and sees the failure), so it needs no witness. + scheduled = caller == CALLER_SCHEDULED + run_witness = nightly_run_witness(account, kind) if scheduled else None + try: + resolved = accounts_mod.resolve_account_profile_cached(account) + if resolved is None and scheduled: + # The cache serves only from the warm snapshot and returns None once + # it is past `accounts._PROBE_TTL_SECS`. `start_async` is + # fire-and-forget, so this runner thread resolves LATER than the + # nightly loop's live `probe_identity` check; a gap longer than the + # TTL would audit a just-verified nightly as failed. The loop names + # the `(profile, region)` it freshly probed in `params`, so fall back + # to it rather than re-failing. No wrong-account risk: the live + # `sts:GetCallerIdentity` in `_authorize_upload` below still verifies + # the account before anything leaves. The owner path keeps the + # cache-only resolve: its route pre-flight has just warmed the snapshot. + verified_profile = params.get(JOB_PARAM_PROFILE) or "" + verified_region = params.get(JOB_PARAM_REGION) or "" + if verified_profile and verified_region: + resolved = (verified_profile, verified_region) + if resolved is None: + raise RuntimeError( + "no working connection for this account — reconnect it, " + "then run the backup again" + ) + profile, region = resolved + # Authorize BEFORE discovery, not just before the upload. `find_drive` + # reaches AWS to resolve the bucket by tags, so with consent withdrawn or + # the app disabled the old order sent tagging-API requests on the owner's + # credentials before any gate had run -- unauthorized calls made in the + # course of refusing the work. The gate needs no bucket, so nothing forces + # it to wait for discovery. + # + # This does NOT replace the pre-upload re-check inside `work`: an archive + # build takes minutes, and consent can be withdrawn during it. This one + # decides whether we may touch AWS at all; that one decides whether the + # bytes may leave. Both are needed, and both audit through the same helper. + _authorize_upload(account, profile, region, caller=caller, payload_kind=kind) + bucket = storage.find_drive(profile, region, account=account) + if not bucket: + # The scheduled loop checks the drive BEFORE claiming a run, so a + # scheduled run never reaches here with no drive; this guards the + # generic `_jobs` entry point, where it is an honest failure for a + # request that named a target with nowhere to push. + raise RuntimeError("this account has no drive yet; nothing was sent to AWS") + # A scheduled run spends the owner's money with nobody present, so it + # records that another install already backs up to this drive -- evidence + # a later panel view cannot reconstruct, since the console's `others` + # count is LIVE. A NOTICE, not a gate: the keys carry an install id, so + # two installs write to two prefixes and refusing would silently stop + # backing one machine up. ONE list call, against the snapshot prefix only + # (so the snapshot kind carries it, not sessions): a both-due wake must + # not pay a second LIST, and a sessions-only wake must not pay a + # snapshot-prefix LIST for a question about a run that is not happening. + # Never raises -- a listing failure is one less log line, not a missed + # nightly. + if scheduled and kind == KIND_SNAPSHOT: + try: + others = other_install_ids(profile, region, bucket, account=account) + except Exception: + others = [] + logger.debug("aws-control nightly: shared-drive check failed", exc_info=True) + if others: + logger.warning( + "aws-control nightly: %d other install(s) also back up to this " + "drive (%s); each writes under its own prefix, so this run proceeds", + len(others), + ", ".join(others[:5]), + ) + _nightly_audit("backup_shared_drive", f"installs={len(others)}", "invoked") + # Resolved by NAME at call time, not captured at registration: the module + # attribute stays the single definition of what a snapshot backup is. + work = run_snapshot_backup if kind == KIND_SNAPSHOT else run_sessions_backup + # Both callers reach this one runner now: an owner click through the app's + # route or the `_jobs` surface, and the nightly loop, which claims its run + # through `start_async` and names CALLER_SCHEDULED in `params` so this run + # applies the unattended gates rather than attributing a 03:00 push to a + # human who is not present. + # + # A scheduled run has no HTTP handler to add the audit the dashboard layer + # puts on every owner-driven mutation, so it emits the same invoked / + # (succeeded|unchanged) / failed trail here. The split matters: a run that + # found the tree unchanged sent nothing, so recording it as a push would + # file a SEL entry against a key no bytes reached tonight -- which the SDK's + # own `done` record cannot tell apart from an ordinary upload. + if scheduled: + _nightly_audit("backup_nightly", _nightly_subject(kind), "invoked") + record = work(account, profile, region, bucket, caller=caller) + if scheduled: + if isinstance(record, dict) and record.get("uploaded") is False: + _nightly_audit("backup_nightly", str(record.get("key", "")), "unchanged") + else: + key = str(record.get("key", "")) if isinstance(record, dict) else "" + _nightly_audit("backup_nightly", key, "succeeded") + except Exception as exc: + # A SCHEDULED failure feeds the loop's backoff so a deterministic fault + # is not re-attempted every wake (`due_for_nightly` reads this record, + # not the SDK run), then re-raises so `_execute` still records the run + # `failed`. The loop only claims a scheduled run once a drive exists, so + # "no drive yet" never reaches here as a scheduled fault. A TEARDOWN is + # not a fault: app disable / gateway stop surface as the stop-event + # refusal `_authorize_upload` raises once `_STOP` is set, and counting + # that would let a restart push the next night out, so it is recorded + # nowhere. A scheduled-only precondition the owner controls -- the + # unattended grant withdrawn, or scheduled transcripts blocked here + # (`ScheduledUploadWithheld`) -- is likewise not a fault: it is the owner's + # own setting declining the upload, so it is audited `withheld` and NOT + # recorded into the backoff. This is the rule an ADOPTED run keeps: because + # both callers share one `(kind, dedupe_key)`, an owner's "Back up now" + # during a nightly adopts the in-flight scheduled run (the SDK keeps the + # first caller's `CALLER_SCHEDULED` params; they do not merge), so without + # this branch an owner surfacing one of these gates would see a failure they + # did not cause and watch the backoff grow. An owner run just re-raises -- + # its failure is the owner's to see, not the schedule's. + if scheduled and isinstance(exc, ScheduledUploadWithheld): + # Owner-controlled withdrawal, not a fault: audited as withheld and + # NOT recorded, so a withdrawn grant cannot push the next night out. + _nightly_audit("backup_nightly", _nightly_subject(kind), "withheld") + elif scheduled and not _STOP.is_set(): + _nightly_audit("backup_nightly", _nightly_subject(kind), "failed", error=str(exc)) + record_nightly_failure(account, kind, str(exc), run_witness=run_witness) + elif scheduled: + # Teardown, not a fault: audited as such and NOT recorded, so a clean + # shutdown cannot push the next night out. + _nightly_audit("backup_nightly", _nightly_subject(kind), "cancelled") + raise return _run @@ -2360,7 +2542,6 @@ def _holders(name: str) -> tuple[str, ...]: _checked, _install_folders, list_remote_backups, - other_install_ids, read_remote_label, ) from kiro_crew.apps.builtins.aws_control.backend.backup_parts.egress_text import ( # noqa: F401 @@ -2400,7 +2581,6 @@ def _holders(name: str) -> tuple[str, ...]: SESSIONS_LAYER_B_SCOPE_KEY, SESSIONS_LAYER_B_SCOPE_WITH_CONVERSATIONS, _audit_layer_b_grant, - sel, set_sessions_layer_b, ) from kiro_crew.apps.builtins.aws_control.backend.backup_parts.ledger import ( # noqa: F401 @@ -2432,9 +2612,7 @@ def _holders(name: str) -> tuple[str, ...]: nightly_enabled, nightly_failures, nightly_retry_delay_secs, - nightly_run_witness, nightly_sessions_enabled, - record_nightly_failure, scheduled_sessions_blocked_code, scheduled_sessions_blocked_reason, set_nightly, @@ -2511,8 +2689,6 @@ def _holders(name: str) -> tuple[str, ...]: kind_unavailable_reason, ) from kiro_crew.apps.builtins.aws_control.backend.backup_parts.uploads import ( # noqa: F401 - _STOP, - CALLER_SCHEDULED, SEL_OP_BASELINE_PROBE, SEL_OP_RETENTION, SEL_OP_UPLOAD, diff --git a/src/kiro_crew/apps/builtins/aws_control/backend/backup_parts/uploads.py b/src/kiro_crew/apps/builtins/aws_control/backend/backup_parts/uploads.py index 5940dadc5392..50029527d898 100644 --- a/src/kiro_crew/apps/builtins/aws_control/backend/backup_parts/uploads.py +++ b/src/kiro_crew/apps/builtins/aws_control/backend/backup_parts/uploads.py @@ -36,6 +36,33 @@ logger = logging.getLogger(_FACADE_MODULE) +class ScheduledUploadWithheld(RuntimeError): + """A scheduled-only precondition said "do not upload" — a policy outcome, not a fault. + + Raised by the two re-checks in :func:`_authorize_upload` that only a + ``CALLER_SCHEDULED`` run is subject to: the per-kind unattended grant does + not hold, or a scheduled transcript upload is blocked here + (redaction turned on mid-build). These are the owner's own settings deciding + the bytes may not leave — not a connection, credential, consent, or AWS + failure. + + It is a distinct type because the drive gate cannot otherwise tell the two + apart: every refusal raises, and :func:`backup._run`'s scheduled handler + feeds a raised exception into ``record_nightly_failure`` so a deterministic + fault is not re-attempted every wake. A withheld-by-policy run must NOT grow + that backoff — it did not fail, the owner withdrew the authorization — and + the distinction matters most on the ADOPTED path: because both callers share + one ``(kind, dedupe_key)``, an owner's "Back up now" during a nightly adopts + the in-flight scheduled run (the SDK returns the first caller's run; params + do not merge, so it stays ``CALLER_SCHEDULED``). If that adopted run met one + of these scheduled-only gates and the refusal counted as a nightly failure, + the owner would see a failure they did not cause and watch the backoff grow. + Subclassing ``RuntimeError`` keeps every existing ``except RuntimeError`` and + the Job SDK's own ``failed`` recording unchanged — the run is still a failed + SDK run — this type only steers the backoff decision. + """ + + #: Who triggered an upload, as the SEL record names them. #: #: An attribution field only earns its place if it DISTINGUISHES, so neither of @@ -50,6 +77,35 @@ CALLER_SCHEDULED = f"app:{APP_NAME}" +#: Job SDK ``params`` key under which a run names its caller role. The run's +#: target already rides in ``dedupe_key`` (the account); this is the one other +#: fact a runner needs the start to carry, so a single registered runner can +#: serve both the owner click and the nightly loop. A run that omits it is the +#: owner path -- the safe default, since only the scheduled loop sets it and the +#: owner path is the fuller one at the drive gate. +JOB_PARAM_CALLER = "caller" + + +#: The profile/region the SCHEDULED loop already verified LIVE, carried so the +#: runner can fall back to it. The runner re-resolves the drive per run (the +#: tag-discovered rule this app documents), but ``resolve_account_profile_cached`` +#: serves only from the in-process snapshot and returns ``None`` once that snapshot +#: is past ``accounts._PROBE_TTL_SECS``. ``start_async`` is fire-and-forget, so the +#: runner thread resolves LATER than the loop's ``probe_identity`` check; if that +#: gap crosses the TTL, a nightly the loop JUST verified resolves to ``None`` and is +#: audited as failed. The loop already holds a freshly probed ``(profile, region)`` +#: (``use_cache=False``), so it names them here and the scheduled path falls back to +#: them when the cache has expired -- the "let the scheduled path use the profile it +#: verified" fix. No wrong-account risk: ``_authorize_upload`` still re-verifies the +#: LIVE account with ``sts:GetCallerIdentity`` before any byte leaves. Only the +#: scheduled loop sets these; an owner run omits them and the route's pre-flight has +#: just warmed the snapshot, so the cached resolve answers for it. +JOB_PARAM_PROFILE = "profile" + + +JOB_PARAM_REGION = "region" + + #: SEL operation names for the two decisions the backup engine asks #: :func:`_authorize_upload` to make. #: @@ -96,6 +152,7 @@ def _refuse_upload( caller: str, outcome: str = "denied", operation: str = SEL_OP_UPLOAD, + withheld: bool = False, ) -> NoReturn: """Record why an upload was refused in the SEL, then refuse. @@ -134,6 +191,15 @@ def _refuse_upload( denied upload is a denial recorded against a transfer that completed. See :data:`SEL_OP_UPLOAD` and :data:`SEL_OP_RETENTION`. + ``withheld`` distinguishes a scheduled-only PRECONDITION the owner controls + (the unattended grant is withdrawn, or scheduled transcripts are blocked + because redaction is on) from a genuine fault. Both still + raise and both still audit, but a withheld refusal raises + :class:`ScheduledUploadWithheld` so ``backup._run`` does not grow the nightly + backoff for a run the owner chose to stop -- which matters most for the + adopted-run case an owner's "Back up now" click lands on. The SEL ``outcome`` + stays ``denied``: it is still an access decision that refused the bytes. + Best-effort, like the route's audit: a failed audit must never convert a refusal into an upload. """ @@ -148,6 +214,12 @@ def _refuse_upload( ) except Exception: logger.debug("aws-control SEL audit failed", exc_info=True) + if withheld: + # A scheduled-only precondition the owner controls said "do not upload". + # Distinct from a fault so `backup._run` does not feed it into the nightly + # backoff; see `ScheduledUploadWithheld`. Still an exception, so the Job + # SDK records the run `failed` and no upload proceeds. + raise ScheduledUploadWithheld(reason) raise RuntimeError(reason) @@ -278,6 +350,7 @@ def _authorize_upload( account, "the unattended grant for this payload no longer holds; upload refused", caller=caller, + withheld=True, ) # The same re-read, for the other precondition a scheduled transcript upload # stands on. The grant above answers "does the owner still want this sent"; @@ -299,6 +372,7 @@ def _authorize_upload( account, f"a scheduled transcript upload is no longer allowed here: {blocked_now}", caller=caller, + withheld=True, ) # Last, and deliberately after every other check: app teardown. A worker # thread cannot be killed, so cancelling the loop's await leaves the archive diff --git a/src/kiro_crew/apps/builtins/aws_control/hooks.py b/src/kiro_crew/apps/builtins/aws_control/hooks.py index f67492c409e6..736f78356d0b 100644 --- a/src/kiro_crew/apps/builtins/aws_control/hooks.py +++ b/src/kiro_crew/apps/builtins/aws_control/hooks.py @@ -1,18 +1,21 @@ """Lifecycle hooks — the nightly backup loop. -One background task, started on enable, that wakes every half hour and runs -the snapshot backup when it is due (nightly toggle on AND >23 h since the -last run AND not inside the retry backoff a run of failed attempts earns -- -see ``backup.due_for_nightly``). Every AWS-reaching step keeps -the same guards the HTTP path has: consent fails closed (a silent skip plus -a log line, never an unconfirmed charge), and the drive is tag-discovered -per run rather than trusted from memory. - -The wake interval is a due-CHECK interval and never a retry interval. A failed -attempt is recorded, by ``_failed_attempt``, so the due-check can tell a fault it -has already met from a first one; without that record the loop re-attempted a -deterministic failure on every wake forever, because recording only successes left -the state file with nothing to distinguish "never ran" from "keeps breaking". +One background task, started on enable, that wakes every half hour and CLAIMS +the snapshot backup through the Job SDK when it is due (nightly toggle on AND +>23 h since the last run AND not inside the retry backoff -- see +``backup.due_for_nightly``). The due-check keeps the same guards the HTTP path +has: consent fails closed (a silent skip plus a log line, never an unconfirmed +charge), and the account is resolved through the same healthy-first policy. + +Claiming the run through ``sdk.start_async`` rather than running the backup on +a bare thread is what makes a nightly a RECORDED run: it dedupes against a +manual click for the same account (both share the SDK's ``(kind, dedupe_key)`` +index, so they do not both do the paid upload), and it is visible to the Backup +row, which derives its busy state from SDK records. The shared runner re-resolves +the drive per run and applies the unattended gates, because the loop names +``CALLER_SCHEDULED`` in the start's ``params``. + +The wake interval is a due-CHECK interval and never a retry interval. The loop runs against the REGISTRY DEFAULT account only — the same account the consent card confirms, resolved through the same healthy-first policy, so @@ -28,94 +31,71 @@ from __future__ import annotations import asyncio -import functools import logging from typing import Any from kiro_crew import aws_consent from kiro_crew.apps.builtins.aws_control.backend import accounts as accounts_mod from kiro_crew.apps.builtins.aws_control.backend import backup as backup_mod -from kiro_crew.apps.builtins.aws_control.backend import storage as storage_mod -from kiro_crew.sel import sel +from kiro_crew.apps.job_sdk import JobError, UnknownJobKind, get_sdk logger = logging.getLogger(__name__) _CHECK_INTERVAL_SECS = 30 * 60 -_task: asyncio.Task[None] | None = None +def _nightly_setup_failure(account: str, kind: str, exc: BaseException, witness: Any) -> None: + """Record and audit one scheduled wake that failed BEFORE a run was claimed. + + The loop resolves the drive itself, before it claims any run (see + :func:`_run_once`), so a lookup that *raises* there never reaches the runner's + own ``failed`` handler -- it would escape the wake into :func:`_loop`'s + catch-all warning with no backoff and no audit, the one failure shape that + re-attempts every wake forever. This is the loop-side equivalent of the + runner's post-claim ``except`` branch, and it is deliberately the SAME two + writes, through the SAME helpers the runner uses: the ``failed`` SEL audit via + :func:`backup._nightly_audit` under :func:`backup._nightly_subject` -- so a + setup failure and a push failure for one kind never land under different + subjects or through a second copy of the audit call -- and + :func:`backup.record_nightly_failure`, so ``due_for_nightly`` can tell a fault + it has already met from a new one. + + ``witness`` is :func:`backup.nightly_run_witness` read BEFORE the lookup, so the + recorder refuses to write over a run slot that moved since. Never raises: it is + already on a failed-wake path, and the audit is best-effort. + """ + backup_mod._nightly_audit( + "backup_nightly", backup_mod._nightly_subject(kind), "failed", error=str(exc) + ) + backup_mod.record_nightly_failure(account, kind, str(exc), run_witness=witness) -def _audit(operation: str, resources: str, outcome: str, *, error: str = "") -> None: - """SEL record for an UNATTENDED backup step. - The HTTP handlers get their audit from the dashboard layer; this loop has no - request, so without this the only unattended S3 mutation in the app would be - the one operation with no trail. Best-effort by the same rule the handlers - use: an audit failure must never abort the backup. - """ - try: - sel().log_api_access( - caller="aws-control-nightly", - operation=f"aws_control.{operation}", - outcome=outcome, - source=backup_mod.APP_NAME, - resources=resources[:200], - error=error[:200], - ) - except Exception: - logger.debug("aws-control nightly SEL audit failed", exc_info=True) - - -async def _note_shared_drive(profile: str, region: str, bucket: str, account: str) -> None: - """Record that another install also backs up here. Then carry on. - - A NOTICE, not a gate, and that is a deliberate departure from the reported - issue's own suggestion that the loop refuse. Refusing would make the drive - single-owner, and single-owner scheduling means exactly one machine keeps - getting backed up while the other silently stops -- discovering that at - restore time is worse than the unattributed pile this change replaces. Once - the keys carry an install id there is nothing left to collide: two installs - write to two prefixes, and two machines each keeping their own memory backed - up is what the owner asked for by pointing both at one account. - - So the value here is EVIDENCE, and specifically evidence a later panel view - cannot reconstruct. The console's ``others`` count is LIVE: it says what the - drive looks like when a human opens the page. This record says what the - UNATTENDED run observed at the moment it spent the owner's money -- and the - nightly is the one path in this app that spends it with nobody present, so "at - this run, the drive already held another install's archives" is an audit fact - about that spend rather than a line for someone to read. The consumer is a human - after the fact, which is what the whole SEL trail is for. - - ONE list call, against the snapshot prefix only. This loop uploads snapshots - and nothing else, so sweeping the sessions prefix too would have cost a second - paid call per scheduled run to answer a question about a run that is not - happening. - - Never raises. A listing failure must not stop the backup it was only - annotating; the outcome is one less log line, not a missed nightly. +def _nightly_setup_cancelled(kind: str) -> None: + """Audit one scheduled wake that was CANCELLED before a run was claimed. + + A cancel is teardown, not a fault: unlike :func:`_nightly_setup_failure` this + writes only the ``cancelled`` SEL record and does NOT back off, so an + interrupted wake leaves a trail that it was stopped without poisoning the next + one. It mirrors the per-kind ``cancelled`` audit the loop-side setup emitted + before the backup moved to the Job SDK, kept so teardown during the drive + lookup stays visible in the SEL trail -- through the same + :func:`backup._nightly_audit` the fault path uses, so there is one audit call, + not a second copy. Never raises: the audit is best-effort and the + ``CancelledError`` must propagate regardless. """ - try: - others = await asyncio.to_thread( - backup_mod.other_install_ids, profile, region, bucket, account=account - ) - except Exception: - logger.debug("aws-control nightly: shared-drive check failed", exc_info=True) - return - if not others: - return - logger.warning( - "aws-control nightly: %d other install(s) also back up to this drive (%s); " - "each writes under its own prefix, so this run proceeds -- restore attributes " - "archives by that prefix", - len(others), - ", ".join(others[:5]), - ) - _audit("backup_shared_drive", f"installs={len(others)}", "invoked") + backup_mod._nightly_audit("backup_nightly", backup_mod._nightly_subject(kind), "cancelled") + + +_task: asyncio.Task[None] | None = None + +async def _run_once(sdk: Any) -> None: + """One due-check + claim of each due backup through the Job SDK. -async def _run_once() -> None: - """One due-check + backup attempt. Every failure is a log line, not a crash.""" + Takes the SDK rather than reaching for it, so the due-check and claim stay + testable without a live app context. The loop fetches the live SDK each wake + (see :func:`_loop`); a direct caller passes one in. + """ # Same resolution the consent card and the HTTP handlers use, so the key this # unattended loop runs under is the key the grant was recorded for. A raw # registry-default read would pick an unhealthy default over the account's @@ -170,13 +150,8 @@ async def _run_once() -> None: ) if not allowed: return # refuse_and_log already logged + audited - # The kinds this wake is actually for, derived once. The shared setup below - # can fail before any push, and a failure there must name the kinds that were - # due rather than one fixed kind: a transcripts-only wake reaches the same - # block, so a hardcoded `backup/snapshots` would record a snapshot that was - # never due, and the SEL trail is append-only. Deriving the list here also - # means the subjects a failure is audited against and the kinds actually - # pushed below cannot drift apart -- they are the same list. + # The kinds this wake is actually for. A transcripts-only wake must not claim + # a snapshot run, so each due bit gates its own kind independently. due_kinds = [ kind for kind, is_due in ( @@ -185,156 +160,100 @@ async def _run_once() -> None: ) if is_due ] - # The run slot's identity per due kind, read BEFORE anything is attempted. This is - # the window a failure write has to be judged against: a run recorded inside it is - # positive evidence that backups are reaching the drive, and the recorder refuses to - # write a failure over it. Captured here rather than inside the handler because by - # then the window has closed. + # Check the drive BEFORE claiming any run, as the loop did before this change. + # The drive is tag-discovered per wake, not trusted from memory; until one + # exists there is nowhere to push, so no run is claimed. Claiming one anyway + # would leave a `done` SDK run on the Backup row every 30 minutes for a backup + # that uploaded nothing -- and that newest `done` run would clear an owner's + # earlier `lastFailed` (e.g. their own "no drive yet" click). A missing drive + # is "nothing to attempt yet", so the wake simply returns, recording nothing. + # + # A drive lookup that RAISES is a different fact from one that returns no drive. + # It is a fault, and because it happens here -- before any run is claimed -- the + # runner's own failure handler never sees it. Left unhandled it escapes into + # `_loop`'s catch-all warning with no backoff and no audit, so a deterministically + # failing lookup re-attempts every wake forever (the exact shape + # `record_nightly_failure` exists to close). So each due kind's run slot is + # witnessed BEFORE the lookup -- the window a failure write is judged against, + # which has closed by the time the handler runs -- and a raise records+audits a + # `failed` attempt per due kind, matching the runner's post-claim branch. A + # CancelledError is teardown, not a fault, so it audits a `cancelled` trail per + # due kind -- so the interrupt stays visible in the SEL record -- but does NOT + # back off. witnesses = { kind: await asyncio.to_thread(backup_mod.nightly_run_witness, account, kind) for kind in due_kinds } try: - bucket = await asyncio.to_thread(storage_mod.find_drive, profile, region, account=account) - if not bucket: - logger.info("aws-control nightly: no drive bucket yet; skipping") - return - await _note_shared_drive(profile, region, bucket, account) + bucket = await asyncio.to_thread( + backup_mod.storage.find_drive, profile, region, account=account + ) except asyncio.CancelledError: for kind in due_kinds: - _audit("backup_nightly", _audit_subject(kind), "cancelled") + _nightly_setup_cancelled(kind) raise except Exception as exc: for kind in due_kinds: - await _failed_attempt(kind, account, exc, witnesses[kind]) - logger.warning("aws-control nightly backup failed", exc_info=True) + await asyncio.to_thread(_nightly_setup_failure, account, kind, exc, witnesses[kind]) + logger.warning("aws-control nightly: drive lookup failed", exc_info=True) return + if not bucket: + logger.info("aws-control nightly: no drive yet; skipping") + return + # Claim each due kind through the Job SDK, the SAME call the owner click makes, + # rather than running the backup on a bare thread. Claiming through the SDK + # is what makes the run recorded and dedupable: + # + # * dedupe_key=account makes a nightly run and a manual click for one account + # adopt one another through the SDK's (kind, dedupe_key) index, so they do + # not both do the paid upload. + # * a claimed run is what the Backup row reads, so a nightly is visible like + # any other run instead of leaving the row idle while it uploads. + # * params names CALLER_SCHEDULED so the shared runner applies the + # unattended-only gates and attributes the spend to the schedule, not to a + # dashboard owner who is not present. + # + # Fire-and-forget: start_async returns once the run is claimed, and the SDK + # record carries the outcome (done / failed) that the row and the audit trail + # read. A refused start (mid-teardown, or no runner registered) is a logged + # skip, not a crash -- the next wake tries again. for kind in due_kinds: - await _push_nightly(kind, account, profile, region, bucket) - - -def _audit_subject(kind: str) -> str: - """The SEL subject for one backup kind. - - One spelling, used by the shared setup's failure handlers and by - :func:`_push_nightly` alike. Two copies of this expression is how a run's - setup failure and its push end up filed under different subjects for the - same kind, which is exactly the misattribution this exists to prevent. - """ - return f"backup/{backup_mod.KIND_SUBPATHS[kind]}" - - -async def _failed_attempt( - kind: str, - account: str, - exc: BaseException, - run_witness: Any, -) -> None: - """Audit one kind's failed unattended attempt AND record it. One spelling. - - The two live together because they are two halves of the same fact, and they had - different fates before this existed: the audit went to SEL, where a human reads - it after the event, and nothing at all went to state, where the loop reads it on - the next wake. So a nightly failing deterministically produced a growing pile of - ``failed`` audit records and a due-check that could not see any of them, and - re-attempted every half hour indefinitely -- staging the whole data home into a - fresh temporary directory each time, and burying every other warning in the - gateway log at that cadence. - - ONE helper rather than the same pair written at each site, for the reason - :func:`_audit_subject` itself gives: this module has two places a nightly attempt - can fail -- the shared setup, and each kind's own push -- and two copies is how - one of them ends up auditing a failure it never counted, or counting one it never - audited. The backoff would then depend on WHICH way the run broke. - - ``run_witness`` is ``backup.nightly_run_witness`` read BEFORE the attempt began, and - the recorder refuses to write when the run slot has moved since. Every caller reads - it at the top of its own attempt rather than here, because by the time this helper - runs the attempt is already over and the window it has to witness has closed. - - Cancellation deliberately does NOT come here. A cancelled attempt is teardown, - not a fault: the owner disabled the app or the gateway is stopping, and counting - that as a failed attempt would have a clean shutdown push the next night out. - Those branches keep their own audit call and record nothing. - """ - _audit("backup_nightly", _audit_subject(kind), "failed", error=str(exc)) - # Off the loop, like every other state read in this module: the recorder takes - # the sidecar lock and rewrites the document, which is real blocking file I/O. - await asyncio.to_thread( - functools.partial( - backup_mod.record_nightly_failure, - account, - kind, - str(exc), - run_witness=run_witness, - ) - ) - - -async def _push_nightly(kind: str, account: str, profile: str, region: str, bucket: str) -> None: - """Push one due nightly kind, audited around the call. - - Each kind gets its OWN try/except rather than sharing one. The two payloads - have nothing in common but the drive they land in, so a snapshot that fails - must not cost the transcripts their window, and the reverse. A shared handler - would turn one failure into two skipped nights. - """ - subject = _audit_subject(kind) - # Read before the attempt, for the reason `_run_once` gives at its own capture: - # the window this witnesses is the attempt's whole duration, so it cannot be - # read from the handler after the attempt has ended. - run_witness = await asyncio.to_thread(backup_mod.nightly_run_witness, account, kind) - runner = ( - backup_mod.run_snapshot_backup - if kind == backup_mod.KIND_SNAPSHOT - else backup_mod.run_sessions_backup - ) - try: - # The nightly path never touches an HTTP handler, so the audit the - # dashboard layer adds to every owner-driven mutation is simply absent - # here -- an unattended export would leave no SEL trace of having run, - # succeeded or failed. Emit the same three-part record the handlers do, - # around the call, so the trail does not depend on who triggered it. - _audit("backup_nightly", subject, "invoked") - record = await asyncio.to_thread( - functools.partial( - runner, - account, - profile, - region, - bucket, - # Nobody is at the keyboard at 03:00. An upload refused here must - # not be recorded against the dashboard owner, which is what a - # hardcoded interactive caller inside the gate would have done. - caller=backup_mod.CALLER_SCHEDULED, - ) - ) - if record.get("uploaded") is False: - # A run that found the tree unchanged sent nothing, so recording it as a - # push would put a SEL entry and a log line against a key no bytes reached - # tonight -- indistinguishable, to whoever reads the audit trail, from an - # ordinary upload. The key is still named because it is the archive this - # run stood on: it is what the drive still holds for this kind. - _audit("backup_nightly", str(record.get("key", "")), "unchanged") - logger.info( - "aws-control nightly backup: tree unchanged since %s, nothing uploaded", - record.get("key", ""), + try: + await sdk.start_async( + kind, + dedupe_key=account, + params={ + backup_mod.JOB_PARAM_CALLER: backup_mod.CALLER_SCHEDULED, + # The profile/region this loop JUST verified live + # (`probe_identity(use_cache=False)` above). The runner + # re-resolves per run, but its sync cache returns None once the + # snapshot is past the TTL -- and `start_async` is + # fire-and-forget, so the runner resolves later than this check. + # Carrying the verified pair lets the scheduled path fall back to + # it rather than audit a just-verified nightly as failed. The + # live account re-check in `_authorize_upload` still guards wrong + # accounts. + backup_mod.JOB_PARAM_PROFILE: profile, + backup_mod.JOB_PARAM_REGION: region, + }, ) - else: - _audit("backup_nightly", str(record.get("key", "")), "succeeded") - logger.info("aws-control nightly backup pushed: %s", record.get("key", "")) - except asyncio.CancelledError: - _audit("backup_nightly", subject, "cancelled") - raise - except Exception as exc: - await _failed_attempt(kind, account, exc, run_witness) - logger.warning("aws-control nightly backup failed: %s", kind, exc_info=True) + except (JobError, UnknownJobKind): + logger.warning("aws-control nightly: could not claim %s run", kind, exc_info=True) async def _loop() -> None: while True: try: - await _run_once() + # Fetch the LIVE SDK each wake rather than capturing one: a re-enable + # builds a fresh AppContext and therefore a fresh JobSDK, and the loop + # outlives that cycle. `get_sdk` returns whichever SDK is registered + # under this app right now (the same one the route reads), so the loop + # never holds a stale handle. Absent means the `jobs` permission is not + # granted; `_register_job_runners` already logged that, so here it is a + # quiet skip until the next wake. + sdk = get_sdk(backup_mod.APP_NAME) + if sdk is not None: + await _run_once(sdk) except asyncio.CancelledError: raise except Exception: diff --git a/test/test_aws_control_app.py b/test/test_aws_control_app.py index 82169d3ee974..65eeba7ee0a6 100644 --- a/test/test_aws_control_app.py +++ b/test/test_aws_control_app.py @@ -672,9 +672,10 @@ async def refuse_and_log(service: str, *, profile: str, region: str) -> bool: AsyncMock(return_value=aws_consent.Identity(ok=True, account=self.ACCOUNT)), ), mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=True), + mock.patch.object(hooks.backup_mod, "due_for_sessions_nightly", return_value=False), mock.patch.object(hooks.aws_consent, "refuse_and_log", refuse_and_log), ): - asyncio.run(hooks._run_once()) + asyncio.run(hooks._run_once(mock.Mock(start_async=AsyncMock()))) operation = asyncio.run(accounts_mod.resolve_account_profile(self.ACCOUNT)) assert gated == [("good", "eu-west-1")] @@ -687,18 +688,18 @@ def test_the_nightly_loop_skips_when_no_key_is_healthy(self): _registry([_entry("broken", account=self.ACCOUNT)], default="broken"), {"broken": _identity(False, detail="ExpiredToken")}, ) + sdk = mock.Mock(start_async=AsyncMock()) with ( mock.patch.object(accounts_mod, "list_accounts", AsyncMock(return_value=snapshot)), mock.patch.object(hooks.aws_consent, "probe_identity") as probe, mock.patch.object(hooks.backup_mod, "due_for_nightly") as due, - mock.patch.object(hooks, "_audit") as audit, ): - asyncio.run(hooks._run_once()) - # No probe, no due-check, no consent check, no audit: there is no key to + asyncio.run(hooks._run_once(sdk)) + # No probe, no due-check, no consent check, no claim: there is no key to # run under, so the loop stops before it can name one. probe.assert_not_called() due.assert_not_called() - audit.assert_not_called() + sdk.start_async.assert_not_called() def test_the_voice_services_do_not_touch_the_account_snapshot(self): """Polly and Transcribe read their own config; only s3/ce use the registry.""" @@ -2370,9 +2371,10 @@ def test_reenabling_clears_the_teardown_signal(self): backup.clear_stop() assert not backup._STOP.is_set() - def test_nightly_backup_emits_sel_records(self): + def test_nightly_backup_claims_the_run_through_the_sdk(self): from kiro_crew.apps.builtins.aws_control import hooks + sdk = mock.Mock(start_async=AsyncMock(return_value="run-1")) with ( mock.patch.object( hooks.accounts_mod, @@ -2385,25 +2387,30 @@ def test_nightly_backup_emits_sel_records(self): AsyncMock(return_value=aws_consent.Identity(ok=True, account=ACCOUNT)), ), mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=True), + mock.patch.object(hooks.backup_mod, "due_for_sessions_nightly", return_value=False), mock.patch.object(hooks.aws_consent, "refuse_and_log", AsyncMock(return_value=True)), - mock.patch.object( - hooks.storage_mod, "find_drive", return_value="kirocrew-drive-abc123def456" - ), - mock.patch.object( - hooks.backup_mod, - "run_snapshot_backup", - return_value={"key": "snapshots/x.tar.gz"}, - ), - mock.patch.object(hooks, "_audit") as audit, + mock.patch.object(hooks.backup_mod.storage, "find_drive", return_value="bkt"), ): - asyncio.run(hooks._run_once()) - - outcomes = [c.args[2] for c in audit.call_args_list] - assert "invoked" in outcomes and "succeeded" in outcomes + asyncio.run(hooks._run_once(sdk)) + + # The run is a RECORDED SDK run, claimed under the + # account and the scheduled caller. The run's own SEL audit is the + # runner's job, pinned in test_aws_control_backup_job.py. + sdk.start_async.assert_awaited_once() + call = sdk.start_async.await_args + assert call.args[0] == hooks.backup_mod.KIND_SNAPSHOT + assert call.kwargs["dedupe_key"] == ACCOUNT + assert call.kwargs["params"] == { + hooks.backup_mod.JOB_PARAM_CALLER: hooks.backup_mod.CALLER_SCHEDULED, + hooks.backup_mod.JOB_PARAM_PROFILE: "p", + hooks.backup_mod.JOB_PARAM_REGION: "us-west-2", + } - def test_nightly_backup_failure_is_audited(self): + def test_a_refused_claim_does_not_crash_the_loop(self): from kiro_crew.apps.builtins.aws_control import hooks + from kiro_crew.apps.job_sdk import JobError + sdk = mock.Mock(start_async=AsyncMock(side_effect=JobError("shutting down"))) with ( mock.patch.object( hooks.accounts_mod, @@ -2416,19 +2423,13 @@ def test_nightly_backup_failure_is_audited(self): AsyncMock(return_value=aws_consent.Identity(ok=True, account=ACCOUNT)), ), mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=True), + mock.patch.object(hooks.backup_mod, "due_for_sessions_nightly", return_value=False), mock.patch.object(hooks.aws_consent, "refuse_and_log", AsyncMock(return_value=True)), - mock.patch.object( - hooks.storage_mod, "find_drive", return_value="kirocrew-drive-abc123def456" - ), - mock.patch.object( - hooks.backup_mod, "run_snapshot_backup", side_effect=RuntimeError("boom") - ), - mock.patch.object(hooks, "_audit") as audit, + mock.patch.object(hooks.backup_mod.storage, "find_drive", return_value="bkt"), ): - asyncio.run(hooks._run_once()) # swallowed, not raised + asyncio.run(hooks._run_once(sdk)) # swallowed, not raised - outcomes = [c.args[2] for c in audit.call_args_list] - assert "failed" in outcomes + sdk.start_async.assert_awaited_once() class TestRound23Junctions: diff --git a/test/test_aws_control_backup_composition_contract.py b/test/test_aws_control_backup_composition_contract.py index 8aec87c86cb1..059ae9adeb74 100644 --- a/test/test_aws_control_backup_composition_contract.py +++ b/test/test_aws_control_backup_composition_contract.py @@ -86,6 +86,9 @@ "INSTALL_KEY", "IO", "JOB_KINDS", + "JOB_PARAM_CALLER", + "JOB_PARAM_PROFILE", + "JOB_PARAM_REGION", "KEY_SEP", "KIND_SESSIONS", "KIND_SNAPSHOT", @@ -120,6 +123,7 @@ "SESSIONS_LAYER_B_SCOPE_WITH_CONVERSATIONS", "STAGING_NAME_MAX_BYTES", "STATE_DIR_LEAF", + "ScheduledUploadWithheld", "UnprovenArchive", "_AUTHORIZE_TIMEOUT_SECS", "_CAN_PIN_TRAVERSAL", diff --git a/test/test_aws_control_backup_job.py b/test/test_aws_control_backup_job.py index d3eba4f97483..cb95d12bc2aa 100644 --- a/test/test_aws_control_backup_job.py +++ b/test/test_aws_control_backup_job.py @@ -202,18 +202,19 @@ def test_a_refusal_is_attributed_to_whoever_triggered_it( def test_the_two_entry_points_pass_different_callers(self) -> None: """The constants only matter if the call sites actually differ. - Pinned by reading the sources: the Job SDK runner exists because an owner - asked through an owner-gated route, and the nightly loop has no owner at - all. If both ever named the same constant, the field would be decoration. + One registered runner now serves both callers: the owner click carries no + caller param (the owner default), and the nightly loop names + ``CALLER_SCHEDULED`` in the start's params. If both ever resolved to the + same constant, the field would be decoration. """ assert backup.CALLER_OWNER != backup.CALLER_SCHEDULED - assert "CALLER_OWNER" in inspect.getsource(backup.make_job_runner) - # The nightly names its caller in `_push_nightly`, the per-kind helper - # `_run_once` delegates each due push to, rather than in `_run_once` - # itself. What the loop actually passes is pinned behaviourally in - # test_aws_control_nightly_sessions.py, which reads the runner's kwargs; - # this half only asserts the two entry points name different constants. - assert "CALLER_SCHEDULED" in inspect.getsource(hooks._push_nightly) + # The runner names BOTH: it defaults to the owner and switches to the + # schedule when the caller param says so. + runner_src = inspect.getsource(backup.make_job_runner) + assert "CALLER_OWNER" in runner_src + assert "CALLER_SCHEDULED" in runner_src + # The loop is the one that states the scheduled caller, in `_run_once`. + assert "CALLER_SCHEDULED" in inspect.getsource(hooks._run_once) def test_teardown_is_recorded_but_not_as_an_access_denial(self) -> None: """Every refusal leaves a record; only access decisions are denials. @@ -490,13 +491,35 @@ def _post_run(kind: str, *, sdk: object | None = None, start=None): def _await_terminal(sdk: job_sdk.JobSDK, run_id: str, timeout: float = 5.0) -> job_sdk.JobRun: - """Block until the worker has written its terminal record.""" + """Block until the worker has written its terminal record AND released its slot. + + The terminal record is written before the live-table entry and its dedupe key + are dropped (``job_sdk`` writes the terminal status, then under its lock pops + ``_live`` and the ``(kind, dedupe_key)`` claim). A caller that returned on the + terminal record alone could fire a second ``start`` for the same + ``(kind, dedupe_key)`` inside that window, where the still-held claim makes the + SDK ADOPT the finished run instead of starting a fresh one -- so a test driving + two sequential runs of one account would see the second silently fold into the + first. Waiting for the run id to leave ``_live`` closes that window: the pop of + ``_live`` and the pop of the dedupe key happen in the same locked block, so once + the run is gone from the live set the key is free and the next ``start`` begins + a new run. ``_live`` is read directly because it is the authoritative in-process + signal for the claim; ``list_active`` reads the durable store, which the terminal + write has already cleared, so it cannot witness this window. + """ deadline = time.monotonic() + timeout + settled: job_sdk.JobRun | None = None while time.monotonic() < deadline: run = sdk.get(run_id) if run is not None and run.is_terminal: - return run + settled = run + if run_id not in sdk._live: + return run time.sleep(0.01) + if settled is not None: + raise AssertionError( + f"run reached {settled.status} but its slot was not released within {timeout}s" + ) run = sdk.get(run_id) raise AssertionError(f"run did not settle within {timeout}s: {run}") @@ -1018,3 +1041,388 @@ def test_the_status_route_still_serves_the_ledger(self): ) assert resp.status == 200 assert _payload(resp)["runs"] == entry + + +def _scheduled_params() -> dict[str, str]: + """The params a nightly start carries so the runner takes the scheduled path.""" + return {backup.JOB_PARAM_CALLER: backup.CALLER_SCHEDULED} + + +class TestRunnerReadsTheCallerFromParams: + """One registered runner serves both callers. The caller role rides in the + start's params; absent means the owner path.""" + + def test_a_scheduled_start_runs_as_the_schedule(self, sdk): + # The nightly loop names CALLER_SCHEDULED in params. The runner must pass + # that through to the gate and the work, so an unattended push is never + # attributed to the dashboard owner. + with ( + _resolvable(), + mock.patch.object(backup, "_authorize_upload", return_value=None) as gate, + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET), + mock.patch.object(backup, "other_install_ids", return_value=[]), + mock.patch.object(backup, "run_snapshot_backup") as work, + ): + run_id = sdk.start(backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT, params=_scheduled_params()) + run = _await_terminal(sdk, run_id) + assert run.status == job_sdk.DONE + assert gate.call_args.kwargs["caller"] == backup.CALLER_SCHEDULED + assert work.call_args.kwargs["caller"] == backup.CALLER_SCHEDULED + + def test_a_start_with_no_caller_param_runs_as_the_owner(self, sdk): + # The manual route carries no caller param. Absent is the owner path -- + # the safe default, and the behaviour every pre-existing runner test + # already relies on. + with ( + _resolvable(), + mock.patch.object(backup, "_authorize_upload", return_value=None), + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET), + mock.patch.object(backup, "run_snapshot_backup") as work, + ): + run_id = sdk.start(backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT) + _await_terminal(sdk, run_id) + assert work.call_args.kwargs["caller"] == backup.CALLER_OWNER + + +class TestScheduledPathFallsBackToTheVerifiedProfile: + """Finding 2: `resolve_account_profile_cached` serves only from the warm + snapshot and returns None once it is past `_PROBE_TTL_SECS`. `start_async` is + fire-and-forget, so the runner resolves later than the loop's live probe; a gap + longer than the TTL would audit a just-verified nightly as failed. The loop + names the (profile, region) it freshly probed in params, so a scheduled run + falls back to it when the cache has expired.""" + + def test_a_scheduled_run_uses_the_verified_profile_when_the_cache_expired(self, sdk): + # The cache answers None (snapshot past the TTL), but the scheduled start + # carries the profile/region the loop just verified. The run must proceed + # on those rather than fail "no working connection". + verified = { + backup.JOB_PARAM_CALLER: backup.CALLER_SCHEDULED, + backup.JOB_PARAM_PROFILE: PROFILE, + backup.JOB_PARAM_REGION: REGION, + } + with ( + mock.patch.object(accounts_mod, "resolve_account_profile_cached", return_value=None), + mock.patch.object(backup, "_authorize_upload", return_value=None) as gate, + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET) as find, + mock.patch.object(backup, "other_install_ids", return_value=[]), + mock.patch.object(backup, "run_snapshot_backup") as work, + ): + run_id = sdk.start(backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT, params=verified) + run = _await_terminal(sdk, run_id) + assert run.status == job_sdk.DONE + # The verified pair flowed into the live-account gate, the drive discovery + # and the work -- the bucket is still re-discovered, not carried. + assert gate.call_args.args[1:] == (PROFILE, REGION) + find.assert_called_once_with(PROFILE, REGION, account=ACCOUNT) + assert work.call_args.args[1:3] == (PROFILE, REGION) + + def test_the_owner_path_does_not_fall_back_on_an_expired_cache(self, sdk): + # The fallback is scheduled-only: an owner run carries no profile/region + # param and its route pre-flight has just warmed the snapshot, so an + # expired cache for it is the honest "reconnect" failure, not a fallback. + with ( + mock.patch.object(accounts_mod, "resolve_account_profile_cached", return_value=None), + mock.patch.object(backup.storage, "find_drive") as find, + mock.patch.object(backup, "run_snapshot_backup") as work, + ): + run_id = sdk.start(backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT) + run = _await_terminal(sdk, run_id) + assert run.status == job_sdk.FAILED + assert "no working connection" in run.error + find.assert_not_called() + work.assert_not_called() + + def test_a_scheduled_run_prefers_the_live_cache_when_it_is_warm(self, sdk): + # When the cache IS warm the fallback must not shadow it: the per-run + # re-resolve is still the primary source, and the carried pair is only a + # fallback for the expired case. + live = ("live-prof", "eu-central-1") + verified = { + backup.JOB_PARAM_CALLER: backup.CALLER_SCHEDULED, + backup.JOB_PARAM_PROFILE: PROFILE, + backup.JOB_PARAM_REGION: REGION, + } + with ( + mock.patch.object(accounts_mod, "resolve_account_profile_cached", return_value=live), + mock.patch.object(backup, "_authorize_upload", return_value=None) as gate, + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET), + mock.patch.object(backup, "other_install_ids", return_value=[]), + mock.patch.object(backup, "run_snapshot_backup") as work, + ): + run_id = sdk.start(backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT, params=verified) + run = _await_terminal(sdk, run_id) + assert run.status == job_sdk.DONE + assert gate.call_args.args[1:] == live + assert work.call_args.args[1:3] == live + + +class TestScheduledFailureFeedsTheBackoff: + """A nightly that fails must be recorded so the loop's due-check backs off: + without this record a deterministic fault re-attempts every half hour. An + owner failure is NOT recorded -- someone is present to see it.""" + + def test_a_failed_scheduled_run_records_the_attempt(self, sdk): + backup.set_nightly(ACCOUNT, True) + + def _one_failed_wake(): + with ( + _resolvable(), + mock.patch.object(backup, "_authorize_upload", return_value=None), + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET), + mock.patch.object(backup, "other_install_ids", return_value=[]), + mock.patch.object( + backup, "run_snapshot_backup", side_effect=RuntimeError("push broke") + ), + ): + run_id = sdk.start( + backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT, params=_scheduled_params() + ) + return _await_terminal(sdk, run_id) + + run = _one_failed_wake() + # The run itself is still recorded failed -- the backoff is in addition to, + # not instead of, the honest terminal state the Backup row reads. + assert run.status == job_sdk.FAILED + snap = backup.nightly_failures(ACCOUNT).get(backup.KIND_SNAPSHOT, {}) + assert snap.get("consecutive") == 1 + # The error text is carried, so an operator reading the record learns WHAT + # keeps failing rather than only that something does. + assert "push broke" in snap.get("error", "") + + # THE regression this backoff exists to prevent. The first retry delay row + # is zero by design, so a single failure stays DUE; the second failing wake + # is the one that must withhold the next attempt. Without the record feeding + # the due-check this reads True on every wake -- the loop re-attempting a + # deterministic fault every half hour. + run = _one_failed_wake() + assert run.status == job_sdk.FAILED + assert backup.nightly_failures(ACCOUNT)[backup.KIND_SNAPSHOT]["consecutive"] == 2 + assert backup.due_for_nightly(ACCOUNT) is False + + def test_a_successful_scheduled_run_clears_the_backoff(self, sdk): + # The other half of the loop the backoff serves: once the fault clears, a + # completing run must make the account eligible again rather than serving + # out a wait measured for a fault that is over. The clear lives inside the + # real `_record_run`, so the runner is driven to it through a success. + backup.set_nightly(ACCOUNT, True) + for _ in range(3): + backup.record_nightly_failure( + ACCOUNT, + backup.KIND_SNAPSHOT, + "eio", + run_witness=backup.nightly_run_witness(ACCOUNT, backup.KIND_SNAPSHOT), + ) + assert backup.due_for_nightly(ACCOUNT) is False + + def _completing(account, profile, region, bucket, *, caller): + return backup._record_run( + account, backup.KIND_SNAPSHOT, "snapshots/i/a.tar.gz", 9, "fp", "v1" + ) + + with ( + _resolvable(), + mock.patch.object(backup, "_authorize_upload", return_value=None), + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET), + mock.patch.object(backup, "other_install_ids", return_value=[]), + mock.patch.object(backup, "run_snapshot_backup", side_effect=_completing), + ): + run_id = sdk.start(backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT, params=_scheduled_params()) + run = _await_terminal(sdk, run_id) + assert run.status == job_sdk.DONE + assert backup.nightly_failures(ACCOUNT) == {} + + def test_an_owner_failure_does_not_feed_the_backoff(self, sdk): + with ( + _resolvable(), + mock.patch.object(backup, "_authorize_upload", return_value=None), + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET), + mock.patch.object( + backup, "run_snapshot_backup", side_effect=RuntimeError("push broke") + ), + ): + run_id = sdk.start(backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT) + run = _await_terminal(sdk, run_id) + assert run.status == job_sdk.FAILED + assert backup.nightly_failures(ACCOUNT).get(backup.KIND_SNAPSHOT) is None + + def test_a_teardown_does_not_feed_the_backoff(self, sdk): + # App disable / gateway stop surfaces as a failure while _STOP is set. + # Counting it would let a restart push the next night out, so it must not + # be recorded even though the caller is the schedule. + with ( + _resolvable(), + mock.patch.object(backup, "_authorize_upload", return_value=None), + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET), + mock.patch.object(backup, "other_install_ids", return_value=[]), + mock.patch.object( + backup, "run_snapshot_backup", side_effect=RuntimeError("shutting down") + ), + ): + backup.signal_stop() + try: + run_id = sdk.start( + backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT, params=_scheduled_params() + ) + run = _await_terminal(sdk, run_id) + finally: + backup.clear_stop() + assert run.status == job_sdk.FAILED + assert backup.nightly_failures(ACCOUNT).get(backup.KIND_SNAPSHOT) is None + + def test_a_withheld_scheduled_run_does_not_feed_the_backoff(self, sdk): + # Finding 1: the owner withdrawing the unattended grant (or turning + # redaction on) mid-build raises ScheduledUploadWithheld from the gate -- + # a policy "do not upload", not a fault. Counting it would grow the + # schedule's backoff for a run the owner chose to stop. This is the rule + # an ADOPTED run keeps: an owner's "Back up now" during a nightly adopts + # the in-flight scheduled run (CALLER_SCHEDULED, params do not merge), so + # if that run met the gate the owner would otherwise see a backoff grow + # for a failure they did not cause. + from kiro_crew.apps.builtins.aws_control.backend.backup_parts.uploads import ( + ScheduledUploadWithheld, + ) + + with ( + _resolvable(), + mock.patch.object( + backup, + "_authorize_upload", + side_effect=ScheduledUploadWithheld( + "the unattended grant for this payload no longer holds; upload refused" + ), + ), + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET), + mock.patch.object(backup, "other_install_ids", return_value=[]), + mock.patch.object(backup, "run_snapshot_backup") as work, + ): + run_id = sdk.start(backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT, params=_scheduled_params()) + run = _await_terminal(sdk, run_id) + # The SDK run is still failed -- the bytes did not leave -- but the backoff + # is NOT fed, so the next wake is due exactly as before. + assert run.status == job_sdk.FAILED + assert backup.nightly_failures(ACCOUNT).get(backup.KIND_SNAPSHOT) is None + # The gate refused before the upload, so the work never ran. + work.assert_not_called() + + def test_a_plain_scheduled_fault_still_feeds_the_backoff(self, sdk): + # The converse of the test above, so the withheld carve-out cannot be + # "never record anything scheduled": a genuine fault (not a withheld + # policy refusal) still records so a deterministic break backs off. + backup.set_nightly(ACCOUNT, True) + with ( + _resolvable(), + mock.patch.object(backup, "_authorize_upload", return_value=None), + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET), + mock.patch.object(backup, "other_install_ids", return_value=[]), + mock.patch.object( + backup, "run_snapshot_backup", side_effect=RuntimeError("push broke") + ), + ): + run_id = sdk.start(backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT, params=_scheduled_params()) + run = _await_terminal(sdk, run_id) + assert run.status == job_sdk.FAILED + assert backup.nightly_failures(ACCOUNT)[backup.KIND_SNAPSHOT]["consecutive"] == 1 + + """A scheduled run has no HTTP handler, so it emits its own SEL trail: the + invoked/(succeeded|unchanged)/failed record the dashboard layer adds to an owner + mutation, plus the shared-drive observation. The spec documents both as decisions, + so they are pinned here rather than left to the SDK's generic run record (which + cannot tell an unchanged night from an upload).""" + + def _audit_outcomes(self, log): + return [ + (c.kwargs["operation"], c.kwargs["outcome"]) for c in log.log_api_access.call_args_list + ] + + def test_a_succeeded_run_audits_invoked_then_succeeded(self, sdk): + log = mock.MagicMock() + with ( + _resolvable(), + mock.patch.object(backup, "_authorize_upload", return_value=None), + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET), + mock.patch.object(backup, "other_install_ids", return_value=[]), + mock.patch.object(backup, "sel", lambda: log), + mock.patch.object( + backup, "run_snapshot_backup", return_value={"uploaded": True, "key": "k/a.tar.gz"} + ), + ): + run_id = sdk.start(backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT, params=_scheduled_params()) + _await_terminal(sdk, run_id) + outcomes = self._audit_outcomes(log) + assert ("aws_control.backup_nightly", "invoked") in outcomes + assert ("aws_control.backup_nightly", "succeeded") in outcomes + assert ("aws_control.backup_nightly", "unchanged") not in outcomes + + def test_an_unchanged_run_audits_unchanged_not_succeeded(self, sdk): + log = mock.MagicMock() + with ( + _resolvable(), + mock.patch.object(backup, "_authorize_upload", return_value=None), + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET), + mock.patch.object(backup, "other_install_ids", return_value=[]), + mock.patch.object(backup, "sel", lambda: log), + mock.patch.object( + backup, + "run_snapshot_backup", + return_value={"uploaded": False, "key": "k/held.tar.gz"}, + ), + ): + run_id = sdk.start(backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT, params=_scheduled_params()) + _await_terminal(sdk, run_id) + outcomes = self._audit_outcomes(log) + assert ("aws_control.backup_nightly", "unchanged") in outcomes + assert ("aws_control.backup_nightly", "succeeded") not in outcomes + + def test_another_install_on_the_drive_is_audited_and_the_run_proceeds(self, sdk): + log = mock.MagicMock() + with ( + _resolvable(), + mock.patch.object(backup, "_authorize_upload", return_value=None), + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET), + mock.patch.object(backup, "other_install_ids", return_value=["inst-xyz"]), + mock.patch.object(backup, "sel", lambda: log), + mock.patch.object( + backup, "run_snapshot_backup", return_value={"uploaded": True, "key": "k/a.tar.gz"} + ) as work, + ): + run_id = sdk.start(backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT, params=_scheduled_params()) + run = _await_terminal(sdk, run_id) + # A NOTICE, not a gate: the backup still runs with another install present. + assert run.status == job_sdk.DONE + work.assert_called_once() + assert ("aws_control.backup_shared_drive", "invoked") in self._audit_outcomes(log) + + def test_an_owner_run_emits_no_nightly_audit(self, sdk): + # The owner path gets its audit from the HTTP layer; the runner must not + # double-file an unattended record for a run a person triggered. + log = mock.MagicMock() + with ( + _resolvable(), + mock.patch.object(backup, "_authorize_upload", return_value=None), + mock.patch.object(backup.storage, "find_drive", return_value=BUCKET), + mock.patch.object(backup, "other_install_ids", return_value=["inst-xyz"]), + mock.patch.object(backup, "sel", lambda: log), + mock.patch.object( + backup, "run_snapshot_backup", return_value={"uploaded": True, "key": "k/a.tar.gz"} + ), + ): + run_id = sdk.start(backup.KIND_SNAPSHOT, dedupe_key=ACCOUNT) + _await_terminal(sdk, run_id) + ops = [c.kwargs["operation"] for c in log.log_api_access.call_args_list] + assert "aws_control.backup_nightly" not in ops + assert "aws_control.backup_shared_drive" not in ops + + +class TestSharedDriveNoticeCostsOneListCall: + """The shared-drive check answers one question — "does another install write + here" — and one prefix answers it, so it stays at a single paid LIST however + many kinds the nightly pushes. This pins `other_install_ids`'s bound directly; + the runner only calls it for the snapshot kind, so the whole scheduled wake + pays at most this one call.""" + + def test_the_notice_costs_exactly_one_paid_list_call(self): + with mock.patch.object(backup, "_install_folders", return_value=set()) as folders: + backup.other_install_ids("p", "us-west-2", "bkt", account="111122223333") + assert folders.call_count == 1 + assert folders.call_args.args[3] == backup.KIND_SNAPSHOT diff --git a/test/test_aws_control_hooks.py b/test/test_aws_control_hooks.py index 05bf5c21532b..e3900f6972e6 100644 --- a/test/test_aws_control_hooks.py +++ b/test/test_aws_control_hooks.py @@ -1,19 +1,17 @@ """Coverage for the aws_control nightly backup lifecycle hooks. -``test_aws_control_app.py::TestRound22Hardening`` already pins the two -end-of-loop SEL cases (full success emits ``invoked``+``succeeded``; a raising -backup is audited ``failed``) by patching ``hooks._audit`` wholesale. Those -leave the interesting parts of the module untested: every EARLY-RETURN guard in -``_run_once`` that makes the loop fail closed, the ``_audit`` body itself -(including its best-effort swallow), the ``_loop`` supervisor's error -swallowing, and ``on_startup`` / ``on_shutdown`` idempotence. This file covers -exactly those, and never leaves a real background task running past a test. +The loop CLAIMS each due backup through the Job SDK +(``start_async(kind, dedupe_key=account, params={caller: scheduled})``) rather +than running it on a bare thread. This file covers the due-check's early-return +guards, the claim itself, the ``_loop`` supervisor's error swallowing, and +``on_startup`` / ``on_shutdown`` idempotence. The runner's own behaviour (reading +the caller from params, the scheduled failure-backoff) is pinned in +``test_aws_control_backup_job.py``, where the runner lives. """ from __future__ import annotations import asyncio -import contextlib from unittest import mock from unittest.mock import AsyncMock @@ -21,6 +19,7 @@ from kiro_crew import aws_consent from kiro_crew.apps.builtins.aws_control import hooks +from kiro_crew.apps.job_sdk import JobError, UnknownJobKind ACCOUNT = "111122223333" @@ -36,68 +35,47 @@ async def _never() -> None: await asyncio.Event().wait() -class TestAuditBody: - """The SEL helper the two Round-22 tests patch OUT -- exercised here for real - so its success path and its must-never-raise contract are both pinned.""" - - def test_audit_forwards_a_three_part_record_to_sel(self): - # The unattended loop has no request, so this call is the ONLY audit - # trail for a nightly S3 mutation. Confirm every field the handlers rely - # on is forwarded, and that resources/error are truncated to 200 chars - # so an oversized value cannot blow past the SEL column bound. - with mock.patch.object(hooks, "sel") as sel_factory: - hooks._audit("backup_nightly", "r" * 300, "invoked", error="e" * 300) - call = sel_factory.return_value.log_api_access.call_args - assert call.kwargs["operation"] == "aws_control.backup_nightly" - assert call.kwargs["outcome"] == "invoked" - assert call.kwargs["caller"] == "aws-control-nightly" - assert len(call.kwargs["resources"]) == 200 - assert len(call.kwargs["error"]) == 200 - - def test_audit_swallows_a_failing_sel(self): - # A broken audit backend must never abort a backup -- the audit is - # best-effort by the same rule the HTTP handlers use. A raise here would - # crash the loop and silently stop nightly backups. - with mock.patch.object(hooks, "sel", side_effect=RuntimeError("sel down")): - hooks._audit("backup_nightly", "backup/snapshots", "invoked") # no raise - - def _run(coro): """Drive one coroutine to completion on a throwaway loop.""" return asyncio.run(coro) +def _fake_sdk(): + """A JobSDK stand-in whose ``start_async`` records its calls.""" + sdk = mock.Mock() + sdk.start_async = AsyncMock(return_value="run-123") + return sdk + + class TestRunOnceEarlyReturns: """Each guard makes the loop fail CLOSED: a missing precondition is a log - line and a return, never an unconfirmed AWS call. One test per guard.""" + line and a return, never a claimed run. One test per guard. ``start_async`` + must stay untouched in every one.""" def test_no_healthy_registered_key_skips(self): # With no working key resolved there is nothing the loop may run under, # so it must return before probing identity. The resolution is the one # the grant was recorded against, so a key it rejects is a key the # consent gate would refuse anyway. + sdk = _fake_sdk() with ( mock.patch.object( hooks.accounts_mod, "resolve_default_account_profile", AsyncMock(return_value=None) ), mock.patch.object(hooks.aws_consent, "probe_identity") as probe, ): - _run(hooks._run_once()) + _run(hooks._run_once(sdk)) probe.assert_not_called() + sdk.start_async.assert_not_called() def test_the_nightly_identity_probe_bypasses_the_cache(self): """The account this loop keys backups by must not come from memory. - ``resolve_default_account_profile`` reaches ``_fold_profile``, which probes - every registry entry with the cache ON, so a cached probe here answers from - an entry primed moments earlier -- the comment's "live probe" would never - run live. A repoint inside that window keys ``due_for_nightly``, - ``find_drive`` and the snapshot record to the wrong account, unattended. - - Asserted on the KEYWORD rather than on a real cache because the loop's own - guards stub the probe out; the cache behaviour itself is pinned where the - cache lives. + A repoint inside a cached-probe window keys ``due_for_nightly`` and the + claim to the wrong account, unattended. Asserted on the KEYWORD because + the loop's own guards stub the probe out. """ + sdk = _fake_sdk() with ( mock.patch.object( hooks.accounts_mod, @@ -110,13 +88,13 @@ def test_the_nightly_identity_probe_bypasses_the_cache(self): AsyncMock(return_value=aws_consent.Identity(ok=False, account="")), ) as probe, ): - _run(hooks._run_once()) + _run(hooks._run_once(sdk)) assert probe.await_args.kwargs.get("use_cache") is False def test_unresolved_identity_skips(self): # A profile NAME is not an account; if the live probe cannot resolve one - # (ok False or empty account) the loop cannot key backup state, so it - # returns before the due-check. + # the loop cannot key backup state, so it returns before the due-check. + sdk = _fake_sdk() with ( mock.patch.object( hooks.accounts_mod, @@ -130,12 +108,14 @@ def test_unresolved_identity_skips(self): ), mock.patch.object(hooks.backup_mod, "due_for_nightly") as due, ): - _run(hooks._run_once()) + _run(hooks._run_once(sdk)) due.assert_not_called() + sdk.start_async.assert_not_called() def test_identity_ok_but_no_account_skips(self): # ok True with an empty account is still unresolved -- the guard checks # both, so this distinct branch must also return before due-check. + sdk = _fake_sdk() with ( mock.patch.object( hooks.accounts_mod, @@ -149,13 +129,13 @@ def test_identity_ok_but_no_account_skips(self): ), mock.patch.object(hooks.backup_mod, "due_for_nightly") as due, ): - _run(hooks._run_once()) + _run(hooks._run_once(sdk)) due.assert_not_called() def test_not_due_skips_before_consent(self): - # The half-hourly wake is cheap; most wakes are not due. A not-due run - # must return before touching consent so a nightly-disabled account is - # never even asked to spend money. + # Most wakes are not due. A not-due run must return before touching + # consent so a nightly-disabled account is never asked to spend money. + sdk = _fake_sdk() with ( mock.patch.object( hooks.accounts_mod, @@ -168,15 +148,19 @@ def test_not_due_skips_before_consent(self): AsyncMock(return_value=aws_consent.Identity(ok=True, account=ACCOUNT)), ), mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=False), + mock.patch.object(hooks.backup_mod, "due_for_sessions_nightly", return_value=False), + mock.patch.object(hooks.backup_mod, "nightly_sessions_enabled", return_value=False), mock.patch.object(hooks.aws_consent, "refuse_and_log") as refuse, ): - _run(hooks._run_once()) + _run(hooks._run_once(sdk)) refuse.assert_not_called() + sdk.start_async.assert_not_called() - def test_consent_refused_skips_before_drive_lookup(self): + def test_consent_refused_skips_before_claiming(self): # Consent fails closed: if refuse_and_log returns False the run stops - # (it already logged + audited), before find_drive is called, so a - # revoked grant produces no S3 read and no upload. + # (it already logged + audited), before any run is claimed, so a revoked + # grant produces no SDK run and no upload. + sdk = _fake_sdk() with ( mock.patch.object( hooks.accounts_mod, @@ -189,16 +173,19 @@ def test_consent_refused_skips_before_drive_lookup(self): AsyncMock(return_value=aws_consent.Identity(ok=True, account=ACCOUNT)), ), mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=True), + mock.patch.object(hooks.backup_mod, "due_for_sessions_nightly", return_value=False), mock.patch.object(hooks.aws_consent, "refuse_and_log", AsyncMock(return_value=False)), - mock.patch.object(hooks.storage_mod, "find_drive") as find, ): - _run(hooks._run_once()) - find.assert_not_called() + _run(hooks._run_once(sdk)) + sdk.start_async.assert_not_called() + - def test_no_drive_bucket_skips_before_backup(self): - # The drive is tag-discovered per run, not trusted from memory. Until one - # exists there is nowhere to push, so the run returns before invoking the - # snapshot backup (and before the "invoked" audit). +class TestClaimsTheRunThroughTheSdk: + """Past every guard, each due kind is CLAIMED through the SDK. The account is + the dedupe key and the scheduled caller rides in params, so a nightly and a + manual click adopt one another and the Backup row sees the run.""" + + def _drive(self, sdk, *, snapshot_due, sessions_due, bucket="bkt"): with ( mock.patch.object( hooks.accounts_mod, @@ -210,30 +197,75 @@ def test_no_drive_bucket_skips_before_backup(self): "probe_identity", AsyncMock(return_value=aws_consent.Identity(ok=True, account=ACCOUNT)), ), - mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=True), + mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=snapshot_due), + mock.patch.object( + hooks.backup_mod, "due_for_sessions_nightly", return_value=sessions_due + ), mock.patch.object(hooks.aws_consent, "refuse_and_log", AsyncMock(return_value=True)), - mock.patch.object(hooks.storage_mod, "find_drive", return_value=""), - mock.patch.object(hooks.backup_mod, "run_snapshot_backup") as backup, - mock.patch.object(hooks, "_audit") as audit, + mock.patch.object(hooks.backup_mod.storage, "find_drive", return_value=bucket), ): - _run(hooks._run_once()) - backup.assert_not_called() - audit.assert_not_called() - - -class TestSharedDriveNotice: - """A shared drive is a NOTICE, never a gate. Two installs pointed at one - account each write under their own prefix, so the nightly loop records the - fact and PROCEEDS -- refusing would silently stop backing one machine up.""" - - def _proceeding_run(self, *, others): - """Run ``_run_once`` past every guard to the backup, with the - shared-drive check returning (or raising) ``others``.""" - others_patch = ( - mock.patch.object(hooks.backup_mod, "other_install_ids", side_effect=others) - if isinstance(others, Exception) - else mock.patch.object(hooks.backup_mod, "other_install_ids", return_value=others) - ) + _run(hooks._run_once(sdk)) + + def test_a_due_snapshot_is_claimed_with_the_account_and_scheduled_caller(self): + sdk = _fake_sdk() + self._drive(sdk, snapshot_due=True, sessions_due=False) + sdk.start_async.assert_awaited_once() + call = sdk.start_async.await_args + assert call.args[0] == hooks.backup_mod.KIND_SNAPSHOT + # The account is the dedupe key: this is what makes a nightly run and a + # manual click for one account adopt one another instead of both paying. + assert call.kwargs["dedupe_key"] == ACCOUNT + # The scheduled-caller param is what tells the shared runner to apply the + # unattended-only gates and to feed the failure backoff. The verified + # profile/region the loop just probed ride alongside it, so a scheduled run + # can fall back to them if the runner's sync cache has expired past the TTL + # by the time the fire-and-forget start reaches it. + assert call.kwargs["params"] == { + hooks.backup_mod.JOB_PARAM_CALLER: hooks.backup_mod.CALLER_SCHEDULED, + hooks.backup_mod.JOB_PARAM_PROFILE: "p", + hooks.backup_mod.JOB_PARAM_REGION: "us-west-2", + } + + def test_both_due_kinds_are_each_claimed_once(self): + # Snapshot and sessions dedupe on (kind, account) independently, so a wake + # due for both claims two runs -- one per kind -- not one. + sdk = _fake_sdk() + self._drive(sdk, snapshot_due=True, sessions_due=True) + kinds = [c.args[0] for c in sdk.start_async.await_args_list] + assert kinds == [hooks.backup_mod.KIND_SNAPSHOT, hooks.backup_mod.KIND_SESSIONS] + assert all(c.kwargs["dedupe_key"] == ACCOUNT for c in sdk.start_async.await_args_list) + + def test_no_drive_yet_claims_no_run(self): + # The drive is tag-discovered per wake. Until one exists there is nowhere to + # push, so the loop claims NOTHING -- as it did before this change. Claiming + # a run anyway would leave a `done` SDK run on the Backup row every wake for + # a backup that uploaded nothing, and that newest `done` run would clear an + # owner's earlier `lastFailed`. A missing drive is "nothing to attempt yet". + sdk = _fake_sdk() + self._drive(sdk, snapshot_due=True, sessions_due=True, bucket="") + sdk.start_async.assert_not_awaited() + + def test_a_refused_start_is_a_log_line_not_a_crash(self): + # The runtime can refuse a start (mid-teardown, or no runner registered). + # Nothing reached AWS, so it is a logged skip and the next wake tries + # again -- it must not propagate out of the loop body. + for exc in (JobError("shutting down"), UnknownJobKind("no runner")): + sdk = _fake_sdk() + sdk.start_async = AsyncMock(side_effect=exc) + self._drive(sdk, snapshot_due=True, sessions_due=False) # no raise + sdk.start_async.assert_awaited_once() + + +class TestADriveLookupThatRaisesIsRecorded: + """A drive lookup that RAISES (not returns no drive) is a fault, and it + happens in the loop BEFORE any run is claimed, so the runner's own failure + handler never sees it. It must still back off and audit per due kind -- or a + deterministically failing lookup re-attempts every wake forever. A setup + fault and a push fault for one kind are the same invariant: both back off and + both audit `failed`. A cancel is teardown, not a fault: it does NOT back off, + but it still audits `cancelled` per due kind so the interrupt stays visible.""" + + def _run_once_with_raising_find_drive(self, sdk, *, snapshot_due, sessions_due): with ( mock.patch.object( hooks.accounts_mod, @@ -245,61 +277,51 @@ def _proceeding_run(self, *, others): "probe_identity", AsyncMock(return_value=aws_consent.Identity(ok=True, account=ACCOUNT)), ), - mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=True), + mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=snapshot_due), + mock.patch.object( + hooks.backup_mod, "due_for_sessions_nightly", return_value=sessions_due + ), mock.patch.object(hooks.aws_consent, "refuse_and_log", AsyncMock(return_value=True)), - mock.patch.object(hooks.storage_mod, "find_drive", return_value="kirocrew-drive-abc"), - others_patch, mock.patch.object( - hooks.backup_mod, "run_snapshot_backup", return_value={"key": "snapshots/x"} - ) as backup, - mock.patch.object(hooks, "_audit") as audit, + hooks.backup_mod.storage, + "find_drive", + side_effect=RuntimeError("tagging API error"), + ), + mock.patch.object(hooks.backup_mod, "nightly_run_witness", return_value=("proc", 3)), + mock.patch.object(hooks.backup_mod, "record_nightly_failure") as rec, + mock.patch.object(hooks.backup_mod, "sel") as sel, ): - _run(hooks._run_once()) - return backup, audit - - def test_another_install_present_still_backs_up_and_audits_the_shared_drive(self): - # The decisive pin: the run does NOT refuse. The backup still runs, and a - # `backup_shared_drive` record is emitted so the shared state is visible - # in the same SEL trail as the upload it precedes. - backup, audit = self._proceeding_run(others=["b" * 32]) - backup.assert_called_once() - shared = [c for c in audit.call_args_list if c.args[0] == "backup_shared_drive"] - assert shared, "the shared-drive notice was not audited" - assert shared[0].args[2] == "invoked" - - def test_a_failing_shared_drive_check_never_blocks_the_backup(self): - # The notice is best-effort: if other_install_ids raises, the check is - # one less log line, never a missed nightly. The backup still runs and no - # shared-drive record is emitted. - backup, audit = self._proceeding_run(others=RuntimeError("list denied")) - backup.assert_called_once() - assert not [c for c in audit.call_args_list if c.args[0] == "backup_shared_drive"] - - def test_no_other_install_emits_no_shared_drive_record(self): - # A drive only this install writes to is not shared, so there is nothing - # to record. The backup runs; the shared-drive audit does not fire. - backup, audit = self._proceeding_run(others=[]) - backup.assert_called_once() - assert not [c for c in audit.call_args_list if c.args[0] == "backup_shared_drive"] - - def test_the_notice_costs_exactly_one_paid_list_call(self): - # One prefix is enough to ANSWER the question the notice asks -- "does - # another install write to this drive" -- so the sweep stays at one paid - # LIST however many kinds the nightly pushes. Sweeping every prefix spent - # an extra paid LIST per scheduled run to re-derive an answer already in - # hand, a real cost on the one path that spends without a human present, - # which is exactly where an unread call is least defensible. - with mock.patch.object(hooks.backup_mod, "_install_folders", return_value=set()) as folders: - hooks.backup_mod.other_install_ids("p", "us-west-2", "bkt", account="111122223333") - assert folders.call_count == 1 - assert folders.call_args.args[3] == hooks.backup_mod.KIND_SNAPSHOT - - -class TestRunOnceCancellation: - """A cancel during the backup is audited as ``cancelled`` and re-raised, so - teardown stops the loop cleanly AND leaves a trail that it was interrupted.""" - - def test_cancelled_backup_is_audited_and_reraised(self): + _run(hooks._run_once(sdk)) + return rec, sel + + def test_both_due_kinds_each_record_and_audit_the_failure(self): + # The lookup raises for a wake due for BOTH kinds. Each due kind must get + # its own backoff record AND its own `failed` audit -- a lookup that keeps + # failing is otherwise the one failure shape that never backs off. + sdk = _fake_sdk() + rec, sel = self._run_once_with_raising_find_drive(sdk, snapshot_due=True, sessions_due=True) + # No run is claimed -- there is no drive to push to. + sdk.start_async.assert_not_awaited() + # One backoff record per due kind, each witnessed and carrying the error. + kinds_recorded = {c.args[1] for c in rec.call_args_list} + assert kinds_recorded == { + hooks.backup_mod.KIND_SNAPSHOT, + hooks.backup_mod.KIND_SESSIONS, + } + assert all(c.kwargs["run_witness"] == ("proc", 3) for c in rec.call_args_list) + # One `failed` SEL audit per due kind. + audits = sel.return_value.log_api_access.call_args_list + assert len(audits) == 2 + assert all(c.kwargs["outcome"] == "failed" for c in audits) + assert all(c.kwargs["operation"] == "aws_control.backup_nightly" for c in audits) + + def test_a_cancel_during_the_lookup_audits_cancelled_but_does_not_back_off(self): + # Teardown is not a fault: a cancel while resolving the drive is the app + # being disabled / the gateway stopping, and counting it as a failed + # attempt would push the next night's backup out -- so it does NOT back + # off. But teardown must still leave a trail that it was interrupted, so + # it audits `cancelled` per due kind before propagating the cancel. + sdk = _fake_sdk() with ( mock.patch.object( hooks.accounts_mod, @@ -312,21 +334,28 @@ def test_cancelled_backup_is_audited_and_reraised(self): AsyncMock(return_value=aws_consent.Identity(ok=True, account=ACCOUNT)), ), mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=True), + mock.patch.object(hooks.backup_mod, "due_for_sessions_nightly", return_value=True), mock.patch.object(hooks.aws_consent, "refuse_and_log", AsyncMock(return_value=True)), - mock.patch.object(hooks.storage_mod, "find_drive", return_value="kirocrew-drive-abc"), mock.patch.object( - hooks.backup_mod, - "run_snapshot_backup", + hooks.backup_mod.storage, + "find_drive", side_effect=asyncio.CancelledError(), ), - mock.patch.object(hooks, "_audit") as audit, + mock.patch.object(hooks.backup_mod, "nightly_run_witness", return_value=("proc", 3)), + mock.patch.object(hooks.backup_mod, "record_nightly_failure") as rec, + mock.patch.object(hooks.backup_mod, "sel") as sel, ): with pytest.raises(asyncio.CancelledError): - _run(hooks._run_once()) - outcomes = [c.args[2] for c in audit.call_args_list] - # "invoked" fired before the cancel, then "cancelled" -- never "failed", - # because a cancel is not an error to swallow. - assert outcomes == ["invoked", "cancelled"] + _run(hooks._run_once(sdk)) + # A cancel is teardown, not a fault: no backoff, no run claimed. + rec.assert_not_called() + sdk.start_async.assert_not_awaited() + # But the interrupt stays visible: one `cancelled` SEL audit per due kind, + # never `failed`. + audits = sel.return_value.log_api_access.call_args_list + assert len(audits) == 2 + assert all(c.kwargs["outcome"] == "cancelled" for c in audits) + assert all(c.kwargs["operation"] == "aws_control.backup_nightly" for c in audits) class TestLoopSupervisor: @@ -336,17 +365,17 @@ class TestLoopSupervisor: def test_loop_swallows_run_once_error_then_sleeps(self): # First pass raises (swallowed), so control reaches the sleep; we cancel # AT the sleep to break the otherwise-infinite loop deterministically. - # Reaching the sleep is the proof the error was swallowed, not raised. calls = {"n": 0} - async def _boom() -> None: + async def _boom(_sdk) -> None: calls["n"] += 1 raise RuntimeError("bad night") async def _drive() -> None: with ( + mock.patch.object(hooks, "get_sdk", return_value=_fake_sdk()), mock.patch.object(hooks, "_run_once", side_effect=_boom), - mock.patch.object( + mock.patch.object( # flake-ok: patching asyncio.sleep is the deterministic way to break _loop's infinite wait hooks.asyncio, "sleep", AsyncMock(side_effect=asyncio.CancelledError()) ), ): @@ -360,12 +389,37 @@ def test_loop_propagates_cancel_from_run_once(self): # A cancel raised by _run_once itself must exit the loop, not be caught # by the broad Exception handler (CancelledError is re-raised first). async def _drive() -> None: - with mock.patch.object(hooks, "_run_once", side_effect=asyncio.CancelledError()): + with ( + mock.patch.object(hooks, "get_sdk", return_value=_fake_sdk()), + mock.patch.object(hooks, "_run_once", side_effect=asyncio.CancelledError()), + ): with pytest.raises(asyncio.CancelledError): await hooks._loop() _run(_drive()) + def test_loop_skips_run_once_when_no_sdk_is_registered(self): + # Without the `jobs` permission the SDK is absent; the loop must not call + # _run_once with None -- it waits for the next wake. + ran = {"n": 0} + + async def _count(_sdk) -> None: + ran["n"] += 1 + + async def _drive() -> None: + with ( + mock.patch.object(hooks, "get_sdk", return_value=None), + mock.patch.object(hooks, "_run_once", side_effect=_count), + mock.patch.object( # flake-ok: patching asyncio.sleep is the deterministic way to break _loop's infinite wait + hooks.asyncio, "sleep", AsyncMock(side_effect=asyncio.CancelledError()) + ), + ): + with pytest.raises(asyncio.CancelledError): + await hooks._loop() + + _run(_drive()) + assert ran["n"] == 0 + class TestStartupShutdown: """Enable/disable idempotence. Every test patches out the real loop body so @@ -380,12 +434,12 @@ async def _drive() -> None: hooks._task = None with ( mock.patch.object(hooks, "_loop", _never), + mock.patch.object(hooks, "_register_job_runners", AsyncMock()), mock.patch.object(hooks.backup_mod, "clear_stop") as clear, ): await hooks.on_startup(None) assert hooks._task is not None clear.assert_called_once() - # Do not let the created task outlive the test. await hooks.on_shutdown(None) with mock.patch.object(hooks.backup_mod, "signal_stop"): @@ -394,11 +448,13 @@ async def _drive() -> None: def test_on_startup_is_idempotent_while_running(self): # A second enable while the worker is live must be a no-op: it must NOT - # spawn a second loop or re-clear the stop, or an enable/disable/enable - # cycle could leave two workers or lose a teardown signal. + # spawn a second loop or re-clear the stop. async def _drive() -> None: hooks._task = None - with mock.patch.object(hooks, "_loop", _never): + with ( + mock.patch.object(hooks, "_loop", _never), + mock.patch.object(hooks, "_register_job_runners", AsyncMock()), + ): with mock.patch.object(hooks.backup_mod, "clear_stop"): await hooks.on_startup(None) first = hooks._task @@ -421,6 +477,7 @@ async def _drive() -> None: hooks._task = done # a completed task/future: .done() is True with ( mock.patch.object(hooks, "_loop", _never), + mock.patch.object(hooks, "_register_job_runners", AsyncMock()), mock.patch.object(hooks.backup_mod, "clear_stop") as clear, ): await hooks.on_startup(None) @@ -436,15 +493,16 @@ def test_on_shutdown_signals_stop_and_cancels(self): # upload) AND cancel the task (so the awaiting loop unblocks). Both, and # then _task must be cleared so a later enable can start fresh. async def _drive() -> None: - with mock.patch.object(hooks, "_loop", _never): + with ( + mock.patch.object(hooks, "_loop", _never), + mock.patch.object(hooks, "_register_job_runners", AsyncMock()), + ): with mock.patch.object(hooks.backup_mod, "clear_stop"): await hooks.on_startup(None) task = hooks._task with mock.patch.object(hooks.backup_mod, "signal_stop") as signal: await hooks.on_shutdown(None) signal.assert_called_once() - # Let the just-cancelled task settle: the cancel is a request until - # the loop runs it, so awaiting is what proves it actually stops. with pytest.raises(asyncio.CancelledError): await task assert task.cancelled() @@ -463,276 +521,3 @@ async def _drive() -> None: assert hooks._task is None _run(_drive()) - - -class TestNightlyOutcomeIsToldApart: - """A nightly that sent nothing must not be recorded as a push. - - ``run_snapshot_backup`` now returns a record whose ``uploaded`` is False when the - source tree has not changed since the archive already in the drive. The audit trail - and the log line are where an operator actually reads what happened, so recording - that run as ``succeeded`` with "backup pushed: " would make a night on which no - bytes left indistinguishable from an ordinary upload -- against a key that really is - in the drive, so nothing downstream looks wrong either. - - Both directions, because an outcome field that always reports one value is not - telling anything apart. - """ - - @staticmethod - def _drive(record: dict): - """Run one nightly with every precondition satisfied, returning the audit mock.""" - with ( - mock.patch.object( - hooks.accounts_mod, - "resolve_default_account_profile", - AsyncMock(return_value=("p", "us-west-2")), - ), - mock.patch.object( - hooks.aws_consent, - "probe_identity", - AsyncMock(return_value=aws_consent.Identity(ok=True, account=ACCOUNT)), - ), - mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=True), - mock.patch.object(hooks.aws_consent, "refuse_and_log", AsyncMock(return_value=True)), - mock.patch.object( - hooks.storage_mod, "find_drive", return_value="kirocrew-drive-abc123def456" - ), - mock.patch.object(hooks.backup_mod, "run_snapshot_backup", return_value=record), - mock.patch.object(hooks, "_audit") as audit, - ): - _run(hooks._run_once()) - return audit - - def test_an_unchanged_night_is_audited_as_unchanged_not_succeeded(self): - audit = self._drive( - {"key": "snapshots/x.tar.gz", "uploaded": False, "bytes": 10, "tree": "t1"} - ) - outcomes = [c.args[2] for c in audit.call_args_list] - assert "unchanged" in outcomes - assert "succeeded" not in outcomes - # The key is still named: it is the archive the drive holds for this kind, which - # is what makes the record useful rather than merely honest. - keys = [c.args[1] for c in audit.call_args_list] - assert "snapshots/x.tar.gz" in keys - - def test_a_real_upload_is_still_audited_as_succeeded(self): - audit = self._drive({"key": "snapshots/x.tar.gz", "uploaded": True, "bytes": 10}) - outcomes = [c.args[2] for c in audit.call_args_list] - assert "succeeded" in outcomes - assert "unchanged" not in outcomes - - def test_a_record_without_the_field_is_treated_as_a_push(self): - # A record from a path that does not set `uploaded` at all must keep the old - # meaning rather than silently becoming "unchanged" -- the branch tests - # `is False`, so only an explicit skip takes the new wording. - audit = self._drive({"key": "snapshots/x.tar.gz"}) - outcomes = [c.args[2] for c in audit.call_args_list] - assert "succeeded" in outcomes - assert "unchanged" not in outcomes - - -class TestFailedAttemptIsRecorded: - """The loop's failure handlers now WRITE, not only audit. - - Before this the two halves had different fates: the audit went to SEL, where a - human reads it after the event, and nothing at all went to state, where the - due-check reads it on the next wake. So a nightly failing deterministically - produced a growing pile of ``failed`` audit records and a due-check that could - not see any of them, and re-attempted every half hour indefinitely. - - State is isolated per test, and the recorder is the REAL one: patching it out - would leave these tests asserting that a mock was called, which cannot show that - the due-check the loop actually reads has anything to read. - """ - - @pytest.fixture(autouse=True) - def _isolated_state(self, tmp_path, monkeypatch): - monkeypatch.setattr(hooks.backup_mod, "_state_path", lambda: tmp_path / "backup.json") - yield - - @contextlib.contextmanager - def _resolved(self): - """The guards up to the due-check, all passed. - - A context manager rather than a tuple of patches: a parenthesized ``with`` - cannot star-unpack, and the alternative -- returning the tuple and entering - it by hand -- would put an ``ExitStack`` in every test for no gain. - """ - with ( - mock.patch.object( - hooks.accounts_mod, - "resolve_default_account_profile", - AsyncMock(return_value=("p", "us-west-2")), - ), - mock.patch.object( - hooks.aws_consent, - "probe_identity", - AsyncMock(return_value=aws_consent.Identity(ok=True, account=ACCOUNT)), - ), - mock.patch.object(hooks.aws_consent, "refuse_and_log", AsyncMock(return_value=True)), - ): - yield - - def test_a_failing_push_records_the_attempt_and_withholds_the_next_wake(self): - # THE regression, driven through the loop rather than through the recorder, - # so it covers the wiring as well as the predicate: a deterministic fault - # must leave the account NOT due on the wake that follows. With nothing - # recorded this assertion reads True, which is the loop re-attempting every - # half hour for as long as the fault lasts. - hooks.backup_mod.set_nightly(ACCOUNT, True) - for _ in range(2): - with ( - self._resolved(), - mock.patch.object(hooks.storage_mod, "find_drive", return_value="drive-abc"), - mock.patch.object(hooks.backup_mod, "other_install_ids", return_value=[]), - mock.patch.object( - hooks.backup_mod, - "run_snapshot_backup", - side_effect=RuntimeError("snapshot build failed (rc=2)"), - ), - ): - _run(hooks._run_once()) - recorded = hooks.backup_mod.nightly_failures(ACCOUNT) - assert recorded[hooks.backup_mod.KIND_SNAPSHOT]["consecutive"] == 2 - # The error text is carried, so an operator reading the record learns WHAT - # keeps failing rather than only that something does. - assert "rc=2" in recorded[hooks.backup_mod.KIND_SNAPSHOT]["error"] - assert hooks.backup_mod.due_for_nightly(ACCOUNT) is False - - def test_the_failure_is_audited_and_recorded_together(self): - # One helper writes both, so the backoff cannot depend on which way the run - # broke. Asserting the pair -- not just the write -- is what pins them as - # one fact rather than two that happen to fire today. - hooks.backup_mod.set_nightly(ACCOUNT, True) - with ( - self._resolved(), - mock.patch.object(hooks.storage_mod, "find_drive", return_value="drive-abc"), - mock.patch.object(hooks.backup_mod, "other_install_ids", return_value=[]), - mock.patch.object( - hooks.backup_mod, "run_snapshot_backup", side_effect=RuntimeError("eio") - ), - mock.patch.object(hooks, "_audit") as audit, - ): - _run(hooks._run_once()) - failed = [c for c in audit.call_args_list if c.args[2] == "failed"] - assert len(failed) == 1 - assert failed[0].args[1] == "backup/snapshots" - recorded = hooks.backup_mod.nightly_failures(ACCOUNT) - assert recorded[hooks.backup_mod.KIND_SNAPSHOT]["consecutive"] == 1 - - def test_a_cancel_records_nothing(self): - # A cancelled attempt is teardown -- the owner disabled the app, or the - # gateway is stopping -- not a fault. Counting it would let a clean shutdown - # push the next night out, and a gateway restarted often enough would keep - # the nightly permanently backed off without anything ever having failed. - hooks.backup_mod.set_nightly(ACCOUNT, True) - with ( - self._resolved(), - mock.patch.object(hooks.storage_mod, "find_drive", return_value="drive-abc"), - mock.patch.object(hooks.backup_mod, "other_install_ids", return_value=[]), - mock.patch.object( - hooks.backup_mod, "run_snapshot_backup", side_effect=asyncio.CancelledError() - ), - ): - with pytest.raises(asyncio.CancelledError): - _run(hooks._run_once()) - assert hooks.backup_mod.nightly_failures(ACCOUNT) == {} - assert hooks.backup_mod.due_for_nightly(ACCOUNT) is True - - def test_a_failing_push_on_an_account_with_run_history_still_records(self): - # Every other case here starts with an EMPTY run slot, so a `_push_nightly` that passed a constant `None` witness - # instead of reading one would still match (absent vs absent) and every test - # would pass. With a prior run present, a constant `None` mismatches the real - # slot and the recorder refuses -- so the backoff would silently never engage - # for any account that has ever backed up successfully, which is most of them. - hooks.backup_mod.set_nightly(ACCOUNT, True) - hooks.backup_mod._record_run( - ACCOUNT, hooks.backup_mod.KIND_SNAPSHOT, "snapshots/i/old.tar.gz", 5, "fp", "v0" - ) - assert hooks.backup_mod.nightly_run_witness(ACCOUNT, hooks.backup_mod.KIND_SNAPSHOT) - with ( - self._resolved(), - mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=True), - mock.patch.object(hooks.storage_mod, "find_drive", return_value="drive-abc"), - mock.patch.object(hooks.backup_mod, "other_install_ids", return_value=[]), - mock.patch.object( - hooks.backup_mod, "run_snapshot_backup", side_effect=RuntimeError("eio") - ), - ): - _run(hooks._run_once()) - recorded = hooks.backup_mod.nightly_failures(ACCOUNT) - assert recorded[hooks.backup_mod.KIND_SNAPSHOT]["consecutive"] == 1 - - def test_a_shared_setup_failure_records_every_due_kind(self): - # The loop has TWO places an attempt can fail, and this is the other one: - # the setup shared by both kinds, before either push. It already audited per - # due kind; it must record per due kind too, or a drive lookup that keeps - # failing is the one failure shape that never backs off. - with ( - self._resolved(), - mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=True), - mock.patch.object(hooks.backup_mod, "due_for_sessions_nightly", return_value=True), - mock.patch.object( - hooks.storage_mod, "find_drive", side_effect=RuntimeError("tag lookup denied") - ), - ): - _run(hooks._run_once()) - recorded = hooks.backup_mod.nightly_failures(ACCOUNT) - assert set(recorded) == { - hooks.backup_mod.KIND_SNAPSHOT, - hooks.backup_mod.KIND_SESSIONS, - } - assert all(row["consecutive"] == 1 for row in recorded.values()) - - def test_a_cancelled_shared_setup_records_nothing(self): - # The cancel branch of the same handler. It is a separate `except` clause, so - # it is separately capable of regressing into recording a teardown. - with ( - self._resolved(), - mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=True), - mock.patch.object( - hooks.storage_mod, "find_drive", side_effect=asyncio.CancelledError() - ), - ): - with pytest.raises(asyncio.CancelledError): - _run(hooks._run_once()) - assert hooks.backup_mod.nightly_failures(ACCOUNT) == {} - - def test_a_successful_run_after_failures_clears_the_backoff(self): - # End to end through the loop: the fault clears, the next run completes, and - # the account is immediately eligible again rather than serving out a wait - # measured for a fault that is over. The runner calls the REAL `_record_run`, - # because the clear lives inside it -- a stubbed runner could not show the - # loop's own success clearing anything. - hooks.backup_mod.set_nightly(ACCOUNT, True) - for _ in range(3): - hooks.backup_mod.record_nightly_failure( - ACCOUNT, - hooks.backup_mod.KIND_SNAPSHOT, - "eio", - run_witness=hooks.backup_mod.nightly_run_witness( - ACCOUNT, hooks.backup_mod.KIND_SNAPSHOT - ), - ) - assert hooks.backup_mod.due_for_nightly(ACCOUNT) is False - - def _completing(account, profile, region, bucket, *, caller): - return hooks.backup_mod._record_run( - account, - hooks.backup_mod.KIND_SNAPSHOT, - "snapshots/i/a.tar.gz", - 9, - "fp", - "v1", - ) - - with ( - self._resolved(), - mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=True), - mock.patch.object(hooks.storage_mod, "find_drive", return_value="drive-abc"), - mock.patch.object(hooks.backup_mod, "other_install_ids", return_value=[]), - mock.patch.object(hooks.backup_mod, "run_snapshot_backup", side_effect=_completing), - ): - _run(hooks._run_once()) - assert hooks.backup_mod.nightly_failures(ACCOUNT) == {} diff --git a/test/test_aws_control_nightly_sessions.py b/test/test_aws_control_nightly_sessions.py index 132013d5357b..6fff3ddcb299 100644 --- a/test/test_aws_control_nightly_sessions.py +++ b/test/test_aws_control_nightly_sessions.py @@ -14,11 +14,12 @@ 2. **The window is its own window.** Due-ness is keyed on the SESSIONS run stamp, so a snapshot that ran an hour ago does not make the transcripts look backed up, and a wake proceeds when either kind is due. -3. **The payload is unchanged.** The loop calls ``run_sessions_backup``, the - same function the owner-triggered archive calls, with the same arguments and - the scheduled caller. Scheduling an existing mechanism decides nothing new - about what is in the archive or how it is redacted; those stay properties of - that function, and of the issue that governs it. +3. **The payload is unchanged.** The loop CLAIMS the sessions run through the + Job SDK (``start_async``, scheduled caller in params), and the registered + runner calls ``run_sessions_backup`` -- the same function the owner-triggered + archive calls. Scheduling an existing mechanism decides nothing new about what + is in the archive or how it is redacted; those stay properties of that + function, and of the issue that governs it. """ from __future__ import annotations @@ -150,15 +151,21 @@ def _run(coro): return asyncio.run(coro) -def _loop_patches(*, snapshot_due: bool, sessions_due: bool) -> list: - """Every guard ``_run_once`` passes before a push, with both due-checks pinned. +def _fake_sdk(): + sdk = mock.Mock() + sdk.start_async = AsyncMock(return_value="run-1") + return sdk - The backup runners are deliberately NOT in here: which of them is expected to - be called is the assertion in most of these tests, so each test states its - own. Entered through an ``ExitStack`` because this is a list, and the - parenthesized ``with`` form cannot unpack one. + +def _drive_loop(sdk, *, snapshot_due: bool, sessions_due: bool) -> None: + """Run ``_run_once`` past every guard with both due-checks pinned. + + The loop CLAIMS each due kind through ``sdk.start_async``. What the claimed + run then does (apply the scheduled gate, honour the transcript grant, record + a failure into the backoff) is the runner's, pinned in + test_aws_control_backup_job.py. """ - return [ + with ( mock.patch.object( hooks.accounts_mod, "resolve_default_account_profile", @@ -172,156 +179,42 @@ def _loop_patches(*, snapshot_due: bool, sessions_due: bool) -> list: mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=snapshot_due), mock.patch.object(hooks.backup_mod, "due_for_sessions_nightly", return_value=sessions_due), mock.patch.object(hooks.aws_consent, "refuse_and_log", AsyncMock(return_value=True)), - mock.patch.object(hooks.storage_mod, "find_drive", return_value="kirocrew-drive-abc"), - mock.patch.object(hooks.backup_mod, "other_install_ids", return_value=[]), - ] - - -def _drive(*, snapshot_due: bool, sessions_due: bool, **runners): - """Run ``_run_once`` with the given runner patches, returning them by name. - - ``runners`` maps a ``backup_mod`` attribute name to the mock it should be - replaced by, so a test names only what it cares about and reads the result - back under the same name. - """ - with contextlib.ExitStack() as stack: - for patcher in _loop_patches(snapshot_due=snapshot_due, sessions_due=sessions_due): - stack.enter_context(patcher) - entered = { - name: stack.enter_context(mock.patch.object(hooks.backup_mod, name, new)) - for name, new in runners.items() + mock.patch.object(hooks.backup_mod.storage, "find_drive", return_value="bkt"), + ): + _run(hooks._run_once(sdk)) + + +class TestTheNightlyLoopClaimsTranscripts: + """The scheduling itself: a due transcript window CLAIMS the sessions run + through the SDK, as the scheduled caller, independently of the snapshot.""" + + def test_a_due_transcript_window_claims_the_sessions_run(self): + # A transcript window that opens on a day the snapshot already ran must + # still claim the sessions run -- the two kinds are independent. + sdk = _fake_sdk() + _drive_loop(sdk, snapshot_due=False, sessions_due=True) + sdk.start_async.assert_awaited_once() + call = sdk.start_async.await_args + assert call.args[0] == backup.KIND_SESSIONS + assert call.kwargs["dedupe_key"] == ACCOUNT + # Nobody is at the keyboard: the scheduled caller rides in params so the + # runner attributes the push to the schedule, not the dashboard owner, and + # the verified profile/region ride alongside it as the runner's TTL-expiry + # fallback. + assert call.kwargs["params"] == { + backup.JOB_PARAM_CALLER: backup.CALLER_SCHEDULED, + backup.JOB_PARAM_PROFILE: "p", + backup.JOB_PARAM_REGION: "us-west-2", } - audit = stack.enter_context(mock.patch.object(hooks, "_audit")) - _run(hooks._run_once()) - return entered, audit - - -class TestTheNightlyLoopCarriesTranscripts: - """The scheduling itself: a due transcript window pushes the same archive - the owner-triggered path pushes, as the scheduled caller.""" - - def test_a_due_transcript_window_pushes_the_sessions_archive(self): - # Red on base twice over: base has no sessions branch at all, and base - # returns before consent when the SNAPSHOT is not due -- so a transcript - # window that opens on a day the snapshot already ran is skipped. - ran, _audit = _drive( - snapshot_due=False, - sessions_due=True, - run_snapshot_backup=mock.Mock(return_value={"key": "snapshots/x"}), - run_sessions_backup=mock.Mock(return_value={"key": "sessions/x"}), - ) - ran["run_snapshot_backup"].assert_not_called() - ran["run_sessions_backup"].assert_called_once() - call = ran["run_sessions_backup"].call_args - assert call.args[0] == ACCOUNT - # Nobody is at the keyboard: the upload must not be recorded against the - # dashboard owner, which is what the interactive caller would have done. - assert call.kwargs["caller"] == backup.CALLER_SCHEDULED - - def test_an_unauthorized_transcript_window_pushes_nothing(self): - # The grant is what decides. With it off the loop still runs its snapshot - # and the archive is never produced, so the sensitive payload cannot leave - # the host on a grant nobody gave. - ran, _audit = _drive( - snapshot_due=True, - sessions_due=False, - run_snapshot_backup=mock.Mock(return_value={"key": "snapshots/x"}), - run_sessions_backup=mock.Mock(), - ) - ran["run_snapshot_backup"].assert_called_once() - ran["run_sessions_backup"].assert_not_called() - - def test_neither_due_asks_nobody_to_spend(self): - # Both windows closed must return before consent, so a nightly-disabled - # account is never even asked to spend money. Keeps the existing - # early-return property now that two bits feed it. - with ( - mock.patch.object( - hooks.accounts_mod, - "resolve_default_account_profile", - AsyncMock(return_value=("p", "us-west-2")), - ), - mock.patch.object( - hooks.aws_consent, - "probe_identity", - AsyncMock(return_value=aws_consent.Identity(ok=True, account=ACCOUNT)), - ), - mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=False), - mock.patch.object(hooks.backup_mod, "due_for_sessions_nightly", return_value=False), - mock.patch.object(hooks.aws_consent, "refuse_and_log") as refuse, - ): - _run(hooks._run_once()) - refuse.assert_not_called() - - def test_a_failed_snapshot_does_not_cost_the_transcripts_their_window(self): - # The reason each kind owns its try/except. Sharing one handler turns a - # snapshot failure into a second skipped night for a payload that had - # nothing to do with it. - ran, audit = _drive( - snapshot_due=True, - sessions_due=True, - run_snapshot_backup=mock.Mock(side_effect=RuntimeError("no disk")), - run_sessions_backup=mock.Mock(return_value={"key": "sessions/x"}), - ) - ran["run_snapshot_backup"].assert_called_once() - ran["run_sessions_backup"].assert_called_once() - assert [c.args[2] for c in audit.call_args_list] == [ - "invoked", - "failed", - "invoked", - "succeeded", - ] - - def test_a_failed_archive_does_not_cost_the_snapshot_its_window(self): - # The mirror direction, and the one that matters for cost: the archive is - # the large payload, so it is the likelier one to time out, and a timeout - # there must not stop the small backup that would have succeeded. - ran, audit = _drive( - snapshot_due=True, - sessions_due=True, - run_snapshot_backup=mock.Mock(return_value={"key": "snapshots/x"}), - run_sessions_backup=mock.Mock(side_effect=RuntimeError("timed out")), - ) - ran["run_snapshot_backup"].assert_called_once() - ran["run_sessions_backup"].assert_called_once() - assert [c.args[2] for c in audit.call_args_list] == [ - "invoked", - "succeeded", - "invoked", - "failed", - ] - - def test_the_transcript_push_is_audited_under_its_own_subject(self): - # An unattended upload of the most sensitive payload in the product must - # be distinguishable in the SEL trail from the snapshot beside it, or an - # incident review cannot tell which bytes left the host. - _ran, audit = _drive( - snapshot_due=False, - sessions_due=True, - run_sessions_backup=mock.Mock(side_effect=RuntimeError("denied")), - ) - assert [c.args[1] for c in audit.call_args_list] == [ - "backup/sessions", - "backup/sessions", - ] - - def test_a_cancel_during_the_archive_is_audited_and_reraised(self): - # Teardown must stop the loop cleanly AND leave a trail that it was - # interrupted -- and a cancel is never swallowed as a failure. - with contextlib.ExitStack() as stack: - for patcher in _loop_patches(snapshot_due=False, sessions_due=True): - stack.enter_context(patcher) - stack.enter_context( - mock.patch.object( - hooks.backup_mod, - "run_sessions_backup", - side_effect=asyncio.CancelledError(), - ) - ) - audit = stack.enter_context(mock.patch.object(hooks, "_audit")) - with pytest.raises(asyncio.CancelledError): - _run(hooks._run_once()) - assert [c.args[2] for c in audit.call_args_list] == ["invoked", "cancelled"] + + def test_each_due_kind_is_claimed_as_its_own_run(self): + # Snapshot and sessions dedupe on (kind, account) independently, so a wake + # due for both claims two runs. A failure in one is then the SDK's to + # record against that run alone -- one cannot cost the other its window. + sdk = _fake_sdk() + _drive_loop(sdk, snapshot_due=True, sessions_due=True) + kinds = [c.args[0] for c in sdk.start_async.await_args_list] + assert kinds == [backup.KIND_SNAPSHOT, backup.KIND_SESSIONS] class TestAGrantCountsOnlyWhenItIsARealYes: @@ -640,79 +533,32 @@ def test_the_owner_triggered_archive_is_untouched_by_the_gap(self, monkeypatch): assert backup.kind_unavailable_reason(backup.KIND_SESSIONS) is None -class TestASetupFailureIsAuditedAgainstWhatWasActuallyDue: - """A failure before any push names the kinds this wake was for. - - The setup block -- drive discovery and the shared-drive note -- runs whenever - EITHER kind is due, so a transcripts-only wake reaches it too. A fixed - ``backup/snapshots`` subject there is therefore a record of a snapshot that - was never due, and the SEL trail is append-only, so nothing later corrects it. - """ - - def _fail_setup(self, *, snapshot_due, sessions_due, exc): - with contextlib.ExitStack() as stack: - for patcher in _loop_patches(snapshot_due=snapshot_due, sessions_due=sessions_due): - stack.enter_context(patcher) - stack.enter_context(mock.patch.object(hooks.storage_mod, "find_drive", side_effect=exc)) - audit = stack.enter_context(mock.patch.object(hooks, "_audit")) - if isinstance(exc, asyncio.CancelledError): - with pytest.raises(asyncio.CancelledError): - _run(hooks._run_once()) - else: - _run(hooks._run_once()) - return [c.args[1] for c in audit.call_args_list] - - def test_a_transcripts_only_wake_does_not_file_its_failure_as_a_snapshot(self): - # The finding itself. Before the fix this recorded "backup/snapshots" on a - # run the snapshot was never due for. - subjects = self._fail_setup( - snapshot_due=False, sessions_due=True, exc=RuntimeError("tagging API down") - ) - assert subjects == ["backup/sessions"] - - def test_a_snapshot_only_wake_still_files_against_the_snapshot(self): - # The direction that was already right must stay right, or the fix is a - # swap rather than a correction. - subjects = self._fail_setup( - snapshot_due=True, sessions_due=False, exc=RuntimeError("tagging API down") - ) - assert subjects == ["backup/snapshots"] - - def test_a_wake_for_both_kinds_names_both(self): - # One failure stopped two due payloads, so both are owed a record. A - # single entry would leave the other kind looking like it never came due. - subjects = self._fail_setup( - snapshot_due=True, sessions_due=True, exc=RuntimeError("tagging API down") - ) - assert subjects == ["backup/snapshots", "backup/sessions"] - - def test_a_cancel_during_setup_is_attributed_the_same_way(self): - # Teardown takes the other handler, and it carried the identical hardcoded - # subject, so fixing only the exception path would leave the same false - # record reachable by cancelling. - subjects = self._fail_setup( - snapshot_due=False, sessions_due=True, exc=asyncio.CancelledError() - ) - assert subjects == ["backup/sessions"] - - def test_the_subject_helper_is_the_only_spelling(self): - # Two copies of this expression is how a setup failure and its push end up - # filed under different subjects for the same kind. Derived from the - # constants so a kind added later cannot quietly miss a mapping. - for kind in backup.JOB_KINDS: - assert hooks._audit_subject(kind) == f"backup/{backup.KIND_SUBPATHS[kind]}" - - class TestAWithheldTranscriptNightlySaysWhy: """Turning the grant on and getting nothing must not be silent.""" + def _guards(self): + return ( + mock.patch.object( + hooks.accounts_mod, + "resolve_default_account_profile", + AsyncMock(return_value=("p", "us-west-2")), + ), + mock.patch.object( + hooks.aws_consent, + "probe_identity", + AsyncMock(return_value=aws_consent.Identity(ok=True, account=ACCOUNT)), + ), + mock.patch.object(hooks.backup_mod, "due_for_nightly", return_value=False), + mock.patch.object(hooks.backup_mod, "due_for_sessions_nightly", return_value=False), + ) + def test_the_redaction_gap_is_logged_when_it_withholds_the_run(self, caplog): # `due_for_sessions_nightly` answering False is the correct scheduling # decision, but on its own it leaves an operator who turned transcripts ON # watching a nightly that never runs with no statement of which of their # settings withheld it. with contextlib.ExitStack() as stack: - for patcher in _loop_patches(snapshot_due=False, sessions_due=False): + for patcher in self._guards(): stack.enter_context(patcher) stack.enter_context( mock.patch.object(hooks.backup_mod, "nightly_sessions_enabled", return_value=True) @@ -725,7 +571,7 @@ def test_the_redaction_gap_is_logged_when_it_withholds_the_run(self, caplog): ) ) with caplog.at_level("INFO", logger=hooks.logger.name): - _run(hooks._run_once()) + _run(hooks._run_once(_fake_sdk())) assert "transcripts withheld" in caplog.text assert "cannot be redacted" in caplog.text @@ -733,13 +579,13 @@ def test_nothing_is_logged_when_the_grant_is_off(self, caplog): # An install that never asked for transcripts is not withholding anything, # so a line here would be noise on every wake of every such install. with contextlib.ExitStack() as stack: - for patcher in _loop_patches(snapshot_due=False, sessions_due=False): + for patcher in self._guards(): stack.enter_context(patcher) stack.enter_context( mock.patch.object(hooks.backup_mod, "nightly_sessions_enabled", return_value=False) ) with caplog.at_level("INFO", logger=hooks.logger.name): - _run(hooks._run_once()) + _run(hooks._run_once(_fake_sdk())) assert "transcripts withheld" not in caplog.text