From c7ff7db304d781f3e7f7dfd97d0bf5101c8ac241 Mon Sep 17 00:00:00 2001 From: gh-autofix#2887 Date: Mon, 31 Aug 2026 17:15:42 +0000 Subject: [PATCH] feat(telemetry): instrument session lifetimes, tool call latency, and subsystem events Three populations that ran entirely unmeasured: how long a session lives and why it ended, how long each tool round-trip takes, and how often the subsystems behind a turn actually do their work. kirocrew.session.duration + kirocrew.session.started. Each session start drops a small crumb under /metrics/open-sessions/; a clean end reads it, emits the lifetime with the teardown path's own end_reason, and unlinks it. Whatever is still on disk at the next boot belongs to a session that never reached a teardown path, so gateway startup emits it as end_reason=crashed -- the population that would otherwise be missing entirely, since a crashed gateway runs no teardown. Consuming the crumb is what makes the accounting exact: the six teardown paths overlap (the idle sweep calls reset), and a back-filled session cannot be counted again on the boot after that. kirocrew.tool.call.duration. The turn histogram makes a slow turn visible but never says which part was slow. tool_kind is normalised against an allowlist, never the tool name: MCP tool names are unbounded and the ACP kind field arrives verbatim from the agent, so an un-normalised label is a cardinality bomb. An MCP-served call is labelled by its transport. AcpClient and the shared dispatch parser are sibling implementations of the same protocol, so both are instrumented; one process-global registry keyed by toolCallId, popped on the terminal status, keeps that layering at exactly one sample per call. Seven business counters at their own subsystem call sites, through the metrics/events facade: subagents spawned (carrying the concurrency it was admitted at, so the max over that attribute is the high-water mark), cron fires, artifacts created, workflow runs, context compactions, MCP stub reconnects, and tool-approval decisions. The approval counter generalises the backend-child permission pair to every surface's gate; those two are left exactly as they are, since they measure a different population. Both new histograms are registered in provider._HISTOGRAM_BUCKETS_MS, session duration under a new minutes-to-days boundary family and tool calls under a sub-millisecond-to-an-hour one -- without an entry each would fall back to OTEL's 10s default ceiling and report a floored percentile as a real latency. Refs #7232 #7257 --- .github/coverage-baselines/backend.txt | 14 + docs/system-specs/modules/metrics.md | 59 +- docs/system-specs/modules/session.md | 37 + src/kiro_crew/acp/_dispatch.py | 23 +- src/kiro_crew/acp/client.py | 26 + src/kiro_crew/artifacts.py | 9 + src/kiro_crew/cron.py | 12 + src/kiro_crew/hooks.py | 37 + src/kiro_crew/mcp_gateway/stub.py | 2 + src/kiro_crew/metrics/events.py | 59 ++ src/kiro_crew/metrics/provider.py | 30 +- src/kiro_crew/metrics/sessions.py | 692 +++++++++++++ src/kiro_crew/metrics/tool_calls.py | 247 +++++ src/kiro_crew/session_allocation.py | 36 + src/kiro_crew/session_background.py | 24 + src/kiro_crew/session_compaction.py | 22 + src/kiro_crew/session_lifecycle.py | 54 +- src/kiro_crew/slack/gateway.py | 34 + src/kiro_crew/subagent_manager/admission.py | 25 + src/kiro_crew/workflows/runner.py | 11 + test/metrics/test_business_counters.py | 278 +++++ test/metrics/test_session_duration.py | 1014 +++++++++++++++++++ test/metrics/test_tool_call_duration.py | 276 +++++ 23 files changed, 3012 insertions(+), 9 deletions(-) create mode 100644 src/kiro_crew/metrics/sessions.py create mode 100644 src/kiro_crew/metrics/tool_calls.py create mode 100644 test/metrics/test_business_counters.py create mode 100644 test/metrics/test_session_duration.py create mode 100644 test/metrics/test_tool_call_duration.py diff --git a/.github/coverage-baselines/backend.txt b/.github/coverage-baselines/backend.txt index fa27ebe2a2f6..fd558ff9054d 100644 --- a/.github/coverage-baselines/backend.txt +++ b/.github/coverage-baselines/backend.txt @@ -41,6 +41,20 @@ src/kiro_crew/apps/builtins/personal_shopper/backend/routes.py 71.5 # 186/260 src/kiro_crew/apps/builtins/workflows/server.py 61.2 # 82/134 src/kiro_crew/apps/hooks_integration.py 43.0 # 65/151 src/kiro_crew/apps/route_registry.py 51.9 # 67/129 +# NOT a missing-tests entry, and NOT this PR's debt -- it exempts a CI-side +# coverage ATTRIBUTION defect in a file no PR here touches, tracked as #7597. +# The tests exist and pass: TestFleetProbe at test/test_pipeline_conductor_agent.py:179, +# 47 tests green, and the file measures 86% (201/229) when coverage is scoped by +# PATH. Under the lane's own `--cov=kiro_crew` (scoped by import name) the same +# passing tests attribute nothing to it -- locally it does not appear among the +# 1214 reported rows at all, and the 34/229 recorded below is exactly the +# import-only footprint of a bare exec_module. Adding tests cannot move this +# number, which is why the floor's usual remedy does not apply. +# REMOVE THIS ENTRY when #7597 is fixed. The gate enforces that itself: once +# attribution works the file reports >= 82.0% (floor + its 2.0pp band), which is +# the `graduated` verdict -- an ::error:: demanding this line be deleted. So the +# exemption cannot outlive the defect silently. +src/kiro_crew/builtin_skills/pipeline-conductor/scripts/fleet_probe.py 14.8 # 34/229 src/kiro_crew/channel.py 70.5 # 258/366 src/kiro_crew/cli_chat.py 29.5 # 46/156 src/kiro_crew/cli_cloud.py 61.2 # 178/291 diff --git a/docs/system-specs/modules/metrics.md b/docs/system-specs/modules/metrics.md index 187352135863..6daa3a6b4e5d 100644 --- a/docs/system-specs/modules/metrics.md +++ b/docs/system-specs/modules/metrics.md @@ -7,7 +7,10 @@ The trunk is designed so future work is purely adding instrument calls at call sites — never changing this plumbing. **Default OFF** (`telemetry.enabled: false`): all metric call sites are cheap no-ops and nothing is written or exported, byte-identical to no telemetry (mirrors the `mcp_gateway.enabled` / -`skills.lazy_load` opt-in convention). +`skills.lazy_load` opt-in convention). Any instrument that needs durable state to +survive a process death — currently only the session-lifetime breadcrumb — must +gate that state on the same consent, and fail closed when consent cannot be +read, so this sentence stays literally true. Source: `src/kiro_crew/metrics/` — `schema.py`, `recorder.py`, `provider.py`, `local_exporter.py`, `http_metrics.py`. Tests: `test/metrics/`. @@ -141,9 +144,10 @@ to `0` (cap disabled) rather than being interpreted as "prune everything". **Default posture — nothing collected, nothing leaves the host.** `telemetry.enabled` defaults `false`, so every metric call site is a cheap no-op -and no file is written. Even once local collection is enabled, `otlp_endpoint` -defaults empty, so **no data ever leaves the machine unless the operator -explicitly sets an OTLP endpoint.** +and no file is written — including the session-lifetime breadcrumb, which is +gated on the same consent and fails closed. Even once local collection is +enabled, `otlp_endpoint` defaults empty, so **no data ever leaves the machine +unless the operator explicitly sets an OTLP endpoint.** **Easy opt-in (four equivalent ways):** - **Config flag:** set `"telemetry": {"enabled": true}` in `~/.kiro/crew/config.json`. @@ -343,10 +347,13 @@ Tests: `test/metrics/test_resource_attrs.py`. | `kirocrew.session.startup.duration` | histogram (ms) | `outcome` (`ready` / `auth_required` / `error`), `spawned` (bool), `backend` (`kiro`) + `phase` (`total` / `spawn_init` / `session_new` / `session_load` / `set_model`), `channel` (conversation source), `resumed` (bool) on the kiro path | Two sites. **claude**: `acp/client.py::AcpClient.ensure_ready()` — times cold-start (spawn + session init) and emits in a `finally` so every exit path is measured, with no `phase` attr. **kiro** (default): `providers/acp.py::_emit_kiro_startup_metric` — one `phase=total` point PLUS one point per internal phase; `spawned` is unconditionally `True` because `_start_kiro_runtime_impl` always spawns a fresh runtime (the warm fast-path returns before reaching either site and is NOT measured). `outcome` defaults to `"error"` so an unexpected exception is never mislabeled `"ready"`. Consumers MUST treat only the end-to-end point (`phase` absent or `total`) as a startup — the phase points are components of one startup. `channel` comes from `messaging.link::telemetry_channel_of`, a closed label set (an unrecognised key classifies as `other`, never the key itself) answering WHICH surface paid the cost; `resumed` separates the `session/load` path from `session/new`. `session_load` is recorded only when a resume was attempted, and `session_new` only when `create_session` actually ran, so a resumed startup never reports a near-zero `session_new`. | | `kirocrew.session.pool.decision` | counter | `outcome` (`hit` / `miss_empty` / `bypass_resume` / `bypass_stateless` / `bypass_cwd` / `bypass_effort` / `bypass_env` / `disabled` / `other`), `channel` | `session.py::SessionManager._record_pool_decision`, one point per `get_or_create` warm-pool decision. Exactly one reason is reported per decision — the disqualifiers form a disjunction, so branch order picks the reported reason, not the outcome. Deliberately a counter rather than an attribute on the startup histogram: "was the pool used" and "how long did startup take" are separate questions, and crossing them would multiply every phase series. `bypass_resume` quantifies how often a `resume_sid` disqualifies a session from the pool. Values are pinned by `session.POOL_DECISIONS`. | | `kirocrew.session.resume.outcome` | counter | `outcome` (`loaded` / `fallback_replay` / `no_session_file`), `channel` | `providers/acp.py::_emit_kiro_startup_metric`, emitted only when a resume was attempted. Distinguishes a lossless native resume from the degraded fallback (fresh `session/new` plus history replay on the Kiro Crew side, taken when `session/load` exhausts `_RESUME_MAX_ATTEMPTS` against a stale lock) and from a resume skipped because the session file was gone. | +| `kirocrew.session.duration` | histogram (ms) | `end_reason` (`reset` / `removed` / `unclaimed` / `destroyed` / `discarded` / `shutdown` / `retired` / `recycled` / `evicted` / `crashed`), `session_source` (via `messaging.link::telemetry_channel_of`) | `metrics/sessions.py::record_session_ended` for every clean end; `metrics/sessions.py::backfill_crashed_sessions` for `crashed`. **Why the reason is passed in, never derived.** There is no single teardown funnel: `session_lifecycle.py` holds six end paths that each pop the registry themselves (`reset` / `remove` / `remove_if_unclaimed` / `destroy` / `discard_conversation` / `close_all`), and four more pop it outside all of those — `retire_kiro_identity_sessions` reports `retired` for the sessions it tears down on an identity change, `reload_provider_factory` reports `retired` too for the registry it clears on a provider switch (both are "this process is no longer the right host for these sessions", which is why they share a label rather than minting a second one), `session_compaction.py::CompactionCoordinator._recycle_held` reports `recycled` (context overflow replacing the provider in place), `session_background.py::recycle_heartbeat` also reports `recycled` (the background runtime replacing its own session at cycle end), and the two dead-provider evictions in `session_allocation.py` report `evicted`. So each site states its own `end_reason`, the same contract `metrics/turns.py` holds its callers to. Every path calls this in the SAME event-loop tick as its `_sessions.pop`, before its first `await`: the call is non-blocking (an in-memory pop, an in-memory histogram record, one `unlink`), and a later call lets a racing cold start register a SUCCESSOR under the reused key whose record the predecessor's teardown then consumes. `reset` is a PATH label rather than a cause — the idle sweep and a slot reset both reach teardown through it, and the finer causes are already counted by `kirocrew.session.idle_expired` and `kirocrew.watchdog.recovery.outcome`; the other four are separate precisely because they never take the `reset` route. **EVERY registry removal must record an end, and that is a correctness requirement rather than a completeness one.** An unrecorded removal does not merely lose a sample: the crumb below survives it, so the next boot reports the session as `crashed` — a failure that never happened, injected into the one population this histogram exists to measure. Four such paths shipped unrecorded and were caught in review, which is why the gate for this is `test_every_registry_removal_records_an_end`, a fail-closed AST walk over every module that mutates the registry (recognising `pop`, `del` and `clear`) rather than a hand-kept list of method names that a new path silently escapes. **The crumb is what makes a crash countable.** A start time held only in memory dies with the process, which is exactly the population most worth measuring, because a crash runs no teardown path at all. So `record_session_started` drops one small JSON file under `/metrics/open-sessions/`, named by SHA-256 digest of the session key PLUS the writing pid and the start time, with the key stored inside (keys carry channel ids and slot names and are not filesystem-safe; `session_map.json` already holds every key in the same data home, so this is not new exposure). **The name identifies the writer and the generation, not just the key, and that is what makes every unlink here safe.** A session key is not unique across processes — `BACKGROUND_KEY` is a fixed constant, so a `kirocrew run` and the gateway hold the same key at the same time with a live session each — so while the name was the digest alone they shared one file, and whichever ended first unlinked the other's record; the locks in this module are THREAD locks and never spanned processes, so nothing here could have prevented it. With the writer and the generation in the name, a crumb can only ever be removed by the session that wrote it. Three consequences follow. A clean end unlinks the ONE generation it can name, and an end with no live table entry unlinks NOTHING — the files sharing that digest belong to other processes or earlier runs, and removing them is precisely the bug the name now prevents. A start that displaces a live generation of the same key, and an eviction under `_MAX_LIVE_SESSIONS`, must each reap the generation they orphan, because once the table forgets a generation nothing can ever name its file again; both ride the same awaited worker hop as the start's own write, so those unlinks stay off the event loop without opening a second cancellation window. And the backfill legitimately finds SEVERAL files for one key, each a distinct session judged on its own recorded owner. Whatever survives to the next boot is a session that never ended cleanly, which `backfill_crashed_sessions(started_before)` emits as `crashed` and unlinks; it is also the only reader of these files, so an unparseable crumb is reaped there rather than by an end unlinking blind. `started_before` is the calling process's own start time, so a crumb at or after it is left alone. **The crumb also records WHO owns it — the writing process's pid plus a per-process start identifier — and the backfill skips any crumb whose owner is still running.** The cutoff alone was not enough: it only ever protected THIS process's own crumbs, and `cli.py` and `eval/runner.py` each build their own session manager against the same data home, so a `kirocrew run` session started before the gateway booted was read as a crash. That both invented a sample and deleted the live crumb, losing the real crash that session might later suffer. The start identifier is what makes pid reuse safe: a recycled pid belongs to a different process and must not look like the owner, and that includes reuse of the CURRENT process's own pid — a container hands the gateway PID 1 on every restart, so a crashed predecessor's crumb arrives carrying this process's pid, and skipping it on the strength of the pid alone meant its crash was never emitted and its file never cleaned on any later boot either. **Liveness comes from `platform_compat.pid_exists`; the start identifier only refines it.** `get_process_start_id` returns None on Windows and for any process the caller may not introspect, and its own contract says a None must NOT be read as a mismatch — using it as the liveness test judged every owner dead on that entire platform and reaped live sibling sessions there. The ownership check fails CLOSED — a live pid whose identity cannot be read counts as running, so an ambiguous crumb waits for a later boot rather than becoming a fabricated crash; losing a real crash sample costs one data point, inventing one corrupts the population this instrument reports. A crumb with no ownership recorded (written before this field existed) falls back to the cutoff alone. **A registration that never became a session is discarded, not ended:** `discard_session_start` pops the table entry and unlinks that generation's crumb WITHOUT emitting, for the cancellation window between inserting a session into the registry and finishing its start record — a session that never lived has no lifetime to report, and it must not be left as a crumb the next boot calls a crash. It is deliberately not a twelfth `end_reason`, because those describe how a live session ENDED. **Exactly-once accounting.** A crumb is consumed at most once, so the teardown paths — which are NOT mutually exclusive, the idle sweep calls `reset` — cannot double-count between them, and a backfilled session cannot be counted again on the boot after. The clean path deliberately reads neither the crumb nor the popped entry's `created_at`: consuming a DIFFERENT record than the crash path would let a clean end's leftover crumb be re-emitted as `crashed`, and `created_at` is reset when a provider is recycled in place, so it measures provider age rather than registry residency. **A crashed session's END time** is its transcript mtime (`/sessions/.jsonl`, appended as the conversation runs, so it is the last moment the session was observably alive and the honest post-hoc maximum). A session with no transcript — a subagent leaves only a replay log — yields no end time and is unlinked WITHOUT an emit rather than recorded as a 0ms lifetime: absence reads as "no data", a recorded zero renders as a real zero. **Two locks, split by what they may be held across.** `_live_lock` guards the in-memory start table ONLY and is never held across I/O, because every end takes it on the event loop; `_crumb_io_lock` orders the two worker-thread paths that touch crumb FILES — the writer, and the backfill's read-decide-unlink — and nothing on the loop may take it. Coupling them was a real stall: the backfill held one lock across a crumb read plus an unlink on a worker while a teardown waited for that same lock on the loop. The end path needs no crumb lock at all, because ownership already partitions the work — the backfill skips every crumb whose owner is live, so the crumbs a session ends in THIS process and the crumbs the backfill reaps are disjoint sets. It is also worth stating that these are THREAD locks and so never protected anything against a sibling PROCESS; that is exactly what the ownership field is for. The write itself runs on a worker thread (`asyncio.to_thread`) and is AWAITED while the caller still holds the session registry lock, which is what satisfies two constraints that look opposed: the syscalls never run on the event loop, where a slow or network-homed data home would stall every gateway task behind one session insertion, and nothing can interleave, because starts, ends and successors all serialise on that registry lock. Three earlier shapes each satisfied one and broke the other — a fire-and-forget pool write let a late writer delete a successor's crumb (an ended-key set, then a generation token compared before and after the write, both failed, because every check still acted on a path the successor SHARED), and writing inline fixed the ordering by putting filesystem I/O on the loop. Naming the file after its writer and generation is what finally removed that shared path, so the post-write check now removes only this writer's own file. The END path's unlink stays inline and synchronous — one syscall, outside every lock — because moving it behind an await reopens the race the same-tick rule exists to prevent: a successor registering during those awaits has its record consumed by its predecessor's teardown. Deferring that unlink to a worker was considered and declined: making it reliable requires `record_session_ended` to become async across roughly ten teardown sites, each gaining a cancellation window, and the residual it would buy back is now bounded to one case — a process killed between the pop and the unlink leaves a crumb the next boot counts as a crash. **The crumb is written only under telemetry consent**, checked fail-closed: it exists solely to feed this instrument, which is a no-op without consent, so writing one on an unopted host would persist state nothing can read against a documented default of collecting nothing. Bounded throughout: `_MAX_BACKFILL_EMITS` (2000) caps a runaway crumb directory while still unlinking everything it walks, and `_MAX_LIVE_SESSIONS` (4096) drops the oldest half of the table, unlinking each dropped generation's crumb as it goes — both cost samples, never memory. Own bucket family `_SESSION_BUCKETS_MS`. | +| `kirocrew.session.started` | counter | `session_source` (via `messaging.link::telemetry_channel_of`) | `metrics/sessions.py::record_session_started`, one increment per session admitted to the registry, called from the sites that insert into `_sessions` (`session_allocation.py` cold-start and resume paths, `session_background.py`). The denominator the `kirocrew.session.duration` population is a subset of: not every registry removal records an end, so `started` counts admissions while `duration` counts the endings actually observed, and the gap between the two is itself readable. `session_source` is the SAME `telemetry_channel_of` derivation the duration histogram and `metrics/turns.py` use, so all three group by one bounded label set (an unrecognised key classifies as `other`, never the key itself). The increment and the `_live_starts` write are in-memory and therefore safe under the session registry lock; only the crumb touches disk, off-thread. | | `kirocrew.turn.duration` | histogram (ms) | `outcome` (`ok` / `timeout` / `cancelled` / `tool_stall` / `stale_recover` / `stall_exhausted` / `error` / `unclassified`), `session_source` (via `messaging.link::telemetry_channel_of`), `model` + `provider` (omitted when the caller cannot resolve them) | **Two emit owners, exactly one sample per turn.** (1) `dashboard/handlers/usage.py::persist_token_record_async` — the call EVERY dispatch surface already makes once per turn at EVENT_COMPLETE, which is what makes cron, the heartbeat, memory consolidation, subagents, task-runner steps, workflow stages and the messaging channels present in this metric at all; before that they emitted nothing and the page reported the interactive median as the system's health. It reuses the built row's own `duration_ms`, so the row store and the histogram cannot disagree about one turn. (2) `dashboard/chat_runner.py::_emit_turn_metric` in `_run_chat`, which passes `emit_metric=False` to its persist call and emits itself — that persist sits behind `usage_has_billing`, so a turn that timed out having billed nothing writes no row and would otherwise lose its sample, and only this surface holds the EFFECTIVE session key (a linked channel conversation is attributed to its channel, not to the `slot.key` the row is filed under) and the spent-recovery-budget knowledge that labels `stall_exhausted`. `metrics/turns.py::turn_outcome` owns the mapping (`""`/`None`/`end_turn`/`stop`/`completed` → ok; the two watchdog stop reasons map to their own outcomes — checked BEFORE the `timeout` substring — so a recovered stall is never counted as a generic fault and the stall population stays visible; a stall arriving with its 3-attempt recovery budget already spent, or on a NESTED turn (`_prompt_depth > 0`, which the recovery branches never re-queue — it dies with "please retry"), labels `stall_exhausted`, which IS a terminal fault to the aggregator, so the recovered-stall exclusion cannot hide a session that dies needing user action). `cancelled` is a user pressing Stop, matched by EXACT equality against `STOP_REASON_CANCELLED` so the watchdog's `"error: cancel unacked"` still reaches the error branch; it used to fold into `error`, which put every deliberate cancel into `fault_rate`'s numerator, and it is now excluded from the numerator while REMAINING in the denominator (the turn ran). `unclassified` marks a turn whose surface passed a bare `TurnUsage` and so had no stop reason to give; it is in neither the numerator nor the denominator of `fault_rate` (in the numerator it would invent a fault for every clean background turn; in the denominator alone it would dilute the rate towards zero as background traffic grows), and its count is readable as `outcome["unclassified"]`. One histogram powers turn latency p50/p90 AND fault rate. The value is `duration_ms or elapsed_ms`: the acp provider always reports `TurnUsage.duration_ms == 0` (only claude_code fills it), so the caller must pass the locally measured wall clock as `elapsed_ms` or nothing is ever emitted. A still-zero value skips the emit deliberately — absence renders as "no data", whereas a recorded 0 would render as a plausible 0ms p50. **What it measures:** the wall clock starts at turn start, so a turn parked on an interactive tool-approval prompt counts operator thinking time. No finer-grained source exists on the acp path, so this is "turn wall-clock", not pure model latency — a high p90 can mean slow approvals rather than a slow model. | | `kirocrew.turn.tokens` | counter (token) | `direction` (`input` / `output`), `model` + `provider` | Same two owners and the same exactly-once split as `kirocrew.turn.duration`, emitted from `metrics/turns.py::emit_turn_usage` off the same built row, so volume and latency describe one population. ONE instrument with a `direction` attribute rather than an `.input`/`.output` pair: the two series then carry an identical attribute set by construction (a `model` added to one cannot be forgotten on the other), the dashboard's counter path already reports every attribute combination under `by_attr`, and `direction` is a two-value enum so the cardinality cost is exactly 2x. **Positive-only.** Each backend fills only the dimensions it bills in; the kiro/acp backend reports zero for every token field, so emitting the zeros would publish a full series of them per turn and a recorded 0 reads as a measured zero rather than as "this backend does not report here". A negative value is also dropped — a monotonic counter cannot take it back. | | `kirocrew.turn.credits` | histogram (credit) | `model` + `provider` | The turn's billed amount on a credit-billing backend (kiro/acp). Emitted only when non-zero, which is what makes it mutually exclusive with `cost_usd` on a given host (`acp/types.py`: "Consumers read whichever is non-zero"). Two instruments rather than one with a `currency` attribute because credits and dollars are not the same quantity: percentiles over the union have no unit. Own bucket family `_CREDIT_BUCKETS` (0.025–2500), calibrated against 17,240 real per-turn rows (min 0.03, p10 0.076, p50 6.8, p90 53, p99 155, max 658). NOT in `_HISTOGRAM_BUCKETS_MS`, and claimed by name in `_aggregate` so it never reaches the `*_ms` surface — see the bucket-boundaries section. | | `kirocrew.turn.cost_usd` | histogram (usd) | `model` + `provider` | The same amount on a dollar-billing backend (claude_code / bedrock), same non-zero gate, same owners. Own bucket family `_USD_BUCKETS` (0.001–100). This is the one bucket array with no local calibration — a credit-billing host reports zero for every `cost_usd` row by construction — so it is sized from published per-token pricing against the observed token range and is worth re-checking once a dollar-billing host reports. Reported through `_amount_stats`, which keeps six decimals so a sub-cent turn does not round to `0.0`. | +| `kirocrew.tool.call.duration` | histogram (ms) | `tool_kind` (`read` / `fetch` / `search` / `edit` / `write` / `create` / `delete` / `move` / `execute` / `think` / `switch_mode` / `client_built_in` / `mcp` / `other`), `outcome` (`completed` / `failed` / `cancelled`) | **Round-trip latency of one tool call, keyed by kind and never by tool name.** The tool NAME is deliberately not an attribute: MCP names are unbounded (any installed server contributes its own) and the recorder caches one instrument per distinct attribute value with no eviction, so a name-valued label is a cardinality bomb. `tool_kind` is the ACP `kind` normalised by `metrics/tool_calls.py::classify_tool_kind` and pinned to the closed `TOOL_KINDS` set; the `kind` field arrives verbatim from the agent, so an unrecognised or agent-authored kind — including an empty one — folds to `other`, which is what stops a tool name minting a series. An MCP-served call is labelled `mcp` whatever kind it reported, decided by the trusted `_meta.kiro.mcpServerName` rather than the reported kind: an MCP server's kind vocabulary is its own, and "this call left the process over MCP" is the more useful fact than a builtin's finer verb. `outcome` carries exactly the three terminal ACP statuses — a non-terminal update (`pending` / `in_progress` / absent) records nothing and leaves the clock running, so no fourth value is reachable. A `<= 0` span skips the emit, the same rule `kirocrew.turn.duration` follows. The clock is `time.perf_counter`, NOT `time.monotonic`: on Windows the latter advances in ~15.6ms ticks, so any call completing inside one tick measured exactly 0.0 and was dropped by that guard — which silently hid every sub-tick tool call on the platform, i.e. most cached reads. `perf_counter` is the highest-resolution monotonic clock available everywhere, so the guard keeps meaning "unmeasurable" rather than "fast". **Site: two parser layers, one registry, one sample per call.** There is no single choke point, because the two backends parse through different code: the shared builders in `acp/_dispatch.py` (`_build_tool_call_event` starts the clock once the trusted MCP identity is resolved; `_build_tool_result_event` stamps the terminal status BEFORE its output parse can return None, so an output-less completion is still measured), where the scope arrives as `cache_scope` forwarded by `acp/session_handle.py` as the frame's `sessionId`; and `acp/client.py::AcpClient._extract_tool_call_update` with its result-extracting sibling, which shape the same frames inline and scope on the client's own `_session_id`. Every other surface consumes the emitted `AcpEvent` stream downstream of these two, so instrumenting both parsers covers all of them. The registry is keyed `scope|tool_call_id`, spelled exactly as `_dispatch`'s own tool-input cache key: a backend-assigned `toolCallId` is unique only WITHIN one backend session while one runtime hosts many, so the scope is what stops two sessions reusing an id from colliding, and a finish in the wrong scope never steals another session's entry. Both layers derive one frame's scope from the same session id, so whichever sees a frame second pops the entry the first opened and the call is sampled once rather than twice — the disjoint-parser assumption failing would then be a harmless miss instead of a double count. A start is idempotent per scoped id (the first start time and kind win, so a `tool_call_update` refinement cannot shrink the span), a finish with no matching start emits nothing, and the registry is bounded (`_MAX_OPEN_CALLS`, dropped wholesale on overflow: missing samples for in-flight calls, never a leak). The watchdog's `acp/liveness.py::ToolCallState.dispatch_ts` is deliberately NOT reused — it is a single last-tool-wins slot, right for stall attribution but wrong for a histogram because interleaved calls overwrite each other, and it exists on only one of the two paths. Own bucket family `_TOOL_CALL_BUCKETS_MS`. Best-effort throughout: a tool call never fails because its telemetry did. | | `kirocrew.watchdog.action` | counter | `action` (`deferral` / `probe` / `cancel`), `verdict` (`working` / `dead` / `unknown` / `stuck_input`), `evidence_class` (`established_flat` / `mcp_flat` / `shell` / `shell_absent` / `wait` / `degraded`), `window` (`narrowed` / `extended` / `standard`), `agent_override` (bool) | `acp/session_handle.py::AcpSessionHandle._emit_watchdog_metric`, one point per watchdog DECISION in `_dispatch_events`: `deferral` from `_log_working_deferral` (rides its 10-min rate limit, so an hours-long WORKING build contributes a bounded handful of points, not one per tick), `probe` at the stale-probe send, `cancel` before `_end_stalled_tool`. `evidence_class` is `_watchdog_evidence_class` — a prefix/shape bucket of the free-form oracle evidence (pids/deltas/commands never emitted). `window` encodes the effective window selection: `narrowed` = a tool-branch evidence TAG reduced the suspect window below the build-scale default (1h) — `established_flat` to the model-silent budget (minutes), `shell_absent` to the ordinary silence budget (`stale_window_secs`, 300s) because the shell command has no process to its name; `extended` = model-wait-branch `established_flat` extended the stale window from 300s (`stale_window_secs`) to 900s (`model_silent_probe_secs`) for a non-streamed server-side think; `standard` = ordinary window in all other cases. `agent_override` is the per-agent watchdog-override BOOLEAN from the `WatchdogSettings` snapshot — deliberately NOT the agent name (free-form ⇒ cardinality bomb; per-agent joins happen via the row store's `agent` + `stop_reason` fields below). Guardrail query: `action=cancel, evidence_class=mcp_flat, window=standard` must not increase — the narrowed window may only affect `established_flat` and `shell_absent`. | | `kirocrew.watchdog.idle.duration` | histogram (ms) | `action`, `evidence_class` | Same emit helper, same decision points; value = the branch's idle clock (`_tool_idle` / `_stale_idle`) at decision time, converted to ms at the emit site because the dashboard's generic aggregation reports every histogram under `*_ms` keys (a seconds instrument would render 1000x off). Answers whether 900s is right for LLM-shaped stalls (idle-at-action distribution per evidence class). Own bucket family `_WATCHDOG_IDLE_BUCKETS_MS` (1s–4h, densest at the 300/900/3600-second window boundaries). | | `kirocrew.watchdog.recovery.outcome` | counter | `mechanism` (`stale_recover` / `tool_stall`), `outcome` (`recovered` / `exhausted`), `attempt_bucket` (1–3) | `dashboard/chat_runner.py::_emit_recovery_outcome`, derived from the per-slot retry budgets the stop-reason branches maintain (`slot._stale_recovery_retries` / `slot._tool_stall_retries`). `exhausted` emits in the stall branches when a budget hits its cap ("start a new chat"); `recovered` emits at the budget-reset block when a turn completes with outcome `ok` while a stall budget is armed — the stall branches return early, so an armed budget reaching that reset is by construction a completed recovery cycle (gated on `ok` so a user cancel of the recovery turn never counts as a recovery). `attempt_bucket` clamps to the 3-attempt cap (closed enum, mirrors the CLI's `attempt_number_bucket`). Every `recovered` point is one prevented hang. Fault accounting for the exhausted case lives on the turn histogram, not here: the final turn of an exhausted cycle labels `stall_exhausted` (see `kirocrew.turn.duration`), so a dead session counts toward `fault_rate` while this counter stays pure mechanism telemetry. | @@ -355,6 +362,13 @@ Tests: `test/metrics/test_resource_attrs.py`. | `kirocrew.mcp.lazy_load.count` / `.duration` | counter + histogram (ms) | `transport` (`stdio`) | `mcp_gateway/gatewayd.py::_emit_lazy_load_metrics` — legacy lazy-spawn path (also emits backend.acquire). | | `kirocrew.mcp.warm_pool.acquire` | counter | `result` (`hit` / `miss`) | `mcp_gateway/prewarm.py::HotKeyStore.record_outcome` (emitted outside the lock). | | `kirocrew.skill.lazy_load.count` / `.duration` | counter + histogram (ms) | `hit` (bool) | `skills.py::SkillsLoader.load_skill` via `_emit_lazy_load_metric` (best-effort; never breaks skill loading). | +| `kirocrew.subagent.spawned` | counter | `concurrency` (int — the live running count INCLUDING this spawn, bounded by the manager's own concurrency cap, so a MAX over it is the high-water mark without a second instrument), `batched` (bool — the spawn is a member of a fan-out wave rather than a lone one) | `subagent_manager/admission.py::SpawnAdmissionCoordinator._log_spawned_impl` — the confirmed-start funnel, reached only after a spawn is admitted, so a rejected or never-started spawn is not counted (a promise an admission-time increment could not make). The `emit_counter` import is function-local, not module-level: `bind_component_globals` rebinds each `*_impl`'s `__globals__` to the owning namespace, where a module-level import here would not be visible. | +| `kirocrew.cron.fires` | counter | `kind` (`agent` / `script` / `command` — the dispatch shape; `script` and `command` bypass the model entirely, so this is the split between a job that costs tokens and one that costs none), `trigger` (`scheduled` / `manual`) | `cron.py::CronService._run_job_isolated`, one increment per execution, emitted BEFORE the jitter sleep so a run cancelled during jitter still counts as fired. No job id, name, message or schedule string reaches the attributes. | +| `kirocrew.artifact.created` | counter | `kind` (the closed `ALLOWED_KINDS` set), `source` (the closed `ALLOWED_SOURCES` set), `kind_auto` (bool — the kind was inferred from the content rather than pinned by the caller) | `artifacts.py::ArtifactStore.create`, emitted after the write and its change notification, so a failed create contributes nothing. `kind` and `source` are exactly the values `_validate_kind` / `_validate_source` already close over, so no slug or free-form label can enter the series. | +| `kirocrew.workflow.runs` | counter | `authored` (bool — the run writes its own script from an intent instead of executing supplied source), `replay` (bool — a restart-subtree reusing cached agent results rather than re-calling the model) | `workflows/runner.py::WorkflowRunner.run`, the one place every run passes before executing anything, foreground or background (the background entry point drives this same method). Both attributes are booleans over run mode; no intent text, script or run id is emitted. | +| `kirocrew.context.compactions` | counter | `success` (bool — whether the compaction ATTEMPT itself completed) | `session_compaction.py::CompactionCoordinator._fire_compact_callback`, one point per compaction that reached a verdict. Placed on this method rather than inside the callback because every compaction passes through here whether or not a surface registered a compact callback, and the early return below would otherwise drop exactly the surfaces that register none. The sum is compaction ATTEMPTS. `success` is deliberately NOT an effectiveness measure: the coordinator learns whether the backend finished the compaction, not how much context it actually freed, and no before/after context reading is taken, so a compaction can complete while reclaiming little. **The counter's `success` is also not the callback's.** `_recycle_held` fires this funnel with `success=True` because the SESSION now has headroom — which is what the callback needs to know — but it is reached exactly when an in-place `/compact` FAILED and the provider had to be replaced instead. Counting that as a successful compaction would report the failure mode as the success case, so the counter reports `False` there. The recycling marker is set for the whole of `_recycle_held` and popped only after this call, which is what separates the two populations at this point; `test_a_failed_compact_that_recycles_is_not_counted_successful` pins that ordering. | +| `kirocrew.mcp.reconnects` | counter | `pool` (bool — the reconnecting stub was a warm-pool bridge rather than a directly owned one) | `mcp_gateway/stub.py::_reconnect`, one increment per stub reconnect to a restarted gateway daemon, emitted after the replayed handshake is delivered. Each increment is a bridge that died and was rebuilt under a live session, so a rising rate is daemon instability that sessions are absorbing silently. The daemon detail string and the per-session reconnect ordinal go to the log line only, never the attributes. | +| `kirocrew.approval.decisions` | counter | `decision` (`allow` / `auto_approve` / `deny` — the gate's own bounded action constant), `security_deny` (bool — separates a hard security refusal from a policy-state deny) | `hooks.py::ToolHookResult._count`, called by the four result factories and nowhere else, so it fires exactly once per gate consultation. A surface that OVERRIDES the gate constructs a result directly (the chat runner downgrading an auto-approve to an interactive card, for instance) and is deliberately NOT counted: counting every construction would report one request as two decisions and keep a count for a verdict that was discarded. Counting from the factories rather than from `__post_init__` behind a flag is what removes the flag — it had no reader but the counter, so it was state carried purely to signal. Both attributes are bounded by construction — no deny reason, tool name or command reaches the recorder. | | `kirocrew.gateway.boot.duration` | histogram (ms) | `server` (`dashboard` / `api`), `outcome` (`ready`) | `dashboard/server.py::start_dashboard` / `start_api_server` — boot-to-ready: wall-clock from the server's `start_time` until full init completes and it is about to accept traffic. Emitted via `metrics/http_metrics.py::record_boot_to_ready`. Best-effort; never blocks startup. | | `kirocrew.gateway.request.duration` | histogram (ms) | `method` (fixed HTTP-verb allowlist, else `OTHER`), `route_template` (matched aiohttp canonical TEMPLATE, e.g. `/api/artifacts/{slug}`, else `__unknown__`), `status_class` (`1xx`..`5xx` / `other`) | `metrics/http_metrics.py::make_route_latency_middleware` — outermost gateway middleware on BOTH `start_dashboard` and `start_api_server`. Times full in-gateway HTTP handling; upgraded WebSocket connections and `text/event-stream` SSE responses are excluded so connection/turn lifetime cannot pollute request latency. **Bounded cardinality** (see below). | @@ -448,13 +462,35 @@ whether or not it runs crons). A pod is not this case: it boots with its own `KIROCREW_HOME` and copies no crons, sessions, or databases, so its inventory belongs to a different install and publishing it is correct. +### The counters beside `stats.py`, and why they are not the same numbers + +`stats.py`'s in-memory singleton already counts adjacent ground: `sessions_created`, +`subagents_spawned` and its tool-approval trio, all consumed by one dashboard +system handler. Those tallies are NOT merged into the counters above, and the +counters are not derived from them. The singleton is a process-local count that +resets on restart and answers "what has this gateway done since it came up"; the +instruments above are time-series carrying attributes, survive a restart, and +answer "what is the rate, broken down by shape". Merging them would mean giving +the singleton attributes and durability it has no consumer for. + +The hazard the split creates is that the names are near-identical while the +POPULATIONS differ, so the two numbers can legitimately disagree and be read as a +bug. `kirocrew.approval.decisions` is the sharpest case: it counts gate VERDICTS, +every consultation including the auto-approves and denies that never surface to a +person, whereas the singleton's approval trio accrues on the surface where a human +acted. `kirocrew.subagent.spawned` fires on the confirmed-start funnel after +admission, so it excludes rejected spawns a request-time tally would include. +Each row above therefore states which population its counter measures; a +comparison across the two systems is only meaningful once those definitions are +read. + ### Histogram bucket boundaries: per instrument, not shared Boundaries live in two metric-name → boundaries maps in `metrics/provider.py` — `_HISTOGRAM_BUCKETS_MS` for millisecond instruments and `_HISTOGRAM_BUCKETS_BY_UNIT` for everything else — merged by `histogram_bounds()`, from which one `View(instrument_name=…)` is built per -instrument. Five families, each sized to its instrument's measured range: +instrument. Eight families, each sized to its instrument's measured range: | Family | Range | Unit | Instruments | |---|---|---|---| @@ -462,9 +498,22 @@ instrument. Five families, each sized to its instrument's measured range: | `_STARTUP_BUCKETS_MS` | 1ms – 60s | ms | `session.startup`, `chat.first_token`, `mcp.lazy_load`, `gateway.boot` | | `_TURN_BUCKETS_MS` | 1s – 1h | ms | `turn.duration` | | `_WATCHDOG_IDLE_BUCKETS_MS` | 1s – 4h | ms | `watchdog.idle` | +| `_TOOL_CALL_BUCKETS_MS` | 0.5ms – 1h | ms | `tool.call.duration` | +| `_SESSION_BUCKETS_MS` | 1s – 7d | ms | `session.duration` | | `_CREDIT_BUCKETS` | 0.025 – 2500 | credit | `turn.credits` | | `_USD_BUCKETS` | 0.001 – 100 | usd | `turn.cost_usd` | +`_TOOL_CALL_BUCKETS_MS` spans a sub-millisecond builtin read to a multi-minute +build in one instrument, so it keeps both ends: a `_FAST` ceiling would floor +every long shell command into the overflow bucket, and a `_TURN` floor would +collapse the entire builtin population onto boundary one. `_SESSION_BUCKETS_MS` +is the only family measured in hours and days — a speculative session removed +before its first turn lives seconds, while a dashboard tab left open across a +working week is ordinary — so it is densest from a minute to a few hours where +interactive sessions land, keeps sub-minute bounds because the short-lived +teardown paths (`unclaimed`, `destroyed`) are a real population, and carries a +7-day ceiling so a long-lived tab is not floored into overflow. + **Why the unit map is separate.** The dashboard's generic histogram aggregation reports every statistic under `*_ms` keys and the frontend formats those with a millisecond suffix, so `_HISTOGRAM_BUCKETS_MS` membership is a claim that the diff --git a/docs/system-specs/modules/session.md b/docs/system-specs/modules/session.md index d08fdc80e228..fd7879003aed 100644 --- a/docs/system-specs/modules/session.md +++ b/docs/system-specs/modules/session.md @@ -855,6 +855,43 @@ start_pool() └── _ensure_background() → BACKGROUND_KEY session (persistent) ``` +## Removing a session from the registry: record the end + +**Every path that removes an entry from `_sessions` must report that removal to +`metrics/sessions.py`** — with `record_session_ended(key, end_reason=...)` for a +session that lived, or `discard_session_start(key)` for a registration being +rolled back before it ever became one. This is a correctness requirement on +lifecycle code, not a telemetry nicety, and it is documented here rather than only +in the metrics module because the people who can break it are editing this +subsystem. + +The reason it matters more than a missing sample: a session start writes a +breadcrumb file that survives process death, which is what lets a session killed +with its gateway be counted at all. A removal that reports nothing leaves that +breadcrumb behind, and the next boot reads a surviving breadcrumb as a crash. So +an unrecorded removal does not lose a data point — it **fabricates a crash that +never happened**, inside the one population the instrument exists to expose. + +Practical rules when you add or change a removal path: + +- Report it in the SAME event-loop tick as the `pop` / `del` / `clear`, before the + path's first `await`. Later than that, a racing cold start can register a + successor under the same key and have its record consumed by the predecessor's + teardown. +- Give it its own `end_reason` if it is genuinely a different event. The enum is + closed and lives in `metrics/sessions.py`; reusing a label merges two + populations, and a metric is not a good reason to grow a lifecycle signature. +- A registration cancelled mid-flight is NOT an end. Use `discard_session_start`, + which consumes the breadcrumb without emitting a lifetime. + +`test/metrics/test_session_duration.py::TestEveryRegistryRemovalRecordsAnEnd` +enforces this. It is fail-closed and discovers its own scope: an AST walk over +every module under `src/kiro_crew` that mentions `_sessions`, recognising the +`pop`, `del` and `clear` spellings, so a new removal path anywhere fails the gate +the day it lands rather than waiting to be added to a list. A container that +merely shares the attribute name can be exempted with a stated reason, and the +exemption self-voids if that module ever starts writing breadcrumbs. + ## Security: PreToolUse Command Enforcement Command denial is enforced by Kiro Crew's bundled `hooks.py` `PreToolUse` gate, diff --git a/src/kiro_crew/acp/_dispatch.py b/src/kiro_crew/acp/_dispatch.py index 75020cee5701..e82213fa1c4b 100644 --- a/src/kiro_crew/acp/_dispatch.py +++ b/src/kiro_crew/acp/_dispatch.py @@ -56,6 +56,7 @@ AcpEvent, JsonRpcMessage, ) +from kiro_crew.metrics.tool_calls import note_tool_call_started, record_tool_call_finished from kiro_crew.security import redact_credentials, redact_exfiltration_urls logger = logging.getLogger(__name__) @@ -879,6 +880,16 @@ def _build_tool_call_event( _mcp_server_name = _kiro_mcp_server_name(update) if tool_call_id and mcp_server_name_cache is not None: mcp_server_name_cache[_ck] = _mcp_server_name + # Round-trip clock for kirocrew.tool.call.duration. Placed after the trusted + # MCP identity is resolved so an MCP call is classified by its transport + # rather than by the kind it reported; the terminal status is stamped in + # _build_tool_result_event. Keyed by the SAME origin scope as the caches + # above, because one runtime hosts many sessions and a backend-assigned + # toolCallId is unique only within one of them. Idempotent per scoped id, so + # the tool_call_update refinements that follow cannot restart the clock. + note_tool_call_started( + tool_call_id, kind=kind, mcp_server_name=_mcp_server_name, scope=cache_scope + ) # Same lifecycle for the trusted tool name (_meta.kiro.toolName) so the # permission event can reconstruct the canonical mcp____ for # per-tool governance in the app-own-server auto-approve. @@ -992,17 +1003,25 @@ def _mcp_content_text(payload: dict[str, Any]) -> str | None: return "\n".join(parts) -def _build_tool_result_event(update: dict[str, Any]) -> AcpEvent | None: +def _build_tool_result_event(update: dict[str, Any], cache_scope: str = "") -> AcpEvent | None: """Build an ``EVENT_TOOL_RESULT`` from a ``tool_call_update`` carrying output. Two output shapes: ``content[].content.text`` blocks (stream mid-turn), or ``rawOutput.items[]`` (``Text`` / ``Json.stdout``) on ``status=completed``. Returns None when the update carries no output (refinement-only updates are handled by :func:`_build_tool_refinement_event`). + + ``cache_scope`` is the emitting session's origin scope, forwarded only so the + duration histogram closes the same registry entry its start opened. """ tool_use_id = update.get("toolCallId", "") if not tool_use_id: return None + # Before the output parsing below, which returns None for an output-less + # update: a tool that completed with no output is still a completed + # round-trip. A non-terminal status is a no-op here, so a mid-stream update + # leaves the clock running for the real completion. + record_tool_call_finished(tool_use_id, status=update.get("status"), scope=cache_scope) # Parts are collected RAW and redaction runs once over their JOIN, before # the single 8000-char bound. Both orderings matter: bounding first can # split a credential at a cut into fragments no redaction regex matches, @@ -1340,7 +1359,7 @@ def parse_session_update( ) return events if kind == UPDATE_TOOL_CALL_UPDATE: - result = _build_tool_result_event(update) + result = _build_tool_result_event(update, cache_scope) if result is not None: events.append(result) refine = _build_tool_refinement_event( diff --git a/src/kiro_crew/acp/client.py b/src/kiro_crew/acp/client.py index 02b6407ff2f2..0a9de73c7772 100644 --- a/src/kiro_crew/acp/client.py +++ b/src/kiro_crew/acp/client.py @@ -134,6 +134,7 @@ from kiro_crew.kiro_cli import known_kiro_cli_dirs, resolve_kiro_cli from kiro_crew.mcp_gateway.claim import schedule_claim from kiro_crew.mcp_gateway.session_servers import pooled_session_servers +from kiro_crew.metrics.tool_calls import note_tool_call_started, record_tool_call_finished from kiro_crew.resource_status import inject_xdist_auto_cap from kiro_crew.sandbox import ( RLIMIT_PROFILE_SESSION_HOST, @@ -5760,6 +5761,21 @@ def _extract_tool_event(self, msg: JsonRpcMessage) -> AcpEvent | None: ) # Build initial tool input string from raw params tool_call_id = update.get("toolCallId", "") + # Start the round-trip clock here rather than at the yield: this is + # the first moment Kiro Crew sees the call, and the same id's terminal + # status is stamped in _extract_tool_call_update below. Both of this + # class's message loops reach this method, so instrumenting it covers + # them without a second call site in each. + note_tool_call_started( + tool_call_id, + kind=kind, + mcp_server_name=_kiro_mcp_server_name(update), + # getattr: a real client always carries _session_id, but this + # extractor is also driven directly with lightweight test + # doubles (test_acp_tool_identity), and telemetry must not be + # the reason such a double stops working. + scope=getattr(self, "_session_id", "") or "", + ) input_str = "" if tool_call_id and raw_input: input_str = ( @@ -5887,6 +5903,16 @@ def _extract_tool_call_update(self, msg: JsonRpcMessage) -> AcpEvent | None: tool_use_id = update.get("toolCallId", "") if not tool_use_id: return None + # Stamped before the output parsing below, which returns None for an + # output-less update: a tool that completes with no output is still a + # completed round-trip and must not be dropped from the histogram. A + # non-terminal status is a no-op, so a mid-stream update leaves the clock + # running for the real completion. + record_tool_call_finished( + tool_use_id, + status=update.get("status"), + scope=getattr(self, "_session_id", "") or "", + ) output_parts: list[str] = [] diff --git a/src/kiro_crew/artifacts.py b/src/kiro_crew/artifacts.py index 61a11ac233c3..1e4ebd692f59 100644 --- a/src/kiro_crew/artifacts.py +++ b/src/kiro_crew/artifacts.py @@ -60,6 +60,7 @@ WebAppTeardown, webapp_metadata_from_dict, ) +from kiro_crew.metrics.events import ARTIFACTS_CREATED, emit_counter from kiro_crew.publish_provider import DEFAULT_PROVIDER from kiro_crew.security import is_sensitive_path @@ -1174,6 +1175,14 @@ def create( self._write_artifact(art, content) logger.info("artifact created: slug=%s name=%s kind=%s", slug, name, kind) self._fire_change("upsert", slug) + # After the write, so a failed create contributes nothing. ``kind`` and + # ``source`` are the values ``_validate_kind`` / ``_validate_source`` + # already restrict to closed sets, and ``kind_auto`` says whether the + # kind was inferred rather than pinned by the caller. + emit_counter( + ARTIFACTS_CREATED, + {"kind": kind, "source": source, "kind_auto": bool(kind_auto)}, + ) return art def create_image( diff --git a/src/kiro_crew/cron.py b/src/kiro_crew/cron.py index da2d7e0f1127..b2b2d4f280f6 100644 --- a/src/kiro_crew/cron.py +++ b/src/kiro_crew/cron.py @@ -52,6 +52,7 @@ from kiro_crew.constants import env_flag_enabled from kiro_crew.cron_history import CronHistoryStore, CronRunRecord from kiro_crew.executors import _CRON_QUEUE_WAIT_SECS, cron_gate_budget, subprocess_executor +from kiro_crew.metrics.events import CRON_FIRES, emit_counter from kiro_crew.resource_status import admission_check from kiro_crew.validation import CHANNEL_MAX_LEN, MAX_CRON_MESSAGE, MAX_SHORT_STRING @@ -3302,6 +3303,17 @@ async def _run_job_isolated(self, job: CronJob) -> None: started_at = meta[0] if meta else time.time() trigger = meta[1] if meta else "scheduled" self._job_start_times[job.id] = started_at + # One increment per execution, before the jitter sleep so a run cancelled + # during jitter still counts as fired. ``kind`` is the dispatch shape -- + # ``script`` and ``command`` bypass the model entirely, so this is the + # split between jobs that cost tokens and jobs that cost none. + emit_counter( + CRON_FIRES, + { + "kind": "script" if job.script else "command" if job.command else "agent", + "trigger": trigger, + }, + ) # Apply jitter to spread execution unless strict_schedule is set or manual jitter = self._compute_jitter(job) if trigger != "manual" else 0 self._job_jitter[job.id] = jitter diff --git a/src/kiro_crew/hooks.py b/src/kiro_crew/hooks.py index 046c8790ea54..fe59a10a2fe7 100644 --- a/src/kiro_crew/hooks.py +++ b/src/kiro_crew/hooks.py @@ -139,17 +139,53 @@ class ToolHookResult: #: classify a sensitive-path or exfiltration deny as non-security. security_deny: bool = True + @staticmethod + def _count(action: str, security_deny: bool) -> None: + """Count one gate verdict. Best-effort; never changes the decision. + + Called by the four factories below and nowhere else, which is what makes + it fire exactly once per gate consultation. A surface that OVERRIDES the + gate constructs a result directly (``chat_runner`` downgrades an + auto-approve to an interactive card this way, twice) and so is not + counted: the gate was consulted once, and counting every constructed + object would report one request as two decisions AND keep a count for a + verdict that was then discarded. + + The two rejected alternatives were instrumenting the 23 exits of + ``HookManager.on_tool_call`` (23 call sites on a security path) and + counting in ``__post_init__`` behind a ``from_gate`` flag -- the flag had + no reader other than the counter itself, so it was state carried purely to + signal, where calling this from the four factories says the same thing + with no field at all. + + ``action`` is one of three module constants and ``security_deny`` a bool, + so the series is bounded by construction -- no reason string, tool name or + command reaches the recorder. + """ + try: + from kiro_crew.metrics.events import APPROVAL_DECISIONS, emit_counter + + emit_counter( + APPROVAL_DECISIONS, + {"decision": action, "security_deny": bool(security_deny)}, + ) + except Exception: # a tool decision must never fail on its telemetry + logger.debug("approval decision counter failed", exc_info=True) + @staticmethod def allow() -> ToolHookResult: + ToolHookResult._count(TOOL_ALLOW, False) return ToolHookResult(action=TOOL_ALLOW) @staticmethod def auto_approve() -> ToolHookResult: + ToolHookResult._count(TOOL_AUTO_APPROVE, False) return ToolHookResult(action=TOOL_AUTO_APPROVE) @staticmethod def deny(reason: str) -> ToolHookResult: """Deny on a hard security check — the attempt is the problem.""" + ToolHookResult._count(TOOL_DENY, True) return ToolHookResult(action=TOOL_DENY, reason=reason, security_deny=True) @staticmethod @@ -160,6 +196,7 @@ def deny_policy(reason: str) -> ToolHookResult: durable budget (cron auto-pause) does not treat a governance ceiling as a defect in what it attempted. """ + ToolHookResult._count(TOOL_DENY, False) return ToolHookResult(action=TOOL_DENY, reason=reason, security_deny=False) diff --git a/src/kiro_crew/mcp_gateway/stub.py b/src/kiro_crew/mcp_gateway/stub.py index 185156163150..6d9e72ad6285 100644 --- a/src/kiro_crew/mcp_gateway/stub.py +++ b/src/kiro_crew/mcp_gateway/stub.py @@ -40,6 +40,7 @@ from kiro_crew.mcp_gateway import transport from kiro_crew.mcp_gateway.hashing import hash_command, hash_effective_env from kiro_crew.mcp_gateway.pool import READ_BUFFER_LIMIT_BYTES, PoolKey +from kiro_crew.metrics.events import MCP_RECONNECTS, emit_counter logger = logging.getLogger(__name__) @@ -1399,6 +1400,7 @@ async def _pause() -> bool: return None session.note_init_result(json.loads(forward)) session.reconnects += 1 + emit_counter(MCP_RECONNECTS, {"pool": bool(pool_label)}) logger.info( "stub reconnected to a restarted gateway (%s, reconnect #%d) pool=%s", detail, diff --git a/src/kiro_crew/metrics/events.py b/src/kiro_crew/metrics/events.py index c4c6494f2e51..d60e77ead9ac 100644 --- a/src/kiro_crew/metrics/events.py +++ b/src/kiro_crew/metrics/events.py @@ -58,3 +58,62 @@ def emit_counter(name: str, attrs: dict[str, str | int | bool | float]) -> None: #: Idle-sweep expiries — ``turn_active=True`` means the sweep killed a #: runtime mid-turn, the teardown signature of the original incidents. SESSION_IDLE_EXPIRED = "kirocrew.session.idle_expired" + + +# --------------------------------------------------------------------------- +# Business-event series — "did this subsystem do its job, and how often" +# --------------------------------------------------------------------------- +# Each of these is emitted at its own subsystem's call site rather than from a +# central observer, because there is no frame every one of them crosses. What +# they share is the attribute rule above: every value is a constant from a +# closed set the emitting site owns, never an id, a name, or a count that grows +# with input. + +#: One per subagent whose spawn actually started. ``concurrency`` is the live +#: running count INCLUDING this one (the counter is emitted after the admission +#: increment), so it reads 1 for a lone subagent -- an integer bounded by the +#: spawn cap, which keeps the series small and makes the aggregator's MAX over it +#: the concurrency high-water mark. A separate high-water instrument would need +#: its own reset semantics and could not be read per-attribute like this. +SUBAGENTS_SPAWNED = "kirocrew.subagent.spawned" + +#: One per cron job execution. ``kind`` separates the three dispatch shapes +#: (``agent`` runs an LLM turn, ``script`` and ``command`` bypass the model +#: entirely), which is the difference between a job that costs tokens and one +#: that costs none. ``trigger`` says whether the schedule or a human fired it. +CRON_FIRES = "kirocrew.cron.fires" + +#: One per artifact created. ``kind`` is the artifact's validated type and +#: ``source`` the validated origin — both already closed sets in ``artifacts``. +ARTIFACTS_CREATED = "kirocrew.artifact.created" + +#: One per dynamic-workflow run started, foreground or background (the +#: background entry point drives the same method). ``authored`` marks a run that +#: writes its own script from an intent; ``replay`` marks a restart-subtree that +#: reuses cached agent results instead of re-calling the model. +WORKFLOW_RUNS = "kirocrew.workflow.runs" + +#: One per context compaction that reached a verdict, so the sum is compaction +#: ATTEMPTS. ``success`` reports whether the compaction itself completed, NOT how +#: much context it reclaimed -- no before/after reading is taken -- and it is +#: False when an in-place compaction failed and the session was recycled instead. +CONTEXT_COMPACTIONS = "kirocrew.context.compactions" + +#: One per MCP stub reconnect to a restarted daemon. Each increment is a +#: bridge that died and was rebuilt under a live session, so a rising rate is +#: daemon instability that sessions are absorbing silently. +MCP_RECONNECTS = "kirocrew.mcp.reconnects" + +#: One per tool-approval decision from the per-surface gate +#: (``hooks.HookManager.on_tool_call``), which every surface consults before a +#: tool runs. ``decision`` is the gate's own bounded action: ``auto_approve`` +#: skipped the human, ``deny`` refused without asking, and ``allow`` is the +#: branch that falls through TO an interactive prompt — so the ``allow`` slice +#: is approvals shown and the ``deny`` slice is approvals denied. +#: ``security_deny`` separates a hard security refusal from a policy-state one. +#: +#: This generalises the backend-child pair above (:data:`CHILD_PERMISSION_DENIED` +#: / :data:`CHILD_PERMISSION_ROUTED`) to every approval prompt. Those two stay +#: exactly as they are: they measure a specific hang-resilience fix on the +#: child-permission path, and their population is not this one's. +APPROVAL_DECISIONS = "kirocrew.approval.decisions" diff --git a/src/kiro_crew/metrics/provider.py b/src/kiro_crew/metrics/provider.py index 3a4204278720..782ecd458bc9 100644 --- a/src/kiro_crew/metrics/provider.py +++ b/src/kiro_crew/metrics/provider.py @@ -184,7 +184,7 @@ def _load_otel() -> bool: # sizing one array for both costs either resolution at the fast end or truth at # the slow end. # -# Three families, each sized to its instrument's MEASURED range: +# Each family below is sized to its instrument's MEASURED range: # Sub-ms through a minute — pooled acquires, skill loads, HTTP requests. # These are dominated by ~1ms values, so the fine end matters. The 60s ceiling @@ -228,6 +228,32 @@ def _load_otel() -> bool: 600000, 900000, 1800000, 3600000, 5400000, 7200000, 10800000, 14400000, ] +# Sub-ms through one hour — a single tool round-trip. The widest span of any +# family here, and both ends are real: a cached file read returns in well under +# a millisecond while a build or test run invoked through the execute tool runs +# for minutes. The fine end is copied from _FAST_BUCKETS_MS because reads +# dominate the population by count, and the ceiling matches _TURN_BUCKETS_MS +# because a tool call cannot outlive the turn that contains it. +_TOOL_CALL_BUCKETS_MS: list[float] = [ + 0.5, 1, 2, 5, 10, 25, 50, 100, 250, 500, + 1000, 2500, 5000, 10000, 30000, 60000, + 120000, 300000, 600000, 1800000, 3600000, +] + +# One second through one week — session lifetime. Sessions are the only +# instrument here measured in hours and days: a speculative session removed +# before its first turn lives seconds, while a dashboard tab left open across a +# working week is ordinary. Resolution is densest from a minute to a few hours, +# where interactive sessions land, and the 7-day ceiling exists so a long-lived +# tab is not floored into an overflow bucket. Sub-minute bounds are kept because +# the short-lived teardown paths (unclaimed, destroyed) are a real population +# whose distribution would otherwise collapse onto one boundary. +_SESSION_BUCKETS_MS: list[float] = [ + 1000, 5000, 15000, 30000, 60000, 300000, 900000, 1800000, + 3600000, 7200000, 14400000, 28800000, 43200000, + 86400000, 172800000, 259200000, 604800000, +] + # Instrument name -> boundaries. This map is the COMPLETE set of kirocrew # duration histograms: the Views below are built from it and there is no # catch-all, because the OTEL SDK applies EVERY matching View rather than the @@ -266,6 +292,8 @@ def _load_otel() -> bool: "kirocrew.mcp.lazy_load.duration": _STARTUP_BUCKETS_MS, "kirocrew.gateway.boot.duration": _STARTUP_BUCKETS_MS, "kirocrew.turn.duration": _TURN_BUCKETS_MS, + "kirocrew.tool.call.duration": _TOOL_CALL_BUCKETS_MS, + "kirocrew.session.duration": _SESSION_BUCKETS_MS, "kirocrew.watchdog.idle.duration": _WATCHDOG_IDLE_BUCKETS_MS, # Embedding queue wait + inference time. Both are ms and both predate this # map; neither name ends in `.duration`, which is precisely why the guard diff --git a/src/kiro_crew/metrics/sessions.py b/src/kiro_crew/metrics/sessions.py new file mode 100644 index 000000000000..0e375c3b6864 --- /dev/null +++ b/src/kiro_crew/metrics/sessions.py @@ -0,0 +1,692 @@ +"""``kirocrew.session.duration`` + ``kirocrew.session.started``. + +How long a session lives, and why it ended. Together they answer questions the +existing session instruments cannot: ``kirocrew.session.startup.duration`` +measures the cold start of the agent PROCESS, and +``kirocrew.session.idle_expired`` counts one specific teardown cause, but +nothing recorded how long a session actually lasted or how its lifetimes split +across the ways a session can end. + +**Why the reason is passed in, never derived.** There is no single teardown +function. ``session_lifecycle.SessionLifecycleService`` exposes six distinct end +paths -- ``reset``, ``remove``, ``remove_if_unclaimed``, ``destroy``, +``discard_conversation`` and ``close_all`` -- and each pops the registry itself +rather than delegating to a shared funnel. Two more paths pop it WITHOUT going +through any of those: ``retire_kiro_identity_sessions`` (an identity change +retires every idle process on the old account) and +``CompactionCoordinator._recycle_held`` (context overflow replaces the provider +in place). So each path states its own reason, the same contract +``metrics/turns.py`` holds its callers to. + +``reset`` is the widest of them: the idle sweep and a slot reset both reach +teardown through it, so ``end_reason=reset`` is a path label, not a cause. That +is deliberate -- the finer causes behind it are already counted separately +(``kirocrew.session.idle_expired``, ``kirocrew.watchdog.recovery.outcome``), and +a metric must not be the reason a lifecycle signature every surface calls grows +a parameter. The identity retire and the compaction recycle are NOT inside it and +carry their own labels, because a metric that reported them as ``reset`` would be +claiming a teardown route they never take. + +**Why a breadcrumb on disk.** A start time held only in memory dies with the +process, which is exactly the population most worth measuring: a session that +ends because the gateway crashed never runs any teardown path, so it would +contribute no sample at all and the histogram would describe only orderly +shutdowns. Each start therefore drops one small JSON file under +``/metrics/open-sessions/``. The clean path does NOT read that file -- +it takes the start time from the in-memory table, so an end can run in the same +tick as the registry removal it reports -- it only unlinks it. Whatever is still +there at the next boot belongs to a session that never ended cleanly, so +:func:`backfill_crashed_sessions` reads it, emits it as ``end_reason=crashed`` +and unlinks it. The crumb is written only while telemetry consent is in force, +since it exists solely to feed an instrument that is a no-op without it. + +**Why not the transcript's existing close marker.** A transcript's metadata line +already carries ``created_at``, and the dashboard tab-close path stamps +``closed`` / ``closed_at`` beside it. Neither substitutes for the crumb. That +stamp is written by the dashboard close path ALONE, so a channel, cron, subagent +or task-runner session never gets one; and its ABSENCE is ambiguous by +construction -- a transcript with no ``closed`` is equally a crashed session, a +live-but-idle one, and one that was never a dashboard tab. There is no positive +crash flag on disk today, which is what the crumb supplies, for every surface. +Boot has no session sweep to piggyback on either: the restore path is seed-driven +from ``open_slots.json`` and never walks the transcript directory. + +**Why the clean path consumes the crumb rather than reading it.** The +popped registry entry does carry an epoch ``created_at``, so a clean end could +compute its own duration without touching disk. It deliberately does not, for two +reasons. The clean and crashed paths must consume the SAME record or they +double-count: a crumb left behind by a clean end is indistinguishable at the next +boot from one left by a crash, and would be emitted a second time as +``crashed``. And that field is RESET when a provider is recycled in place, so it +measures the provider's age rather than the session's residency in the registry. + +That unlink is what makes the accounting exact. A crumb is consumed at most +once, so the six teardown paths cannot double-count a session between them (the +idle sweep's ``reset`` is a real instance of this: the sweep and the path it +calls both sit on the same session), and a backfilled session cannot be counted +again on the boot after that. + +**Where a crashed session's END time comes from.** The crumb is written once and +never rewritten, so its own mtime is its start. The session's transcript +(``/sessions/.jsonl``) is appended to as the conversation runs, +so its mtime is the last moment the session was observably alive -- the honest +maximum available after the fact. A session with no transcript on disk (a +subagent leaves only a replay log) yields no end time, so it is unlinked without +an emit rather than recorded as a plausible-looking zero: an absent sample reads +as "no data", a zero renders as a real 0ms lifetime. + +Telemetry must never break the instrumented path, so every function here +swallows its failures after a debug log, and every attribute value is a +constant from the closed enums below. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import logging +import os +import threading +import time +from pathlib import Path + +logger = logging.getLogger(__name__) + +#: Session lifetime, in ms. Registered in ``metrics/provider.py``'s bucket table +#: under its own boundary family: this is the only kirocrew histogram whose range +#: is minutes to days, so it shares no family with the request/turn instruments. +SESSION_DURATION_METRIC = "kirocrew.session.duration" + +#: One increment per session admitted to the registry -- the denominator the +#: duration histogram's population is a subset of. +SESSION_STARTED_COUNTER = "kirocrew.session.started" + +# --------------------------------------------------------------------------- +# end_reason -- closed enum, one member per teardown path +# --------------------------------------------------------------------------- + +#: ``SessionLifecycleService.reset`` -- the widest path (idle sweep, slot reset, +#: watchdog recycle). A compaction recycle is NOT here: it pops the registry +#: itself and reports :data:`END_REASON_RECYCLED`. See the module docstring. +END_REASON_RESET = "reset" +#: ``remove`` -- shut down, session-map entry deliberately preserved. +END_REASON_REMOVED = "removed" +#: ``remove_if_unclaimed`` -- a speculative session whose first turn never came. +END_REASON_UNCLAIMED = "unclaimed" +#: ``destroy`` -- session and its persistence entry both gone. +END_REASON_DESTROYED = "destroyed" +#: ``discard_conversation`` -- native conversation dropped, linkage kept. +END_REASON_DISCARDED = "discarded" +#: ``close_all`` -- gateway shutdown drained the registry. +END_REASON_SHUTDOWN = "shutdown" +#: ``retire_kiro_identity_sessions`` -- the account behind the session changed, +#: so every idle process on the old identity is torn down. Its own label rather +#: than folded into ``reset``: it pops the registry directly and never calls +#: ``reset``, and an identity change is a different event from an idle sweep. +#: ``reload_provider_factory`` reports this too for the registry it clears on a +#: provider switch -- both are "this process is no longer the right host for these +#: sessions", which is why they share a label rather than minting a second one. +END_REASON_RETIRED = "retired" +#: ``CompactionCoordinator._recycle_held`` -- context overflow replaced the +#: provider in place. Also pops directly rather than routing through ``reset``. +END_REASON_RECYCLED = "recycled" +#: A registry entry dropped because its provider was already dead +#: (``SessionAllocationService._evict_stale_session`` and the same check inside +#: ``get_or_create``). Its own label rather than folded into ``reset``: nothing +#: was torn down here, the process had already gone, so this population answers +#: "how often did a session die under us and get noticed on the next lookup" +#: rather than "how often did we end one". Left unrecorded it would be worse than +#: missing -- the crumb would survive and be reported as ``crashed`` at the next +#: boot, which is a different claim about a different boot. +END_REASON_EVICTED = "evicted" +#: Backfilled at boot from a crumb no teardown path ever consumed. +END_REASON_CRASHED = "crashed" + +#: Every value ``end_reason`` can carry. The drift gate in +#: ``test/metrics/test_session_duration.py`` harvests this set, so a new path +#: cannot start emitting a label the tests do not know about. +END_REASONS = frozenset( + { + END_REASON_RESET, + END_REASON_REMOVED, + END_REASON_UNCLAIMED, + END_REASON_DESTROYED, + END_REASON_DISCARDED, + END_REASON_SHUTDOWN, + END_REASON_RETIRED, + END_REASON_RECYCLED, + END_REASON_EVICTED, + END_REASON_CRASHED, + } +) + +# A runaway crumb directory must not turn boot into a stat storm, so the backfill +# emits at most this many samples per boot. Everything it walks is unlinked +# either way, so a directory that somehow grew past the cap is drained rather +# than left to be re-walked at every boot from then on. +_MAX_BACKFILL_EMITS = 2000 + +_CRUMB_DIR_NAME = "open-sessions" + +# Start time per LIVE session key -- the authority for the normal path, so an end +# never has to read the disk and can therefore run in the same tick as the +# registry removal it reports. The crumb on disk exists only to survive process +# death. Last write wins: a key re-entering the registry is a new session whose +# lifetime must be its own (see record_session_started). +# +# **This lock guards the TABLE ONLY and is never held across I/O.** It is taken on +# the event loop (every end runs there, in the same tick as its registry pop), so +# holding it across a crumb read or write would park the loop behind filesystem +# latency -- which is exactly what happened when it also guarded the crumb: the +# backfill held it across a read plus an unlink on a worker thread while a +# teardown waited for it on the loop. Crumb file ordering is _crumb_io_lock's job. +# +# Bounded because not every registry removal records an end, so a stale entry is +# possible; the cap discards the oldest half rather than growing for the process +# lifetime, which costs samples and never memory. +_MAX_LIVE_SESSIONS = 4096 +_live_lock = threading.Lock() +_live_starts: dict[str, float] = {} + +# Orders the two WORKER-THREAD paths that touch crumb files: the writer, and the +# backfill's read-decide-unlink. Both run off the loop, so a hold across +# filesystem latency costs a worker and never the gateway's responsiveness -- +# which is the whole reason it is a second lock rather than _live_lock. +# +# Nothing on the event loop may take this. The end path does not need to: with +# ownership recorded in each crumb, the backfill skips every crumb whose owner is +# live, so a session ending in THIS process and the crumbs the backfill reaps are +# disjoint sets. It is also worth being explicit that this is a THREAD lock and +# therefore never protected anything against a sibling PROCESS -- that is what the +# ownership field is for. +_crumb_io_lock = threading.Lock() + + +def _crumb_dir() -> Path: + """``/metrics/open-sessions`` -- created on demand.""" + from kiro_crew.config.paths import config_dir + + return config_dir() / "metrics" / _CRUMB_DIR_NAME + + +def _crumb_path(session_key: str, started_at: float) -> Path: + """Path of ONE generation of *session_key*'s crumb. + + Session keys carry channel ids and slot names and are not constrained to + filesystem-safe characters, so the key is never used as a filename. The key + itself is stored INSIDE the file, which is what lets the backfill find the + session's transcript; it is not new exposure, since ``session_map.json`` + already holds every session key in the same data home. + + **The name identifies the WRITER and the GENERATION, not just the key, and + that is what makes every unlink in this module safe.** Keying the file on the + session key alone gave two live sessions one shared path, because a session + key is not unique across processes: ``BACKGROUND_KEY`` is a fixed constant, so + a ``kirocrew run`` and the gateway hold the same key at the same time, each + with its own session behind it. Whichever ended first unlinked the file, and + the survivor's crash then went unrecorded -- and the thread locks here never + spanned processes, so nothing in this module could have prevented it. With the + pid and the start time in the name, a crumb can only ever be removed by the + session it belongs to, so a stale or late unlink cannot reach a live session's + record. It also means the backfill legitimately finds SEVERAL files for one + key: each is a distinct session, judged on its own recorded owner. + """ + digest = hashlib.sha256(session_key.encode("utf-8", "surrogatepass")).hexdigest() + return _crumb_dir() / f"{digest[:32]}-{os.getpid()}-{int(started_at * 1e6)}.json" + + +def _session_source(session_key: str) -> str: + """Bounded label for which surface owns *session_key*. + + ``telemetry_channel_of`` exists for exactly this question and never returns + the key itself, so cardinality is bounded whatever the caller passes. Same + derivation ``metrics/turns.py`` uses, so the two instruments group by the + same label. + """ + from kiro_crew.messaging.link import telemetry_channel_of + + return telemetry_channel_of(session_key) + + +async def record_session_started(session_key: str) -> None: + """Record one session start; never raises. + + The start time goes into an in-memory table keyed by session key. The crumb on + disk is written on a worker thread, awaited before returning. The table is + what the normal path reads; the crumb exists ONLY so a start can outlive its + process (see :func:`backfill_crashed_sessions`). + + **The table entry is overwritten, and so is the crumb.** A key re-entering + the registry is a NEW session, and its lifetime must be measured from ITS + start, not from a predecessor's -- so last-writer-wins is what keeps a + successor's lifetime its own. + + **Every registry removal must record an end.** Not because a missing sample + would matter on its own, but because the crumb turns an unrecorded removal + into a WRONG one: the crumb survives, and the next boot reports the session as + ``crashed``. So this is not merely a lost sample -- it manufactures a failure + that never happened, in exactly the population the histogram exists to + measure. ``test_every_registry_removal_records_an_end`` is a fail-closed gate + over every removal site in ``src/`` for that reason. + + **Why this is a coroutine, and awaited under the caller's lock.** Two + constraints look opposed and are not. The write must not run on the event + loop, because a slow or network-homed data home would stall every gateway + task behind one session insertion -- the same reason ``SessionMap`` offloads + its own persist to a worker rather than writing inline. But it also must not + be fire-and-forget, because a writer that outlives its critical section can + land after its session ended or after a SUCCESSOR registered under the same + key, and every attempt to detect that after the fact was itself racy. Awaiting + a worker hop while the caller still holds the session registry lock satisfies + both: the syscalls happen off the loop, and no start, end or successor can + interleave, because they all serialise on that same lock. + """ + if not session_key: + return + started_at = time.time() + orphaned: list[tuple[str, float]] = [] + with _live_lock: + previous = _live_starts.get(session_key) + if previous is not None and previous != started_at: + # A start SUPERSEDING a live generation of the same key. The displaced + # generation's crumb is named after it, so once the table forgets it no + # end can ever name it again -- reap it here or it survives to the next + # boot as a crash that never happened. Under one shared filename this + # case needed no handling, because the successor's write simply + # overwrote the file. + orphaned.append((session_key, previous)) + _live_starts[session_key] = started_at + if len(_live_starts) > _MAX_LIVE_SESSIONS: + # A registry this large is not real; drop the oldest half rather than + # growing for the process lifetime. Costs samples, never memory. + oldest = sorted(_live_starts.items(), key=lambda kv: kv[1]) + for stale_key, stale_at in oldest[: len(oldest) // 2]: + if _live_starts.pop(stale_key, None) is not None: + orphaned.append((stale_key, stale_at)) + if _crumbs_enabled(): + try: + # The orphaned generations ride the SAME worker hop as this start's own + # write. Dropping a table entry destroys the only record of which + # generation this process owns, so an orphaned crumb no end can name + # would survive to the next boot and be reported as a crash that never + # happened -- and doing those unlinks here rather than on the loop + # keeps up to half a table's worth of filesystem I/O off it without + # opening a second cancellation window. + await asyncio.to_thread(_write_crumb, session_key, started_at, tuple(orphaned)) + except Exception: + logger.debug("session start crumb handoff failed", exc_info=True) + # Counted LAST, after the only cancellable point in this function. Emitted + # before it, a cancellation during the handoff -- which the caller answers by + # rolling the registry entry back and discarding the start -- would leave this + # counter reporting a session that never existed, and `started` is the + # denominator every session-scoped rate is read against. + try: + from kiro_crew.metrics.events import emit_counter + + emit_counter(SESSION_STARTED_COUNTER, {"session_source": _session_source(session_key)}) + except Exception: + logger.debug("session started counter failed", exc_info=True) + + +def discard_session_start(session_key: str) -> None: + """Undo a start that never became a live session; never raises. + + For the cancellation window between registering a session and finishing its + start record: the caller is about to hard-kill the provider and roll the + registry back, so there is no session left to have a lifetime. Emits NOTHING + -- a session that never lived has no duration worth reporting -- but it must + still consume the crumb, because a crumb with no session behind it is read at + the next boot as a crash that never happened. + + Deliberately not one of the ``end_reason`` values: those describe how a live + session ENDED, and inventing a twelfth member for "never started" would put a + non-session into the lifetime histogram's population. + """ + if not session_key: + return + with _live_lock: + started_at = _live_starts.pop(session_key, None) + if started_at is None: + # Nothing of OURS to remove, so remove nothing. The crumbs sharing this + # key's digest belong to other processes or to earlier runs of this one, + # and deleting those is exactly the bug the generation-scoped name exists + # to prevent -- a discard that cannot name its own generation has no crumb + # to consume, because it never wrote one. + return + # Outside the lock, for the reason record_session_ended gives. + _unlink(_crumb_path(session_key, started_at)) + + +def _crumbs_enabled() -> bool: + """True while telemetry consent is in force; never raises. + + The crumb exists solely to feed :data:`SESSION_DURATION_METRIC`, and that + instrument is a no-op without consent -- so writing one on an unopted host + would persist state that nothing can ever read, against a documented default + of collecting nothing. Fails CLOSED: if consent cannot be determined, no file + is written. + """ + try: + from kiro_crew.metrics.provider import get_recorder + + return bool(get_recorder().enabled) + except Exception: + logger.debug("crumb consent check failed; skipping crumb", exc_info=True) + return False + + +def _is_current_generation(session_key: str, started_at: float) -> bool: + """True while *started_at* is still the generation installed for *session_key*. + + False once the key has been popped by an end or a discard, or overwritten by a + successor's start -- each a reason an in-flight crumb write must not stand. + """ + with _live_lock: + return _live_starts.get(session_key) == started_at + + +def _write_crumb( + session_key: str, + started_at: float, + orphaned: tuple[tuple[str, float], ...] = (), +) -> None: + """Persist *session_key*'s open-session crumb; never raises. + + **Runs on a worker thread and self-corrects, which is what makes the caller's + await safe to cancel.** Cancelling an ``asyncio.to_thread`` abandons the await + but does NOT stop the thread, so the caller's rollback can unlink before this + write lands and leave an orphan crumb the next boot reports as a crash. Rather + than make the await non-cancellable, this checks the generation on BOTH sides + of the write: it declines to write once its own ``started_at`` is no longer the + installed generation, and removes what it just wrote if that changed while the + write was in flight -- so an end, a discard and a cancellation are all covered + by the same test. + + **The post-write unlink removes only this writer's own file, and the NAME is + what guarantees that.** Unlinking a shared path unconditionally is what made + this unsafe before: while every generation of a key wrote to one path, a late + predecessor could delete a successor's crumb, and the read-back check that + tried to detect it afterwards was only ever a narrowing -- it could not help + across processes at all. Now the path encodes the writer and the generation, + so the file this removes cannot belong to anyone else. + + Takes ``_crumb_io_lock`` (crumb files) and, inside it, ``_live_lock`` (the + table). That order is the only one used anywhere, so the two cannot deadlock: + no path takes the table lock and then the I/O lock. + """ + try: + from kiro_crew.atomic_write import atomic_write + + path = _crumb_path(session_key, started_at) + payload = json.dumps( + { + "key": session_key, + "started_at": started_at, + # WHO owns this session. Without it a crumb is only "some session + # started before I booted", and the gateway's boot scan cannot tell + # a casualty of the last run from a session running RIGHT NOW in a + # sibling process -- `kirocrew run` and the eval runner each build + # their own SessionManager against the same data home. Reaping one + # of those both invents a crash and destroys the live crumb, so the + # real crash that session may later suffer goes unrecorded. + # ``start_id`` guards pid reuse: a recycled pid belongs to a + # different process and must not look like this one. + "pid": os.getpid(), + "start_id": _own_start_id(), + }, + ensure_ascii=True, + ) + with _crumb_io_lock: + # Generations this start displaced -- superseded under the same key, or + # evicted by the cap -- go first, and unconditionally: they must be + # removed even when this start's own write turns out to be superseded + # below, because once the table has forgotten a generation nothing can + # ever name its file again. + for stale_key, stale_at in orphaned: + _unlink(_crumb_path(stale_key, stale_at)) + if not _is_current_generation(session_key, started_at): + return + path.parent.mkdir(parents=True, exist_ok=True) + atomic_write(path, payload) + if not _is_current_generation(session_key, started_at): + _unlink(path) + except Exception: + logger.debug("session start crumb write failed", exc_info=True) + + +def _own_start_id() -> str: + """This process's start identifier, or ``""`` when unavailable.""" + try: + from kiro_crew import platform_compat + + return platform_compat.get_process_start_id(os.getpid()) or "" + except Exception: + return "" + + +def _owner_still_running(pid: object, start_id: object) -> bool: + """True when *pid* is alive AND is still the process that wrote the crumb. + + Fails CLOSED -- an undecidable owner counts as RUNNING, so an ambiguous crumb + is left on disk for a later boot rather than reaped into a fabricated crash. + Losing a real crash sample costs one data point; inventing one corrupts the + population this instrument exists to report. + + **Liveness comes from ``pid_exists``, identity only refines it.** + ``get_process_start_id`` returns None on Windows and on any process it may not + introspect, and its own contract says a None must NOT be read as a mismatch -- + so using it as the liveness test would judge every owner dead on an entire + platform and reap live sibling sessions there. It is used only to catch pid + REUSE, and only when both sides of the comparison are actually available -- + including reuse of THIS process's own pid, which a container restart turns + from a freak collision into the normal case. + """ + if not isinstance(pid, int) or pid <= 0: + # A crumb from before ownership was recorded. It cannot be attributed, so + # the cutoff is all that protects it -- treat it as not-running and let the + # caller's cutoff decide, which is the pre-ownership behaviour. + return False + if pid == os.getpid(): + # NOT necessarily this process. A container hands the gateway the same pid + # on every restart -- PID 1 -- so a crumb a crashed predecessor wrote + # arrives here looking like one of our own. Answering True would skip it on + # this boot and on every boot after, so its crash is never emitted and its + # file is never cleaned: in a container that collision is deterministic, + # not the coincidence it looks like on a developer's machine. Comparing the + # start identifiers separates the two, and only when BOTH sides are + # readable -- an unknown identity still fails closed to running. + own = _own_start_id() + if isinstance(start_id, str) and start_id and own: + return start_id == own + return True + try: + from kiro_crew import platform_compat + + if not platform_compat.pid_exists(pid): + return False # definitively gone: a casualty to be reported + live_start_id = platform_compat.get_process_start_id(pid) + except Exception: + return True + if not isinstance(start_id, str) or not start_id or not live_start_id: + # SOMETHING holds that pid and the identities cannot be compared (Windows, + # a pre-identity crumb, or a process we may not introspect). Fail closed. + return True + return live_start_id == start_id + + +def record_session_ended(session_key: str, *, end_reason: str) -> None: + """Emit *session_key*'s lifetime; never raises. + + **Non-blocking, so it belongs in the same tick as the registry removal it + reports.** The start time is popped from the in-memory table, the histogram + is recorded in memory, and the only disk work is a single ``unlink``. That + is what lets every teardown path call this immediately after its + ``_sessions.pop`` and BEFORE its first ``await``. The earlier version read + the crumb off disk here, which forced the call to the end of teardown -- and + a replacement session registering under the same key during those awaits then + had its crumb consumed by its predecessor's teardown. + + Popping the table entry is also the double-count defence: the teardown paths + are not mutually exclusive (the idle sweep calls ``reset``), so whichever + reaches the session first is the one that records it, and a second call finds + nothing. + **The crumb unlink runs INLINE, and outside every lock.** It is one ``unlink`` + syscall, so it costs this tick almost nothing, and handing it to the + maintenance pool instead was wrong twice over. A queued unlink can land AFTER a + successor registered under the same key and wrote its own crumb, deleting a + live session's crumb and losing its later crash. And + ``shutdown_maintenance_executor`` drains with ``cancel_futures=True``, so at + ``close_all`` -- exactly when that pool is flooded with teardown work -- the + unlink is cancelled outright, leaving behind the crumb of a session that ended + cleanly for the next boot to back-fill as ``crashed``: orderly shutdown would + inflate the very population this instrument exists to measure. + + It holds no lock while doing it, and must not. This runs on the event loop, so + waiting on ``_crumb_io_lock`` -- which worker threads hold across filesystem + latency -- would park the gateway behind a crumb read. No lock is needed + either: the backfill skips every crumb whose owner is live, so it never looks + at one this process wrote, and an in-flight writer for this key self-corrects + by re-checking the generation this function has just popped. + """ + if not session_key or end_reason not in END_REASONS: + return + with _live_lock: + started_at = _live_starts.pop(session_key, None) + if started_at is None: + # No generation of ours to consume, so nothing is unlinked. A removal with + # no live entry never wrote a crumb in this process -- the entry was + # evicted (which unlinks at eviction time) or the start was never recorded + # -- and the files sharing this digest belong to other processes or earlier + # runs. Unlinking those unconditionally was the sibling-clobbering bug. + return + # Unlinked OUTSIDE the lock: this runs on the event loop, and the table lock is + # never held across I/O. Removing the file even when the duration cannot be + # emitted is deliberate -- the session is over, so its crumb must not survive + # for the next boot to read as a crash. The path names this writer and this + # generation, so no concurrent session's record is reachable from here. + _unlink(_crumb_path(session_key, started_at)) + try: + _emit_duration(session_key, time.time() - started_at, end_reason) + except Exception: + logger.debug("session end emit failed", exc_info=True) + + +def backfill_crashed_sessions(started_before: float | None = None) -> int: + """Emit ``end_reason=crashed`` for crumbs no teardown path consumed. + + *started_before* is an epoch cutoff: a crumb whose ``started_at`` is at or + after it is left untouched. Callers pass their own process start time, which + is what makes this safe to run OFF the boot path -- without the cutoff the + scan had to complete before this process opened its first session, or it + would back-fill a crumb it had just written as though a previous run had + crashed. With it, ordering is irrelevant: only crumbs that predate this + process can ever be claimed. ``None`` disables the cutoff (tests). + + Returns the number of samples emitted (0 on any failure); never raises. + """ + emitted = 0 + try: + crumb_dir = _crumb_dir() + if not crumb_dir.is_dir(): + return 0 + for path in sorted(crumb_dir.glob("*.json")): + # Read, judge and unlink under ONE lock hold, which now serialises this + # scan against THIS process's own crumb writer rather than guarding an + # identity race. The race it used to guard is gone: a path names one + # writer and one generation, so a session registering mid-scan gets a + # file of its own and the successor this once could have deleted no + # longer shares a name with anything here. + with _crumb_io_lock: + key, started_at, pid, start_id = _read_crumb(path) + if _owner_still_running(pid, start_id): + # A LIVE session, in this process or a sibling. Reaping it + # would both invent a crash and delete the evidence of the + # real one it may yet suffer. + continue + if ( + started_at is not None + and started_before is not None + and started_at >= started_before + ): + # Pre-ownership crumb from this run: still live, not a casualty. + continue + _unlink(path) + if not key or started_at is None or emitted >= _MAX_BACKFILL_EMITS: + continue + ended_at = _last_activity(key) + if ended_at is None: + continue + if _emit_duration(key, ended_at - started_at, END_REASON_CRASHED): + emitted += 1 + except Exception: + logger.debug("crashed-session backfill failed", exc_info=True) + return emitted + + +def _emit_duration(session_key: str, seconds: float, end_reason: str) -> bool: + """Record one histogram sample. Returns whether a sample was emitted. + + A non-positive lifetime is skipped rather than recorded, for the reason + ``metrics/turns.py`` gives: an absent sample reads as "no data" on the + Telemetry page, while a recorded 0 renders as a plausible 0ms lifetime. + """ + if seconds <= 0: + return False + try: + from kiro_crew.metrics.provider import get_recorder + + get_recorder().histogram( + SESSION_DURATION_METRIC, + seconds * 1000.0, + unit="ms", + attrs={"end_reason": end_reason, "session_source": _session_source(session_key)}, + description="Session lifetime (ms), by how the session ended.", + ) + return True + except Exception: + logger.debug("session duration emit failed", exc_info=True) + return False + + +def _read_crumb(path: Path) -> tuple[str, float | None, object, object]: + """Return ``(session_key, started_at, pid, start_id)`` from *path*. + + ``("", None, None, None)`` when the file is missing, unreadable or malformed. + ``pid`` / ``start_id`` are absent on crumbs written before ownership existed. + """ + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return "", None, None, None + if not isinstance(payload, dict): + return "", None, None, None + key = payload.get("key") + started = payload.get("started_at") + if not isinstance(key, str) or not isinstance(started, (int, float)): + return "", None, None, None + return key, float(started), payload.get("pid"), payload.get("start_id") + + +def _last_activity(session_key: str) -> float | None: + """Mtime of *session_key*'s transcript -- when it was last observably alive. + + ``None`` when the session has no transcript on disk, which is normal rather + than an error: a subagent run leaves only a replay log. The caller skips the + emit in that case (see the module docstring). + """ + try: + from kiro_crew.config.paths import config_dir + from kiro_crew.history import SESSIONS_DIR_NAME, transcript_stem + + path = config_dir() / SESSIONS_DIR_NAME / f"{transcript_stem(session_key)}.jsonl" + return path.stat().st_mtime + except Exception: + return None + + +def _unlink(path: Path) -> None: + try: + path.unlink(missing_ok=True) + except OSError: + logger.debug("session crumb unlink failed for %s", path.name, exc_info=True) diff --git a/src/kiro_crew/metrics/tool_calls.py b/src/kiro_crew/metrics/tool_calls.py new file mode 100644 index 000000000000..f8cf5c0b66cd --- /dev/null +++ b/src/kiro_crew/metrics/tool_calls.py @@ -0,0 +1,247 @@ +"""``kirocrew.tool.call.duration`` -- how long each tool round-trip takes. + +The turn histogram already reports how long a whole agent loop takes, which +makes a slow turn visible but never says which part was slow. A turn is model +calls plus every tool round-trip, so without this instrument a 4-minute turn is +indistinguishable between "the model was slow" and "one shell command took four +minutes", and the tool population that dominates real latency cannot be seen at +all. + +**Cardinality: the tool NAME is never an attribute.** MCP tool names are +unbounded -- any server the user installs contributes its own -- and the +recorder caches one instrument per distinct name with no eviction, so a +name-valued attribute is a cardinality bomb (``metrics/schema.py``). The label +is a normalised ``tool_kind`` instead, and normalisation is not optional +politeness: the ACP ``kind`` field arrives verbatim from the agent, which +``hooks.py`` already documents when it explains why its auto-approve decision is +an ALLOW-list rather than a denylist. :func:`classify_tool_kind` applies the +same reasoning here -- a kind outside :data:`TOOL_KINDS` becomes ``other``, so +an agent cannot mint series by inventing kinds. + +An MCP tool is classified ``mcp`` whatever kind it claims. The kind an MCP +server reports is its own vocabulary, so keeping it would put third-party +strings in the allowlist's path; and "this call left the process over MCP" is +the more useful fact anyway, since it is the difference between a builtin and a +round-trip through a server that can be slow or wedged. + +**Two layers, one registry, exactly one sample.** There is no single choke point +to instrument, because the two backends parse tool frames in different places. +The kiro backend runs on ``AcpRuntime`` + ``AcpSessionHandle``, which parses +through the shared ``acp/_dispatch.py`` builders; the claude backend (and the app +worker pools that build a client directly, e.g. ``knowledge/llm_pool``) stays on +``acp/client.py``, which re-implements the same shaping inline and never calls +``parse_session_update``. ``providers/acp.py`` constructs an ``AcpClient`` and +then swaps in an ``AcpSessionProvider`` at startup for the kiro path, so both +parsers are live in a normal install and both are instrumented here. + +Every surface -- dashboard, Slack, Discord, cron, subagents, task runner, +workflow -- sits DOWNSTREAM of those two: they consume the emitted ``AcpEvent`` +stream rather than re-parsing frames. So instrumenting both parsers covers every +surface, and nothing else needs a call site. + +The start times live in ONE process-global registry keyed by ``toolCallId``, and +a finish POPS its entry: a call with no recorded start emits nothing. So if a +frame is ever seen by both layers, the first finish records it and the second is +a no-op -- the layering cannot double-count. + +**Why not reuse the watchdog's existing dispatch clock.** +``acp/liveness.py::ToolCallState`` already stamps ``dispatch_ts`` on a tool call, +and ``AcpSessionHandle`` clears it on the result. It is deliberately NOT reused: +``_inflight_tool`` is a SINGLE SLOT holding the most recent call, which is the +right shape for stall attribution (the oracle only asks "what are we waiting on +now") and the wrong shape for a histogram -- interleaved or overlapping tool +calls overwrite each other, so durations would be attributed to the wrong call +and some would never be recorded at all. A ``toolCallId``-keyed registry is what +makes every call its own sample. It also exists only on the kiro path, so it +could not serve the claude one. + +Every function is best-effort and swallows its own failures: a tool call must +never fail because its telemetry did. +""" + +from __future__ import annotations + +import logging +import threading +import time + +logger = logging.getLogger(__name__) + +#: Registered in ``metrics/provider.py``'s bucket table -- a tool call spans +#: sub-millisecond file reads to multi-minute builds, so it carries its own +#: boundary family rather than sharing the request or turn ones. +TOOL_CALL_METRIC = "kirocrew.tool.call.duration" + +#: ``tool_kind`` for a call that went out over MCP, whatever kind it reported. +TOOL_KIND_MCP = "mcp" + +#: ``tool_kind`` for a kind this build does not recognise -- including the ACP +#: protocol's own literal ``"other"``, an empty kind, and anything an agent +#: invents. Folding them together is what bounds the series. +TOOL_KIND_OTHER = "other" + +#: The recognised ACP tool kinds, plus Kiro Crew's own ``client_built_in`` (used +#: when kiro-cli's security filter cancels a tool use). Union of the two sets +#: ``hooks.py`` maintains for its approval gate, so this allowlist and the gate's +#: vocabulary cannot silently diverge. +TOOL_KINDS = frozenset( + { + "read", + "fetch", + "search", + "edit", + "write", + "create", + "delete", + "move", + "execute", + "think", + "switch_mode", + "client_built_in", + TOOL_KIND_MCP, + TOOL_KIND_OTHER, + } +) + +#: Terminal ACP tool-call statuses. A non-terminal update (``pending``, +#: ``in_progress``) is not an outcome, so it records nothing and leaves the +#: call's start time in place for its real completion. +OUTCOME_COMPLETED = "completed" +OUTCOME_FAILED = "failed" +OUTCOME_CANCELLED = "cancelled" + +#: Every value ``outcome`` can carry. Exactly the terminal statuses: an unknown +#: status is not folded into ``other`` because :func:`record_tool_call_finished` +#: returns without emitting for anything outside this set, so no such sample can +#: exist. (Contrast :data:`TOOL_KIND_OTHER`, which IS reachable -- an unknown +#: kind still belongs to a call whose duration is recorded.) +OUTCOMES = frozenset({OUTCOME_COMPLETED, OUTCOME_FAILED, OUTCOME_CANCELLED}) + +_TERMINAL_STATUSES = frozenset({OUTCOME_COMPLETED, OUTCOME_FAILED, OUTCOME_CANCELLED}) + +# A turn cannot legitimately hold this many tool calls open at once, so a +# registry that reaches the cap is evidence of frames whose completion never +# arrived (a killed process, a cancelled turn). It is dropped wholesale rather +# than grown for the life of the process -- the same trade ``AcpClient`` makes +# for its ``_skill_read_noted`` bookkeeping. The cost is missing samples for the +# calls that were in flight, never a leak. +_MAX_OPEN_CALLS = 4096 + +_lock = threading.Lock() +_open_calls: dict[str, tuple[float, str]] = {} + + +def _registry_key(tool_call_id: str, scope: str) -> str: + """Origin-scoped registry key, spelled exactly as ``_dispatch``'s caches are. + + ``toolCallId`` is assigned by the BACKEND and is unique only within one + backend session, while this registry is process-global and one ``AcpRuntime`` + hosts many sessions. Without the scope two sessions reusing an id collide: + the second start is dropped as a duplicate and the first finish pops the + shared entry, so one sample is misattributed and the other is lost. + + ``acp/_dispatch.py`` already solved this for its tool-input caches with + ``_ck = f"{cache_scope}|{tool_call_id}"``; this is the same key so the two + cannot disagree about what "the same tool call" means. + """ + return f"{scope}|{tool_call_id}" if scope else tool_call_id + + +def classify_tool_kind(kind: str | None, *, mcp_server_name: str | None = None) -> str: + """Map an ACP ``kind`` onto a member of :data:`TOOL_KINDS`. + + ``mcp_server_name`` wins when set: see the module docstring for why an MCP + call is labelled by its transport rather than by the kind it claims. + """ + if mcp_server_name: + return TOOL_KIND_MCP + if kind and kind in TOOL_KINDS: + return kind + return TOOL_KIND_OTHER + + +def note_tool_call_started( + tool_call_id: str | None, + *, + kind: str | None = None, + mcp_server_name: str | None = None, + scope: str = "", +) -> None: + """Record when *tool_call_id* started, with its normalised kind. + + *scope* is the emitting session's cache scope -- see :func:`_registry_key` + for why a process-global registry cannot key on the raw backend id. + + Idempotent per scoped id: an id already open keeps its ORIGINAL start time, + so the ``tool_call_update`` refinements that follow a call (and the second + layer, if it also sees the frame) cannot reset the clock and shrink the + measurement. + """ + if not tool_call_id: + return + try: + resolved = classify_tool_kind(kind, mcp_server_name=mcp_server_name) + key = _registry_key(tool_call_id, scope) + with _lock: + if key in _open_calls: + return + if len(_open_calls) >= _MAX_OPEN_CALLS: + _open_calls.clear() + _open_calls[key] = (time.perf_counter(), resolved) + except Exception: + logger.debug("tool call start bookkeeping failed", exc_info=True) + + +def record_tool_call_finished( + tool_call_id: str | None, *, status: str | None, scope: str = "" +) -> None: + """Emit one ``kirocrew.tool.call.duration`` sample; never raises. + + A non-terminal *status* returns without touching the registry, so the call + is still measured when its real completion arrives. A terminal status POPS + the entry, which is what makes the two instrumented layers add up to exactly + one sample per call. *scope* must match the one its start was recorded under. + """ + if not tool_call_id or not status or status not in _TERMINAL_STATUSES: + return + try: + with _lock: + started = _open_calls.pop(_registry_key(tool_call_id, scope), None) + if started is None: + return + started_at, resolved_kind = started + # ``perf_counter``, not ``monotonic``: on Windows the latter advances in + # ~15.6ms ticks, so a call that completes inside one tick measures exactly + # 0.0 and the guard below drops it. That silently hid EVERY sub-tick tool + # call on Windows -- which is most cached reads -- and turned the whole + # platform's fast population into no data. ``perf_counter`` is the + # highest-resolution clock available on every platform and is equally + # monotonic, so the guard keeps meaning "unmeasurable" rather than "fast". + elapsed_ms = (time.perf_counter() - started_at) * 1000.0 + if elapsed_ms <= 0: + # Same rule as ``metrics/turns.py``: an absent sample reads as "no + # data", while a recorded 0 renders as a plausible 0ms tool call. + return + from kiro_crew.metrics.provider import get_recorder + + get_recorder().histogram( + TOOL_CALL_METRIC, + elapsed_ms, + unit="ms", + attrs={"tool_kind": resolved_kind, "outcome": status}, + description="Tool call round-trip duration (ms), by tool kind.", + ) + except Exception: + logger.debug("tool call duration emit failed", exc_info=True) + + +def open_call_count() -> int: + """Number of tool calls currently awaiting a terminal status (tests only).""" + with _lock: + return len(_open_calls) + + +def reset_open_calls() -> None: + """Drop all in-flight bookkeeping. For test isolation, not production use.""" + with _lock: + _open_calls.clear() diff --git a/src/kiro_crew/session_allocation.py b/src/kiro_crew/session_allocation.py index 4a37407a9b91..2dd08681519f 100644 --- a/src/kiro_crew/session_allocation.py +++ b/src/kiro_crew/session_allocation.py @@ -19,6 +19,13 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Protocol, cast +from kiro_crew.metrics.sessions import ( + END_REASON_EVICTED, + discard_session_start, + record_session_ended, + record_session_started, +) + if TYPE_CHECKING: from kiro_crew.providers.base import LLMProvider else: @@ -490,6 +497,9 @@ async def _evict_stale_session(self, key: str, session: Any) -> None: if self._sessions.get(key) is session: del self._sessions[key] dead = session.provider + # Same tick as the removal. Left unrecorded, the start crumb + # survives and the next boot calls this a crash. + record_session_ended(key, end_reason=END_REASON_EVICTED) if dead is not None: await asyncio.to_thread(self._deps.unlink_session_queue, session) try: @@ -565,6 +575,18 @@ async def open_task_session( ) self._sessions[key] = session won_race_session = session + try: + await record_session_started(key) + except BaseException: + # This await is the only suspension point between registering + # the session and returning it. Cancelled here, the caller + # hard-kills the provider while the entry stays visible, so a + # claimant can be handed a session whose process is already + # dying -- and the crumb would outlive it into a false crash. + if self._sessions.get(key) is session: + del self._sessions[key] + discard_session_start(key) + raise if duplicate is not None: try: await duplicate.shutdown() @@ -1089,6 +1111,10 @@ async def get_or_create( stale_provider = session.provider stale_session = session del self._sessions[key] + # Same tick as the removal. Left unrecorded, the + # start crumb survives and the next boot calls this + # a crash rather than an eviction. + record_session_ended(key, end_reason=END_REASON_EVICTED) if alive: session.last_used = time.monotonic() if ( @@ -1384,6 +1410,16 @@ def resolve_claim_watchdog() -> tuple[str, object]: ): owner._session_map.clear_sid(key) self._sessions[key] = session + try: + await record_session_started(key) + except BaseException: + # See open_task_session: a cancellation here would leave a + # registered session whose provider the caller is about to + # kill, plus a crumb the next boot reads as a crash. + if self._sessions.get(key) is session: + del self._sessions[key] + discard_session_start(key) + raise self._deps.logger.info( "New session: %s agent=%s resumed=%s provider_switch=%s (total=%d)", key, diff --git a/src/kiro_crew/session_background.py b/src/kiro_crew/session_background.py index c29a5ead949d..0d65db8af96d 100644 --- a/src/kiro_crew/session_background.py +++ b/src/kiro_crew/session_background.py @@ -20,6 +20,13 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Protocol +from kiro_crew.metrics.sessions import ( + END_REASON_RECYCLED, + discard_session_start, + record_session_ended, + record_session_started, +) + if TYPE_CHECKING: from kiro_crew.acp.types import AcpEvent from kiro_crew.providers.base import LLMProvider @@ -256,6 +263,17 @@ async def _ensure_background(self) -> None: agent=background_agent, ) self._owner._sessions[background_key] = sess + try: + await record_session_started(background_key) + except BaseException: + # Cancelled between registering and recording: roll the entry + # back so no claimant sees a session the caller is tearing + # down, and consume the crumb so it cannot become a false + # crash at the next boot. + if self._owner._sessions.get(background_key) is sess: + del self._owner._sessions[background_key] + discard_session_start(background_key) + raise logger.info("Background session created") return # Racing registration lost, or shutdown began while we were starting: @@ -619,6 +637,12 @@ async def recycle_heartbeat(self) -> None: # deliberately cycle-scoped, never per concurrently gathered task. async with self._owner._lock: old = self._owner._sessions.pop(heartbeat_key, None) + if old: + # Same tick as the pop, before the shutdown await. An unrecorded + # removal here does not merely lose a sample: the start crumb + # survives and the next boot reports this cleanly recycled + # session as crashed. + record_session_ended(heartbeat_key, end_reason=END_REASON_RECYCLED) if old: await old.provider.shutdown() diff --git a/src/kiro_crew/session_compaction.py b/src/kiro_crew/session_compaction.py index d4695afbabd2..dae0c7f1f4b1 100644 --- a/src/kiro_crew/session_compaction.py +++ b/src/kiro_crew/session_compaction.py @@ -20,6 +20,9 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Protocol +from kiro_crew.metrics.events import CONTEXT_COMPACTIONS, emit_counter +from kiro_crew.metrics.sessions import END_REASON_RECYCLED, record_session_ended + if TYPE_CHECKING: # Type-only: importing providers.base from this leaf at runtime enters the # providers -> acp package -> runtime -> session_pid -> providers cycle. @@ -381,6 +384,10 @@ async def _recycle_held(self, key: str, session: Any, pct: float) -> None: popped = None if owner._sessions.get(key) is session: popped = owner._sessions.pop(key, None) + # Same tick as the pop. Only this branch records: on the + # other one the registry already holds a SUCCESSOR under + # this key, whose start must stay its own. + record_session_ended(key, end_reason=END_REASON_RECYCLED) await asyncio.to_thread(self._deps.unlink_session_queue, session) if popped is None: @@ -544,6 +551,21 @@ async def _reset_still_critical( async def _fire_compact_callback(self, key: str, pct: float, *, success: bool) -> None: """Mark reinjection and invoke the compact callback, swallowing errors.""" + # Every compaction that reached a verdict passes here, whether or not a + # callback is registered, so this is where the counter belongs: the early + # return below would otherwise drop the surfaces that register none. + # + # The counter's success is NOT the callback's success. ``_recycle_held`` + # fires this with success=True because the SESSION now has headroom, which + # is what the callback needs to know -- but it is reached exactly when an + # in-place /compact FAILED and the provider had to be replaced instead. + # Counting that as a successful compaction would report the failure mode + # as the success case. The recycling marker is set for the whole of + # ``_recycle_held`` and popped only after this call, so it is what + # separates the two populations here; ``test_a_failed_compact_that_recycles + # _is_not_counted_successful`` pins that ordering. + recycled = key in self._owner._recycling + emit_counter(CONTEXT_COMPACTIONS, {"success": bool(success) and not recycled}) # Recycling destroys this session, so its successor receives startup # context normally. The identity guard also avoids flagging a racing # replacement while the old provider is being reaped. diff --git a/src/kiro_crew/session_lifecycle.py b/src/kiro_crew/session_lifecycle.py index e7029060274c..fdcd9f83ffd6 100644 --- a/src/kiro_crew/session_lifecycle.py +++ b/src/kiro_crew/session_lifecycle.py @@ -20,6 +20,17 @@ from dataclasses import dataclass, field from typing import Any, Literal, Protocol +from kiro_crew.metrics.sessions import ( + END_REASON_DESTROYED, + END_REASON_DISCARDED, + END_REASON_REMOVED, + END_REASON_RESET, + END_REASON_RETIRED, + END_REASON_SHUTDOWN, + END_REASON_UNCLAIMED, + record_session_ended, +) + CancelOutcome = Literal["acked", "timeout", "no_turn", "error"] StopOutcome = Literal["soft", "hard", "idle"] ProviderFactory = Callable[..., Any] @@ -324,6 +335,11 @@ async def reload_provider_factory(self) -> None: # path does not rewrite session-map or compaction state here. stale = list(owner._sessions.items()) owner._sessions.clear() + # Same tick as the clear. This removal had no end record, so a + # replacement under a reused key inherited the old start and + # reported a lifetime spanning two sessions. + for stale_key, _stale_sess in stale: + record_session_ended(stale_key, end_reason=END_REASON_RETIRED) # Shutdown remains outside both locks. Queue unlinking and companion # runtime release are intentionally not added to this historical path. for key, sess in stale: @@ -369,6 +385,14 @@ async def reset( self._suppress_replay.discard(key) owner._compact_pending_verdict.pop(key, None) self._origin_links.pop(key, None) + if session is not None: + # Same event-loop tick as the pop, for the reason the clear_sid + # call below documents: the awaits further down let a racing cold + # start register a SUCCESSOR under this key, and recording the + # end after them would consume the successor's start instead of + # this session's. Non-blocking by design (an in-memory pop plus + # an in-memory histogram record), so it is safe under the lock. + record_session_ended(key, end_reason=END_REASON_RESET) if clear_conversation and session is not None: # This stays in the same event-loop tick as the registry pop, so a # racing cold start cannot have published a successor SID yet. @@ -478,6 +502,10 @@ async def remove(self, key: str) -> None: self._suppress_replay.discard(key) owner._compact_pending_verdict.pop(key, None) self._origin_links.pop(key, None) + if session is not None: + # Same tick as the pop: see reset for why recording after the + # teardown awaits would consume a successor's start. + record_session_ended(key, end_reason=END_REASON_REMOVED) if session: await asyncio.to_thread(self._deps.get_unlink_session_queue(), session) await session.provider.shutdown() @@ -524,6 +552,10 @@ async def retire_kiro_identity_sessions(self) -> tuple[list[str], bool]: owner._compact_cooldown_until.pop(key, None) self._suppress_replay.discard(key) self._origin_links.pop(key, None) + # Same tick as the removal, not down in the shutdown loop + # below: that loop awaits, and a replacement session can + # register under this key while it does. + record_session_ended(key, end_reason=END_REASON_RETIRED) # Do not clear _compact_pending_verdict: the identity # recycle historically preserves that deferred verdict. doomed.append((key, sess.provider)) @@ -638,6 +670,8 @@ async def remove_if_unclaimed(self, key: str) -> bool: self._suppress_replay.discard(key) owner._compact_pending_verdict.pop(key, None) self._origin_links.pop(key, None) + # Same tick as the removal: see reset. + record_session_ended(key, end_reason=END_REASON_UNCLAIMED) await asyncio.to_thread(self._deps.get_unlink_session_queue(), session) await session.provider.shutdown() await owner.release_subagent_runtime(key) @@ -665,6 +699,9 @@ async def destroy(self, key: str) -> None: owner.set_autocompact_pct(key, None) # _origin_links deliberately survives destroy; existing callers # rely on the historical asymmetry with reset/remove. + if session is not None: + # Same tick as the pop: see reset. + record_session_ended(key, end_reason=END_REASON_DESTROYED) try: if session: await asyncio.to_thread(self._deps.get_unlink_session_queue(), session) @@ -720,6 +757,9 @@ async def discard_conversation( self._suppress_replay.discard(key) else: self._suppress_replay.add(key) + if session is not None: + # Same tick as the pop, exactly like the clear_sid below. + record_session_ended(key, end_reason=END_REASON_DISCARDED) # Same tick as the pop — no await between the two, so a cold start racing # this teardown cannot have registered a replacement sid for the key in # between, and this clear therefore cannot erase a SUCCESSOR's pointer. @@ -934,6 +974,10 @@ async def close_all(self, drain_timeout: float | None = None) -> None: owner._compact_cooldown_until.clear() self._suppress_replay.clear() owner._compact_pending_verdict.clear() + # Same tick as the clear: recording is non-blocking, so the whole + # drained set is accounted for before anything can await. + for closed_key in sessions: + record_session_ended(closed_key, end_reason=END_REASON_SHUTDOWN) # Provider shutdown can enqueue multiple blocking process-maintenance # jobs, so keep the original bounded fan-out. @@ -1105,7 +1149,14 @@ async def _eager_respawn(self, key: str) -> None: self._deps.logger.debug("Eager respawn failed for %s", key, exc_info=True) async def drain_all_providers(self) -> list[Any]: - """Pop every registered session and return its providers.""" + """Pop every registered session and return its providers. + + Records an end per popped key even though the one current caller drains + an already-empty registry (it calls ``reload_provider_factory`` first, + which clears and records). An unrecorded mass pop here would not lose + samples, it would manufacture crashes: every crumb left behind is + reported as ``crashed`` at the next boot. + """ owner = self._owner providers: list[Any] = [] popped: list[_SessionEntry] = [] @@ -1116,6 +1167,7 @@ async def drain_all_providers(self) -> list[Any]: if session: providers.append(session.provider) popped.append(session) + record_session_ended(key, end_reason=END_REASON_SHUTDOWN) # Filesystem unlink stays outside the registry lock. for session in popped: await asyncio.to_thread(self._deps.get_unlink_session_queue(), session) diff --git a/src/kiro_crew/slack/gateway.py b/src/kiro_crew/slack/gateway.py index 934627d21a6d..21ccfdc35820 100644 --- a/src/kiro_crew/slack/gateway.py +++ b/src/kiro_crew/slack/gateway.py @@ -10184,6 +10184,40 @@ def _probe_persistence() -> str | None: cleanup_orphaned_sessions() + # Same "previous run left residue" concern as the orphan sweep above, for + # telemetry rather than processes: any open-session crumb on disk belongs + # to a session that never reached a teardown path, so it is emitted as + # end_reason=crashed. + # + # Deliberately NOT awaited here. The scan is a glob plus a read/stat/unlink + # per crumb, so its cost scales with accumulated user data, and running it + # inline would delay readiness in proportion to that. It goes to a worker + # thread on a tracked task instead, and the process-start cutoff means it + # cannot mistake a session THIS process opens in the meantime for a + # casualty of the last one -- so it no longer has to finish before the + # gateway starts serving. + _telemetry_backfill_cutoff = time.time() + + async def _backfill_unclean_session_telemetry() -> None: + try: + from kiro_crew.metrics.sessions import backfill_crashed_sessions + + count = await asyncio.to_thread( + backfill_crashed_sessions, _telemetry_backfill_cutoff + ) + if count: + logger.info( + "Telemetry: back-filled %d unclean session lifetime(s) " + "from the previous run", + count, + ) + except Exception: # telemetry must never break boot + logger.debug("unclean-session telemetry backfill failed", exc_info=True) + + _backfill_task = asyncio.create_task(_backfill_unclean_session_telemetry()) + self._background_tasks.add(_backfill_task) + _backfill_task.add_done_callback(self._background_tasks.discard) + # Fill the sandbox probe cache BEFORE any on-loop spawn path can reach # detect_backend(). Waiting (off-loop) rather than firing-and-forgetting # is what makes that guarantee hold: a fire-and-forget prewarm leaves the diff --git a/src/kiro_crew/subagent_manager/admission.py b/src/kiro_crew/subagent_manager/admission.py index 0cb79a6a1add..4fa4bdbf31f2 100644 --- a/src/kiro_crew/subagent_manager/admission.py +++ b/src/kiro_crew/subagent_manager/admission.py @@ -790,6 +790,31 @@ def _log_spawned_impl(self, info: SubagentInfo) -> None: logger.warning("Failed to create agent folder for %s", info.id, exc_info=True) Stats().inc_subagent_spawned() + # Beside that stat, and for the same reason: this is the confirmed-start + # funnel. Every path reaches it only AFTER the spawn is approved -- the + # approval path calls it once the user allows and returns earlier on a + # rejection -- so a rejected or unstarted spawn is never counted, which + # the admission-time increment could not promise. ``concurrency`` is the + # live running count, bounded by ``_max_concurrent``, so the aggregator's + # MAX over that attribute is the concurrency high-water mark without a + # second instrument. + # + # Imported HERE, not at module scope: ``bind_component_globals`` rebinds + # every ``*_impl`` function's ``__globals__`` to ``subagent``'s namespace + # for patch compatibility, so a module-level import in this file is not + # visible from inside this function at all. + try: + from kiro_crew.metrics.events import SUBAGENTS_SPAWNED, emit_counter + + emit_counter( + SUBAGENTS_SPAWNED, + { + "concurrency": self._manager._running_count, + "batched": bool(getattr(info, "batch_id", "")), + }, + ) + except Exception: + logger.debug("subagent spawned counter failed", exc_info=True) sel().log_tool_invocation( session_key=info.parent_session_key, source="subagent", diff --git a/src/kiro_crew/workflows/runner.py b/src/kiro_crew/workflows/runner.py index 716847a35b6f..ff28c7c581b0 100644 --- a/src/kiro_crew/workflows/runner.py +++ b/src/kiro_crew/workflows/runner.py @@ -37,6 +37,8 @@ from dataclasses import dataclass, field from typing import Any, AsyncIterator, Awaitable, Callable, Optional +from kiro_crew.metrics.events import WORKFLOW_RUNS, emit_counter + from . import BudgetExceeded, WorkflowEvent from .context import DEFAULT_MAX_AGENTS_PER_RUN, AgentCounter, Budget, build_safe_globals from .dsl import parallel as _parallel @@ -660,6 +662,15 @@ def emit(ev: WorkflowEvent) -> WorkflowEvent: "outcome": "started", }, ) + # Beside the audit, and for the same reason: this is the one place every + # run passes before executing anything, foreground or background + # (``run_background`` drives this method). ``authored`` marks a run whose + # script this run writes from an intent; ``replay`` marks a + # restart-subtree that reuses cached agent results. + emit_counter( + WORKFLOW_RUNS, + {"authored": bool(intent and not source), "replay": bool(replay_results)}, + ) # 0. Author-in-run: if we were handed an intent and no source, turn # the intent into a validated script HERE, as a visible "Authoring" phase. diff --git a/test/metrics/test_business_counters.py b/test/metrics/test_business_counters.py new file mode 100644 index 000000000000..f38d5cdf10c4 --- /dev/null +++ b/test/metrics/test_business_counters.py @@ -0,0 +1,278 @@ +"""The business-event counters, driven through their production call sites. + +Each subsystem emits from its own call site because no frame is common to all of +them, so what these tests hold is the property that makes that safe: the name is +the shared constant, and every attribute VALUE is a member of a closed set +(``metrics/schema.py`` -- the recorder caches one instrument per name and never +evicts, so an unbounded value is a cardinality bomb). +""" + +import ast +from pathlib import Path +from unittest.mock import patch + +import pytest + +from kiro_crew.metrics import events as ev + + +class _CapturingRecorder: + def __init__(self) -> None: + self.counters: list = [] + + def counter(self, name, value=1, *, attrs=None, **kwargs) -> None: + self.counters.append({"name": name, "value": value, "attrs": dict(attrs or {})}) + + def histogram(self, name, value, **kwargs) -> None: # pragma: no cover - unused + pass + + +@pytest.fixture +def rec(): + r = _CapturingRecorder() + with patch("kiro_crew.metrics.provider.get_recorder", return_value=r): + yield r + + +def _named(rec, name): + return [c for c in rec.counters if c["name"] == name] + + +class TestApprovalDecisions: + """Every ``ToolHookResult`` construction is a gate verdict.""" + + def test_allow_is_the_branch_that_reaches_a_prompt(self, rec): + from kiro_crew.hooks import TOOL_ALLOW, ToolHookResult + + ToolHookResult.allow() + call = _named(rec, ev.APPROVAL_DECISIONS)[-1] + assert call["attrs"]["decision"] == TOOL_ALLOW + + def test_auto_approve_is_counted_separately(self, rec): + from kiro_crew.hooks import TOOL_AUTO_APPROVE, ToolHookResult + + ToolHookResult.auto_approve() + assert _named(rec, ev.APPROVAL_DECISIONS)[-1]["attrs"]["decision"] == TOOL_AUTO_APPROVE + + def test_a_security_deny_and_a_policy_deny_are_distinguishable(self, rec): + from kiro_crew.hooks import ToolHookResult + + ToolHookResult.deny("blocked") + ToolHookResult.deny_policy("ceiling") + calls = _named(rec, ev.APPROVAL_DECISIONS)[-2:] + assert calls[0]["attrs"]["security_deny"] is True + assert calls[1]["attrs"]["security_deny"] is False + + def test_the_deny_reason_never_reaches_the_attributes(self, rec): + """A reason carries paths and commands -- unbounded, and often sensitive.""" + from kiro_crew.hooks import ToolHookResult + + ToolHookResult.deny("refusing to read /home/someone/.aws/credentials") + call = _named(rec, ev.APPROVAL_DECISIONS)[-1] + assert set(call["attrs"]) == {"decision", "security_deny"} + + def test_the_gate_still_returns_its_verdict_when_telemetry_raises(self): + from kiro_crew.hooks import TOOL_DENY, ToolHookResult + + with patch("kiro_crew.metrics.events.emit_counter", side_effect=RuntimeError("boom")): + result = ToolHookResult.deny("blocked") + assert result.action == TOOL_DENY + assert result.reason == "blocked" + + def test_the_existing_child_permission_counters_are_untouched(self): + """Those measure a specific hang-resilience fix, not this population.""" + assert ev.CHILD_PERMISSION_DENIED == "kirocrew.acp.child_permission.denied" + assert ev.CHILD_PERMISSION_ROUTED == "kirocrew.acp.child_permission.routed" + + def test_a_surface_downgrade_of_the_verdict_is_not_a_second_decision(self, rec): + """Found in review: chat_runner rebuilds the result to downgrade it.""" + from kiro_crew.hooks import TOOL_ALLOW, ToolHookResult + + ToolHookResult.auto_approve() # the gate's verdict + before = len(_named(rec, ev.APPROVAL_DECISIONS)) + ToolHookResult(action=TOOL_ALLOW) # the surface overriding it + after = len(_named(rec, ev.APPROVAL_DECISIONS)) + assert after == before, "one consultation must count once, not once per object" + + def test_every_gate_exit_is_counted(self): + """All 18 returns in on_tool_call go through a factory; nothing else does.""" + import ast + import inspect + + from kiro_crew import hooks + + tree = ast.parse(inspect.getsource(hooks)) + factories = set() + direct = 0 + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.name == "on_tool_call": + for call in ast.walk(node): + if not isinstance(call, ast.Call): + continue + fn = call.func + if isinstance(fn, ast.Attribute) and isinstance(fn.value, ast.Name): + if fn.value.id == "ToolHookResult": + factories.add(fn.attr) + elif isinstance(fn, ast.Name) and fn.id == "ToolHookResult": + direct += 1 + assert factories, "the gate must return through the counted factories" + assert factories <= {"allow", "auto_approve", "deny", "deny_policy"} + assert direct == 0, "a direct construction inside the gate would go uncounted" + + +class TestSpawnCounterPlacement: + """Found in review: the admission-time emit counted rejected spawns.""" + + def test_the_counter_sits_at_the_confirmed_start_funnel(self): + import ast + import inspect + + from kiro_crew.subagent_manager import admission + + tree = ast.parse(inspect.getsource(admission)) + holders = [] + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef): + for sub in ast.walk(node): + if not isinstance(sub, ast.Call): + continue + fn = sub.func + name = fn.id if isinstance(fn, ast.Name) else getattr(fn, "attr", "") + if name == "emit_counter": + holders.append(node.name) + assert holders == ["_log_spawned_impl"], ( + "the spawn counter must live at _log_spawned_impl, which every path " + f"reaches only after approval; found in {holders}" + ) + + def test_it_sits_beside_the_repo_s_own_spawn_stat(self): + import inspect + + from kiro_crew.subagent_manager import admission + + body = ( + inspect.getsource(admission._SpawnAdmission._log_spawned_impl) + if hasattr(admission, "_SpawnAdmission") + else inspect.getsource(admission) + ) + assert "inc_subagent_spawned" in body + assert "SUBAGENTS_SPAWNED" in body + + +class TestCompactions: + @staticmethod + def _fire(success, *, recycling=False): + """Drive the production funnel with the minimum state it reads.""" + import asyncio + from types import SimpleNamespace + from typing import Any, cast + + from kiro_crew.session_compaction import CompactionCoordinator + + coordinator = CompactionCoordinator.__new__(CompactionCoordinator) + stub = cast(Any, coordinator) + stub._owner = SimpleNamespace( + _recycling={"k": object()} if recycling else {}, + mark_needs_reinjection=lambda key: None, + ) + stub.state = SimpleNamespace(on_compacted=None) + asyncio.run( + CompactionCoordinator._fire_compact_callback(coordinator, "k", 0.9, success=success) + ) + + def test_a_compaction_verdict_is_counted(self, rec): + self._fire(True) + assert _named(rec, ev.CONTEXT_COMPACTIONS)[-1]["attrs"] == {"success": True} + + def test_a_failed_compaction_is_still_an_attempt(self, rec): + self._fire(False) + assert _named(rec, ev.CONTEXT_COMPACTIONS)[-1]["attrs"] == {"success": False} + + def test_a_failed_compact_that_recycles_is_not_counted_successful(self, rec): + """Review round 6: the counter's success is not the callback's success. + + ``_recycle_held`` is reached exactly when an in-place ``/compact`` FAILED + and the provider had to be replaced instead, yet it fires this funnel with + ``success=True`` because the SESSION now has headroom -- which is what the + callback needs to know. Counting that as a successful compaction reports + the failure mode as the success case. The recycling marker is set for the + whole of ``_recycle_held`` and popped only afterwards, so it is what + separates the two populations at this point; if that ordering ever moves, + this test is what notices. + """ + self._fire(True, recycling=True) + assert _named(rec, ev.CONTEXT_COMPACTIONS)[-1]["attrs"] == {"success": False} + + def test_a_surface_with_no_callback_is_still_counted(self, rec): + """The early return below the emit would otherwise drop those surfaces.""" + self._fire(True) + assert len(_named(rec, ev.CONTEXT_COMPACTIONS)) == 1 + + +class TestCallSitesAreWired: + """Each counter must be referenced by the module that owns its event.""" + + OWNERS = { + "SUBAGENTS_SPAWNED": "subagent_manager/admission.py", + "CRON_FIRES": "cron.py", + "ARTIFACTS_CREATED": "artifacts.py", + "WORKFLOW_RUNS": "workflows/runner.py", + "CONTEXT_COMPACTIONS": "session_compaction.py", + "MCP_RECONNECTS": "mcp_gateway/stub.py", + "APPROVAL_DECISIONS": "hooks.py", + } + + @pytest.mark.parametrize("const,rel", sorted(OWNERS.items())) + def test_the_owning_module_emits_the_counter(self, const, rel): + src = (Path(ev.__file__).resolve().parent.parent / rel).read_text(encoding="utf-8") + assert const in src, f"{rel} must emit {const}" + assert "emit_counter" in src + + def test_every_declared_counter_has_an_owner(self): + """A constant with no call site is a metric nobody can read.""" + declared = { + name + for name, value in vars(ev).items() + if name.isupper() and isinstance(value, str) and value.startswith("kirocrew.") + } + # The hang-resilience series predates this change and is emitted from + # acp/ + session.py; only the business series is asserted here. + business = set(self.OWNERS) + assert business <= declared + + +class TestAttributeValuesAreBounded: + """No emit may hand the recorder a value derived from free-form input.""" + + #: Names a call site may read for an attribute value. Anything else -- a + #: session key, a tool name, a slug, a reason string -- is unbounded. + ALLOWED_CALLS = {"bool", "len", "int"} + + @pytest.mark.parametrize( + "rel", + [ + "subagent_manager/admission.py", + "cron.py", + "artifacts.py", + "workflows/runner.py", + "session_compaction.py", + "mcp_gateway/stub.py", + "hooks.py", + ], + ) + def test_no_emit_passes_an_f_string_or_a_concatenation(self, rel): + path = Path(ev.__file__).resolve().parent.parent / rel + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + func = node.func + name = func.id if isinstance(func, ast.Name) else getattr(func, "attr", "") + if name != "emit_counter": + continue + attrs = node.args[1] if len(node.args) > 1 else None + assert isinstance(attrs, ast.Dict), f"{rel}: attrs must be a literal dict" + for value in attrs.values: + assert not isinstance( + value, (ast.JoinedStr, ast.BinOp) + ), f"{rel}: an interpolated attribute value is unbounded" diff --git a/test/metrics/test_session_duration.py b/test/metrics/test_session_duration.py new file mode 100644 index 000000000000..093d2408fd3d --- /dev/null +++ b/test/metrics/test_session_duration.py @@ -0,0 +1,1014 @@ +"""``kirocrew.session.duration`` + ``kirocrew.session.started``. + +Drives the REAL production helpers in ``metrics/sessions.py`` with a patched +recorder and a redirected data home, so the crumb lifecycle, the exactly-once +consumption, the end_reason enum and the crashed back-fill all live in +production code -- a change there fails these tests instead of passing green. +""" + +import asyncio +import json +import os +import time +from pathlib import Path +from unittest.mock import patch + +import pytest + +from kiro_crew.metrics import sessions as sess + + +class _CapturingRecorder: + """Stand-in recorder that captures histogram() and counter() calls.""" + + #: The crumb is only written under telemetry consent, and this double stands + #: in for a CONSENTED recorder. ``TestConsentGate`` drives the other state. + enabled = True + + def __init__(self) -> None: + self.hist: list = [] + self.counters: list = [] + + def histogram(self, name, value, *, unit="ms", attrs=None, **kwargs) -> None: + self.hist.append({"name": name, "value": value, "unit": unit, "attrs": dict(attrs or {})}) + + def counter(self, name, value=1, *, attrs=None, **kwargs) -> None: + self.counters.append({"name": name, "value": value, "attrs": dict(attrs or {})}) + + +@pytest.fixture(autouse=True) +def clean_live_registry(): + """The live-start table is process-global; leaking it across tests is ordering bugs.""" + with sess._live_lock: + sess._live_starts.clear() + yield + with sess._live_lock: + sess._live_starts.clear() + + +@pytest.fixture +def home(tmp_path, monkeypatch): + """Point the crumb directory and transcript store at a temp data home.""" + monkeypatch.setattr("kiro_crew.config.paths.config_dir", lambda: tmp_path) + return tmp_path + + +@pytest.fixture +def rec(): + r = _CapturingRecorder() + with patch("kiro_crew.metrics.provider.get_recorder", return_value=r): + yield r + + +def _crumbs(home): + d = home / "metrics" / "open-sessions" + return sorted(d.glob("*.json")) if d.is_dir() else [] + + +def _unlink_is_outside_the_table_lock(src: str) -> bool: + """True when no ``_unlink(`` call is nested inside a ``with _live_lock:`` block. + + Indentation-aware rather than a substring window: the block is the run of + lines indented deeper than the ``with`` itself, which is what "inside" means. + """ + lines = src.splitlines() + for i, line in enumerate(lines): + if line.strip() != "with _live_lock:": + continue + indent = len(line) - len(line.lstrip()) + for follower in lines[i + 1 :]: + if not follower.strip(): + continue + if len(follower) - len(follower.lstrip()) <= indent: + break # block ended + if "_unlink(" in follower: + return False + return True + + +def _start(key): + """Drive ``record_session_started`` from a sync test. + + It is a coroutine because the crumb write is awaited on a worker thread -- off + the event loop, but ordered, because the caller holds the session registry + lock across the await. ``asyncio.run`` gives it the loop that + ``asyncio.to_thread`` needs. + """ + asyncio.run(sess.record_session_started(key)) + + +def _write_crumb(home, key, started_at, *, owner_pid=None): + """Leave behind the crumb a CRASHED process would leave. + + Goes through the production writer, which only writes while the key's + generation is installed, then drops the generation again -- the crash itself, + which takes the whole in-memory table with it and is precisely why the crumb + on disk is the only surviving evidence. + + The writer stamps THIS process as the owner, and the backfill skips crumbs + whose owner is still alive -- correctly, since a live owner means a live + session. So the ownership is rewritten to *owner_pid* (default: absent, i.e. a + crumb from before ownership was recorded) to model a process that is gone. + """ + with sess._live_lock: + sess._live_starts[key] = started_at + sess._write_crumb(key, started_at) + with sess._live_lock: + sess._live_starts.pop(key, None) + found = _crumbs(home) + assert found, "helper must have produced a crumb" + payload = json.loads(found[0].read_text()) + if owner_pid is None: + payload.pop("pid", None) + payload.pop("start_id", None) + else: + payload["pid"] = owner_pid + found[0].write_text(json.dumps(payload)) + + +def _duration_calls(rec): + return [c for c in rec.hist if c["name"] == "kirocrew.session.duration"] + + +class TestStart: + def test_start_counts_and_leaves_a_crumb(self, home, rec): + _start("dashboard:chat-1") + names = [c["name"] for c in rec.counters] + assert "kirocrew.session.started" in names + assert len(_crumbs(home)) == 1 + + def test_start_labels_the_surface(self, home, rec): + _start("dashboard:chat-1") + started = [c for c in rec.counters if c["name"] == "kirocrew.session.started"][-1] + assert started["attrs"]["session_source"] == "dashboard" + + def test_crumb_is_named_by_digest_not_by_the_key(self, home, rec): + # A session key can carry path separators and channel punctuation; a + # key-named file would escape the directory or fail to create at all. + _start("slack:C123/../../etc/passwd") + crumbs = _crumbs(home) + assert len(crumbs) == 1 + assert "passwd" not in crumbs[0].name + + def test_a_second_start_overwrites_the_first(self, home, rec): + """Deliberate reversal, found in review round 2. + + Registry removal has no single choke point and not every remover records + an end, so a stale entry is possible. A key re-entering the registry is a + NEW session whose lifetime must be its own -- keeping the predecessor's + start would report a lifetime spanning two sessions. + """ + _start("dashboard:chat-1") + first = sess._live_starts["dashboard:chat-1"] + time.sleep(0.01) + _start("dashboard:chat-1") + assert sess._live_starts["dashboard:chat-1"] > first + + def test_empty_key_is_a_no_op(self, home, rec): + _start("") + assert not rec.counters + assert not _crumbs(home) + + +class TestEnd: + @staticmethod + def _start(key, seconds_ago): + """Register a live session the way production does, aged by seconds. + + The crumb's NAME carries its generation, so ageing the table alone would + leave a file whose generation disagrees with the table -- a state + production cannot reach, and one the end path would then rightly refuse to + touch. The aged generation is therefore installed in both places. + """ + _start(key) + started_at = time.time() - seconds_ago + with sess._live_lock: + previous = sess._live_starts.get(key) + sess._live_starts[key] = started_at + if previous is not None: + sess._crumb_path(key, previous).unlink(missing_ok=True) + sess._write_crumb(key, started_at) + + def test_end_emits_the_lifetime_and_consumes_the_crumb(self, home, rec): + self._start("dashboard:chat-1", 60) + assert _crumbs(home), "the start must have left a crumb to consume" + sess.record_session_ended("dashboard:chat-1", end_reason=sess.END_REASON_RESET) + calls = _duration_calls(rec) + assert len(calls) == 1 + assert calls[0]["unit"] == "ms" + assert 55_000 < calls[0]["value"] < 70_000 + assert calls[0]["attrs"] == {"end_reason": "reset", "session_source": "dashboard"} + assert _crumbs(home) == [], "the end must consume the crumb" + + def test_the_end_record_touches_no_disk(self): + """It runs in the same tick as the registry pop, so it cannot block. + + Review round 2: reading the crumb here forced the call to the end of + teardown, and a replacement session registering under the same key during + those awaits had its start consumed by its predecessor. + """ + import inspect + + body = inspect.getsource(sess.record_session_ended) + assert "_live_starts.pop" in body + assert "_read_crumb" not in body + assert "read_text" not in body + + def test_a_second_end_emits_nothing(self, home, rec): + """The teardown paths overlap -- the idle sweep calls reset.""" + self._start("dashboard:chat-1", 60) + sess.record_session_ended("dashboard:chat-1", end_reason=sess.END_REASON_RESET) + sess.record_session_ended("dashboard:chat-1", end_reason=sess.END_REASON_SHUTDOWN) + assert len(_duration_calls(rec)) == 1 + + def test_end_without_a_crumb_emits_nothing(self, home, rec): + sess.record_session_ended("dashboard:never-started", end_reason=sess.END_REASON_RESET) + assert not _duration_calls(rec) + + def test_unknown_end_reason_is_refused(self, home, rec): + """An unbounded label would mint a series; the enum is the gate.""" + _write_crumb(home, "dashboard:chat-1", time.time() - 60) + sess.record_session_ended("dashboard:chat-1", end_reason="whatever-i-like") + assert not _duration_calls(rec) + assert _crumbs(home), "a refused reason must not consume the crumb" + + def test_non_positive_lifetime_is_skipped(self, home, rec): + # A LIVE start in the future, so the end actually reaches the emit and is + # rejected there. Planting only a crumb would return early at the pop and + # pass without ever exercising the guard. + self._start("dashboard:chat-1", -3600) + sess.record_session_ended("dashboard:chat-1", end_reason=sess.END_REASON_RESET) + assert not _duration_calls(rec) + + +class TestCrashedBackfill: + def _transcript(self, home, key, mtime): + from kiro_crew.history import SESSIONS_DIR_NAME, transcript_stem + + d = home / SESSIONS_DIR_NAME + d.mkdir(parents=True, exist_ok=True) + path = d / f"{transcript_stem(key)}.jsonl" + path.write_text("{}\n") + import os + + os.utime(path, (mtime, mtime)) + return path + + def test_leftover_crumb_becomes_a_crashed_sample(self, home, rec): + started = time.time() - 7200 + _write_crumb(home, "dashboard:chat-1", started) + self._transcript(home, "dashboard:chat-1", started + 3600) + assert sess.backfill_crashed_sessions() == 1 + calls = _duration_calls(rec) + assert len(calls) == 1 + assert calls[0]["attrs"]["end_reason"] == "crashed" + assert 3_500_000 < calls[0]["value"] < 3_700_000 + + def test_backfill_consumes_the_crumb_so_it_cannot_recur(self, home, rec): + started = time.time() - 7200 + _write_crumb(home, "dashboard:chat-1", started) + self._transcript(home, "dashboard:chat-1", started + 3600) + sess.backfill_crashed_sessions() + assert not _crumbs(home) + assert sess.backfill_crashed_sessions() == 0 + assert len(_duration_calls(rec)) == 1 + + def test_no_transcript_means_no_sample_but_the_crumb_still_goes(self, home, rec): + _write_crumb(home, "subagent:run-1", time.time() - 7200) + assert sess.backfill_crashed_sessions() == 0 + assert not _duration_calls(rec) + assert not _crumbs(home) + + def test_missing_directory_is_not_an_error(self, home, rec): + assert sess.backfill_crashed_sessions() == 0 + + def test_a_cleanly_ended_session_is_never_backfilled(self, home, rec): + started = time.time() - 7200 + _start("dashboard:chat-1") + with sess._live_lock: + previous = sess._live_starts.get("dashboard:chat-1") + sess._live_starts["dashboard:chat-1"] = started + # Age the FILE with the table: the crumb's name carries its generation. + sess._crumb_path("dashboard:chat-1", previous).unlink(missing_ok=True) + sess._write_crumb("dashboard:chat-1", started) + assert _crumbs(home), "the start must have left a crumb to consume" + self._transcript(home, "dashboard:chat-1", started + 3600) + sess.record_session_ended("dashboard:chat-1", end_reason=sess.END_REASON_SHUTDOWN) + assert _crumbs(home) == [], "the end must consume the crumb" + assert sess.backfill_crashed_sessions() == 0 + calls = _duration_calls(rec) + assert len(calls) == 1 + assert calls[0]["attrs"]["end_reason"] == "shutdown" + + def test_a_corrupt_crumb_is_consumed_here_without_emitting(self, home, rec): + """The backfill owns this now, because it is the only reader of the files. + + The end path used to consume an unparseable crumb as a side effect of + unlinking blind, which was the same blind unlink that could delete a live + sibling's record. Now that an end only ever removes the generation it can + name, the file that no session can claim is the backfill's to reap -- and + it must still be reaped, or every boot re-walks it forever. + """ + _write_crumb(home, "dashboard:chat-1", time.time() - 60) + _crumbs(home)[0].write_text("not json at all") + assert sess.backfill_crashed_sessions() == 0 + assert not _duration_calls(rec) + assert not _crumbs(home), "a corrupt crumb must not be re-walked every boot" + + +class TestWriterSelfCorrection: + """The worker cannot be cancelled, so it has to clean up after itself.""" + + def test_a_writer_landing_after_a_discard_leaves_no_crumb(self, home, rec): + """Review round 9: cancelling an ``asyncio.to_thread`` does not stop it. + + The await is abandoned, the worker is not -- so the caller's rollback can + unlink BEFORE the write lands, and the crumb the worker then writes is read + at the next boot as a crash that never happened. The worker self-corrects + instead of the await being made non-cancellable: it re-checks the + generation after writing, and a discard has already popped it. + """ + started = time.time() + with sess._live_lock: + sess._live_starts["dashboard:chat-1"] = started + sess.discard_session_start("dashboard:chat-1") + # The worker runs LATE, after the rollback already cleaned up. + sess._write_crumb("dashboard:chat-1", started) + assert not _crumbs(home), "a crumb written after a discard would backfill as crashed" + + def test_a_late_writer_never_deletes_a_successors_crumb(self, home, rec): + """A predecessor can only ever remove its OWN file, by construction. + + An earlier attempt unlinked the path unconditionally, so a predecessor + finishing late deleted the crumb its SUCCESSOR had just written under the + same key -- losing that live session's later crash. Comparing the recorded + ``started_at`` after the write only narrowed that window, and across + PROCESSES it did nothing at all, because two processes holding one key still + shared one filename and these are thread locks. + + The name now carries the writer and the generation, so the successor's + record is not addressable from the predecessor's writer. The successor is + still registered DURING the predecessor's write, the interleaving that used + to reach the broken post-write check. + """ + import kiro_crew.atomic_write as aw + + key = "dashboard:chat-1" + predecessor = time.time() - 3600 + successor = time.time() + with sess._live_lock: + sess._live_starts[key] = predecessor + real_write = aw.atomic_write + + def _successor_lands_mid_write(path, data, *a, **k): + real_write(path, data, *a, **k) + # A successor takes the key and writes ITS OWN crumb while the + # predecessor's write is still inside its critical section. + with sess._live_lock: + sess._live_starts[key] = successor + real_write( + sess._crumb_path(key, successor), + json.dumps({"key": key, "started_at": successor, "pid": os.getpid()}), + ) + + with patch.object(aw, "atomic_write", _successor_lands_mid_write): + sess._write_crumb(key, predecessor) + + crumbs = _crumbs(home) + assert len(crumbs) == 1, "the successor's crumb must survive the predecessor" + assert json.loads(crumbs[0].read_text())["started_at"] == successor + + def test_two_generations_of_one_key_never_share_a_file(self, home, rec): + """The premise the round-4 through round-9 findings all rested on. + + A session key is not unique across processes -- ``BACKGROUND_KEY`` is a + fixed constant, so a ``kirocrew run`` and the gateway hold the same key at + once, each with a live session behind it. While the filename was the key's + digest alone they shared one file, and whichever ended first unlinked the + other's record. + """ + key = "_bg" + first = time.time() - 60 + second = time.time() + assert sess._crumb_path(key, first) != sess._crumb_path(key, second) + + def test_a_superseding_start_reaps_the_generation_it_displaced(self, home, rec): + """Named generations make this a case that needs handling. + + Under one shared filename a second start simply overwrote the file. Now the + displaced generation has a name of its own that the table no longer holds, + so nothing could ever unlink it again -- it would reach the next boot and be + reported as a crash that never happened. + """ + key = "dashboard:chat-1" + _start(key) + first = sess._live_starts[key] + time.sleep(0.01) + _start(key) + second = sess._live_starts[key] + assert second > first + assert not sess._crumb_path(key, first).exists(), "the displaced crumb must be reaped" + assert sess._crumb_path(key, second).exists() + assert len(_crumbs(home)) == 1 + + def test_the_lock_order_is_io_then_table_everywhere(self): + """Nested locks deadlock only if two paths disagree about the order. + + The writer takes ``_crumb_io_lock`` and then ``_live_lock`` through the + generation check. Nothing may take them the other way round. + """ + import inspect + + for name in ("record_session_ended", "discard_session_start", "_write_crumb"): + src = inspect.getsource(getattr(sess, name)) + body = src.split('"""', 2)[-1] if '"""' in src else src + io_at = body.find("with _crumb_io_lock") + live_at = body.find("with _live_lock") + if io_at >= 0 and live_at >= 0: + assert io_at < live_at, f"{name} takes the locks in the wrong order" + + +class TestCrossProcessOwnership: + """Review round 7: a crumb with no owner cannot say WHOSE session it is. + + ``kirocrew run`` and the eval runner each build their own SessionManager + against the same data home, so a gateway boot sees their crumbs too. The + start-time cutoff only ever protected THIS process's own crumbs, so a sibling's + live session -- started before the gateway booted -- was read as a crash: it + both invented a sample and deleted the live crumb, losing the real crash that + session might later suffer. + """ + + def test_a_crumb_owned_by_a_live_process_is_never_reaped(self, home, rec): + _write_crumb(home, "dashboard:chat-1", time.time() - 7200, owner_pid=os.getpid()) + assert sess.backfill_crashed_sessions() == 0, "a live owner is not a casualty" + assert _crumbs(home), "and its crumb must survive for its own teardown" + + @staticmethod + def _restamp(home, start_id): + """Give the one crumb on disk an explicit owning start identifier.""" + crumb = _crumbs(home)[0] + payload = json.loads(crumb.read_text()) + payload["start_id"] = start_id + crumb.write_text(json.dumps(payload)) + + def test_our_own_pid_with_a_foreign_start_id_is_a_dead_predecessor(self, home, rec): + """Review round 10: in a container the gateway is PID 1 on every restart. + + A crashed predecessor's crumb therefore arrives carrying THIS process's pid, + and the own-pid fast path used to answer "still running" without comparing + identities at all -- so that crumb was skipped on this boot and on every + boot after: its crash was never emitted and its file was never cleaned. The + other review lane read this as a coincidental pid collision, but under a + container's fixed PID 1 it is deterministic. + """ + started = time.time() - 7200 + _write_crumb(home, "dashboard:chat-1", started, owner_pid=os.getpid()) + self._restamp(home, "the-process-that-crashed") + TestCrashedBackfill()._transcript(home, "dashboard:chat-1", started + 3600) + with patch( + "kiro_crew.platform_compat.get_process_start_id", + side_effect=lambda pid: "this-process", + ): + assert sess.backfill_crashed_sessions() == 1 + assert _duration_calls(rec)[0]["attrs"]["end_reason"] == "crashed" + assert not _crumbs(home), "and it must not be re-walked on every later boot" + + def test_our_own_pid_with_a_matching_start_id_is_still_us(self, home, rec): + """The refinement must not start reaping this process's own live crumbs.""" + _write_crumb(home, "dashboard:chat-1", time.time() - 7200, owner_pid=os.getpid()) + self._restamp(home, "this-process") + with patch( + "kiro_crew.platform_compat.get_process_start_id", + side_effect=lambda pid: "this-process", + ): + assert sess.backfill_crashed_sessions() == 0 + assert _crumbs(home), "a live owner's crumb must survive" + + def test_our_own_pid_with_an_unreadable_identity_fails_closed(self, home, rec): + """Windows returns None here, and a None must never read as a mismatch.""" + _write_crumb(home, "dashboard:chat-1", time.time() - 7200, owner_pid=os.getpid()) + self._restamp(home, "written-where-it-could-be-read") + with patch( + "kiro_crew.platform_compat.get_process_start_id", + side_effect=lambda pid: None, + ): + assert sess.backfill_crashed_sessions() == 0 + assert _crumbs(home), "an unreadable identity must not become a fabricated crash" + + def test_a_crumb_whose_owner_is_gone_is_still_claimed(self, home, rec): + started = time.time() - 7200 + _write_crumb(home, "dashboard:chat-1", started, owner_pid=424242) + TestCrashedBackfill()._transcript(home, "dashboard:chat-1", started + 3600) + with patch( + "kiro_crew.platform_compat.get_process_start_id", + side_effect=lambda pid: None if pid == 424242 else "self", + ): + assert sess.backfill_crashed_sessions() == 1 + assert _duration_calls(rec)[0]["attrs"]["end_reason"] == "crashed" + + def test_an_undecidable_owner_fails_closed(self, home, rec): + """A live pid whose identity we cannot read is left alone, not reaped. + + This is the Windows case, and it is why liveness comes from ``pid_exists`` + rather than from the start identifier: ``get_process_start_id`` returns + None on Windows and for any process it may not introspect, and its own + contract says a None must NOT be read as a mismatch. Reading it as one + judged every owner dead on that platform and reaped live sibling sessions. + + Losing a real crash sample costs one data point; inventing one corrupts + the population this instrument exists to report. + """ + _write_crumb(home, "dashboard:chat-1", time.time() - 7200, owner_pid=424242) + with patch("kiro_crew.platform_compat.pid_exists", return_value=True): + with patch("kiro_crew.platform_compat.get_process_start_id", return_value=None): + assert sess.backfill_crashed_sessions() == 0 + assert _crumbs(home), "an ambiguous crumb waits for a later boot" + + def test_a_live_pid_whose_identity_differs_is_a_recycled_pid(self, home, rec): + """Pid reuse must not make a dead session look alive.""" + started = time.time() - 7200 + _write_crumb(home, "dashboard:chat-1", started, owner_pid=424242) + payload = json.loads(_crumbs(home)[0].read_text()) + payload["start_id"] = "the-original-process" + _crumbs(home)[0].write_text(json.dumps(payload)) + TestCrashedBackfill()._transcript(home, "dashboard:chat-1", started + 3600) + with patch("kiro_crew.platform_compat.pid_exists", return_value=True): + with patch( + "kiro_crew.platform_compat.get_process_start_id", + return_value="a-different-process", + ): + assert sess.backfill_crashed_sessions() == 1 + assert _duration_calls(rec)[0]["attrs"]["end_reason"] == "crashed" + + def test_the_writer_stamps_its_own_identity(self, home, rec): + _start("dashboard:chat-1") + payload = json.loads(_crumbs(home)[0].read_text()) + assert payload["pid"] == os.getpid() + assert isinstance(payload["start_id"], str) + + +class TestContract: + def test_every_end_reason_constant_is_in_the_enum(self): + """A new path must not emit a label the tests do not know about.""" + declared = { + value + for name, value in vars(sess).items() + if name.startswith("END_REASON_") and isinstance(value, str) + } + assert declared == set(sess.END_REASONS) + + def test_the_lifecycle_module_uses_only_enum_members(self): + """The teardown paths import their labels rather than spelling them.""" + from kiro_crew import session_lifecycle + + for name in ( + "END_REASON_RESET", + "END_REASON_REMOVED", + "END_REASON_UNCLAIMED", + "END_REASON_DESTROYED", + "END_REASON_DISCARDED", + "END_REASON_SHUTDOWN", + ): + assert getattr(session_lifecycle, name) in sess.END_REASONS + + def test_the_duration_histogram_has_registered_bounds(self): + from kiro_crew.metrics.provider import _HISTOGRAM_BUCKETS_MS + + bounds = _HISTOGRAM_BUCKETS_MS[sess.SESSION_DURATION_METRIC] + # Minutes to days: a week-long dashboard tab must not land in +Inf, and + # a seconds-long unclaimed session must not collapse onto bound one. + assert bounds[0] <= 1000 + assert bounds[-1] >= 7 * 24 * 60 * 60 * 1000 + + +class TestReviewFixes: + """Regressions found in review -- each of these was a real defect.""" + + def test_the_crumb_write_is_off_the_loop_but_still_ordered(self, home, rec): + """Three earlier shapes each satisfied one constraint and broke the other. + + Fire-and-forget on a pool kept the loop clear but let a writer land after + its session ended, or after a SUCCESSOR registered under the same key -- + and detecting that after the fact was itself racy, because the post-write + check acted on the path the successor SHARES. Writing inline fixed the + ordering and put filesystem I/O on the event loop, where a slow data home + stalls every gateway task behind one session insertion. + + Awaiting a worker hop satisfies both: the syscalls leave the loop, and + nothing can interleave because the caller holds the session registry lock + across the await. This pins each half -- the hop, the await, and the + writer taking ``_live_lock`` for the backfill it races. + """ + import inspect + + body = inspect.getsource(sess.record_session_started) + assert "await asyncio.to_thread(" in body, "the write must be awaited off-loop" + assert "_write_crumb" in body, "the awaited hop must be the crumb writer" + assert "run_in_executor(" not in body, "no loop-bound future" + assert not hasattr(sess, "_submit"), "the fire-and-forget pool submitter must stay gone" + writer_src = inspect.getsource(sess._write_crumb) + assert "with _crumb_io_lock:" in writer_src, "the writer must take the I/O lock" + assert "with _live_lock:" not in writer_src, "the table lock must not span file I/O" + + # The loop-side paths must not wait on a lock a worker holds across I/O. + for fn in (sess.record_session_ended, sess.discard_session_start): + body = inspect.getsource(fn) + # Acquisition, not any mention: both docstrings discuss the I/O lock + # precisely to record why they must never take it. + assert "with _crumb_io_lock" not in body, f"{fn.__name__} runs on the loop" + assert "_crumb_io_lock.acquire" not in body, f"{fn.__name__} runs on the loop" + assert _unlink_is_outside_the_table_lock(body), ( + f"{fn.__name__} must unlink OUTSIDE the table lock, or the loop " + "parks behind whatever holds it" + ) + + # And the end consumes whatever the start left. + _start("dashboard:chat-1") + assert _crumbs(home) + sess.record_session_ended("dashboard:chat-1", end_reason=sess.END_REASON_RESET) + assert not _crumbs(home) + assert sess.backfill_crashed_sessions() == 0 + + def test_a_later_session_under_the_same_key_still_gets_a_crumb(self, home, rec): + """An ended key must not be suppressed for the rest of the process.""" + sess.record_session_ended("dashboard:chat-1", end_reason=sess.END_REASON_RESET) + _start("dashboard:chat-1") + assert len(_crumbs(home)) == 1 + + def test_a_superseded_start_leaves_only_the_successors_crumb(self, home, rec): + """Review rounds 3-5 all lived here; the inline write ended the class. + + A deferred write could land after its own session ended, or after a + SUCCESSOR registered under the same key. Guarding it with an ended-key + set, then with a generation compared before and after the write, both + failed -- the post-write check acted on the path the successor SHARES, so + a late predecessor deleted the crumb its successor had just written and + lost that session's crash. Writing under the lock that installs the start + removes the interleaving rather than detecting it: there is no window in + which two starts are both mid-write. + """ + _start("dashboard:chat-1") + first = sess._live_starts["dashboard:chat-1"] + time.sleep(0.01) + _start("dashboard:chat-1") + current = sess._live_starts["dashboard:chat-1"] + assert current > first + crumbs = _crumbs(home) + assert len(crumbs) == 1, "one key is one crumb, whatever the generation" + assert json.loads(crumbs[0].read_text())["started_at"] == current + + def test_the_end_unlinks_inline_so_the_unlink_cannot_be_cancelled(self): + """Review round 4: a pooled unlink was wrong in both directions. + + Queued, it can land after a successor wrote its own crumb and delete it, + losing that session's later crash. And ``shutdown_maintenance_executor`` + drains with ``cancel_futures=True``, so at ``close_all`` -- when the pool + is flooded with teardown work -- the unlink is dropped entirely, leaving + a cleanly ended session's crumb for the next boot to call ``crashed``. + """ + import inspect + + body = inspect.getsource(sess.record_session_ended) + assert "_submit(" not in body, "the unlink must not be handed to the pool" + assert "_unlink(_crumb_path(" in body, "the end must unlink its own generation" + + def test_the_backfill_never_claims_a_crumb_from_this_process(self, home, rec): + """The cutoff is what lets the scan run off the boot path.""" + cutoff = time.time() + _write_crumb(home, "dashboard:chat-1", cutoff + 5) + assert sess.backfill_crashed_sessions(cutoff) == 0 + assert _crumbs(home), "a live crumb must survive the scan" + + def test_the_backfill_still_claims_an_older_crumb_under_a_cutoff(self, home, rec): + started = time.time() - 7200 + _write_crumb(home, "dashboard:chat-1", started) + TestCrashedBackfill()._transcript(home, "dashboard:chat-1", started + 3600) + assert sess.backfill_crashed_sessions(time.time()) == 1 + assert _duration_calls(rec)[0]["attrs"]["end_reason"] == "crashed" + + def test_retire_and_recycle_have_their_own_reasons(self): + """They pop the registry directly and never route through reset.""" + assert sess.END_REASON_RETIRED in sess.END_REASONS + assert sess.END_REASON_RECYCLED in sess.END_REASONS + assert sess.END_REASON_RETIRED != sess.END_REASON_RESET + assert sess.END_REASON_RECYCLED != sess.END_REASON_RESET + + def test_the_identity_retire_path_records_an_end(self): + from kiro_crew import session_lifecycle + + found = TestTeardownPathsAreWired._reasons_recorded_by("retire_kiro_identity_sessions") + assert found == ["END_REASON_RETIRED"] + assert session_lifecycle.END_REASON_RETIRED in sess.END_REASONS + + def test_the_compaction_recycle_path_records_an_end(self): + import ast + import inspect + + from kiro_crew import session_compaction + + tree = ast.parse(inspect.getsource(session_compaction)) + found = [] + for node in ast.walk(tree): + if isinstance(node, ast.AsyncFunctionDef) and node.name == "_recycle_held": + for call in ast.walk(node): + if not isinstance(call, ast.Call): + continue + fn = call.func + name = fn.id if isinstance(fn, ast.Name) else getattr(fn, "attr", "") + if name == "record_session_ended": + found.append( + [ + k.value.id + for k in call.keywords + if k.arg == "end_reason" and isinstance(k.value, ast.Name) + ] + ) + # Exactly one: the branch that owns the pop. The other branch's key + # already holds a SUCCESSOR whose crumb must not be consumed here. + assert found == [["END_REASON_RECYCLED"]] + + def test_the_boot_hook_no_longer_blocks_readiness(self): + from pathlib import Path + + src = (Path(sess.__file__).resolve().parent.parent / "slack" / "gateway.py").read_text( + encoding="utf-8" + ) + i = src.index("_backfill_unclean_session_telemetry") + window = src[i : i + 900] + assert "asyncio.to_thread" in window, "the scan must not run on the loop" + assert "create_task" in window, "the scan must not be awaited before readiness" + assert "_background_tasks" in window, "the task must be tracked, not fire-and-forget" + + def test_the_crumb_write_is_never_a_loop_bound_future(self): + """Windows CI failed with 'Event loop is closed' on the first version. + + That version used ``loop.run_in_executor``, whose future is owned by the + running loop, so a fire-and-forget write outliving its loop raised when + the result was set. The write is off-loop again now, but AWAITED rather + than fired and forgotten, which is what keeps it ordered as well as + non-blocking. + """ + import inspect + + body = inspect.getsource(sess.record_session_started) + assert "run_in_executor(" not in body + assert "asyncio.to_thread" in body + + def test_a_start_writes_its_crumb_with_no_gateway_loop_of_its_own(self, home, rec): + _start("cron:job-1") + assert _crumbs(home), "a session start must leave a crumb" + + def test_no_crumb_is_written_without_telemetry_consent(self, home): + """First Principles round 6: an unopted install must write nothing. + + The crumb feeds only ``kirocrew.session.duration``, which is a no-op + without consent -- so writing one on an unopted host persists state that + nothing can ever read, against a documented default of collecting nothing. + Round 5 relaxed the spec sentence to permit it; that was the wrong + direction, and this pins the behaviour instead. + """ + + class _Off: + enabled = False + + def histogram(self, *a, **k): + pass + + def counter(self, *a, **k): + pass + + with patch("kiro_crew.metrics.provider.get_recorder", return_value=_Off()): + _start("dashboard:chat-1") + assert not _crumbs(home), "consent is off, so no crumb may reach the disk" + + def test_the_consent_check_fails_closed(self, home): + """An unreadable consent state must not default to writing.""" + with patch("kiro_crew.metrics.provider.get_recorder", side_effect=RuntimeError("boom")): + _start("dashboard:chat-1") + assert not _crumbs(home) + + +class TestEveryRegistryRemovalRecordsAnEnd: + """A fail-closed gate over the whole source tree, not a hand-kept list. + + Review round 5 found FOUR registry removals with no end record, and the + hand-maintained list in ``TestTeardownPathsAreWired`` could not have caught + any of them: it enumerates six methods of one module by name, so a seventh + path -- or any pop outside that module -- is invisible to it. That is the + wrong shape for this invariant, because an unrecorded removal is not a lost + sample. The start crumb survives it, and the next boot reports the session as + ``crashed``: a failure that never happened, in the exact population the + histogram exists to measure. + + So this walks every module that mutates the registry and requires each + mutating function to record an end. A NEW removal path fails this test by + default, which is the only version of this gate that stays true as the + lifecycle grows. + """ + + #: Functions that remove from the registry WITHOUT consuming the crumb, + #: deliberately. Each entry needs a reason, and the reason has to survive review. + ALLOWED_SILENT: dict[str, str] = {} + + #: Modules whose ``_sessions`` is a DIFFERENT container that merely shares the + #: attribute name, with the reason it is out of scope. Discovery is by name, so + #: it cannot tell two containers apart on its own -- but an exemption here is + #: NOT a permanent hole: ``test_each_unrelated_registry_exemption_still_holds`` + #: re-derives the claim, so the day such a module starts participating in the + #: crumb lifecycle its exemption fails instead of quietly covering for it. + UNRELATED_REGISTRIES = { + "connections.warm": ( + "the shared warm OAuth mint's dict[int, _WarmSession], keyed by " + "activation id -- not the session registry's dict[str, _SessionEntry], " + "and it writes no crumb, so a removal there cannot leave one behind" + ), + } + + @staticmethod + def _module_source(module_name): + import importlib + import inspect + + return inspect.getsource(importlib.import_module(f"kiro_crew.{module_name}")) + + @staticmethod + def _owning_modules(): + """Every module under ``src/kiro_crew`` that mutates the session registry. + + DISCOVERED, not listed. Design review's objection to a hardcoded list was + exact: session-metric correctness depends on every current AND FUTURE + removal path reporting itself, and a list of four module names cannot see a + fifth. Walking the tree means a removal added anywhere is in scope the day + it lands, which is the only version of this gate that stays true as the + lifecycle grows. + """ + import kiro_crew + + root = Path(kiro_crew.__file__).resolve().parent + found = [] + for path in sorted(root.rglob("*.py")): + if "__pycache__" in path.parts: + continue + try: + if "_sessions" not in path.read_text(encoding="utf-8"): + continue + except OSError: + continue + rel = path.relative_to(root).with_suffix("") + found.append(".".join(rel.parts)) + return found + + @staticmethod + def _removal_functions(module_name): + """Every function in *module_name* that removes a registry entry. + + Returns ``{function name: consumes_the_crumb}``. Recognises the three + removal spellings the tree actually uses -- ``_sessions.pop``, ``del + _sessions[...]`` and ``_sessions.clear()`` -- so a path that switches + spelling does not fall out of the gate. A removal consumes its crumb + either by recording an end (a session that lived) or by discarding the + start (a registration rolled back before it became a session); both leave + no crumb behind, which is what the invariant is actually about. + """ + import ast + import importlib + import inspect + + module = importlib.import_module(f"kiro_crew.{module_name}") + tree = ast.parse(inspect.getsource(module)) + out: dict[str, bool] = {} + consumers = {"record_session_ended", "discard_session_start"} + + def _is_registry(node): + return isinstance(node, ast.Attribute) and node.attr == "_sessions" + + for fn in ast.walk(tree): + if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + removes = records = False + for node in ast.walk(fn): + if isinstance(node, ast.Call): + func = node.func + name = func.id if isinstance(func, ast.Name) else getattr(func, "attr", "") + if name in {"pop", "clear"} and _is_registry(getattr(func, "value", None)): + removes = True + if name in consumers: + records = True + elif isinstance(node, ast.Delete): + for target in node.targets: + if isinstance(target, ast.Subscript) and _is_registry(target.value): + removes = True + if removes: + out[fn.name] = records + return out + + def test_every_registry_removal_records_an_end(self): + silent = [] + for module_name in self._owning_modules(): + if module_name in self.UNRELATED_REGISTRIES: + continue + for fn_name, records in self._removal_functions(module_name).items(): + if not records and fn_name not in self.ALLOWED_SILENT: + silent.append(f"{module_name}.{fn_name}") + assert not silent, ( + "these registry removals record no end, so every session they drop " + f"is reported as crashed at the next boot: {sorted(silent)}" + ) + + def test_each_unrelated_registry_exemption_still_holds(self): + """An exemption must void itself if the module joins the crumb lifecycle.""" + for module_name, reason in self.UNRELATED_REGISTRIES.items(): + assert reason, f"{module_name} needs a reason, not a bare exemption" + src = self._module_source(module_name) + assert "record_session_started" not in src, ( + f"{module_name} now writes session crumbs, so its exemption is " + f"void and its removals must consume them. Stale reason: {reason}" + ) + + def test_the_gate_can_actually_fail(self): + """A gate that has stopped detecting anything reads as a green signal.""" + modules = self._owning_modules() + assert "session_lifecycle" in modules, "discovery missed the lifecycle module" + removals = self._removal_functions("session_lifecycle") + assert removals, "the AST walk found no registry removals at all" + assert "drain_all_providers" in removals, "the mass-pop path must be in scope" + + +class TestTeardownPathsAreWired: + """Each teardown path must record, with a reason of its own. + + A source-level gate rather than a driven one: these six methods are the + lifecycle service's own registry pops, and standing each of them up needs a + stub owner, provider, executor and platform layer -- which would pin the + stubs, not the wiring. The behaviour they delegate to is driven directly + above; what this holds is that they still delegate at all, and that no two + paths report the same reason (which would silently merge two populations). + + Coverage that every removal records AT ALL belongs to + ``TestEveryRegistryRemovalRecordsAnEnd`` above; this one is about the reasons + being distinct. + """ + + EXPECTED = { + "reset": "END_REASON_RESET", + "remove": "END_REASON_REMOVED", + "remove_if_unclaimed": "END_REASON_UNCLAIMED", + "destroy": "END_REASON_DESTROYED", + "discard_conversation": "END_REASON_DISCARDED", + "close_all": "END_REASON_SHUTDOWN", + } + + @staticmethod + def _reasons_recorded_by(method_name): + import ast + import inspect + + from kiro_crew import session_lifecycle + + tree = ast.parse(inspect.getsource(session_lifecycle)) + found: list[str] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.AsyncFunctionDef) or node.name != method_name: + continue + for call in ast.walk(node): + if not isinstance(call, ast.Call): + continue + func = call.func + name = func.id if isinstance(func, ast.Name) else getattr(func, "attr", "") + if name != "record_session_ended": + continue + for kw in call.keywords: + if kw.arg == "end_reason" and isinstance(kw.value, ast.Name): + found.append(kw.value.id) + return found + + @pytest.mark.parametrize("method,const", sorted(EXPECTED.items())) + def test_the_path_records_with_its_own_reason(self, method, const): + assert self._reasons_recorded_by(method) == [const] + + def test_no_two_paths_share_a_reason(self): + reasons = [c for m in self.EXPECTED for c in self._reasons_recorded_by(m)] + assert len(reasons) == len(set(reasons)) + + def test_the_boot_backfill_is_wired_into_gateway_startup(self): + from pathlib import Path + + gateway = Path(sess.__file__).resolve().parent.parent / "slack" / "gateway.py" + src = gateway.read_text(encoding="utf-8") + assert "backfill_crashed_sessions" in src + # Ordering against the orphan sweep is deliberately NOT asserted any + # more: review showed an inline pre-ready scan delays readiness in + # proportion to accumulated crumbs, so it moved to a worker thread. What + # replaces the ordering requirement is the process-start cutoff, which + # TestReviewFixes pins directly -- the scan can no longer mistake a + # session THIS process opened for a casualty of the last one. + assert "_telemetry_backfill_cutoff" in src + + def test_the_registry_insertions_record_a_start(self): + from pathlib import Path + + root = Path(sess.__file__).resolve().parent.parent + for rel in ("session_allocation.py", "session_background.py"): + src = (root / rel).read_text(encoding="utf-8") + assert "record_session_started" in src, rel diff --git a/test/metrics/test_tool_call_duration.py b/test/metrics/test_tool_call_duration.py new file mode 100644 index 000000000000..68593e88ff25 --- /dev/null +++ b/test/metrics/test_tool_call_duration.py @@ -0,0 +1,276 @@ +"""``kirocrew.tool.call.duration``. + +Drives the REAL production helpers in ``metrics/tool_calls.py`` plus the two +parser layers that call them, so the kind normalisation, the terminal-status +rule and the exactly-once accounting across layers live in production code. +""" + +from unittest.mock import patch + +import pytest + +from kiro_crew.metrics import tool_calls as tc + + +class _CapturingRecorder: + def __init__(self) -> None: + self.hist: list = [] + + def histogram(self, name, value, *, unit="ms", attrs=None, **kwargs) -> None: + self.hist.append({"name": name, "value": value, "unit": unit, "attrs": dict(attrs or {})}) + + +@pytest.fixture(autouse=True) +def clean_registry(): + tc.reset_open_calls() + yield + tc.reset_open_calls() + + +@pytest.fixture +def rec(): + r = _CapturingRecorder() + with patch("kiro_crew.metrics.provider.get_recorder", return_value=r): + yield r + + +def _calls(rec): + return [c for c in rec.hist if c["name"] == "kirocrew.tool.call.duration"] + + +class TestKindClassification: + @pytest.mark.parametrize( + "kind", ["read", "fetch", "search", "edit", "write", "create", "delete", "move", "execute"] + ) + def test_known_kinds_pass_through(self, kind): + assert tc.classify_tool_kind(kind) == kind + + def test_an_mcp_call_is_labelled_by_its_transport(self): + # The kind an MCP server reports is its own vocabulary; keeping it would + # put third-party strings in the allowlist's path. + assert tc.classify_tool_kind("read", mcp_server_name="some-server") == "mcp" + assert tc.classify_tool_kind("whatever", mcp_server_name="srv") == "mcp" + + def test_an_agent_authored_kind_cannot_mint_a_series(self): + """``kind`` arrives verbatim from the agent -- an allowlist is the bound.""" + assert tc.classify_tool_kind("totally_new_kind_9000") == "other" + assert tc.classify_tool_kind("unknown") == "other" + + def test_absent_kind_is_other(self): + assert tc.classify_tool_kind(None) == "other" + assert tc.classify_tool_kind("") == "other" + + def test_every_classification_is_inside_the_declared_set(self): + for candidate in (None, "", "read", "unknown", "execute", "nonsense"): + assert tc.classify_tool_kind(candidate) in tc.TOOL_KINDS + + +class TestRoundTrip: + def test_a_completed_call_is_recorded_once(self, rec): + tc.note_tool_call_started("t1", kind="execute") + tc.record_tool_call_finished("t1", status="completed") + calls = _calls(rec) + assert len(calls) == 1 + assert calls[0]["unit"] == "ms" + assert calls[0]["attrs"] == {"tool_kind": "execute", "outcome": "completed"} + assert calls[0]["value"] > 0 + + @pytest.mark.parametrize("status", ["completed", "failed", "cancelled"]) + def test_every_terminal_status_is_an_outcome(self, rec, status): + tc.note_tool_call_started("t1", kind="read") + tc.record_tool_call_finished("t1", status=status) + assert _calls(rec)[0]["attrs"]["outcome"] == status + + @pytest.mark.parametrize("status", ["pending", "in_progress", None, ""]) + def test_a_non_terminal_update_leaves_the_clock_running(self, rec, status): + tc.note_tool_call_started("t1", kind="read") + tc.record_tool_call_finished("t1", status=status) + assert not _calls(rec) + assert tc.open_call_count() == 1 + tc.record_tool_call_finished("t1", status="completed") + assert len(_calls(rec)) == 1 + + def test_a_finish_with_no_start_records_nothing(self, rec): + """This is what makes the two instrumented layers add up to one sample.""" + tc.record_tool_call_finished("never-started", status="completed") + assert not _calls(rec) + + def test_a_second_finish_records_nothing(self, rec): + tc.note_tool_call_started("t1", kind="read") + tc.record_tool_call_finished("t1", status="completed") + tc.record_tool_call_finished("t1", status="completed") + assert len(_calls(rec)) == 1 + + def test_a_repeated_start_does_not_restart_the_clock(self, rec): + tc.note_tool_call_started("t1", kind="execute") + tc.note_tool_call_started("t1", kind="read") + tc.record_tool_call_finished("t1", status="completed") + # The refinement's kind loses to the original, which is what keeps the + # measured span the whole round-trip. + assert _calls(rec)[0]["attrs"]["tool_kind"] == "execute" + + def test_an_empty_id_is_a_no_op(self, rec): + tc.note_tool_call_started("", kind="read") + tc.record_tool_call_finished("", status="completed") + assert not _calls(rec) + assert tc.open_call_count() == 0 + + def test_the_registry_is_bounded(self, rec): + for i in range(tc._MAX_OPEN_CALLS + 5): + tc.note_tool_call_started(f"t{i}", kind="read") + assert tc.open_call_count() <= tc._MAX_OPEN_CALLS + + +class TestParserWiring: + """The production parsers must be the ones driving the registry.""" + + def test_the_shared_dispatch_parser_opens_and_closes_a_call(self, rec): + from kiro_crew.acp import _dispatch + + _dispatch._build_tool_call_event( + {"sessionUpdate": "tool_call", "toolCallId": "tc-1", "kind": "execute", "title": "sh"}, + None, + ) + assert tc.open_call_count() == 1 + _dispatch._build_tool_result_event( + {"toolCallId": "tc-1", "status": "completed", "content": []}, + ) + calls = _calls(rec) + assert len(calls) == 1 + assert calls[0]["attrs"] == {"tool_kind": "execute", "outcome": "completed"} + + def test_an_mcp_served_call_is_labelled_mcp_by_the_parser(self, rec): + """The trusted ``_meta.kiro.mcpServerName``, not the reported kind.""" + from kiro_crew.acp import _dispatch + + _dispatch._build_tool_call_event( + { + "sessionUpdate": "tool_call", + "toolCallId": "tc-m", + "kind": "read", + "title": "t", + "_meta": {"kiro": {"mcpServerName": "srv", "toolName": "x"}}, + }, + None, + ) + _dispatch._build_tool_result_event({"toolCallId": "tc-m", "status": "completed"}) + assert _calls(rec)[0]["attrs"]["tool_kind"] == "mcp" + + def test_an_output_less_completion_is_still_measured(self, rec): + """_build_tool_result_event returns None with no output; the sample stays.""" + from kiro_crew.acp import _dispatch + + _dispatch._build_tool_call_event( + {"sessionUpdate": "tool_call", "toolCallId": "tc-2", "kind": "read", "title": "r"}, + None, + ) + event = _dispatch._build_tool_result_event({"toolCallId": "tc-2", "status": "completed"}) + assert event is None + assert len(_calls(rec)) == 1 + + def test_both_layers_together_still_yield_one_sample(self, rec): + """A frame seen by AcpClient and by the shared parser is not doubled.""" + from kiro_crew.acp import _dispatch + + _dispatch._build_tool_call_event( + {"sessionUpdate": "tool_call", "toolCallId": "tc-3", "kind": "read", "title": "r"}, + None, + ) + for _ in range(2): + _dispatch._build_tool_result_event({"toolCallId": "tc-3", "status": "completed"}) + assert len(_calls(rec)) == 1 + + +class TestScopeIsolation: + """Found in review: toolCallId is backend-scoped, the registry is global.""" + + def test_two_sessions_reusing_an_id_each_get_their_own_sample(self, rec): + tc.note_tool_call_started("t1", kind="execute", scope="sessA") + tc.note_tool_call_started("t1", kind="read", scope="sessB") + assert tc.open_call_count() == 2, "the second start must not look like a duplicate" + tc.record_tool_call_finished("t1", status="completed", scope="sessA") + tc.record_tool_call_finished("t1", status="completed", scope="sessB") + kinds = sorted(c["attrs"]["tool_kind"] for c in _calls(rec)) + assert kinds == ["execute", "read"], "each session keeps its own kind" + + def test_a_finish_in_the_wrong_scope_does_not_steal_the_entry(self, rec): + tc.note_tool_call_started("t1", kind="execute", scope="sessA") + tc.record_tool_call_finished("t1", status="completed", scope="sessB") + assert not _calls(rec) + assert tc.open_call_count() == 1 + tc.record_tool_call_finished("t1", status="completed", scope="sessA") + assert len(_calls(rec)) == 1 + + def test_the_key_matches_the_dispatch_cache_spelling(self): + """Same composition as _dispatch's _ck, so the two cannot disagree.""" + assert tc._registry_key("tc-1", "sess") == "sess|tc-1" + assert tc._registry_key("tc-1", "") == "tc-1" + + def test_both_layers_derive_one_frames_scope_from_the_same_session_id(self, rec): + """Design review: agreeing scopes are what make the second layer a no-op. + + Layer one takes the scope from the frame's ``sessionId``, which + ``session_handle`` forwards as ``cache_scope``; layer two reads the + client's own ``_session_id``. For one frame those are the same value, so + whichever layer sees it second pops the entry the first opened and the + call is sampled once. Were they ever to diverge, the disjoint-parser + assumption would fail by DOUBLE-counting rather than by harmlessly + missing -- which is why this is pinned at both the source and the + behaviour. + """ + import inspect + + from kiro_crew.acp import _dispatch + from kiro_crew.acp.client import AcpClient + + # Source: neither layer may invent a scope of its own. + layer_two = inspect.getsource(AcpClient._extract_tool_call_update) + assert 'scope=getattr(self, "_session_id", "") or ""' in layer_two + assert "scope=cache_scope" in inspect.getsource(_dispatch._build_tool_call_event) + + # Behaviour: layer two opens the call, layer one closes it, one sample. + sid = "acp-sess-7" + client = AcpClient.__new__(AcpClient) # scope only; no process is spawned + client._session_id = sid + tc.note_tool_call_started( + "tc-x", kind="read", scope=getattr(client, "_session_id", "") or "" + ) + assert tc.open_call_count() == 1 + _dispatch._build_tool_result_event({"toolCallId": "tc-x", "status": "completed"}, sid) + assert len(_calls(rec)) == 1 + assert tc.open_call_count() == 0, "the cross-layer pop must consume the entry" + + def test_the_dispatch_parser_threads_its_cache_scope_through(self, rec): + from kiro_crew.acp import _dispatch + + for scope in ("sessA", "sessB"): + _dispatch._build_tool_call_event( + { + "sessionUpdate": "tool_call", + "toolCallId": "dup", + "kind": "execute" if scope == "sessA" else "read", + "title": "t", + }, + None, + cache_scope=scope, + ) + assert tc.open_call_count() == 2 + for scope in ("sessA", "sessB"): + _dispatch._build_tool_result_event({"toolCallId": "dup", "status": "completed"}, scope) + assert len(_calls(rec)) == 2 + + +class TestContract: + def test_the_histogram_has_registered_bounds(self): + from kiro_crew.metrics.provider import _HISTOGRAM_BUCKETS_MS + + bounds = _HISTOGRAM_BUCKETS_MS[tc.TOOL_CALL_METRIC] + # Sub-ms reads keep resolution; a multi-minute build stays off +Inf. + assert bounds[0] < 1 + assert bounds[-1] >= 60 * 60 * 1000 + + def test_no_tool_name_reaches_the_attributes(self, rec): + """MCP tool names are unbounded, so they must never be a label.""" + tc.note_tool_call_started("t1", kind="read", mcp_server_name="srv") + tc.record_tool_call_finished("t1", status="completed") + assert set(_calls(rec)[0]["attrs"]) == {"tool_kind", "outcome"}