Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/system-specs/modules/slack-gateway.md

Large diffs are not rendered by default.

108 changes: 102 additions & 6 deletions src/kiro_crew/dashboard/loop_watchdog.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,12 @@
from kiro_crew.dashboard.stall_enrichment import collect_stall_enrichment, describe_loop_thread
from kiro_crew.diag.threads import _parse_stat, _read_text, _task_dir
from kiro_crew.platform_compat import (
MEMORY_PRESSURE_WARN,
arm_process_alarm,
awake_now,
boottime_now,
memory_pressure_level,
proc_pageins_for_pid,
process_alarm_available,
)

Expand All @@ -139,6 +142,31 @@
#: paging still ends the process.
PAGING_GRACE_MAX_SECS: float = 600.0

#: The macOS grace cap, shorter than the Linux one on purpose. The macOS
#: page-in signal (:func:`read_loop_pageins_under_pressure`) is process-wide,
#: not per-thread, so it cannot tell "the loop thread is paging" from "some
#: other thread in the process is paging". A shorter cap bounds how long a
#: genuinely wedged loop can be kept alive by an unrelated thread's paging.
PAGING_GRACE_MAX_SECS_MACOS: float = 120.0

#: The WARNING a paging deferral logs, once per silence, in two forms selected
#: by the grace cap -- already how the macOS watchdog is distinguished without a
#: platform branch (see :data:`PAGING_GRACE_MAX_SECS_MACOS`). The Linux signal
#: is the loop thread's own major-fault count, so its message names the thread
#: and rules out a wedge. The macOS signal is the whole process's page-in count,
#: which cannot attribute paging to the loop thread, so its message makes no
#: such claim. Each takes three %-args: silence seconds, count delta, budget.
_PAGING_MESSAGE_LINUX: str = (
"event loop silent %.1fs while its thread is paging (%d major faults "
"since the last poll): the host is short of memory, not the loop "
"wedged, so the stall exit is deferred for up to %.0fs of silence"
)
_PAGING_MESSAGE_MACOS: str = (
"event loop silent %.1fs while the process is paging (%d page-ins "
"under memory pressure since the last poll): the stall exit is "
"deferred for up to %.0fs of silence"
)


def read_thread_major_faults(native_thread_id: int | None) -> int | None:
"""Major page faults taken so far by one thread of this process.
Expand All @@ -156,6 +184,48 @@ def read_thread_major_faults(native_thread_id: int | None) -> int | None:
return major_faults if isinstance(major_faults, int) else None


def read_loop_pageins_under_pressure(
_native_thread_id: int | None,
*,
pressure_level: "Callable[[], int | None]" = memory_pressure_level,
pageins: "Callable[[int], int | None]" = proc_pageins_for_pid,
warn_level: int = MEMORY_PRESSURE_WARN,
) -> int | None:
"""macOS analogue of :func:`read_thread_major_faults` for the paging deferral.

Plugs into the same ``loop_major_faults`` hook, so :meth:`LoopStallWatchdog.
_defer_exit_while_paging` needs no change. It returns the process's
``ri_pageins`` count **only while the host is under memory pressure**
(``memory_pressure_level`` at or above WARN); otherwise it returns ``None``.

The ``None``-when-not-pressured choice is deliberate and load-bearing. The
deferral acts on a *rise* between two readings, and ``None`` is already its
"no reading -- reset the baseline, defer nothing" path. Returning a flat
``0`` instead would manufacture a huge false rise on the first pressured poll
after an unpressured one (``0`` -> true count) and defer a genuinely wedged
loop. So the pressure gate folds into the reader as: no pressure -> ``None``
-> no baseline -> no deferral.

This is also why a stall the compressor serves (pressure high, disk pageins
flat) still exits on the normal budget: the count does not rise, so nothing
defers. The signal only ever *extends* the budget on real disk-pagein
progress; it can never keep a flat-pagein stall alive.

The signal is process-wide, not per-thread -- macOS exposes no public
per-thread page-in count -- so ``_native_thread_id`` is accepted to match
:func:`read_thread_major_faults`'s signature and ignored. That coarseness
is bounded by the shorter :data:`PAGING_GRACE_MAX_SECS_MACOS`.

Off macOS, or on any unreadable signal, returns ``None``. The two readers
are injectable so the deferral can be driven deterministically in tests
without ever inducing real memory pressure.
"""
level = pressure_level()
if level is None or level < warn_level:
return None
return pageins(os.getpid())


#: The mechanism the last :func:`_default_arm_later` armed the dump-then-exit
#: on — ``"alarm"`` or ``"faulthandler"`` — or ``None`` while it holds no
#: pending deadline. Process-wide like the two deadlines it names
Expand Down Expand Up @@ -371,7 +441,13 @@ class LoopStallWatchdog:
:func:`kiro_crew.dashboard.stall_enrichment.collect_stall_enrichment`.
loop_major_faults: Reads the loop thread's major-fault count from its
native thread id, ``None`` when unknown; injectable for tests.
Defaults to :func:`read_thread_major_faults`.
Defaults to :func:`read_thread_major_faults`. macOS passes
:func:`read_loop_pageins_under_pressure` instead.
paging_grace_max: Longest the exit may be deferred past ``exit_after``
while the loop keeps paging. ``None`` resolves to the module
:data:`PAGING_GRACE_MAX_SECS` at construction; macOS passes the
shorter :data:`PAGING_GRACE_MAX_SECS_MACOS` because its page-in
signal is process-wide rather than per-thread.
log: Logger, injectable for tests.
"""

Expand Down Expand Up @@ -399,6 +475,7 @@ def __init__(
enrich_after: float = 15.0,
enrich: "Callable[[float], list[str]] | None" = None,
loop_major_faults: Callable[[int | None], int | None] = read_thread_major_faults,
paging_grace_max: float | None = None,
log: logging.Logger | None = None,
) -> None:
self._stall_after = stall_after
Expand Down Expand Up @@ -432,6 +509,27 @@ def __init__(
# The kernel's id for that same thread, which names its procfs row.
self._loop_native_thread_id: int | None = None
self._loop_major_faults = loop_major_faults
# The grace cap for THIS instance, resolved from the module default at
# construction when not given. Reading the module name here (not as a
# default-arg value, which binds once at def time) keeps a
# ``monkeypatch.setattr(loop_watchdog, "PAGING_GRACE_MAX_SECS", ...)``
# done before construction effective, and lets the macOS watchdog pass
# the shorter ``PAGING_GRACE_MAX_SECS_MACOS`` without a platform branch
# inside the hot path.
self._paging_grace_max = (
PAGING_GRACE_MAX_SECS if paging_grace_max is None else paging_grace_max
)
# The WARNING a paging deferral logs, once per silence. Selected from
# the grace cap, the same signal that already distinguishes the macOS
# watchdog without a platform branch: the macOS cap pairs with the
# process-wide page-in message, every other cap with the Linux
# per-thread major-fault message. Takes three %-args: silence seconds,
# the count delta, total budget.
self._paging_log_message = (
_PAGING_MESSAGE_MACOS
if self._paging_grace_max == PAGING_GRACE_MAX_SECS_MACOS
else _PAGING_MESSAGE_LINUX
)
# Counts beats, so a paging deferral can tell that the loop beat after
# it sampled and must leave the beat's fresh deadline alone.
self._beat_generation = 0
Expand Down Expand Up @@ -755,20 +853,18 @@ def _defer_exit_while_paging(self, silence: float) -> None:
return
if major_faults <= baseline[1]:
return
remaining_budget = exit_after + PAGING_GRACE_MAX_SECS - silence
remaining_budget = exit_after + self._paging_grace_max - silence
if remaining_budget <= 0:
return
if not self._rearm_alarm(min(exit_after, remaining_budget), unless_beat_since=generation):
return
if self._paging_logged_generation != generation:
self._paging_logged_generation = generation
self._log.warning(
"event loop silent %.1fs while its thread is paging (%d major faults "
"since the last poll): the host is short of memory, not the loop "
"wedged, so the stall exit is deferred for up to %.0fs of silence",
self._paging_log_message,
silence,
major_faults - baseline[1],
exit_after + PAGING_GRACE_MAX_SECS,
exit_after + self._paging_grace_max,
)

def claim_lag_enrichment(self, lag: float) -> bool:
Expand Down
4 changes: 4 additions & 0 deletions src/kiro_crew/dashboard/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,10 @@
release_site,
)
from kiro_crew.dashboard.loop_watchdog import LoopStallWatchdog # noqa: F401
from kiro_crew.dashboard.loop_watchdog import ( # noqa: F401
PAGING_GRACE_MAX_SECS_MACOS,
read_loop_pageins_under_pressure,
)
from kiro_crew.dashboard.origin import ( # noqa: F401
AUDIT_CLAIMED_KEY,
PROBE_PATHS,
Expand Down
16 changes: 15 additions & 1 deletion src/kiro_crew/dashboard/server_runtime/heartbeat.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

if TYPE_CHECKING:
from kiro_crew.dashboard.server import (
PAGING_GRACE_MAX_SECS_MACOS,
DashboardState,
LoopStallWatchdog,
attribute_dump,
Expand All @@ -27,6 +28,7 @@
newest_dump_with_stacks,
open_dump_file,
platform_compat,
read_loop_pageins_under_pressure,
record_healthy_boot,
register_pre_exec_stop,
resolve_loop_stall_exit_after,
Expand Down Expand Up @@ -79,7 +81,19 @@ async def _open_loop_watchdog(_launch_environment: dict[str, str]) -> LoopStallW
# Config failure must not erase the managed-service grace that protects
# the process while its config filesystem is itself under pressure.
_exit_after = float(resolve_loop_stall_exit_after(environ=_launch_environment))
watchdog = LoopStallWatchdog(dump_file=_dump_file, exit_after=_exit_after)
# On macOS the Linux per-thread major-fault reader returns None, so the
# paging deferral never fires. Swap in the process-wide page-in reader
# (gated on memory pressure) with a shorter grace cap, since that signal
# cannot attribute paging to the loop thread specifically.
if platform_compat.IS_MACOS:
watchdog = LoopStallWatchdog(
dump_file=_dump_file,
exit_after=_exit_after,
loop_major_faults=read_loop_pageins_under_pressure,
paging_grace_max=PAGING_GRACE_MAX_SECS_MACOS,
)
else:
watchdog = LoopStallWatchdog(dump_file=_dump_file, exit_after=_exit_after)

async def _stop_before_exec() -> None:
# An in-place restart can exec right after a recovered stall, before
Expand Down
3 changes: 3 additions & 0 deletions src/kiro_crew/platform_compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -605,6 +605,7 @@ def __dir__() -> list[str]:
_DARWIN_PTI_RESIDENT_SIZE_OFFSET,
_DARWIN_PTI_TOTAL_SYSTEM_OFFSET,
_DARWIN_PTI_TOTAL_USER_OFFSET,
_DARWIN_RI_PAGEINS_OFFSET,
_DARWIN_RI_PHYS_FOOTPRINT_OFFSET,
_DARWIN_RUSAGE_INFO_V2,
_DARWIN_RUSAGE_INFO_V2_SIZE,
Expand Down Expand Up @@ -639,6 +640,7 @@ def __dir__() -> list[str]:
_darwin_peer_is_loopback,
_darwin_process_cpu_nanos,
_darwin_process_cwd,
_darwin_process_pageins,
_darwin_process_phys_footprint_bytes,
_darwin_process_start_identity,
_darwin_process_start_microtime,
Expand Down Expand Up @@ -676,6 +678,7 @@ def __dir__() -> list[str]:
pids_holding_file,
proc_child_map,
proc_cpu_jiffies_for_pids,
proc_pageins_for_pid,
proc_phys_footprint_bytes_for_pid,
read_proc_stat,
read_proc_status_int,
Expand Down
45 changes: 45 additions & 0 deletions src/kiro_crew/platform_compat_proctable.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,13 @@ class ProcessStartIdentity(NamedTuple):
_DARWIN_RUSAGE_INFO_V2 = 2
_DARWIN_RUSAGE_INFO_V2_SIZE = 16 + 18 * 8
_DARWIN_RI_PHYS_FOOTPRINT_OFFSET = 16 + 7 * 8
# ``ri_pageins`` is the fifth uint64 in ``rusage_info_v2`` (user time, system
# time, the two wakeup counters, then pageins), so 16 bytes of ``ri_uuid`` plus
# four uint64 fields. It counts the disk pageins this process has taken since it
# started -- a monotonic process-wide counter, NOT per-thread, and it does NOT
# include compressor decompressions (macOS serves most memory pressure from the
# compressor, so this can stay flat while the process stalls paging).
_DARWIN_RI_PAGEINS_OFFSET = 16 + 4 * 8

_darwin_libproc: Any = None
_darwin_libproc_loaded = False
Expand Down Expand Up @@ -232,6 +239,44 @@ def proc_phys_footprint_bytes_for_pid(pid: int) -> int | None:
return _darwin_process_phys_footprint_bytes(pid)


def _darwin_process_pageins(pid: int) -> int | None:
"""macOS ``ri_pageins`` of *pid* via ``proc_pid_rusage``, or None when unreadable.

The count of disk pageins the process has taken since it started: a
monotonic, process-wide counter. It is NOT per-thread, and it does NOT
count compressor decompressions, so under memory pressure served entirely
from the compressor it can stay flat while the process stalls paging.
Needs no entitlement for a same-uid process and never spawns a subprocess.
"""
lib = _darwin_libproc_handle()
rusage = getattr(lib, "proc_pid_rusage", None) if lib is not None else None
if rusage is None:
return None
try:
buf = ctypes.create_string_buffer(_DARWIN_RUSAGE_INFO_V2_SIZE)
if rusage(pid, _DARWIN_RUSAGE_INFO_V2, buf) != 0:
return None
off = _DARWIN_RI_PAGEINS_OFFSET
# Both x86_64 and arm64 macOS are little-endian.
return int.from_bytes(buf.raw[off : off + 8], "little")
except Exception:
return None


def proc_pageins_for_pid(pid: int) -> int | None:
"""``ri_pageins`` of *pid* on macOS; None elsewhere or unreadable.

The process-wide disk page-in count (see ``_darwin_process_pageins``). Other
platforms have no equivalent and answer None. A rise between two readings of
one silence is the macOS analogue of the Linux loop-stall watchdog's
per-thread major-fault delta -- coarser, because it is process-wide, which
is why the deferral it feeds uses a shorter grace cap.
"""
if not IS_MACOS:
return None
return _darwin_process_pageins(pid)


def darwin_libproc_available() -> bool:
"""True when ``libproc`` loads, so the macOS memory and tree probes can answer."""
return _darwin_libproc_handle() is not None
Expand Down
Loading
Loading